mirror of
https://github.com/QwenLM/qwen-code.git
synced 2025-12-19 09:33:53 +00:00
test(nonInteractiveCli): add tests and remove unused cost info
This commit is contained in:
1479
packages/cli/src/nonInteractive/io/BaseJsonOutputAdapter.test.ts
Normal file
1479
packages/cli/src/nonInteractive/io/BaseJsonOutputAdapter.test.ts
Normal file
File diff suppressed because it is too large
Load Diff
@@ -56,7 +56,6 @@ export interface ResultOptions {
|
||||
readonly apiDurationMs: number;
|
||||
readonly numTurns: number;
|
||||
readonly usage?: ExtendedUsage;
|
||||
readonly totalCostUsd?: number;
|
||||
readonly stats?: SessionMetrics;
|
||||
readonly summary?: string;
|
||||
readonly subtype?: string;
|
||||
@@ -1020,7 +1019,6 @@ export abstract class BaseJsonOutputAdapter {
|
||||
duration_ms: options.durationMs,
|
||||
duration_api_ms: options.apiDurationMs,
|
||||
num_turns: options.numTurns,
|
||||
total_cost_usd: options.totalCostUsd ?? 0,
|
||||
usage,
|
||||
permission_denials: [],
|
||||
error: { message: errorMessage },
|
||||
@@ -1037,7 +1035,6 @@ export abstract class BaseJsonOutputAdapter {
|
||||
duration_api_ms: options.apiDurationMs,
|
||||
num_turns: options.numTurns,
|
||||
result: resultText,
|
||||
total_cost_usd: options.totalCostUsd ?? 0,
|
||||
usage,
|
||||
permission_denials: [],
|
||||
};
|
||||
@@ -1075,7 +1072,6 @@ export abstract class BaseJsonOutputAdapter {
|
||||
duration_ms: 0,
|
||||
duration_api_ms: 0,
|
||||
num_turns: numTurns,
|
||||
total_cost_usd: 0,
|
||||
usage,
|
||||
permission_denials: [],
|
||||
error: { message: errorMessage },
|
||||
|
||||
@@ -392,7 +392,6 @@ describe('JsonOutputAdapter', () => {
|
||||
durationMs: 1000,
|
||||
apiDurationMs: 800,
|
||||
numTurns: 1,
|
||||
totalCostUsd: 0.01,
|
||||
});
|
||||
|
||||
expect(stdoutWriteSpy).toHaveBeenCalled();
|
||||
@@ -414,7 +413,6 @@ describe('JsonOutputAdapter', () => {
|
||||
expect(resultMessage.result).toBe('Response text');
|
||||
expect(resultMessage.duration_ms).toBe(1000);
|
||||
expect(resultMessage.num_turns).toBe(1);
|
||||
expect(resultMessage.total_cost_usd).toBe(0.01);
|
||||
});
|
||||
|
||||
it('should emit error result', () => {
|
||||
@@ -424,7 +422,6 @@ describe('JsonOutputAdapter', () => {
|
||||
durationMs: 500,
|
||||
apiDurationMs: 300,
|
||||
numTurns: 1,
|
||||
totalCostUsd: 0.005,
|
||||
});
|
||||
|
||||
const output = stdoutWriteSpy.mock.calls[0][0] as string;
|
||||
|
||||
@@ -612,7 +612,6 @@ describe('StreamJsonOutputAdapter', () => {
|
||||
durationMs: 1000,
|
||||
apiDurationMs: 800,
|
||||
numTurns: 1,
|
||||
totalCostUsd: 0.01,
|
||||
});
|
||||
|
||||
expect(stdoutWriteSpy).toHaveBeenCalled();
|
||||
@@ -625,7 +624,6 @@ describe('StreamJsonOutputAdapter', () => {
|
||||
expect(parsed.result).toBe('Response text');
|
||||
expect(parsed.duration_ms).toBe(1000);
|
||||
expect(parsed.num_turns).toBe(1);
|
||||
expect(parsed.total_cost_usd).toBe(0.01);
|
||||
});
|
||||
|
||||
it('should emit error result', () => {
|
||||
@@ -636,7 +634,6 @@ describe('StreamJsonOutputAdapter', () => {
|
||||
durationMs: 500,
|
||||
apiDurationMs: 300,
|
||||
numTurns: 1,
|
||||
totalCostUsd: 0.005,
|
||||
});
|
||||
|
||||
const output = stdoutWriteSpy.mock.calls[0][0] as string;
|
||||
|
||||
@@ -608,7 +608,6 @@ class SessionManager {
|
||||
apiDurationMs,
|
||||
numTurns,
|
||||
usage: undefined,
|
||||
totalCostUsd: undefined,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -36,7 +36,6 @@ export interface ModelUsage {
|
||||
cacheReadInputTokens: number;
|
||||
cacheCreationInputTokens: number;
|
||||
webSearchRequests: number;
|
||||
costUSD: number;
|
||||
contextWindow: number;
|
||||
}
|
||||
|
||||
@@ -162,7 +161,6 @@ export interface CLIResultMessageSuccess {
|
||||
duration_api_ms: number;
|
||||
num_turns: number;
|
||||
result: string;
|
||||
total_cost_usd: number;
|
||||
usage: ExtendedUsage;
|
||||
modelUsage?: Record<string, ModelUsage>;
|
||||
permission_denials: CLIPermissionDenial[];
|
||||
@@ -178,7 +176,6 @@ export interface CLIResultMessageError {
|
||||
duration_ms: number;
|
||||
duration_api_ms: number;
|
||||
num_turns: number;
|
||||
total_cost_usd: number;
|
||||
usage: ExtendedUsage;
|
||||
modelUsage?: Record<string, ModelUsage>;
|
||||
permission_denials: CLIPermissionDenial[];
|
||||
|
||||
@@ -173,6 +173,45 @@ describe('runNonInteractive', () => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
/**
|
||||
* Creates a default mock SessionMetrics object.
|
||||
* Can be overridden in individual tests if needed.
|
||||
*/
|
||||
function createMockMetrics(
|
||||
overrides?: Partial<SessionMetrics>,
|
||||
): SessionMetrics {
|
||||
return {
|
||||
models: {},
|
||||
tools: {
|
||||
totalCalls: 0,
|
||||
totalSuccess: 0,
|
||||
totalFail: 0,
|
||||
totalDurationMs: 0,
|
||||
totalDecisions: {
|
||||
accept: 0,
|
||||
reject: 0,
|
||||
modify: 0,
|
||||
auto_accept: 0,
|
||||
},
|
||||
byName: {},
|
||||
},
|
||||
files: {
|
||||
totalLinesAdded: 0,
|
||||
totalLinesRemoved: 0,
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets up the default mock for uiTelemetryService.getMetrics().
|
||||
* Should be called in beforeEach or at the start of tests that need metrics.
|
||||
*/
|
||||
function setupMetricsMock(overrides?: Partial<SessionMetrics>): void {
|
||||
const mockMetrics = createMockMetrics(overrides);
|
||||
vi.mocked(uiTelemetryService.getMetrics).mockReturnValue(mockMetrics);
|
||||
}
|
||||
|
||||
async function* createStreamFromEvents(
|
||||
events: ServerGeminiStreamEvent[],
|
||||
): AsyncGenerator<ServerGeminiStreamEvent> {
|
||||
@@ -475,27 +514,7 @@ describe('runNonInteractive', () => {
|
||||
createStreamFromEvents(events),
|
||||
);
|
||||
(mockConfig.getOutputFormat as Mock).mockReturnValue(OutputFormat.JSON);
|
||||
const mockMetrics: SessionMetrics = {
|
||||
models: {},
|
||||
tools: {
|
||||
totalCalls: 0,
|
||||
totalSuccess: 0,
|
||||
totalFail: 0,
|
||||
totalDurationMs: 0,
|
||||
totalDecisions: {
|
||||
accept: 0,
|
||||
reject: 0,
|
||||
modify: 0,
|
||||
auto_accept: 0,
|
||||
},
|
||||
byName: {},
|
||||
},
|
||||
files: {
|
||||
totalLinesAdded: 0,
|
||||
totalLinesRemoved: 0,
|
||||
},
|
||||
};
|
||||
vi.mocked(uiTelemetryService.getMetrics).mockReturnValue(mockMetrics);
|
||||
setupMetricsMock();
|
||||
|
||||
await runNonInteractive(
|
||||
mockConfig,
|
||||
@@ -527,7 +546,9 @@ describe('runNonInteractive', () => {
|
||||
);
|
||||
expect(resultMessage).toBeTruthy();
|
||||
expect(resultMessage?.result).toBe('Hello World');
|
||||
expect(resultMessage?.stats).toEqual(mockMetrics);
|
||||
// Get the actual metrics that were used
|
||||
const actualMetrics = vi.mocked(uiTelemetryService.getMetrics)();
|
||||
expect(resultMessage?.stats).toEqual(actualMetrics);
|
||||
});
|
||||
|
||||
it('should write JSON output with stats for tool-only commands (no text response)', async () => {
|
||||
@@ -568,8 +589,7 @@ describe('runNonInteractive', () => {
|
||||
.mockReturnValueOnce(createStreamFromEvents(secondCallEvents));
|
||||
|
||||
(mockConfig.getOutputFormat as Mock).mockReturnValue(OutputFormat.JSON);
|
||||
const mockMetrics: SessionMetrics = {
|
||||
models: {},
|
||||
setupMetricsMock({
|
||||
tools: {
|
||||
totalCalls: 1,
|
||||
totalSuccess: 1,
|
||||
@@ -596,12 +616,7 @@ describe('runNonInteractive', () => {
|
||||
},
|
||||
},
|
||||
},
|
||||
files: {
|
||||
totalLinesAdded: 0,
|
||||
totalLinesRemoved: 0,
|
||||
},
|
||||
};
|
||||
vi.mocked(uiTelemetryService.getMetrics).mockReturnValue(mockMetrics);
|
||||
});
|
||||
|
||||
await runNonInteractive(
|
||||
mockConfig,
|
||||
@@ -651,27 +666,7 @@ describe('runNonInteractive', () => {
|
||||
createStreamFromEvents(events),
|
||||
);
|
||||
(mockConfig.getOutputFormat as Mock).mockReturnValue(OutputFormat.JSON);
|
||||
const mockMetrics: SessionMetrics = {
|
||||
models: {},
|
||||
tools: {
|
||||
totalCalls: 0,
|
||||
totalSuccess: 0,
|
||||
totalFail: 0,
|
||||
totalDurationMs: 0,
|
||||
totalDecisions: {
|
||||
accept: 0,
|
||||
reject: 0,
|
||||
modify: 0,
|
||||
auto_accept: 0,
|
||||
},
|
||||
byName: {},
|
||||
},
|
||||
files: {
|
||||
totalLinesAdded: 0,
|
||||
totalLinesRemoved: 0,
|
||||
},
|
||||
};
|
||||
vi.mocked(uiTelemetryService.getMetrics).mockReturnValue(mockMetrics);
|
||||
setupMetricsMock();
|
||||
|
||||
await runNonInteractive(
|
||||
mockConfig,
|
||||
@@ -703,11 +698,14 @@ describe('runNonInteractive', () => {
|
||||
);
|
||||
expect(resultMessage).toBeTruthy();
|
||||
expect(resultMessage?.result).toBe('');
|
||||
expect(resultMessage?.stats).toEqual(mockMetrics);
|
||||
// Get the actual metrics that were used
|
||||
const actualMetrics = vi.mocked(uiTelemetryService.getMetrics)();
|
||||
expect(resultMessage?.stats).toEqual(actualMetrics);
|
||||
});
|
||||
|
||||
it('should handle errors in JSON format', async () => {
|
||||
(mockConfig.getOutputFormat as Mock).mockReturnValue(OutputFormat.JSON);
|
||||
setupMetricsMock();
|
||||
const testError = new Error('Invalid input provided');
|
||||
|
||||
mockGeminiClient.sendMessageStream.mockImplementation(() => {
|
||||
@@ -753,6 +751,7 @@ describe('runNonInteractive', () => {
|
||||
|
||||
it('should handle FatalInputError with custom exit code in JSON format', async () => {
|
||||
(mockConfig.getOutputFormat as Mock).mockReturnValue(OutputFormat.JSON);
|
||||
setupMetricsMock();
|
||||
const fatalError = new FatalInputError('Invalid command syntax provided');
|
||||
|
||||
mockGeminiClient.sendMessageStream.mockImplementation(() => {
|
||||
@@ -950,6 +949,7 @@ describe('runNonInteractive', () => {
|
||||
it('should emit stream-json envelopes when output format is stream-json', async () => {
|
||||
(mockConfig.getOutputFormat as Mock).mockReturnValue('stream-json');
|
||||
(mockConfig.getIncludePartialMessages as Mock).mockReturnValue(false);
|
||||
setupMetricsMock();
|
||||
|
||||
const writes: string[] = [];
|
||||
processStdoutSpy.mockImplementation((chunk: string | Uint8Array) => {
|
||||
@@ -1065,6 +1065,25 @@ describe('runNonInteractive', () => {
|
||||
it('should include usage metadata and API duration in stream-json result', async () => {
|
||||
(mockConfig.getOutputFormat as Mock).mockReturnValue('stream-json');
|
||||
(mockConfig.getIncludePartialMessages as Mock).mockReturnValue(false);
|
||||
setupMetricsMock({
|
||||
models: {
|
||||
'test-model': {
|
||||
api: {
|
||||
totalRequests: 1,
|
||||
totalErrors: 0,
|
||||
totalLatencyMs: 500,
|
||||
},
|
||||
tokens: {
|
||||
prompt: 11,
|
||||
candidates: 5,
|
||||
total: 16,
|
||||
cached: 3,
|
||||
thoughts: 0,
|
||||
tool: 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const writes: string[] = [];
|
||||
processStdoutSpy.mockImplementation((chunk: string | Uint8Array) => {
|
||||
@@ -1125,6 +1144,7 @@ describe('runNonInteractive', () => {
|
||||
it('should not emit user message when userMessage option is provided (stream-json input binding)', async () => {
|
||||
(mockConfig.getOutputFormat as Mock).mockReturnValue('stream-json');
|
||||
(mockConfig.getIncludePartialMessages as Mock).mockReturnValue(false);
|
||||
setupMetricsMock();
|
||||
|
||||
const writes: string[] = [];
|
||||
processStdoutSpy.mockImplementation((chunk: string | Uint8Array) => {
|
||||
@@ -1198,6 +1218,7 @@ describe('runNonInteractive', () => {
|
||||
it('should emit tool results as user messages in stream-json format', async () => {
|
||||
(mockConfig.getOutputFormat as Mock).mockReturnValue('stream-json');
|
||||
(mockConfig.getIncludePartialMessages as Mock).mockReturnValue(false);
|
||||
setupMetricsMock();
|
||||
|
||||
const writes: string[] = [];
|
||||
processStdoutSpy.mockImplementation((chunk: string | Uint8Array) => {
|
||||
@@ -1297,6 +1318,7 @@ describe('runNonInteractive', () => {
|
||||
it('should emit tool errors in tool_result blocks in stream-json format', async () => {
|
||||
(mockConfig.getOutputFormat as Mock).mockReturnValue('stream-json');
|
||||
(mockConfig.getIncludePartialMessages as Mock).mockReturnValue(false);
|
||||
setupMetricsMock();
|
||||
|
||||
const writes: string[] = [];
|
||||
processStdoutSpy.mockImplementation((chunk: string | Uint8Array) => {
|
||||
@@ -1390,6 +1412,7 @@ describe('runNonInteractive', () => {
|
||||
it('should emit partial messages when includePartialMessages is true', async () => {
|
||||
(mockConfig.getOutputFormat as Mock).mockReturnValue('stream-json');
|
||||
(mockConfig.getIncludePartialMessages as Mock).mockReturnValue(true);
|
||||
setupMetricsMock();
|
||||
|
||||
const writes: string[] = [];
|
||||
processStdoutSpy.mockImplementation((chunk: string | Uint8Array) => {
|
||||
@@ -1446,6 +1469,7 @@ describe('runNonInteractive', () => {
|
||||
it('should handle thinking blocks in stream-json format', async () => {
|
||||
(mockConfig.getOutputFormat as Mock).mockReturnValue('stream-json');
|
||||
(mockConfig.getIncludePartialMessages as Mock).mockReturnValue(false);
|
||||
setupMetricsMock();
|
||||
|
||||
const writes: string[] = [];
|
||||
processStdoutSpy.mockImplementation((chunk: string | Uint8Array) => {
|
||||
@@ -1503,6 +1527,7 @@ describe('runNonInteractive', () => {
|
||||
it('should handle multiple tool calls in stream-json format', async () => {
|
||||
(mockConfig.getOutputFormat as Mock).mockReturnValue('stream-json');
|
||||
(mockConfig.getIncludePartialMessages as Mock).mockReturnValue(false);
|
||||
setupMetricsMock();
|
||||
|
||||
const writes: string[] = [];
|
||||
processStdoutSpy.mockImplementation((chunk: string | Uint8Array) => {
|
||||
@@ -1613,6 +1638,7 @@ describe('runNonInteractive', () => {
|
||||
it('should handle userMessage with text content blocks in stream-json input mode', async () => {
|
||||
(mockConfig.getOutputFormat as Mock).mockReturnValue('stream-json');
|
||||
(mockConfig.getIncludePartialMessages as Mock).mockReturnValue(false);
|
||||
setupMetricsMock();
|
||||
|
||||
const writes: string[] = [];
|
||||
processStdoutSpy.mockImplementation((chunk: string | Uint8Array) => {
|
||||
|
||||
@@ -36,10 +36,9 @@ import {
|
||||
import {
|
||||
normalizePartList,
|
||||
extractPartsFromUserMessage,
|
||||
extractUsageFromGeminiClient,
|
||||
calculateApproximateCost,
|
||||
buildSystemMessage,
|
||||
createTaskToolProgressHandler,
|
||||
computeUsageFromMetrics,
|
||||
} from './utils/nonInteractiveHelpers.js';
|
||||
|
||||
/**
|
||||
@@ -315,8 +314,10 @@ export async function runNonInteractive(
|
||||
}
|
||||
currentMessages = [{ role: 'user', parts: toolResponseParts }];
|
||||
} else {
|
||||
const usage = extractUsageFromGeminiClient(geminiClient);
|
||||
// For JSON and STREAM_JSON modes, compute usage from metrics
|
||||
if (adapter) {
|
||||
const metrics = uiTelemetryService.getMetrics();
|
||||
const usage = computeUsageFromMetrics(metrics);
|
||||
// Get stats for JSON format output
|
||||
const stats =
|
||||
outputFormat === OutputFormat.JSON
|
||||
@@ -328,20 +329,21 @@ export async function runNonInteractive(
|
||||
apiDurationMs: totalApiDurationMs,
|
||||
numTurns: turnCount,
|
||||
usage,
|
||||
totalCostUsd: calculateApproximateCost(usage),
|
||||
stats,
|
||||
});
|
||||
} else {
|
||||
// Text output mode
|
||||
// Text output mode - no usage needed
|
||||
process.stdout.write('\n');
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
const usage = extractUsageFromGeminiClient(config.getGeminiClient());
|
||||
// For JSON and STREAM_JSON modes, compute usage from metrics
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (adapter) {
|
||||
const metrics = uiTelemetryService.getMetrics();
|
||||
const usage = computeUsageFromMetrics(metrics);
|
||||
// Get stats for JSON format output
|
||||
const stats =
|
||||
outputFormat === OutputFormat.JSON
|
||||
@@ -354,7 +356,6 @@ export async function runNonInteractive(
|
||||
numTurns: turnCount,
|
||||
errorMessage: message,
|
||||
usage,
|
||||
totalCostUsd: calculateApproximateCost(usage),
|
||||
stats,
|
||||
});
|
||||
}
|
||||
|
||||
1150
packages/cli/src/utils/nonInteractiveHelpers.test.ts
Normal file
1150
packages/cli/src/utils/nonInteractiveHelpers.test.ts
Normal file
File diff suppressed because it is too large
Load Diff
@@ -11,19 +11,20 @@ import type {
|
||||
OutputUpdateHandler,
|
||||
ToolCallRequestInfo,
|
||||
ToolCallResponseInfo,
|
||||
SessionMetrics,
|
||||
} from '@qwen-code/qwen-code-core';
|
||||
import { ToolErrorType } from '@qwen-code/qwen-code-core';
|
||||
import type { Part, PartListUnion } from '@google/genai';
|
||||
import type {
|
||||
CLIUserMessage,
|
||||
Usage,
|
||||
ExtendedUsage,
|
||||
PermissionMode,
|
||||
CLISystemMessage,
|
||||
} from '../nonInteractive/types.js';
|
||||
import { CommandService } from '../services/CommandService.js';
|
||||
import { BuiltinCommandLoader } from '../services/BuiltinCommandLoader.js';
|
||||
import type { JsonOutputAdapterInterface } from '../nonInteractive/io/BaseJsonOutputAdapter.js';
|
||||
import { computeSessionStats } from '../ui/utils/computeStats.js';
|
||||
|
||||
/**
|
||||
* Normalizes various part list formats into a consistent Part[] array.
|
||||
@@ -147,20 +148,38 @@ export function extractUsageFromGeminiClient(
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates approximate cost for API usage.
|
||||
* Currently returns 0 as a placeholder - cost calculation logic can be added here.
|
||||
* Computes Usage information from SessionMetrics using computeSessionStats.
|
||||
* Aggregates token usage across all models in the session.
|
||||
*
|
||||
* @param usage - Usage information from API response
|
||||
* @returns Approximate cost in USD or undefined if not calculable
|
||||
* @param metrics - Session metrics from uiTelemetryService
|
||||
* @returns Usage object with token counts
|
||||
*/
|
||||
export function calculateApproximateCost(
|
||||
usage: Usage | ExtendedUsage | undefined,
|
||||
): number | undefined {
|
||||
if (!usage) {
|
||||
return undefined;
|
||||
export function computeUsageFromMetrics(metrics: SessionMetrics): Usage {
|
||||
const stats = computeSessionStats(metrics);
|
||||
const { models } = metrics;
|
||||
|
||||
// Sum up output tokens (candidates) and total tokens across all models
|
||||
const totalOutputTokens = Object.values(models).reduce(
|
||||
(acc, model) => acc + model.tokens.candidates,
|
||||
0,
|
||||
);
|
||||
const totalTokens = Object.values(models).reduce(
|
||||
(acc, model) => acc + model.tokens.total,
|
||||
0,
|
||||
);
|
||||
|
||||
const usage: Usage = {
|
||||
input_tokens: stats.totalPromptTokens,
|
||||
output_tokens: totalOutputTokens,
|
||||
cache_read_input_tokens: stats.totalCachedTokens,
|
||||
};
|
||||
|
||||
// Only include total_tokens if it's greater than 0
|
||||
if (totalTokens > 0) {
|
||||
usage.total_tokens = totalTokens;
|
||||
}
|
||||
// TODO: Implement actual cost calculation based on token counts and model pricing
|
||||
return 0;
|
||||
|
||||
return usage;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user