abbyswag/qwen-code
1
0
Fork 0
mirror of https://github.com/QwenLM/qwen-code.git synced 2025-12-20 08:47:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

# 🚀 Sync Gemini CLI v0.2.1 - Major Feature Update (#483)

Browse Source
This commit is contained in:
tanzhenxin
2025-09-01 14:48:55 +08:00
committed by GitHub
parent 1610c1586e
commit 2572faf726
292 changed files with 19401 additions and 5941 deletions
Download Patch File Download Diff File Expand all files Collapse all files

181
packages/core/src/telemetry/clearcut-logger/clearcut-logger.test.ts
View File

@@ -4,6 +4,7 @@
* SPDX-License-Identifier: Apache-2.0
*/
import 'vitest';
import {
vi,
describe,
@@ -13,8 +14,13 @@ import {
beforeAll,
afterAll,
} from 'vitest';
import { ClearcutLogger, LogEventEntry, TEST_ONLY } from './clearcut-logger.js';
import {
ClearcutLogger,
LogEvent,
LogEventEntry,
EventNames,
TEST_ONLY,
} from './clearcut-logger.js';
import { ConfigParameters } from '../../config/config.js';
import * as userAccount from '../../utils/user_account.js';
import * as userId from '../../utils/user_id.js';
@@ -22,6 +28,48 @@ import { EventMetadataKey } from './event-metadata-key.js';
import { makeFakeConfig } from '../../test-utils/config.js';
import { http, HttpResponse } from 'msw';
import { server } from '../../mocks/msw.js';
import { makeChatCompressionEvent } from '../types.js';
interface CustomMatchers<R = unknown> {
toHaveMetadataValue: ([key, value]: [EventMetadataKey, string]) => R;
toHaveEventName: (name: EventNames) => R;
}
declare module 'vitest' {
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-empty-object-type
interface Matchers<T = any> extends CustomMatchers<T> {}
}
expect.extend({
toHaveEventName(received: LogEventEntry[], name: EventNames) {
const { isNot } = this;
const event = JSON.parse(received[0].source_extension_json) as LogEvent;
const pass = event.event_name === (name as unknown as string);
return {
pass,
message: () =>
`event name ${event.event_name} does${isNot ? ' not ' : ''} match ${name}}`,
};
},
toHaveMetadataValue(
received: LogEventEntry[],
[key, value]: [EventMetadataKey, string],
) {
const { isNot } = this;
const event = JSON.parse(received[0].source_extension_json) as LogEvent;
const metadata = event['event_metadata'][0];
const data = metadata.find((m) => m.gemini_cli_key === key)?.value;
const pass = data !== undefined && data === value;
return {
pass,
message: () =>
`event ${received} does${isNot ? ' not' : ''} have ${value}}`,
};
},
});
vi.mock('../../utils/user_account');
vi.mock('../../utils/user_id');
@@ -47,6 +95,7 @@ describe('ClearcutLogger', () => {
const CLEARCUT_URL = 'https://play.googleapis.com/log';
const MOCK_DATE = new Date('2025-01-02T00:00:00.000Z');
const EXAMPLE_RESPONSE = `["${NEXT_WAIT_MS}",null,[[["ANDROID_BACKUP",0],["BATTERY_STATS",0],["SMART_SETUP",0],["TRON",0]],-3334737594024971225],[]]`;
// A helper to get the internal events array for testing
const getEvents = (l: ClearcutLogger): LogEventEntry[][] =>
l['events'].toArray() as LogEventEntry[][];
@@ -130,9 +179,9 @@ describe('ClearcutLogger', () => {
lifetimeGoogleAccounts: 9001,
});
const event = logger?.createLogEvent('abc', []);
const event = logger?.createLogEvent(EventNames.API_ERROR, []);
expect(event?.event_metadata[0][0]).toEqual({
expect(event?.event_metadata[0]).toContainEqual({
gemini_cli_key: EventMetadataKey.GEMINI_CLI_GOOGLE_ACCOUNTS_COUNT,
value: '9001',
});
@@ -143,23 +192,23 @@ describe('ClearcutLogger', () => {
vi.stubEnv('GITHUB_SHA', '8675309');
const event = logger?.createLogEvent('abc', []);
const event = logger?.createLogEvent(EventNames.CHAT_COMPRESSION, []);
expect(event?.event_metadata[0][1]).toEqual({
expect(event?.event_metadata[0]).toContainEqual({
gemini_cli_key: EventMetadataKey.GEMINI_CLI_SURFACE,
value: 'GitHub',
});
});
it('honors the value from env.SURFACE over all others', () => {
it('logs the current surface', () => {
const { logger } = setup({});
vi.stubEnv('TERM_PROGRAM', 'vscode');
vi.stubEnv('SURFACE', 'ide-1234');
const event = logger?.createLogEvent('abc', []);
const event = logger?.createLogEvent(EventNames.API_ERROR, []);
expect(event?.event_metadata[0][1]).toEqual({
expect(event?.event_metadata[0]).toContainEqual({
gemini_cli_key: EventMetadataKey.GEMINI_CLI_SURFACE,
value: 'ide-1234',
});
@@ -202,7 +251,7 @@ describe('ClearcutLogger', () => {
expectedValue: 'cloudshell',
},
])(
'logs the current surface for as $expectedValue, preempting vscode detection',
'logs the current surface as $expectedValue, preempting vscode detection',
({ env, expectedValue }) => {
const { logger } = setup({});
@@ -224,9 +273,8 @@ describe('ClearcutLogger', () => {
vi.stubEnv(key, value);
}
vi.stubEnv('TERM_PROGRAM', 'vscode');
const event = logger?.createLogEvent('abc', []);
expect(event?.event_metadata[0][1]).toEqual({
const event = logger?.createLogEvent(EventNames.API_ERROR, []);
expect(event?.event_metadata[0][3]).toEqual({
gemini_cli_key: EventMetadataKey.GEMINI_CLI_SURFACE,
value: expectedValue,
});
@@ -234,10 +282,34 @@ describe('ClearcutLogger', () => {
);
});
describe('logChatCompressionEvent', () => {
it('logs an event with proper fields', () => {
const { logger } = setup();
logger?.logChatCompressionEvent(
makeChatCompressionEvent({
tokens_before: 9001,
tokens_after: 8000,
}),
);
const events = getEvents(logger!);
expect(events.length).toBe(1);
expect(events[0]).toHaveEventName(EventNames.CHAT_COMPRESSION);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_COMPRESSION_TOKENS_BEFORE,
'9001',
]);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_COMPRESSION_TOKENS_AFTER,
'8000',
]);
});
});
describe('enqueueLogEvent', () => {
it('should add events to the queue', () => {
const { logger } = setup();
logger!.enqueueLogEvent({ test: 'event1' });
logger!.enqueueLogEvent(logger!.createLogEvent(EventNames.API_ERROR));
expect(getEventsSize(logger!)).toBe(1);
});
@@ -245,27 +317,43 @@ describe('ClearcutLogger', () => {
const { logger } = setup();
for (let i = 0; i < TEST_ONLY.MAX_EVENTS; i++) {
logger!.enqueueLogEvent({ event_id: i });
logger!.enqueueLogEvent(
logger!.createLogEvent(EventNames.API_ERROR, [
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_AI_ADDED_LINES,
value: `${i}`,
},
]),
);
}
expect(getEventsSize(logger!)).toBe(TEST_ONLY.MAX_EVENTS);
const firstEvent = JSON.parse(
getEvents(logger!)[0][0].source_extension_json,
);
expect(firstEvent.event_id).toBe(0);
let events = getEvents(logger!);
expect(events.length).toBe(TEST_ONLY.MAX_EVENTS);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_AI_ADDED_LINES,
'0',
]);
// This should push out the first event
logger!.enqueueLogEvent({ event_id: TEST_ONLY.MAX_EVENTS });
logger!.enqueueLogEvent(
logger!.createLogEvent(EventNames.API_ERROR, [
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_AI_ADDED_LINES,
value: `${TEST_ONLY.MAX_EVENTS}`,
},
]),
);
events = getEvents(logger!);
expect(events.length).toBe(TEST_ONLY.MAX_EVENTS);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_AI_ADDED_LINES,
'1',
]);
expect(getEventsSize(logger!)).toBe(TEST_ONLY.MAX_EVENTS);
const newFirstEvent = JSON.parse(
getEvents(logger!)[0][0].source_extension_json,
);
expect(newFirstEvent.event_id).toBe(1);
const lastEvent = JSON.parse(
getEvents(logger!)[TEST_ONLY.MAX_EVENTS - 1][0].source_extension_json,
);
expect(lastEvent.event_id).toBe(TEST_ONLY.MAX_EVENTS);
expect(events.at(TEST_ONLY.MAX_EVENTS - 1)).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_AI_ADDED_LINES,
`${TEST_ONLY.MAX_EVENTS}`,
]);
});
});
@@ -277,7 +365,7 @@ describe('ClearcutLogger', () => {
},
});
logger!.enqueueLogEvent({ event_id: 1 });
logger!.enqueueLogEvent(logger!.createLogEvent(EventNames.API_ERROR));
const response = await logger!.flushToClearcut();
@@ -287,7 +375,7 @@ describe('ClearcutLogger', () => {
it('should clear events on successful flush', async () => {
const { logger } = setup();
logger!.enqueueLogEvent({ event_id: 1 });
logger!.enqueueLogEvent(logger!.createLogEvent(EventNames.API_ERROR));
const response = await logger!.flushToClearcut();
expect(getEvents(logger!)).toEqual([]);
@@ -298,8 +386,8 @@ describe('ClearcutLogger', () => {
const { logger } = setup();
server.resetHandlers(http.post(CLEARCUT_URL, () => HttpResponse.error()));
logger!.enqueueLogEvent({ event_id: 1 });
logger!.enqueueLogEvent({ event_id: 2 });
logger!.enqueueLogEvent(logger!.createLogEvent(EventNames.API_REQUEST));
logger!.enqueueLogEvent(logger!.createLogEvent(EventNames.API_ERROR));
expect(getEventsSize(logger!)).toBe(2);
const x = logger!.flushToClearcut();
@@ -307,7 +395,9 @@ describe('ClearcutLogger', () => {
expect(getEventsSize(logger!)).toBe(2);
const events = getEvents(logger!);
expect(JSON.parse(events[0][0].source_extension_json).event_id).toBe(1);
expect(events.length).toBe(2);
expect(events[0]).toHaveEventName(EventNames.API_REQUEST);
});
it('should handle an HTTP error and requeue events', async () => {
@@ -326,23 +416,22 @@ describe('ClearcutLogger', () => {
),
);
logger!.enqueueLogEvent({ event_id: 1 });
logger!.enqueueLogEvent({ event_id: 2 });
logger!.enqueueLogEvent(logger!.createLogEvent(EventNames.API_REQUEST));
logger!.enqueueLogEvent(logger!.createLogEvent(EventNames.API_ERROR));
expect(getEvents(logger!).length).toBe(2);
await logger!.flushToClearcut();
expect(getEvents(logger!).length).toBe(2);
const events = getEvents(logger!);
expect(JSON.parse(events[0][0].source_extension_json).event_id).toBe(1);
expect(events[0]).toHaveEventName(EventNames.API_REQUEST);
});
});
describe('requeueFailedEvents logic', () => {
it('should limit the number of requeued events to max_retry_events', () => {
const { logger } = setup();
const maxRetryEvents = TEST_ONLY.MAX_RETRY_EVENTS;
const eventsToLogCount = maxRetryEvents + 5;
const eventsToLogCount = TEST_ONLY.MAX_RETRY_EVENTS + 5;
const eventsToSend: LogEventEntry[][] = [];
for (let i = 0; i < eventsToLogCount; i++) {
eventsToSend.push([
@@ -355,13 +444,13 @@ describe('ClearcutLogger', () => {
requeueFailedEvents(logger!, eventsToSend);
expect(getEventsSize(logger!)).toBe(maxRetryEvents);
expect(getEventsSize(logger!)).toBe(TEST_ONLY.MAX_RETRY_EVENTS);
const firstRequeuedEvent = JSON.parse(
getEvents(logger!)[0][0].source_extension_json,
);
) as { event_id: string };
// The last `maxRetryEvents` are kept. The oldest of those is at index `eventsToLogCount - maxRetryEvents`.
expect(firstRequeuedEvent.event_id).toBe(
eventsToLogCount - maxRetryEvents,
eventsToLogCount - TEST_ONLY.MAX_RETRY_EVENTS,
);
});
@@ -371,7 +460,7 @@ describe('ClearcutLogger', () => {
const spaceToLeave = 5;
const initialEventCount = maxEvents - spaceToLeave;
for (let i = 0; i < initialEventCount; i++) {
logger!.enqueueLogEvent({ event_id: `initial_${i}` });
logger!.enqueueLogEvent(logger!.createLogEvent(EventNames.API_ERROR));
}
expect(getEventsSize(logger!)).toBe(initialEventCount);
@@ -398,7 +487,7 @@ describe('ClearcutLogger', () => {
// The first element in the deque is the one with id 'failed_5'.
const firstRequeuedEvent = JSON.parse(
getEvents(logger!)[0][0].source_extension_json,
);
) as { event_id: string };
expect(firstRequeuedEvent.event_id).toBe('failed_5');
});
});

305
packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts
View File

@@ -7,19 +7,21 @@
import { HttpsProxyAgent } from 'https-proxy-agent';
import {
StartSessionEvent,
EndSessionEvent,
UserPromptEvent,
ToolCallEvent,
ApiRequestEvent,
ApiResponseEvent,
ApiErrorEvent,
FlashFallbackEvent,
LoopDetectedEvent,
NextSpeakerCheckEvent,
SlashCommandEvent,
MalformedJsonResponseEvent,
IdeConnectionEvent,
KittySequenceOverflowEvent,
ChatCompressionEvent,
InvalidChunkEvent,
ContentRetryEvent,
ContentRetryFailureEvent,
} from '../types.js';
import { EventMetadataKey } from './event-metadata-key.js';
import { Config } from '../../config/config.js';
@@ -30,22 +32,29 @@ import {
} from '../../utils/user_account.js';
import { getInstallationId } from '../../utils/user_id.js';
import { FixedDeque } from 'mnemonist';
import { GIT_COMMIT_INFO, CLI_VERSION } from '../../generated/git-commit.js';
import { DetectedIde, detectIde } from '../../ide/detect-ide.js';
const start_session_event_name = 'start_session';
const new_prompt_event_name = 'new_prompt';
const tool_call_event_name = 'tool_call';
const api_request_event_name = 'api_request';
const api_response_event_name = 'api_response';
const api_error_event_name = 'api_error';
const end_session_event_name = 'end_session';
const flash_fallback_event_name = 'flash_fallback';
const loop_detected_event_name = 'loop_detected';
const next_speaker_check_event_name = 'next_speaker_check';
const slash_command_event_name = 'slash_command';
const malformed_json_response_event_name = 'malformed_json_response';
const ide_connection_event_name = 'ide_connection';
const kitty_sequence_overflow_event_name = 'kitty_sequence_overflow';
export enum EventNames {
START_SESSION = 'start_session',
NEW_PROMPT = 'new_prompt',
TOOL_CALL = 'tool_call',
API_REQUEST = 'api_request',
API_RESPONSE = 'api_response',
API_ERROR = 'api_error',
END_SESSION = 'end_session',
FLASH_FALLBACK = 'flash_fallback',
LOOP_DETECTED = 'loop_detected',
NEXT_SPEAKER_CHECK = 'next_speaker_check',
SLASH_COMMAND = 'slash_command',
MALFORMED_JSON_RESPONSE = 'malformed_json_response',
IDE_CONNECTION = 'ide_connection',
KITTY_SEQUENCE_OVERFLOW = 'kitty_sequence_overflow',
CHAT_COMPRESSION = 'chat_compression',
INVALID_CHUNK = 'invalid_chunk',
CONTENT_RETRY = 'content_retry',
CONTENT_RETRY_FAILURE = 'content_retry_failure',
}
export interface LogResponse {
nextRequestWaitMs?: number;
@@ -57,7 +66,7 @@ export interface LogEventEntry {
}
export interface EventValue {
gemini_cli_key: EventMetadataKey | string;
gemini_cli_key: EventMetadataKey;
value: string;
}
@@ -86,11 +95,11 @@ export interface LogRequest {
* methods might have in their runtimes.
*/
function determineSurface(): string {
if (process.env.SURFACE) {
return process.env.SURFACE;
} else if (process.env.GITHUB_SHA) {
if (process.env['SURFACE']) {
return process.env['SURFACE'];
} else if (process.env['GITHUB_SHA']) {
return 'GitHub';
} else if (process.env.TERM_PROGRAM === 'vscode') {
} else if (process.env['TERM_PROGRAM'] === 'vscode') {
return detectIde() || DetectedIde.VSCode;
} else {
return 'SURFACE_NOT_SET';
@@ -124,6 +133,8 @@ const MAX_RETRY_EVENTS = 100;
export class ClearcutLogger {
private static instance: ClearcutLogger;
private config?: Config;
private sessionData: EventValue[] = [];
private promptId: string = '';
/**
* Queue of pending events that need to be flushed to the server. New events
@@ -150,6 +161,7 @@ export class ClearcutLogger {
private constructor(config?: Config) {
this.config = config;
this.events = new FixedDeque<LogEventEntry[]>(Array, MAX_EVENTS);
this.promptId = config?.getSessionId() ?? '';
}
static getInstance(config?: Config): ClearcutLogger | undefined {
@@ -167,7 +179,7 @@ export class ClearcutLogger {
ClearcutLogger.instance = undefined;
}
enqueueLogEvent(event: object): void {
enqueueLogEvent(event: LogEvent): void {
try {
// Manually handle overflow for FixedDeque, which throws when full.
const wasAtCapacity = this.events.size >= MAX_EVENTS;
@@ -195,15 +207,18 @@ export class ClearcutLogger {
}
}
createLogEvent(name: string, data: EventValue[]): LogEvent {
createLogEvent(eventName: EventNames, data: EventValue[] = []): LogEvent {
const email = getCachedGoogleAccount();
data = addDefaultFields(data);
if (eventName !== EventNames.START_SESSION) {
data.push(...this.sessionData);
}
data = this.addDefaultFields(data);
const logEvent: LogEvent = {
console_type: 'GEMINI_CLI',
application: 102, // GEMINI_CLI
event_name: name,
event_name: eventName as string,
event_metadata: [data],
};
@@ -314,10 +329,6 @@ export class ClearcutLogger {
gemini_cli_key: EventMetadataKey.GEMINI_CLI_START_SESSION_MODEL,
value: event.model,
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_SESSION_ID,
value: this.config?.getSessionId() ?? '',
},
{
gemini_cli_key:
EventMetadataKey.GEMINI_CLI_START_SESSION_EMBEDDING_MODEL,
@@ -375,35 +386,25 @@ export class ClearcutLogger {
value: event.telemetry_log_user_prompts_enabled.toString(),
},
];
this.sessionData = data;
// Flush start event immediately
this.enqueueLogEvent(this.createLogEvent(start_session_event_name, data));
this.enqueueLogEvent(this.createLogEvent(EventNames.START_SESSION, data));
this.flushToClearcut().catch((error) => {
console.debug('Error flushing to Clearcut:', error);
});
}
logNewPromptEvent(event: UserPromptEvent): void {
this.promptId = event.prompt_id;
const data: EventValue[] = [
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_USER_PROMPT_LENGTH,
value: JSON.stringify(event.prompt_length),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_SESSION_ID,
value: this.config?.getSessionId() ?? '',
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_PROMPT_ID,
value: JSON.stringify(event.prompt_id),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_AUTH_TYPE,
value: JSON.stringify(event.auth_type),
},
];
this.enqueueLogEvent(this.createLogEvent(new_prompt_event_name, data));
this.enqueueLogEvent(this.createLogEvent(EventNames.NEW_PROMPT, data));
this.flushIfNeeded();
}
@@ -413,10 +414,6 @@ export class ClearcutLogger {
gemini_cli_key: EventMetadataKey.GEMINI_CLI_TOOL_CALL_NAME,
value: JSON.stringify(event.function_name),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_PROMPT_ID,
value: JSON.stringify(event.prompt_id),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_TOOL_CALL_DECISION,
value: JSON.stringify(event.decision),
@@ -437,6 +434,10 @@ export class ClearcutLogger {
gemini_cli_key: EventMetadataKey.GEMINI_CLI_TOOL_CALL_ERROR_TYPE,
value: JSON.stringify(event.error_type),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_TOOL_TYPE,
value: JSON.stringify(event.tool_type),
},
];
if (event.metadata) {
@@ -457,7 +458,7 @@ export class ClearcutLogger {
}
}
const logEvent = this.createLogEvent(tool_call_event_name, data);
const logEvent = this.createLogEvent(EventNames.TOOL_CALL, data);
this.enqueueLogEvent(logEvent);
this.flushIfNeeded();
}
@@ -468,13 +469,9 @@ export class ClearcutLogger {
gemini_cli_key: EventMetadataKey.GEMINI_CLI_API_REQUEST_MODEL,
value: JSON.stringify(event.model),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_PROMPT_ID,
value: JSON.stringify(event.prompt_id),
},
];
this.enqueueLogEvent(this.createLogEvent(api_request_event_name, data));
this.enqueueLogEvent(this.createLogEvent(EventNames.API_REQUEST, data));
this.flushIfNeeded();
}
@@ -484,10 +481,6 @@ export class ClearcutLogger {
gemini_cli_key: EventMetadataKey.GEMINI_CLI_API_RESPONSE_MODEL,
value: JSON.stringify(event.model),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_PROMPT_ID,
value: JSON.stringify(event.prompt_id),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_API_RESPONSE_STATUS_CODE,
value: JSON.stringify(event.status_code),
@@ -525,13 +518,9 @@ export class ClearcutLogger {
EventMetadataKey.GEMINI_CLI_API_RESPONSE_TOOL_TOKEN_COUNT,
value: JSON.stringify(event.tool_token_count),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_AUTH_TYPE,
value: JSON.stringify(event.auth_type),
},
];
this.enqueueLogEvent(this.createLogEvent(api_response_event_name, data));
this.enqueueLogEvent(this.createLogEvent(EventNames.API_RESPONSE, data));
this.flushIfNeeded();
}
@@ -541,10 +530,6 @@ export class ClearcutLogger {
gemini_cli_key: EventMetadataKey.GEMINI_CLI_API_ERROR_MODEL,
value: JSON.stringify(event.model),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_PROMPT_ID,
value: JSON.stringify(event.prompt_id),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_API_ERROR_TYPE,
value: JSON.stringify(event.error_type),
@@ -557,29 +542,31 @@ export class ClearcutLogger {
gemini_cli_key: EventMetadataKey.GEMINI_CLI_API_ERROR_DURATION_MS,
value: JSON.stringify(event.duration_ms),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_AUTH_TYPE,
value: JSON.stringify(event.auth_type),
},
];
this.enqueueLogEvent(this.createLogEvent(api_error_event_name, data));
this.enqueueLogEvent(this.createLogEvent(EventNames.API_ERROR, data));
this.flushIfNeeded();
}
logFlashFallbackEvent(event: FlashFallbackEvent): void {
logChatCompressionEvent(event: ChatCompressionEvent): void {
const data: EventValue[] = [
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_AUTH_TYPE,
value: JSON.stringify(event.auth_type),
gemini_cli_key: EventMetadataKey.GEMINI_CLI_COMPRESSION_TOKENS_BEFORE,
value: `${event.tokens_before}`,
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_SESSION_ID,
value: this.config?.getSessionId() ?? '',
gemini_cli_key: EventMetadataKey.GEMINI_CLI_COMPRESSION_TOKENS_AFTER,
value: `${event.tokens_after}`,
},
];
this.enqueueLogEvent(this.createLogEvent(flash_fallback_event_name, data));
this.enqueueLogEvent(
this.createLogEvent(EventNames.CHAT_COMPRESSION, data),
);
}
logFlashFallbackEvent(): void {
this.enqueueLogEvent(this.createLogEvent(EventNames.FLASH_FALLBACK, []));
this.flushToClearcut().catch((error) => {
console.debug('Error flushing to Clearcut:', error);
});
@@ -587,26 +574,18 @@ export class ClearcutLogger {
logLoopDetectedEvent(event: LoopDetectedEvent): void {
const data: EventValue[] = [
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_PROMPT_ID,
value: JSON.stringify(event.prompt_id),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_LOOP_DETECTED_TYPE,
value: JSON.stringify(event.loop_type),
},
];
this.enqueueLogEvent(this.createLogEvent(loop_detected_event_name, data));
this.enqueueLogEvent(this.createLogEvent(EventNames.LOOP_DETECTED, data));
this.flushIfNeeded();
}
logNextSpeakerCheck(event: NextSpeakerCheckEvent): void {
const data: EventValue[] = [
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_PROMPT_ID,
value: JSON.stringify(event.prompt_id),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_RESPONSE_FINISH_REASON,
value: JSON.stringify(event.finish_reason),
@@ -615,14 +594,10 @@ export class ClearcutLogger {
gemini_cli_key: EventMetadataKey.GEMINI_CLI_NEXT_SPEAKER_CHECK_RESULT,
value: JSON.stringify(event.result),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_SESSION_ID,
value: this.config?.getSessionId() ?? '',
},
];
this.enqueueLogEvent(
this.createLogEvent(next_speaker_check_event_name, data),
this.createLogEvent(EventNames.NEXT_SPEAKER_CHECK, data),
);
this.flushIfNeeded();
}
@@ -649,7 +624,7 @@ export class ClearcutLogger {
});
}
this.enqueueLogEvent(this.createLogEvent(slash_command_event_name, data));
this.enqueueLogEvent(this.createLogEvent(EventNames.SLASH_COMMAND, data));
this.flushIfNeeded();
}
@@ -663,7 +638,7 @@ export class ClearcutLogger {
];
this.enqueueLogEvent(
this.createLogEvent(malformed_json_response_event_name, data),
this.createLogEvent(EventNames.MALFORMED_JSON_RESPONSE, data),
);
this.flushIfNeeded();
}
@@ -676,7 +651,7 @@ export class ClearcutLogger {
},
];
this.enqueueLogEvent(this.createLogEvent(ide_connection_event_name, data));
this.enqueueLogEvent(this.createLogEvent(EventNames.IDE_CONNECTION, data));
this.flushIfNeeded();
}
@@ -693,26 +668,125 @@ export class ClearcutLogger {
];
this.enqueueLogEvent(
this.createLogEvent(kitty_sequence_overflow_event_name, data),
this.createLogEvent(EventNames.KITTY_SEQUENCE_OVERFLOW, data),
);
this.flushIfNeeded();
}
logEndSessionEvent(event: EndSessionEvent): void {
const data: EventValue[] = [
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_SESSION_ID,
value: event?.session_id?.toString() ?? '',
},
];
logEndSessionEvent(): void {
// Flush immediately on session end.
this.enqueueLogEvent(this.createLogEvent(end_session_event_name, data));
this.enqueueLogEvent(this.createLogEvent(EventNames.END_SESSION, []));
this.flushToClearcut().catch((error) => {
console.debug('Error flushing to Clearcut:', error);
});
}
logInvalidChunkEvent(event: InvalidChunkEvent): void {
const data: EventValue[] = [];
if (event.error_message) {
data.push({
gemini_cli_key: EventMetadataKey.GEMINI_CLI_INVALID_CHUNK_ERROR_MESSAGE,
value: event.error_message,
});
}
this.enqueueLogEvent(this.createLogEvent(EventNames.INVALID_CHUNK, data));
this.flushIfNeeded();
}
logContentRetryEvent(event: ContentRetryEvent): void {
const data: EventValue[] = [
{
gemini_cli_key:
EventMetadataKey.GEMINI_CLI_CONTENT_RETRY_ATTEMPT_NUMBER,
value: String(event.attempt_number),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_CONTENT_RETRY_ERROR_TYPE,
value: event.error_type,
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_CONTENT_RETRY_DELAY_MS,
value: String(event.retry_delay_ms),
},
];
this.enqueueLogEvent(this.createLogEvent(EventNames.CONTENT_RETRY, data));
this.flushIfNeeded();
}
logContentRetryFailureEvent(event: ContentRetryFailureEvent): void {
const data: EventValue[] = [
{
gemini_cli_key:
EventMetadataKey.GEMINI_CLI_CONTENT_RETRY_FAILURE_TOTAL_ATTEMPTS,
value: String(event.total_attempts),
},
{
gemini_cli_key:
EventMetadataKey.GEMINI_CLI_CONTENT_RETRY_FAILURE_FINAL_ERROR_TYPE,
value: event.final_error_type,
},
];
if (event.total_duration_ms) {
data.push({
gemini_cli_key:
EventMetadataKey.GEMINI_CLI_CONTENT_RETRY_FAILURE_TOTAL_DURATION_MS,
value: String(event.total_duration_ms),
});
}
this.enqueueLogEvent(
this.createLogEvent(EventNames.CONTENT_RETRY_FAILURE, data),
);
this.flushIfNeeded();
}
/**
* Adds default fields to data, and returns a new data array. This fields
* should exist on all log events.
*/
addDefaultFields(data: EventValue[]): EventValue[] {
const totalAccounts = getLifetimeGoogleAccounts();
const surface = determineSurface();
const defaultLogMetadata: EventValue[] = [
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_SESSION_ID,
value: this.config?.getSessionId() ?? '',
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_AUTH_TYPE,
value: JSON.stringify(
this.config?.getContentGeneratorConfig()?.authType,
),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_GOOGLE_ACCOUNTS_COUNT,
value: `${totalAccounts}`,
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_SURFACE,
value: surface,
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_VERSION,
value: CLI_VERSION,
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_GIT_COMMIT_HASH,
value: GIT_COMMIT_INFO,
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_PROMPT_ID,
value: this.promptId,
},
];
return [...data, ...defaultLogMetadata];
}
getProxyAgent() {
const proxyUrl = this.config?.getProxy();
if (!proxyUrl) return undefined;
@@ -726,8 +800,7 @@ export class ClearcutLogger {
}
shutdown() {
const event = new EndSessionEvent(this.config);
this.logEndSessionEvent(event);
this.logEndSessionEvent();
}
private requeueFailedEvents(eventsToSend: LogEventEntry[][]): void {
@@ -781,26 +854,6 @@ export class ClearcutLogger {
}
}
/**
* Adds default fields to data, and returns a new data array. This fields
* should exist on all log events.
*/
function addDefaultFields(data: EventValue[]): EventValue[] {
const totalAccounts = getLifetimeGoogleAccounts();
const surface = determineSurface();
const defaultLogMetadata: EventValue[] = [
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_GOOGLE_ACCOUNTS_COUNT,
value: `${totalAccounts}`,
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_SURFACE,
value: surface,
},
];
return [...data, ...defaultLogMetadata];
}
export const TEST_ONLY = {
MAX_RETRY_EVENTS,
MAX_EVENTS,

106
packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts
View File

@@ -157,6 +157,12 @@ export enum EventMetadataKey {
// Logs the session id
GEMINI_CLI_SESSION_ID = 40,
// Logs the Gemini CLI version
GEMINI_CLI_VERSION = 54,
// Logs the Gemini CLI Git commit hash
GEMINI_CLI_GIT_COMMIT_HASH = 55,
// ==========================================================================
// Loop Detected Event Keys
// ===========================================================================
@@ -217,22 +223,96 @@ export enum EventMetadataKey {
// Kitty Sequence Overflow Event Keys
// ===========================================================================
// Logs the truncated kitty sequence.
GEMINI_CLI_KITTY_TRUNCATED_SEQUENCE = 52,
// Logs the length of the kitty sequence that overflowed.
GEMINI_CLI_KITTY_SEQUENCE_LENGTH = 53,
// Logs the truncated kitty sequence.
GEMINI_CLI_KITTY_TRUNCATED_SEQUENCE = 52,
}
// ==========================================================================
// Conversation Finished Event Keys
// ===========================================================================
export function getEventMetadataKey(
keyName: string,
): EventMetadataKey | undefined {
// Access the enum member by its string name
const key = EventMetadataKey[keyName as keyof typeof EventMetadataKey];
// Logs the approval mode of the session.
GEMINI_CLI_APPROVAL_MODE = 58,
// Check if the result is a valid enum member (not undefined and is a number)
if (typeof key === 'number') {
return key;
}
return undefined;
// Logs the number of turns
GEMINI_CLI_CONVERSATION_TURN_COUNT = 59,
// Logs the number of tokens before context window compression.
GEMINI_CLI_COMPRESSION_TOKENS_BEFORE = 60,
// Logs the number of tokens after context window compression.
GEMINI_CLI_COMPRESSION_TOKENS_AFTER = 61,
// Logs tool type whether it is mcp or native.
GEMINI_CLI_TOOL_TYPE = 62,
// Logs name of MCP tools as comma separated string
GEMINI_CLI_START_SESSION_MCP_TOOLS = 65,
// ==========================================================================
// Research Event Keys
// ===========================================================================
// Logs the research opt-in status (true/false)
GEMINI_CLI_RESEARCH_OPT_IN_STATUS = 66,
// Logs the contact email for research participation
GEMINI_CLI_RESEARCH_CONTACT_EMAIL = 67,
// Logs the user ID for research events
GEMINI_CLI_RESEARCH_USER_ID = 68,
// Logs the type of research feedback
GEMINI_CLI_RESEARCH_FEEDBACK_TYPE = 69,
// Logs the content of research feedback
GEMINI_CLI_RESEARCH_FEEDBACK_CONTENT = 70,
// Logs survey responses for research feedback (JSON stringified)
GEMINI_CLI_RESEARCH_SURVEY_RESPONSES = 71,
// ==========================================================================
// File Operation Event Keys
// ===========================================================================
// Logs the programming language of the project.
GEMINI_CLI_PROGRAMMING_LANGUAGE = 56,
// Logs the operation type of the file operation.
GEMINI_CLI_FILE_OPERATION_TYPE = 57,
// Logs the number of lines in the file operation.
GEMINI_CLI_FILE_OPERATION_LINES = 72,
// Logs the mimetype of the file in the file operation.
GEMINI_CLI_FILE_OPERATION_MIMETYPE = 73,
// Logs the extension of the file in the file operation.
GEMINI_CLI_FILE_OPERATION_EXTENSION = 74,
// ==========================================================================
// Content Streaming Event Keys
// ===========================================================================
// Logs the error message for an invalid chunk.
GEMINI_CLI_INVALID_CHUNK_ERROR_MESSAGE = 75,
// Logs the attempt number for a content retry.
GEMINI_CLI_CONTENT_RETRY_ATTEMPT_NUMBER = 76,
// Logs the error type for a content retry.
GEMINI_CLI_CONTENT_RETRY_ERROR_TYPE = 77,
// Logs the delay in milliseconds for a content retry.
GEMINI_CLI_CONTENT_RETRY_DELAY_MS = 78,
// Logs the total number of attempts for a content retry failure.
GEMINI_CLI_CONTENT_RETRY_FAILURE_TOTAL_ATTEMPTS = 79,
// Logs the final error type for a content retry failure.
GEMINI_CLI_CONTENT_RETRY_FAILURE_FINAL_ERROR_TYPE = 80,
// Logs the total duration in milliseconds for a content retry failure.
GEMINI_CLI_CONTENT_RETRY_FAILURE_TOTAL_DURATION_MS = 81,
}

9
packages/core/src/telemetry/constants.ts
View File

@@ -16,6 +16,11 @@ export const EVENT_FLASH_FALLBACK = 'qwen-code.flash_fallback';
export const EVENT_NEXT_SPEAKER_CHECK = 'qwen-code.next_speaker_check';
export const EVENT_SLASH_COMMAND = 'qwen-code.slash_command';
export const EVENT_IDE_CONNECTION = 'qwen-code.ide_connection';
export const EVENT_CHAT_COMPRESSION = 'qwen-code.chat_compression';
export const EVENT_INVALID_CHUNK = 'qwen-code.chat.invalid_chunk';
export const EVENT_CONTENT_RETRY = 'qwen-code.chat.content_retry';
export const EVENT_CONTENT_RETRY_FAILURE =
'qwen-code.chat.content_retry_failure';
export const METRIC_TOOL_CALL_COUNT = 'qwen-code.tool.call.count';
export const METRIC_TOOL_CALL_LATENCY = 'qwen-code.tool.call.latency';
@@ -24,3 +29,7 @@ export const METRIC_API_REQUEST_LATENCY = 'qwen-code.api.request.latency';
export const METRIC_TOKEN_USAGE = 'qwen-code.token.usage';
export const METRIC_SESSION_COUNT = 'qwen-code.session.count';
export const METRIC_FILE_OPERATION_COUNT = 'qwen-code.file.operation.count';
export const METRIC_INVALID_CHUNK_COUNT = 'qwen-code.chat.invalid_chunk.count';
export const METRIC_CONTENT_RETRY_COUNT = 'qwen-code.chat.content_retry.count';
export const METRIC_CONTENT_RETRY_FAILURE_COUNT =
'qwen-code.chat.content_retry_failure.count';

3
packages/core/src/telemetry/index.ts
View File

@@ -29,6 +29,7 @@ export {
logFlashFallback,
logSlashCommand,
logKittySequenceOverflow,
logChatCompression,
} from './loggers.js';
export {
StartSessionEvent,
@@ -44,6 +45,8 @@ export {
SlashCommandEvent,
makeSlashCommandEvent,
SlashCommandStatus,
ChatCompressionEvent,
makeChatCompressionEvent,
} from './types.js';
export { SpanStatusCode, ValueType } from '@opentelemetry/api';
export { SemanticAttributes } from '@opentelemetry/semantic-conventions';

3
packages/core/src/telemetry/integration.test.circular.ts
View File

@@ -72,7 +72,8 @@ describe('Circular Reference Integration Test', () => {
const logger = QwenLogger.getInstance(mockConfig);
expect(() => {
logger?.enqueueLogEvent(problematicEvent);
// eslint-disable-next-line @typescript-eslint/no-explicit-any
logger?.enqueueLogEvent(problematicEvent as any);
}).not.toThrow();
});

53
packages/core/src/telemetry/loggers.test.ts
View File

@@ -34,6 +34,7 @@ import {
logUserPrompt,
logToolCall,
logFlashFallback,
logChatCompression,
} from './loggers.js';
import { ToolCallDecision } from './tool-call-decision.js';
import {
@@ -43,12 +44,15 @@ import {
ToolCallEvent,
UserPromptEvent,
FlashFallbackEvent,
makeChatCompressionEvent,
} from './types.js';
import * as metrics from './metrics.js';
import * as sdk from './sdk.js';
import { vi, describe, beforeEach, it, expect } from 'vitest';
import { GenerateContentResponseUsageMetadata } from '@google/genai';
import * as uiTelemetry from './uiTelemetry.js';
import { makeFakeConfig } from '../test-utils/config.js';
import { QwenLogger } from './qwen-logger/qwen-logger.js';
describe('loggers', () => {
const mockLogger = {
@@ -68,6 +72,45 @@ describe('loggers', () => {
vi.setSystemTime(new Date('2025-01-01T00:00:00.000Z'));
});
describe('logChatCompression', () => {
beforeEach(() => {
vi.spyOn(metrics, 'recordChatCompressionMetrics');
vi.spyOn(QwenLogger.prototype, 'logChatCompressionEvent');
});
it('logs the chat compression event to QwenLogger', () => {
const mockConfig = makeFakeConfig();
const event = makeChatCompressionEvent({
tokens_before: 9001,
tokens_after: 9000,
});
logChatCompression(mockConfig, event);
expect(QwenLogger.prototype.logChatCompressionEvent).toHaveBeenCalledWith(
event,
);
});
it('records the chat compression event to OTEL', () => {
const mockConfig = makeFakeConfig();
logChatCompression(
mockConfig,
makeChatCompressionEvent({
tokens_before: 9001,
tokens_after: 9000,
}),
);
expect(metrics.recordChatCompressionMetrics).toHaveBeenCalledWith(
mockConfig,
{ tokens_before: 9001, tokens_after: 9000 },
);
});
});
describe('logCliConfiguration', () => {
it('should log the cli configuration', () => {
const mockConfig = {
@@ -484,6 +527,7 @@ describe('loggers', () => {
success: true,
decision: ToolCallDecision.ACCEPT,
prompt_id: 'prompt-id-1',
tool_type: 'native',
},
});
@@ -493,6 +537,7 @@ describe('loggers', () => {
100,
true,
ToolCallDecision.ACCEPT,
'native',
);
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
@@ -547,6 +592,7 @@ describe('loggers', () => {
success: false,
decision: ToolCallDecision.REJECT,
prompt_id: 'prompt-id-2',
tool_type: 'native',
},
});
@@ -556,6 +602,7 @@ describe('loggers', () => {
100,
false,
ToolCallDecision.REJECT,
'native',
);
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
@@ -613,6 +660,7 @@ describe('loggers', () => {
success: true,
decision: ToolCallDecision.MODIFY,
prompt_id: 'prompt-id-3',
tool_type: 'native',
},
});
@@ -622,6 +670,7 @@ describe('loggers', () => {
100,
true,
ToolCallDecision.MODIFY,
'native',
);
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
@@ -677,6 +726,7 @@ describe('loggers', () => {
duration_ms: 100,
success: true,
prompt_id: 'prompt-id-4',
tool_type: 'native',
},
});
@@ -686,6 +736,7 @@ describe('loggers', () => {
100,
true,
undefined,
'native',
);
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
@@ -746,6 +797,7 @@ describe('loggers', () => {
error_type: ToolErrorType.UNKNOWN,
'error.type': ToolErrorType.UNKNOWN,
prompt_id: 'prompt-id-5',
tool_type: 'native',
},
});
@@ -755,6 +807,7 @@ describe('loggers', () => {
100,
false,
undefined,
'native',
);
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({

131
packages/core/src/telemetry/loggers.ts
View File

@@ -7,7 +7,6 @@
import { LogAttributes, LogRecord, logs } from '@opentelemetry/api-logs';
import { SemanticAttributes } from '@opentelemetry/semantic-conventions';
import { Config } from '../config/config.js';
import { safeJsonStringify } from '../utils/safeJsonStringify.js';
import {
EVENT_API_ERROR,
EVENT_API_REQUEST,
@@ -20,15 +19,11 @@ import {
EVENT_TOOL_CALL,
EVENT_USER_PROMPT,
SERVICE_NAME,
EVENT_CHAT_COMPRESSION,
EVENT_INVALID_CHUNK,
EVENT_CONTENT_RETRY,
EVENT_CONTENT_RETRY_FAILURE,
} from './constants.js';
import {
recordApiErrorMetrics,
recordApiResponseMetrics,
recordTokenUsageMetrics,
recordToolCallMetrics,
} from './metrics.js';
import { QwenLogger } from './qwen-logger/qwen-logger.js';
import { isTelemetrySdkInitialized } from './sdk.js';
import {
ApiErrorEvent,
ApiRequestEvent,
@@ -42,8 +37,25 @@ import {
StartSessionEvent,
ToolCallEvent,
UserPromptEvent,
ChatCompressionEvent,
InvalidChunkEvent,
ContentRetryEvent,
ContentRetryFailureEvent,
} from './types.js';
import { UiEvent, uiTelemetryService } from './uiTelemetry.js';
import {
recordApiErrorMetrics,
recordTokenUsageMetrics,
recordApiResponseMetrics,
recordToolCallMetrics,
recordChatCompressionMetrics,
recordInvalidChunk,
recordContentRetry,
recordContentRetryFailure,
} from './metrics.js';
import { QwenLogger } from './qwen-logger/qwen-logger.js';
import { isTelemetrySdkInitialized } from './sdk.js';
import { uiTelemetryService, UiEvent } from './uiTelemetry.js';
import { safeJsonStringify } from '../utils/safeJsonStringify.js';
const shouldLogUserPrompts = (config: Config): boolean =>
config.getTelemetryLogPromptsEnabled();
@@ -98,7 +110,7 @@ export function logUserPrompt(config: Config, event: UserPromptEvent): void {
};
if (shouldLogUserPrompts(config)) {
attributes.prompt = event.prompt;
attributes['prompt'] = event.prompt;
}
const logger = logs.getLogger(SERVICE_NAME);
@@ -145,6 +157,7 @@ export function logToolCall(config: Config, event: ToolCallEvent): void {
event.duration_ms,
event.success,
event.decision,
event.tool_type,
);
}
@@ -247,7 +260,7 @@ export function logApiResponse(config: Config, event: ApiResponseEvent): void {
'event.timestamp': new Date().toISOString(),
};
if (event.response_text) {
attributes.response_text = event.response_text;
attributes['response_text'] = event.response_text;
}
if (event.error) {
attributes['error.message'] = event.error;
@@ -380,6 +393,31 @@ export function logIdeConnection(
logger.emit(logRecord);
}
export function logChatCompression(
config: Config,
event: ChatCompressionEvent,
): void {
QwenLogger.getInstance(config)?.logChatCompressionEvent(event);
const attributes: LogAttributes = {
...getCommonAttributes(config),
...event,
'event.name': EVENT_CHAT_COMPRESSION,
};
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: `Chat compression (Saved ${event.tokens_before - event.tokens_after} tokens)`,
attributes,
};
logger.emit(logRecord);
recordChatCompressionMetrics(config, {
tokens_before: event.tokens_before,
tokens_after: event.tokens_after,
});
}
export function logKittySequenceOverflow(
config: Config,
event: KittySequenceOverflowEvent,
@@ -397,3 +435,72 @@ export function logKittySequenceOverflow(
};
logger.emit(logRecord);
}
export function logInvalidChunk(
config: Config,
event: InvalidChunkEvent,
): void {
QwenLogger.getInstance(config)?.logInvalidChunkEvent(event);
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {
...getCommonAttributes(config),
'event.name': EVENT_INVALID_CHUNK,
'event.timestamp': event['event.timestamp'],
};
if (event.error_message) {
attributes['error.message'] = event.error_message;
}
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: `Invalid chunk received from stream.`,
attributes,
};
logger.emit(logRecord);
recordInvalidChunk(config);
}
export function logContentRetry(
config: Config,
event: ContentRetryEvent,
): void {
QwenLogger.getInstance(config)?.logContentRetryEvent(event);
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {
...getCommonAttributes(config),
...event,
'event.name': EVENT_CONTENT_RETRY,
};
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: `Content retry attempt ${event.attempt_number} due to ${event.error_type}.`,
attributes,
};
logger.emit(logRecord);
recordContentRetry(config);
}
export function logContentRetryFailure(
config: Config,
event: ContentRetryFailureEvent,
): void {
QwenLogger.getInstance(config)?.logContentRetryFailureEvent(event);
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {
...getCommonAttributes(config),
...event,
'event.name': EVENT_CONTENT_RETRY_FAILURE,
};
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: `All content retries failed after ${event.total_attempts} attempts.`,
attributes,
};
logger.emit(logRecord);
recordContentRetryFailure(config);
}

50
packages/core/src/telemetry/metrics.test.ts
View File

@@ -14,6 +14,7 @@ import type {
} from '@opentelemetry/api';
import { Config } from '../config/config.js';
import { FileOperation } from './metrics.js';
import { makeFakeConfig } from '../test-utils/config.js';
const mockCounterAddFn: Mock<
(value: number, attributes?: Attributes, context?: Context) => void
@@ -28,18 +29,18 @@ const mockCreateHistogramFn: Mock<
(name: string, options?: unknown) => Histogram
> = vi.fn();
const mockCounterInstance = {
const mockCounterInstance: Counter = {
add: mockCounterAddFn,
} as unknown as Counter;
} as Partial<Counter> as Counter;
const mockHistogramInstance = {
const mockHistogramInstance: Histogram = {
record: mockHistogramRecordFn,
} as unknown as Histogram;
} as Partial<Histogram> as Histogram;
const mockMeterInstance = {
const mockMeterInstance: Meter = {
createCounter: mockCreateCounterFn.mockReturnValue(mockCounterInstance),
createHistogram: mockCreateHistogramFn.mockReturnValue(mockHistogramInstance),
} as unknown as Meter;
} as Partial<Meter> as Meter;
function originalOtelMockFactory() {
return {
@@ -49,15 +50,19 @@ function originalOtelMockFactory() {
ValueType: {
INT: 1,
},
diag: {
setLogger: vi.fn(),
},
};
}
vi.mock('@opentelemetry/api', originalOtelMockFactory);
vi.mock('@opentelemetry/api');
describe('Telemetry Metrics', () => {
let initializeMetricsModule: typeof import('./metrics.js').initializeMetrics;
let recordTokenUsageMetricsModule: typeof import('./metrics.js').recordTokenUsageMetrics;
let recordFileOperationMetricModule: typeof import('./metrics.js').recordFileOperationMetric;
let recordChatCompressionMetricsModule: typeof import('./metrics.js').recordChatCompressionMetrics;
beforeEach(async () => {
vi.resetModules();
@@ -71,6 +76,8 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule = metricsJsModule.initializeMetrics;
recordTokenUsageMetricsModule = metricsJsModule.recordTokenUsageMetrics;
recordFileOperationMetricModule = metricsJsModule.recordFileOperationMetric;
recordChatCompressionMetricsModule =
metricsJsModule.recordChatCompressionMetrics;
const otelApiModule = await import('@opentelemetry/api');
@@ -85,6 +92,35 @@ describe('Telemetry Metrics', () => {
mockCreateHistogramFn.mockReturnValue(mockHistogramInstance);
});
describe('recordChatCompressionMetrics', () => {
it('does not record metrics if not initialized', () => {
const lol = makeFakeConfig({});
recordChatCompressionMetricsModule(lol, {
tokens_after: 100,
tokens_before: 200,
});
expect(mockCounterAddFn).not.toHaveBeenCalled();
});
it('records token compression with the correct attributes', () => {
const config = makeFakeConfig({});
initializeMetricsModule(config);
recordChatCompressionMetricsModule(config, {
tokens_after: 100,
tokens_before: 200,
});
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
'session.id': 'test-session-id',
tokens_after: 100,
tokens_before: 200,
});
});
});
describe('recordTokenUsageMetrics', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',

83
packages/core/src/telemetry/metrics.ts
View File

@@ -21,6 +21,10 @@ import {
METRIC_TOKEN_USAGE,
METRIC_SESSION_COUNT,
METRIC_FILE_OPERATION_COUNT,
EVENT_CHAT_COMPRESSION,
METRIC_INVALID_CHUNK_COUNT,
METRIC_CONTENT_RETRY_COUNT,
METRIC_CONTENT_RETRY_FAILURE_COUNT,
} from './constants.js';
import { Config } from '../config/config.js';
import { DiffStat } from '../tools/tools.js';
@@ -38,6 +42,10 @@ let apiRequestCounter: Counter | undefined;
let apiRequestLatencyHistogram: Histogram | undefined;
let tokenUsageCounter: Counter | undefined;
let fileOperationCounter: Counter | undefined;
let chatCompressionCounter: Counter | undefined;
let invalidChunkCounter: Counter | undefined;
let contentRetryCounter: Counter | undefined;
let contentRetryFailureCounter: Counter | undefined;
let isMetricsInitialized = false;
function getCommonAttributes(config: Config): Attributes {
@@ -88,6 +96,28 @@ export function initializeMetrics(config: Config): void {
description: 'Counts file operations (create, read, update).',
valueType: ValueType.INT,
});
chatCompressionCounter = meter.createCounter(EVENT_CHAT_COMPRESSION, {
description: 'Counts chat compression events.',
valueType: ValueType.INT,
});
// New counters for content errors
invalidChunkCounter = meter.createCounter(METRIC_INVALID_CHUNK_COUNT, {
description: 'Counts invalid chunks received from a stream.',
valueType: ValueType.INT,
});
contentRetryCounter = meter.createCounter(METRIC_CONTENT_RETRY_COUNT, {
description: 'Counts retries due to content errors (e.g., empty stream).',
valueType: ValueType.INT,
});
contentRetryFailureCounter = meter.createCounter(
METRIC_CONTENT_RETRY_FAILURE_COUNT,
{
description: 'Counts occurrences of all content retries failing.',
valueType: ValueType.INT,
},
);
const sessionCounter = meter.createCounter(METRIC_SESSION_COUNT, {
description: 'Count of CLI sessions started.',
valueType: ValueType.INT,
@@ -96,12 +126,24 @@ export function initializeMetrics(config: Config): void {
isMetricsInitialized = true;
}
export function recordChatCompressionMetrics(
config: Config,
args: { tokens_before: number; tokens_after: number },
) {
if (!chatCompressionCounter || !isMetricsInitialized) return;
chatCompressionCounter.add(1, {
...getCommonAttributes(config),
...args,
});
}
export function recordToolCallMetrics(
config: Config,
functionName: string,
durationMs: number,
success: boolean,
decision?: 'accept' | 'reject' | 'modify' | 'auto_accept',
tool_type?: 'native' | 'mcp',
): void {
if (!toolCallCounter || !toolCallLatencyHistogram || !isMetricsInitialized)
return;
@@ -111,6 +153,7 @@ export function recordToolCallMetrics(
function_name: functionName,
success,
decision,
tool_type,
};
toolCallCounter.add(1, metricAttributes);
toolCallLatencyHistogram.record(durationMs, {
@@ -197,14 +240,40 @@ export function recordFileOperationMetric(
...getCommonAttributes(config),
operation,
};
if (lines !== undefined) attributes.lines = lines;
if (mimetype !== undefined) attributes.mimetype = mimetype;
if (extension !== undefined) attributes.extension = extension;
if (lines !== undefined) attributes['lines'] = lines;
if (mimetype !== undefined) attributes['mimetype'] = mimetype;
if (extension !== undefined) attributes['extension'] = extension;
if (diffStat !== undefined) {
attributes.ai_added_lines = diffStat.ai_added_lines;
attributes.ai_removed_lines = diffStat.ai_removed_lines;
attributes.user_added_lines = diffStat.user_added_lines;
attributes.user_removed_lines = diffStat.user_removed_lines;
attributes['ai_added_lines'] = diffStat.ai_added_lines;
attributes['ai_removed_lines'] = diffStat.ai_removed_lines;
attributes['user_added_lines'] = diffStat.user_added_lines;
attributes['user_removed_lines'] = diffStat.user_removed_lines;
}
fileOperationCounter.add(1, attributes);
}
// --- New Metric Recording Functions ---
/**
* Records a metric for when an invalid chunk is received from a stream.
*/
export function recordInvalidChunk(config: Config): void {
if (!invalidChunkCounter || !isMetricsInitialized) return;
invalidChunkCounter.add(1, getCommonAttributes(config));
}
/**
* Records a metric for when a retry is triggered due to a content error.
*/
export function recordContentRetry(config: Config): void {
if (!contentRetryCounter || !isMetricsInitialized) return;
contentRetryCounter.add(1, getCommonAttributes(config));
}
/**
* Records a metric for when all content error retries have failed for a request.
*/
export function recordContentRetryFailure(config: Config): void {
if (!contentRetryFailureCounter || !isMetricsInitialized) return;
contentRetryFailureCounter.add(1, getCommonAttributes(config));
}

64
packages/core/src/telemetry/qwen-logger/qwen-logger.ts
View File

@@ -23,6 +23,10 @@ import {
MalformedJsonResponseEvent,
IdeConnectionEvent,
KittySequenceOverflowEvent,
ChatCompressionEvent,
InvalidChunkEvent,
ContentRetryEvent,
ContentRetryFailureEvent,
} from '../types.js';
import {
RumEvent,
@@ -205,7 +209,7 @@ export class QwenLogger {
return {
app: {
id: RUN_APP_ID,
env: process.env.DEBUG ? 'dev' : 'prod',
env: process.env['DEBUG'] ? 'dev' : 'prod',
version: version || 'unknown',
type: 'cli',
},
@@ -225,7 +229,9 @@ export class QwenLogger {
auth_type: authType,
model: this.config?.getModel(),
base_url:
authType === AuthType.USE_OPENAI ? process.env.OPENAI_BASE_URL : '',
authType === AuthType.USE_OPENAI
? process.env['OPENAI_BASE_URL']
: '',
},
_v: `qwen-code@${version}`,
};
@@ -568,6 +574,60 @@ export class QwenLogger {
this.flushIfNeeded();
}
logChatCompressionEvent(event: ChatCompressionEvent): void {
const rumEvent = this.createActionEvent('compression', 'chat_compression', {
snapshots: JSON.stringify({
tokens_before: event.tokens_before,
tokens_after: event.tokens_after,
}),
});
this.enqueueLogEvent(rumEvent);
this.flushIfNeeded();
}
logInvalidChunkEvent(event: InvalidChunkEvent): void {
const rumEvent = this.createExceptionEvent('error', 'invalid_chunk', {
subtype: 'invalid_chunk',
message: event.error_message,
});
this.enqueueLogEvent(rumEvent);
this.flushIfNeeded();
}
logContentRetryEvent(event: ContentRetryEvent): void {
const rumEvent = this.createActionEvent('retry', 'content_retry', {
snapshots: JSON.stringify({
attempt_number: event.attempt_number,
error_type: event.error_type,
retry_delay_ms: event.retry_delay_ms,
}),
});
this.enqueueLogEvent(rumEvent);
this.flushIfNeeded();
}
logContentRetryFailureEvent(event: ContentRetryFailureEvent): void {
const rumEvent = this.createExceptionEvent(
'error',
'content_retry_failure',
{
subtype: 'content_retry_failure',
message: `Content retry failed after ${event.total_attempts} attempts`,
snapshots: JSON.stringify({
total_attempts: event.total_attempts,
final_error_type: event.final_error_type,
total_duration_ms: event.total_duration_ms,
}),
},
);
this.enqueueLogEvent(rumEvent);
this.flushIfNeeded();
}
logEndSessionEvent(_event: EndSessionEvent): void {
const applicationEvent = this.createViewEvent('session', 'session_end', {});

104
packages/core/src/telemetry/sdk.test.ts Normal file
View File

@@ -0,0 +1,104 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import { Config } from '../config/config.js';
import { initializeTelemetry, shutdownTelemetry } from './sdk.js';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc';
import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-grpc';
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc';
import { OTLPTraceExporter as OTLPTraceExporterHttp } from '@opentelemetry/exporter-trace-otlp-http';
import { OTLPLogExporter as OTLPLogExporterHttp } from '@opentelemetry/exporter-logs-otlp-http';
import { OTLPMetricExporter as OTLPMetricExporterHttp } from '@opentelemetry/exporter-metrics-otlp-http';
import { NodeSDK } from '@opentelemetry/sdk-node';
vi.mock('@opentelemetry/exporter-trace-otlp-grpc');
vi.mock('@opentelemetry/exporter-logs-otlp-grpc');
vi.mock('@opentelemetry/exporter-metrics-otlp-grpc');
vi.mock('@opentelemetry/exporter-trace-otlp-http');
vi.mock('@opentelemetry/exporter-logs-otlp-http');
vi.mock('@opentelemetry/exporter-metrics-otlp-http');
vi.mock('@opentelemetry/sdk-node');
describe('Telemetry SDK', () => {
let mockConfig: Config;
beforeEach(() => {
vi.clearAllMocks();
mockConfig = {
getTelemetryEnabled: () => true,
getTelemetryOtlpEndpoint: () => 'http://localhost:4317',
getTelemetryOtlpProtocol: () => 'grpc',
getTelemetryOutfile: () => undefined,
getDebugMode: () => false,
getSessionId: () => 'test-session',
} as unknown as Config;
});
afterEach(async () => {
await shutdownTelemetry(mockConfig);
});
it('should use gRPC exporters when protocol is grpc', () => {
initializeTelemetry(mockConfig);
expect(OTLPTraceExporter).toHaveBeenCalledWith({
url: 'http://localhost:4317',
compression: 'gzip',
});
expect(OTLPLogExporter).toHaveBeenCalledWith({
url: 'http://localhost:4317',
compression: 'gzip',
});
expect(OTLPMetricExporter).toHaveBeenCalledWith({
url: 'http://localhost:4317',
compression: 'gzip',
});
expect(NodeSDK.prototype.start).toHaveBeenCalled();
});
it('should use HTTP exporters when protocol is http', () => {
vi.spyOn(mockConfig, 'getTelemetryEnabled').mockReturnValue(true);
vi.spyOn(mockConfig, 'getTelemetryOtlpProtocol').mockReturnValue('http');
vi.spyOn(mockConfig, 'getTelemetryOtlpEndpoint').mockReturnValue(
'http://localhost:4318',
);
initializeTelemetry(mockConfig);
expect(OTLPTraceExporterHttp).toHaveBeenCalledWith({
url: 'http://localhost:4318/',
});
expect(OTLPLogExporterHttp).toHaveBeenCalledWith({
url: 'http://localhost:4318/',
});
expect(OTLPMetricExporterHttp).toHaveBeenCalledWith({
url: 'http://localhost:4318/',
});
expect(NodeSDK.prototype.start).toHaveBeenCalled();
});
it('should parse gRPC endpoint correctly', () => {
vi.spyOn(mockConfig, 'getTelemetryOtlpEndpoint').mockReturnValue(
'https://my-collector.com',
);
initializeTelemetry(mockConfig);
expect(OTLPTraceExporter).toHaveBeenCalledWith(
expect.objectContaining({ url: 'https://my-collector.com' }),
);
});
it('should parse HTTP endpoint correctly', () => {
vi.spyOn(mockConfig, 'getTelemetryOtlpProtocol').mockReturnValue('http');
vi.spyOn(mockConfig, 'getTelemetryOtlpEndpoint').mockReturnValue(
'https://my-collector.com',
);
initializeTelemetry(mockConfig);
expect(OTLPTraceExporterHttp).toHaveBeenCalledWith(
expect.objectContaining({ url: 'https://my-collector.com/' }),
);
});
});

107
packages/core/src/telemetry/sdk.ts
View File

@@ -8,10 +8,13 @@ import { DiagConsoleLogger, DiagLogLevel, diag } from '@opentelemetry/api';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc';
import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-grpc';
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc';
import { OTLPTraceExporter as OTLPTraceExporterHttp } from '@opentelemetry/exporter-trace-otlp-http';
import { OTLPLogExporter as OTLPLogExporterHttp } from '@opentelemetry/exporter-logs-otlp-http';
import { OTLPMetricExporter as OTLPMetricExporterHttp } from '@opentelemetry/exporter-metrics-otlp-http';
import { CompressionAlgorithm } from '@opentelemetry/otlp-exporter-base';
import { NodeSDK } from '@opentelemetry/sdk-node';
import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';
import { Resource } from '@opentelemetry/resources';
import { resourceFromAttributes } from '@opentelemetry/resources';
import {
BatchSpanProcessor,
ConsoleSpanExporter,
@@ -44,8 +47,9 @@ export function isTelemetrySdkInitialized(): boolean {
return telemetryInitialized;
}
function parseGrpcEndpoint(
function parseOtlpEndpoint(
otlpEndpointSetting: string | undefined,
protocol: 'grpc' | 'http',
): string | undefined {
if (!otlpEndpointSetting) {
return undefined;
@@ -55,9 +59,13 @@ function parseGrpcEndpoint(
try {
const url = new URL(trimmedEndpoint);
// OTLP gRPC exporters expect an endpoint in the format scheme://host:port
// The `origin` property provides this, stripping any path, query, or hash.
return url.origin;
if (protocol === 'grpc') {
// OTLP gRPC exporters expect an endpoint in the format scheme://host:port
// The `origin` property provides this, stripping any path, query, or hash.
return url.origin;
}
// For http, use the full href.
return url.href;
} catch (error) {
diag.error('Invalid OTLP endpoint URL provided:', trimmedEndpoint, error);
return undefined;
@@ -69,55 +77,82 @@ export function initializeTelemetry(config: Config): void {
return;
}
const resource = new Resource({
const resource = resourceFromAttributes({
[SemanticResourceAttributes.SERVICE_NAME]: SERVICE_NAME,
[SemanticResourceAttributes.SERVICE_VERSION]: process.version,
'session.id': config.getSessionId(),
});
const otlpEndpoint = config.getTelemetryOtlpEndpoint();
const grpcParsedEndpoint = parseGrpcEndpoint(otlpEndpoint);
const useOtlp = !!grpcParsedEndpoint;
const otlpProtocol = config.getTelemetryOtlpProtocol();
const parsedEndpoint = parseOtlpEndpoint(otlpEndpoint, otlpProtocol);
const useOtlp = !!parsedEndpoint;
const telemetryOutfile = config.getTelemetryOutfile();
const spanExporter = useOtlp
? new OTLPTraceExporter({
url: grpcParsedEndpoint,
let spanExporter:
| OTLPTraceExporter
| OTLPTraceExporterHttp
| FileSpanExporter
| ConsoleSpanExporter;
let logExporter:
| OTLPLogExporter
| OTLPLogExporterHttp
| FileLogExporter
| ConsoleLogRecordExporter;
let metricReader: PeriodicExportingMetricReader;
if (useOtlp) {
if (otlpProtocol === 'http') {
spanExporter = new OTLPTraceExporterHttp({
url: parsedEndpoint,
});
logExporter = new OTLPLogExporterHttp({
url: parsedEndpoint,
});
metricReader = new PeriodicExportingMetricReader({
exporter: new OTLPMetricExporterHttp({
url: parsedEndpoint,
}),
exportIntervalMillis: 10000,
});
} else {
// grpc
spanExporter = new OTLPTraceExporter({
url: parsedEndpoint,
compression: CompressionAlgorithm.GZIP,
})
: telemetryOutfile
? new FileSpanExporter(telemetryOutfile)
: new ConsoleSpanExporter();
const logExporter = useOtlp
? new OTLPLogExporter({
url: grpcParsedEndpoint,
});
logExporter = new OTLPLogExporter({
url: parsedEndpoint,
compression: CompressionAlgorithm.GZIP,
})
: telemetryOutfile
? new FileLogExporter(telemetryOutfile)
: new ConsoleLogRecordExporter();
const metricReader = useOtlp
? new PeriodicExportingMetricReader({
});
metricReader = new PeriodicExportingMetricReader({
exporter: new OTLPMetricExporter({
url: grpcParsedEndpoint,
url: parsedEndpoint,
compression: CompressionAlgorithm.GZIP,
}),
exportIntervalMillis: 10000,
})
: telemetryOutfile
? new PeriodicExportingMetricReader({
exporter: new FileMetricExporter(telemetryOutfile),
exportIntervalMillis: 10000,
})
: new PeriodicExportingMetricReader({
exporter: new ConsoleMetricExporter(),
exportIntervalMillis: 10000,
});
});
}
} else if (telemetryOutfile) {
spanExporter = new FileSpanExporter(telemetryOutfile);
logExporter = new FileLogExporter(telemetryOutfile);
metricReader = new PeriodicExportingMetricReader({
exporter: new FileMetricExporter(telemetryOutfile),
exportIntervalMillis: 10000,
});
} else {
spanExporter = new ConsoleSpanExporter();
logExporter = new ConsoleLogRecordExporter();
metricReader = new PeriodicExportingMetricReader({
exporter: new ConsoleMetricExporter(),
exportIntervalMillis: 10000,
});
}
sdk = new NodeSDK({
resource,
spanProcessors: [new BatchSpanProcessor(spanExporter)],
logRecordProcessor: new BatchLogRecordProcessor(logExporter),
logRecordProcessors: [new BatchLogRecordProcessor(logExporter)],
metricReader,
instrumentations: [new HttpInstrumentation()],
});

87
packages/core/src/telemetry/types.ts
View File

@@ -7,6 +7,7 @@
import { GenerateContentResponseUsageMetadata } from '@google/genai';
import { Config } from '../config/config.js';
import { CompletedToolCall } from '../core/coreToolScheduler.js';
import { DiscoveredMCPTool } from '../tools/mcp-tool.js';
import { FileDiff } from '../tools/tools.js';
import { AuthType } from '../core/contentGenerator.js';
import {
@@ -14,7 +15,7 @@ import {
ToolCallDecision,
} from './tool-call-decision.js';
interface BaseTelemetryEvent {
export interface BaseTelemetryEvent {
'event.name': string;
/** Current timestamp in ISO 8601 format */
'event.timestamp': string;
@@ -114,6 +115,7 @@ export class ToolCallEvent implements BaseTelemetryEvent {
error?: string;
error_type?: string;
prompt_id: string;
tool_type: 'native' | 'mcp';
// eslint-disable-next-line @typescript-eslint/no-explicit-any
metadata?: { [key: string]: any };
@@ -130,6 +132,10 @@ export class ToolCallEvent implements BaseTelemetryEvent {
this.error = call.response.error?.message;
this.error_type = call.response.errorType;
this.prompt_id = call.request.prompt_id;
this.tool_type =
typeof call.tool !== 'undefined' && call.tool instanceof DiscoveredMCPTool
? 'mcp'
: 'native';
if (
call.status === 'success' &&
@@ -298,7 +304,7 @@ export class NextSpeakerCheckEvent implements BaseTelemetryEvent {
export interface SlashCommandEvent extends BaseTelemetryEvent {
'event.name': 'slash_command';
'event.timestamp': string; // ISO 8106
'event.timestamp': string;
command: string;
subcommand?: string;
status?: SlashCommandStatus;
@@ -323,6 +329,25 @@ export enum SlashCommandStatus {
ERROR = 'error',
}
export interface ChatCompressionEvent extends BaseTelemetryEvent {
'event.name': 'chat_compression';
'event.timestamp': string;
tokens_before: number;
tokens_after: number;
}
export function makeChatCompressionEvent({
tokens_before,
tokens_after,
}: Omit<ChatCompressionEvent, CommonFields>): ChatCompressionEvent {
return {
'event.name': 'chat_compression',
'event.timestamp': new Date().toISOString(),
tokens_before,
tokens_after,
};
}
export class MalformedJsonResponseEvent implements BaseTelemetryEvent {
'event.name': 'malformed_json_response';
'event.timestamp': string;
@@ -366,6 +391,59 @@ export class KittySequenceOverflowEvent {
}
}
// Add these new event interfaces
export class InvalidChunkEvent implements BaseTelemetryEvent {
'event.name': 'invalid_chunk';
'event.timestamp': string;
error_message?: string; // Optional: validation error details
constructor(error_message?: string) {
this['event.name'] = 'invalid_chunk';
this['event.timestamp'] = new Date().toISOString();
this.error_message = error_message;
}
}
export class ContentRetryEvent implements BaseTelemetryEvent {
'event.name': 'content_retry';
'event.timestamp': string;
attempt_number: number;
error_type: string; // e.g., 'EmptyStreamError'
retry_delay_ms: number;
constructor(
attempt_number: number,
error_type: string,
retry_delay_ms: number,
) {
this['event.name'] = 'content_retry';
this['event.timestamp'] = new Date().toISOString();
this.attempt_number = attempt_number;
this.error_type = error_type;
this.retry_delay_ms = retry_delay_ms;
}
}
export class ContentRetryFailureEvent implements BaseTelemetryEvent {
'event.name': 'content_retry_failure';
'event.timestamp': string;
total_attempts: number;
final_error_type: string;
total_duration_ms?: number; // Optional: total time spent retrying
constructor(
total_attempts: number,
final_error_type: string,
total_duration_ms?: number,
) {
this['event.name'] = 'content_retry_failure';
this['event.timestamp'] = new Date().toISOString();
this.total_attempts = total_attempts;
this.final_error_type = final_error_type;
this.total_duration_ms = total_duration_ms;
}
}
export type TelemetryEvent =
| StartSessionEvent
| EndSessionEvent
@@ -380,4 +458,7 @@ export type TelemetryEvent =
| KittySequenceOverflowEvent
| MalformedJsonResponseEvent
| IdeConnectionEvent
| SlashCommandEvent;
| SlashCommandEvent
| InvalidChunkEvent
| ContentRetryEvent
| ContentRetryFailureEvent;

76
packages/core/src/telemetry/uiTelemetry.test.ts
View File

@@ -43,7 +43,7 @@ const createFakeCompletedToolCall = (
status: 'success',
request,
tool,
invocation: tool.build({}),
invocation: tool.build({ param: 'test' }),
response: {
callId: request.callId,
responseParts: {
@@ -108,6 +108,10 @@ describe('UiTelemetryService', () => {
},
byName: {},
},
files: {
totalLinesAdded: 0,
totalLinesRemoved: 0,
},
});
expect(service.getLastPromptTokenCount()).toBe(0);
});
@@ -342,9 +346,9 @@ describe('UiTelemetryService', () => {
ToolConfirmationOutcome.ProceedOnce,
);
service.addEvent({
...JSON.parse(JSON.stringify(new ToolCallEvent(toolCall))),
...structuredClone(new ToolCallEvent(toolCall)),
'event.name': EVENT_TOOL_CALL,
});
} as ToolCallEvent & { 'event.name': typeof EVENT_TOOL_CALL });
const metrics = service.getMetrics();
const { tools } = metrics;
@@ -376,9 +380,9 @@ describe('UiTelemetryService', () => {
ToolConfirmationOutcome.Cancel,
);
service.addEvent({
...JSON.parse(JSON.stringify(new ToolCallEvent(toolCall))),
...structuredClone(new ToolCallEvent(toolCall)),
'event.name': EVENT_TOOL_CALL,
});
} as ToolCallEvent & { 'event.name': typeof EVENT_TOOL_CALL });
const metrics = service.getMetrics();
const { tools } = metrics;
@@ -410,9 +414,9 @@ describe('UiTelemetryService', () => {
ToolConfirmationOutcome.ModifyWithEditor,
);
service.addEvent({
...JSON.parse(JSON.stringify(new ToolCallEvent(toolCall))),
...structuredClone(new ToolCallEvent(toolCall)),
'event.name': EVENT_TOOL_CALL,
});
} as ToolCallEvent & { 'event.name': typeof EVENT_TOOL_CALL });
const metrics = service.getMetrics();
const { tools } = metrics;
@@ -426,9 +430,9 @@ describe('UiTelemetryService', () => {
it('should process a ToolCallEvent without a decision', () => {
const toolCall = createFakeCompletedToolCall('test_tool', true, 100);
service.addEvent({
...JSON.parse(JSON.stringify(new ToolCallEvent(toolCall))),
...structuredClone(new ToolCallEvent(toolCall)),
'event.name': EVENT_TOOL_CALL,
});
} as ToolCallEvent & { 'event.name': typeof EVENT_TOOL_CALL });
const metrics = service.getMetrics();
const { tools } = metrics;
@@ -462,13 +466,13 @@ describe('UiTelemetryService', () => {
);
service.addEvent({
...JSON.parse(JSON.stringify(new ToolCallEvent(toolCall1))),
...structuredClone(new ToolCallEvent(toolCall1)),
'event.name': EVENT_TOOL_CALL,
});
} as ToolCallEvent & { 'event.name': typeof EVENT_TOOL_CALL });
service.addEvent({
...JSON.parse(JSON.stringify(new ToolCallEvent(toolCall2))),
...structuredClone(new ToolCallEvent(toolCall2)),
'event.name': EVENT_TOOL_CALL,
});
} as ToolCallEvent & { 'event.name': typeof EVENT_TOOL_CALL });
const metrics = service.getMetrics();
const { tools } = metrics;
@@ -497,13 +501,13 @@ describe('UiTelemetryService', () => {
const toolCall1 = createFakeCompletedToolCall('tool_A', true, 100);
const toolCall2 = createFakeCompletedToolCall('tool_B', false, 200);
service.addEvent({
...JSON.parse(JSON.stringify(new ToolCallEvent(toolCall1))),
...structuredClone(new ToolCallEvent(toolCall1)),
'event.name': EVENT_TOOL_CALL,
});
} as ToolCallEvent & { 'event.name': typeof EVENT_TOOL_CALL });
service.addEvent({
...JSON.parse(JSON.stringify(new ToolCallEvent(toolCall2))),
...structuredClone(new ToolCallEvent(toolCall2)),
'event.name': EVENT_TOOL_CALL,
});
} as ToolCallEvent & { 'event.name': typeof EVENT_TOOL_CALL });
const metrics = service.getMetrics();
const { tools } = metrics;
@@ -629,4 +633,42 @@ describe('UiTelemetryService', () => {
expect(spy).toHaveBeenCalledOnce();
});
});
describe('Tool Call Event with Line Count Metadata', () => {
it('should aggregate valid line count metadata', () => {
const toolCall = createFakeCompletedToolCall('test_tool', true, 100);
const event = {
...structuredClone(new ToolCallEvent(toolCall)),
'event.name': EVENT_TOOL_CALL,
metadata: {
ai_added_lines: 10,
ai_removed_lines: 5,
},
} as ToolCallEvent & { 'event.name': typeof EVENT_TOOL_CALL };
service.addEvent(event);
const metrics = service.getMetrics();
expect(metrics.files.totalLinesAdded).toBe(10);
expect(metrics.files.totalLinesRemoved).toBe(5);
});
it('should ignore null/undefined values in line count metadata', () => {
const toolCall = createFakeCompletedToolCall('test_tool', true, 100);
const event = {
...structuredClone(new ToolCallEvent(toolCall)),
'event.name': EVENT_TOOL_CALL,
metadata: {
ai_added_lines: null,
ai_removed_lines: undefined,
},
} as ToolCallEvent & { 'event.name': typeof EVENT_TOOL_CALL };
service.addEvent(event);
const metrics = service.getMetrics();
expect(metrics.files.totalLinesAdded).toBe(0);
expect(metrics.files.totalLinesRemoved).toBe(0);
});
});
});

20
packages/core/src/telemetry/uiTelemetry.ts
View File

@@ -63,6 +63,10 @@ export interface SessionMetrics {
};
byName: Record<string, ToolCallStats>;
};
files: {
totalLinesAdded: number;
totalLinesRemoved: number;
};
}
const createInitialModelMetrics = (): ModelMetrics => ({
@@ -96,6 +100,10 @@ const createInitialMetrics = (): SessionMetrics => ({
},
byName: {},
},
files: {
totalLinesAdded: 0,
totalLinesRemoved: 0,
},
});
export class UiTelemetryService extends EventEmitter {
@@ -171,7 +179,7 @@ export class UiTelemetryService extends EventEmitter {
}
private processToolCall(event: ToolCallEvent) {
const { tools } = this.#metrics;
const { tools, files } = this.#metrics;
tools.totalCalls++;
tools.totalDurationMs += event.duration_ms;
@@ -209,6 +217,16 @@ export class UiTelemetryService extends EventEmitter {
tools.totalDecisions[event.decision]++;
toolStats.decisions[event.decision]++;
}
// Aggregate line count data from metadata
if (event.metadata) {
if (event.metadata['ai_added_lines'] !== undefined) {
files.totalLinesAdded += event.metadata['ai_added_lines'];
}
if (event.metadata['ai_removed_lines'] !== undefined) {
files.totalLinesRemoved += event.metadata['ai_removed_lines'];
}
}
}
}