abbyswag/qwen-code
1
0
Fork 0
mirror of https://github.com/QwenLM/qwen-code.git synced 2025-12-21 01:07:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

pre-release commit

Browse Source
This commit is contained in:
koalazf.99
2025-07-22 19:59:07 +08:00
parent c5dee4bb17
commit a9d6965bef
485 changed files with 111444 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

484
packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts Normal file
View File

@@ -0,0 +1,484 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { Buffer } from 'buffer';
import * as https from 'https';
import {
StartSessionEvent,
EndSessionEvent,
UserPromptEvent,
ToolCallEvent,
ApiRequestEvent,
ApiResponseEvent,
ApiErrorEvent,
FlashFallbackEvent,
LoopDetectedEvent,
} from '../types.js';
import { EventMetadataKey } from './event-metadata-key.js';
import { Config } from '../../config/config.js';
import { getInstallationId } from '../../utils/user_id.js';
import {
getCachedGoogleAccount,
getLifetimeGoogleAccounts,
} from '../../utils/user_account.js';
import { safeJsonStringify } from '../../utils/safeJsonStringify.js';
const start_session_event_name = 'start_session';
const new_prompt_event_name = 'new_prompt';
const tool_call_event_name = 'tool_call';
const api_request_event_name = 'api_request';
const api_response_event_name = 'api_response';
const api_error_event_name = 'api_error';
const end_session_event_name = 'end_session';
const flash_fallback_event_name = 'flash_fallback';
const loop_detected_event_name = 'loop_detected';
export interface LogResponse {
nextRequestWaitMs?: number;
}
// Singleton class for batch posting log events to Clearcut. When a new event comes in, the elapsed time
// is checked and events are flushed to Clearcut if at least a minute has passed since the last flush.
export class ClearcutLogger {
private static instance: ClearcutLogger;
private config?: Config;
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Clearcut expects this format.
private readonly events: any = [];
private last_flush_time: number = Date.now();
private flush_interval_ms: number = 1000 * 60; // Wait at least a minute before flushing events.
private constructor(config?: Config) {
this.config = config;
}
static getInstance(config?: Config): ClearcutLogger | undefined {
if (config === undefined || !config?.getUsageStatisticsEnabled())
return undefined;
if (!ClearcutLogger.instance) {
ClearcutLogger.instance = new ClearcutLogger(config);
}
return ClearcutLogger.instance;
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Clearcut expects this format.
enqueueLogEvent(event: any): void {
this.events.push([
{
event_time_ms: Date.now(),
source_extension_json: safeJsonStringify(event),
},
]);
}
createLogEvent(name: string, data: object[]): object {
const email = getCachedGoogleAccount();
const totalAccounts = getLifetimeGoogleAccounts();
data.push({
gemini_cli_key: EventMetadataKey.GEMINI_CLI_GOOGLE_ACCOUNTS_COUNT,
value: totalAccounts.toString(),
});
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const logEvent: any = {
console_type: 'GEMINI_CLI',
application: 102,
event_name: name,
event_metadata: [data] as object[],
};
// Should log either email or install ID, not both. See go/cloudmill-1p-oss-instrumentation#define-sessionable-id
if (email) {
logEvent.client_email = email;
} else {
logEvent.client_install_id = getInstallationId();
}
return logEvent;
}
flushIfNeeded(): void {
if (Date.now() - this.last_flush_time < this.flush_interval_ms) {
return;
}
this.flushToClearcut().catch((error) => {
console.debug('Error flushing to Clearcut:', error);
});
}
flushToClearcut(): Promise<LogResponse> {
if (this.config?.getDebugMode()) {
console.log('Flushing log events to Clearcut.');
}
const eventsToSend = [...this.events];
this.events.length = 0;
return new Promise<Buffer>((resolve, reject) => {
const request = [
{
log_source_name: 'CONCORD',
request_time_ms: Date.now(),
log_event: eventsToSend,
},
];
const body = safeJsonStringify(request);
const options = {
hostname: 'play.googleapis.com',
path: '/log',
method: 'POST',
headers: { 'Content-Length': Buffer.byteLength(body) },
};
const bufs: Buffer[] = [];
const req = https.request(options, (res) => {
res.on('data', (buf) => bufs.push(buf));
res.on('end', () => {
resolve(Buffer.concat(bufs));
});
});
req.on('error', (e) => {
if (this.config?.getDebugMode()) {
console.log('Clearcut POST request error: ', e);
}
// Add the events back to the front of the queue to be retried.
this.events.unshift(...eventsToSend);
reject(e);
});
req.end(body);
})
.then((buf: Buffer) => {
try {
this.last_flush_time = Date.now();
return this.decodeLogResponse(buf) || {};
} catch (error: unknown) {
console.error('Error flushing log events:', error);
return {};
}
})
.catch((error: unknown) => {
// Handle all errors to prevent unhandled promise rejections
console.error('Error flushing log events:', error);
// Return empty response to maintain the Promise<LogResponse> contract
return {};
});
}
// Visible for testing. Decodes protobuf-encoded response from Clearcut server.
decodeLogResponse(buf: Buffer): LogResponse | undefined {
// TODO(obrienowen): return specific errors to facilitate debugging.
if (buf.length < 1) {
return undefined;
}
// The first byte of the buffer is `field<<3 | type`. We're looking for field
// 1, with type varint, represented by type=0. If the first byte isn't 8, that
// means field 1 is missing or the message is corrupted. Either way, we return
// undefined.
if (buf.readUInt8(0) !== 8) {
return undefined;
}
let ms = BigInt(0);
let cont = true;
// In each byte, the most significant bit is the continuation bit. If it's
// set, we keep going. The lowest 7 bits, are data bits. They are concatenated
// in reverse order to form the final number.
for (let i = 1; cont && i < buf.length; i++) {
const byte = buf.readUInt8(i);
ms |= BigInt(byte & 0x7f) << BigInt(7 * (i - 1));
cont = (byte & 0x80) !== 0;
}
if (cont) {
// We have fallen off the buffer without seeing a terminating byte. The
// message is corrupted.
return undefined;
}
const returnVal = {
nextRequestWaitMs: Number(ms),
};
return returnVal;
}
logStartSessionEvent(event: StartSessionEvent): void {
const data = [
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_START_SESSION_MODEL,
value: event.model,
},
{
gemini_cli_key:
EventMetadataKey.GEMINI_CLI_START_SESSION_EMBEDDING_MODEL,
value: event.embedding_model,
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_START_SESSION_SANDBOX,
value: event.sandbox_enabled.toString(),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_START_SESSION_CORE_TOOLS,
value: event.core_tools_enabled,
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_START_SESSION_APPROVAL_MODE,
value: event.approval_mode,
},
{
gemini_cli_key:
EventMetadataKey.GEMINI_CLI_START_SESSION_API_KEY_ENABLED,
value: event.api_key_enabled.toString(),
},
{
gemini_cli_key:
EventMetadataKey.GEMINI_CLI_START_SESSION_VERTEX_API_ENABLED,
value: event.vertex_ai_enabled.toString(),
},
{
gemini_cli_key:
EventMetadataKey.GEMINI_CLI_START_SESSION_DEBUG_MODE_ENABLED,
value: event.debug_enabled.toString(),
},
{
gemini_cli_key:
EventMetadataKey.GEMINI_CLI_START_SESSION_VERTEX_API_ENABLED,
value: event.vertex_ai_enabled.toString(),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_START_SESSION_MCP_SERVERS,
value: event.mcp_servers,
},
{
gemini_cli_key:
EventMetadataKey.GEMINI_CLI_START_SESSION_VERTEX_API_ENABLED,
value: event.vertex_ai_enabled.toString(),
},
{
gemini_cli_key:
EventMetadataKey.GEMINI_CLI_START_SESSION_TELEMETRY_ENABLED,
value: event.telemetry_enabled.toString(),
},
{
gemini_cli_key:
EventMetadataKey.GEMINI_CLI_START_SESSION_TELEMETRY_LOG_USER_PROMPTS_ENABLED,
value: event.telemetry_log_user_prompts_enabled.toString(),
},
];
// Flush start event immediately
this.enqueueLogEvent(this.createLogEvent(start_session_event_name, data));
this.flushToClearcut().catch((error) => {
console.debug('Error flushing to Clearcut:', error);
});
}
logNewPromptEvent(event: UserPromptEvent): void {
const data = [
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_USER_PROMPT_LENGTH,
value: JSON.stringify(event.prompt_length),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_PROMPT_ID,
value: JSON.stringify(event.prompt_id),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_AUTH_TYPE,
value: JSON.stringify(event.auth_type),
},
];
this.enqueueLogEvent(this.createLogEvent(new_prompt_event_name, data));
this.flushIfNeeded();
}
logToolCallEvent(event: ToolCallEvent): void {
const data = [
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_TOOL_CALL_NAME,
value: JSON.stringify(event.function_name),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_PROMPT_ID,
value: JSON.stringify(event.prompt_id),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_TOOL_CALL_DECISION,
value: JSON.stringify(event.decision),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_TOOL_CALL_SUCCESS,
value: JSON.stringify(event.success),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_TOOL_CALL_DURATION_MS,
value: JSON.stringify(event.duration_ms),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_TOOL_ERROR_MESSAGE,
value: JSON.stringify(event.error),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_TOOL_CALL_ERROR_TYPE,
value: JSON.stringify(event.error_type),
},
];
const logEvent = this.createLogEvent(tool_call_event_name, data);
this.enqueueLogEvent(logEvent);
this.flushIfNeeded();
}
logApiRequestEvent(event: ApiRequestEvent): void {
const data = [
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_API_REQUEST_MODEL,
value: JSON.stringify(event.model),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_PROMPT_ID,
value: JSON.stringify(event.prompt_id),
},
];
this.enqueueLogEvent(this.createLogEvent(api_request_event_name, data));
this.flushIfNeeded();
}
logApiResponseEvent(event: ApiResponseEvent): void {
const data = [
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_API_RESPONSE_MODEL,
value: JSON.stringify(event.model),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_PROMPT_ID,
value: JSON.stringify(event.prompt_id),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_API_RESPONSE_STATUS_CODE,
value: JSON.stringify(event.status_code),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_API_RESPONSE_DURATION_MS,
value: JSON.stringify(event.duration_ms),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_API_ERROR_MESSAGE,
value: JSON.stringify(event.error),
},
{
gemini_cli_key:
EventMetadataKey.GEMINI_CLI_API_RESPONSE_INPUT_TOKEN_COUNT,
value: JSON.stringify(event.input_token_count),
},
{
gemini_cli_key:
EventMetadataKey.GEMINI_CLI_API_RESPONSE_OUTPUT_TOKEN_COUNT,
value: JSON.stringify(event.output_token_count),
},
{
gemini_cli_key:
EventMetadataKey.GEMINI_CLI_API_RESPONSE_CACHED_TOKEN_COUNT,
value: JSON.stringify(event.cached_content_token_count),
},
{
gemini_cli_key:
EventMetadataKey.GEMINI_CLI_API_RESPONSE_THINKING_TOKEN_COUNT,
value: JSON.stringify(event.thoughts_token_count),
},
{
gemini_cli_key:
EventMetadataKey.GEMINI_CLI_API_RESPONSE_TOOL_TOKEN_COUNT,
value: JSON.stringify(event.tool_token_count),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_AUTH_TYPE,
value: JSON.stringify(event.auth_type),
},
];
this.enqueueLogEvent(this.createLogEvent(api_response_event_name, data));
this.flushIfNeeded();
}
logApiErrorEvent(event: ApiErrorEvent): void {
const data = [
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_API_ERROR_MODEL,
value: JSON.stringify(event.model),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_PROMPT_ID,
value: JSON.stringify(event.prompt_id),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_API_ERROR_TYPE,
value: JSON.stringify(event.error_type),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_API_ERROR_STATUS_CODE,
value: JSON.stringify(event.status_code),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_API_ERROR_DURATION_MS,
value: JSON.stringify(event.duration_ms),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_AUTH_TYPE,
value: JSON.stringify(event.auth_type),
},
];
this.enqueueLogEvent(this.createLogEvent(api_error_event_name, data));
this.flushIfNeeded();
}
logFlashFallbackEvent(event: FlashFallbackEvent): void {
const data = [
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_AUTH_TYPE,
value: JSON.stringify(event.auth_type),
},
];
this.enqueueLogEvent(this.createLogEvent(flash_fallback_event_name, data));
this.flushToClearcut().catch((error) => {
console.debug('Error flushing to Clearcut:', error);
});
}
logLoopDetectedEvent(event: LoopDetectedEvent): void {
const data = [
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_LOOP_DETECTED_TYPE,
value: JSON.stringify(event.loop_type),
},
];
this.enqueueLogEvent(this.createLogEvent(loop_detected_event_name, data));
this.flushIfNeeded();
}
logEndSessionEvent(event: EndSessionEvent): void {
const data = [
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_END_SESSION_ID,
value: event?.session_id?.toString() ?? '',
},
];
// Flush immediately on session end.
this.enqueueLogEvent(this.createLogEvent(end_session_event_name, data));
this.flushToClearcut().catch((error) => {
console.debug('Error flushing to Clearcut:', error);
});
}
shutdown() {
const event = new EndSessionEvent(this.config);
this.logEndSessionEvent(event);
}
}

173
packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts Normal file
View File

@@ -0,0 +1,173 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
// Defines valid event metadata keys for Clearcut logging.
export enum EventMetadataKey {
GEMINI_CLI_KEY_UNKNOWN = 0,
// ==========================================================================
// Start Session Event Keys
// ===========================================================================
// Logs the model id used in the session.
GEMINI_CLI_START_SESSION_MODEL = 1,
// Logs the embedding model id used in the session.
GEMINI_CLI_START_SESSION_EMBEDDING_MODEL = 2,
// Logs the sandbox that was used in the session.
GEMINI_CLI_START_SESSION_SANDBOX = 3,
// Logs the core tools that were enabled in the session.
GEMINI_CLI_START_SESSION_CORE_TOOLS = 4,
// Logs the approval mode that was used in the session.
GEMINI_CLI_START_SESSION_APPROVAL_MODE = 5,
// Logs whether an API key was used in the session.
GEMINI_CLI_START_SESSION_API_KEY_ENABLED = 6,
// Logs whether the Vertex API was used in the session.
GEMINI_CLI_START_SESSION_VERTEX_API_ENABLED = 7,
// Logs whether debug mode was enabled in the session.
GEMINI_CLI_START_SESSION_DEBUG_MODE_ENABLED = 8,
// Logs the MCP servers that were enabled in the session.
GEMINI_CLI_START_SESSION_MCP_SERVERS = 9,
// Logs whether user-collected telemetry was enabled in the session.
GEMINI_CLI_START_SESSION_TELEMETRY_ENABLED = 10,
// Logs whether prompt collection was enabled for user-collected telemetry.
GEMINI_CLI_START_SESSION_TELEMETRY_LOG_USER_PROMPTS_ENABLED = 11,
// Logs whether the session was configured to respect gitignore files.
GEMINI_CLI_START_SESSION_RESPECT_GITIGNORE = 12,
// ==========================================================================
// User Prompt Event Keys
// ===========================================================================
// Logs the length of the prompt.
GEMINI_CLI_USER_PROMPT_LENGTH = 13,
// ==========================================================================
// Tool Call Event Keys
// ===========================================================================
// Logs the function name.
GEMINI_CLI_TOOL_CALL_NAME = 14,
// Logs the user's decision about how to handle the tool call.
GEMINI_CLI_TOOL_CALL_DECISION = 15,
// Logs whether the tool call succeeded.
GEMINI_CLI_TOOL_CALL_SUCCESS = 16,
// Logs the tool call duration in milliseconds.
GEMINI_CLI_TOOL_CALL_DURATION_MS = 17,
// Logs the tool call error message, if any.
GEMINI_CLI_TOOL_ERROR_MESSAGE = 18,
// Logs the tool call error type, if any.
GEMINI_CLI_TOOL_CALL_ERROR_TYPE = 19,
// ==========================================================================
// GenAI API Request Event Keys
// ===========================================================================
// Logs the model id of the request.
GEMINI_CLI_API_REQUEST_MODEL = 20,
// ==========================================================================
// GenAI API Response Event Keys
// ===========================================================================
// Logs the model id of the API call.
GEMINI_CLI_API_RESPONSE_MODEL = 21,
// Logs the status code of the response.
GEMINI_CLI_API_RESPONSE_STATUS_CODE = 22,
// Logs the duration of the API call in milliseconds.
GEMINI_CLI_API_RESPONSE_DURATION_MS = 23,
// Logs the error message of the API call, if any.
GEMINI_CLI_API_ERROR_MESSAGE = 24,
// Logs the input token count of the API call.
GEMINI_CLI_API_RESPONSE_INPUT_TOKEN_COUNT = 25,
// Logs the output token count of the API call.
GEMINI_CLI_API_RESPONSE_OUTPUT_TOKEN_COUNT = 26,
// Logs the cached token count of the API call.
GEMINI_CLI_API_RESPONSE_CACHED_TOKEN_COUNT = 27,
// Logs the thinking token count of the API call.
GEMINI_CLI_API_RESPONSE_THINKING_TOKEN_COUNT = 28,
// Logs the tool use token count of the API call.
GEMINI_CLI_API_RESPONSE_TOOL_TOKEN_COUNT = 29,
// ==========================================================================
// GenAI API Error Event Keys
// ===========================================================================
// Logs the model id of the API call.
GEMINI_CLI_API_ERROR_MODEL = 30,
// Logs the error type.
GEMINI_CLI_API_ERROR_TYPE = 31,
// Logs the status code of the error response.
GEMINI_CLI_API_ERROR_STATUS_CODE = 32,
// Logs the duration of the API call in milliseconds.
GEMINI_CLI_API_ERROR_DURATION_MS = 33,
// ==========================================================================
// End Session Event Keys
// ===========================================================================
// Logs the end of a session.
GEMINI_CLI_END_SESSION_ID = 34,
// ==========================================================================
// Shared Keys
// ===========================================================================
// Logs the Prompt Id
GEMINI_CLI_PROMPT_ID = 35,
// Logs the Auth type for the prompt, api responses and errors.
GEMINI_CLI_AUTH_TYPE = 36,
// Logs the total number of Google accounts ever used.
GEMINI_CLI_GOOGLE_ACCOUNTS_COUNT = 37,
// ==========================================================================
// Loop Detected Event Keys
// ===========================================================================
// Logs the type of loop detected.
GEMINI_CLI_LOOP_DETECTED_TYPE = 38,
}
export function getEventMetadataKey(
keyName: string,
): EventMetadataKey | undefined {
// Access the enum member by its string name
const key = EventMetadataKey[keyName as keyof typeof EventMetadataKey];
// Check if the result is a valid enum member (not undefined and is a number)
if (typeof key === 'number') {
return key;
}
return undefined;
}

23
packages/core/src/telemetry/constants.ts Normal file
View File

@@ -0,0 +1,23 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
export const SERVICE_NAME = 'gemini-cli';
export const EVENT_USER_PROMPT = 'gemini_cli.user_prompt';
export const EVENT_TOOL_CALL = 'gemini_cli.tool_call';
export const EVENT_API_REQUEST = 'gemini_cli.api_request';
export const EVENT_API_ERROR = 'gemini_cli.api_error';
export const EVENT_API_RESPONSE = 'gemini_cli.api_response';
export const EVENT_CLI_CONFIG = 'gemini_cli.config';
export const EVENT_FLASH_FALLBACK = 'gemini_cli.flash_fallback';
export const METRIC_TOOL_CALL_COUNT = 'gemini_cli.tool.call.count';
export const METRIC_TOOL_CALL_LATENCY = 'gemini_cli.tool.call.latency';
export const METRIC_API_REQUEST_COUNT = 'gemini_cli.api.request.count';
export const METRIC_API_REQUEST_LATENCY = 'gemini_cli.api.request.latency';
export const METRIC_TOKEN_USAGE = 'gemini_cli.token.usage';
export const METRIC_SESSION_COUNT = 'gemini_cli.session.count';
export const METRIC_FILE_OPERATION_COUNT = 'gemini_cli.file.operation.count';

43
packages/core/src/telemetry/index.ts Normal file
View File

@@ -0,0 +1,43 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
export enum TelemetryTarget {
GCP = 'gcp',
LOCAL = 'local',
}
const DEFAULT_TELEMETRY_TARGET = TelemetryTarget.LOCAL;
const DEFAULT_OTLP_ENDPOINT = 'http://localhost:4317';
export { DEFAULT_TELEMETRY_TARGET, DEFAULT_OTLP_ENDPOINT };
export {
initializeTelemetry,
shutdownTelemetry,
isTelemetrySdkInitialized,
} from './sdk.js';
export {
logCliConfiguration,
logUserPrompt,
logToolCall,
logApiRequest,
logApiError,
logApiResponse,
logFlashFallback,
} from './loggers.js';
export {
StartSessionEvent,
EndSessionEvent,
UserPromptEvent,
ToolCallEvent,
ApiRequestEvent,
ApiErrorEvent,
ApiResponseEvent,
TelemetryEvent,
FlashFallbackEvent,
} from './types.js';
export { SpanStatusCode, ValueType } from '@opentelemetry/api';
export { SemanticAttributes } from '@opentelemetry/semantic-conventions';
export * from './uiTelemetry.js';

62
packages/core/src/telemetry/integration.test.circular.ts Normal file
View File

@@ -0,0 +1,62 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
/**
* Integration test to verify circular reference handling with proxy agents
*/
import { describe, it, expect } from 'vitest';
import { ClearcutLogger } from './clearcut-logger/clearcut-logger.js';
import { Config } from '../config/config.js';
describe('Circular Reference Integration Test', () => {
it('should handle HttpsProxyAgent-like circular references in clearcut logging', () => {
// Create a mock config with proxy
const mockConfig = {
getTelemetryEnabled: () => true,
getUsageStatisticsEnabled: () => true,
getSessionId: () => 'test-session',
getModel: () => 'test-model',
getEmbeddingModel: () => 'test-embedding',
getDebugMode: () => false,
getProxy: () => 'http://proxy.example.com:8080',
} as unknown as Config;
// Simulate the structure that causes the circular reference error
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const proxyAgentLike: any = {
sockets: {},
options: { proxy: 'http://proxy.example.com:8080' },
};
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const socketLike: any = {
_httpMessage: {
agent: proxyAgentLike,
socket: null,
},
};
socketLike._httpMessage.socket = socketLike; // Create circular reference
proxyAgentLike.sockets['cloudcode-pa.googleapis.com:443'] = [socketLike];
// Create an event that would contain this circular structure
const problematicEvent = {
error: new Error('Network error'),
function_args: {
filePath: '/test/file.txt',
httpAgent: proxyAgentLike, // This would cause the circular reference
},
};
// Test that ClearcutLogger can handle this
const logger = ClearcutLogger.getInstance(mockConfig);
expect(() => {
logger?.enqueueLogEvent(problematicEvent);
}).not.toThrow();
});
});

119
packages/core/src/telemetry/loggers.test.circular.ts Normal file
View File

@@ -0,0 +1,119 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
/**
* Test to verify circular reference handling in telemetry logging
*/
import { describe, it, expect } from 'vitest';
import { logToolCall } from './loggers.js';
import { ToolCallEvent } from './types.js';
import { Config } from '../config/config.js';
import { CompletedToolCall } from '../core/coreToolScheduler.js';
import { ToolCallRequestInfo, ToolCallResponseInfo } from '../core/turn.js';
import { Tool } from '../tools/tools.js';
describe('Circular Reference Handling', () => {
it('should handle circular references in tool function arguments', () => {
// Create a mock config
const mockConfig = {
getTelemetryEnabled: () => true,
getUsageStatisticsEnabled: () => true,
getSessionId: () => 'test-session',
getModel: () => 'test-model',
getEmbeddingModel: () => 'test-embedding',
getDebugMode: () => false,
} as unknown as Config;
// Create an object with circular references (similar to HttpsProxyAgent)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const circularObject: any = {
sockets: {},
agent: null,
};
circularObject.agent = circularObject; // Create circular reference
circularObject.sockets['test-host'] = [
{ _httpMessage: { agent: circularObject } },
];
// Create a mock CompletedToolCall with circular references in function_args
const mockRequest: ToolCallRequestInfo = {
callId: 'test-call-id',
name: 'ReadFile',
args: circularObject, // This would cause the original error
isClientInitiated: false,
prompt_id: 'test-prompt-id',
};
const mockResponse: ToolCallResponseInfo = {
callId: 'test-call-id',
responseParts: [{ text: 'test result' }],
resultDisplay: undefined,
error: undefined, // undefined means success
};
const mockCompletedToolCall: CompletedToolCall = {
status: 'success',
request: mockRequest,
response: mockResponse,
tool: {} as Tool,
durationMs: 100,
};
// Create a tool call event with circular references in function_args
const event = new ToolCallEvent(mockCompletedToolCall);
// This should not throw an error
expect(() => {
logToolCall(mockConfig, event);
}).not.toThrow();
});
it('should handle normal objects without circular references', () => {
const mockConfig = {
getTelemetryEnabled: () => true,
getUsageStatisticsEnabled: () => true,
getSessionId: () => 'test-session',
getModel: () => 'test-model',
getEmbeddingModel: () => 'test-embedding',
getDebugMode: () => false,
} as unknown as Config;
const normalObject = {
filePath: '/test/path',
options: { encoding: 'utf8' },
};
const mockRequest: ToolCallRequestInfo = {
callId: 'test-call-id',
name: 'ReadFile',
args: normalObject,
isClientInitiated: false,
prompt_id: 'test-prompt-id',
};
const mockResponse: ToolCallResponseInfo = {
callId: 'test-call-id',
responseParts: [{ text: 'test result' }],
resultDisplay: undefined,
error: undefined, // undefined means success
};
const mockCompletedToolCall: CompletedToolCall = {
status: 'success',
request: mockRequest,
response: mockResponse,
tool: {} as Tool,
durationMs: 100,
};
const event = new ToolCallEvent(mockCompletedToolCall);
expect(() => {
logToolCall(mockConfig, event);
}).not.toThrow();
});
});

753
packages/core/src/telemetry/loggers.test.ts Normal file
View File

@@ -0,0 +1,753 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import {
AuthType,
CompletedToolCall,
ContentGeneratorConfig,
EditTool,
ErroredToolCall,
GeminiClient,
ToolConfirmationOutcome,
ToolRegistry,
} from '../index.js';
import { logs } from '@opentelemetry/api-logs';
import { SemanticAttributes } from '@opentelemetry/semantic-conventions';
import { Config } from '../config/config.js';
import {
EVENT_API_REQUEST,
EVENT_API_RESPONSE,
EVENT_CLI_CONFIG,
EVENT_TOOL_CALL,
EVENT_USER_PROMPT,
EVENT_FLASH_FALLBACK,
} from './constants.js';
import {
logApiRequest,
logApiResponse,
logCliConfiguration,
logUserPrompt,
logToolCall,
logFlashFallback,
} from './loggers.js';
import {
ApiRequestEvent,
ApiResponseEvent,
StartSessionEvent,
ToolCallDecision,
ToolCallEvent,
UserPromptEvent,
FlashFallbackEvent,
} from './types.js';
import * as metrics from './metrics.js';
import * as sdk from './sdk.js';
import { vi, describe, beforeEach, it, expect } from 'vitest';
import { GenerateContentResponseUsageMetadata } from '@google/genai';
import * as uiTelemetry from './uiTelemetry.js';
describe('loggers', () => {
const mockLogger = {
emit: vi.fn(),
};
const mockUiEvent = {
addEvent: vi.fn(),
};
beforeEach(() => {
vi.spyOn(sdk, 'isTelemetrySdkInitialized').mockReturnValue(true);
vi.spyOn(logs, 'getLogger').mockReturnValue(mockLogger);
vi.spyOn(uiTelemetry.uiTelemetryService, 'addEvent').mockImplementation(
mockUiEvent.addEvent,
);
vi.useFakeTimers();
vi.setSystemTime(new Date('2025-01-01T00:00:00.000Z'));
});
describe('logCliConfiguration', () => {
it('should log the cli configuration', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getModel: () => 'test-model',
getEmbeddingModel: () => 'test-embedding-model',
getSandbox: () => true,
getCoreTools: () => ['ls', 'read-file'],
getApprovalMode: () => 'default',
getContentGeneratorConfig: () => ({
model: 'test-model',
apiKey: 'test-api-key',
authType: AuthType.USE_VERTEX_AI,
}),
getTelemetryEnabled: () => true,
getUsageStatisticsEnabled: () => true,
getTelemetryLogPromptsEnabled: () => true,
getFileFilteringRespectGitIgnore: () => true,
getFileFilteringAllowBuildArtifacts: () => false,
getDebugMode: () => true,
getMcpServers: () => ({
'test-server': {
command: 'test-command',
},
}),
getQuestion: () => 'test-question',
getTargetDir: () => 'target-dir',
getProxy: () => 'http://test.proxy.com:8080',
} as unknown as Config;
const startSessionEvent = new StartSessionEvent(mockConfig);
logCliConfiguration(mockConfig, startSessionEvent);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'CLI configuration loaded.',
attributes: {
'session.id': 'test-session-id',
'event.name': EVENT_CLI_CONFIG,
'event.timestamp': '2025-01-01T00:00:00.000Z',
model: 'test-model',
embedding_model: 'test-embedding-model',
sandbox_enabled: true,
core_tools_enabled: 'ls,read-file',
approval_mode: 'default',
api_key_enabled: true,
vertex_ai_enabled: true,
log_user_prompts_enabled: true,
file_filtering_respect_git_ignore: true,
debug_mode: true,
mcp_servers: 'test-server',
},
});
});
});
describe('logUserPrompt', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getTelemetryEnabled: () => true,
getTelemetryLogPromptsEnabled: () => true,
getUsageStatisticsEnabled: () => true,
} as unknown as Config;
it('should log a user prompt', () => {
const event = new UserPromptEvent(
11,
'prompt-id-8',
AuthType.USE_VERTEX_AI,
'test-prompt',
);
logUserPrompt(mockConfig, event);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'User prompt. Length: 11.',
attributes: {
'session.id': 'test-session-id',
'event.name': EVENT_USER_PROMPT,
'event.timestamp': '2025-01-01T00:00:00.000Z',
prompt_length: 11,
prompt: 'test-prompt',
},
});
});
it('should not log prompt if disabled', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getTelemetryEnabled: () => true,
getTelemetryLogPromptsEnabled: () => false,
getTargetDir: () => 'target-dir',
getUsageStatisticsEnabled: () => true,
} as unknown as Config;
const event = new UserPromptEvent(
11,
'test-prompt',
AuthType.CLOUD_SHELL,
);
logUserPrompt(mockConfig, event);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'User prompt. Length: 11.',
attributes: {
'session.id': 'test-session-id',
'event.name': EVENT_USER_PROMPT,
'event.timestamp': '2025-01-01T00:00:00.000Z',
prompt_length: 11,
},
});
});
});
describe('logApiResponse', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getTargetDir: () => 'target-dir',
getUsageStatisticsEnabled: () => true,
getTelemetryEnabled: () => true,
getTelemetryLogPromptsEnabled: () => true,
} as Config;
const mockMetrics = {
recordApiResponseMetrics: vi.fn(),
recordTokenUsageMetrics: vi.fn(),
};
beforeEach(() => {
vi.spyOn(metrics, 'recordApiResponseMetrics').mockImplementation(
mockMetrics.recordApiResponseMetrics,
);
vi.spyOn(metrics, 'recordTokenUsageMetrics').mockImplementation(
mockMetrics.recordTokenUsageMetrics,
);
});
it('should log an API response with all fields', () => {
const usageData: GenerateContentResponseUsageMetadata = {
promptTokenCount: 17,
candidatesTokenCount: 50,
cachedContentTokenCount: 10,
thoughtsTokenCount: 5,
toolUsePromptTokenCount: 2,
};
const event = new ApiResponseEvent(
'test-model',
100,
'prompt-id-1',
AuthType.LOGIN_WITH_GOOGLE,
usageData,
'test-response',
);
logApiResponse(mockConfig, event);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'API response from test-model. Status: 200. Duration: 100ms.',
attributes: {
'session.id': 'test-session-id',
'event.name': EVENT_API_RESPONSE,
'event.timestamp': '2025-01-01T00:00:00.000Z',
[SemanticAttributes.HTTP_STATUS_CODE]: 200,
model: 'test-model',
status_code: 200,
duration_ms: 100,
input_token_count: 17,
output_token_count: 50,
cached_content_token_count: 10,
thoughts_token_count: 5,
tool_token_count: 2,
total_token_count: 0,
response_text: 'test-response',
prompt_id: 'prompt-id-1',
auth_type: 'oauth-personal',
},
});
expect(mockMetrics.recordApiResponseMetrics).toHaveBeenCalledWith(
mockConfig,
'test-model',
100,
200,
undefined,
);
expect(mockMetrics.recordTokenUsageMetrics).toHaveBeenCalledWith(
mockConfig,
'test-model',
50,
'output',
);
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
...event,
'event.name': EVENT_API_RESPONSE,
'event.timestamp': '2025-01-01T00:00:00.000Z',
});
});
it('should log an API response with an error', () => {
const usageData: GenerateContentResponseUsageMetadata = {
promptTokenCount: 17,
candidatesTokenCount: 50,
cachedContentTokenCount: 10,
thoughtsTokenCount: 5,
toolUsePromptTokenCount: 2,
};
const event = new ApiResponseEvent(
'test-model',
100,
'prompt-id-1',
AuthType.USE_GEMINI,
usageData,
'test-response',
'test-error',
);
logApiResponse(mockConfig, event);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'API response from test-model. Status: 200. Duration: 100ms.',
attributes: {
'session.id': 'test-session-id',
...event,
'event.name': EVENT_API_RESPONSE,
'event.timestamp': '2025-01-01T00:00:00.000Z',
'error.message': 'test-error',
},
});
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
...event,
'event.name': EVENT_API_RESPONSE,
'event.timestamp': '2025-01-01T00:00:00.000Z',
});
});
});
describe('logApiRequest', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getTargetDir: () => 'target-dir',
getUsageStatisticsEnabled: () => true,
getTelemetryEnabled: () => true,
getTelemetryLogPromptsEnabled: () => true,
} as Config;
it('should log an API request with request_text', () => {
const event = new ApiRequestEvent(
'test-model',
'prompt-id-7',
'This is a test request',
);
logApiRequest(mockConfig, event);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'API request to test-model.',
attributes: {
'session.id': 'test-session-id',
'event.name': EVENT_API_REQUEST,
'event.timestamp': '2025-01-01T00:00:00.000Z',
model: 'test-model',
request_text: 'This is a test request',
prompt_id: 'prompt-id-7',
},
});
});
it('should log an API request without request_text', () => {
const event = new ApiRequestEvent('test-model', 'prompt-id-6');
logApiRequest(mockConfig, event);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'API request to test-model.',
attributes: {
'session.id': 'test-session-id',
'event.name': EVENT_API_REQUEST,
'event.timestamp': '2025-01-01T00:00:00.000Z',
model: 'test-model',
prompt_id: 'prompt-id-6',
},
});
});
});
describe('logFlashFallback', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getUsageStatisticsEnabled: () => true,
} as unknown as Config;
it('should log flash fallback event', () => {
const event = new FlashFallbackEvent(AuthType.USE_VERTEX_AI);
logFlashFallback(mockConfig, event);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'Switching to flash as Fallback.',
attributes: {
'session.id': 'test-session-id',
'event.name': EVENT_FLASH_FALLBACK,
'event.timestamp': '2025-01-01T00:00:00.000Z',
auth_type: 'vertex-ai',
},
});
});
});
describe('logToolCall', () => {
const cfg1 = {
getSessionId: () => 'test-session-id',
getTargetDir: () => 'target-dir',
getGeminiClient: () => mockGeminiClient,
} as Config;
const cfg2 = {
getSessionId: () => 'test-session-id',
getTargetDir: () => 'target-dir',
getProxy: () => 'http://test.proxy.com:8080',
getContentGeneratorConfig: () =>
({ model: 'test-model' }) as ContentGeneratorConfig,
getModel: () => 'test-model',
getEmbeddingModel: () => 'test-embedding-model',
getWorkingDir: () => 'test-working-dir',
getSandbox: () => true,
getCoreTools: () => ['ls', 'read-file'],
getApprovalMode: () => 'default',
getTelemetryLogPromptsEnabled: () => true,
getFileFilteringRespectGitIgnore: () => true,
getFileFilteringAllowBuildArtifacts: () => false,
getDebugMode: () => true,
getMcpServers: () => ({
'test-server': {
command: 'test-command',
},
}),
getQuestion: () => 'test-question',
getToolRegistry: () => new ToolRegistry(cfg1),
getFullContext: () => false,
getUserMemory: () => 'user-memory',
} as unknown as Config;
const mockGeminiClient = new GeminiClient(cfg2);
const mockConfig = {
getSessionId: () => 'test-session-id',
getTargetDir: () => 'target-dir',
getGeminiClient: () => mockGeminiClient,
getUsageStatisticsEnabled: () => true,
getTelemetryEnabled: () => true,
getTelemetryLogPromptsEnabled: () => true,
} as Config;
const mockMetrics = {
recordToolCallMetrics: vi.fn(),
};
beforeEach(() => {
vi.spyOn(metrics, 'recordToolCallMetrics').mockImplementation(
mockMetrics.recordToolCallMetrics,
);
mockLogger.emit.mockReset();
});
it('should log a tool call with all fields', () => {
const call: CompletedToolCall = {
status: 'success',
request: {
name: 'test-function',
args: {
arg1: 'value1',
arg2: 2,
},
callId: 'test-call-id',
isClientInitiated: true,
prompt_id: 'prompt-id-1',
},
response: {
callId: 'test-call-id',
responseParts: 'test-response',
resultDisplay: undefined,
error: undefined,
},
tool: new EditTool(mockConfig),
durationMs: 100,
outcome: ToolConfirmationOutcome.ProceedOnce,
};
const event = new ToolCallEvent(call);
logToolCall(mockConfig, event);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'Tool call: test-function. Decision: accept. Success: true. Duration: 100ms.',
attributes: {
'session.id': 'test-session-id',
'event.name': EVENT_TOOL_CALL,
'event.timestamp': '2025-01-01T00:00:00.000Z',
function_name: 'test-function',
function_args: JSON.stringify(
{
arg1: 'value1',
arg2: 2,
},
null,
2,
),
duration_ms: 100,
success: true,
decision: ToolCallDecision.ACCEPT,
prompt_id: 'prompt-id-1',
},
});
expect(mockMetrics.recordToolCallMetrics).toHaveBeenCalledWith(
mockConfig,
'test-function',
100,
true,
ToolCallDecision.ACCEPT,
);
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
...event,
'event.name': EVENT_TOOL_CALL,
'event.timestamp': '2025-01-01T00:00:00.000Z',
});
});
it('should log a tool call with a reject decision', () => {
const call: ErroredToolCall = {
status: 'error',
request: {
name: 'test-function',
args: {
arg1: 'value1',
arg2: 2,
},
callId: 'test-call-id',
isClientInitiated: true,
prompt_id: 'prompt-id-2',
},
response: {
callId: 'test-call-id',
responseParts: 'test-response',
resultDisplay: undefined,
error: undefined,
},
durationMs: 100,
outcome: ToolConfirmationOutcome.Cancel,
};
const event = new ToolCallEvent(call);
logToolCall(mockConfig, event);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'Tool call: test-function. Decision: reject. Success: false. Duration: 100ms.',
attributes: {
'session.id': 'test-session-id',
'event.name': EVENT_TOOL_CALL,
'event.timestamp': '2025-01-01T00:00:00.000Z',
function_name: 'test-function',
function_args: JSON.stringify(
{
arg1: 'value1',
arg2: 2,
},
null,
2,
),
duration_ms: 100,
success: false,
decision: ToolCallDecision.REJECT,
prompt_id: 'prompt-id-2',
},
});
expect(mockMetrics.recordToolCallMetrics).toHaveBeenCalledWith(
mockConfig,
'test-function',
100,
false,
ToolCallDecision.REJECT,
);
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
...event,
'event.name': EVENT_TOOL_CALL,
'event.timestamp': '2025-01-01T00:00:00.000Z',
});
});
it('should log a tool call with a modify decision', () => {
const call: CompletedToolCall = {
status: 'success',
request: {
name: 'test-function',
args: {
arg1: 'value1',
arg2: 2,
},
callId: 'test-call-id',
isClientInitiated: true,
prompt_id: 'prompt-id-3',
},
response: {
callId: 'test-call-id',
responseParts: 'test-response',
resultDisplay: undefined,
error: undefined,
},
outcome: ToolConfirmationOutcome.ModifyWithEditor,
tool: new EditTool(mockConfig),
durationMs: 100,
};
const event = new ToolCallEvent(call);
logToolCall(mockConfig, event);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'Tool call: test-function. Decision: modify. Success: true. Duration: 100ms.',
attributes: {
'session.id': 'test-session-id',
'event.name': EVENT_TOOL_CALL,
'event.timestamp': '2025-01-01T00:00:00.000Z',
function_name: 'test-function',
function_args: JSON.stringify(
{
arg1: 'value1',
arg2: 2,
},
null,
2,
),
duration_ms: 100,
success: true,
decision: ToolCallDecision.MODIFY,
prompt_id: 'prompt-id-3',
},
});
expect(mockMetrics.recordToolCallMetrics).toHaveBeenCalledWith(
mockConfig,
'test-function',
100,
true,
ToolCallDecision.MODIFY,
);
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
...event,
'event.name': EVENT_TOOL_CALL,
'event.timestamp': '2025-01-01T00:00:00.000Z',
});
});
it('should log a tool call without a decision', () => {
const call: CompletedToolCall = {
status: 'success',
request: {
name: 'test-function',
args: {
arg1: 'value1',
arg2: 2,
},
callId: 'test-call-id',
isClientInitiated: true,
prompt_id: 'prompt-id-4',
},
response: {
callId: 'test-call-id',
responseParts: 'test-response',
resultDisplay: undefined,
error: undefined,
},
tool: new EditTool(mockConfig),
durationMs: 100,
};
const event = new ToolCallEvent(call);
logToolCall(mockConfig, event);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'Tool call: test-function. Success: true. Duration: 100ms.',
attributes: {
'session.id': 'test-session-id',
'event.name': EVENT_TOOL_CALL,
'event.timestamp': '2025-01-01T00:00:00.000Z',
function_name: 'test-function',
function_args: JSON.stringify(
{
arg1: 'value1',
arg2: 2,
},
null,
2,
),
duration_ms: 100,
success: true,
prompt_id: 'prompt-id-4',
},
});
expect(mockMetrics.recordToolCallMetrics).toHaveBeenCalledWith(
mockConfig,
'test-function',
100,
true,
undefined,
);
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
...event,
'event.name': EVENT_TOOL_CALL,
'event.timestamp': '2025-01-01T00:00:00.000Z',
});
});
it('should log a failed tool call with an error', () => {
const call: ErroredToolCall = {
status: 'error',
request: {
name: 'test-function',
args: {
arg1: 'value1',
arg2: 2,
},
callId: 'test-call-id',
isClientInitiated: true,
prompt_id: 'prompt-id-5',
},
response: {
callId: 'test-call-id',
responseParts: 'test-response',
resultDisplay: undefined,
error: {
name: 'test-error-type',
message: 'test-error',
},
},
durationMs: 100,
};
const event = new ToolCallEvent(call);
logToolCall(mockConfig, event);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'Tool call: test-function. Success: false. Duration: 100ms.',
attributes: {
'session.id': 'test-session-id',
'event.name': EVENT_TOOL_CALL,
'event.timestamp': '2025-01-01T00:00:00.000Z',
function_name: 'test-function',
function_args: JSON.stringify(
{
arg1: 'value1',
arg2: 2,
},
null,
2,
),
duration_ms: 100,
success: false,
error: 'test-error',
'error.message': 'test-error',
error_type: 'test-error-type',
'error.type': 'test-error-type',
prompt_id: 'prompt-id-5',
},
});
expect(mockMetrics.recordToolCallMetrics).toHaveBeenCalledWith(
mockConfig,
'test-function',
100,
false,
undefined,
);
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
...event,
'event.name': EVENT_TOOL_CALL,
'event.timestamp': '2025-01-01T00:00:00.000Z',
});
});
});
});

311
packages/core/src/telemetry/loggers.ts Normal file
View File

@@ -0,0 +1,311 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { logs, LogRecord, LogAttributes } from '@opentelemetry/api-logs';
import { SemanticAttributes } from '@opentelemetry/semantic-conventions';
import { Config } from '../config/config.js';
import {
EVENT_API_ERROR,
EVENT_API_REQUEST,
EVENT_API_RESPONSE,
EVENT_CLI_CONFIG,
EVENT_TOOL_CALL,
EVENT_USER_PROMPT,
EVENT_FLASH_FALLBACK,
SERVICE_NAME,
} from './constants.js';
import {
ApiErrorEvent,
ApiRequestEvent,
ApiResponseEvent,
StartSessionEvent,
ToolCallEvent,
UserPromptEvent,
FlashFallbackEvent,
LoopDetectedEvent,
} from './types.js';
import {
recordApiErrorMetrics,
recordTokenUsageMetrics,
recordApiResponseMetrics,
recordToolCallMetrics,
} from './metrics.js';
import { isTelemetrySdkInitialized } from './sdk.js';
import { uiTelemetryService, UiEvent } from './uiTelemetry.js';
import { ClearcutLogger } from './clearcut-logger/clearcut-logger.js';
import { safeJsonStringify } from '../utils/safeJsonStringify.js';
const shouldLogUserPrompts = (config: Config): boolean =>
config.getTelemetryLogPromptsEnabled();
function getCommonAttributes(config: Config): LogAttributes {
return {
'session.id': config.getSessionId(),
};
}
export function logCliConfiguration(
config: Config,
event: StartSessionEvent,
): void {
ClearcutLogger.getInstance(config)?.logStartSessionEvent(event);
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {
...getCommonAttributes(config),
'event.name': EVENT_CLI_CONFIG,
'event.timestamp': new Date().toISOString(),
model: event.model,
embedding_model: event.embedding_model,
sandbox_enabled: event.sandbox_enabled,
core_tools_enabled: event.core_tools_enabled,
approval_mode: event.approval_mode,
api_key_enabled: event.api_key_enabled,
vertex_ai_enabled: event.vertex_ai_enabled,
log_user_prompts_enabled: event.telemetry_log_user_prompts_enabled,
file_filtering_respect_git_ignore: event.file_filtering_respect_git_ignore,
debug_mode: event.debug_enabled,
mcp_servers: event.mcp_servers,
};
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: 'CLI configuration loaded.',
attributes,
};
logger.emit(logRecord);
}
export function logUserPrompt(config: Config, event: UserPromptEvent): void {
ClearcutLogger.getInstance(config)?.logNewPromptEvent(event);
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {
...getCommonAttributes(config),
'event.name': EVENT_USER_PROMPT,
'event.timestamp': new Date().toISOString(),
prompt_length: event.prompt_length,
};
if (shouldLogUserPrompts(config)) {
attributes.prompt = event.prompt;
}
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: `User prompt. Length: ${event.prompt_length}.`,
attributes,
};
logger.emit(logRecord);
}
export function logToolCall(config: Config, event: ToolCallEvent): void {
const uiEvent = {
...event,
'event.name': EVENT_TOOL_CALL,
'event.timestamp': new Date().toISOString(),
} as UiEvent;
uiTelemetryService.addEvent(uiEvent);
ClearcutLogger.getInstance(config)?.logToolCallEvent(event);
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {
...getCommonAttributes(config),
...event,
'event.name': EVENT_TOOL_CALL,
'event.timestamp': new Date().toISOString(),
function_args: safeJsonStringify(event.function_args, 2),
};
if (event.error) {
attributes['error.message'] = event.error;
if (event.error_type) {
attributes['error.type'] = event.error_type;
}
}
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: `Tool call: ${event.function_name}${event.decision ? `. Decision: ${event.decision}` : ''}. Success: ${event.success}. Duration: ${event.duration_ms}ms.`,
attributes,
};
logger.emit(logRecord);
recordToolCallMetrics(
config,
event.function_name,
event.duration_ms,
event.success,
event.decision,
);
}
export function logApiRequest(config: Config, event: ApiRequestEvent): void {
ClearcutLogger.getInstance(config)?.logApiRequestEvent(event);
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {
...getCommonAttributes(config),
...event,
'event.name': EVENT_API_REQUEST,
'event.timestamp': new Date().toISOString(),
};
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: `API request to ${event.model}.`,
attributes,
};
logger.emit(logRecord);
}
export function logFlashFallback(
config: Config,
event: FlashFallbackEvent,
): void {
ClearcutLogger.getInstance(config)?.logFlashFallbackEvent(event);
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {
...getCommonAttributes(config),
...event,
'event.name': EVENT_FLASH_FALLBACK,
'event.timestamp': new Date().toISOString(),
};
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: `Switching to flash as Fallback.`,
attributes,
};
logger.emit(logRecord);
}
export function logApiError(config: Config, event: ApiErrorEvent): void {
const uiEvent = {
...event,
'event.name': EVENT_API_ERROR,
'event.timestamp': new Date().toISOString(),
} as UiEvent;
uiTelemetryService.addEvent(uiEvent);
ClearcutLogger.getInstance(config)?.logApiErrorEvent(event);
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {
...getCommonAttributes(config),
...event,
'event.name': EVENT_API_ERROR,
'event.timestamp': new Date().toISOString(),
['error.message']: event.error,
model_name: event.model,
duration: event.duration_ms,
};
if (event.error_type) {
attributes['error.type'] = event.error_type;
}
if (typeof event.status_code === 'number') {
attributes[SemanticAttributes.HTTP_STATUS_CODE] = event.status_code;
}
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: `API error for ${event.model}. Error: ${event.error}. Duration: ${event.duration_ms}ms.`,
attributes,
};
logger.emit(logRecord);
recordApiErrorMetrics(
config,
event.model,
event.duration_ms,
event.status_code,
event.error_type,
);
}
export function logApiResponse(config: Config, event: ApiResponseEvent): void {
const uiEvent = {
...event,
'event.name': EVENT_API_RESPONSE,
'event.timestamp': new Date().toISOString(),
} as UiEvent;
uiTelemetryService.addEvent(uiEvent);
ClearcutLogger.getInstance(config)?.logApiResponseEvent(event);
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {
...getCommonAttributes(config),
...event,
'event.name': EVENT_API_RESPONSE,
'event.timestamp': new Date().toISOString(),
};
if (event.response_text) {
attributes.response_text = event.response_text;
}
if (event.error) {
attributes['error.message'] = event.error;
} else if (event.status_code) {
if (typeof event.status_code === 'number') {
attributes[SemanticAttributes.HTTP_STATUS_CODE] = event.status_code;
}
}
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: `API response from ${event.model}. Status: ${event.status_code || 'N/A'}. Duration: ${event.duration_ms}ms.`,
attributes,
};
logger.emit(logRecord);
recordApiResponseMetrics(
config,
event.model,
event.duration_ms,
event.status_code,
event.error,
);
recordTokenUsageMetrics(
config,
event.model,
event.input_token_count,
'input',
);
recordTokenUsageMetrics(
config,
event.model,
event.output_token_count,
'output',
);
recordTokenUsageMetrics(
config,
event.model,
event.cached_content_token_count,
'cache',
);
recordTokenUsageMetrics(
config,
event.model,
event.thoughts_token_count,
'thought',
);
recordTokenUsageMetrics(config, event.model, event.tool_token_count, 'tool');
}
export function logLoopDetected(
config: Config,
event: LoopDetectedEvent,
): void {
ClearcutLogger.getInstance(config)?.logLoopDetectedEvent(event);
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {
...getCommonAttributes(config),
...event,
};
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: `Loop detected. Type: ${event.loop_type}.`,
attributes,
};
logger.emit(logRecord);
}

225
packages/core/src/telemetry/metrics.test.ts Normal file
View File

@@ -0,0 +1,225 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect, vi, beforeEach, type Mock } from 'vitest';
import type {
Counter,
Meter,
Attributes,
Context,
Histogram,
} from '@opentelemetry/api';
import { Config } from '../config/config.js';
import { FileOperation } from './metrics.js';
const mockCounterAddFn: Mock<
(value: number, attributes?: Attributes, context?: Context) => void
> = vi.fn();
const mockHistogramRecordFn: Mock<
(value: number, attributes?: Attributes, context?: Context) => void
> = vi.fn();
const mockCreateCounterFn: Mock<(name: string, options?: unknown) => Counter> =
vi.fn();
const mockCreateHistogramFn: Mock<
(name: string, options?: unknown) => Histogram
> = vi.fn();
const mockCounterInstance = {
add: mockCounterAddFn,
} as unknown as Counter;
const mockHistogramInstance = {
record: mockHistogramRecordFn,
} as unknown as Histogram;
const mockMeterInstance = {
createCounter: mockCreateCounterFn.mockReturnValue(mockCounterInstance),
createHistogram: mockCreateHistogramFn.mockReturnValue(mockHistogramInstance),
} as unknown as Meter;
function originalOtelMockFactory() {
return {
metrics: {
getMeter: vi.fn(),
},
ValueType: {
INT: 1,
},
};
}
vi.mock('@opentelemetry/api', originalOtelMockFactory);
describe('Telemetry Metrics', () => {
let initializeMetricsModule: typeof import('./metrics.js').initializeMetrics;
let recordTokenUsageMetricsModule: typeof import('./metrics.js').recordTokenUsageMetrics;
let recordFileOperationMetricModule: typeof import('./metrics.js').recordFileOperationMetric;
beforeEach(async () => {
vi.resetModules();
vi.doMock('@opentelemetry/api', () => {
const actualApi = originalOtelMockFactory();
(actualApi.metrics.getMeter as Mock).mockReturnValue(mockMeterInstance);
return actualApi;
});
const metricsJsModule = await import('./metrics.js');
initializeMetricsModule = metricsJsModule.initializeMetrics;
recordTokenUsageMetricsModule = metricsJsModule.recordTokenUsageMetrics;
recordFileOperationMetricModule = metricsJsModule.recordFileOperationMetric;
const otelApiModule = await import('@opentelemetry/api');
mockCounterAddFn.mockClear();
mockCreateCounterFn.mockClear();
mockCreateHistogramFn.mockClear();
mockHistogramRecordFn.mockClear();
(otelApiModule.metrics.getMeter as Mock).mockClear();
(otelApiModule.metrics.getMeter as Mock).mockReturnValue(mockMeterInstance);
mockCreateCounterFn.mockReturnValue(mockCounterInstance);
mockCreateHistogramFn.mockReturnValue(mockHistogramInstance);
});
describe('recordTokenUsageMetrics', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
} as unknown as Config;
it('should not record metrics if not initialized', () => {
recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 100, 'input');
expect(mockCounterAddFn).not.toHaveBeenCalled();
});
it('should record token usage with the correct attributes', () => {
initializeMetricsModule(mockConfig);
recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 100, 'input');
expect(mockCounterAddFn).toHaveBeenCalledTimes(2);
expect(mockCounterAddFn).toHaveBeenNthCalledWith(1, 1, {
'session.id': 'test-session-id',
});
expect(mockCounterAddFn).toHaveBeenNthCalledWith(2, 100, {
'session.id': 'test-session-id',
model: 'gemini-pro',
type: 'input',
});
});
it('should record token usage for different types', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 50, 'output');
expect(mockCounterAddFn).toHaveBeenCalledWith(50, {
'session.id': 'test-session-id',
model: 'gemini-pro',
type: 'output',
});
recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 25, 'thought');
expect(mockCounterAddFn).toHaveBeenCalledWith(25, {
'session.id': 'test-session-id',
model: 'gemini-pro',
type: 'thought',
});
recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 75, 'cache');
expect(mockCounterAddFn).toHaveBeenCalledWith(75, {
'session.id': 'test-session-id',
model: 'gemini-pro',
type: 'cache',
});
recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 125, 'tool');
expect(mockCounterAddFn).toHaveBeenCalledWith(125, {
'session.id': 'test-session-id',
model: 'gemini-pro',
type: 'tool',
});
});
it('should handle different models', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
recordTokenUsageMetricsModule(mockConfig, 'gemini-ultra', 200, 'input');
expect(mockCounterAddFn).toHaveBeenCalledWith(200, {
'session.id': 'test-session-id',
model: 'gemini-ultra',
type: 'input',
});
});
});
describe('recordFileOperationMetric', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
} as unknown as Config;
it('should not record metrics if not initialized', () => {
recordFileOperationMetricModule(
mockConfig,
FileOperation.CREATE,
10,
'text/plain',
'txt',
);
expect(mockCounterAddFn).not.toHaveBeenCalled();
});
it('should record file creation with all attributes', () => {
initializeMetricsModule(mockConfig);
recordFileOperationMetricModule(
mockConfig,
FileOperation.CREATE,
10,
'text/plain',
'txt',
);
expect(mockCounterAddFn).toHaveBeenCalledTimes(2);
expect(mockCounterAddFn).toHaveBeenNthCalledWith(1, 1, {
'session.id': 'test-session-id',
});
expect(mockCounterAddFn).toHaveBeenNthCalledWith(2, 1, {
'session.id': 'test-session-id',
operation: FileOperation.CREATE,
lines: 10,
mimetype: 'text/plain',
extension: 'txt',
});
});
it('should record file read with minimal attributes', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
recordFileOperationMetricModule(mockConfig, FileOperation.READ);
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
'session.id': 'test-session-id',
operation: FileOperation.READ,
});
});
it('should record file update with some attributes', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
recordFileOperationMetricModule(
mockConfig,
FileOperation.UPDATE,
undefined,
'application/javascript',
);
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
'session.id': 'test-session-id',
operation: FileOperation.UPDATE,
mimetype: 'application/javascript',
});
});
});
});

202
packages/core/src/telemetry/metrics.ts Normal file
View File

@@ -0,0 +1,202 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import {
metrics,
Attributes,
ValueType,
Meter,
Counter,
Histogram,
} from '@opentelemetry/api';
import {
SERVICE_NAME,
METRIC_TOOL_CALL_COUNT,
METRIC_TOOL_CALL_LATENCY,
METRIC_API_REQUEST_COUNT,
METRIC_API_REQUEST_LATENCY,
METRIC_TOKEN_USAGE,
METRIC_SESSION_COUNT,
METRIC_FILE_OPERATION_COUNT,
} from './constants.js';
import { Config } from '../config/config.js';
export enum FileOperation {
CREATE = 'create',
READ = 'read',
UPDATE = 'update',
}
let cliMeter: Meter | undefined;
let toolCallCounter: Counter | undefined;
let toolCallLatencyHistogram: Histogram | undefined;
let apiRequestCounter: Counter | undefined;
let apiRequestLatencyHistogram: Histogram | undefined;
let tokenUsageCounter: Counter | undefined;
let fileOperationCounter: Counter | undefined;
let isMetricsInitialized = false;
function getCommonAttributes(config: Config): Attributes {
return {
'session.id': config.getSessionId(),
};
}
export function getMeter(): Meter | undefined {
if (!cliMeter) {
cliMeter = metrics.getMeter(SERVICE_NAME);
}
return cliMeter;
}
export function initializeMetrics(config: Config): void {
if (isMetricsInitialized) return;
const meter = getMeter();
if (!meter) return;
toolCallCounter = meter.createCounter(METRIC_TOOL_CALL_COUNT, {
description: 'Counts tool calls, tagged by function name and success.',
valueType: ValueType.INT,
});
toolCallLatencyHistogram = meter.createHistogram(METRIC_TOOL_CALL_LATENCY, {
description: 'Latency of tool calls in milliseconds.',
unit: 'ms',
valueType: ValueType.INT,
});
apiRequestCounter = meter.createCounter(METRIC_API_REQUEST_COUNT, {
description: 'Counts API requests, tagged by model and status.',
valueType: ValueType.INT,
});
apiRequestLatencyHistogram = meter.createHistogram(
METRIC_API_REQUEST_LATENCY,
{
description: 'Latency of API requests in milliseconds.',
unit: 'ms',
valueType: ValueType.INT,
},
);
tokenUsageCounter = meter.createCounter(METRIC_TOKEN_USAGE, {
description: 'Counts the total number of tokens used.',
valueType: ValueType.INT,
});
fileOperationCounter = meter.createCounter(METRIC_FILE_OPERATION_COUNT, {
description: 'Counts file operations (create, read, update).',
valueType: ValueType.INT,
});
const sessionCounter = meter.createCounter(METRIC_SESSION_COUNT, {
description: 'Count of CLI sessions started.',
valueType: ValueType.INT,
});
sessionCounter.add(1, getCommonAttributes(config));
isMetricsInitialized = true;
}
export function recordToolCallMetrics(
config: Config,
functionName: string,
durationMs: number,
success: boolean,
decision?: 'accept' | 'reject' | 'modify',
): void {
if (!toolCallCounter || !toolCallLatencyHistogram || !isMetricsInitialized)
return;
const metricAttributes: Attributes = {
...getCommonAttributes(config),
function_name: functionName,
success,
decision,
};
toolCallCounter.add(1, metricAttributes);
toolCallLatencyHistogram.record(durationMs, {
...getCommonAttributes(config),
function_name: functionName,
});
}
export function recordTokenUsageMetrics(
config: Config,
model: string,
tokenCount: number,
type: 'input' | 'output' | 'thought' | 'cache' | 'tool',
): void {
if (!tokenUsageCounter || !isMetricsInitialized) return;
tokenUsageCounter.add(tokenCount, {
...getCommonAttributes(config),
model,
type,
});
}
export function recordApiResponseMetrics(
config: Config,
model: string,
durationMs: number,
statusCode?: number | string,
error?: string,
): void {
if (
!apiRequestCounter ||
!apiRequestLatencyHistogram ||
!isMetricsInitialized
)
return;
const metricAttributes: Attributes = {
...getCommonAttributes(config),
model,
status_code: statusCode ?? (error ? 'error' : 'ok'),
};
apiRequestCounter.add(1, metricAttributes);
apiRequestLatencyHistogram.record(durationMs, {
...getCommonAttributes(config),
model,
});
}
export function recordApiErrorMetrics(
config: Config,
model: string,
durationMs: number,
statusCode?: number | string,
errorType?: string,
): void {
if (
!apiRequestCounter ||
!apiRequestLatencyHistogram ||
!isMetricsInitialized
)
return;
const metricAttributes: Attributes = {
...getCommonAttributes(config),
model,
status_code: statusCode ?? 'error',
error_type: errorType ?? 'unknown',
};
apiRequestCounter.add(1, metricAttributes);
apiRequestLatencyHistogram.record(durationMs, {
...getCommonAttributes(config),
model,
});
}
export function recordFileOperationMetric(
config: Config,
operation: FileOperation,
lines?: number,
mimetype?: string,
extension?: string,
): void {
if (!fileOperationCounter || !isMetricsInitialized) return;
const attributes: Attributes = {
...getCommonAttributes(config),
operation,
};
if (lines !== undefined) attributes.lines = lines;
if (mimetype !== undefined) attributes.mimetype = mimetype;
if (extension !== undefined) attributes.extension = extension;
fileOperationCounter.add(1, attributes);
}

137
packages/core/src/telemetry/sdk.ts Normal file
View File

@@ -0,0 +1,137 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { DiagConsoleLogger, DiagLogLevel, diag } from '@opentelemetry/api';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc';
import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-grpc';
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc';
import { CompressionAlgorithm } from '@opentelemetry/otlp-exporter-base';
import { NodeSDK } from '@opentelemetry/sdk-node';
import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';
import { Resource } from '@opentelemetry/resources';
import {
BatchSpanProcessor,
ConsoleSpanExporter,
} from '@opentelemetry/sdk-trace-node';
import {
BatchLogRecordProcessor,
ConsoleLogRecordExporter,
} from '@opentelemetry/sdk-logs';
import {
ConsoleMetricExporter,
PeriodicExportingMetricReader,
} from '@opentelemetry/sdk-metrics';
import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
import { Config } from '../config/config.js';
import { SERVICE_NAME } from './constants.js';
import { initializeMetrics } from './metrics.js';
import { ClearcutLogger } from './clearcut-logger/clearcut-logger.js';
// For troubleshooting, set the log level to DiagLogLevel.DEBUG
diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.INFO);
let sdk: NodeSDK | undefined;
let telemetryInitialized = false;
export function isTelemetrySdkInitialized(): boolean {
return telemetryInitialized;
}
function parseGrpcEndpoint(
otlpEndpointSetting: string | undefined,
): string | undefined {
if (!otlpEndpointSetting) {
return undefined;
}
// Trim leading/trailing quotes that might come from env variables
const trimmedEndpoint = otlpEndpointSetting.replace(/^["']|["']$/g, '');
try {
const url = new URL(trimmedEndpoint);
// OTLP gRPC exporters expect an endpoint in the format scheme://host:port
// The `origin` property provides this, stripping any path, query, or hash.
return url.origin;
} catch (error) {
diag.error('Invalid OTLP endpoint URL provided:', trimmedEndpoint, error);
return undefined;
}
}
export function initializeTelemetry(config: Config): void {
if (telemetryInitialized || !config.getTelemetryEnabled()) {
return;
}
const resource = new Resource({
[SemanticResourceAttributes.SERVICE_NAME]: SERVICE_NAME,
[SemanticResourceAttributes.SERVICE_VERSION]: process.version,
'session.id': config.getSessionId(),
});
const otlpEndpoint = config.getTelemetryOtlpEndpoint();
const grpcParsedEndpoint = parseGrpcEndpoint(otlpEndpoint);
const useOtlp = !!grpcParsedEndpoint;
const spanExporter = useOtlp
? new OTLPTraceExporter({
url: grpcParsedEndpoint,
compression: CompressionAlgorithm.GZIP,
})
: new ConsoleSpanExporter();
const logExporter = useOtlp
? new OTLPLogExporter({
url: grpcParsedEndpoint,
compression: CompressionAlgorithm.GZIP,
})
: new ConsoleLogRecordExporter();
const metricReader = useOtlp
? new PeriodicExportingMetricReader({
exporter: new OTLPMetricExporter({
url: grpcParsedEndpoint,
compression: CompressionAlgorithm.GZIP,
}),
exportIntervalMillis: 10000,
})
: new PeriodicExportingMetricReader({
exporter: new ConsoleMetricExporter(),
exportIntervalMillis: 10000,
});
sdk = new NodeSDK({
resource,
spanProcessors: [new BatchSpanProcessor(spanExporter)],
logRecordProcessor: new BatchLogRecordProcessor(logExporter),
metricReader,
instrumentations: [new HttpInstrumentation()],
});
try {
sdk.start();
console.log('OpenTelemetry SDK started successfully.');
telemetryInitialized = true;
initializeMetrics(config);
} catch (error) {
console.error('Error starting OpenTelemetry SDK:', error);
}
process.on('SIGTERM', shutdownTelemetry);
process.on('SIGINT', shutdownTelemetry);
}
export async function shutdownTelemetry(): Promise<void> {
if (!telemetryInitialized || !sdk) {
return;
}
try {
ClearcutLogger.getInstance()?.shutdown();
await sdk.shutdown();
console.log('OpenTelemetry SDK shut down successfully.');
} catch (error) {
console.error('Error shutting down SDK:', error);
} finally {
telemetryInitialized = false;
}
}

64
packages/core/src/telemetry/telemetry.test.ts Normal file
View File

@@ -0,0 +1,64 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import {
initializeTelemetry,
shutdownTelemetry,
isTelemetrySdkInitialized,
} from './sdk.js';
import { Config } from '../config/config.js';
import { NodeSDK } from '@opentelemetry/sdk-node';
vi.mock('@opentelemetry/sdk-node');
vi.mock('../config/config.js');
describe('telemetry', () => {
let mockConfig: Config;
let mockNodeSdk: NodeSDK;
beforeEach(() => {
vi.resetAllMocks();
mockConfig = new Config({
sessionId: 'test-session-id',
model: 'test-model',
targetDir: '/test/dir',
debugMode: false,
cwd: '/test/dir',
});
vi.spyOn(mockConfig, 'getTelemetryEnabled').mockReturnValue(true);
vi.spyOn(mockConfig, 'getTelemetryOtlpEndpoint').mockReturnValue(
'http://localhost:4317',
);
vi.spyOn(mockConfig, 'getSessionId').mockReturnValue('test-session-id');
mockNodeSdk = {
start: vi.fn(),
shutdown: vi.fn().mockResolvedValue(undefined),
} as unknown as NodeSDK;
vi.mocked(NodeSDK).mockImplementation(() => mockNodeSdk);
});
afterEach(async () => {
// Ensure we shut down telemetry even if a test fails.
if (isTelemetrySdkInitialized()) {
await shutdownTelemetry();
}
});
it('should initialize the telemetry service', () => {
initializeTelemetry(mockConfig);
expect(NodeSDK).toHaveBeenCalled();
expect(mockNodeSdk.start).toHaveBeenCalled();
});
it('should shutdown the telemetry service', async () => {
initializeTelemetry(mockConfig);
await shutdownTelemetry();
expect(mockNodeSdk.shutdown).toHaveBeenCalled();
});
});

275
packages/core/src/telemetry/types.ts Normal file
View File

@@ -0,0 +1,275 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { GenerateContentResponseUsageMetadata } from '@google/genai';
import { Config } from '../config/config.js';
import { CompletedToolCall } from '../core/coreToolScheduler.js';
import { ToolConfirmationOutcome } from '../tools/tools.js';
import { AuthType } from '../core/contentGenerator.js';
export enum ToolCallDecision {
ACCEPT = 'accept',
REJECT = 'reject',
MODIFY = 'modify',
}
export function getDecisionFromOutcome(
outcome: ToolConfirmationOutcome,
): ToolCallDecision {
switch (outcome) {
case ToolConfirmationOutcome.ProceedOnce:
case ToolConfirmationOutcome.ProceedAlways:
case ToolConfirmationOutcome.ProceedAlwaysServer:
case ToolConfirmationOutcome.ProceedAlwaysTool:
return ToolCallDecision.ACCEPT;
case ToolConfirmationOutcome.ModifyWithEditor:
return ToolCallDecision.MODIFY;
case ToolConfirmationOutcome.Cancel:
default:
return ToolCallDecision.REJECT;
}
}
export class StartSessionEvent {
'event.name': 'cli_config';
'event.timestamp': string; // ISO 8601
model: string;
embedding_model: string;
sandbox_enabled: boolean;
core_tools_enabled: string;
approval_mode: string;
api_key_enabled: boolean;
vertex_ai_enabled: boolean;
debug_enabled: boolean;
mcp_servers: string;
telemetry_enabled: boolean;
telemetry_log_user_prompts_enabled: boolean;
file_filtering_respect_git_ignore: boolean;
constructor(config: Config) {
const generatorConfig = config.getContentGeneratorConfig();
const mcpServers = config.getMcpServers();
let useGemini = false;
let useVertex = false;
if (generatorConfig && generatorConfig.authType) {
useGemini = generatorConfig.authType === AuthType.USE_GEMINI;
useVertex = generatorConfig.authType === AuthType.USE_VERTEX_AI;
}
this['event.name'] = 'cli_config';
this.model = config.getModel();
this.embedding_model = config.getEmbeddingModel();
this.sandbox_enabled =
typeof config.getSandbox() === 'string' || !!config.getSandbox();
this.core_tools_enabled = (config.getCoreTools() ?? []).join(',');
this.approval_mode = config.getApprovalMode();
this.api_key_enabled = useGemini || useVertex;
this.vertex_ai_enabled = useVertex;
this.debug_enabled = config.getDebugMode();
this.mcp_servers = mcpServers ? Object.keys(mcpServers).join(',') : '';
this.telemetry_enabled = config.getTelemetryEnabled();
this.telemetry_log_user_prompts_enabled =
config.getTelemetryLogPromptsEnabled();
this.file_filtering_respect_git_ignore =
config.getFileFilteringRespectGitIgnore();
}
}
export class EndSessionEvent {
'event.name': 'end_session';
'event.timestamp': string; // ISO 8601
session_id?: string;
constructor(config?: Config) {
this['event.name'] = 'end_session';
this['event.timestamp'] = new Date().toISOString();
this.session_id = config?.getSessionId();
}
}
export class UserPromptEvent {
'event.name': 'user_prompt';
'event.timestamp': string; // ISO 8601
prompt_length: number;
prompt_id: string;
auth_type?: string;
prompt?: string;
constructor(
prompt_length: number,
prompt_Id: string,
auth_type?: string,
prompt?: string,
) {
this['event.name'] = 'user_prompt';
this['event.timestamp'] = new Date().toISOString();
this.prompt_length = prompt_length;
this.prompt_id = prompt_Id;
this.auth_type = auth_type;
this.prompt = prompt;
}
}
export class ToolCallEvent {
'event.name': 'tool_call';
'event.timestamp': string; // ISO 8601
function_name: string;
function_args: Record<string, unknown>;
duration_ms: number;
success: boolean;
decision?: ToolCallDecision;
error?: string;
error_type?: string;
prompt_id: string;
constructor(call: CompletedToolCall) {
this['event.name'] = 'tool_call';
this['event.timestamp'] = new Date().toISOString();
this.function_name = call.request.name;
this.function_args = call.request.args;
this.duration_ms = call.durationMs ?? 0;
this.success = call.status === 'success';
this.decision = call.outcome
? getDecisionFromOutcome(call.outcome)
: undefined;
this.error = call.response.error?.message;
this.error_type = call.response.error?.name;
this.prompt_id = call.request.prompt_id;
}
}
export class ApiRequestEvent {
'event.name': 'api_request';
'event.timestamp': string; // ISO 8601
model: string;
prompt_id: string;
request_text?: string;
constructor(model: string, prompt_id: string, request_text?: string) {
this['event.name'] = 'api_request';
this['event.timestamp'] = new Date().toISOString();
this.model = model;
this.prompt_id = prompt_id;
this.request_text = request_text;
}
}
export class ApiErrorEvent {
'event.name': 'api_error';
'event.timestamp': string; // ISO 8601
model: string;
error: string;
error_type?: string;
status_code?: number | string;
duration_ms: number;
prompt_id: string;
auth_type?: string;
constructor(
model: string,
error: string,
duration_ms: number,
prompt_id: string,
auth_type?: string,
error_type?: string,
status_code?: number | string,
) {
this['event.name'] = 'api_error';
this['event.timestamp'] = new Date().toISOString();
this.model = model;
this.error = error;
this.error_type = error_type;
this.status_code = status_code;
this.duration_ms = duration_ms;
this.prompt_id = prompt_id;
this.auth_type = auth_type;
}
}
export class ApiResponseEvent {
'event.name': 'api_response';
'event.timestamp': string; // ISO 8601
model: string;
status_code?: number | string;
duration_ms: number;
error?: string;
input_token_count: number;
output_token_count: number;
cached_content_token_count: number;
thoughts_token_count: number;
tool_token_count: number;
total_token_count: number;
response_text?: string;
prompt_id: string;
auth_type?: string;
constructor(
model: string,
duration_ms: number,
prompt_id: string,
auth_type?: string,
usage_data?: GenerateContentResponseUsageMetadata,
response_text?: string,
error?: string,
) {
this['event.name'] = 'api_response';
this['event.timestamp'] = new Date().toISOString();
this.model = model;
this.duration_ms = duration_ms;
this.status_code = 200;
this.input_token_count = usage_data?.promptTokenCount ?? 0;
this.output_token_count = usage_data?.candidatesTokenCount ?? 0;
this.cached_content_token_count = usage_data?.cachedContentTokenCount ?? 0;
this.thoughts_token_count = usage_data?.thoughtsTokenCount ?? 0;
this.tool_token_count = usage_data?.toolUsePromptTokenCount ?? 0;
this.total_token_count = usage_data?.totalTokenCount ?? 0;
this.response_text = response_text;
this.error = error;
this.prompt_id = prompt_id;
this.auth_type = auth_type;
}
}
export class FlashFallbackEvent {
'event.name': 'flash_fallback';
'event.timestamp': string; // ISO 8601
auth_type: string;
constructor(auth_type: string) {
this['event.name'] = 'flash_fallback';
this['event.timestamp'] = new Date().toISOString();
this.auth_type = auth_type;
}
}
export enum LoopType {
CONSECUTIVE_IDENTICAL_TOOL_CALLS = 'consecutive_identical_tool_calls',
CHANTING_IDENTICAL_SENTENCES = 'chanting_identical_sentences',
}
export class LoopDetectedEvent {
'event.name': 'loop_detected';
'event.timestamp': string; // ISO 8601
loop_type: LoopType;
constructor(loop_type: LoopType) {
this['event.name'] = 'loop_detected';
this['event.timestamp'] = new Date().toISOString();
this.loop_type = loop_type;
}
}
export type TelemetryEvent =
| StartSessionEvent
| EndSessionEvent
| UserPromptEvent
| ToolCallEvent
| ApiRequestEvent
| ApiErrorEvent
| ApiResponseEvent
| FlashFallbackEvent
| LoopDetectedEvent;

511
packages/core/src/telemetry/uiTelemetry.test.ts Normal file
View File

@@ -0,0 +1,511 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { UiTelemetryService } from './uiTelemetry.js';
import {
ApiErrorEvent,
ApiResponseEvent,
ToolCallEvent,
ToolCallDecision,
} from './types.js';
import {
EVENT_API_ERROR,
EVENT_API_RESPONSE,
EVENT_TOOL_CALL,
} from './constants.js';
import {
CompletedToolCall,
ErroredToolCall,
SuccessfulToolCall,
} from '../core/coreToolScheduler.js';
import { Tool, ToolConfirmationOutcome } from '../tools/tools.js';
const createFakeCompletedToolCall = (
name: string,
success: boolean,
duration = 100,
outcome?: ToolConfirmationOutcome,
error?: Error,
): CompletedToolCall => {
const request = {
callId: `call_${name}_${Date.now()}`,
name,
args: { foo: 'bar' },
isClientInitiated: false,
prompt_id: 'prompt-id-1',
};
if (success) {
return {
status: 'success',
request,
tool: { name } as Tool, // Mock tool
response: {
callId: request.callId,
responseParts: {
functionResponse: {
id: request.callId,
name,
response: { output: 'Success!' },
},
},
error: undefined,
resultDisplay: 'Success!',
},
durationMs: duration,
outcome,
} as SuccessfulToolCall;
} else {
return {
status: 'error',
request,
response: {
callId: request.callId,
responseParts: {
functionResponse: {
id: request.callId,
name,
response: { error: 'Tool failed' },
},
},
error: error || new Error('Tool failed'),
resultDisplay: 'Failure!',
},
durationMs: duration,
outcome,
} as ErroredToolCall;
}
};
describe('UiTelemetryService', () => {
let service: UiTelemetryService;
beforeEach(() => {
service = new UiTelemetryService();
});
it('should have correct initial metrics', () => {
const metrics = service.getMetrics();
expect(metrics).toEqual({
models: {},
tools: {
totalCalls: 0,
totalSuccess: 0,
totalFail: 0,
totalDurationMs: 0,
totalDecisions: {
[ToolCallDecision.ACCEPT]: 0,
[ToolCallDecision.REJECT]: 0,
[ToolCallDecision.MODIFY]: 0,
},
byName: {},
},
});
expect(service.getLastPromptTokenCount()).toBe(0);
});
it('should emit an update event when an event is added', () => {
const spy = vi.fn();
service.on('update', spy);
const event = {
'event.name': EVENT_API_RESPONSE,
model: 'gemini-2.5-pro',
duration_ms: 500,
input_token_count: 10,
output_token_count: 20,
total_token_count: 30,
cached_content_token_count: 5,
thoughts_token_count: 2,
tool_token_count: 3,
} as ApiResponseEvent & { 'event.name': typeof EVENT_API_RESPONSE };
service.addEvent(event);
expect(spy).toHaveBeenCalledOnce();
const { metrics, lastPromptTokenCount } = spy.mock.calls[0][0];
expect(metrics).toBeDefined();
expect(lastPromptTokenCount).toBe(10);
});
describe('API Response Event Processing', () => {
it('should process a single ApiResponseEvent', () => {
const event = {
'event.name': EVENT_API_RESPONSE,
model: 'gemini-2.5-pro',
duration_ms: 500,
input_token_count: 10,
output_token_count: 20,
total_token_count: 30,
cached_content_token_count: 5,
thoughts_token_count: 2,
tool_token_count: 3,
} as ApiResponseEvent & { 'event.name': typeof EVENT_API_RESPONSE };
service.addEvent(event);
const metrics = service.getMetrics();
expect(metrics.models['gemini-2.5-pro']).toEqual({
api: {
totalRequests: 1,
totalErrors: 0,
totalLatencyMs: 500,
},
tokens: {
prompt: 10,
candidates: 20,
total: 30,
cached: 5,
thoughts: 2,
tool: 3,
},
});
expect(service.getLastPromptTokenCount()).toBe(10);
});
it('should aggregate multiple ApiResponseEvents for the same model', () => {
const event1 = {
'event.name': EVENT_API_RESPONSE,
model: 'gemini-2.5-pro',
duration_ms: 500,
input_token_count: 10,
output_token_count: 20,
total_token_count: 30,
cached_content_token_count: 5,
thoughts_token_count: 2,
tool_token_count: 3,
} as ApiResponseEvent & {
'event.name': typeof EVENT_API_RESPONSE;
};
const event2 = {
'event.name': EVENT_API_RESPONSE,
model: 'gemini-2.5-pro',
duration_ms: 600,
input_token_count: 15,
output_token_count: 25,
total_token_count: 40,
cached_content_token_count: 10,
thoughts_token_count: 4,
tool_token_count: 6,
} as ApiResponseEvent & {
'event.name': typeof EVENT_API_RESPONSE;
};
service.addEvent(event1);
service.addEvent(event2);
const metrics = service.getMetrics();
expect(metrics.models['gemini-2.5-pro']).toEqual({
api: {
totalRequests: 2,
totalErrors: 0,
totalLatencyMs: 1100,
},
tokens: {
prompt: 25,
candidates: 45,
total: 70,
cached: 15,
thoughts: 6,
tool: 9,
},
});
expect(service.getLastPromptTokenCount()).toBe(15);
});
it('should handle ApiResponseEvents for different models', () => {
const event1 = {
'event.name': EVENT_API_RESPONSE,
model: 'gemini-2.5-pro',
duration_ms: 500,
input_token_count: 10,
output_token_count: 20,
total_token_count: 30,
cached_content_token_count: 5,
thoughts_token_count: 2,
tool_token_count: 3,
} as ApiResponseEvent & {
'event.name': typeof EVENT_API_RESPONSE;
};
const event2 = {
'event.name': EVENT_API_RESPONSE,
model: 'gemini-2.5-flash',
duration_ms: 1000,
input_token_count: 100,
output_token_count: 200,
total_token_count: 300,
cached_content_token_count: 50,
thoughts_token_count: 20,
tool_token_count: 30,
} as ApiResponseEvent & {
'event.name': typeof EVENT_API_RESPONSE;
};
service.addEvent(event1);
service.addEvent(event2);
const metrics = service.getMetrics();
expect(metrics.models['gemini-2.5-pro']).toBeDefined();
expect(metrics.models['gemini-2.5-flash']).toBeDefined();
expect(metrics.models['gemini-2.5-pro'].api.totalRequests).toBe(1);
expect(metrics.models['gemini-2.5-flash'].api.totalRequests).toBe(1);
expect(service.getLastPromptTokenCount()).toBe(100);
});
});
describe('API Error Event Processing', () => {
it('should process a single ApiErrorEvent', () => {
const event = {
'event.name': EVENT_API_ERROR,
model: 'gemini-2.5-pro',
duration_ms: 300,
error: 'Something went wrong',
} as ApiErrorEvent & { 'event.name': typeof EVENT_API_ERROR };
service.addEvent(event);
const metrics = service.getMetrics();
expect(metrics.models['gemini-2.5-pro']).toEqual({
api: {
totalRequests: 1,
totalErrors: 1,
totalLatencyMs: 300,
},
tokens: {
prompt: 0,
candidates: 0,
total: 0,
cached: 0,
thoughts: 0,
tool: 0,
},
});
});
it('should aggregate ApiErrorEvents and ApiResponseEvents', () => {
const responseEvent = {
'event.name': EVENT_API_RESPONSE,
model: 'gemini-2.5-pro',
duration_ms: 500,
input_token_count: 10,
output_token_count: 20,
total_token_count: 30,
cached_content_token_count: 5,
thoughts_token_count: 2,
tool_token_count: 3,
} as ApiResponseEvent & {
'event.name': typeof EVENT_API_RESPONSE;
};
const errorEvent = {
'event.name': EVENT_API_ERROR,
model: 'gemini-2.5-pro',
duration_ms: 300,
error: 'Something went wrong',
} as ApiErrorEvent & { 'event.name': typeof EVENT_API_ERROR };
service.addEvent(responseEvent);
service.addEvent(errorEvent);
const metrics = service.getMetrics();
expect(metrics.models['gemini-2.5-pro']).toEqual({
api: {
totalRequests: 2,
totalErrors: 1,
totalLatencyMs: 800,
},
tokens: {
prompt: 10,
candidates: 20,
total: 30,
cached: 5,
thoughts: 2,
tool: 3,
},
});
});
});
describe('Tool Call Event Processing', () => {
it('should process a single successful ToolCallEvent', () => {
const toolCall = createFakeCompletedToolCall(
'test_tool',
true,
150,
ToolConfirmationOutcome.ProceedOnce,
);
service.addEvent({
...JSON.parse(JSON.stringify(new ToolCallEvent(toolCall))),
'event.name': EVENT_TOOL_CALL,
});
const metrics = service.getMetrics();
const { tools } = metrics;
expect(tools.totalCalls).toBe(1);
expect(tools.totalSuccess).toBe(1);
expect(tools.totalFail).toBe(0);
expect(tools.totalDurationMs).toBe(150);
expect(tools.totalDecisions[ToolCallDecision.ACCEPT]).toBe(1);
expect(tools.byName['test_tool']).toEqual({
count: 1,
success: 1,
fail: 0,
durationMs: 150,
decisions: {
[ToolCallDecision.ACCEPT]: 1,
[ToolCallDecision.REJECT]: 0,
[ToolCallDecision.MODIFY]: 0,
},
});
});
it('should process a single failed ToolCallEvent', () => {
const toolCall = createFakeCompletedToolCall(
'test_tool',
false,
200,
ToolConfirmationOutcome.Cancel,
);
service.addEvent({
...JSON.parse(JSON.stringify(new ToolCallEvent(toolCall))),
'event.name': EVENT_TOOL_CALL,
});
const metrics = service.getMetrics();
const { tools } = metrics;
expect(tools.totalCalls).toBe(1);
expect(tools.totalSuccess).toBe(0);
expect(tools.totalFail).toBe(1);
expect(tools.totalDurationMs).toBe(200);
expect(tools.totalDecisions[ToolCallDecision.REJECT]).toBe(1);
expect(tools.byName['test_tool']).toEqual({
count: 1,
success: 0,
fail: 1,
durationMs: 200,
decisions: {
[ToolCallDecision.ACCEPT]: 0,
[ToolCallDecision.REJECT]: 1,
[ToolCallDecision.MODIFY]: 0,
},
});
});
it('should process a ToolCallEvent with modify decision', () => {
const toolCall = createFakeCompletedToolCall(
'test_tool',
true,
250,
ToolConfirmationOutcome.ModifyWithEditor,
);
service.addEvent({
...JSON.parse(JSON.stringify(new ToolCallEvent(toolCall))),
'event.name': EVENT_TOOL_CALL,
});
const metrics = service.getMetrics();
const { tools } = metrics;
expect(tools.totalDecisions[ToolCallDecision.MODIFY]).toBe(1);
expect(tools.byName['test_tool'].decisions[ToolCallDecision.MODIFY]).toBe(
1,
);
});
it('should process a ToolCallEvent without a decision', () => {
const toolCall = createFakeCompletedToolCall('test_tool', true, 100);
service.addEvent({
...JSON.parse(JSON.stringify(new ToolCallEvent(toolCall))),
'event.name': EVENT_TOOL_CALL,
});
const metrics = service.getMetrics();
const { tools } = metrics;
expect(tools.totalDecisions).toEqual({
[ToolCallDecision.ACCEPT]: 0,
[ToolCallDecision.REJECT]: 0,
[ToolCallDecision.MODIFY]: 0,
});
expect(tools.byName['test_tool'].decisions).toEqual({
[ToolCallDecision.ACCEPT]: 0,
[ToolCallDecision.REJECT]: 0,
[ToolCallDecision.MODIFY]: 0,
});
});
it('should aggregate multiple ToolCallEvents for the same tool', () => {
const toolCall1 = createFakeCompletedToolCall(
'test_tool',
true,
100,
ToolConfirmationOutcome.ProceedOnce,
);
const toolCall2 = createFakeCompletedToolCall(
'test_tool',
false,
150,
ToolConfirmationOutcome.Cancel,
);
service.addEvent({
...JSON.parse(JSON.stringify(new ToolCallEvent(toolCall1))),
'event.name': EVENT_TOOL_CALL,
});
service.addEvent({
...JSON.parse(JSON.stringify(new ToolCallEvent(toolCall2))),
'event.name': EVENT_TOOL_CALL,
});
const metrics = service.getMetrics();
const { tools } = metrics;
expect(tools.totalCalls).toBe(2);
expect(tools.totalSuccess).toBe(1);
expect(tools.totalFail).toBe(1);
expect(tools.totalDurationMs).toBe(250);
expect(tools.totalDecisions[ToolCallDecision.ACCEPT]).toBe(1);
expect(tools.totalDecisions[ToolCallDecision.REJECT]).toBe(1);
expect(tools.byName['test_tool']).toEqual({
count: 2,
success: 1,
fail: 1,
durationMs: 250,
decisions: {
[ToolCallDecision.ACCEPT]: 1,
[ToolCallDecision.REJECT]: 1,
[ToolCallDecision.MODIFY]: 0,
},
});
});
it('should handle ToolCallEvents for different tools', () => {
const toolCall1 = createFakeCompletedToolCall('tool_A', true, 100);
const toolCall2 = createFakeCompletedToolCall('tool_B', false, 200);
service.addEvent({
...JSON.parse(JSON.stringify(new ToolCallEvent(toolCall1))),
'event.name': EVENT_TOOL_CALL,
});
service.addEvent({
...JSON.parse(JSON.stringify(new ToolCallEvent(toolCall2))),
'event.name': EVENT_TOOL_CALL,
});
const metrics = service.getMetrics();
const { tools } = metrics;
expect(tools.totalCalls).toBe(2);
expect(tools.totalSuccess).toBe(1);
expect(tools.totalFail).toBe(1);
expect(tools.byName['tool_A']).toBeDefined();
expect(tools.byName['tool_B']).toBeDefined();
expect(tools.byName['tool_A'].count).toBe(1);
expect(tools.byName['tool_B'].count).toBe(1);
});
});
});

207
packages/core/src/telemetry/uiTelemetry.ts Normal file
View File

@@ -0,0 +1,207 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { EventEmitter } from 'events';
import {
EVENT_API_ERROR,
EVENT_API_RESPONSE,
EVENT_TOOL_CALL,
} from './constants.js';
import {
ApiErrorEvent,
ApiResponseEvent,
ToolCallEvent,
ToolCallDecision,
} from './types.js';
export type UiEvent =
| (ApiResponseEvent & { 'event.name': typeof EVENT_API_RESPONSE })
| (ApiErrorEvent & { 'event.name': typeof EVENT_API_ERROR })
| (ToolCallEvent & { 'event.name': typeof EVENT_TOOL_CALL });
export interface ToolCallStats {
count: number;
success: number;
fail: number;
durationMs: number;
decisions: {
[ToolCallDecision.ACCEPT]: number;
[ToolCallDecision.REJECT]: number;
[ToolCallDecision.MODIFY]: number;
};
}
export interface ModelMetrics {
api: {
totalRequests: number;
totalErrors: number;
totalLatencyMs: number;
};
tokens: {
prompt: number;
candidates: number;
total: number;
cached: number;
thoughts: number;
tool: number;
};
}
export interface SessionMetrics {
models: Record<string, ModelMetrics>;
tools: {
totalCalls: number;
totalSuccess: number;
totalFail: number;
totalDurationMs: number;
totalDecisions: {
[ToolCallDecision.ACCEPT]: number;
[ToolCallDecision.REJECT]: number;
[ToolCallDecision.MODIFY]: number;
};
byName: Record<string, ToolCallStats>;
};
}
const createInitialModelMetrics = (): ModelMetrics => ({
api: {
totalRequests: 0,
totalErrors: 0,
totalLatencyMs: 0,
},
tokens: {
prompt: 0,
candidates: 0,
total: 0,
cached: 0,
thoughts: 0,
tool: 0,
},
});
const createInitialMetrics = (): SessionMetrics => ({
models: {},
tools: {
totalCalls: 0,
totalSuccess: 0,
totalFail: 0,
totalDurationMs: 0,
totalDecisions: {
[ToolCallDecision.ACCEPT]: 0,
[ToolCallDecision.REJECT]: 0,
[ToolCallDecision.MODIFY]: 0,
},
byName: {},
},
});
export class UiTelemetryService extends EventEmitter {
#metrics: SessionMetrics = createInitialMetrics();
#lastPromptTokenCount = 0;
addEvent(event: UiEvent) {
switch (event['event.name']) {
case EVENT_API_RESPONSE:
this.processApiResponse(event);
break;
case EVENT_API_ERROR:
this.processApiError(event);
break;
case EVENT_TOOL_CALL:
this.processToolCall(event);
break;
default:
// We should not emit update for any other event metric.
return;
}
this.emit('update', {
metrics: this.#metrics,
lastPromptTokenCount: this.#lastPromptTokenCount,
});
}
getMetrics(): SessionMetrics {
return this.#metrics;
}
getLastPromptTokenCount(): number {
return this.#lastPromptTokenCount;
}
private getOrCreateModelMetrics(modelName: string): ModelMetrics {
if (!this.#metrics.models[modelName]) {
this.#metrics.models[modelName] = createInitialModelMetrics();
}
return this.#metrics.models[modelName];
}
private processApiResponse(event: ApiResponseEvent) {
const modelMetrics = this.getOrCreateModelMetrics(event.model);
modelMetrics.api.totalRequests++;
modelMetrics.api.totalLatencyMs += event.duration_ms;
modelMetrics.tokens.prompt += event.input_token_count;
modelMetrics.tokens.candidates += event.output_token_count;
modelMetrics.tokens.total += event.total_token_count;
modelMetrics.tokens.cached += event.cached_content_token_count;
modelMetrics.tokens.thoughts += event.thoughts_token_count;
modelMetrics.tokens.tool += event.tool_token_count;
this.#lastPromptTokenCount = event.input_token_count;
}
private processApiError(event: ApiErrorEvent) {
const modelMetrics = this.getOrCreateModelMetrics(event.model);
modelMetrics.api.totalRequests++;
modelMetrics.api.totalErrors++;
modelMetrics.api.totalLatencyMs += event.duration_ms;
}
private processToolCall(event: ToolCallEvent) {
const { tools } = this.#metrics;
tools.totalCalls++;
tools.totalDurationMs += event.duration_ms;
if (event.success) {
tools.totalSuccess++;
} else {
tools.totalFail++;
}
if (!tools.byName[event.function_name]) {
tools.byName[event.function_name] = {
count: 0,
success: 0,
fail: 0,
durationMs: 0,
decisions: {
[ToolCallDecision.ACCEPT]: 0,
[ToolCallDecision.REJECT]: 0,
[ToolCallDecision.MODIFY]: 0,
},
};
}
const toolStats = tools.byName[event.function_name];
toolStats.count++;
toolStats.durationMs += event.duration_ms;
if (event.success) {
toolStats.success++;
} else {
toolStats.fail++;
}
if (event.decision) {
tools.totalDecisions[event.decision]++;
toolStats.decisions[event.decision]++;
}
}
}
export const uiTelemetryService = new UiTelemetryService();