abbyswag/qwen-code
1
0
Fork 0
mirror of https://github.com/QwenLM/qwen-code.git synced 2025-12-20 16:57:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Clearcut logging - initial implementation (#1274)

Browse Source 
Flag-guarded initial implementation of a clearcut logger to collect telemetry data and send it to Concord for dashboards, etc.
This commit is contained in:
owenofbrien
2025-06-22 09:26:48 -05:00
committed by GitHub
parent c9950b3cb2
commit 4cfab0a893
23 changed files with 1051 additions and 335 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
packages/core/src/telemetry/loggers.ts
View File

@@ -20,6 +20,7 @@ import {
ApiErrorEvent,
ApiRequestEvent,
ApiResponseEvent,
StartSessionEvent,
ToolCallEvent,
UserPromptEvent,
} from './types.js';
@@ -30,15 +31,10 @@ import {
recordToolCallMetrics,
} from './metrics.js';
import { isTelemetrySdkInitialized } from './sdk.js';
import { ToolConfirmationOutcome } from '../tools/tools.js';
import {
GenerateContentResponse,
GenerateContentResponseUsageMetadata,
} from '@google/genai';
import { AuthType } from '../core/contentGenerator.js';
import { ClearcutLogger } from './clearcut-logger/clearcut-logger.js';
const shouldLogUserPrompts = (config: Config): boolean =>
config.getTelemetryLogPromptsEnabled() ?? false;
config.getTelemetryLogPromptsEnabled();
function getCommonAttributes(config: Config): LogAttributes {
return {
@@ -46,59 +42,30 @@ function getCommonAttributes(config: Config): LogAttributes {
};
}
export enum ToolCallDecision {
ACCEPT = 'accept',
REJECT = 'reject',
MODIFY = 'modify',
}
export function getDecisionFromOutcome(
outcome: ToolConfirmationOutcome,
): ToolCallDecision {
switch (outcome) {
case ToolConfirmationOutcome.ProceedOnce:
case ToolConfirmationOutcome.ProceedAlways:
case ToolConfirmationOutcome.ProceedAlwaysServer:
case ToolConfirmationOutcome.ProceedAlwaysTool:
return ToolCallDecision.ACCEPT;
case ToolConfirmationOutcome.ModifyWithEditor:
return ToolCallDecision.MODIFY;
case ToolConfirmationOutcome.Cancel:
default:
return ToolCallDecision.REJECT;
}
}
export function logCliConfiguration(config: Config): void {
export function logCliConfiguration(
config: Config,
event: StartSessionEvent,
): void {
ClearcutLogger.getInstance(config)?.logStartSessionEvent(event);
if (!isTelemetrySdkInitialized()) return;
const generatorConfig = config.getContentGeneratorConfig();
let useGemini = false;
let useVertex = false;
if (generatorConfig && generatorConfig.authType) {
useGemini = generatorConfig.authType === AuthType.USE_GEMINI;
useVertex = generatorConfig.authType === AuthType.USE_VERTEX_AI;
}
const mcpServers = config.getMcpServers();
const attributes: LogAttributes = {
...getCommonAttributes(config),
'event.name': EVENT_CLI_CONFIG,
'event.timestamp': new Date().toISOString(),
model: config.getModel(),
embedding_model: config.getEmbeddingModel(),
sandbox_enabled: !!config.getSandbox(),
core_tools_enabled: (config.getCoreTools() ?? []).join(','),
approval_mode: config.getApprovalMode(),
api_key_enabled: useGemini || useVertex,
vertex_ai_enabled: useVertex,
log_user_prompts_enabled: config.getTelemetryLogPromptsEnabled(),
file_filtering_respect_git_ignore:
config.getFileFilteringRespectGitIgnore(),
debug_mode: config.getDebugMode(),
mcp_servers: mcpServers ? Object.keys(mcpServers).join(',') : '',
model: event.model,
embedding_model: event.embedding_model,
sandbox_enabled: event.sandbox_enabled,
core_tools_enabled: event.core_tools_enabled,
approval_mode: event.approval_mode,
api_key_enabled: event.api_key_enabled,
vertex_ai_enabled: event.vertex_ai_enabled,
log_user_prompts_enabled: event.telemetry_log_user_prompts_enabled,
file_filtering_respect_git_ignore: event.file_filtering_respect_git_ignore,
debug_mode: event.debug_enabled,
mcp_servers: event.mcp_servers,
};
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: 'CLI configuration loaded.',
@@ -107,12 +74,8 @@ export function logCliConfiguration(config: Config): void {
logger.emit(logRecord);
}
export function logUserPrompt(
config: Config,
event: Omit<UserPromptEvent, 'event.name' | 'event.timestamp' | 'prompt'> & {
prompt: string;
},
): void {
export function logUserPrompt(config: Config, event: UserPromptEvent): void {
ClearcutLogger.getInstance(config)?.logNewPromptEvent(event);
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {
@@ -134,22 +97,16 @@ export function logUserPrompt(
logger.emit(logRecord);
}
export function logToolCall(
config: Config,
event: Omit<ToolCallEvent, 'event.name' | 'event.timestamp' | 'decision'>,
outcome?: ToolConfirmationOutcome,
): void {
export function logToolCall(config: Config, event: ToolCallEvent): void {
ClearcutLogger.getInstance(config)?.logToolCallEvent(event);
if (!isTelemetrySdkInitialized()) return;
const decision = outcome ? getDecisionFromOutcome(outcome) : undefined;
const attributes: LogAttributes = {
...getCommonAttributes(config),
...event,
'event.name': EVENT_TOOL_CALL,
'event.timestamp': new Date().toISOString(),
function_args: JSON.stringify(event.function_args, null, 2),
decision,
};
if (event.error) {
attributes['error.message'] = event.error;
@@ -157,9 +114,10 @@ export function logToolCall(
attributes['error.type'] = event.error_type;
}
}
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: `Tool call: ${event.function_name}${decision ? `. Decision: ${decision}` : ''}. Success: ${event.success}. Duration: ${event.duration_ms}ms.`,
body: `Tool call: ${event.function_name}${event.decision ? `. Decision: ${event.decision}` : ''}. Success: ${event.success}. Duration: ${event.duration_ms}ms.`,
attributes,
};
logger.emit(logRecord);
@@ -168,21 +126,21 @@ export function logToolCall(
event.function_name,
event.duration_ms,
event.success,
decision,
event.decision,
);
}
export function logApiRequest(
config: Config,
event: Omit<ApiRequestEvent, 'event.name' | 'event.timestamp'>,
): void {
export function logApiRequest(config: Config, event: ApiRequestEvent): void {
ClearcutLogger.getInstance(config)?.logApiRequestEvent(event);
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {
...getCommonAttributes(config),
...event,
'event.name': EVENT_API_REQUEST,
'event.timestamp': new Date().toISOString(),
};
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: `API request to ${event.model}.`,
@@ -191,17 +149,18 @@ export function logApiRequest(
logger.emit(logRecord);
}
export function logApiError(
config: Config,
event: Omit<ApiErrorEvent, 'event.name' | 'event.timestamp'>,
): void {
export function logApiError(config: Config, event: ApiErrorEvent): void {
ClearcutLogger.getInstance(config)?.logApiErrorEvent(event);
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {
...getCommonAttributes(config),
...event,
'event.name': EVENT_API_ERROR,
'event.timestamp': new Date().toISOString(),
['error.message']: event.error,
model_name: event.model,
duration: event.duration_ms,
};
if (event.error_type) {
@@ -226,10 +185,8 @@ export function logApiError(
);
}
export function logApiResponse(
config: Config,
event: Omit<ApiResponseEvent, 'event.name' | 'event.timestamp'>,
): void {
export function logApiResponse(config: Config, event: ApiResponseEvent): void {
ClearcutLogger.getInstance(config)?.logApiResponseEvent(event);
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {
...getCommonAttributes(config),
@@ -287,15 +244,3 @@ export function logApiResponse(
);
recordTokenUsageMetrics(config, event.model, event.tool_token_count, 'tool');
}
export function getFinalUsageMetadata(
chunks: GenerateContentResponse[],
): GenerateContentResponseUsageMetadata | undefined {
// Only the last streamed item has the final token count.
const lastChunkWithMetadata = chunks
.slice()
.reverse()
.find((chunk) => chunk.usageMetadata);
return lastChunkWithMetadata?.usageMetadata;
}