Files
qwen-code/packages/core/src/telemetry/metrics.ts
2025-10-23 09:27:04 +08:00

750 lines
23 KiB
TypeScript

/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import type { Attributes, Meter, Counter, Histogram } from '@opentelemetry/api';
import { diag, metrics, ValueType } from '@opentelemetry/api';
import { SERVICE_NAME, EVENT_CHAT_COMPRESSION } from './constants.js';
import type { Config } from '../config/config.js';
import type { ModelSlashCommandEvent } from './types.js';
const TOOL_CALL_COUNT = `${SERVICE_NAME}.tool.call.count`;
const TOOL_CALL_LATENCY = `${SERVICE_NAME}.tool.call.latency`;
const API_REQUEST_COUNT = `${SERVICE_NAME}.api.request.count`;
const API_REQUEST_LATENCY = `${SERVICE_NAME}.api.request.latency`;
const TOKEN_USAGE = `${SERVICE_NAME}.token.usage`;
const SESSION_COUNT = `${SERVICE_NAME}.session.count`;
const FILE_OPERATION_COUNT = `${SERVICE_NAME}.file.operation.count`;
const INVALID_CHUNK_COUNT = `${SERVICE_NAME}.chat.invalid_chunk.count`;
const CONTENT_RETRY_COUNT = `${SERVICE_NAME}.chat.content_retry.count`;
const CONTENT_RETRY_FAILURE_COUNT = `${SERVICE_NAME}.chat.content_retry_failure.count`;
const MODEL_SLASH_COMMAND_CALL_COUNT = `${SERVICE_NAME}.slash_command.model.call_count`;
export const SUBAGENT_EXECUTION_COUNT = `${SERVICE_NAME}.subagent.execution.count`;
// Performance Monitoring Metrics
const STARTUP_TIME = `${SERVICE_NAME}.startup.duration`;
const MEMORY_USAGE = `${SERVICE_NAME}.memory.usage`;
const CPU_USAGE = `${SERVICE_NAME}.cpu.usage`;
const TOOL_QUEUE_DEPTH = `${SERVICE_NAME}.tool.queue.depth`;
const TOOL_EXECUTION_BREAKDOWN = `${SERVICE_NAME}.tool.execution.breakdown`;
const TOKEN_EFFICIENCY = `${SERVICE_NAME}.token.efficiency`;
const API_REQUEST_BREAKDOWN = `${SERVICE_NAME}.api.request.breakdown`;
const PERFORMANCE_SCORE = `${SERVICE_NAME}.performance.score`;
const REGRESSION_DETECTION = `${SERVICE_NAME}.performance.regression`;
const REGRESSION_PERCENTAGE_CHANGE = `${SERVICE_NAME}.performance.regression.percentage_change`;
const BASELINE_COMPARISON = `${SERVICE_NAME}.performance.baseline.comparison`;
const baseMetricDefinition = {
getCommonAttributes: (config: Config): Attributes => ({
'session.id': config.getSessionId(),
}),
};
const COUNTER_DEFINITIONS = {
[TOOL_CALL_COUNT]: {
description: 'Counts tool calls, tagged by function name and success.',
valueType: ValueType.INT,
assign: (c: Counter) => (toolCallCounter = c),
attributes: {} as {
function_name: string;
success: boolean;
decision?: 'accept' | 'reject' | 'modify' | 'auto_accept';
tool_type?: 'native' | 'mcp';
},
},
[API_REQUEST_COUNT]: {
description: 'Counts API requests, tagged by model and status.',
valueType: ValueType.INT,
assign: (c: Counter) => (apiRequestCounter = c),
attributes: {} as {
model: string;
status_code?: number | string;
error_type?: string;
},
},
[TOKEN_USAGE]: {
description: 'Counts the total number of tokens used.',
valueType: ValueType.INT,
assign: (c: Counter) => (tokenUsageCounter = c),
attributes: {} as {
model: string;
type: 'input' | 'output' | 'thought' | 'cache' | 'tool';
},
},
[SESSION_COUNT]: {
description: 'Count of CLI sessions started.',
valueType: ValueType.INT,
assign: (c: Counter) => (sessionCounter = c),
attributes: {} as Record<string, never>,
},
[FILE_OPERATION_COUNT]: {
description: 'Counts file operations (create, read, update).',
valueType: ValueType.INT,
assign: (c: Counter) => (fileOperationCounter = c),
attributes: {} as {
operation: FileOperation;
lines?: number;
mimetype?: string;
extension?: string;
programming_language?: string;
},
},
[INVALID_CHUNK_COUNT]: {
description: 'Counts invalid chunks received from a stream.',
valueType: ValueType.INT,
assign: (c: Counter) => (invalidChunkCounter = c),
attributes: {} as Record<string, never>,
},
[CONTENT_RETRY_COUNT]: {
description: 'Counts retries due to content errors (e.g., empty stream).',
valueType: ValueType.INT,
assign: (c: Counter) => (contentRetryCounter = c),
attributes: {} as Record<string, never>,
},
[CONTENT_RETRY_FAILURE_COUNT]: {
description: 'Counts occurrences of all content retries failing.',
valueType: ValueType.INT,
assign: (c: Counter) => (contentRetryFailureCounter = c),
attributes: {} as Record<string, never>,
},
[MODEL_SLASH_COMMAND_CALL_COUNT]: {
description: 'Counts model slash command calls.',
valueType: ValueType.INT,
assign: (c: Counter) => (modelSlashCommandCallCounter = c),
attributes: {} as {
'slash_command.model.model_name': string;
},
},
[EVENT_CHAT_COMPRESSION]: {
description: 'Counts chat compression events.',
valueType: ValueType.INT,
assign: (c: Counter) => (chatCompressionCounter = c),
attributes: {} as {
tokens_before: number;
tokens_after: number;
},
},
} as const;
const HISTOGRAM_DEFINITIONS = {
[TOOL_CALL_LATENCY]: {
description: 'Latency of tool calls in milliseconds.',
unit: 'ms',
valueType: ValueType.INT,
assign: (h: Histogram) => (toolCallLatencyHistogram = h),
attributes: {} as {
function_name: string;
},
},
[API_REQUEST_LATENCY]: {
description: 'Latency of API requests in milliseconds.',
unit: 'ms',
valueType: ValueType.INT,
assign: (h: Histogram) => (apiRequestLatencyHistogram = h),
attributes: {} as {
model: string;
},
},
} as const;
const PERFORMANCE_COUNTER_DEFINITIONS = {
[REGRESSION_DETECTION]: {
description: 'Performance regression detection events.',
valueType: ValueType.INT,
assign: (c: Counter) => (regressionDetectionCounter = c),
attributes: {} as {
metric: string;
severity: 'low' | 'medium' | 'high';
current_value: number;
baseline_value: number;
},
},
} as const;
const PERFORMANCE_HISTOGRAM_DEFINITIONS = {
[STARTUP_TIME]: {
description:
'CLI startup time in milliseconds, broken down by initialization phase.',
unit: 'ms',
valueType: ValueType.DOUBLE,
assign: (h: Histogram) => (startupTimeHistogram = h),
attributes: {} as {
phase: string;
details?: Record<string, string | number | boolean>;
},
},
[MEMORY_USAGE]: {
description: 'Memory usage in bytes.',
unit: 'bytes',
valueType: ValueType.INT,
assign: (h: Histogram) => (memoryUsageGauge = h),
attributes: {} as {
memory_type: MemoryMetricType;
component?: string;
},
},
[CPU_USAGE]: {
description: 'CPU usage percentage.',
unit: 'percent',
valueType: ValueType.DOUBLE,
assign: (h: Histogram) => (cpuUsageGauge = h),
attributes: {} as {
component?: string;
},
},
[TOOL_QUEUE_DEPTH]: {
description: 'Number of tools in execution queue.',
unit: 'count',
valueType: ValueType.INT,
assign: (h: Histogram) => (toolQueueDepthGauge = h),
attributes: {} as Record<string, never>,
},
[TOOL_EXECUTION_BREAKDOWN]: {
description: 'Tool execution time breakdown by phase in milliseconds.',
unit: 'ms',
valueType: ValueType.INT,
assign: (h: Histogram) => (toolExecutionBreakdownHistogram = h),
attributes: {} as {
function_name: string;
phase: ToolExecutionPhase;
},
},
[TOKEN_EFFICIENCY]: {
description:
'Token efficiency metrics (tokens per operation, cache hit rate, etc.).',
unit: 'ratio',
valueType: ValueType.DOUBLE,
assign: (h: Histogram) => (tokenEfficiencyHistogram = h),
attributes: {} as {
model: string;
metric: string;
context?: string;
},
},
[API_REQUEST_BREAKDOWN]: {
description: 'API request time breakdown by phase in milliseconds.',
unit: 'ms',
valueType: ValueType.INT,
assign: (h: Histogram) => (apiRequestBreakdownHistogram = h),
attributes: {} as {
model: string;
phase: ApiRequestPhase;
},
},
[PERFORMANCE_SCORE]: {
description: 'Composite performance score (0-100).',
unit: 'score',
valueType: ValueType.DOUBLE,
assign: (h: Histogram) => (performanceScoreGauge = h),
attributes: {} as {
category: string;
baseline?: number;
},
},
[REGRESSION_PERCENTAGE_CHANGE]: {
description:
'Percentage change compared to baseline for detected regressions.',
unit: 'percent',
valueType: ValueType.DOUBLE,
assign: (h: Histogram) => (regressionPercentageChangeHistogram = h),
attributes: {} as {
metric: string;
severity: 'low' | 'medium' | 'high';
current_value: number;
baseline_value: number;
},
},
[BASELINE_COMPARISON]: {
description:
'Performance comparison to established baseline (percentage change).',
unit: 'percent',
valueType: ValueType.DOUBLE,
assign: (h: Histogram) => (baselineComparisonHistogram = h),
attributes: {} as {
metric: string;
category: string;
current_value: number;
baseline_value: number;
},
},
} as const;
type AllMetricDefs = typeof COUNTER_DEFINITIONS &
typeof HISTOGRAM_DEFINITIONS &
typeof PERFORMANCE_COUNTER_DEFINITIONS &
typeof PERFORMANCE_HISTOGRAM_DEFINITIONS;
export type MetricDefinitions = {
[K in keyof AllMetricDefs]: {
attributes: AllMetricDefs[K]['attributes'];
};
};
export enum FileOperation {
CREATE = 'create',
READ = 'read',
UPDATE = 'update',
}
export enum PerformanceMetricType {
STARTUP = 'startup',
MEMORY = 'memory',
CPU = 'cpu',
TOOL_EXECUTION = 'tool_execution',
API_REQUEST = 'api_request',
TOKEN_EFFICIENCY = 'token_efficiency',
}
export enum MemoryMetricType {
HEAP_USED = 'heap_used',
HEAP_TOTAL = 'heap_total',
EXTERNAL = 'external',
RSS = 'rss',
}
export enum ToolExecutionPhase {
VALIDATION = 'validation',
PREPARATION = 'preparation',
EXECUTION = 'execution',
RESULT_PROCESSING = 'result_processing',
}
export enum ApiRequestPhase {
REQUEST_PREPARATION = 'request_preparation',
NETWORK_LATENCY = 'network_latency',
RESPONSE_PROCESSING = 'response_processing',
TOKEN_PROCESSING = 'token_processing',
}
let cliMeter: Meter | undefined;
let toolCallCounter: Counter | undefined;
let toolCallLatencyHistogram: Histogram | undefined;
let apiRequestCounter: Counter | undefined;
let apiRequestLatencyHistogram: Histogram | undefined;
let tokenUsageCounter: Counter | undefined;
let sessionCounter: Counter | undefined;
let fileOperationCounter: Counter | undefined;
let chatCompressionCounter: Counter | undefined;
let invalidChunkCounter: Counter | undefined;
let contentRetryCounter: Counter | undefined;
let contentRetryFailureCounter: Counter | undefined;
let subagentExecutionCounter: Counter | undefined;
let modelSlashCommandCallCounter: Counter | undefined;
// Performance Monitoring Metrics
let startupTimeHistogram: Histogram | undefined;
let memoryUsageGauge: Histogram | undefined; // Using Histogram until ObservableGauge is available
let cpuUsageGauge: Histogram | undefined;
let toolQueueDepthGauge: Histogram | undefined;
let toolExecutionBreakdownHistogram: Histogram | undefined;
let tokenEfficiencyHistogram: Histogram | undefined;
let apiRequestBreakdownHistogram: Histogram | undefined;
let performanceScoreGauge: Histogram | undefined;
let regressionDetectionCounter: Counter | undefined;
let regressionPercentageChangeHistogram: Histogram | undefined;
let baselineComparisonHistogram: Histogram | undefined;
let isMetricsInitialized = false;
let isPerformanceMonitoringEnabled = false;
export function getMeter(): Meter | undefined {
if (!cliMeter) {
cliMeter = metrics.getMeter(SERVICE_NAME);
}
return cliMeter;
}
export function initializeMetrics(config: Config): void {
if (isMetricsInitialized) return;
const meter = getMeter();
if (!meter) return;
// Initialize core metrics
Object.entries(COUNTER_DEFINITIONS).forEach(
([name, { description, valueType, assign }]) => {
assign(meter.createCounter(name, { description, valueType }));
},
);
subagentExecutionCounter = meter.createCounter(SUBAGENT_EXECUTION_COUNT, {
description:
'Counts subagent execution events, tagged by status and subagent name.',
valueType: ValueType.INT,
});
Object.entries(HISTOGRAM_DEFINITIONS).forEach(
([name, { description, unit, valueType, assign }]) => {
assign(meter.createHistogram(name, { description, unit, valueType }));
},
);
// Increment session counter after all metrics are initialized
sessionCounter?.add(1, baseMetricDefinition.getCommonAttributes(config));
// Initialize performance monitoring metrics if enabled
initializePerformanceMonitoring(config);
isMetricsInitialized = true;
}
export function recordChatCompressionMetrics(
config: Config,
attributes: MetricDefinitions[typeof EVENT_CHAT_COMPRESSION]['attributes'],
) {
if (!chatCompressionCounter || !isMetricsInitialized) return;
chatCompressionCounter.add(1, {
...baseMetricDefinition.getCommonAttributes(config),
...attributes,
});
}
export function recordToolCallMetrics(
config: Config,
durationMs: number,
attributes: MetricDefinitions[typeof TOOL_CALL_COUNT]['attributes'],
): void {
if (!toolCallCounter || !toolCallLatencyHistogram || !isMetricsInitialized)
return;
const metricAttributes: Attributes = {
...baseMetricDefinition.getCommonAttributes(config),
...attributes,
};
toolCallCounter.add(1, metricAttributes);
toolCallLatencyHistogram.record(durationMs, {
...baseMetricDefinition.getCommonAttributes(config),
function_name: attributes.function_name,
});
}
export function recordTokenUsageMetrics(
config: Config,
tokenCount: number,
attributes: MetricDefinitions[typeof TOKEN_USAGE]['attributes'],
): void {
if (!tokenUsageCounter || !isMetricsInitialized) return;
tokenUsageCounter.add(tokenCount, {
...baseMetricDefinition.getCommonAttributes(config),
...attributes,
});
}
export function recordApiResponseMetrics(
config: Config,
durationMs: number,
attributes: MetricDefinitions[typeof API_REQUEST_COUNT]['attributes'],
): void {
if (
!apiRequestCounter ||
!apiRequestLatencyHistogram ||
!isMetricsInitialized
)
return;
const metricAttributes: Attributes = {
...baseMetricDefinition.getCommonAttributes(config),
model: attributes.model,
status_code: attributes.status_code ?? 'ok',
};
apiRequestCounter.add(1, metricAttributes);
apiRequestLatencyHistogram.record(durationMs, {
...baseMetricDefinition.getCommonAttributes(config),
model: attributes.model,
});
}
export function recordApiErrorMetrics(
config: Config,
durationMs: number,
attributes: MetricDefinitions[typeof API_REQUEST_COUNT]['attributes'],
): void {
if (
!apiRequestCounter ||
!apiRequestLatencyHistogram ||
!isMetricsInitialized
)
return;
const metricAttributes: Attributes = {
...baseMetricDefinition.getCommonAttributes(config),
model: attributes.model,
status_code: attributes.status_code ?? 'error',
error_type: attributes.error_type ?? 'unknown',
};
apiRequestCounter.add(1, metricAttributes);
apiRequestLatencyHistogram.record(durationMs, {
...baseMetricDefinition.getCommonAttributes(config),
model: attributes.model,
});
}
export function recordFileOperationMetric(
config: Config,
attributes: MetricDefinitions[typeof FILE_OPERATION_COUNT]['attributes'],
): void {
if (!fileOperationCounter || !isMetricsInitialized) return;
fileOperationCounter.add(1, {
...baseMetricDefinition.getCommonAttributes(config),
...attributes,
});
}
// --- New Metric Recording Functions ---
/**
* Records a metric for when an invalid chunk is received from a stream.
*/
export function recordInvalidChunk(config: Config): void {
if (!invalidChunkCounter || !isMetricsInitialized) return;
invalidChunkCounter.add(1, baseMetricDefinition.getCommonAttributes(config));
}
/**
* Records a metric for when a retry is triggered due to a content error.
*/
export function recordContentRetry(config: Config): void {
if (!contentRetryCounter || !isMetricsInitialized) return;
contentRetryCounter.add(1, baseMetricDefinition.getCommonAttributes(config));
}
/**
* Records a metric for when all content error retries have failed for a request.
*/
export function recordContentRetryFailure(config: Config): void {
if (!contentRetryFailureCounter || !isMetricsInitialized) return;
contentRetryFailureCounter.add(
1,
baseMetricDefinition.getCommonAttributes(config),
);
}
export function recordModelSlashCommand(
config: Config,
event: ModelSlashCommandEvent,
): void {
if (!modelSlashCommandCallCounter || !isMetricsInitialized) return;
modelSlashCommandCallCounter.add(1, {
...baseMetricDefinition.getCommonAttributes(config),
'slash_command.model.model_name': event.model_name,
});
}
// Performance Monitoring Functions
export function initializePerformanceMonitoring(config: Config): void {
const meter = getMeter();
if (!meter) return;
// Check if performance monitoring is enabled in config
// For now, enable performance monitoring when telemetry is enabled
// TODO: Add specific performance monitoring settings to config
isPerformanceMonitoringEnabled = config.getTelemetryEnabled();
if (!isPerformanceMonitoringEnabled) return;
Object.entries(PERFORMANCE_COUNTER_DEFINITIONS).forEach(
([name, { description, valueType, assign }]) => {
assign(meter.createCounter(name, { description, valueType }));
},
);
Object.entries(PERFORMANCE_HISTOGRAM_DEFINITIONS).forEach(
([name, { description, unit, valueType, assign }]) => {
assign(meter.createHistogram(name, { description, unit, valueType }));
},
);
}
export function recordStartupPerformance(
config: Config,
durationMs: number,
attributes: MetricDefinitions[typeof STARTUP_TIME]['attributes'],
): void {
if (!startupTimeHistogram || !isPerformanceMonitoringEnabled) return;
const metricAttributes: Attributes = {
...baseMetricDefinition.getCommonAttributes(config),
phase: attributes.phase,
...attributes.details,
};
startupTimeHistogram.record(durationMs, metricAttributes);
}
export function recordMemoryUsage(
config: Config,
bytes: number,
attributes: MetricDefinitions[typeof MEMORY_USAGE]['attributes'],
): void {
if (!memoryUsageGauge || !isPerformanceMonitoringEnabled) return;
const metricAttributes: Attributes = {
...baseMetricDefinition.getCommonAttributes(config),
...attributes,
};
memoryUsageGauge.record(bytes, metricAttributes);
}
export function recordCpuUsage(
config: Config,
percentage: number,
attributes: MetricDefinitions[typeof CPU_USAGE]['attributes'],
): void {
if (!cpuUsageGauge || !isPerformanceMonitoringEnabled) return;
const metricAttributes: Attributes = {
...baseMetricDefinition.getCommonAttributes(config),
...attributes,
};
cpuUsageGauge.record(percentage, metricAttributes);
}
export function recordToolQueueDepth(config: Config, queueDepth: number): void {
if (!toolQueueDepthGauge || !isPerformanceMonitoringEnabled) return;
const attributes: Attributes = {
...baseMetricDefinition.getCommonAttributes(config),
};
toolQueueDepthGauge.record(queueDepth, attributes);
}
export function recordToolExecutionBreakdown(
config: Config,
durationMs: number,
attributes: MetricDefinitions[typeof TOOL_EXECUTION_BREAKDOWN]['attributes'],
): void {
if (!toolExecutionBreakdownHistogram || !isPerformanceMonitoringEnabled)
return;
const metricAttributes: Attributes = {
...baseMetricDefinition.getCommonAttributes(config),
...attributes,
};
toolExecutionBreakdownHistogram.record(durationMs, metricAttributes);
}
export function recordTokenEfficiency(
config: Config,
value: number,
attributes: MetricDefinitions[typeof TOKEN_EFFICIENCY]['attributes'],
): void {
if (!tokenEfficiencyHistogram || !isPerformanceMonitoringEnabled) return;
const metricAttributes: Attributes = {
...baseMetricDefinition.getCommonAttributes(config),
...attributes,
};
tokenEfficiencyHistogram.record(value, metricAttributes);
}
export function recordApiRequestBreakdown(
config: Config,
durationMs: number,
attributes: MetricDefinitions[typeof API_REQUEST_BREAKDOWN]['attributes'],
): void {
if (!apiRequestBreakdownHistogram || !isPerformanceMonitoringEnabled) return;
const metricAttributes: Attributes = {
...baseMetricDefinition.getCommonAttributes(config),
...attributes,
};
apiRequestBreakdownHistogram.record(durationMs, metricAttributes);
}
export function recordPerformanceScore(
config: Config,
score: number,
attributes: MetricDefinitions[typeof PERFORMANCE_SCORE]['attributes'],
): void {
if (!performanceScoreGauge || !isPerformanceMonitoringEnabled) return;
const metricAttributes: Attributes = {
...baseMetricDefinition.getCommonAttributes(config),
...attributes,
};
performanceScoreGauge.record(score, metricAttributes);
}
export function recordPerformanceRegression(
config: Config,
attributes: MetricDefinitions[typeof REGRESSION_DETECTION]['attributes'],
): void {
if (!regressionDetectionCounter || !isPerformanceMonitoringEnabled) return;
const metricAttributes: Attributes = {
...baseMetricDefinition.getCommonAttributes(config),
...attributes,
};
regressionDetectionCounter.add(1, metricAttributes);
if (attributes.baseline_value !== 0 && regressionPercentageChangeHistogram) {
const percentageChange =
((attributes.current_value - attributes.baseline_value) /
attributes.baseline_value) *
100;
regressionPercentageChangeHistogram.record(
percentageChange,
metricAttributes,
);
}
}
export function recordBaselineComparison(
config: Config,
attributes: MetricDefinitions[typeof BASELINE_COMPARISON]['attributes'],
): void {
if (!baselineComparisonHistogram || !isPerformanceMonitoringEnabled) return;
if (attributes.baseline_value === 0) {
diag.warn('Baseline value is zero, skipping comparison.');
return;
}
const percentageChange =
((attributes.current_value - attributes.baseline_value) /
attributes.baseline_value) *
100;
const metricAttributes: Attributes = {
...baseMetricDefinition.getCommonAttributes(config),
...attributes,
};
baselineComparisonHistogram.record(percentageChange, metricAttributes);
}
// Utility function to check if performance monitoring is enabled
export function isPerformanceMonitoringActive(): boolean {
return isPerformanceMonitoringEnabled && isMetricsInitialized;
}
/**
* Records a metric for subagent execution events.
*/
export function recordSubagentExecutionMetrics(
config: Config,
subagentName: string,
status: 'started' | 'completed' | 'failed' | 'cancelled',
terminateReason?: string,
): void {
if (!subagentExecutionCounter || !isMetricsInitialized) return;
const attributes: Attributes = {
...baseMetricDefinition.getCommonAttributes(config),
subagent_name: subagentName,
status,
};
if (terminateReason) {
attributes['terminate_reason'] = terminateReason;
}
subagentExecutionCounter.add(1, attributes);
}