abbyswag/qwen-code
1
0
Fork 0
mirror of https://github.com/QwenLM/qwen-code.git synced 2025-12-20 08:47:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

telemetry: include user decisions in tool call logs (#966)

Browse Source 
Add the user's decision (accept, reject, modify) to tool call telemetry to better understand user intent. The decision provides crucial context to the `success` metric, as a user can reject a call that would have succeeded or accept one that fails. 

Also prettify the arguments json.

Example: 
![image](https://github.com/user-attachments/assets/251cb9fc-ceaa-4cdd-929c-8de47031aca8)

#750
This commit is contained in:
Jerop Kipruto
2025-06-12 16:48:10 -04:00
committed by GitHub
parent f8863f4d00
commit 6723c72fa5
8 changed files with 339 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

238
packages/core/src/telemetry/loggers.test.ts
View File

@@ -4,6 +4,7 @@
* SPDX-License-Identifier: Apache-2.0
*/
import { ToolConfirmationOutcome } from '../index.js';
import { logs } from '@opentelemetry/api-logs';
import { SemanticAttributes } from '@opentelemetry/semantic-conventions';
import { Config } from '../config/config.js';
@@ -12,6 +13,8 @@ import {
logApiResponse,
logCliConfiguration,
logUserPrompt,
logToolCall,
ToolCallDecision,
} from './loggers.js';
import * as metrics from './metrics.js';
import * as sdk from './sdk.js';
@@ -236,4 +239,239 @@ describe('loggers', () => {
});
});
});
describe('logToolCall', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
} as Config;
const mockMetrics = {
recordToolCallMetrics: vi.fn(),
};
beforeEach(() => {
vi.spyOn(metrics, 'recordToolCallMetrics').mockImplementation(
mockMetrics.recordToolCallMetrics,
);
mockLogger.emit.mockReset();
});
it('should log a tool call with all fields', () => {
const event = {
function_name: 'test-function',
function_args: {
arg1: 'value1',
arg2: 2,
},
duration_ms: 100,
success: true,
};
logToolCall(mockConfig, event, ToolConfirmationOutcome.ProceedOnce);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'Tool call: test-function. Decision: accept. Success: true. Duration: 100ms.',
attributes: {
'session.id': 'test-session-id',
'event.name': 'gemini_cli.tool_call',
'event.timestamp': '2025-01-01T00:00:00.000Z',
function_name: 'test-function',
function_args: JSON.stringify(
{
arg1: 'value1',
arg2: 2,
},
null,
2,
),
duration_ms: 100,
success: true,
decision: ToolCallDecision.ACCEPT,
},
});
expect(mockMetrics.recordToolCallMetrics).toHaveBeenCalledWith(
mockConfig,
'test-function',
100,
true,
ToolCallDecision.ACCEPT,
);
});
it('should log a tool call with a reject decision', () => {
const event = {
function_name: 'test-function',
function_args: {
arg1: 'value1',
arg2: 2,
},
duration_ms: 100,
success: false,
};
logToolCall(mockConfig, event, ToolConfirmationOutcome.Cancel);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'Tool call: test-function. Decision: reject. Success: false. Duration: 100ms.',
attributes: {
'session.id': 'test-session-id',
'event.name': 'gemini_cli.tool_call',
'event.timestamp': '2025-01-01T00:00:00.000Z',
function_name: 'test-function',
function_args: JSON.stringify(
{
arg1: 'value1',
arg2: 2,
},
null,
2,
),
duration_ms: 100,
success: false,
decision: ToolCallDecision.REJECT,
},
});
expect(mockMetrics.recordToolCallMetrics).toHaveBeenCalledWith(
mockConfig,
'test-function',
100,
false,
ToolCallDecision.REJECT,
);
});
it('should log a tool call with a modify decision', () => {
const event = {
function_name: 'test-function',
function_args: {
arg1: 'value1',
arg2: 2,
},
duration_ms: 100,
success: true,
};
logToolCall(mockConfig, event, ToolConfirmationOutcome.ModifyWithEditor);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'Tool call: test-function. Decision: modify. Success: true. Duration: 100ms.',
attributes: {
'session.id': 'test-session-id',
'event.name': 'gemini_cli.tool_call',
'event.timestamp': '2025-01-01T00:00:00.000Z',
function_name: 'test-function',
function_args: JSON.stringify(
{
arg1: 'value1',
arg2: 2,
},
null,
2,
),
duration_ms: 100,
success: true,
decision: ToolCallDecision.MODIFY,
},
});
expect(mockMetrics.recordToolCallMetrics).toHaveBeenCalledWith(
mockConfig,
'test-function',
100,
true,
ToolCallDecision.MODIFY,
);
});
it('should log a tool call without a decision', () => {
const event = {
function_name: 'test-function',
function_args: {
arg1: 'value1',
arg2: 2,
},
duration_ms: 100,
success: true,
};
logToolCall(mockConfig, event);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'Tool call: test-function. Success: true. Duration: 100ms.',
attributes: {
'session.id': 'test-session-id',
'event.name': 'gemini_cli.tool_call',
'event.timestamp': '2025-01-01T00:00:00.000Z',
function_name: 'test-function',
function_args: JSON.stringify(
{
arg1: 'value1',
arg2: 2,
},
null,
2,
),
duration_ms: 100,
success: true,
},
});
expect(mockMetrics.recordToolCallMetrics).toHaveBeenCalledWith(
mockConfig,
'test-function',
100,
true,
undefined,
);
});
it('should log a failed tool call with an error', () => {
const event = {
function_name: 'test-function',
function_args: {
arg1: 'value1',
arg2: 2,
},
duration_ms: 100,
success: false,
error: 'test-error',
error_type: 'test-error-type',
};
logToolCall(mockConfig, event);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'Tool call: test-function. Success: false. Duration: 100ms.',
attributes: {
'session.id': 'test-session-id',
'event.name': 'gemini_cli.tool_call',
'event.timestamp': '2025-01-01T00:00:00.000Z',
function_name: 'test-function',
function_args: JSON.stringify(
{
arg1: 'value1',
arg2: 2,
},
null,
2,
),
duration_ms: 100,
success: false,
error: 'test-error',
'error.message': 'test-error',
error_type: 'test-error-type',
'error.type': 'test-error-type',
},
});
expect(mockMetrics.recordToolCallMetrics).toHaveBeenCalledWith(
mockConfig,
'test-function',
100,
false,
undefined,
);
});
});
});