mirror of
https://github.com/QwenLM/qwen-code.git
synced 2025-12-19 09:33:53 +00:00
Log prompt id when a loop is detected (#4765)
Co-authored-by: N. Taylor Mullen <ntaylormullen@google.com>
This commit is contained in:
@@ -293,7 +293,7 @@ export class GeminiClient {
|
|||||||
originalModel?: string,
|
originalModel?: string,
|
||||||
): AsyncGenerator<ServerGeminiStreamEvent, Turn> {
|
): AsyncGenerator<ServerGeminiStreamEvent, Turn> {
|
||||||
if (this.lastPromptId !== prompt_id) {
|
if (this.lastPromptId !== prompt_id) {
|
||||||
this.loopDetector.reset();
|
this.loopDetector.reset(prompt_id);
|
||||||
this.lastPromptId = prompt_id;
|
this.lastPromptId = prompt_id;
|
||||||
}
|
}
|
||||||
this.sessionTurnCount++;
|
this.sessionTurnCount++;
|
||||||
|
|||||||
@@ -168,21 +168,21 @@ describe('LoopDetectionService', () => {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
service.reset();
|
service.reset('');
|
||||||
for (let i = 0; i < CONTENT_LOOP_THRESHOLD + 2; i++) {
|
for (let i = 0; i < CONTENT_LOOP_THRESHOLD + 2; i++) {
|
||||||
expect(service.addAndCheck(createContentEvent('obj.method()'))).toBe(
|
expect(service.addAndCheck(createContentEvent('obj.method()'))).toBe(
|
||||||
false,
|
false,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
service.reset();
|
service.reset('');
|
||||||
for (let i = 0; i < CONTENT_LOOP_THRESHOLD + 2; i++) {
|
for (let i = 0; i < CONTENT_LOOP_THRESHOLD + 2; i++) {
|
||||||
expect(
|
expect(
|
||||||
service.addAndCheck(createContentEvent('arr.filter().map()')),
|
service.addAndCheck(createContentEvent('arr.filter().map()')),
|
||||||
).toBe(false);
|
).toBe(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
service.reset();
|
service.reset('');
|
||||||
for (let i = 0; i < CONTENT_LOOP_THRESHOLD + 2; i++) {
|
for (let i = 0; i < CONTENT_LOOP_THRESHOLD + 2; i++) {
|
||||||
expect(
|
expect(
|
||||||
service.addAndCheck(
|
service.addAndCheck(
|
||||||
@@ -203,7 +203,7 @@ describe('LoopDetectionService', () => {
|
|||||||
service.addAndCheck(createContentEvent('This is a sentence.')),
|
service.addAndCheck(createContentEvent('This is a sentence.')),
|
||||||
).toBe(true);
|
).toBe(true);
|
||||||
|
|
||||||
service.reset();
|
service.reset('');
|
||||||
for (let i = 0; i < CONTENT_LOOP_THRESHOLD - 1; i++) {
|
for (let i = 0; i < CONTENT_LOOP_THRESHOLD - 1; i++) {
|
||||||
expect(
|
expect(
|
||||||
service.addAndCheck(createContentEvent('Is this a question? ')),
|
service.addAndCheck(createContentEvent('Is this a question? ')),
|
||||||
@@ -213,7 +213,7 @@ describe('LoopDetectionService', () => {
|
|||||||
service.addAndCheck(createContentEvent('Is this a question? ')),
|
service.addAndCheck(createContentEvent('Is this a question? ')),
|
||||||
).toBe(true);
|
).toBe(true);
|
||||||
|
|
||||||
service.reset();
|
service.reset('');
|
||||||
for (let i = 0; i < CONTENT_LOOP_THRESHOLD - 1; i++) {
|
for (let i = 0; i < CONTENT_LOOP_THRESHOLD - 1; i++) {
|
||||||
expect(
|
expect(
|
||||||
service.addAndCheck(createContentEvent('What excitement!\n')),
|
service.addAndCheck(createContentEvent('What excitement!\n')),
|
||||||
|
|||||||
@@ -50,6 +50,7 @@ const SENTENCE_ENDING_PUNCTUATION_REGEX = /[.!?]+(?=\s|$)/;
|
|||||||
*/
|
*/
|
||||||
export class LoopDetectionService {
|
export class LoopDetectionService {
|
||||||
private readonly config: Config;
|
private readonly config: Config;
|
||||||
|
private promptId = '';
|
||||||
|
|
||||||
// Tool call tracking
|
// Tool call tracking
|
||||||
private lastToolCallKey: string | null = null;
|
private lastToolCallKey: string | null = null;
|
||||||
@@ -129,7 +130,10 @@ export class LoopDetectionService {
|
|||||||
if (this.toolCallRepetitionCount >= TOOL_CALL_LOOP_THRESHOLD) {
|
if (this.toolCallRepetitionCount >= TOOL_CALL_LOOP_THRESHOLD) {
|
||||||
logLoopDetected(
|
logLoopDetected(
|
||||||
this.config,
|
this.config,
|
||||||
new LoopDetectedEvent(LoopType.CONSECUTIVE_IDENTICAL_TOOL_CALLS),
|
new LoopDetectedEvent(
|
||||||
|
LoopType.CONSECUTIVE_IDENTICAL_TOOL_CALLS,
|
||||||
|
this.promptId,
|
||||||
|
),
|
||||||
);
|
);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -170,7 +174,10 @@ export class LoopDetectionService {
|
|||||||
if (this.sentenceRepetitionCount >= CONTENT_LOOP_THRESHOLD) {
|
if (this.sentenceRepetitionCount >= CONTENT_LOOP_THRESHOLD) {
|
||||||
logLoopDetected(
|
logLoopDetected(
|
||||||
this.config,
|
this.config,
|
||||||
new LoopDetectedEvent(LoopType.CHANTING_IDENTICAL_SENTENCES),
|
new LoopDetectedEvent(
|
||||||
|
LoopType.CHANTING_IDENTICAL_SENTENCES,
|
||||||
|
this.promptId,
|
||||||
|
),
|
||||||
);
|
);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -234,7 +241,7 @@ Please analyze the conversation history to determine the possibility that the co
|
|||||||
}
|
}
|
||||||
logLoopDetected(
|
logLoopDetected(
|
||||||
this.config,
|
this.config,
|
||||||
new LoopDetectedEvent(LoopType.LLM_DETECTED_LOOP),
|
new LoopDetectedEvent(LoopType.LLM_DETECTED_LOOP, this.promptId),
|
||||||
);
|
);
|
||||||
return true;
|
return true;
|
||||||
} else {
|
} else {
|
||||||
@@ -251,7 +258,8 @@ Please analyze the conversation history to determine the possibility that the co
|
|||||||
/**
|
/**
|
||||||
* Resets all loop detection state.
|
* Resets all loop detection state.
|
||||||
*/
|
*/
|
||||||
reset(): void {
|
reset(promptId: string): void {
|
||||||
|
this.promptId = promptId;
|
||||||
this.resetToolCallCount();
|
this.resetToolCallCount();
|
||||||
this.resetSentenceCount();
|
this.resetSentenceCount();
|
||||||
this.resetLlmCheckTracking();
|
this.resetLlmCheckTracking();
|
||||||
|
|||||||
@@ -481,8 +481,8 @@ export class ClearcutLogger {
|
|||||||
logLoopDetectedEvent(event: LoopDetectedEvent): void {
|
logLoopDetectedEvent(event: LoopDetectedEvent): void {
|
||||||
const data = [
|
const data = [
|
||||||
{
|
{
|
||||||
gemini_cli_key: EventMetadataKey.GEMINI_CLI_SESSION_ID,
|
gemini_cli_key: EventMetadataKey.GEMINI_CLI_PROMPT_ID,
|
||||||
value: this.config?.getSessionId() ?? '',
|
value: JSON.stringify(event.prompt_id),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
gemini_cli_key: EventMetadataKey.GEMINI_CLI_LOOP_DETECTED_TYPE,
|
gemini_cli_key: EventMetadataKey.GEMINI_CLI_LOOP_DETECTED_TYPE,
|
||||||
|
|||||||
@@ -256,11 +256,13 @@ export class LoopDetectedEvent {
|
|||||||
'event.name': 'loop_detected';
|
'event.name': 'loop_detected';
|
||||||
'event.timestamp': string; // ISO 8601
|
'event.timestamp': string; // ISO 8601
|
||||||
loop_type: LoopType;
|
loop_type: LoopType;
|
||||||
|
prompt_id: string;
|
||||||
|
|
||||||
constructor(loop_type: LoopType) {
|
constructor(loop_type: LoopType, prompt_id: string) {
|
||||||
this['event.name'] = 'loop_detected';
|
this['event.name'] = 'loop_detected';
|
||||||
this['event.timestamp'] = new Date().toISOString();
|
this['event.timestamp'] = new Date().toISOString();
|
||||||
this.loop_type = loop_type;
|
this.loop_type = loop_type;
|
||||||
|
this.prompt_id = prompt_id;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user