Log prompt id when a loop is detected (#4765)

Co-authored-by: N. Taylor Mullen <ntaylormullen@google.com>
This commit is contained in:
Sandy Tao
2025-07-23 22:37:28 -07:00
committed by GitHub
parent 6380bfe35c
commit 0ef9c0b792
5 changed files with 23 additions and 13 deletions

View File

@@ -293,7 +293,7 @@ export class GeminiClient {
originalModel?: string, originalModel?: string,
): AsyncGenerator<ServerGeminiStreamEvent, Turn> { ): AsyncGenerator<ServerGeminiStreamEvent, Turn> {
if (this.lastPromptId !== prompt_id) { if (this.lastPromptId !== prompt_id) {
this.loopDetector.reset(); this.loopDetector.reset(prompt_id);
this.lastPromptId = prompt_id; this.lastPromptId = prompt_id;
} }
this.sessionTurnCount++; this.sessionTurnCount++;

View File

@@ -168,21 +168,21 @@ describe('LoopDetectionService', () => {
); );
} }
service.reset(); service.reset('');
for (let i = 0; i < CONTENT_LOOP_THRESHOLD + 2; i++) { for (let i = 0; i < CONTENT_LOOP_THRESHOLD + 2; i++) {
expect(service.addAndCheck(createContentEvent('obj.method()'))).toBe( expect(service.addAndCheck(createContentEvent('obj.method()'))).toBe(
false, false,
); );
} }
service.reset(); service.reset('');
for (let i = 0; i < CONTENT_LOOP_THRESHOLD + 2; i++) { for (let i = 0; i < CONTENT_LOOP_THRESHOLD + 2; i++) {
expect( expect(
service.addAndCheck(createContentEvent('arr.filter().map()')), service.addAndCheck(createContentEvent('arr.filter().map()')),
).toBe(false); ).toBe(false);
} }
service.reset(); service.reset('');
for (let i = 0; i < CONTENT_LOOP_THRESHOLD + 2; i++) { for (let i = 0; i < CONTENT_LOOP_THRESHOLD + 2; i++) {
expect( expect(
service.addAndCheck( service.addAndCheck(
@@ -203,7 +203,7 @@ describe('LoopDetectionService', () => {
service.addAndCheck(createContentEvent('This is a sentence.')), service.addAndCheck(createContentEvent('This is a sentence.')),
).toBe(true); ).toBe(true);
service.reset(); service.reset('');
for (let i = 0; i < CONTENT_LOOP_THRESHOLD - 1; i++) { for (let i = 0; i < CONTENT_LOOP_THRESHOLD - 1; i++) {
expect( expect(
service.addAndCheck(createContentEvent('Is this a question? ')), service.addAndCheck(createContentEvent('Is this a question? ')),
@@ -213,7 +213,7 @@ describe('LoopDetectionService', () => {
service.addAndCheck(createContentEvent('Is this a question? ')), service.addAndCheck(createContentEvent('Is this a question? ')),
).toBe(true); ).toBe(true);
service.reset(); service.reset('');
for (let i = 0; i < CONTENT_LOOP_THRESHOLD - 1; i++) { for (let i = 0; i < CONTENT_LOOP_THRESHOLD - 1; i++) {
expect( expect(
service.addAndCheck(createContentEvent('What excitement!\n')), service.addAndCheck(createContentEvent('What excitement!\n')),

View File

@@ -50,6 +50,7 @@ const SENTENCE_ENDING_PUNCTUATION_REGEX = /[.!?]+(?=\s|$)/;
*/ */
export class LoopDetectionService { export class LoopDetectionService {
private readonly config: Config; private readonly config: Config;
private promptId = '';
// Tool call tracking // Tool call tracking
private lastToolCallKey: string | null = null; private lastToolCallKey: string | null = null;
@@ -129,7 +130,10 @@ export class LoopDetectionService {
if (this.toolCallRepetitionCount >= TOOL_CALL_LOOP_THRESHOLD) { if (this.toolCallRepetitionCount >= TOOL_CALL_LOOP_THRESHOLD) {
logLoopDetected( logLoopDetected(
this.config, this.config,
new LoopDetectedEvent(LoopType.CONSECUTIVE_IDENTICAL_TOOL_CALLS), new LoopDetectedEvent(
LoopType.CONSECUTIVE_IDENTICAL_TOOL_CALLS,
this.promptId,
),
); );
return true; return true;
} }
@@ -170,7 +174,10 @@ export class LoopDetectionService {
if (this.sentenceRepetitionCount >= CONTENT_LOOP_THRESHOLD) { if (this.sentenceRepetitionCount >= CONTENT_LOOP_THRESHOLD) {
logLoopDetected( logLoopDetected(
this.config, this.config,
new LoopDetectedEvent(LoopType.CHANTING_IDENTICAL_SENTENCES), new LoopDetectedEvent(
LoopType.CHANTING_IDENTICAL_SENTENCES,
this.promptId,
),
); );
return true; return true;
} }
@@ -234,7 +241,7 @@ Please analyze the conversation history to determine the possibility that the co
} }
logLoopDetected( logLoopDetected(
this.config, this.config,
new LoopDetectedEvent(LoopType.LLM_DETECTED_LOOP), new LoopDetectedEvent(LoopType.LLM_DETECTED_LOOP, this.promptId),
); );
return true; return true;
} else { } else {
@@ -251,7 +258,8 @@ Please analyze the conversation history to determine the possibility that the co
/** /**
* Resets all loop detection state. * Resets all loop detection state.
*/ */
reset(): void { reset(promptId: string): void {
this.promptId = promptId;
this.resetToolCallCount(); this.resetToolCallCount();
this.resetSentenceCount(); this.resetSentenceCount();
this.resetLlmCheckTracking(); this.resetLlmCheckTracking();

View File

@@ -481,8 +481,8 @@ export class ClearcutLogger {
logLoopDetectedEvent(event: LoopDetectedEvent): void { logLoopDetectedEvent(event: LoopDetectedEvent): void {
const data = [ const data = [
{ {
gemini_cli_key: EventMetadataKey.GEMINI_CLI_SESSION_ID, gemini_cli_key: EventMetadataKey.GEMINI_CLI_PROMPT_ID,
value: this.config?.getSessionId() ?? '', value: JSON.stringify(event.prompt_id),
}, },
{ {
gemini_cli_key: EventMetadataKey.GEMINI_CLI_LOOP_DETECTED_TYPE, gemini_cli_key: EventMetadataKey.GEMINI_CLI_LOOP_DETECTED_TYPE,

View File

@@ -256,11 +256,13 @@ export class LoopDetectedEvent {
'event.name': 'loop_detected'; 'event.name': 'loop_detected';
'event.timestamp': string; // ISO 8601 'event.timestamp': string; // ISO 8601
loop_type: LoopType; loop_type: LoopType;
prompt_id: string;
constructor(loop_type: LoopType) { constructor(loop_type: LoopType, prompt_id: string) {
this['event.name'] = 'loop_detected'; this['event.name'] = 'loop_detected';
this['event.timestamp'] = new Date().toISOString(); this['event.timestamp'] = new Date().toISOString();
this.loop_type = loop_type; this.loop_type = loop_type;
this.prompt_id = prompt_id;
} }
} }