test(integration): add failing test for stdin context with prompt (#6158)

This commit is contained in:
Allen Hutchison
2025-08-20 16:52:27 -07:00
committed by GitHub
parent 653267a64f
commit a590a033be
4 changed files with 202 additions and 27 deletions

View File

@@ -93,7 +93,9 @@ export function validateModelOutput(
if (missingContent.length > 0) {
console.warn(
`Warning: LLM did not include expected content in response: ${missingContent.join(', ')}.`,
`Warning: LLM did not include expected content in response: ${missingContent.join(
', ',
)}.`,
'This is not ideal but not a test failure.',
);
console.warn(
@@ -141,10 +143,7 @@ export class TestRig {
mkdirSync(geminiDir, { recursive: true });
// In sandbox mode, use an absolute path for telemetry inside the container
// The container mounts the test directory at the same path as the host
const telemetryPath =
env.GEMINI_SANDBOX && env.GEMINI_SANDBOX !== 'false'
? join(this.testDir, 'telemetry.log') // Absolute path in test directory
: env.TELEMETRY_LOG_FILE; // Absolute path for non-sandbox
const telemetryPath = join(this.testDir, 'telemetry.log'); // Always use test directory for telemetry
const settings = {
telemetry: {
@@ -322,11 +321,8 @@ export class TestRig {
}
async waitForTelemetryReady() {
// In sandbox mode, telemetry is written to a relative path in the test directory
const logFilePath =
env.GEMINI_SANDBOX && env.GEMINI_SANDBOX !== 'false'
? join(this.testDir!, 'telemetry.log')
: env.TELEMETRY_LOG_FILE;
// Telemetry is always written to the test directory
const logFilePath = join(this.testDir!, 'telemetry.log');
if (!logFilePath) return;
@@ -347,6 +343,52 @@ export class TestRig {
);
}
async waitForTelemetryEvent(eventName: string, timeout?: number) {
if (!timeout) {
timeout = this.getDefaultTimeout();
}
await this.waitForTelemetryReady();
return this.poll(
() => {
const logFilePath = join(this.testDir!, 'telemetry.log');
if (!logFilePath || !fs.existsSync(logFilePath)) {
return false;
}
const content = readFileSync(logFilePath, 'utf-8');
const jsonObjects = content
.split(/}\n{/)
.map((obj, index, array) => {
// Add back the braces we removed during split
if (index > 0) obj = '{' + obj;
if (index < array.length - 1) obj = obj + '}';
return obj.trim();
})
.filter((obj) => obj);
for (const jsonStr of jsonObjects) {
try {
const logData = JSON.parse(jsonStr);
if (
logData.attributes &&
logData.attributes['event.name'] === `gemini_cli.${eventName}`
) {
return true;
}
} catch {
// ignore
}
}
return false;
},
timeout,
100,
);
}
async waitForToolCall(toolName: string, timeout?: number) {
// Use environment-specific timeout
if (!timeout) {
@@ -566,11 +608,8 @@ export class TestRig {
}
}
// In sandbox mode, telemetry is written to a relative path in the test directory
const logFilePath =
env.GEMINI_SANDBOX && env.GEMINI_SANDBOX !== 'false'
? join(this.testDir!, 'telemetry.log')
: env.TELEMETRY_LOG_FILE;
// Telemetry is always written to the test directory
const logFilePath = join(this.testDir!, 'telemetry.log');
if (!logFilePath) {
console.warn(`TELEMETRY_LOG_FILE environment variable not set`);
@@ -587,7 +626,7 @@ export class TestRig {
// Split the content into individual JSON objects
// They are separated by "}\n{"
const jsonObjects = content
.split(/}\s*\n\s*{/)
.split(/}\n{/)
.map((obj, index, array) => {
// Add back the braces we removed during split
if (index > 0) obj = '{' + obj;
@@ -626,14 +665,47 @@ export class TestRig {
} catch (e) {
// Skip objects that aren't valid JSON
if (env.VERBOSE === 'true') {
console.error(
'Failed to parse telemetry object:',
(e as Error).message,
);
console.error('Failed to parse telemetry object:', e);
}
}
}
return logs;
}
readLastApiRequest(): Record<string, unknown> | null {
// Telemetry is always written to the test directory
const logFilePath = join(this.testDir!, 'telemetry.log');
if (!logFilePath || !fs.existsSync(logFilePath)) {
return null;
}
const content = readFileSync(logFilePath, 'utf-8');
const jsonObjects = content
.split(/}\n{/)
.map((obj, index, array) => {
if (index > 0) obj = '{' + obj;
if (index < array.length - 1) obj = obj + '}';
return obj.trim();
})
.filter((obj) => obj);
let lastApiRequest = null;
for (const jsonStr of jsonObjects) {
try {
const logData = JSON.parse(jsonStr);
if (
logData.attributes &&
logData.attributes['event.name'] === 'gemini_cli.api_request'
) {
lastApiRequest = logData;
}
} catch {
// ignore
}
}
return lastApiRequest;
}
}