abbyswag/qwen-code
1
0
Fork 0
mirror of https://github.com/QwenLM/qwen-code.git synced 2025-12-19 09:33:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Sync upstream Gemini-CLI v0.8.2 (#838)

Browse Source
This commit is contained in:
tanzhenxin
2025-10-23 09:27:04 +08:00
committed by GitHub
parent 096fabb5d6
commit eb95c131be
644 changed files with 70389 additions and 23709 deletions
Download Patch File Download Diff File Expand all files Collapse all files

116
integration-tests/context-compress-interactive.test.ts Normal file
View File

@@ -0,0 +1,116 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { expect, describe, it, beforeEach, afterEach } from 'vitest';
import { TestRig, type } from './test-helper.js';
describe('Interactive Mode', () => {
let rig: TestRig;
beforeEach(() => {
rig = new TestRig();
});
afterEach(async () => {
await rig.cleanup();
});
it.skipIf(process.platform === 'win32')(
'should trigger chat compression with /compress command',
async () => {
await rig.setup('interactive-compress-test');
const { ptyProcess } = rig.runInteractive();
let fullOutput = '';
ptyProcess.onData((data) => (fullOutput += data));
const authDialogAppeared = await rig.waitForText(
'How would you like to authenticate',
5000,
);
// select the second option if auth dialog come's up
if (authDialogAppeared) {
ptyProcess.write('2');
}
// Wait for the app to be ready
const isReady = await rig.waitForText('Type your message', 15000);
expect(
isReady,
'CLI did not start up in interactive mode correctly',
).toBe(true);
const longPrompt =
'Dont do anything except returning a 1000 token long paragragh with the <name of the scientist who discovered theory of relativity> at the end to indicate end of response. This is a moderately long sentence.';
await type(ptyProcess, longPrompt);
await type(ptyProcess, '\r');
await rig.waitForText('einstein', 25000);
await type(ptyProcess, '/compress');
// A small delay to allow React to re-render the command list.
await new Promise((resolve) => setTimeout(resolve, 100));
await type(ptyProcess, '\r');
const foundEvent = await rig.waitForTelemetryEvent(
'chat_compression',
90000,
);
expect(foundEvent, 'chat_compression telemetry event was not found').toBe(
true,
);
},
);
it.skipIf(process.platform === 'win32')(
'should handle compression failure on token inflation',
async () => {
await rig.setup('interactive-compress-test');
const { ptyProcess } = rig.runInteractive();
let fullOutput = '';
ptyProcess.onData((data) => (fullOutput += data));
const authDialogAppeared = await rig.waitForText(
'How would you like to authenticate',
5000,
);
// select the second option if auth dialog come's up
if (authDialogAppeared) {
ptyProcess.write('2');
}
// Wait for the app to be ready
const isReady = await rig.waitForText('Type your message', 25000);
expect(
isReady,
'CLI did not start up in interactive mode correctly',
).toBe(true);
await type(ptyProcess, '/compress');
await new Promise((resolve) => setTimeout(resolve, 100));
await type(ptyProcess, '\r');
const foundEvent = await rig.waitForTelemetryEvent(
'chat_compression',
90000,
);
expect(foundEvent).toBe(true);
const compressionFailed = await rig.waitForText(
'compression was not beneficial',
25000,
);
expect(compressionFailed).toBe(true);
},
);
});

129
integration-tests/ctrl-c-exit.test.ts Normal file
View File

@@ -0,0 +1,129 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect } from 'vitest';
import { TestRig } from './test-helper.js';
import * as fs from 'node:fs';
import * as path from 'node:path';
describe('Ctrl+C exit', () => {
// (#9782) Temporarily disabling on windows because it is failing on main and every
// PR, which is potentially hiding other failures
it.skipIf(process.platform === 'win32')(
'should exit gracefully on second Ctrl+C',
async () => {
const rig = new TestRig();
await rig.setup('should exit gracefully on second Ctrl+C');
const { ptyProcess, promise } = rig.runInteractive();
let output = '';
ptyProcess.onData((data) => {
output += data;
});
// Wait for the app to be ready by looking for the initial prompt indicator
await rig.poll(() => output.includes('▶'), 5000, 100);
// Send first Ctrl+C
ptyProcess.write(String.fromCharCode(3));
// Wait for the exit prompt
await rig.poll(
() => output.includes('Press Ctrl+C again to exit'),
1500,
50,
);
// Send second Ctrl+C
ptyProcess.write(String.fromCharCode(3));
const result = await promise;
// Expect a graceful exit (code 0)
expect(
result.exitCode,
`Process exited with code ${result.exitCode}. Output: ${result.output}`,
).toBe(0);
// Check that the quitting message is displayed
const quittingMessage = 'Agent powering down. Goodbye!';
// The regex below is intentionally matching the ESC control character (\x1b)
// to strip ANSI color codes from the terminal output.
// eslint-disable-next-line no-control-regex
const cleanOutput = output.replace(/\x1b\[[0-9;]*m/g, '');
expect(cleanOutput).toContain(quittingMessage);
},
);
it.skipIf(process.platform === 'win32')(
'should exit gracefully on second Ctrl+C when calling a tool',
async () => {
const rig = new TestRig();
await rig.setup(
'should exit gracefully on second Ctrl+C when calling a tool',
);
const childProcessFile = 'child_process_file.txt';
rig.createFile(
'wait.js',
`setTimeout(() => require('fs').writeFileSync('${childProcessFile}', 'done'), 5000)`,
);
const { ptyProcess, promise } = rig.runInteractive();
let output = '';
ptyProcess.onData((data) => {
output += data;
});
// Wait for the app to be ready by looking for the initial prompt indicator
await rig.poll(() => output.includes('▶'), 5000, 100);
ptyProcess.write('use the tool to run "node -e wait.js"\n');
await rig.poll(() => output.includes('Shell'), 5000, 100);
// Send first Ctrl+C
ptyProcess.write(String.fromCharCode(3));
// Wait for the exit prompt
await rig.poll(
() => output.includes('Press Ctrl+C again to exit'),
1500,
50,
);
// Send second Ctrl+C
ptyProcess.write(String.fromCharCode(3));
const result = await promise;
// Expect a graceful exit (code 0)
expect(
result.exitCode,
`Process exited with code ${result.exitCode}. Output: ${result.output}`,
).toBe(0);
// Check that the quitting message is displayed
const quittingMessage = 'Agent powering down. Goodbye!';
// The regex below is intentionally matching the ESC control character (\x1b)
// to strip ANSI color codes from the terminal output.
// eslint-disable-next-line no-control-regex
const cleanOutput = output.replace(/\x1b\[[0-9;]*m/g, '');
expect(cleanOutput).toContain(quittingMessage);
// Check that the child process was terminated and did not create the file.
const childProcessFileExists = fs.existsSync(
path.join(rig.testDir!, childProcessFile),
);
expect(
childProcessFileExists,
'Child process file should not exist',
).toBe(false);
},
);
});

121
integration-tests/edit.test.ts
View File

@@ -61,4 +61,125 @@ describe('edit', () => {
console.log('File edited successfully. New content:', newFileContent);
}
});
it('should handle $ literally when replacing text ending with $', async () => {
const rig = new TestRig();
await rig.setup(
'should handle $ literally when replacing text ending with $',
);
const fileName = 'regex.yml';
const originalContent = "| select('match', '^[sv]d[a-z]$')\n";
const expectedContent = "| select('match', '^[sv]d[a-z]$') # updated\n";
rig.createFile(fileName, originalContent);
const prompt =
"Open regex.yml and append ' # updated' after the line containing ^[sv]d[a-z]$ without breaking the $ character.";
const result = await rig.run(prompt);
const foundToolCall = await rig.waitForToolCall('edit');
if (!foundToolCall) {
printDebugInfo(rig, result);
}
expect(foundToolCall, 'Expected to find an edit tool call').toBeTruthy();
validateModelOutput(result, ['regex.yml'], 'Replace $ literal test');
const newFileContent = rig.readFile(fileName);
expect(newFileContent).toBe(expectedContent);
});
it('should fail safely when old_string is not found', async () => {
const rig = new TestRig();
await rig.setup('should fail safely when old_string is not found');
const fileName = 'no_match.txt';
const fileContent = 'hello world';
rig.createFile(fileName, fileContent);
const prompt = `replace "goodbye" with "farewell" in ${fileName}`;
await rig.run(prompt);
await rig.waitForTelemetryReady();
const toolLogs = rig.readToolLogs();
const editAttempt = toolLogs.find((log) => log.toolRequest.name === 'edit');
const readAttempt = toolLogs.find(
(log) => log.toolRequest.name === 'read_file',
);
// VERIFY: The model must have at least tried to read the file or perform an edit.
expect(
readAttempt || editAttempt,
'Expected model to attempt a read_file or edit',
).toBeDefined();
// If the model tried to edit, that specific attempt must have failed.
if (editAttempt) {
if (editAttempt.toolRequest.success) {
console.error('The edit tool succeeded when it was expected to fail');
console.error('Tool call args:', editAttempt.toolRequest.args);
}
expect(
editAttempt.toolRequest.success,
'If edit is called, it must fail',
).toBe(false);
}
// CRITICAL: The final content of the file must be unchanged.
const newFileContent = rig.readFile(fileName);
expect(newFileContent).toBe(fileContent);
});
it('should insert a multi-line block of text', async () => {
const rig = new TestRig();
await rig.setup('should insert a multi-line block of text');
const fileName = 'insert_block.js';
const originalContent = 'function hello() {\n // INSERT_CODE_HERE\n}';
const newBlock = "console.log('hello');\n console.log('world');";
const expectedContent = `function hello() {\n ${newBlock}\n}`;
rig.createFile(fileName, originalContent);
const prompt = `In ${fileName}, replace "// INSERT_CODE_HERE" with:\n${newBlock}`;
const result = await rig.run(prompt);
const foundToolCall = await rig.waitForToolCall('edit');
if (!foundToolCall) {
printDebugInfo(rig, result);
}
expect(foundToolCall, 'Expected to find an edit tool call').toBeTruthy();
const newFileContent = rig.readFile(fileName);
expect(newFileContent.replace(/\r\n/g, '\n')).toBe(
expectedContent.replace(/\r\n/g, '\n'),
);
});
it('should delete a block of text', async () => {
const rig = new TestRig();
await rig.setup('should delete a block of text');
const fileName = 'delete_block.txt';
const blockToDelete =
'## DELETE THIS ##\nThis is a block of text to delete.\n## END DELETE ##';
const originalContent = `Hello\n${blockToDelete}\nWorld`;
// When deleting the block, a newline remains from the original structure (Hello\n + \nWorld)
rig.createFile(fileName, originalContent);
const prompt = `In ${fileName}, delete the entire block from "## DELETE THIS ##" to "## END DELETE ##" including the markers.`;
const result = await rig.run(prompt);
const foundToolCall = await rig.waitForToolCall('edit');
if (!foundToolCall) {
printDebugInfo(rig, result);
}
expect(foundToolCall, 'Expected to find an edit tool call').toBeTruthy();
const newFileContent = rig.readFile(fileName);
// Accept either 1 or 2 newlines between Hello and World
expect(newFileContent).toMatch(/^Hello\n\n?World$/);
});
});

52
integration-tests/extensions-install.test.ts Normal file
View File

@@ -0,0 +1,52 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { expect, test } from 'vitest';
import { TestRig } from './test-helper.js';
import { writeFileSync } from 'node:fs';
import { join } from 'node:path';
const extension = `{
"name": "test-extension",
"version": "0.0.1"
}`;
const extensionUpdate = `{
"name": "test-extension",
"version": "0.0.2"
}`;
test('installs a local extension, verifies a command, and updates it', async () => {
const rig = new TestRig();
rig.setup('extension install test');
const testServerPath = join(rig.testDir!, 'qwen-extension.json');
writeFileSync(testServerPath, extension);
try {
await rig.runCommand(['extensions', 'uninstall', 'test-extension']);
} catch {
/* empty */
}
const result = await rig.runCommand(
['extensions', 'install', `${rig.testDir!}`],
{ stdin: 'y\n' },
);
expect(result).toContain('test-extension');
const listResult = await rig.runCommand(['extensions', 'list']);
expect(listResult).toContain('test-extension');
writeFileSync(testServerPath, extensionUpdate);
const updateResult = await rig.runCommand([
'extensions',
'update',
`test-extension`,
]);
expect(updateResult).toContain('0.0.2');
await rig.runCommand(['extensions', 'uninstall', 'test-extension']);
await rig.cleanup();
});

85
integration-tests/file-system-interactive.test.ts Normal file
View File

@@ -0,0 +1,85 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { expect, describe, it, beforeEach, afterEach } from 'vitest';
import { TestRig, type, printDebugInfo } from './test-helper.js';
describe('Interactive file system', () => {
let rig: TestRig;
beforeEach(() => {
rig = new TestRig();
});
afterEach(async () => {
await rig.cleanup();
});
it.skipIf(process.platform === 'win32')(
'should perform a read-then-write sequence',
async () => {
const fileName = 'version.txt';
await rig.setup('interactive-read-then-write');
rig.createFile(fileName, '1.0.0');
const { ptyProcess } = rig.runInteractive();
const authDialogAppeared = await rig.waitForText(
'How would you like to authenticate',
5000,
);
// select the second option if auth dialog come's up
if (authDialogAppeared) {
ptyProcess.write('2');
}
// Wait for the app to be ready
const isReady = await rig.waitForText('Type your message', 15000);
expect(
isReady,
'CLI did not start up in interactive mode correctly',
).toBe(true);
// Step 1: Read the file
const readPrompt = `Read the version from ${fileName}`;
await type(ptyProcess, readPrompt);
await type(ptyProcess, '\r');
const readCall = await rig.waitForToolCall('read_file', 30000);
expect(readCall, 'Expected to find a read_file tool call').toBe(true);
const containsExpectedVersion = await rig.waitForText('1.0.0', 15000);
expect(
containsExpectedVersion,
'Expected to see version "1.0.0" in output',
).toBe(true);
// Step 2: Write the file
const writePrompt = `now change the version to 1.0.1 in the file`;
await type(ptyProcess, writePrompt);
await type(ptyProcess, '\r');
const toolCall = await rig.waitForAnyToolCall(
['write_file', 'edit'],
30000,
);
if (!toolCall) {
printDebugInfo(rig, rig._interactiveOutput, {
toolCall,
});
}
expect(toolCall, 'Expected to find a write_file or edit tool call').toBe(
true,
);
const newFileContent = rig.readFile(fileName);
expect(newFileContent).toBe('1.0.1');
},
);
});

167
integration-tests/file-system.test.ts
View File

@@ -5,6 +5,8 @@
*/
import { describe, it, expect } from 'vitest';
import { existsSync } from 'node:fs';
import * as path from 'node:path';
import { TestRig, printDebugInfo, validateModelOutput } from './test-helper.js';
describe('file-system', () => {
@@ -86,4 +88,169 @@ describe('file-system', () => {
console.log('File written successfully with hello message.');
}
});
it('should correctly handle file paths with spaces', async () => {
const rig = new TestRig();
await rig.setup('should correctly handle file paths with spaces');
const fileName = 'my test file.txt';
const result = await rig.run(`write "hello" to "${fileName}"`);
const foundToolCall = await rig.waitForToolCall('write_file');
if (!foundToolCall) {
printDebugInfo(rig, result);
}
expect(
foundToolCall,
'Expected to find a write_file tool call',
).toBeTruthy();
const newFileContent = rig.readFile(fileName);
expect(newFileContent).toBe('hello');
});
it('should perform a read-then-write sequence', async () => {
const rig = new TestRig();
await rig.setup('should perform a read-then-write sequence');
const fileName = 'version.txt';
rig.createFile(fileName, '1.0.0');
const prompt = `Read the version from ${fileName} and write the next version 1.0.1 back to the file.`;
const result = await rig.run(prompt);
await rig.waitForTelemetryReady();
const toolLogs = rig.readToolLogs();
const readCall = toolLogs.find(
(log) => log.toolRequest.name === 'read_file',
);
const writeCall = toolLogs.find(
(log) =>
log.toolRequest.name === 'write_file' ||
log.toolRequest.name === 'replace',
);
if (!readCall || !writeCall) {
printDebugInfo(rig, result, { readCall, writeCall });
}
expect(readCall, 'Expected to find a read_file tool call').toBeDefined();
expect(
writeCall,
'Expected to find a write_file or replace tool call',
).toBeDefined();
const newFileContent = rig.readFile(fileName);
expect(newFileContent).toBe('1.0.1');
});
it.skip('should replace multiple instances of a string', async () => {
const rig = new TestRig();
await rig.setup('should replace multiple instances of a string');
const fileName = 'ambiguous.txt';
const fileContent = 'Hey there, \ntest line\ntest line';
const expectedContent = 'Hey there, \nnew line\nnew line';
rig.createFile(fileName, fileContent);
const result = await rig.run(
`replace "test line" with "new line" in ${fileName}`,
);
const foundToolCall = await rig.waitForAnyToolCall([
'replace',
'write_file',
]);
if (!foundToolCall) {
printDebugInfo(rig, result);
}
expect(
foundToolCall,
'Expected to find a replace or write_file tool call',
).toBeTruthy();
const toolLogs = rig.readToolLogs();
const successfulEdit = toolLogs.some(
(log) =>
(log.toolRequest.name === 'replace' ||
log.toolRequest.name === 'write_file') &&
log.toolRequest.success,
);
if (!successfulEdit) {
console.error(
'Expected a successful edit tool call, but none was found.',
);
printDebugInfo(rig, result);
}
expect(successfulEdit, 'Expected a successful edit tool call').toBeTruthy();
const newFileContent = rig.readFile(fileName);
expect(newFileContent).toBe(expectedContent);
});
it('should fail safely when trying to edit a non-existent file', async () => {
const rig = new TestRig();
await rig.setup(
'should fail safely when trying to edit a non-existent file',
);
const fileName = 'non_existent.txt';
const result = await rig.run(`In ${fileName}, replace "a" with "b"`);
await rig.waitForTelemetryReady();
const toolLogs = rig.readToolLogs();
const readAttempt = toolLogs.find(
(log) => log.toolRequest.name === 'read_file',
);
const writeAttempt = toolLogs.find(
(log) => log.toolRequest.name === 'write_file',
);
const successfulReplace = toolLogs.find(
(log) => log.toolRequest.name === 'replace' && log.toolRequest.success,
);
// The model can either investigate (and fail) or do nothing.
// If it chose to investigate by reading, that read must have failed.
if (readAttempt && readAttempt.toolRequest.success) {
console.error(
'A read_file attempt succeeded for a non-existent file when it should have failed.',
);
printDebugInfo(rig, result);
}
if (readAttempt) {
expect(
readAttempt.toolRequest.success,
'If model tries to read the file, that attempt must fail',
).toBe(false);
}
// CRITICAL: Verify that no matter what the model did, it never successfully
// wrote or replaced anything.
if (writeAttempt) {
console.error(
'A write_file attempt was made when no file should be written.',
);
printDebugInfo(rig, result);
}
expect(
writeAttempt,
'write_file should not have been called',
).toBeUndefined();
if (successfulReplace) {
console.error('A successful replace occurred when it should not have.');
printDebugInfo(rig, result);
}
expect(
successfulReplace,
'A successful replace should not have occurred',
).toBeUndefined();
// Final verification: ensure the file was not created.
const filePath = path.join(rig.testDir!, fileName);
const fileExists = existsSync(filePath);
expect(fileExists, 'The non-existent file should not be created').toBe(
false,
);
});
});

4
integration-tests/globalSetup.ts
View File

@@ -22,7 +22,7 @@ import { fileURLToPath } from 'node:url';
import * as os from 'node:os';
import {
GEMINI_CONFIG_DIR,
QWEN_CONFIG_DIR,
DEFAULT_CONTEXT_FILENAME,
} from '../packages/core/src/tools/memoryTool.js';
@@ -33,7 +33,7 @@ let runDir = ''; // Make runDir accessible in teardown
const memoryFilePath = join(
os.homedir(),
GEMINI_CONFIG_DIR,
QWEN_CONFIG_DIR,
DEFAULT_CONTEXT_FILENAME,
);
let originalMemoryContent: string | null = null;

201
integration-tests/ide-client.test.ts
View File

@@ -1,201 +0,0 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import * as net from 'node:net';
import * as child_process from 'node:child_process';
import { IdeClient } from '../packages/core/src/ide/ide-client.js';
import { TestMcpServer } from './test-mcp-server.js';
describe.skip('IdeClient', () => {
it('reads port from file and connects', async () => {
const server = new TestMcpServer();
const port = await server.start();
const pid = process.pid;
const portFile = path.join(os.tmpdir(), `qwen-code-ide-server-${pid}.json`);
fs.writeFileSync(portFile, JSON.stringify({ port }));
process.env['QWEN_CODE_IDE_WORKSPACE_PATH'] = process.cwd();
process.env['TERM_PROGRAM'] = 'vscode';
const ideClient = await IdeClient.getInstance();
await ideClient.connect();
expect(ideClient.getConnectionStatus()).toEqual({
status: 'connected',
details: undefined,
});
fs.unlinkSync(portFile);
await server.stop();
delete process.env['QWEN_CODE_IDE_WORKSPACE_PATH'];
});
});
const getFreePort = (): Promise<number> => {
return new Promise((resolve, reject) => {
const server = net.createServer();
server.unref();
server.on('error', reject);
server.listen(0, () => {
const port = (server.address() as net.AddressInfo).port;
server.close(() => {
resolve(port);
});
});
});
};
describe('IdeClient fallback connection logic', () => {
let server: TestMcpServer;
let envPort: number;
let pid: number;
let portFile: string;
beforeEach(async () => {
pid = process.pid;
portFile = path.join(os.tmpdir(), `qwen-code-ide-server-${pid}.json`);
server = new TestMcpServer();
envPort = await server.start();
process.env['QWEN_CODE_IDE_SERVER_PORT'] = String(envPort);
process.env['TERM_PROGRAM'] = 'vscode';
process.env['QWEN_CODE_IDE_WORKSPACE_PATH'] = process.cwd();
// Reset instance
(IdeClient as unknown as { instance: IdeClient | undefined }).instance =
undefined;
});
afterEach(async () => {
await server.stop();
delete process.env['QWEN_CODE_IDE_SERVER_PORT'];
delete process.env['QWEN_CODE_IDE_WORKSPACE_PATH'];
if (fs.existsSync(portFile)) {
fs.unlinkSync(portFile);
}
});
it('connects using env var when port file does not exist', async () => {
// Ensure port file doesn't exist
if (fs.existsSync(portFile)) {
fs.unlinkSync(portFile);
}
const ideClient = await IdeClient.getInstance();
await ideClient.connect();
expect(ideClient.getConnectionStatus()).toEqual({
status: 'connected',
details: undefined,
});
});
it('falls back to env var when connection with port from file fails', async () => {
const filePort = await getFreePort();
// Write port file with a port that is not listening
fs.writeFileSync(portFile, JSON.stringify({ port: filePort }));
const ideClient = await IdeClient.getInstance();
await ideClient.connect();
expect(ideClient.getConnectionStatus()).toEqual({
status: 'connected',
details: undefined,
});
});
});
describe.skip('getIdeProcessId', () => {
let child: child_process.ChildProcess;
afterEach(() => {
if (child) {
child.kill();
}
});
it('should return the pid of the parent process', async () => {
// We need to spawn a child process that will run the test
// so that we can check that getIdeProcessId returns the pid of the parent
const parentPid = process.pid;
const output = await new Promise<string>((resolve, reject) => {
child = child_process.spawn(
'node',
[
'-e',
`
const { getIdeProcessId } = require('../packages/core/src/ide/process-utils.js');
getIdeProcessId().then(pid => console.log(pid));
`,
],
{
stdio: ['pipe', 'pipe', 'pipe'],
},
);
let out = '';
child.stdout?.on('data', (data) => {
out += data.toString();
});
child.on('close', (code) => {
if (code === 0) {
resolve(out.trim());
} else {
reject(new Error(`Child process exited with code ${code}`));
}
});
});
expect(parseInt(output, 10)).toBe(parentPid);
}, 10000);
});
describe('IdeClient with proxy', () => {
let mcpServer: TestMcpServer;
let proxyServer: net.Server;
let mcpServerPort: number;
let proxyServerPort: number;
beforeEach(async () => {
mcpServer = new TestMcpServer();
mcpServerPort = await mcpServer.start();
proxyServer = net.createServer().listen();
proxyServerPort = (proxyServer.address() as net.AddressInfo).port;
vi.stubEnv('QWEN_CODE_IDE_SERVER_PORT', String(mcpServerPort));
vi.stubEnv('TERM_PROGRAM', 'vscode');
vi.stubEnv('QWEN_CODE_IDE_WORKSPACE_PATH', process.cwd());
// Reset instance
(IdeClient as unknown as { instance: IdeClient | undefined }).instance =
undefined;
});
afterEach(async () => {
(await IdeClient.getInstance()).disconnect();
await mcpServer.stop();
proxyServer.close();
vi.unstubAllEnvs();
});
it('should connect to IDE server when HTTP_PROXY, HTTPS_PROXY and NO_PROXY are set', async () => {
vi.stubEnv('HTTP_PROXY', `http://localhost:${proxyServerPort}`);
vi.stubEnv('HTTPS_PROXY', `http://localhost:${proxyServerPort}`);
vi.stubEnv('NO_PROXY', 'example.com,127.0.0.1,::1');
const ideClient = await IdeClient.getInstance();
await ideClient.connect();
expect(ideClient.getConnectionStatus()).toEqual({
status: 'connected',
details: undefined,
});
});
});

89
integration-tests/json-output.test.ts Normal file
View File

@@ -0,0 +1,89 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { expect, describe, it, beforeEach, afterEach } from 'vitest';
import { TestRig } from './test-helper.js';
describe('JSON output', () => {
let rig: TestRig;
beforeEach(async () => {
rig = new TestRig();
await rig.setup('json-output-test');
});
afterEach(async () => {
await rig.cleanup();
});
it('should return a valid JSON with response and stats', async () => {
const result = await rig.run(
'What is the capital of France?',
'--output-format',
'json',
);
const parsed = JSON.parse(result);
expect(parsed).toHaveProperty('response');
expect(typeof parsed.response).toBe('string');
expect(parsed.response.toLowerCase()).toContain('paris');
expect(parsed).toHaveProperty('stats');
expect(typeof parsed.stats).toBe('object');
});
it('should return a JSON error for enforced auth mismatch before running', async () => {
process.env['GOOGLE_GENAI_USE_GCA'] = 'true';
await rig.setup('json-output-auth-mismatch', {
settings: {
security: { auth: { enforcedType: 'gemini-api-key' } },
},
});
let thrown: Error | undefined;
try {
await rig.run('Hello', '--output-format', 'json');
expect.fail('Expected process to exit with error');
} catch (e) {
thrown = e as Error;
} finally {
delete process.env['GOOGLE_GENAI_USE_GCA'];
}
expect(thrown).toBeDefined();
const message = (thrown as Error).message;
// Use a regex to find the first complete JSON object in the string
const jsonMatch = message.match(/{[\s\S]*}/);
// Fail if no JSON-like text was found
expect(
jsonMatch,
'Expected to find a JSON object in the error output',
).toBeTruthy();
let payload;
try {
// Parse the matched JSON string
payload = JSON.parse(jsonMatch![0]);
} catch (parseError) {
console.error('Failed to parse the following JSON:', jsonMatch![0]);
throw new Error(
`Test failed: Could not parse JSON from error message. Details: ${parseError}`,
);
}
expect(payload.error).toBeDefined();
expect(payload.error.type).toBe('Error');
expect(payload.error.code).toBe(1);
expect(payload.error.message).toContain(
'configured auth type is gemini-api-key',
);
expect(payload.error.message).toContain(
'current auth type is oauth-personal',
);
});
});

2
integration-tests/list_directory.test.ts
View File

@@ -29,7 +29,7 @@ describe('list_directory', () => {
50, // check every 50ms
);
const prompt = `Can you list the files in the current directory. Display them in the style of 'ls'`;
const prompt = `Can you list the files in the current directory.`;
const result = await rig.run(prompt);

41
integration-tests/mcp_server_cyclic_schema.test.ts
View File

@@ -5,14 +5,26 @@
*/
/**
* This test verifies we can match maximum schema depth errors from Gemini
* and then detect and warn about the potential tools that caused the error.
* This test verifies we can provide MCP tools with recursive input schemas
* (in JSON, using the $ref keyword) and both the GenAI SDK and the Gemini
* API calls succeed. Note that prior to
* https://github.com/googleapis/js-genai/commit/36f6350705ecafc47eaea3f3eecbcc69512edab7#diff-fdde9372aec859322b7c5a5efe467e0ad25a57210c7229724586ee90ea4f5a30
* the Gemini API call would fail for such tools because the schema was
* passed not as a JSON string but using the Gemini API's tool parameter
* schema object which has stricter typing and recursion restrictions.
* If this test fails, it's likely because either the GenAI SDK or Gemini API
* has become more restrictive about the type of tool parameter schemas that
* are accepted. If this occurs: Gemini CLI previously attempted to detect
* such tools and proactively remove them from the set of tools provided in
* the Gemini API call (as FunctionDeclaration objects). It may be appropriate
* to resurrect that behavior but note that it's difficult to keep the
* GCLI filters in sync with the Gemini API restrictions and behavior.
*/
import { describe, it, beforeAll, expect } from 'vitest';
import { TestRig } from './test-helper.js';
import { join } from 'node:path';
import { writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { beforeAll, describe, expect, it } from 'vitest';
import { TestRig } from './test-helper.js';
// Create a minimal MCP server that doesn't require external dependencies
// This implements the MCP protocol directly using Node.js built-ins
@@ -180,15 +192,16 @@ describe('mcp server with cyclic tool schema is detected', () => {
}
});
it('should error and suggest disabling the cyclic tool', async () => {
// Just run any command to trigger the schema depth error.
// If this test starts failing, check `isSchemaDepthError` from
// geminiChat.ts to see if it needs to be updated.
// Or, possibly it could mean that gemini has fixed the issue.
const output = await rig.run('hello');
it('mcp tool with cyclic schema should be accessible', async () => {
const mcp_list_output = await rig.runCommand(['mcp', 'list']);
expect(output).toMatch(
/Skipping tool 'tool_with_cyclic_schema' from MCP server 'cyclic-schema-server' because it has missing types in its parameter schema/,
);
// Verify the cyclic schema server is configured
expect(mcp_list_output).toContain('cyclic-schema-server');
});
it('gemini api call should be successful with cyclic mcp tool schema', async () => {
// Run any command and verify that we get a non-error response from
// the Gemini API.
await rig.run('hello');
});
});

62
integration-tests/mixed-input-crash.test.ts Normal file
View File

@@ -0,0 +1,62 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect } from 'vitest';
import { TestRig } from './test-helper.js';
describe('mixed input crash prevention', () => {
it('should not crash when using mixed prompt inputs', async () => {
const rig = new TestRig();
rig.setup('should not crash when using mixed prompt inputs');
// Test: echo "say '1'." | gemini --prompt-interactive="say '2'." say '3'.
const stdinContent = "say '1'.";
try {
await rig.run(
{ stdin: stdinContent },
'--prompt-interactive',
"say '2'.",
"say '3'.",
);
throw new Error('Expected the command to fail, but it succeeded');
} catch (error: unknown) {
expect(error).toBeInstanceOf(Error);
const err = error as Error;
expect(err.message).toContain('Process exited with code 1');
expect(err.message).toContain(
'--prompt-interactive flag cannot be used when input is piped',
);
expect(err.message).not.toContain('setRawMode is not a function');
expect(err.message).not.toContain('unexpected critical error');
}
const lastRequest = rig.readLastApiRequest();
expect(lastRequest).toBeNull();
});
it('should provide clear error message for mixed input', async () => {
const rig = new TestRig();
rig.setup('should provide clear error message for mixed input');
try {
await rig.run(
{ stdin: 'test input' },
'--prompt-interactive',
'test prompt',
);
throw new Error('Expected the command to fail, but it succeeded');
} catch (error: unknown) {
expect(error).toBeInstanceOf(Error);
const err = error as Error;
expect(err.message).toContain(
'--prompt-interactive flag cannot be used when input is piped',
);
}
});
});

10
integration-tests/read_many_files.test.ts
View File

@@ -14,7 +14,7 @@ describe('read_many_files', () => {
rig.createFile('file1.txt', 'file 1 content');
rig.createFile('file2.txt', 'file 2 content');
const prompt = `Please use read_many_files to read file1.txt and file2.txt and show me what's in them`;
const prompt = `Use the read_many_files tool to read the contents of file1.txt and file2.txt and then print the contents of each file.`;
const result = await rig.run(prompt);
@@ -41,11 +41,7 @@ describe('read_many_files', () => {
'Expected to find either read_many_files or multiple read_file tool calls',
).toBeTruthy();
// Validate model output - will throw if no output, warn if missing expected content
validateModelOutput(
result,
['file 1 content', 'file 2 content'],
'Read many files test',
);
// Validate model output - will throw if no output
validateModelOutput(result, null, 'Read many files test');
});
});

60
integration-tests/run_shell_command.test.ts
View File

@@ -67,4 +67,64 @@ describe('run_shell_command', () => {
// Validate model output - will throw if no output, warn if missing expected content
validateModelOutput(result, 'test-stdin', 'Shell command stdin test');
});
it('should propagate environment variables to the child process', async () => {
const rig = new TestRig();
await rig.setup('should propagate environment variables');
const varName = 'GEMINI_CLI_TEST_VAR';
const varValue = `test-value-${Math.random().toString(36).substring(7)}`;
process.env[varName] = varValue;
try {
const prompt = `Use echo to learn the value of the environment variable named ${varName} and tell me what it is.`;
const result = await rig.run(prompt);
const foundToolCall = await rig.waitForToolCall('run_shell_command');
if (!foundToolCall || !result.includes(varValue)) {
printDebugInfo(rig, result, {
'Found tool call': foundToolCall,
'Contains varValue': result.includes(varValue),
});
}
expect(
foundToolCall,
'Expected to find a run_shell_command tool call',
).toBeTruthy();
validateModelOutput(result, varValue, 'Env var propagation test');
expect(result).toContain(varValue);
} finally {
delete process.env[varName];
}
});
it('should run a platform-specific file listing command', async () => {
const rig = new TestRig();
await rig.setup('should run platform-specific file listing');
const fileName = `test-file-${Math.random().toString(36).substring(7)}.txt`;
rig.createFile(fileName, 'test content');
const prompt = `Run a shell command to list the files in the current directory and tell me what they are.`;
const result = await rig.run(prompt);
const foundToolCall = await rig.waitForToolCall('run_shell_command');
// Debugging info
if (!foundToolCall || !result.includes(fileName)) {
printDebugInfo(rig, result, {
'Found tool call': foundToolCall,
'Contains fileName': result.includes(fileName),
});
}
expect(
foundToolCall,
'Expected to find a run_shell_command tool call',
).toBeTruthy();
validateModelOutput(result, fileName, 'Platform-specific listing test');
expect(result).toContain(fileName);
});
});

4
integration-tests/save_memory.test.ts
View File

@@ -8,7 +8,9 @@ import { describe, it, expect } from 'vitest';
import { TestRig, printDebugInfo, validateModelOutput } from './test-helper.js';
describe('save_memory', () => {
it('should be able to save to memory', async () => {
// Skipped due to flaky model behavior - the model sometimes answers the question
// directly without calling the save_memory tool, even when prompted to "remember"
it.skip('should be able to save to memory', async () => {
const rig = new TestRig();
await rig.setup('should be able to save to memory');

156
integration-tests/shell-service.test.ts
View File

@@ -1,156 +0,0 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect, beforeAll } from 'vitest';
import { ShellExecutionService } from '../packages/core/src/services/shellExecutionService.js';
import * as fs from 'node:fs/promises';
import * as path from 'node:path';
import { vi } from 'vitest';
describe('ShellExecutionService programmatic integration tests', () => {
let testDir: string;
beforeAll(async () => {
// Create a dedicated directory for this test suite to avoid conflicts.
testDir = path.join(
process.env['INTEGRATION_TEST_FILE_DIR']!,
'shell-service-tests',
);
await fs.mkdir(testDir, { recursive: true });
});
it('should execute a simple cross-platform command (echo)', async () => {
const command = 'echo "hello from the service"';
const onOutputEvent = vi.fn();
const abortController = new AbortController();
const handle = await ShellExecutionService.execute(
command,
testDir,
onOutputEvent,
abortController.signal,
false,
);
const result = await handle.result;
expect(result.error).toBeNull();
expect(result.exitCode).toBe(0);
// Output can vary slightly between shells (e.g., quotes), so check for inclusion.
expect(result.output).toContain('hello from the service');
});
it.runIf(process.platform === 'win32')(
'should execute "dir" on Windows',
async () => {
const testFile = 'test-file-windows.txt';
await fs.writeFile(path.join(testDir, testFile), 'windows test');
const command = 'dir';
const onOutputEvent = vi.fn();
const abortController = new AbortController();
const handle = await ShellExecutionService.execute(
command,
testDir,
onOutputEvent,
abortController.signal,
false,
);
const result = await handle.result;
expect(result.error).toBeNull();
expect(result.exitCode).toBe(0);
expect(result.output).toContain(testFile);
},
);
it.skipIf(process.platform === 'win32')(
'should execute "ls -l" on Unix',
async () => {
const testFile = 'test-file-unix.txt';
await fs.writeFile(path.join(testDir, testFile), 'unix test');
const command = 'ls -l';
const onOutputEvent = vi.fn();
const abortController = new AbortController();
const handle = await ShellExecutionService.execute(
command,
testDir,
onOutputEvent,
abortController.signal,
false,
);
const result = await handle.result;
expect(result.error).toBeNull();
expect(result.exitCode).toBe(0);
expect(result.output).toContain(testFile);
},
);
it('should abort a running process', async () => {
// A command that runs for a bit. 'sleep' on unix, 'timeout' on windows.
const command = process.platform === 'win32' ? 'timeout /t 20' : 'sleep 20';
const onOutputEvent = vi.fn();
const abortController = new AbortController();
const handle = await ShellExecutionService.execute(
command,
testDir,
onOutputEvent,
abortController.signal,
false,
);
// Abort shortly after starting
setTimeout(() => abortController.abort(), 50);
const result = await handle.result;
// For debugging the flaky test.
console.log('Abort test result:', result);
expect(result.aborted).toBe(true);
// A clean exit is exitCode 0 and no signal. If the process was truly
// aborted, it should not have exited cleanly.
const exitedCleanly = result.exitCode === 0 && result.signal === null;
expect(exitedCleanly, 'Process should not have exited cleanly').toBe(false);
});
it('should propagate environment variables to the child process', async () => {
const varName = 'QWEN_CODE_TEST_VAR';
const varValue = `test-value`;
process.env[varName] = varValue;
try {
const command =
process.platform === 'win32' ? `echo %${varName}%` : `echo $${varName}`;
const onOutputEvent = vi.fn();
const abortController = new AbortController();
const handle = await ShellExecutionService.execute(
command,
testDir,
onOutputEvent,
abortController.signal,
false,
);
const result = await handle.result;
expect(result.error).toBeNull();
expect(result.exitCode).toBe(0);
expect(result.output).toContain(varValue);
} finally {
// Clean up the env var to prevent side-effects on other tests.
delete process.env[varName];
}
});
});

19
integration-tests/simple-mcp-server.test.ts
View File

@@ -189,6 +189,25 @@ describe('simple-mcp-server', () => {
const { chmodSync } = await import('node:fs');
chmodSync(testServerPath, 0o755);
}
// Poll for script for up to 5s
const { accessSync, constants } = await import('node:fs');
const isReady = await rig.poll(
() => {
try {
accessSync(testServerPath, constants.F_OK);
return true;
} catch {
return false;
}
},
5000, // Max wait 5 seconds
100, // Poll every 100ms
);
if (!isReady) {
throw new Error('MCP server script was not ready in time.');
}
});
it('should add two numbers', async () => {

26
integration-tests/telemetry.test.ts Normal file
View File

@@ -0,0 +1,26 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect } from 'vitest';
import { TestRig } from './test-helper.js';
describe('telemetry', () => {
it('should emit a metric and a log event', async () => {
const rig = new TestRig();
rig.setup('should emit a metric and a log event');
// Run a simple command that should trigger telemetry
await rig.run('just saying hi');
// Verify that a user_prompt event was logged
const hasUserPromptEvent = await rig.waitForTelemetryEvent('user_prompt');
expect(hasUserPromptEvent).toBe(true);
// Verify that a cli_command_count metric was emitted
const cliCommandCountMetric = rig.readMetric('session.count');
expect(cliCommandCountMetric).not.toBeNull();
});
});

380
integration-tests/test-helper.ts
View File

@@ -5,13 +5,15 @@
*/
import { execSync, spawn } from 'node:child_process';
import { parse } from 'shell-quote';
import { mkdirSync, writeFileSync, readFileSync } from 'node:fs';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import { env } from 'node:process';
import { DEFAULT_QWEN_MODEL } from '../packages/core/src/config/models.js';
import fs from 'node:fs';
import { EOL } from 'node:os';
import * as pty from '@lydell/node-pty';
import stripAnsi from 'strip-ansi';
const __dirname = dirname(fileURLToPath(import.meta.url));
@@ -112,11 +114,38 @@ export function validateModelOutput(
return true;
}
// Simulates typing a string one character at a time to avoid paste detection.
export async function type(ptyProcess: pty.IPty, text: string) {
const delay = 5;
for (const char of text) {
ptyProcess.write(char);
await new Promise((resolve) => setTimeout(resolve, delay));
}
}
interface ParsedLog {
attributes?: {
'event.name'?: string;
function_name?: string;
function_args?: string;
success?: boolean;
duration_ms?: number;
};
scopeMetrics?: {
metrics: {
descriptor: {
name: string;
};
}[];
}[];
}
export class TestRig {
bundlePath: string;
testDir: string | null;
testName?: string;
_lastRunStdout?: string;
_interactiveOutput = '';
constructor() {
this.bundlePath = join(__dirname, '..', 'bundle/gemini.js');
@@ -140,8 +169,8 @@ export class TestRig {
mkdirSync(this.testDir, { recursive: true });
// Create a settings file to point the CLI to the local collector
const geminiDir = join(this.testDir, '.qwen');
mkdirSync(geminiDir, { recursive: true });
const qwenDir = join(this.testDir, '.qwen');
mkdirSync(qwenDir, { recursive: true });
// In sandbox mode, use an absolute path for telemetry inside the container
// The container mounts the test directory at the same path as the host
const telemetryPath = join(this.testDir, 'telemetry.log'); // Always use test directory for telemetry
@@ -153,12 +182,12 @@ export class TestRig {
otlpEndpoint: '',
outfile: telemetryPath,
},
sandbox:
env['GEMINI_SANDBOX'] !== 'false' ? env['GEMINI_SANDBOX'] : false,
model: DEFAULT_QWEN_MODEL,
sandbox: env.GEMINI_SANDBOX !== 'false' ? env.GEMINI_SANDBOX : false,
...options.settings, // Allow tests to override/add settings
};
writeFileSync(
join(geminiDir, 'settings.json'),
join(qwenDir, 'settings.json'),
JSON.stringify(settings, null, 2),
);
}
@@ -178,13 +207,32 @@ export class TestRig {
execSync('sync', { cwd: this.testDir! });
}
/**
* The command and args to use to invoke Qwen Code CLI. Allows us to switch
* between using the bundled gemini.js (the default) and using the installed
* 'qwen' (used to verify npm bundles).
*/
private _getCommandAndArgs(extraInitialArgs: string[] = []): {
command: string;
initialArgs: string[];
} {
const isNpmReleaseTest =
process.env.INTEGRATION_TEST_USE_INSTALLED_GEMINI === 'true';
const command = isNpmReleaseTest ? 'qwen' : 'node';
const initialArgs = isNpmReleaseTest
? extraInitialArgs
: [this.bundlePath, ...extraInitialArgs];
return { command, initialArgs };
}
run(
promptOrOptions:
| string
| { prompt?: string; stdin?: string; stdinDoesNotEnd?: boolean },
...args: string[]
): Promise<string> {
let command = `node ${this.bundlePath} --yolo`;
const { command, initialArgs } = this._getCommandAndArgs(['--yolo']);
const commandArgs = [...initialArgs];
const execOptions: {
cwd: string;
encoding: 'utf-8';
@@ -195,27 +243,25 @@ export class TestRig {
};
if (typeof promptOrOptions === 'string') {
command += ` --prompt ${JSON.stringify(promptOrOptions)}`;
commandArgs.push('--prompt', promptOrOptions);
} else if (
typeof promptOrOptions === 'object' &&
promptOrOptions !== null
) {
if (promptOrOptions.prompt) {
command += ` --prompt ${JSON.stringify(promptOrOptions.prompt)}`;
commandArgs.push('--prompt', promptOrOptions.prompt);
}
if (promptOrOptions.stdin) {
execOptions.input = promptOrOptions.stdin;
}
}
command += ` ${args.join(' ')}`;
commandArgs.push(...args);
const commandArgs = parse(command);
const node = commandArgs.shift() as string;
const child = spawn(node, commandArgs as string[], {
const child = spawn(command, commandArgs, {
cwd: this.testDir!,
stdio: 'pipe',
env: process.env,
});
let stdout = '';
@@ -291,8 +337,15 @@ export class TestRig {
result = filteredLines.join('\n');
}
// If we have stderr output, include that also
if (stderr) {
// Check if this is a JSON output test - if so, don't include stderr
// as it would corrupt the JSON
const isJsonOutput =
commandArgs.includes('--output-format') &&
commandArgs.includes('json');
// If we have stderr output and it's not a JSON test, include that also
if (stderr && !isJsonOutput) {
result += `\n\nStdErr:\n${stderr}`;
}
@@ -306,6 +359,58 @@ export class TestRig {
return promise;
}
runCommand(
args: string[],
options: { stdin?: string } = {},
): Promise<string> {
const { command, initialArgs } = this._getCommandAndArgs();
const commandArgs = [...initialArgs, ...args];
const child = spawn(command, commandArgs, {
cwd: this.testDir!,
stdio: 'pipe',
});
let stdout = '';
let stderr = '';
if (options.stdin) {
child.stdin!.write(options.stdin);
child.stdin!.end();
}
child.stdout!.on('data', (data: Buffer) => {
stdout += data;
if (env.KEEP_OUTPUT === 'true' || env.VERBOSE === 'true') {
process.stdout.write(data);
}
});
child.stderr!.on('data', (data: Buffer) => {
stderr += data;
if (env.KEEP_OUTPUT === 'true' || env.VERBOSE === 'true') {
process.stderr.write(data);
}
});
const promise = new Promise<string>((resolve, reject) => {
child.on('close', (code: number) => {
if (code === 0) {
this._lastRunStdout = stdout;
let result = stdout;
if (stderr) {
result += `\n\nStdErr:\n${stderr}`;
}
resolve(result);
} else {
reject(new Error(`Process exited with code ${code}:\n${stderr}`));
}
});
});
return promise;
}
readFile(fileName: string) {
const filePath = join(this.testDir!, fileName);
const content = readFileSync(filePath, 'utf-8');
@@ -363,37 +468,12 @@ export class TestRig {
return this.poll(
() => {
const logFilePath = join(this.testDir!, 'telemetry.log');
if (!logFilePath || !fs.existsSync(logFilePath)) {
return false;
}
const content = readFileSync(logFilePath, 'utf-8');
const jsonObjects = content
.split(/}\n{/)
.map((obj, index, array) => {
// Add back the braces we removed during split
if (index > 0) obj = '{' + obj;
if (index < array.length - 1) obj = obj + '}';
return obj.trim();
})
.filter((obj) => obj);
for (const jsonStr of jsonObjects) {
try {
const logData = JSON.parse(jsonStr);
if (
logData.attributes &&
logData.attributes['event.name'] === `gemini_cli.${eventName}`
) {
return true;
}
} catch {
// ignore
}
}
return false;
const logs = this._readAndParseTelemetryLog();
return logs.some(
(logData) =>
logData.attributes &&
logData.attributes['event.name'] === `qwen-code.${eventName}`,
);
},
timeout,
100,
@@ -566,7 +646,7 @@ export class TestRig {
}
} else if (
obj.attributes &&
obj.attributes['event.name'] === 'gemini_cli.tool_call'
obj.attributes['event.name'] === 'qwen-code.tool_call'
) {
logs.push({
timestamp: obj.attributes['event.timestamp'],
@@ -590,6 +670,45 @@ export class TestRig {
return logs;
}
private _readAndParseTelemetryLog(): ParsedLog[] {
// Telemetry is always written to the test directory
const logFilePath = join(this.testDir!, 'telemetry.log');
if (!logFilePath || !fs.existsSync(logFilePath)) {
return [];
}
const content = readFileSync(logFilePath, 'utf-8');
// Split the content into individual JSON objects
// They are separated by "}\n{"
const jsonObjects = content
.split(/}\n{/)
.map((obj, index, array) => {
// Add back the braces we removed during split
if (index > 0) obj = '{' + obj;
if (index < array.length - 1) obj = obj + '}';
return obj.trim();
})
.filter((obj) => obj);
const logs: ParsedLog[] = [];
for (const jsonStr of jsonObjects) {
try {
const logData = JSON.parse(jsonStr);
logs.push(logData);
} catch (e) {
// Skip objects that aren't valid JSON
if (env.VERBOSE === 'true') {
console.error('Failed to parse telemetry object:', e);
}
}
}
return logs;
}
readToolLogs() {
// For Podman, first check if telemetry file exists and has content
// If not, fall back to parsing from stdout
@@ -619,33 +738,7 @@ export class TestRig {
}
}
// Telemetry is always written to the test directory
const logFilePath = join(this.testDir!, 'telemetry.log');
if (!logFilePath) {
console.warn(`TELEMETRY_LOG_FILE environment variable not set`);
return [];
}
// Check if file exists, if not return empty array (file might not be created yet)
if (!fs.existsSync(logFilePath)) {
return [];
}
const content = readFileSync(logFilePath, 'utf-8');
// Split the content into individual JSON objects
// They are separated by "}\n{"
const jsonObjects = content
.split(/}\n{/)
.map((obj, index, array) => {
// Add back the braces we removed during split
if (index > 0) obj = '{' + obj;
if (index < array.length - 1) obj = obj + '}';
return obj.trim();
})
.filter((obj) => obj);
const parsedLogs = this._readAndParseTelemetryLog();
const logs: {
toolRequest: {
name: string;
@@ -655,29 +748,21 @@ export class TestRig {
};
}[] = [];
for (const jsonStr of jsonObjects) {
try {
const logData = JSON.parse(jsonStr);
// Look for tool call logs
if (
logData.attributes &&
logData.attributes['event.name'] === 'qwen-code.tool_call'
) {
const toolName = logData.attributes.function_name;
logs.push({
toolRequest: {
name: toolName,
args: logData.attributes.function_args,
success: logData.attributes.success,
duration_ms: logData.attributes.duration_ms,
},
});
}
} catch (e) {
// Skip objects that aren't valid JSON
if (env['VERBOSE'] === 'true') {
console.error('Failed to parse telemetry object:', e);
}
for (const logData of parsedLogs) {
// Look for tool call logs
if (
logData.attributes &&
logData.attributes['event.name'] === 'qwen-code.tool_call'
) {
const toolName = logData.attributes.function_name;
logs.push({
toolRequest: {
name: toolName,
args: logData.attributes.function_args,
success: logData.attributes.success,
duration_ms: logData.attributes.duration_ms,
},
});
}
}
@@ -685,38 +770,79 @@ export class TestRig {
}
readLastApiRequest(): Record<string, unknown> | null {
// Telemetry is always written to the test directory
const logFilePath = join(this.testDir!, 'telemetry.log');
const logs = this._readAndParseTelemetryLog();
const apiRequests = logs.filter(
(logData) =>
logData.attributes &&
logData.attributes['event.name'] === 'qwen-code.api_request',
);
return apiRequests.pop() || null;
}
if (!logFilePath || !fs.existsSync(logFilePath)) {
return null;
}
const content = readFileSync(logFilePath, 'utf-8');
const jsonObjects = content
.split(/}\n{/)
.map((obj, index, array) => {
if (index > 0) obj = '{' + obj;
if (index < array.length - 1) obj = obj + '}';
return obj.trim();
})
.filter((obj) => obj);
let lastApiRequest = null;
for (const jsonStr of jsonObjects) {
try {
const logData = JSON.parse(jsonStr);
if (
logData.attributes &&
logData.attributes['event.name'] === 'gemini_cli.api_request'
) {
lastApiRequest = logData;
readMetric(metricName: string): Record<string, unknown> | null {
const logs = this._readAndParseTelemetryLog();
for (const logData of logs) {
if (logData.scopeMetrics) {
for (const scopeMetric of logData.scopeMetrics) {
for (const metric of scopeMetric.metrics) {
if (metric.descriptor.name === `qwen-code.${metricName}`) {
return metric;
}
}
}
} catch {
// ignore
}
}
return lastApiRequest;
return null;
}
async waitForText(text: string, timeout?: number): Promise<boolean> {
if (!timeout) {
timeout = this.getDefaultTimeout();
}
return this.poll(
() =>
stripAnsi(this._interactiveOutput)
.toLowerCase()
.includes(text.toLowerCase()),
timeout,
200,
);
}
runInteractive(...args: string[]): {
ptyProcess: pty.IPty;
promise: Promise<{ exitCode: number; signal?: number; output: string }>;
} {
const { command, initialArgs } = this._getCommandAndArgs(['--yolo']);
const commandArgs = [...initialArgs, ...args];
this._interactiveOutput = ''; // Reset output for the new run
const ptyProcess = pty.spawn(command, commandArgs, {
name: 'xterm-color',
cols: 80,
rows: 30,
cwd: this.testDir!,
env: process.env as { [key: string]: string },
});
ptyProcess.onData((data) => {
this._interactiveOutput += data;
if (env.KEEP_OUTPUT === 'true' || env.VERBOSE === 'true') {
process.stdout.write(data);
}
});
const promise = new Promise<{
exitCode: number;
signal?: number;
output: string;
}>((resolve) => {
ptyProcess.onExit(({ exitCode, signal }) => {
resolve({ exitCode, signal, output: this._interactiveOutput });
});
});
return { ptyProcess, promise };
}
}

141
integration-tests/utf-bom-encoding.test.ts Normal file
View File

@@ -0,0 +1,141 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
import { writeFileSync, readFileSync } from 'node:fs';
import { join, resolve } from 'node:path';
import { TestRig } from './test-helper.js';
// Windows skip (Option A: avoid infra scope)
const d = process.platform === 'win32' ? describe.skip : describe;
// BOM encoders
const utf8BOM = (s: string) =>
Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from(s, 'utf8')]);
const utf16LE = (s: string) =>
Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from(s, 'utf16le')]);
const utf16BE = (s: string) => {
const bom = Buffer.from([0xfe, 0xff]);
const le = Buffer.from(s, 'utf16le');
le.swap16();
return Buffer.concat([bom, le]);
};
const utf32LE = (s: string) => {
const bom = Buffer.from([0xff, 0xfe, 0x00, 0x00]);
const cps = Array.from(s, (c) => c.codePointAt(0)!);
const payload = Buffer.alloc(cps.length * 4);
cps.forEach((cp, i) => {
const o = i * 4;
payload[o] = cp & 0xff;
payload[o + 1] = (cp >>> 8) & 0xff;
payload[o + 2] = (cp >>> 16) & 0xff;
payload[o + 3] = (cp >>> 24) & 0xff;
});
return Buffer.concat([bom, payload]);
};
const utf32BE = (s: string) => {
const bom = Buffer.from([0x00, 0x00, 0xfe, 0xff]);
const cps = Array.from(s, (c) => c.codePointAt(0)!);
const payload = Buffer.alloc(cps.length * 4);
cps.forEach((cp, i) => {
const o = i * 4;
payload[o] = (cp >>> 24) & 0xff;
payload[o + 1] = (cp >>> 16) & 0xff;
payload[o + 2] = (cp >>> 8) & 0xff;
payload[o + 3] = cp & 0xff;
});
return Buffer.concat([bom, payload]);
};
let rig: TestRig;
let dir: string;
d('BOM end-to-end integration', () => {
beforeAll(async () => {
rig = new TestRig();
await rig.setup('bom-integration');
dir = rig.testDir!;
});
afterAll(async () => {
await rig.cleanup();
});
async function runAndAssert(
filename: string,
content: Buffer,
expectedText: string | null,
) {
writeFileSync(join(dir, filename), content);
const prompt = `read the file ${filename} and output its exact contents`;
const output = await rig.run(prompt);
await rig.waitForToolCall('read_file');
const lower = output.toLowerCase();
if (expectedText === null) {
expect(
lower.includes('binary') ||
lower.includes('skipped binary file') ||
lower.includes('cannot display'),
).toBeTruthy();
} else {
expect(output.includes(expectedText)).toBeTruthy();
expect(lower.includes('skipped binary file')).toBeFalsy();
}
}
it('UTF-8 BOM', async () => {
await runAndAssert('utf8.txt', utf8BOM('BOM_OK UTF-8'), 'BOM_OK UTF-8');
});
it('UTF-16 LE BOM', async () => {
await runAndAssert(
'utf16le.txt',
utf16LE('BOM_OK UTF-16LE'),
'BOM_OK UTF-16LE',
);
});
it('UTF-16 BE BOM', async () => {
await runAndAssert(
'utf16be.txt',
utf16BE('BOM_OK UTF-16BE'),
'BOM_OK UTF-16BE',
);
});
it('UTF-32 LE BOM', async () => {
await runAndAssert(
'utf32le.txt',
utf32LE('BOM_OK UTF-32LE'),
'BOM_OK UTF-32LE',
);
});
it('UTF-32 BE BOM', async () => {
await runAndAssert(
'utf32be.txt',
utf32BE('BOM_OK UTF-32BE'),
'BOM_OK UTF-32BE',
);
});
it('Can describe a PNG file', async () => {
const imagePath = resolve(
process.cwd(),
'docs/assets/gemini-screenshot.png',
);
const imageContent = readFileSync(imagePath);
const filename = 'gemini-screenshot.png';
writeFileSync(join(dir, filename), imageContent);
const prompt = `What is shown in the image ${filename}?`;
const output = await rig.run(prompt);
await rig.waitForToolCall('read_file');
const lower = output.toLowerCase();
// The response is non-deterministic, so we just check for some
// keywords that are very likely to be in the response.
expect(lower.includes('gemini')).toBeTruthy();
});
});

8
integration-tests/vitest.config.ts
View File

@@ -17,6 +17,12 @@ export default defineConfig({
include: ['**/*.test.ts'],
exclude: ['**/terminal-bench/*.test.ts', '**/node_modules/**'],
retry: 2,
fileParallelism: false,
fileParallelism: true,
poolOptions: {
threads: {
minThreads: 2,
maxThreads: 4,
},
},
},
});