abbyswag/qwen-code
1
0
Fork 0
mirror of https://github.com/QwenLM/qwen-code.git synced 2025-12-21 01:07:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge tag 'v0.1.15' into feature/yiheng/sync-gemini-cli-0.1.15

Browse Source
This commit is contained in:
奕桁
2025-08-01 23:06:11 +08:00
parent 4db2fc9ed6 ed1483b06f
commit b69b2ce376
340 changed files with 36528 additions and 22931 deletions
Download Patch File Download Diff File Expand all files Collapse all files

286
packages/core/src/utils/bfsFileSearch.test.ts
View File

@@ -4,145 +4,189 @@
* SPDX-License-Identifier: Apache-2.0
*/
import * as fs from 'fs';
import { vi, describe, it, expect, beforeEach } from 'vitest';
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import * as fsPromises from 'fs/promises';
import * as gitUtils from './gitUtils.js';
import * as path from 'path';
import * as os from 'os';
import { bfsFileSearch } from './bfsFileSearch.js';
import { FileDiscoveryService } from '../services/fileDiscoveryService.js';
vi.mock('fs');
vi.mock('fs/promises');
vi.mock('./gitUtils.js');
const createMockDirent = (name: string, isFile: boolean): fs.Dirent => {
const dirent = new fs.Dirent();
dirent.name = name;
dirent.isFile = () => isFile;
dirent.isDirectory = () => !isFile;
return dirent;
};
// Type for the specific overload we're using
type ReaddirWithFileTypes = (
path: fs.PathLike,
options: { withFileTypes: true },
) => Promise<fs.Dirent[]>;
describe('bfsFileSearch', () => {
beforeEach(() => {
vi.resetAllMocks();
let testRootDir: string;
async function createEmptyDir(...pathSegments: string[]) {
const fullPath = path.join(testRootDir, ...pathSegments);
await fsPromises.mkdir(fullPath, { recursive: true });
return fullPath;
}
async function createTestFile(content: string, ...pathSegments: string[]) {
const fullPath = path.join(testRootDir, ...pathSegments);
await fsPromises.mkdir(path.dirname(fullPath), { recursive: true });
await fsPromises.writeFile(fullPath, content);
return fullPath;
}
beforeEach(async () => {
testRootDir = await fsPromises.mkdtemp(
path.join(os.tmpdir(), 'bfs-file-search-test-'),
);
});
afterEach(async () => {
await fsPromises.rm(testRootDir, { recursive: true, force: true });
});
it('should find a file in the root directory', async () => {
const mockFs = vi.mocked(fsPromises);
const mockReaddir = mockFs.readdir as unknown as ReaddirWithFileTypes;
vi.mocked(mockReaddir).mockResolvedValue([
createMockDirent('file1.txt', true),
createMockDirent('file2.txt', true),
]);
const result = await bfsFileSearch('/test', { fileName: 'file1.txt' });
expect(result).toEqual(['/test/file1.txt']);
const targetFilePath = await createTestFile('content', 'target.txt');
const result = await bfsFileSearch(testRootDir, { fileName: 'target.txt' });
expect(result).toEqual([targetFilePath]);
});
it('should find a file in a subdirectory', async () => {
const mockFs = vi.mocked(fsPromises);
const mockReaddir = mockFs.readdir as unknown as ReaddirWithFileTypes;
vi.mocked(mockReaddir).mockImplementation(async (dir) => {
if (dir === '/test') {
return [createMockDirent('subdir', false)];
}
if (dir === '/test/subdir') {
return [createMockDirent('file1.txt', true)];
}
return [];
});
const result = await bfsFileSearch('/test', { fileName: 'file1.txt' });
expect(result).toEqual(['/test/subdir/file1.txt']);
it('should find a file in a nested directory', async () => {
const targetFilePath = await createTestFile(
'content',
'a',
'b',
'target.txt',
);
const result = await bfsFileSearch(testRootDir, { fileName: 'target.txt' });
expect(result).toEqual([targetFilePath]);
});
it('should ignore specified directories', async () => {
const mockFs = vi.mocked(fsPromises);
const mockReaddir = mockFs.readdir as unknown as ReaddirWithFileTypes;
vi.mocked(mockReaddir).mockImplementation(async (dir) => {
if (dir === '/test') {
return [
createMockDirent('subdir1', false),
createMockDirent('subdir2', false),
];
}
if (dir === '/test/subdir1') {
return [createMockDirent('file1.txt', true)];
}
if (dir === '/test/subdir2') {
return [createMockDirent('file1.txt', true)];
}
return [];
});
const result = await bfsFileSearch('/test', {
fileName: 'file1.txt',
ignoreDirs: ['subdir2'],
});
expect(result).toEqual(['/test/subdir1/file1.txt']);
it('should find multiple files with the same name', async () => {
const targetFilePath1 = await createTestFile('content1', 'a', 'target.txt');
const targetFilePath2 = await createTestFile('content2', 'b', 'target.txt');
const result = await bfsFileSearch(testRootDir, { fileName: 'target.txt' });
result.sort();
expect(result).toEqual([targetFilePath1, targetFilePath2].sort());
});
it('should respect maxDirs limit', async () => {
const mockFs = vi.mocked(fsPromises);
const mockReaddir = mockFs.readdir as unknown as ReaddirWithFileTypes;
vi.mocked(mockReaddir).mockImplementation(async (dir) => {
if (dir === '/test') {
return [
createMockDirent('subdir1', false),
createMockDirent('subdir2', false),
];
}
if (dir === '/test/subdir1') {
return [createMockDirent('file1.txt', true)];
}
if (dir === '/test/subdir2') {
return [createMockDirent('file1.txt', true)];
}
return [];
});
const result = await bfsFileSearch('/test', {
fileName: 'file1.txt',
maxDirs: 2,
});
expect(result).toEqual(['/test/subdir1/file1.txt']);
it('should return an empty array if no file is found', async () => {
await createTestFile('content', 'other.txt');
const result = await bfsFileSearch(testRootDir, { fileName: 'target.txt' });
expect(result).toEqual([]);
});
it('should respect .gitignore files', async () => {
const mockFs = vi.mocked(fsPromises);
const mockGitUtils = vi.mocked(gitUtils);
mockGitUtils.isGitRepository.mockReturnValue(true);
const mockReaddir = mockFs.readdir as unknown as ReaddirWithFileTypes;
vi.mocked(mockReaddir).mockImplementation(async (dir) => {
if (dir === '/test') {
return [
createMockDirent('.gitignore', true),
createMockDirent('subdir1', false),
createMockDirent('subdir2', false),
];
}
if (dir === '/test/subdir1') {
return [createMockDirent('file1.txt', true)];
}
if (dir === '/test/subdir2') {
return [createMockDirent('file1.txt', true)];
}
return [];
it('should ignore directories specified in ignoreDirs', async () => {
await createTestFile('content', 'ignored', 'target.txt');
const targetFilePath = await createTestFile(
'content',
'not-ignored',
'target.txt',
);
const result = await bfsFileSearch(testRootDir, {
fileName: 'target.txt',
ignoreDirs: ['ignored'],
});
vi.mocked(fs).readFileSync.mockReturnValue('subdir2');
expect(result).toEqual([targetFilePath]);
});
const fileService = new FileDiscoveryService('/test');
const result = await bfsFileSearch('/test', {
fileName: 'file1.txt',
fileService,
it('should respect the maxDirs limit and not find the file', async () => {
await createTestFile('content', 'a', 'b', 'c', 'target.txt');
const result = await bfsFileSearch(testRootDir, {
fileName: 'target.txt',
maxDirs: 3,
});
expect(result).toEqual([]);
});
it('should respect the maxDirs limit and find the file', async () => {
const targetFilePath = await createTestFile(
'content',
'a',
'b',
'c',
'target.txt',
);
const result = await bfsFileSearch(testRootDir, {
fileName: 'target.txt',
maxDirs: 4,
});
expect(result).toEqual([targetFilePath]);
});
describe('with FileDiscoveryService', () => {
let projectRoot: string;
beforeEach(async () => {
projectRoot = await createEmptyDir('project');
});
it('should ignore gitignored files', async () => {
await createEmptyDir('project', '.git');
await createTestFile('node_modules/', 'project', '.gitignore');
await createTestFile('content', 'project', 'node_modules', 'target.txt');
const targetFilePath = await createTestFile(
'content',
'project',
'not-ignored',
'target.txt',
);
const fileService = new FileDiscoveryService(projectRoot);
const result = await bfsFileSearch(projectRoot, {
fileName: 'target.txt',
fileService,
fileFilteringOptions: {
respectGitIgnore: true,
respectGeminiIgnore: true,
},
});
expect(result).toEqual([targetFilePath]);
});
it('should ignore geminiignored files', async () => {
await createTestFile('node_modules/', 'project', '.geminiignore');
await createTestFile('content', 'project', 'node_modules', 'target.txt');
const targetFilePath = await createTestFile(
'content',
'project',
'not-ignored',
'target.txt',
);
const fileService = new FileDiscoveryService(projectRoot);
const result = await bfsFileSearch(projectRoot, {
fileName: 'target.txt',
fileService,
fileFilteringOptions: {
respectGitIgnore: false,
respectGeminiIgnore: true,
},
});
expect(result).toEqual([targetFilePath]);
});
it('should not ignore files if respect flags are false', async () => {
await createEmptyDir('project', '.git');
await createTestFile('node_modules/', 'project', '.gitignore');
const target1 = await createTestFile(
'content',
'project',
'node_modules',
'target.txt',
);
const target2 = await createTestFile(
'content',
'project',
'not-ignored',
'target.txt',
);
const fileService = new FileDiscoveryService(projectRoot);
const result = await bfsFileSearch(projectRoot, {
fileName: 'target.txt',
fileService,
fileFilteringOptions: {
respectGitIgnore: false,
respectGeminiIgnore: false,
},
});
expect(result.sort()).toEqual([target1, target2].sort());
});
expect(result).toEqual(['/test/subdir1/file1.txt']);
});
});

11
packages/core/src/utils/bfsFileSearch.ts
View File

@@ -8,7 +8,7 @@ import * as fs from 'fs/promises';
import * as path from 'path';
import { Dirent } from 'fs';
import { FileDiscoveryService } from '../services/fileDiscoveryService.js';
import { FileFilteringOptions } from '../config/config.js';
// Simple console logger for now.
// TODO: Integrate with a more robust server-side logger.
const logger = {
@@ -22,6 +22,7 @@ interface BfsFileSearchOptions {
maxDirs?: number;
debug?: boolean;
fileService?: FileDiscoveryService;
fileFilteringOptions?: FileFilteringOptions;
}
/**
@@ -69,7 +70,13 @@ export async function bfsFileSearch(
for (const entry of entries) {
const fullPath = path.join(currentDir, entry.name);
if (fileService?.shouldGitIgnoreFile(fullPath)) {
if (
fileService?.shouldIgnoreFile(fullPath, {
respectGitIgnore: options.fileFilteringOptions?.respectGitIgnore,
respectGeminiIgnore:
options.fileFilteringOptions?.respectGeminiIgnore,
})
) {
continue;
}

53
packages/core/src/utils/browser.ts Normal file
View File

@@ -0,0 +1,53 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
/**
* Determines if we should attempt to launch a browser for authentication
* based on the user's environment.
*
* This is an adaptation of the logic from the Google Cloud SDK.
* @returns True if the tool should attempt to launch a browser.
*/
export function shouldAttemptBrowserLaunch(): boolean {
// A list of browser names that indicate we should not attempt to open a
// web browser for the user.
const browserBlocklist = ['www-browser'];
const browserEnv = process.env.BROWSER;
if (browserEnv && browserBlocklist.includes(browserEnv)) {
return false;
}
// Common environment variables used in CI/CD or other non-interactive shells.
if (process.env.CI || process.env.DEBIAN_FRONTEND === 'noninteractive') {
return false;
}
// The presence of SSH_CONNECTION indicates a remote session.
// We should not attempt to launch a browser unless a display is explicitly available
// (checked below for Linux).
const isSSH = !!process.env.SSH_CONNECTION;
// On Linux, the presence of a display server is a strong indicator of a GUI.
if (process.platform === 'linux') {
// These are environment variables that can indicate a running compositor on
// Linux.
const displayVariables = ['DISPLAY', 'WAYLAND_DISPLAY', 'MIR_SOCKET'];
const hasDisplay = displayVariables.some((v) => !!process.env[v]);
if (!hasDisplay) {
return false;
}
}
// If in an SSH session on a non-Linux OS (e.g., macOS), don't launch browser.
// The Linux case is handled above (it's allowed if DISPLAY is set).
if (isSSH && process.platform !== 'linux') {
return false;
}
// For non-Linux OSes, we generally assume a GUI is available
// unless other signals (like SSH) suggest otherwise.
// The `open` command's error handling will catch final edge cases.
return true;
}

2
packages/core/src/utils/editCorrector.test.ts
View File

@@ -81,7 +81,7 @@ describe('editCorrector', () => {
it('should correctly count occurrences when substring is longer', () => {
expect(countOccurrences('abc', 'abcdef')).toBe(0);
});
it('should be case sensitive', () => {
it('should be case-sensitive', () => {
expect(countOccurrences('abcABC', 'a')).toBe(1);
expect(countOccurrences('abcABC', 'A')).toBe(1);
});

8
packages/core/src/utils/editCorrector.ts
View File

@@ -77,10 +77,10 @@ function getTimestampFromFunctionId(fcnId: string): number {
/**
* Will look through the gemini client history and determine when the most recent
* edit to a target file occured. If no edit happened, it will return -1
* edit to a target file occurred. If no edit happened, it will return -1
* @param filePath the path to the file
* @param client the geminiClient, so that we can get the history
* @returns a DateTime (as a number) of when the last edit occured, or -1 if no edit was found.
* @returns a DateTime (as a number) of when the last edit occurred, or -1 if no edit was found.
*/
async function findLastEditTimestamp(
filePath: string,
@@ -132,8 +132,8 @@ async function findLastEditTimestamp(
// Use the "blunt hammer" approach to find the file path in the content.
// Note that the tool response data is inconsistent in their formatting
// with successes and errors - so, we just check for the existance
// as the best guess to if error/failed occured with the response.
// with successes and errors - so, we just check for the existence
// as the best guess to if error/failed occurred with the response.
const stringified = JSON.stringify(content);
if (
!stringified.includes('Error') && // only applicable for functionResponse

191
packages/core/src/utils/editor.test.ts
View File

@@ -52,56 +52,99 @@ describe('editor utils', () => {
describe('checkHasEditorType', () => {
const testCases: Array<{
editor: EditorType;
command: string;
win32Command: string;
commands: string[];
win32Commands: string[];
}> = [
{ editor: 'vscode', command: 'code', win32Command: 'code.cmd' },
{ editor: 'vscodium', command: 'codium', win32Command: 'codium.cmd' },
{ editor: 'windsurf', command: 'windsurf', win32Command: 'windsurf' },
{ editor: 'cursor', command: 'cursor', win32Command: 'cursor' },
{ editor: 'vim', command: 'vim', win32Command: 'vim' },
{ editor: 'neovim', command: 'nvim', win32Command: 'nvim' },
{ editor: 'zed', command: 'zed', win32Command: 'zed' },
{ editor: 'vscode', commands: ['code'], win32Commands: ['code.cmd'] },
{
editor: 'vscodium',
commands: ['codium'],
win32Commands: ['codium.cmd'],
},
{
editor: 'windsurf',
commands: ['windsurf'],
win32Commands: ['windsurf'],
},
{ editor: 'cursor', commands: ['cursor'], win32Commands: ['cursor'] },
{ editor: 'vim', commands: ['vim'], win32Commands: ['vim'] },
{ editor: 'neovim', commands: ['nvim'], win32Commands: ['nvim'] },
{ editor: 'zed', commands: ['zed', 'zeditor'], win32Commands: ['zed'] },
];
for (const { editor, command, win32Command } of testCases) {
for (const { editor, commands, win32Commands } of testCases) {
describe(`${editor}`, () => {
it(`should return true if "${command}" command exists on non-windows`, () => {
// Non-windows tests
it(`should return true if first command "${commands[0]}" exists on non-windows`, () => {
Object.defineProperty(process, 'platform', { value: 'linux' });
(execSync as Mock).mockReturnValue(
Buffer.from(`/usr/bin/${command}`),
Buffer.from(`/usr/bin/${commands[0]}`),
);
expect(checkHasEditorType(editor)).toBe(true);
expect(execSync).toHaveBeenCalledWith(`command -v ${command}`, {
expect(execSync).toHaveBeenCalledWith(`command -v ${commands[0]}`, {
stdio: 'ignore',
});
});
it(`should return false if "${command}" command does not exist on non-windows`, () => {
if (commands.length > 1) {
it(`should return true if first command doesn't exist but second command "${commands[1]}" exists on non-windows`, () => {
Object.defineProperty(process, 'platform', { value: 'linux' });
(execSync as Mock)
.mockImplementationOnce(() => {
throw new Error(); // first command not found
})
.mockReturnValueOnce(Buffer.from(`/usr/bin/${commands[1]}`)); // second command found
expect(checkHasEditorType(editor)).toBe(true);
expect(execSync).toHaveBeenCalledTimes(2);
});
}
it(`should return false if none of the commands exist on non-windows`, () => {
Object.defineProperty(process, 'platform', { value: 'linux' });
(execSync as Mock).mockImplementation(() => {
throw new Error();
throw new Error(); // all commands not found
});
expect(checkHasEditorType(editor)).toBe(false);
expect(execSync).toHaveBeenCalledTimes(commands.length);
});
it(`should return true if "${win32Command}" command exists on windows`, () => {
// Windows tests
it(`should return true if first command "${win32Commands[0]}" exists on windows`, () => {
Object.defineProperty(process, 'platform', { value: 'win32' });
(execSync as Mock).mockReturnValue(
Buffer.from(`C:\\Program Files\\...\\${win32Command}`),
Buffer.from(`C:\\Program Files\\...\\${win32Commands[0]}`),
);
expect(checkHasEditorType(editor)).toBe(true);
expect(execSync).toHaveBeenCalledWith(`where.exe ${win32Command}`, {
stdio: 'ignore',
});
expect(execSync).toHaveBeenCalledWith(
`where.exe ${win32Commands[0]}`,
{
stdio: 'ignore',
},
);
});
it(`should return false if "${win32Command}" command does not exist on windows`, () => {
if (win32Commands.length > 1) {
it(`should return true if first command doesn't exist but second command "${win32Commands[1]}" exists on windows`, () => {
Object.defineProperty(process, 'platform', { value: 'win32' });
(execSync as Mock)
.mockImplementationOnce(() => {
throw new Error(); // first command not found
})
.mockReturnValueOnce(
Buffer.from(`C:\\Program Files\\...\\${win32Commands[1]}`),
); // second command found
expect(checkHasEditorType(editor)).toBe(true);
expect(execSync).toHaveBeenCalledTimes(2);
});
}
it(`should return false if none of the commands exist on windows`, () => {
Object.defineProperty(process, 'platform', { value: 'win32' });
(execSync as Mock).mockImplementation(() => {
throw new Error();
throw new Error(); // all commands not found
});
expect(checkHasEditorType(editor)).toBe(false);
expect(execSync).toHaveBeenCalledTimes(win32Commands.length);
});
});
}
@@ -110,31 +153,109 @@ describe('editor utils', () => {
describe('getDiffCommand', () => {
const guiEditors: Array<{
editor: EditorType;
command: string;
win32Command: string;
commands: string[];
win32Commands: string[];
}> = [
{ editor: 'vscode', command: 'code', win32Command: 'code.cmd' },
{ editor: 'vscodium', command: 'codium', win32Command: 'codium.cmd' },
{ editor: 'windsurf', command: 'windsurf', win32Command: 'windsurf' },
{ editor: 'cursor', command: 'cursor', win32Command: 'cursor' },
{ editor: 'zed', command: 'zed', win32Command: 'zed' },
{ editor: 'vscode', commands: ['code'], win32Commands: ['code.cmd'] },
{
editor: 'vscodium',
commands: ['codium'],
win32Commands: ['codium.cmd'],
},
{
editor: 'windsurf',
commands: ['windsurf'],
win32Commands: ['windsurf'],
},
{ editor: 'cursor', commands: ['cursor'], win32Commands: ['cursor'] },
{ editor: 'zed', commands: ['zed', 'zeditor'], win32Commands: ['zed'] },
];
for (const { editor, command, win32Command } of guiEditors) {
it(`should return the correct command for ${editor} on non-windows`, () => {
for (const { editor, commands, win32Commands } of guiEditors) {
// Non-windows tests
it(`should use first command "${commands[0]}" when it exists on non-windows`, () => {
Object.defineProperty(process, 'platform', { value: 'linux' });
(execSync as Mock).mockReturnValue(
Buffer.from(`/usr/bin/${commands[0]}`),
);
const diffCommand = getDiffCommand('old.txt', 'new.txt', editor);
expect(diffCommand).toEqual({
command,
command: commands[0],
args: ['--wait', '--diff', 'old.txt', 'new.txt'],
});
});
it(`should return the correct command for ${editor} on windows`, () => {
Object.defineProperty(process, 'platform', { value: 'win32' });
if (commands.length > 1) {
it(`should use second command "${commands[1]}" when first doesn't exist on non-windows`, () => {
Object.defineProperty(process, 'platform', { value: 'linux' });
(execSync as Mock)
.mockImplementationOnce(() => {
throw new Error(); // first command not found
})
.mockReturnValueOnce(Buffer.from(`/usr/bin/${commands[1]}`)); // second command found
const diffCommand = getDiffCommand('old.txt', 'new.txt', editor);
expect(diffCommand).toEqual({
command: commands[1],
args: ['--wait', '--diff', 'old.txt', 'new.txt'],
});
});
}
it(`should fall back to last command "${commands[commands.length - 1]}" when none exist on non-windows`, () => {
Object.defineProperty(process, 'platform', { value: 'linux' });
(execSync as Mock).mockImplementation(() => {
throw new Error(); // all commands not found
});
const diffCommand = getDiffCommand('old.txt', 'new.txt', editor);
expect(diffCommand).toEqual({
command: win32Command,
command: commands[commands.length - 1],
args: ['--wait', '--diff', 'old.txt', 'new.txt'],
});
});
// Windows tests
it(`should use first command "${win32Commands[0]}" when it exists on windows`, () => {
Object.defineProperty(process, 'platform', { value: 'win32' });
(execSync as Mock).mockReturnValue(
Buffer.from(`C:\\Program Files\\...\\${win32Commands[0]}`),
);
const diffCommand = getDiffCommand('old.txt', 'new.txt', editor);
expect(diffCommand).toEqual({
command: win32Commands[0],
args: ['--wait', '--diff', 'old.txt', 'new.txt'],
});
});
if (win32Commands.length > 1) {
it(`should use second command "${win32Commands[1]}" when first doesn't exist on windows`, () => {
Object.defineProperty(process, 'platform', { value: 'win32' });
(execSync as Mock)
.mockImplementationOnce(() => {
throw new Error(); // first command not found
})
.mockReturnValueOnce(
Buffer.from(`C:\\Program Files\\...\\${win32Commands[1]}`),
); // second command found
const diffCommand = getDiffCommand('old.txt', 'new.txt', editor);
expect(diffCommand).toEqual({
command: win32Commands[1],
args: ['--wait', '--diff', 'old.txt', 'new.txt'],
});
});
}
it(`should fall back to last command "${win32Commands[win32Commands.length - 1]}" when none exist on windows`, () => {
Object.defineProperty(process, 'platform', { value: 'win32' });
(execSync as Mock).mockImplementation(() => {
throw new Error(); // all commands not found
});
const diffCommand = getDiffCommand('old.txt', 'new.txt', editor);
expect(diffCommand).toEqual({
command: win32Commands[win32Commands.length - 1],
args: ['--wait', '--diff', 'old.txt', 'new.txt'],
});
});

33
packages/core/src/utils/editor.ts
View File

@@ -44,21 +44,28 @@ function commandExists(cmd: string): boolean {
}
}
const editorCommands: Record<EditorType, { win32: string; default: string }> = {
vscode: { win32: 'code.cmd', default: 'code' },
vscodium: { win32: 'codium.cmd', default: 'codium' },
windsurf: { win32: 'windsurf', default: 'windsurf' },
cursor: { win32: 'cursor', default: 'cursor' },
vim: { win32: 'vim', default: 'vim' },
neovim: { win32: 'nvim', default: 'nvim' },
zed: { win32: 'zed', default: 'zed' },
/**
* Editor command configurations for different platforms.
* Each editor can have multiple possible command names, listed in order of preference.
*/
const editorCommands: Record<
EditorType,
{ win32: string[]; default: string[] }
> = {
vscode: { win32: ['code.cmd'], default: ['code'] },
vscodium: { win32: ['codium.cmd'], default: ['codium'] },
windsurf: { win32: ['windsurf'], default: ['windsurf'] },
cursor: { win32: ['cursor'], default: ['cursor'] },
vim: { win32: ['vim'], default: ['vim'] },
neovim: { win32: ['nvim'], default: ['nvim'] },
zed: { win32: ['zed'], default: ['zed', 'zeditor'] },
};
export function checkHasEditorType(editor: EditorType): boolean {
const commandConfig = editorCommands[editor];
const command =
const commands =
process.platform === 'win32' ? commandConfig.win32 : commandConfig.default;
return commandExists(command);
return commands.some((cmd) => commandExists(cmd));
}
export function allowEditorTypeInSandbox(editor: EditorType): boolean {
@@ -92,8 +99,12 @@ export function getDiffCommand(
return null;
}
const commandConfig = editorCommands[editor];
const command =
const commands =
process.platform === 'win32' ? commandConfig.win32 : commandConfig.default;
const command =
commands.slice(0, -1).find((cmd) => commandExists(cmd)) ||
commands[commands.length - 1];
switch (editor) {
case 'vscode':
case 'vscodium':

144
packages/core/src/utils/errorReporting.test.ts
View File

@@ -4,36 +4,36 @@
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect, vi, beforeEach, afterEach, Mock } from 'vitest';
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import fs from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import { reportError } from './errorReporting.js';
// Use a type alias for SpyInstance as it's not directly exported
type SpyInstance = ReturnType<typeof vi.spyOn>;
import { reportError } from './errorReporting.js';
import fs from 'node:fs/promises';
import os from 'node:os';
// Mock dependencies
vi.mock('node:fs/promises');
vi.mock('node:os');
describe('reportError', () => {
let consoleErrorSpy: SpyInstance;
const MOCK_TMP_DIR = '/tmp';
let testDir: string;
const MOCK_TIMESTAMP = '2025-01-01T00-00-00-000Z';
beforeEach(() => {
beforeEach(async () => {
// Create a temporary directory for logs
testDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gemini-report-test-'));
vi.resetAllMocks();
consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
(os.tmpdir as Mock).mockReturnValue(MOCK_TMP_DIR);
vi.spyOn(Date.prototype, 'toISOString').mockReturnValue(MOCK_TIMESTAMP);
});
afterEach(() => {
afterEach(async () => {
vi.restoreAllMocks();
// Clean up the temporary directory
await fs.rm(testDir, { recursive: true, force: true });
});
const getExpectedReportPath = (type: string) =>
`${MOCK_TMP_DIR}/gemini-client-error-${type}-${MOCK_TIMESTAMP}.json`;
path.join(testDir, `gemini-client-error-${type}-${MOCK_TIMESTAMP}.json`);
it('should generate a report and log the path', async () => {
const error = new Error('Test error');
@@ -43,22 +43,18 @@ describe('reportError', () => {
const type = 'test-type';
const expectedReportPath = getExpectedReportPath(type);
(fs.writeFile as Mock).mockResolvedValue(undefined);
await reportError(error, baseMessage, context, type, testDir);
await reportError(error, baseMessage, context, type);
// Verify the file was written
const reportContent = await fs.readFile(expectedReportPath, 'utf-8');
const parsedReport = JSON.parse(reportContent);
expect(os.tmpdir).toHaveBeenCalledTimes(1);
expect(fs.writeFile).toHaveBeenCalledWith(
expectedReportPath,
JSON.stringify(
{
error: { message: 'Test error', stack: error.stack },
context,
},
null,
2,
),
);
expect(parsedReport).toEqual({
error: { message: 'Test error', stack: 'Test stack' },
context,
});
// Verify the console log
expect(consoleErrorSpy).toHaveBeenCalledWith(
`${baseMessage} Full report available at: ${expectedReportPath}`,
);
@@ -70,19 +66,15 @@ describe('reportError', () => {
const type = 'general';
const expectedReportPath = getExpectedReportPath(type);
(fs.writeFile as Mock).mockResolvedValue(undefined);
await reportError(error, baseMessage);
await reportError(error, baseMessage, undefined, type, testDir);
const reportContent = await fs.readFile(expectedReportPath, 'utf-8');
const parsedReport = JSON.parse(reportContent);
expect(parsedReport).toEqual({
error: { message: 'Test plain object error' },
});
expect(fs.writeFile).toHaveBeenCalledWith(
expectedReportPath,
JSON.stringify(
{
error: { message: 'Test plain object error' },
},
null,
2,
),
);
expect(consoleErrorSpy).toHaveBeenCalledWith(
`${baseMessage} Full report available at: ${expectedReportPath}`,
);
@@ -94,19 +86,15 @@ describe('reportError', () => {
const type = 'general';
const expectedReportPath = getExpectedReportPath(type);
(fs.writeFile as Mock).mockResolvedValue(undefined);
await reportError(error, baseMessage);
await reportError(error, baseMessage, undefined, type, testDir);
const reportContent = await fs.readFile(expectedReportPath, 'utf-8');
const parsedReport = JSON.parse(reportContent);
expect(parsedReport).toEqual({
error: { message: 'Just a string error' },
});
expect(fs.writeFile).toHaveBeenCalledWith(
expectedReportPath,
JSON.stringify(
{
error: { message: 'Just a string error' },
},
null,
2,
),
);
expect(consoleErrorSpy).toHaveBeenCalledWith(
`${baseMessage} Full report available at: ${expectedReportPath}`,
);
@@ -115,22 +103,15 @@ describe('reportError', () => {
it('should log fallback message if writing report fails', async () => {
const error = new Error('Main error');
const baseMessage = 'Failed operation.';
const writeError = new Error('Failed to write file');
const context = ['some context'];
const type = 'general';
const expectedReportPath = getExpectedReportPath(type);
const nonExistentDir = path.join(testDir, 'non-existent-dir');
(fs.writeFile as Mock).mockRejectedValue(writeError);
await reportError(error, baseMessage, context, type, nonExistentDir);
await reportError(error, baseMessage, context, type);
expect(fs.writeFile).toHaveBeenCalledWith(
expectedReportPath,
expect.any(String),
); // It still tries to write
expect(consoleErrorSpy).toHaveBeenCalledWith(
`${baseMessage} Additionally, failed to write detailed error report:`,
writeError,
expect.any(Error), // The actual write error
);
expect(consoleErrorSpy).toHaveBeenCalledWith(
'Original error that triggered report generation:',
@@ -163,9 +144,7 @@ describe('reportError', () => {
return originalJsonStringify(value, replacer, space);
});
(fs.writeFile as Mock).mockResolvedValue(undefined); // Mock for the minimal report write
await reportError(error, baseMessage, context, type);
await reportError(error, baseMessage, context, type, testDir);
expect(consoleErrorSpy).toHaveBeenCalledWith(
`${baseMessage} Could not stringify report content (likely due to context):`,
@@ -178,15 +157,14 @@ describe('reportError', () => {
expect(consoleErrorSpy).toHaveBeenCalledWith(
'Original context could not be stringified or included in report.',
);
// Check that it attempts to write a minimal report
expect(fs.writeFile).toHaveBeenCalledWith(
expectedMinimalReportPath,
originalJsonStringify(
{ error: { message: error.message, stack: error.stack } },
null,
2,
),
);
// Check that it writes a minimal report
const reportContent = await fs.readFile(expectedMinimalReportPath, 'utf-8');
const parsedReport = JSON.parse(reportContent);
expect(parsedReport).toEqual({
error: { message: error.message, stack: error.stack },
});
expect(consoleErrorSpy).toHaveBeenCalledWith(
`${baseMessage} Partial report (excluding context) available at: ${expectedMinimalReportPath}`,
);
@@ -199,19 +177,15 @@ describe('reportError', () => {
const type = 'general';
const expectedReportPath = getExpectedReportPath(type);
(fs.writeFile as Mock).mockResolvedValue(undefined);
await reportError(error, baseMessage, undefined, type);
await reportError(error, baseMessage, undefined, type, testDir);
const reportContent = await fs.readFile(expectedReportPath, 'utf-8');
const parsedReport = JSON.parse(reportContent);
expect(parsedReport).toEqual({
error: { message: 'Error without context', stack: 'No context stack' },
});
expect(fs.writeFile).toHaveBeenCalledWith(
expectedReportPath,
JSON.stringify(
{
error: { message: 'Error without context', stack: error.stack },
},
null,
2,
),
);
expect(consoleErrorSpy).toHaveBeenCalledWith(
`${baseMessage} Full report available at: ${expectedReportPath}`,
);

3
packages/core/src/utils/errorReporting.ts
View File

@@ -27,10 +27,11 @@ export async function reportError(
baseMessage: string,
context?: Content[] | Record<string, unknown> | unknown[],
type = 'general',
reportingDir = os.tmpdir(), // for testing
): Promise<void> {
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const reportFileName = `gemini-client-error-${type}-${timestamp}.json`;
const reportPath = path.join(os.tmpdir(), reportFileName);
const reportPath = path.join(reportingDir, reportFileName);
let errorToReport: { message: string; stack?: string };
if (error instanceof Error) {

13
packages/core/src/utils/errors.ts
View File

@@ -4,7 +4,11 @@
* SPDX-License-Identifier: Apache-2.0
*/
import { GaxiosError } from 'gaxios';
interface GaxiosError {
response?: {
data?: unknown;
};
}
export function isNodeError(error: unknown): error is NodeJS.ErrnoException {
return error instanceof Error && 'code' in error;
@@ -33,8 +37,9 @@ interface ResponseData {
}
export function toFriendlyError(error: unknown): unknown {
if (error instanceof GaxiosError) {
const data = parseResponseData(error);
if (error && typeof error === 'object' && 'response' in error) {
const gaxiosError = error as GaxiosError;
const data = parseResponseData(gaxiosError);
if (data.error && data.error.message && data.error.code) {
switch (data.error.code) {
case 400:
@@ -58,5 +63,5 @@ function parseResponseData(error: GaxiosError): ResponseData {
if (typeof error.response?.data === 'string') {
return JSON.parse(error.response?.data) as ResponseData;
}
return typeof error.response?.data as ResponseData;
return error.response?.data as ResponseData;
}

74
packages/core/src/utils/fileUtils.test.ts
View File

@@ -42,7 +42,7 @@ describe('fileUtils', () => {
let testImageFilePath: string;
let testPdfFilePath: string;
let testBinaryFilePath: string;
let nonExistentFilePath: string;
let nonexistentFilePath: string;
let directoryPath: string;
beforeEach(() => {
@@ -57,7 +57,7 @@ describe('fileUtils', () => {
testImageFilePath = path.join(tempRootDir, 'image.png');
testPdfFilePath = path.join(tempRootDir, 'document.pdf');
testBinaryFilePath = path.join(tempRootDir, 'app.exe');
nonExistentFilePath = path.join(tempRootDir, 'notfound.txt');
nonexistentFilePath = path.join(tempRootDir, 'nonexistent.txt');
directoryPath = path.join(tempRootDir, 'subdir');
actualNodeFs.mkdirSync(directoryPath, { recursive: true }); // Ensure subdir exists
@@ -142,41 +142,41 @@ describe('fileUtils', () => {
}
});
it('should return false for an empty file', () => {
it('should return false for an empty file', async () => {
actualNodeFs.writeFileSync(filePathForBinaryTest, '');
expect(isBinaryFile(filePathForBinaryTest)).toBe(false);
expect(await isBinaryFile(filePathForBinaryTest)).toBe(false);
});
it('should return false for a typical text file', () => {
it('should return false for a typical text file', async () => {
actualNodeFs.writeFileSync(
filePathForBinaryTest,
'Hello, world!\nThis is a test file with normal text content.',
);
expect(isBinaryFile(filePathForBinaryTest)).toBe(false);
expect(await isBinaryFile(filePathForBinaryTest)).toBe(false);
});
it('should return true for a file with many null bytes', () => {
it('should return true for a file with many null bytes', async () => {
const binaryContent = Buffer.from([
0x48, 0x65, 0x00, 0x6c, 0x6f, 0x00, 0x00, 0x00, 0x00, 0x00,
]); // "He\0llo\0\0\0\0\0"
actualNodeFs.writeFileSync(filePathForBinaryTest, binaryContent);
expect(isBinaryFile(filePathForBinaryTest)).toBe(true);
expect(await isBinaryFile(filePathForBinaryTest)).toBe(true);
});
it('should return true for a file with high percentage of non-printable ASCII', () => {
it('should return true for a file with high percentage of non-printable ASCII', async () => {
const binaryContent = Buffer.from([
0x41, 0x42, 0x01, 0x02, 0x03, 0x04, 0x05, 0x43, 0x44, 0x06,
]); // AB\x01\x02\x03\x04\x05CD\x06
actualNodeFs.writeFileSync(filePathForBinaryTest, binaryContent);
expect(isBinaryFile(filePathForBinaryTest)).toBe(true);
expect(await isBinaryFile(filePathForBinaryTest)).toBe(true);
});
it('should return false if file access fails (e.g., ENOENT)', () => {
it('should return false if file access fails (e.g., ENOENT)', async () => {
// Ensure the file does not exist
if (actualNodeFs.existsSync(filePathForBinaryTest)) {
actualNodeFs.unlinkSync(filePathForBinaryTest);
}
expect(isBinaryFile(filePathForBinaryTest)).toBe(false);
expect(await isBinaryFile(filePathForBinaryTest)).toBe(false);
});
});
@@ -196,64 +196,64 @@ describe('fileUtils', () => {
vi.restoreAllMocks(); // Restore spies on actualNodeFs
});
it('should detect typescript type by extension (ts)', () => {
expect(detectFileType('file.ts')).toBe('text');
expect(detectFileType('file.test.ts')).toBe('text');
it('should detect typescript type by extension (ts)', async () => {
expect(await detectFileType('file.ts')).toBe('text');
expect(await detectFileType('file.test.ts')).toBe('text');
});
it('should detect image type by extension (png)', () => {
it('should detect image type by extension (png)', async () => {
mockMimeLookup.mockReturnValueOnce('image/png');
expect(detectFileType('file.png')).toBe('image');
expect(await detectFileType('file.png')).toBe('image');
});
it('should detect image type by extension (jpeg)', () => {
it('should detect image type by extension (jpeg)', async () => {
mockMimeLookup.mockReturnValueOnce('image/jpeg');
expect(detectFileType('file.jpg')).toBe('image');
expect(await detectFileType('file.jpg')).toBe('image');
});
it('should detect svg type by extension', () => {
expect(detectFileType('image.svg')).toBe('svg');
expect(detectFileType('image.icon.svg')).toBe('svg');
it('should detect svg type by extension', async () => {
expect(await detectFileType('image.svg')).toBe('svg');
expect(await detectFileType('image.icon.svg')).toBe('svg');
});
it('should detect pdf type by extension', () => {
it('should detect pdf type by extension', async () => {
mockMimeLookup.mockReturnValueOnce('application/pdf');
expect(detectFileType('file.pdf')).toBe('pdf');
expect(await detectFileType('file.pdf')).toBe('pdf');
});
it('should detect audio type by extension', () => {
it('should detect audio type by extension', async () => {
mockMimeLookup.mockReturnValueOnce('audio/mpeg');
expect(detectFileType('song.mp3')).toBe('audio');
expect(await detectFileType('song.mp3')).toBe('audio');
});
it('should detect video type by extension', () => {
it('should detect video type by extension', async () => {
mockMimeLookup.mockReturnValueOnce('video/mp4');
expect(detectFileType('movie.mp4')).toBe('video');
expect(await detectFileType('movie.mp4')).toBe('video');
});
it('should detect known binary extensions as binary (e.g. .zip)', () => {
it('should detect known binary extensions as binary (e.g. .zip)', async () => {
mockMimeLookup.mockReturnValueOnce('application/zip');
expect(detectFileType('archive.zip')).toBe('binary');
expect(await detectFileType('archive.zip')).toBe('binary');
});
it('should detect known binary extensions as binary (e.g. .exe)', () => {
it('should detect known binary extensions as binary (e.g. .exe)', async () => {
mockMimeLookup.mockReturnValueOnce('application/octet-stream'); // Common for .exe
expect(detectFileType('app.exe')).toBe('binary');
expect(await detectFileType('app.exe')).toBe('binary');
});
it('should use isBinaryFile for unknown extensions and detect as binary', () => {
it('should use isBinaryFile for unknown extensions and detect as binary', async () => {
mockMimeLookup.mockReturnValueOnce(false); // Unknown mime type
// Create a file that isBinaryFile will identify as binary
const binaryContent = Buffer.from([
0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a,
]);
actualNodeFs.writeFileSync(filePathForDetectTest, binaryContent);
expect(detectFileType(filePathForDetectTest)).toBe('binary');
expect(await detectFileType(filePathForDetectTest)).toBe('binary');
});
it('should default to text if mime type is unknown and content is not binary', () => {
it('should default to text if mime type is unknown and content is not binary', async () => {
mockMimeLookup.mockReturnValueOnce(false); // Unknown mime type
// filePathForDetectTest is already a text file by default from beforeEach
expect(detectFileType(filePathForDetectTest)).toBe('text');
expect(await detectFileType(filePathForDetectTest)).toBe('text');
});
});
@@ -284,7 +284,7 @@ describe('fileUtils', () => {
it('should handle file not found', async () => {
const result = await processSingleFileContent(
nonExistentFilePath,
nonexistentFilePath,
tempRootDir,
);
expect(result.error).toContain('File not found');

53
packages/core/src/utils/fileUtils.ts
View File

@@ -4,8 +4,8 @@
* SPDX-License-Identifier: Apache-2.0
*/
import fs from 'fs';
import path from 'path';
import fs from 'node:fs';
import path from 'node:path';
import { PartUnion } from '@google/genai';
import mime from 'mime-types';
@@ -56,22 +56,24 @@ export function isWithinRoot(
/**
* Determines if a file is likely binary based on content sampling.
* @param filePath Path to the file.
* @returns True if the file appears to be binary.
* @returns Promise that resolves to true if the file appears to be binary.
*/
export function isBinaryFile(filePath: string): boolean {
export async function isBinaryFile(filePath: string): Promise<boolean> {
let fileHandle: fs.promises.FileHandle | undefined;
try {
const fd = fs.openSync(filePath, 'r');
fileHandle = await fs.promises.open(filePath, 'r');
// Read up to 4KB or file size, whichever is smaller
const fileSize = fs.fstatSync(fd).size;
const stats = await fileHandle.stat();
const fileSize = stats.size;
if (fileSize === 0) {
// Empty file is not considered binary for content checking
fs.closeSync(fd);
return false;
}
const bufferSize = Math.min(4096, fileSize);
const buffer = Buffer.alloc(bufferSize);
const bytesRead = fs.readSync(fd, buffer, 0, buffer.length, 0);
fs.closeSync(fd);
const result = await fileHandle.read(buffer, 0, buffer.length, 0);
const bytesRead = result.bytesRead;
if (bytesRead === 0) return false;
@@ -84,21 +86,40 @@ export function isBinaryFile(filePath: string): boolean {
}
// If >30% non-printable characters, consider it binary
return nonPrintableCount / bytesRead > 0.3;
} catch {
} catch (error) {
// Log error for debugging while maintaining existing behavior
console.warn(
`Failed to check if file is binary: ${filePath}`,
error instanceof Error ? error.message : String(error),
);
// If any error occurs (e.g. file not found, permissions),
// treat as not binary here; let higher-level functions handle existence/access errors.
return false;
} finally {
// Safely close the file handle if it was successfully opened
if (fileHandle) {
try {
await fileHandle.close();
} catch (closeError) {
// Log close errors for debugging while continuing with cleanup
console.warn(
`Failed to close file handle for: ${filePath}`,
closeError instanceof Error ? closeError.message : String(closeError),
);
// The important thing is that we attempted to clean up
}
}
}
}
/**
* Detects the type of file based on extension and content.
* @param filePath Path to the file.
* @returns 'text', 'image', 'pdf', 'audio', 'video', or 'binary'.
* @returns Promise that resolves to 'text', 'image', 'pdf', 'audio', 'video', 'binary' or 'svg'.
*/
export function detectFileType(
export async function detectFileType(
filePath: string,
): 'text' | 'image' | 'pdf' | 'audio' | 'video' | 'binary' | 'svg' {
): Promise<'text' | 'image' | 'pdf' | 'audio' | 'video' | 'binary' | 'svg'> {
const ext = path.extname(filePath).toLowerCase();
// The mimetype for "ts" is MPEG transport stream (a video format) but we want
@@ -164,9 +185,9 @@ export function detectFileType(
return 'binary';
}
// Fallback to content-based check if mime type wasn't conclusive for image/pdf
// Fall back to content-based check if mime type wasn't conclusive for image/pdf
// and it's not a known binary extension.
if (isBinaryFile(filePath)) {
if (await isBinaryFile(filePath)) {
return 'binary';
}
@@ -227,7 +248,7 @@ export async function processSingleFileContent(
);
}
const fileType = detectFileType(filePath);
const fileType = await detectFileType(filePath);
const relativePathForDisplay = path
.relative(rootDirectory, filePath)
.replace(/\\/g, '/');

16
packages/core/src/utils/formatters.ts Normal file
View File

@@ -0,0 +1,16 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
export const formatMemoryUsage = (bytes: number): string => {
const gb = bytes / (1024 * 1024 * 1024);
if (bytes < 1024 * 1024) {
return `${(bytes / 1024).toFixed(1)} KB`;
}
if (bytes < 1024 * 1024 * 1024) {
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
}
return `${gb.toFixed(2)} GB`;
};

422
packages/core/src/utils/getFolderStructure.test.ts
View File

@@ -4,341 +4,337 @@
* SPDX-License-Identifier: Apache-2.0
*/
/* eslint-disable @typescript-eslint/no-explicit-any */
import { describe, it, expect, vi, beforeEach, afterEach, Mock } from 'vitest';
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import fsPromises from 'fs/promises';
import * as fs from 'fs';
import { Dirent as FSDirent } from 'fs';
import * as nodePath from 'path';
import * as os from 'os';
import { getFolderStructure } from './getFolderStructure.js';
import * as gitUtils from './gitUtils.js';
import { FileDiscoveryService } from '../services/fileDiscoveryService.js';
vi.mock('path', async (importOriginal) => {
const original = (await importOriginal()) as typeof nodePath;
return {
...original,
resolve: vi.fn((str) => str),
// Other path functions (basename, join, normalize, etc.) will use original implementation
};
});
vi.mock('fs/promises');
vi.mock('fs');
vi.mock('./gitUtils.js');
// Import 'path' again here, it will be the mocked version
import * as path from 'path';
// Helper to create Dirent-like objects for mocking fs.readdir
const createDirent = (name: string, type: 'file' | 'dir'): FSDirent => ({
name,
isFile: () => type === 'file',
isDirectory: () => type === 'dir',
isBlockDevice: () => false,
isCharacterDevice: () => false,
isSymbolicLink: () => false,
isFIFO: () => false,
isSocket: () => false,
path: '',
parentPath: '',
});
describe('getFolderStructure', () => {
beforeEach(() => {
vi.resetAllMocks();
let testRootDir: string;
// path.resolve is now a vi.fn() due to the top-level vi.mock.
// We ensure its implementation is set for each test (or rely on the one from vi.mock).
// vi.resetAllMocks() clears call history but not the implementation set by vi.fn() in vi.mock.
// If we needed to change it per test, we would do it here:
(path.resolve as Mock).mockImplementation((str: string) => str);
async function createEmptyDir(...pathSegments: string[]) {
const fullPath = path.join(testRootDir, ...pathSegments);
await fsPromises.mkdir(fullPath, { recursive: true });
}
// Re-apply/define the mock implementation for fsPromises.readdir for each test
(fsPromises.readdir as Mock).mockImplementation(
async (dirPath: string | Buffer | URL) => {
// path.normalize here will use the mocked path module.
// Since normalize is spread from original, it should be the real one.
const normalizedPath = path.normalize(dirPath.toString());
if (mockFsStructure[normalizedPath]) {
return mockFsStructure[normalizedPath];
}
throw Object.assign(
new Error(
`ENOENT: no such file or directory, scandir '${normalizedPath}'`,
),
{ code: 'ENOENT' },
);
},
async function createTestFile(...pathSegments: string[]) {
const fullPath = path.join(testRootDir, ...pathSegments);
await fsPromises.mkdir(path.dirname(fullPath), { recursive: true });
await fsPromises.writeFile(fullPath, '');
return fullPath;
}
beforeEach(async () => {
testRootDir = await fsPromises.mkdtemp(
path.join(os.tmpdir(), 'folder-structure-test-'),
);
});
afterEach(() => {
vi.restoreAllMocks(); // Restores spies (like fsPromises.readdir) and resets vi.fn mocks (like path.resolve)
afterEach(async () => {
await fsPromises.rm(testRootDir, { recursive: true, force: true });
});
const mockFsStructure: Record<string, FSDirent[]> = {
'/testroot': [
createDirent('file1.txt', 'file'),
createDirent('subfolderA', 'dir'),
createDirent('emptyFolder', 'dir'),
createDirent('.hiddenfile', 'file'),
createDirent('node_modules', 'dir'),
],
'/testroot/subfolderA': [
createDirent('fileA1.ts', 'file'),
createDirent('fileA2.js', 'file'),
createDirent('subfolderB', 'dir'),
],
'/testroot/subfolderA/subfolderB': [createDirent('fileB1.md', 'file')],
'/testroot/emptyFolder': [],
'/testroot/node_modules': [createDirent('somepackage', 'dir')],
'/testroot/manyFilesFolder': Array.from({ length: 10 }, (_, i) =>
createDirent(`file-${i}.txt`, 'file'),
),
'/testroot/manyFolders': Array.from({ length: 5 }, (_, i) =>
createDirent(`folder-${i}`, 'dir'),
),
...Array.from({ length: 5 }, (_, i) => ({
[`/testroot/manyFolders/folder-${i}`]: [
createDirent('child.txt', 'file'),
],
})).reduce((acc, val) => ({ ...acc, ...val }), {}),
'/testroot/deepFolders': [createDirent('level1', 'dir')],
'/testroot/deepFolders/level1': [createDirent('level2', 'dir')],
'/testroot/deepFolders/level1/level2': [createDirent('level3', 'dir')],
'/testroot/deepFolders/level1/level2/level3': [
createDirent('file.txt', 'file'),
],
};
it('should return basic folder structure', async () => {
const structure = await getFolderStructure('/testroot/subfolderA');
const expected = `
Showing up to 20 items (files + folders).
await createTestFile('fileA1.ts');
await createTestFile('fileA2.js');
await createTestFile('subfolderB', 'fileB1.md');
/testroot/subfolderA/
const structure = await getFolderStructure(testRootDir);
expect(structure.trim()).toBe(
`
Showing up to 200 items (files + folders).
${testRootDir}${path.sep}
├───fileA1.ts
├───fileA2.js
└───subfolderB/
└───subfolderB${path.sep}
└───fileB1.md
`.trim();
expect(structure.trim()).toBe(expected);
`.trim(),
);
});
it('should handle an empty folder', async () => {
const structure = await getFolderStructure('/testroot/emptyFolder');
const expected = `
Showing up to 20 items (files + folders).
const structure = await getFolderStructure(testRootDir);
expect(structure.trim()).toBe(
`
Showing up to 200 items (files + folders).
/testroot/emptyFolder/
`.trim();
expect(structure.trim()).toBe(expected.trim());
${testRootDir}${path.sep}
`
.trim()
.trim(),
);
});
it('should ignore folders specified in ignoredFolders (default)', async () => {
const structure = await getFolderStructure('/testroot');
const expected = `
Showing up to 20 items (files + folders). Folders or files indicated with ... contain more items not shown, were ignored, or the display limit (20 items) was reached.
await createTestFile('.hiddenfile');
await createTestFile('file1.txt');
await createEmptyDir('emptyFolder');
await createTestFile('node_modules', 'somepackage', 'index.js');
await createTestFile('subfolderA', 'fileA1.ts');
await createTestFile('subfolderA', 'fileA2.js');
await createTestFile('subfolderA', 'subfolderB', 'fileB1.md');
/testroot/
const structure = await getFolderStructure(testRootDir);
expect(structure.trim()).toBe(
`
Showing up to 200 items (files + folders). Folders or files indicated with ... contain more items not shown, were ignored, or the display limit (200 items) was reached.
${testRootDir}${path.sep}
├───.hiddenfile
├───file1.txt
├───emptyFolder/
├───node_modules/...
└───subfolderA/
├───emptyFolder${path.sep}
├───node_modules${path.sep}...
└───subfolderA${path.sep}
├───fileA1.ts
├───fileA2.js
└───subfolderB/
└───subfolderB${path.sep}
└───fileB1.md
`.trim();
expect(structure.trim()).toBe(expected);
`.trim(),
);
});
it('should ignore folders specified in custom ignoredFolders', async () => {
const structure = await getFolderStructure('/testroot', {
await createTestFile('.hiddenfile');
await createTestFile('file1.txt');
await createEmptyDir('emptyFolder');
await createTestFile('node_modules', 'somepackage', 'index.js');
await createTestFile('subfolderA', 'fileA1.ts');
const structure = await getFolderStructure(testRootDir, {
ignoredFolders: new Set(['subfolderA', 'node_modules']),
});
const expected = `
Showing up to 20 items (files + folders). Folders or files indicated with ... contain more items not shown, were ignored, or the display limit (20 items) was reached.
Showing up to 200 items (files + folders). Folders or files indicated with ... contain more items not shown, were ignored, or the display limit (200 items) was reached.
/testroot/
${testRootDir}${path.sep}
├───.hiddenfile
├───file1.txt
├───emptyFolder/
├───node_modules/...
└───subfolderA/...
├───emptyFolder${path.sep}
├───node_modules${path.sep}...
└───subfolderA${path.sep}...
`.trim();
expect(structure.trim()).toBe(expected);
});
it('should filter files by fileIncludePattern', async () => {
const structure = await getFolderStructure('/testroot/subfolderA', {
await createTestFile('fileA1.ts');
await createTestFile('fileA2.js');
await createTestFile('subfolderB', 'fileB1.md');
const structure = await getFolderStructure(testRootDir, {
fileIncludePattern: /\.ts$/,
});
const expected = `
Showing up to 20 items (files + folders).
Showing up to 200 items (files + folders).
/testroot/subfolderA/
${testRootDir}${path.sep}
├───fileA1.ts
└───subfolderB/
└───subfolderB${path.sep}
`.trim();
expect(structure.trim()).toBe(expected);
});
it('should handle maxItems truncation for files within a folder', async () => {
const structure = await getFolderStructure('/testroot/subfolderA', {
await createTestFile('fileA1.ts');
await createTestFile('fileA2.js');
await createTestFile('subfolderB', 'fileB1.md');
const structure = await getFolderStructure(testRootDir, {
maxItems: 3,
});
const expected = `
Showing up to 3 items (files + folders).
/testroot/subfolderA/
${testRootDir}${path.sep}
├───fileA1.ts
├───fileA2.js
└───subfolderB/
└───subfolderB${path.sep}
`.trim();
expect(structure.trim()).toBe(expected);
});
it('should handle maxItems truncation for subfolders', async () => {
const structure = await getFolderStructure('/testroot/manyFolders', {
for (let i = 0; i < 5; i++) {
await createTestFile(`folder-${i}`, 'child.txt');
}
const structure = await getFolderStructure(testRootDir, {
maxItems: 4,
});
const expectedRevised = `
Showing up to 4 items (files + folders). Folders or files indicated with ... contain more items not shown, were ignored, or the display limit (4 items) was reached.
/testroot/manyFolders/
├───folder-0/
├───folder-1/
├───folder-2/
├───folder-3/
${testRootDir}${path.sep}
├───folder-0${path.sep}
├───folder-1${path.sep}
├───folder-2${path.sep}
├───folder-3${path.sep}
└───...
`.trim();
expect(structure.trim()).toBe(expectedRevised);
});
it('should handle maxItems that only allows the root folder itself', async () => {
const structure = await getFolderStructure('/testroot/subfolderA', {
await createTestFile('fileA1.ts');
await createTestFile('fileA2.ts');
await createTestFile('subfolderB', 'fileB1.ts');
const structure = await getFolderStructure(testRootDir, {
maxItems: 1,
});
const expectedRevisedMax1 = `
const expected = `
Showing up to 1 items (files + folders). Folders or files indicated with ... contain more items not shown, were ignored, or the display limit (1 items) was reached.
/testroot/subfolderA/
${testRootDir}${path.sep}
├───fileA1.ts
├───...
└───...
`.trim();
expect(structure.trim()).toBe(expectedRevisedMax1);
expect(structure.trim()).toBe(expected);
});
it('should handle non-existent directory', async () => {
// Temporarily make fsPromises.readdir throw ENOENT for this specific path
const originalReaddir = fsPromises.readdir;
(fsPromises.readdir as Mock).mockImplementation(
async (p: string | Buffer | URL) => {
if (p === '/nonexistent') {
throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' });
}
return originalReaddir(p);
},
);
const structure = await getFolderStructure('/nonexistent');
const nonExistentPath = path.join(testRootDir, 'non-existent');
const structure = await getFolderStructure(nonExistentPath);
expect(structure).toContain(
'Error: Could not read directory "/nonexistent"',
`Error: Could not read directory "${nonExistentPath}". Check path and permissions.`,
);
});
it('should handle deep folder structure within limits', async () => {
const structure = await getFolderStructure('/testroot/deepFolders', {
await createTestFile('level1', 'level2', 'level3', 'file.txt');
const structure = await getFolderStructure(testRootDir, {
maxItems: 10,
});
const expected = `
Showing up to 10 items (files + folders).
/testroot/deepFolders/
└───level1/
└───level2/
└───level3/
${testRootDir}${path.sep}
└───level1${path.sep}
└───level2${path.sep}
└───level3${path.sep}
└───file.txt
`.trim();
expect(structure.trim()).toBe(expected);
});
it('should truncate deep folder structure if maxItems is small', async () => {
const structure = await getFolderStructure('/testroot/deepFolders', {
await createTestFile('level1', 'level2', 'level3', 'file.txt');
const structure = await getFolderStructure(testRootDir, {
maxItems: 3,
});
const expected = `
Showing up to 3 items (files + folders).
/testroot/deepFolders/
└───level1/
└───level2/
└───level3/
${testRootDir}${path.sep}
└───level1${path.sep}
└───level2${path.sep}
└───level3${path.sep}
`.trim();
expect(structure.trim()).toBe(expected);
});
});
describe('getFolderStructure gitignore', () => {
beforeEach(() => {
vi.resetAllMocks();
(path.resolve as Mock).mockImplementation((str: string) => str);
(fsPromises.readdir as Mock).mockImplementation(async (p) => {
const path = p.toString();
if (path === '/test/project') {
return [
createDirent('file1.txt', 'file'),
createDirent('node_modules', 'dir'),
createDirent('ignored.txt', 'file'),
createDirent('.qwen', 'dir'),
] as any;
}
if (path === '/test/project/node_modules') {
return [createDirent('some-package', 'dir')] as any;
}
if (path === '/test/project/.gemini') {
return [
createDirent('config.yaml', 'file'),
createDirent('logs.json', 'file'),
] as any;
}
return [];
describe('with gitignore', () => {
beforeEach(async () => {
await fsPromises.mkdir(path.join(testRootDir, '.git'), {
recursive: true,
});
});
(fs.readFileSync as Mock).mockImplementation((p) => {
const path = p.toString();
if (path === '/test/project/.gitignore') {
return 'ignored.txt\nnode_modules/\n.qwen/\n!/.qwen/config.yaml';
}
return '';
it('should ignore files and folders specified in .gitignore', async () => {
await fsPromises.writeFile(
nodePath.join(testRootDir, '.gitignore'),
'ignored.txt\nnode_modules/\n.gemini/*\n!/.gemini/config.yaml',
);
await createTestFile('file1.txt');
await createTestFile('node_modules', 'some-package', 'index.js');
await createTestFile('ignored.txt');
await createTestFile('.gemini', 'config.yaml');
await createTestFile('.gemini', 'logs.json');
const fileService = new FileDiscoveryService(testRootDir);
const structure = await getFolderStructure(testRootDir, {
fileService,
});
expect(structure).not.toContain('ignored.txt');
expect(structure).toContain(`node_modules${path.sep}...`);
expect(structure).not.toContain('logs.json');
expect(structure).toContain('config.yaml');
expect(structure).toContain('file1.txt');
});
vi.mocked(gitUtils.isGitRepository).mockReturnValue(true);
it('should not ignore files if respectGitIgnore is false', async () => {
await fsPromises.writeFile(
nodePath.join(testRootDir, '.gitignore'),
'ignored.txt',
);
await createTestFile('file1.txt');
await createTestFile('ignored.txt');
const fileService = new FileDiscoveryService(testRootDir);
const structure = await getFolderStructure(testRootDir, {
fileService,
fileFilteringOptions: {
respectGeminiIgnore: false,
respectGitIgnore: false,
},
});
expect(structure).toContain('ignored.txt');
expect(structure).toContain('file1.txt');
});
});
it('should ignore files and folders specified in .gitignore', async () => {
const fileService = new FileDiscoveryService('/test/project');
const structure = await getFolderStructure('/test/project', {
fileService,
});
expect(structure).not.toContain('ignored.txt');
expect(structure).toContain('node_modules/...');
expect(structure).not.toContain('logs.json');
});
describe('with geminiignore', () => {
it('should ignore geminiignore files by default', async () => {
await fsPromises.writeFile(
nodePath.join(testRootDir, '.geminiignore'),
'ignored.txt\nnode_modules/\n.gemini/\n!/.gemini/config.yaml',
);
await createTestFile('file1.txt');
await createTestFile('node_modules', 'some-package', 'index.js');
await createTestFile('ignored.txt');
await createTestFile('.gemini', 'config.yaml');
await createTestFile('.gemini', 'logs.json');
it('should not ignore files if respectGitIgnore is false', async () => {
const fileService = new FileDiscoveryService('/test/project');
const structure = await getFolderStructure('/test/project', {
fileService,
respectGitIgnore: false,
const fileService = new FileDiscoveryService(testRootDir);
const structure = await getFolderStructure(testRootDir, {
fileService,
});
expect(structure).not.toContain('ignored.txt');
expect(structure).toContain(`node_modules${path.sep}...`);
expect(structure).not.toContain('logs.json');
});
it('should not ignore files if respectGeminiIgnore is false', async () => {
await fsPromises.writeFile(
nodePath.join(testRootDir, '.geminiignore'),
'ignored.txt\nnode_modules/\n.gemini/\n!/.gemini/config.yaml',
);
await createTestFile('file1.txt');
await createTestFile('node_modules', 'some-package', 'index.js');
await createTestFile('ignored.txt');
await createTestFile('.gemini', 'config.yaml');
await createTestFile('.gemini', 'logs.json');
const fileService = new FileDiscoveryService(testRootDir);
const structure = await getFolderStructure(testRootDir, {
fileService,
fileFilteringOptions: {
respectGeminiIgnore: false,
respectGitIgnore: true, // Explicitly disable gemini ignore only
},
});
expect(structure).toContain('ignored.txt');
// node_modules is still ignored by default
expect(structure).toContain(`node_modules${path.sep}...`);
});
expect(structure).toContain('ignored.txt');
// node_modules is still ignored by default
expect(structure).toContain('node_modules/...');
});
});

73
packages/core/src/utils/getFolderStructure.ts
View File

@@ -9,8 +9,10 @@ import { Dirent } from 'fs';
import * as path from 'path';
import { getErrorMessage, isNodeError } from './errors.js';
import { FileDiscoveryService } from '../services/fileDiscoveryService.js';
import { FileFilteringOptions } from '../config/config.js';
import { DEFAULT_FILE_FILTERING_OPTIONS } from '../config/config.js';
const MAX_ITEMS = 20;
const MAX_ITEMS = 200;
const TRUNCATION_INDICATOR = '...';
const DEFAULT_IGNORED_FOLDERS = new Set(['node_modules', '.git', 'dist']);
@@ -18,7 +20,7 @@ const DEFAULT_IGNORED_FOLDERS = new Set(['node_modules', '.git', 'dist']);
/** Options for customizing folder structure retrieval. */
interface FolderStructureOptions {
/** Maximum number of files and folders combined to display. Defaults to 20. */
/** Maximum number of files and folders combined to display. Defaults to 200. */
maxItems?: number;
/** Set of folder names to ignore completely. Case-sensitive. */
ignoredFolders?: Set<string>;
@@ -26,16 +28,16 @@ interface FolderStructureOptions {
fileIncludePattern?: RegExp;
/** For filtering files. */
fileService?: FileDiscoveryService;
/** Whether to use .gitignore patterns. */
respectGitIgnore?: boolean;
/** File filtering ignore options. */
fileFilteringOptions?: FileFilteringOptions;
}
// Define a type for the merged options where fileIncludePattern remains optional
type MergedFolderStructureOptions = Required<
Omit<FolderStructureOptions, 'fileIncludePattern' | 'fileService'>
> & {
fileIncludePattern?: RegExp;
fileService?: FileDiscoveryService;
fileFilteringOptions?: FileFilteringOptions;
};
/** Represents the full, unfiltered information about a folder and its contents. */
@@ -126,8 +128,13 @@ async function readFullStructure(
}
const fileName = entry.name;
const filePath = path.join(currentPath, fileName);
if (options.respectGitIgnore && options.fileService) {
if (options.fileService.shouldGitIgnoreFile(filePath)) {
if (options.fileService) {
const shouldIgnore =
(options.fileFilteringOptions.respectGitIgnore &&
options.fileService.shouldGitIgnoreFile(filePath)) ||
(options.fileFilteringOptions.respectGeminiIgnore &&
options.fileService.shouldGeminiIgnoreFile(filePath));
if (shouldIgnore) {
continue;
}
}
@@ -160,14 +167,16 @@ async function readFullStructure(
const subFolderName = entry.name;
const subFolderPath = path.join(currentPath, subFolderName);
let isIgnoredByGit = false;
if (options.respectGitIgnore && options.fileService) {
if (options.fileService.shouldGitIgnoreFile(subFolderPath)) {
isIgnoredByGit = true;
}
let isIgnored = false;
if (options.fileService) {
isIgnored =
(options.fileFilteringOptions.respectGitIgnore &&
options.fileService.shouldGitIgnoreFile(subFolderPath)) ||
(options.fileFilteringOptions.respectGeminiIgnore &&
options.fileService.shouldGeminiIgnoreFile(subFolderPath));
}
if (options.ignoredFolders.has(subFolderName) || isIgnoredByGit) {
if (options.ignoredFolders.has(subFolderName) || isIgnored) {
const ignoredSubFolder: FullFolderInfo = {
name: subFolderName,
path: subFolderPath,
@@ -227,7 +236,7 @@ function formatStructure(
// Ignored root nodes ARE printed with a connector.
if (!isProcessingRootNode || node.isIgnored) {
builder.push(
`${currentIndent}${connector}${node.name}/${node.isIgnored ? TRUNCATION_INDICATOR : ''}`,
`${currentIndent}${connector}${node.name}${path.sep}${node.isIgnored ? TRUNCATION_INDICATOR : ''}`,
);
}
@@ -295,7 +304,8 @@ export async function getFolderStructure(
ignoredFolders: options?.ignoredFolders ?? DEFAULT_IGNORED_FOLDERS,
fileIncludePattern: options?.fileIncludePattern,
fileService: options?.fileService,
respectGitIgnore: options?.respectGitIgnore ?? true,
fileFilteringOptions:
options?.fileFilteringOptions ?? DEFAULT_FILE_FILTERING_OPTIONS,
};
try {
@@ -312,34 +322,25 @@ export async function getFolderStructure(
formatStructure(structureRoot, '', true, true, structureLines);
// 3. Build the final output string
const displayPath = resolvedPath.replace(/\\/g, '/');
let disclaimer = '';
// Check if truncation occurred anywhere or if ignored folders are present.
// A simple check: if any node indicates more files/subfolders, or is ignored.
let truncationOccurred = false;
function checkForTruncation(node: FullFolderInfo) {
function isTruncated(node: FullFolderInfo): boolean {
if (node.hasMoreFiles || node.hasMoreSubfolders || node.isIgnored) {
truncationOccurred = true;
return true;
}
if (!truncationOccurred) {
for (const sub of node.subFolders) {
checkForTruncation(sub);
if (truncationOccurred) break;
for (const sub of node.subFolders) {
if (isTruncated(sub)) {
return true;
}
}
}
checkForTruncation(structureRoot);
if (truncationOccurred) {
disclaimer = `Folders or files indicated with ${TRUNCATION_INDICATOR} contain more items not shown, were ignored, or the display limit (${mergedOptions.maxItems} items) was reached.`;
return false;
}
const summary =
`Showing up to ${mergedOptions.maxItems} items (files + folders). ${disclaimer}`.trim();
let summary = `Showing up to ${mergedOptions.maxItems} items (files + folders).`;
const output = `${summary}\n\n${displayPath}/\n${structureLines.join('\n')}`;
return output;
if (isTruncated(structureRoot)) {
summary += ` Folders or files indicated with ${TRUNCATION_INDICATOR} contain more items not shown, were ignored, or the display limit (${mergedOptions.maxItems} items) was reached.`;
}
return `${summary}\n\n${resolvedPath}${path.sep}\n${structureLines.join('\n')}`;
} catch (error: unknown) {
console.error(`Error getting folder structure for ${resolvedPath}:`, error);
return `Error processing directory "${resolvedPath}": ${getErrorMessage(error)}`;

149
packages/core/src/utils/gitIgnoreParser.test.ts
View File

@@ -4,39 +4,43 @@
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest';
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import { GitIgnoreParser } from './gitIgnoreParser.js';
import * as fs from 'fs';
import * as fs from 'fs/promises';
import * as path from 'path';
import { isGitRepository } from './gitUtils.js';
// Mock fs module
vi.mock('fs');
// Mock gitUtils module
vi.mock('./gitUtils.js');
import * as os from 'os';
describe('GitIgnoreParser', () => {
let parser: GitIgnoreParser;
const mockProjectRoot = '/test/project';
let projectRoot: string;
beforeEach(() => {
parser = new GitIgnoreParser(mockProjectRoot);
// Reset mocks before each test
vi.mocked(fs.readFileSync).mockClear();
vi.mocked(isGitRepository).mockReturnValue(true);
async function createTestFile(filePath: string, content = '') {
const fullPath = path.join(projectRoot, filePath);
await fs.mkdir(path.dirname(fullPath), { recursive: true });
await fs.writeFile(fullPath, content);
}
async function setupGitRepo() {
await fs.mkdir(path.join(projectRoot, '.git'), { recursive: true });
}
beforeEach(async () => {
projectRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'gitignore-test-'));
parser = new GitIgnoreParser(projectRoot);
});
afterEach(() => {
vi.restoreAllMocks();
afterEach(async () => {
await fs.rm(projectRoot, { recursive: true, force: true });
});
describe('initialization', () => {
it('should initialize without errors when no .gitignore exists', () => {
it('should initialize without errors when no .gitignore exists', async () => {
await setupGitRepo();
expect(() => parser.loadGitRepoPatterns()).not.toThrow();
});
it('should load .gitignore patterns when file exists', () => {
it('should load .gitignore patterns when file exists', async () => {
await setupGitRepo();
const gitignoreContent = `
# Comment
node_modules/
@@ -44,7 +48,7 @@ node_modules/
/dist
.env
`;
vi.mocked(fs.readFileSync).mockReturnValueOnce(gitignoreContent);
await createTestFile('.gitignore', gitignoreContent);
parser.loadGitRepoPatterns();
@@ -55,41 +59,35 @@ node_modules/
'/dist',
'.env',
]);
expect(parser.isIgnored('node_modules/some-lib')).toBe(true);
expect(parser.isIgnored('src/app.log')).toBe(true);
expect(parser.isIgnored('dist/index.js')).toBe(true);
expect(parser.isIgnored(path.join('node_modules', 'some-lib'))).toBe(
true,
);
expect(parser.isIgnored(path.join('src', 'app.log'))).toBe(true);
expect(parser.isIgnored(path.join('dist', 'index.js'))).toBe(true);
expect(parser.isIgnored('.env')).toBe(true);
});
it('should handle git exclude file', () => {
vi.mocked(fs.readFileSync).mockImplementation((filePath) => {
if (
filePath === path.join(mockProjectRoot, '.git', 'info', 'exclude')
) {
return 'temp/\n*.tmp';
}
throw new Error('ENOENT');
});
it('should handle git exclude file', async () => {
await setupGitRepo();
await createTestFile(
path.join('.git', 'info', 'exclude'),
'temp/\n*.tmp',
);
parser.loadGitRepoPatterns();
expect(parser.getPatterns()).toEqual(['.git', 'temp/', '*.tmp']);
expect(parser.isIgnored('temp/file.txt')).toBe(true);
expect(parser.isIgnored('src/file.tmp')).toBe(true);
expect(parser.isIgnored(path.join('temp', 'file.txt'))).toBe(true);
expect(parser.isIgnored(path.join('src', 'file.tmp'))).toBe(true);
});
it('should handle custom patterns file name', () => {
vi.mocked(isGitRepository).mockReturnValue(false);
vi.mocked(fs.readFileSync).mockImplementation((filePath) => {
if (filePath === path.join(mockProjectRoot, '.geminiignore')) {
return 'temp/\n*.tmp';
}
throw new Error('ENOENT');
});
it('should handle custom patterns file name', async () => {
// No .git directory for this test
await createTestFile('.geminiignore', 'temp/\n*.tmp');
parser.loadPatterns('.geminiignore');
expect(parser.getPatterns()).toEqual(['temp/', '*.tmp']);
expect(parser.isIgnored('temp/file.txt')).toBe(true);
expect(parser.isIgnored('src/file.tmp')).toBe(true);
expect(parser.isIgnored(path.join('temp', 'file.txt'))).toBe(true);
expect(parser.isIgnored(path.join('src', 'file.tmp'))).toBe(true);
});
it('should initialize without errors when no .geminiignore exists', () => {
@@ -98,7 +96,8 @@ node_modules/
});
describe('isIgnored', () => {
beforeEach(() => {
beforeEach(async () => {
await setupGitRepo();
const gitignoreContent = `
node_modules/
*.log
@@ -107,66 +106,88 @@ node_modules/
src/*.tmp
!src/important.tmp
`;
vi.mocked(fs.readFileSync).mockReturnValueOnce(gitignoreContent);
await createTestFile('.gitignore', gitignoreContent);
parser.loadGitRepoPatterns();
});
it('should always ignore .git directory', () => {
expect(parser.isIgnored('.git')).toBe(true);
expect(parser.isIgnored('.git/config')).toBe(true);
expect(parser.isIgnored(path.join(mockProjectRoot, '.git', 'HEAD'))).toBe(
expect(parser.isIgnored(path.join('.git', 'config'))).toBe(true);
expect(parser.isIgnored(path.join(projectRoot, '.git', 'HEAD'))).toBe(
true,
);
});
it('should ignore files matching patterns', () => {
expect(parser.isIgnored('node_modules/package/index.js')).toBe(true);
expect(
parser.isIgnored(path.join('node_modules', 'package', 'index.js')),
).toBe(true);
expect(parser.isIgnored('app.log')).toBe(true);
expect(parser.isIgnored('logs/app.log')).toBe(true);
expect(parser.isIgnored('dist/bundle.js')).toBe(true);
expect(parser.isIgnored(path.join('logs', 'app.log'))).toBe(true);
expect(parser.isIgnored(path.join('dist', 'bundle.js'))).toBe(true);
expect(parser.isIgnored('.env')).toBe(true);
expect(parser.isIgnored('config/.env')).toBe(false); // .env is anchored to root
expect(parser.isIgnored(path.join('config', '.env'))).toBe(false); // .env is anchored to root
});
it('should ignore files with path-specific patterns', () => {
expect(parser.isIgnored('src/temp.tmp')).toBe(true);
expect(parser.isIgnored('other/temp.tmp')).toBe(false);
expect(parser.isIgnored(path.join('src', 'temp.tmp'))).toBe(true);
expect(parser.isIgnored(path.join('other', 'temp.tmp'))).toBe(false);
});
it('should handle negation patterns', () => {
expect(parser.isIgnored('src/important.tmp')).toBe(false);
expect(parser.isIgnored(path.join('src', 'important.tmp'))).toBe(false);
});
it('should not ignore files that do not match patterns', () => {
expect(parser.isIgnored('src/index.ts')).toBe(false);
expect(parser.isIgnored(path.join('src', 'index.ts'))).toBe(false);
expect(parser.isIgnored('README.md')).toBe(false);
});
it('should handle absolute paths correctly', () => {
const absolutePath = path.join(mockProjectRoot, 'node_modules', 'lib');
const absolutePath = path.join(projectRoot, 'node_modules', 'lib');
expect(parser.isIgnored(absolutePath)).toBe(true);
});
it('should handle paths outside project root by not ignoring them', () => {
const outsidePath = path.resolve(mockProjectRoot, '../other/file.txt');
const outsidePath = path.resolve(projectRoot, '..', 'other', 'file.txt');
expect(parser.isIgnored(outsidePath)).toBe(false);
});
it('should handle relative paths correctly', () => {
expect(parser.isIgnored('node_modules/some-package')).toBe(true);
expect(parser.isIgnored('../some/other/file.txt')).toBe(false);
expect(parser.isIgnored(path.join('node_modules', 'some-package'))).toBe(
true,
);
expect(
parser.isIgnored(path.join('..', 'some', 'other', 'file.txt')),
).toBe(false);
});
it('should normalize path separators on Windows', () => {
expect(parser.isIgnored('node_modules\\package')).toBe(true);
expect(parser.isIgnored('src\\temp.tmp')).toBe(true);
expect(parser.isIgnored(path.join('node_modules', 'package'))).toBe(true);
expect(parser.isIgnored(path.join('src', 'temp.tmp'))).toBe(true);
});
it('should handle root path "/" without throwing error', () => {
expect(() => parser.isIgnored('/')).not.toThrow();
expect(parser.isIgnored('/')).toBe(false);
});
it('should handle absolute-like paths without throwing error', () => {
expect(() => parser.isIgnored('/some/path')).not.toThrow();
expect(parser.isIgnored('/some/path')).toBe(false);
});
it('should handle paths that start with forward slash', () => {
expect(() => parser.isIgnored('/node_modules')).not.toThrow();
expect(parser.isIgnored('/node_modules')).toBe(false);
});
});
describe('getIgnoredPatterns', () => {
it('should return the raw patterns added', () => {
it('should return the raw patterns added', async () => {
await setupGitRepo();
const gitignoreContent = '*.log\n!important.log';
vi.mocked(fs.readFileSync).mockReturnValueOnce(gitignoreContent);
await createTestFile('.gitignore', gitignoreContent);
parser.loadGitRepoPatterns();
expect(parser.getPatterns()).toEqual(['.git', '*.log', '!important.log']);

12
packages/core/src/utils/gitIgnoreParser.ts
View File

@@ -57,19 +57,15 @@ export class GitIgnoreParser implements GitIgnoreFilter {
}
isIgnored(filePath: string): boolean {
const relativePath = path.isAbsolute(filePath)
? path.relative(this.projectRoot, filePath)
: filePath;
const resolved = path.resolve(this.projectRoot, filePath);
const relativePath = path.relative(this.projectRoot, resolved);
if (relativePath === '' || relativePath.startsWith('..')) {
return false;
}
let normalizedPath = relativePath.replace(/\\/g, '/');
if (normalizedPath.startsWith('./')) {
normalizedPath = normalizedPath.substring(2);
}
// Even in windows, Ignore expects forward slashes.
const normalizedPath = relativePath.replace(/\\/g, '/');
return this.ig.ignores(normalizedPath);
}

722
packages/core/src/utils/memoryDiscovery.test.ts
View File

@@ -4,604 +4,380 @@
* SPDX-License-Identifier: Apache-2.0
*/
import { vi, describe, it, expect, beforeEach, Mocked } from 'vitest';
import { vi, describe, it, expect, beforeEach, afterEach } from 'vitest';
import * as fsPromises from 'fs/promises';
import * as fsSync from 'fs';
import { Stats, Dirent } from 'fs';
import * as os from 'os';
import * as path from 'path';
import { loadServerHierarchicalMemory } from './memoryDiscovery.js';
import {
GEMINI_CONFIG_DIR,
setGeminiMdFilename,
getCurrentGeminiMdFilename,
DEFAULT_CONTEXT_FILENAME,
} from '../tools/memoryTool.js';
import { FileDiscoveryService } from '../services/fileDiscoveryService.js';
const ORIGINAL_GEMINI_MD_FILENAME_CONST_FOR_TEST = DEFAULT_CONTEXT_FILENAME;
// Mock the entire fs/promises module
vi.mock('fs/promises');
// Mock the parts of fsSync we might use (like constants or existsSync if needed)
vi.mock('fs', async (importOriginal) => {
const actual = await importOriginal<typeof fsSync>();
vi.mock('os', async (importOriginal) => {
const actualOs = await importOriginal<typeof os>();
return {
...actual, // Spread actual to get all exports, including Stats and Dirent if they are classes/constructors
constants: { ...actual.constants }, // Preserve constants
...actualOs,
homedir: vi.fn(),
};
});
vi.mock('os');
describe('loadServerHierarchicalMemory', () => {
const mockFs = fsPromises as Mocked<typeof fsPromises>;
const mockOs = os as Mocked<typeof os>;
let testRootDir: string;
let cwd: string;
let projectRoot: string;
let homedir: string;
const CWD = '/test/project/src';
const PROJECT_ROOT = '/test/project';
const USER_HOME = '/test/userhome';
async function createEmptyDir(fullPath: string) {
await fsPromises.mkdir(fullPath, { recursive: true });
return fullPath;
}
let GLOBAL_GEMINI_DIR: string;
let GLOBAL_GEMINI_FILE: string; // Defined in beforeEach
async function createTestFile(fullPath: string, fileContents: string) {
await fsPromises.mkdir(path.dirname(fullPath), { recursive: true });
await fsPromises.writeFile(fullPath, fileContents);
return path.resolve(testRootDir, fullPath);
}
beforeEach(async () => {
testRootDir = await fsPromises.mkdtemp(
path.join(os.tmpdir(), 'folder-structure-test-'),
);
const fileService = new FileDiscoveryService(PROJECT_ROOT);
beforeEach(() => {
vi.resetAllMocks();
// Set environment variables to indicate test environment
process.env.NODE_ENV = 'test';
process.env.VITEST = 'true';
setGeminiMdFilename(DEFAULT_CONTEXT_FILENAME); // Use defined const
mockOs.homedir.mockReturnValue(USER_HOME);
projectRoot = await createEmptyDir(path.join(testRootDir, 'project'));
cwd = await createEmptyDir(path.join(projectRoot, 'src'));
homedir = await createEmptyDir(path.join(testRootDir, 'userhome'));
vi.mocked(os.homedir).mockReturnValue(homedir);
});
// Define these here to use potentially reset/updated values from imports
GLOBAL_GEMINI_DIR = path.join(USER_HOME, GEMINI_CONFIG_DIR);
GLOBAL_GEMINI_FILE = path.join(
GLOBAL_GEMINI_DIR,
getCurrentGeminiMdFilename(), // Use current filename
);
mockFs.stat.mockRejectedValue(new Error('File not found'));
mockFs.readdir.mockResolvedValue([]);
mockFs.readFile.mockRejectedValue(new Error('File not found'));
mockFs.access.mockRejectedValue(new Error('File not found'));
afterEach(async () => {
// Some tests set this to a different value.
setGeminiMdFilename(DEFAULT_CONTEXT_FILENAME);
// Clean up the temporary directory to prevent resource leaks.
await fsPromises.rm(testRootDir, { recursive: true, force: true });
});
it('should return empty memory and count if no context files are found', async () => {
const { memoryContent, fileCount } = await loadServerHierarchicalMemory(
CWD,
const result = await loadServerHierarchicalMemory(
cwd,
false,
fileService,
new FileDiscoveryService(projectRoot),
);
expect(memoryContent).toBe('');
expect(fileCount).toBe(0);
expect(result).toEqual({
memoryContent: '',
fileCount: 0,
});
});
it('should load only the global context file if present and others are not (default filename)', async () => {
const globalDefaultFile = path.join(
GLOBAL_GEMINI_DIR,
DEFAULT_CONTEXT_FILENAME,
const defaultContextFile = await createTestFile(
path.join(homedir, GEMINI_CONFIG_DIR, DEFAULT_CONTEXT_FILENAME),
'default context content',
);
mockFs.access.mockImplementation(async (p) => {
if (p === globalDefaultFile) {
return undefined;
}
throw new Error('File not found');
});
mockFs.readFile.mockImplementation(async (p) => {
if (p === globalDefaultFile) {
return 'Global memory content';
}
throw new Error('File not found');
});
const { memoryContent, fileCount } = await loadServerHierarchicalMemory(
CWD,
const result = await loadServerHierarchicalMemory(
cwd,
false,
fileService,
new FileDiscoveryService(projectRoot),
);
expect(memoryContent).toBe(
`--- Context from: ${path.relative(CWD, globalDefaultFile)} ---\nGlobal memory content\n--- End of Context from: ${path.relative(CWD, globalDefaultFile)} ---`,
);
expect(fileCount).toBe(1);
expect(mockFs.readFile).toHaveBeenCalledWith(globalDefaultFile, 'utf-8');
expect(result).toEqual({
memoryContent: `--- Context from: ${path.relative(cwd, defaultContextFile)} ---
default context content
--- End of Context from: ${path.relative(cwd, defaultContextFile)} ---`,
fileCount: 1,
});
});
it('should load only the global custom context file if present and filename is changed', async () => {
const customFilename = 'CUSTOM_AGENTS.md';
setGeminiMdFilename(customFilename);
const globalCustomFile = path.join(GLOBAL_GEMINI_DIR, customFilename);
mockFs.access.mockImplementation(async (p) => {
if (p === globalCustomFile) {
return undefined;
}
throw new Error('File not found');
});
mockFs.readFile.mockImplementation(async (p) => {
if (p === globalCustomFile) {
return 'Global custom memory';
}
throw new Error('File not found');
});
const customContextFile = await createTestFile(
path.join(homedir, GEMINI_CONFIG_DIR, customFilename),
'custom context content',
);
const { memoryContent, fileCount } = await loadServerHierarchicalMemory(
CWD,
const result = await loadServerHierarchicalMemory(
cwd,
false,
fileService,
new FileDiscoveryService(projectRoot),
);
expect(memoryContent).toBe(
`--- Context from: ${path.relative(CWD, globalCustomFile)} ---\nGlobal custom memory\n--- End of Context from: ${path.relative(CWD, globalCustomFile)} ---`,
);
expect(fileCount).toBe(1);
expect(mockFs.readFile).toHaveBeenCalledWith(globalCustomFile, 'utf-8');
expect(result).toEqual({
memoryContent: `--- Context from: ${path.relative(cwd, customContextFile)} ---
custom context content
--- End of Context from: ${path.relative(cwd, customContextFile)} ---`,
fileCount: 1,
});
});
it('should load context files by upward traversal with custom filename', async () => {
const customFilename = 'PROJECT_CONTEXT.md';
setGeminiMdFilename(customFilename);
const projectRootCustomFile = path.join(PROJECT_ROOT, customFilename);
const srcCustomFile = path.join(CWD, customFilename);
mockFs.stat.mockImplementation(async (p) => {
if (p === path.join(PROJECT_ROOT, '.git')) {
return { isDirectory: () => true } as Stats;
}
throw new Error('File not found');
});
const projectContextFile = await createTestFile(
path.join(projectRoot, customFilename),
'project context content',
);
const cwdContextFile = await createTestFile(
path.join(cwd, customFilename),
'cwd context content',
);
mockFs.access.mockImplementation(async (p) => {
if (p === projectRootCustomFile || p === srcCustomFile) {
return undefined;
}
throw new Error('File not found');
});
mockFs.readFile.mockImplementation(async (p) => {
if (p === projectRootCustomFile) {
return 'Project root custom memory';
}
if (p === srcCustomFile) {
return 'Src directory custom memory';
}
throw new Error('File not found');
});
const { memoryContent, fileCount } = await loadServerHierarchicalMemory(
CWD,
const result = await loadServerHierarchicalMemory(
cwd,
false,
fileService,
new FileDiscoveryService(projectRoot),
);
const expectedContent =
`--- Context from: ${path.relative(CWD, projectRootCustomFile)} ---\nProject root custom memory\n--- End of Context from: ${path.relative(CWD, projectRootCustomFile)} ---\n\n` +
`--- Context from: ${customFilename} ---\nSrc directory custom memory\n--- End of Context from: ${customFilename} ---`;
expect(memoryContent).toBe(expectedContent);
expect(fileCount).toBe(2);
expect(mockFs.readFile).toHaveBeenCalledWith(
projectRootCustomFile,
'utf-8',
);
expect(mockFs.readFile).toHaveBeenCalledWith(srcCustomFile, 'utf-8');
expect(result).toEqual({
memoryContent: `--- Context from: ${path.relative(cwd, projectContextFile)} ---
project context content
--- End of Context from: ${path.relative(cwd, projectContextFile)} ---
--- Context from: ${path.relative(cwd, cwdContextFile)} ---
cwd context content
--- End of Context from: ${path.relative(cwd, cwdContextFile)} ---`,
fileCount: 2,
});
});
it('should load context files by downward traversal with custom filename', async () => {
const customFilename = 'LOCAL_CONTEXT.md';
setGeminiMdFilename(customFilename);
const subDir = path.join(CWD, 'subdir');
const subDirCustomFile = path.join(subDir, customFilename);
const cwdCustomFile = path.join(CWD, customFilename);
mockFs.access.mockImplementation(async (p) => {
if (p === cwdCustomFile || p === subDirCustomFile) return undefined;
throw new Error('File not found');
});
mockFs.readFile.mockImplementation(async (p) => {
if (p === cwdCustomFile) return 'CWD custom memory';
if (p === subDirCustomFile) return 'Subdir custom memory';
throw new Error('File not found');
});
mockFs.readdir.mockImplementation((async (
p: fsSync.PathLike,
): Promise<Dirent[]> => {
if (p === CWD) {
return [
{
name: customFilename,
isFile: () => true,
isDirectory: () => false,
} as Dirent,
{
name: 'subdir',
isFile: () => false,
isDirectory: () => true,
} as Dirent,
] as Dirent[];
}
if (p === subDir) {
return [
{
name: customFilename,
isFile: () => true,
isDirectory: () => false,
} as Dirent,
] as Dirent[];
}
return [] as Dirent[];
}) as unknown as typeof fsPromises.readdir);
const { memoryContent, fileCount } = await loadServerHierarchicalMemory(
CWD,
false,
fileService,
await createTestFile(
path.join(cwd, 'subdir', customFilename),
'Subdir custom memory',
);
const expectedContent =
`--- Context from: ${customFilename} ---\nCWD custom memory\n--- End of Context from: ${customFilename} ---\n\n` +
`--- Context from: ${path.join('subdir', customFilename)} ---\nSubdir custom memory\n--- End of Context from: ${path.join('subdir', customFilename)} ---`;
await createTestFile(path.join(cwd, customFilename), 'CWD custom memory');
expect(memoryContent).toBe(expectedContent);
expect(fileCount).toBe(2);
const result = await loadServerHierarchicalMemory(
cwd,
false,
new FileDiscoveryService(projectRoot),
);
expect(result).toEqual({
memoryContent: `--- Context from: ${customFilename} ---
CWD custom memory
--- End of Context from: ${customFilename} ---
--- Context from: ${path.join('subdir', customFilename)} ---
Subdir custom memory
--- End of Context from: ${path.join('subdir', customFilename)} ---`,
fileCount: 2,
});
});
it('should load ORIGINAL_GEMINI_MD_FILENAME files by upward traversal from CWD to project root', async () => {
const projectRootGeminiFile = path.join(
PROJECT_ROOT,
ORIGINAL_GEMINI_MD_FILENAME_CONST_FOR_TEST,
const projectRootGeminiFile = await createTestFile(
path.join(projectRoot, DEFAULT_CONTEXT_FILENAME),
'Project root memory',
);
const srcGeminiFile = path.join(
CWD,
ORIGINAL_GEMINI_MD_FILENAME_CONST_FOR_TEST,
const srcGeminiFile = await createTestFile(
path.join(cwd, DEFAULT_CONTEXT_FILENAME),
'Src directory memory',
);
mockFs.stat.mockImplementation(async (p) => {
if (p === path.join(PROJECT_ROOT, '.git')) {
return { isDirectory: () => true } as Stats;
}
throw new Error('File not found');
});
mockFs.access.mockImplementation(async (p) => {
if (p === projectRootGeminiFile || p === srcGeminiFile) {
return undefined;
}
throw new Error('File not found');
});
mockFs.readFile.mockImplementation(async (p) => {
if (p === projectRootGeminiFile) {
return 'Project root memory';
}
if (p === srcGeminiFile) {
return 'Src directory memory';
}
throw new Error('File not found');
});
const { memoryContent, fileCount } = await loadServerHierarchicalMemory(
CWD,
const result = await loadServerHierarchicalMemory(
cwd,
false,
fileService,
new FileDiscoveryService(projectRoot),
);
const expectedContent =
`--- Context from: ${path.relative(CWD, projectRootGeminiFile)} ---\nProject root memory\n--- End of Context from: ${path.relative(CWD, projectRootGeminiFile)} ---\n\n` +
`--- Context from: ${ORIGINAL_GEMINI_MD_FILENAME_CONST_FOR_TEST} ---\nSrc directory memory\n--- End of Context from: ${ORIGINAL_GEMINI_MD_FILENAME_CONST_FOR_TEST} ---`;
expect(memoryContent).toBe(expectedContent);
expect(fileCount).toBe(2);
expect(mockFs.readFile).toHaveBeenCalledWith(
projectRootGeminiFile,
'utf-8',
);
expect(mockFs.readFile).toHaveBeenCalledWith(srcGeminiFile, 'utf-8');
expect(result).toEqual({
memoryContent: `--- Context from: ${path.relative(cwd, projectRootGeminiFile)} ---
Project root memory
--- End of Context from: ${path.relative(cwd, projectRootGeminiFile)} ---
--- Context from: ${path.relative(cwd, srcGeminiFile)} ---
Src directory memory
--- End of Context from: ${path.relative(cwd, srcGeminiFile)} ---`,
fileCount: 2,
});
});
it('should load ORIGINAL_GEMINI_MD_FILENAME files by downward traversal from CWD', async () => {
const subDir = path.join(CWD, 'subdir');
const subDirGeminiFile = path.join(
subDir,
ORIGINAL_GEMINI_MD_FILENAME_CONST_FOR_TEST,
await createTestFile(
path.join(cwd, 'subdir', DEFAULT_CONTEXT_FILENAME),
'Subdir memory',
);
const cwdGeminiFile = path.join(
CWD,
ORIGINAL_GEMINI_MD_FILENAME_CONST_FOR_TEST,
await createTestFile(
path.join(cwd, DEFAULT_CONTEXT_FILENAME),
'CWD memory',
);
mockFs.access.mockImplementation(async (p) => {
if (p === cwdGeminiFile || p === subDirGeminiFile) return undefined;
throw new Error('File not found');
});
mockFs.readFile.mockImplementation(async (p) => {
if (p === cwdGeminiFile) return 'CWD memory';
if (p === subDirGeminiFile) return 'Subdir memory';
throw new Error('File not found');
});
mockFs.readdir.mockImplementation((async (
p: fsSync.PathLike,
): Promise<Dirent[]> => {
if (p === CWD) {
return [
{
name: ORIGINAL_GEMINI_MD_FILENAME_CONST_FOR_TEST,
isFile: () => true,
isDirectory: () => false,
} as Dirent,
{
name: 'subdir',
isFile: () => false,
isDirectory: () => true,
} as Dirent,
] as Dirent[];
}
if (p === subDir) {
return [
{
name: ORIGINAL_GEMINI_MD_FILENAME_CONST_FOR_TEST,
isFile: () => true,
isDirectory: () => false,
} as Dirent,
] as Dirent[];
}
return [] as Dirent[];
}) as unknown as typeof fsPromises.readdir);
const { memoryContent, fileCount } = await loadServerHierarchicalMemory(
CWD,
const result = await loadServerHierarchicalMemory(
cwd,
false,
fileService,
new FileDiscoveryService(projectRoot),
);
const expectedContent =
`--- Context from: ${ORIGINAL_GEMINI_MD_FILENAME_CONST_FOR_TEST} ---\nCWD memory\n--- End of Context from: ${ORIGINAL_GEMINI_MD_FILENAME_CONST_FOR_TEST} ---\n\n` +
`--- Context from: ${path.join('subdir', ORIGINAL_GEMINI_MD_FILENAME_CONST_FOR_TEST)} ---\nSubdir memory\n--- End of Context from: ${path.join('subdir', ORIGINAL_GEMINI_MD_FILENAME_CONST_FOR_TEST)} ---`;
expect(memoryContent).toBe(expectedContent);
expect(fileCount).toBe(2);
expect(result).toEqual({
memoryContent: `--- Context from: ${DEFAULT_CONTEXT_FILENAME} ---
CWD memory
--- End of Context from: ${DEFAULT_CONTEXT_FILENAME} ---
--- Context from: ${path.join('subdir', DEFAULT_CONTEXT_FILENAME)} ---
Subdir memory
--- End of Context from: ${path.join('subdir', DEFAULT_CONTEXT_FILENAME)} ---`,
fileCount: 2,
});
});
it('should load and correctly order global, upward, and downward ORIGINAL_GEMINI_MD_FILENAME files', async () => {
setGeminiMdFilename(ORIGINAL_GEMINI_MD_FILENAME_CONST_FOR_TEST); // Explicitly set for this test
const globalFileToUse = path.join(
GLOBAL_GEMINI_DIR,
ORIGINAL_GEMINI_MD_FILENAME_CONST_FOR_TEST,
const defaultContextFile = await createTestFile(
path.join(homedir, GEMINI_CONFIG_DIR, DEFAULT_CONTEXT_FILENAME),
'default context content',
);
const projectParentDir = path.dirname(PROJECT_ROOT);
const projectParentGeminiFile = path.join(
projectParentDir,
ORIGINAL_GEMINI_MD_FILENAME_CONST_FOR_TEST,
const rootGeminiFile = await createTestFile(
path.join(testRootDir, DEFAULT_CONTEXT_FILENAME),
'Project parent memory',
);
const projectRootGeminiFile = path.join(
PROJECT_ROOT,
ORIGINAL_GEMINI_MD_FILENAME_CONST_FOR_TEST,
const projectRootGeminiFile = await createTestFile(
path.join(projectRoot, DEFAULT_CONTEXT_FILENAME),
'Project root memory',
);
const cwdGeminiFile = path.join(
CWD,
ORIGINAL_GEMINI_MD_FILENAME_CONST_FOR_TEST,
const cwdGeminiFile = await createTestFile(
path.join(cwd, DEFAULT_CONTEXT_FILENAME),
'CWD memory',
);
const subDir = path.join(CWD, 'sub');
const subDirGeminiFile = path.join(
subDir,
ORIGINAL_GEMINI_MD_FILENAME_CONST_FOR_TEST,
const subDirGeminiFile = await createTestFile(
path.join(cwd, 'sub', DEFAULT_CONTEXT_FILENAME),
'Subdir memory',
);
mockFs.stat.mockImplementation(async (p) => {
if (p === path.join(PROJECT_ROOT, '.git')) {
return { isDirectory: () => true } as Stats;
} else if (p === path.join(PROJECT_ROOT, '.gemini')) {
return { isDirectory: () => true } as Stats;
}
throw new Error('File not found');
});
mockFs.access.mockImplementation(async (p) => {
if (
p === globalFileToUse || // Use the dynamically set global file path
p === projectParentGeminiFile ||
p === projectRootGeminiFile ||
p === cwdGeminiFile ||
p === subDirGeminiFile
) {
return undefined;
}
throw new Error('File not found');
});
mockFs.readFile.mockImplementation(async (p) => {
if (p === globalFileToUse) return 'Global memory'; // Use the dynamically set global file path
if (p === projectParentGeminiFile) return 'Project parent memory';
if (p === projectRootGeminiFile) return 'Project root memory';
if (p === cwdGeminiFile) return 'CWD memory';
if (p === subDirGeminiFile) return 'Subdir memory';
throw new Error('File not found');
});
mockFs.readdir.mockImplementation((async (
p: fsSync.PathLike,
): Promise<Dirent[]> => {
if (p === CWD) {
return [
{
name: 'sub',
isFile: () => false,
isDirectory: () => true,
} as Dirent,
] as Dirent[];
}
if (p === subDir) {
return [
{
name: ORIGINAL_GEMINI_MD_FILENAME_CONST_FOR_TEST,
isFile: () => true,
isDirectory: () => false,
} as Dirent,
] as Dirent[];
}
return [] as Dirent[];
}) as unknown as typeof fsPromises.readdir);
const { memoryContent, fileCount } = await loadServerHierarchicalMemory(
CWD,
const result = await loadServerHierarchicalMemory(
cwd,
false,
fileService,
new FileDiscoveryService(projectRoot),
);
const relPathGlobal = path.relative(CWD, GLOBAL_GEMINI_FILE);
const relPathProjectParent = path.relative(CWD, projectParentGeminiFile);
const relPathProjectRoot = path.relative(CWD, projectRootGeminiFile);
const relPathCwd = ORIGINAL_GEMINI_MD_FILENAME_CONST_FOR_TEST;
const relPathSubDir = path.join(
'sub',
ORIGINAL_GEMINI_MD_FILENAME_CONST_FOR_TEST,
);
expect(result).toEqual({
memoryContent: `--- Context from: ${path.relative(cwd, defaultContextFile)} ---
default context content
--- End of Context from: ${path.relative(cwd, defaultContextFile)} ---
const expectedContent = [
`--- Context from: ${relPathGlobal} ---\nGlobal memory\n--- End of Context from: ${relPathGlobal} ---`,
`--- Context from: ${relPathProjectParent} ---\nProject parent memory\n--- End of Context from: ${relPathProjectParent} ---`,
`--- Context from: ${relPathProjectRoot} ---\nProject root memory\n--- End of Context from: ${relPathProjectRoot} ---`,
`--- Context from: ${relPathCwd} ---\nCWD memory\n--- End of Context from: ${relPathCwd} ---`,
`--- Context from: ${relPathSubDir} ---\nSubdir memory\n--- End of Context from: ${relPathSubDir} ---`,
].join('\n\n');
--- Context from: ${path.relative(cwd, rootGeminiFile)} ---
Project parent memory
--- End of Context from: ${path.relative(cwd, rootGeminiFile)} ---
expect(memoryContent).toBe(expectedContent);
expect(fileCount).toBe(5);
--- Context from: ${path.relative(cwd, projectRootGeminiFile)} ---
Project root memory
--- End of Context from: ${path.relative(cwd, projectRootGeminiFile)} ---
--- Context from: ${path.relative(cwd, cwdGeminiFile)} ---
CWD memory
--- End of Context from: ${path.relative(cwd, cwdGeminiFile)} ---
--- Context from: ${path.relative(cwd, subDirGeminiFile)} ---
Subdir memory
--- End of Context from: ${path.relative(cwd, subDirGeminiFile)} ---`,
fileCount: 5,
});
});
it('should ignore specified directories during downward scan', async () => {
const ignoredDir = path.join(CWD, 'node_modules');
const ignoredDirGeminiFile = path.join(
ignoredDir,
ORIGINAL_GEMINI_MD_FILENAME_CONST_FOR_TEST,
); // Corrected
const regularSubDir = path.join(CWD, 'my_code');
const regularSubDirGeminiFile = path.join(
regularSubDir,
ORIGINAL_GEMINI_MD_FILENAME_CONST_FOR_TEST,
await createEmptyDir(path.join(projectRoot, '.git'));
await createTestFile(path.join(projectRoot, '.gitignore'), 'node_modules');
await createTestFile(
path.join(cwd, 'node_modules', DEFAULT_CONTEXT_FILENAME),
'Ignored memory',
);
const regularSubDirGeminiFile = await createTestFile(
path.join(cwd, 'my_code', DEFAULT_CONTEXT_FILENAME),
'My code memory',
);
mockFs.access.mockImplementation(async (p) => {
if (p === regularSubDirGeminiFile) return undefined;
if (p === ignoredDirGeminiFile)
throw new Error('Should not access ignored file');
throw new Error('File not found');
});
mockFs.readFile.mockImplementation(async (p) => {
if (p === regularSubDirGeminiFile) return 'My code memory';
throw new Error('File not found');
});
mockFs.readdir.mockImplementation((async (
p: fsSync.PathLike,
): Promise<Dirent[]> => {
if (p === CWD) {
return [
{
name: 'node_modules',
isFile: () => false,
isDirectory: () => true,
} as Dirent,
{
name: 'my_code',
isFile: () => false,
isDirectory: () => true,
} as Dirent,
] as Dirent[];
}
if (p === regularSubDir) {
return [
{
name: ORIGINAL_GEMINI_MD_FILENAME_CONST_FOR_TEST,
isFile: () => true,
isDirectory: () => false,
} as Dirent,
] as Dirent[];
}
if (p === ignoredDir) {
return [] as Dirent[];
}
return [] as Dirent[];
}) as unknown as typeof fsPromises.readdir);
const { memoryContent, fileCount } = await loadServerHierarchicalMemory(
CWD,
const result = await loadServerHierarchicalMemory(
cwd,
false,
fileService,
new FileDiscoveryService(projectRoot),
[],
{
respectGitIgnore: true,
respectGeminiIgnore: true,
},
);
const expectedContent = `--- Context from: ${path.join('my_code', ORIGINAL_GEMINI_MD_FILENAME_CONST_FOR_TEST)} ---\nMy code memory\n--- End of Context from: ${path.join('my_code', ORIGINAL_GEMINI_MD_FILENAME_CONST_FOR_TEST)} ---`;
expect(memoryContent).toBe(expectedContent);
expect(fileCount).toBe(1);
expect(mockFs.readFile).not.toHaveBeenCalledWith(
ignoredDirGeminiFile,
'utf-8',
);
expect(result).toEqual({
memoryContent: `--- Context from: ${path.relative(cwd, regularSubDirGeminiFile)} ---
My code memory
--- End of Context from: ${path.relative(cwd, regularSubDirGeminiFile)} ---`,
fileCount: 1,
});
});
it('should respect MAX_DIRECTORIES_TO_SCAN_FOR_MEMORY during downward scan', async () => {
it('should respect the maxDirs parameter during downward scan', async () => {
const consoleDebugSpy = vi
.spyOn(console, 'debug')
.mockImplementation(() => {});
const dirNames: Dirent[] = [];
for (let i = 0; i < 250; i++) {
dirNames.push({
name: `deep_dir_${i}`,
isFile: () => false,
isDirectory: () => true,
} as Dirent);
for (let i = 0; i < 100; i++) {
await createEmptyDir(path.join(cwd, `deep_dir_${i}`));
}
mockFs.readdir.mockImplementation((async (
p: fsSync.PathLike,
): Promise<Dirent[]> => {
if (p === CWD) return dirNames;
if (p.toString().startsWith(path.join(CWD, 'deep_dir_')))
return [] as Dirent[];
return [] as Dirent[];
}) as unknown as typeof fsPromises.readdir);
mockFs.access.mockRejectedValue(new Error('not found'));
await loadServerHierarchicalMemory(CWD, true, fileService);
// Pass the custom limit directly to the function
await loadServerHierarchicalMemory(
cwd,
true,
new FileDiscoveryService(projectRoot),
[],
{
respectGitIgnore: true,
respectGeminiIgnore: true,
},
50, // maxDirs
);
expect(consoleDebugSpy).toHaveBeenCalledWith(
expect.stringContaining('[DEBUG] [BfsFileSearch]'),
expect.stringContaining('Scanning [200/200]:'),
expect.stringContaining('Scanning [50/50]:'),
);
consoleDebugSpy.mockRestore();
vi.mocked(console.debug).mockRestore();
const result = await loadServerHierarchicalMemory(
cwd,
false,
new FileDiscoveryService(projectRoot),
);
expect(result).toEqual({
memoryContent: '',
fileCount: 0,
});
});
it('should load extension context file paths', async () => {
const extensionFilePath = '/test/extensions/ext1/GEMINI.md';
mockFs.access.mockImplementation(async (p) => {
if (p === extensionFilePath) {
return undefined;
}
throw new Error('File not found');
});
mockFs.readFile.mockImplementation(async (p) => {
if (p === extensionFilePath) {
return 'Extension memory content';
}
throw new Error('File not found');
});
const extensionFilePath = await createTestFile(
path.join(testRootDir, 'extensions/ext1/GEMINI.md'),
'Extension memory content',
);
const { memoryContent, fileCount } = await loadServerHierarchicalMemory(
CWD,
const result = await loadServerHierarchicalMemory(
cwd,
false,
fileService,
new FileDiscoveryService(projectRoot),
[extensionFilePath],
);
expect(memoryContent).toBe(
`--- Context from: ${path.relative(CWD, extensionFilePath)} ---\nExtension memory content\n--- End of Context from: ${path.relative(CWD, extensionFilePath)} ---`,
);
expect(fileCount).toBe(1);
expect(mockFs.readFile).toHaveBeenCalledWith(extensionFilePath, 'utf-8');
expect(result).toEqual({
memoryContent: `--- Context from: ${path.relative(cwd, extensionFilePath)} ---
Extension memory content
--- End of Context from: ${path.relative(cwd, extensionFilePath)} ---`,
fileCount: 1,
});
});
});

22
packages/core/src/utils/memoryDiscovery.ts
View File

@@ -15,6 +15,10 @@ import {
} from '../tools/memoryTool.js';
import { FileDiscoveryService } from '../services/fileDiscoveryService.js';
import { processImports } from './memoryImportProcessor.js';
import {
DEFAULT_MEMORY_FILE_FILTERING_OPTIONS,
FileFilteringOptions,
} from '../config/config.js';
// Simple console logger, similar to the one previously in CLI's config.ts
// TODO: Integrate with a more robust server-side logger if available/appropriate.
@@ -29,8 +33,6 @@ const logger = {
console.error('[ERROR] [MemoryDiscovery]', ...args),
};
const MAX_DIRECTORIES_TO_SCAN_FOR_MEMORY = 200;
interface GeminiFileContent {
filePath: string;
content: string | null;
@@ -85,6 +87,8 @@ async function getGeminiMdFilePathsInternal(
debugMode: boolean,
fileService: FileDiscoveryService,
extensionContextFilePaths: string[] = [],
fileFilteringOptions: FileFilteringOptions,
maxDirs: number,
): Promise<string[]> {
const allPaths = new Set<string>();
const geminiMdFilenames = getAllGeminiMdFilenames();
@@ -181,11 +185,18 @@ async function getGeminiMdFilePathsInternal(
}
upwardPaths.forEach((p) => allPaths.add(p));
// Merge options with memory defaults, with options taking precedence
const mergedOptions = {
...DEFAULT_MEMORY_FILE_FILTERING_OPTIONS,
...fileFilteringOptions,
};
const downwardPaths = await bfsFileSearch(resolvedCwd, {
fileName: geminiMdFilename,
maxDirs: MAX_DIRECTORIES_TO_SCAN_FOR_MEMORY,
maxDirs,
debug: debugMode,
fileService,
fileFilteringOptions: mergedOptions, // Pass merged options as fileFilter
});
downwardPaths.sort(); // Sort for consistent ordering, though hierarchy might be more complex
if (debugMode && downwardPaths.length > 0)
@@ -282,11 +293,14 @@ export async function loadServerHierarchicalMemory(
debugMode: boolean,
fileService: FileDiscoveryService,
extensionContextFilePaths: string[] = [],
fileFilteringOptions?: FileFilteringOptions,
maxDirs: number = 200,
): Promise<{ memoryContent: string; fileCount: number }> {
if (debugMode)
logger.debug(
`Loading server hierarchical memory for CWD: ${currentWorkingDirectory}`,
);
// For the server, homedir() refers to the server process's home.
// This is consistent with how MemoryTool already finds the global path.
const userHomePath = homedir();
@@ -296,6 +310,8 @@ export async function loadServerHierarchicalMemory(
debugMode,
fileService,
extensionContextFilePaths,
fileFilteringOptions || DEFAULT_MEMORY_FILE_FILTERING_OPTIONS,
maxDirs,
);
if (filePaths.length === 0) {
if (debugMode) logger.debug('No GEMINI.md files found in hierarchy.');

166
packages/core/src/utils/partUtils.test.ts Normal file
View File

@@ -0,0 +1,166 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect } from 'vitest';
import { partToString, getResponseText } from './partUtils.js';
import { GenerateContentResponse, Part } from '@google/genai';
const mockResponse = (
parts?: Array<{ text?: string; functionCall?: unknown }>,
): GenerateContentResponse => ({
candidates: parts
? [{ content: { parts: parts as Part[], role: 'model' }, index: 0 }]
: [],
promptFeedback: { safetyRatings: [] },
text: undefined,
data: undefined,
functionCalls: undefined,
executableCode: undefined,
codeExecutionResult: undefined,
});
describe('partUtils', () => {
describe('partToString (default behavior)', () => {
it('should return empty string for undefined or null', () => {
// @ts-expect-error Testing invalid input
expect(partToString(undefined)).toBe('');
// @ts-expect-error Testing invalid input
expect(partToString(null)).toBe('');
});
it('should return string input unchanged', () => {
expect(partToString('hello')).toBe('hello');
});
it('should concatenate strings from an array', () => {
expect(partToString(['a', 'b'])).toBe('ab');
});
it('should return text property when provided a text part', () => {
expect(partToString({ text: 'hi' })).toBe('hi');
});
it('should return empty string for non-text parts', () => {
const part: Part = { inlineData: { mimeType: 'image/png', data: '' } };
expect(partToString(part)).toBe('');
const part2: Part = { functionCall: { name: 'test' } };
expect(partToString(part2)).toBe('');
});
});
describe('partToString (verbose)', () => {
const verboseOptions = { verbose: true };
it('should return empty string for undefined or null', () => {
// @ts-expect-error Testing invalid input
expect(partToString(undefined, verboseOptions)).toBe('');
// @ts-expect-error Testing invalid input
expect(partToString(null, verboseOptions)).toBe('');
});
it('should return string input unchanged', () => {
expect(partToString('hello', verboseOptions)).toBe('hello');
});
it('should join parts if the value is an array', () => {
const parts = ['hello', { text: ' world' }];
expect(partToString(parts, verboseOptions)).toBe('hello world');
});
it('should return the text property if the part is an object with text', () => {
const part: Part = { text: 'hello world' };
expect(partToString(part, verboseOptions)).toBe('hello world');
});
it('should return descriptive string for videoMetadata part', () => {
const part = { videoMetadata: {} } as Part;
expect(partToString(part, verboseOptions)).toBe('[Video Metadata]');
});
it('should return descriptive string for thought part', () => {
const part = { thought: 'thinking' } as unknown as Part;
expect(partToString(part, verboseOptions)).toBe('[Thought: thinking]');
});
it('should return descriptive string for codeExecutionResult part', () => {
const part = { codeExecutionResult: {} } as Part;
expect(partToString(part, verboseOptions)).toBe(
'[Code Execution Result]',
);
});
it('should return descriptive string for executableCode part', () => {
const part = { executableCode: {} } as Part;
expect(partToString(part, verboseOptions)).toBe('[Executable Code]');
});
it('should return descriptive string for fileData part', () => {
const part = { fileData: {} } as Part;
expect(partToString(part, verboseOptions)).toBe('[File Data]');
});
it('should return descriptive string for functionCall part', () => {
const part = { functionCall: { name: 'myFunction' } } as Part;
expect(partToString(part, verboseOptions)).toBe(
'[Function Call: myFunction]',
);
});
it('should return descriptive string for functionResponse part', () => {
const part = { functionResponse: { name: 'myFunction' } } as Part;
expect(partToString(part, verboseOptions)).toBe(
'[Function Response: myFunction]',
);
});
it('should return descriptive string for inlineData part', () => {
const part = { inlineData: { mimeType: 'image/png', data: '' } } as Part;
expect(partToString(part, verboseOptions)).toBe('<image/png>');
});
it('should return an empty string for an unknown part type', () => {
const part: Part = {};
expect(partToString(part, verboseOptions)).toBe('');
});
it('should handle complex nested arrays with various part types', () => {
const parts = [
'start ',
{ text: 'middle' },
[
{ functionCall: { name: 'func1' } },
' end',
{ inlineData: { mimeType: 'audio/mp3', data: '' } },
],
];
expect(partToString(parts as Part, verboseOptions)).toBe(
'start middle[Function Call: func1] end<audio/mp3>',
);
});
});
describe('getResponseText', () => {
it('should return null when no candidates exist', () => {
const response = mockResponse(undefined);
expect(getResponseText(response)).toBeNull();
});
it('should return concatenated text from first candidate', () => {
const result = mockResponse([{ text: 'a' }, { text: 'b' }]);
expect(getResponseText(result)).toBe('ab');
});
it('should ignore parts without text', () => {
const result = mockResponse([{ functionCall: {} }, { text: 'hello' }]);
expect(getResponseText(result)).toBe('hello');
});
it('should return null when candidate has no parts', () => {
const result = mockResponse([]);
expect(getResponseText(result)).toBeNull();
});
});
});

85
packages/core/src/utils/partUtils.ts Normal file
View File

@@ -0,0 +1,85 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { GenerateContentResponse, PartListUnion, Part } from '@google/genai';
/**
* Converts a PartListUnion into a string.
* If verbose is true, includes summary representations of non-text parts.
*/
export function partToString(
value: PartListUnion,
options?: { verbose?: boolean },
): string {
if (!value) {
return '';
}
if (typeof value === 'string') {
return value;
}
if (Array.isArray(value)) {
return value.map((part) => partToString(part, options)).join('');
}
// Cast to Part, assuming it might contain project-specific fields
const part = value as Part & {
videoMetadata?: unknown;
thought?: string;
codeExecutionResult?: unknown;
executableCode?: unknown;
};
if (options?.verbose) {
if (part.videoMetadata !== undefined) {
return `[Video Metadata]`;
}
if (part.thought !== undefined) {
return `[Thought: ${part.thought}]`;
}
if (part.codeExecutionResult !== undefined) {
return `[Code Execution Result]`;
}
if (part.executableCode !== undefined) {
return `[Executable Code]`;
}
// Standard Part fields
if (part.fileData !== undefined) {
return `[File Data]`;
}
if (part.functionCall !== undefined) {
return `[Function Call: ${part.functionCall.name}]`;
}
if (part.functionResponse !== undefined) {
return `[Function Response: ${part.functionResponse.name}]`;
}
if (part.inlineData !== undefined) {
return `<${part.inlineData.mimeType}>`;
}
}
return part.text ?? '';
}
export function getResponseText(
response: GenerateContentResponse,
): string | null {
if (response.candidates && response.candidates.length > 0) {
const candidate = response.candidates[0];
if (
candidate.content &&
candidate.content.parts &&
candidate.content.parts.length > 0
) {
return candidate.content.parts
.filter((part) => part.text)
.map((part) => part.text)
.join('');
}
}
return null;
}

20
packages/core/src/utils/paths.ts
View File

@@ -11,6 +11,7 @@ import * as crypto from 'crypto';
export const GEMINI_DIR = '.qwen';
export const GOOGLE_ACCOUNTS_FILENAME = 'google_accounts.json';
const TMP_DIR_NAME = 'tmp';
const COMMANDS_DIR_NAME = 'commands';
/**
* Replaces the home directory with a tilde.
@@ -44,7 +45,7 @@ export function shortenPath(filePath: string, maxLen: number = 35): string {
// Handle cases with no segments after root (e.g., "/", "C:\") or only one segment
if (segments.length <= 1) {
// Fallback to simple start/end truncation for very short paths or single segments
// Fall back to simple start/end truncation for very short paths or single segments
const keepLen = Math.floor((maxLen - 3) / 2);
// Ensure keepLen is not negative if maxLen is very small
if (keepLen <= 0) {
@@ -158,3 +159,20 @@ export function getProjectTempDir(projectRoot: string): string {
const hash = getProjectHash(projectRoot);
return path.join(os.homedir(), GEMINI_DIR, TMP_DIR_NAME, hash);
}
/**
* Returns the absolute path to the user-level commands directory.
* @returns The path to the user's commands directory.
*/
export function getUserCommandsDir(): string {
return path.join(os.homedir(), GEMINI_DIR, COMMANDS_DIR_NAME);
}
/**
* Returns the absolute path to the project-level commands directory.
* @param projectRoot The absolute path to the project's root directory.
* @returns The path to the project's commands directory.
*/
export function getProjectCommandsDir(projectRoot: string): string {
return path.join(projectRoot, GEMINI_DIR, COMMANDS_DIR_NAME);
}

9
packages/core/src/utils/quotaErrorDetection.ts
View File

@@ -68,10 +68,6 @@ export function isProQuotaExceededError(error: unknown): boolean {
};
};
if (gaxiosError.response && gaxiosError.response.data) {
console.log(
'[DEBUG] isProQuotaExceededError - checking response data:',
gaxiosError.response.data,
);
if (typeof gaxiosError.response.data === 'string') {
return checkMessage(gaxiosError.response.data);
}
@@ -87,11 +83,6 @@ export function isProQuotaExceededError(error: unknown): boolean {
}
}
}
console.log(
'[DEBUG] isProQuotaExceededError - no matching error format for:',
error,
);
return false;
}

4
packages/core/src/utils/retry.ts
View File

@@ -196,7 +196,7 @@ export async function retryWithBackoff<T>(
// Reset currentDelay for next potential non-429 error, or if Retry-After is not present next time
currentDelay = initialDelayMs;
} else {
// Fallback to exponential backoff with jitter
// Fall back to exponential backoff with jitter
logRetryAttempt(attempt, error, errorStatus);
// Add jitter: +/- 30% of currentDelay
const jitter = currentDelay * 0.3 * (Math.random() * 2 - 1);
@@ -216,7 +216,7 @@ export async function retryWithBackoff<T>(
* @param error The error object.
* @returns The HTTP status code, or undefined if not found.
*/
function getErrorStatus(error: unknown): number | undefined {
export function getErrorStatus(error: unknown): number | undefined {
if (typeof error === 'object' && error !== null) {
if ('status' in error && typeof error.status === 'number') {
return error.status;

10
packages/core/src/utils/schemaValidator.ts
View File

@@ -5,9 +5,11 @@
*/
import { Schema } from '@google/genai';
import * as ajv from 'ajv';
const ajValidator = new ajv.Ajv();
import AjvPkg from 'ajv';
// Ajv's ESM/CJS interop: use 'any' for compatibility as recommended by Ajv docs
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const AjvClass = (AjvPkg as any).default || AjvPkg;
const ajValidator = new AjvClass();
/**
* Simple utility to validate objects against JSON Schemas
@@ -34,7 +36,7 @@ export class SchemaValidator {
/**
* Converts @google/genai's Schema to an object compatible with avj.
* This is necessry because it represents Types as an Enum (with
* This is necessary because it represents Types as an Enum (with
* UPPERCASE values) and minItems and minLength as strings, when they should be numbers.
*/
private static toObjectSchema(schema: Schema): object {

277
packages/core/src/utils/shell-utils.test.ts Normal file
View File

@@ -0,0 +1,277 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { expect, describe, it, beforeEach } from 'vitest';
import {
checkCommandPermissions,
getCommandRoots,
isCommandAllowed,
stripShellWrapper,
} from './shell-utils.js';
import { Config } from '../config/config.js';
let config: Config;
beforeEach(() => {
config = {
getCoreTools: () => [],
getExcludeTools: () => [],
} as unknown as Config;
});
describe('isCommandAllowed', () => {
it('should allow a command if no restrictions are provided', () => {
const result = isCommandAllowed('ls -l', config);
expect(result.allowed).toBe(true);
});
it('should allow a command if it is in the global allowlist', () => {
config.getCoreTools = () => ['ShellTool(ls)'];
const result = isCommandAllowed('ls -l', config);
expect(result.allowed).toBe(true);
});
it('should block a command if it is not in a strict global allowlist', () => {
config.getCoreTools = () => ['ShellTool(ls -l)'];
const result = isCommandAllowed('rm -rf /', config);
expect(result.allowed).toBe(false);
expect(result.reason).toBe(`Command(s) not in the allowed commands list.`);
});
it('should block a command if it is in the blocked list', () => {
config.getExcludeTools = () => ['ShellTool(rm -rf /)'];
const result = isCommandAllowed('rm -rf /', config);
expect(result.allowed).toBe(false);
expect(result.reason).toBe(
`Command 'rm -rf /' is blocked by configuration`,
);
});
it('should prioritize the blocklist over the allowlist', () => {
config.getCoreTools = () => ['ShellTool(rm -rf /)'];
config.getExcludeTools = () => ['ShellTool(rm -rf /)'];
const result = isCommandAllowed('rm -rf /', config);
expect(result.allowed).toBe(false);
expect(result.reason).toBe(
`Command 'rm -rf /' is blocked by configuration`,
);
});
it('should allow any command when a wildcard is in coreTools', () => {
config.getCoreTools = () => ['ShellTool'];
const result = isCommandAllowed('any random command', config);
expect(result.allowed).toBe(true);
});
it('should block any command when a wildcard is in excludeTools', () => {
config.getExcludeTools = () => ['run_shell_command'];
const result = isCommandAllowed('any random command', config);
expect(result.allowed).toBe(false);
expect(result.reason).toBe(
'Shell tool is globally disabled in configuration',
);
});
it('should block a command on the blocklist even with a wildcard allow', () => {
config.getCoreTools = () => ['ShellTool'];
config.getExcludeTools = () => ['ShellTool(rm -rf /)'];
const result = isCommandAllowed('rm -rf /', config);
expect(result.allowed).toBe(false);
expect(result.reason).toBe(
`Command 'rm -rf /' is blocked by configuration`,
);
});
it('should allow a chained command if all parts are on the global allowlist', () => {
config.getCoreTools = () => [
'run_shell_command(echo)',
'run_shell_command(ls)',
];
const result = isCommandAllowed('echo "hello" && ls -l', config);
expect(result.allowed).toBe(true);
});
it('should block a chained command if any part is blocked', () => {
config.getExcludeTools = () => ['run_shell_command(rm)'];
const result = isCommandAllowed('echo "hello" && rm -rf /', config);
expect(result.allowed).toBe(false);
expect(result.reason).toBe(
`Command 'rm -rf /' is blocked by configuration`,
);
});
describe('command substitution', () => {
it('should block command substitution using `$(...)`', () => {
const result = isCommandAllowed('echo $(rm -rf /)', config);
expect(result.allowed).toBe(false);
expect(result.reason).toContain('Command substitution');
});
it('should block command substitution using `<(...)`', () => {
const result = isCommandAllowed('diff <(ls) <(ls -a)', config);
expect(result.allowed).toBe(false);
expect(result.reason).toContain('Command substitution');
});
it('should block command substitution using backticks', () => {
const result = isCommandAllowed('echo `rm -rf /`', config);
expect(result.allowed).toBe(false);
expect(result.reason).toContain('Command substitution');
});
it('should allow substitution-like patterns inside single quotes', () => {
config.getCoreTools = () => ['ShellTool(echo)'];
const result = isCommandAllowed("echo '$(pwd)'", config);
expect(result.allowed).toBe(true);
});
});
});
describe('checkCommandPermissions', () => {
describe('in "Default Allow" mode (no sessionAllowlist)', () => {
it('should return a detailed success object for an allowed command', () => {
const result = checkCommandPermissions('ls -l', config);
expect(result).toEqual({
allAllowed: true,
disallowedCommands: [],
});
});
it('should return a detailed failure object for a blocked command', () => {
config.getExcludeTools = () => ['ShellTool(rm)'];
const result = checkCommandPermissions('rm -rf /', config);
expect(result).toEqual({
allAllowed: false,
disallowedCommands: ['rm -rf /'],
blockReason: `Command 'rm -rf /' is blocked by configuration`,
isHardDenial: true,
});
});
it('should return a detailed failure object for a command not on a strict allowlist', () => {
config.getCoreTools = () => ['ShellTool(ls)'];
const result = checkCommandPermissions('git status && ls', config);
expect(result).toEqual({
allAllowed: false,
disallowedCommands: ['git status'],
blockReason: `Command(s) not in the allowed commands list.`,
isHardDenial: false,
});
});
});
describe('in "Default Deny" mode (with sessionAllowlist)', () => {
it('should allow a command on the sessionAllowlist', () => {
const result = checkCommandPermissions(
'ls -l',
config,
new Set(['ls -l']),
);
expect(result.allAllowed).toBe(true);
});
it('should block a command not on the sessionAllowlist or global allowlist', () => {
const result = checkCommandPermissions(
'rm -rf /',
config,
new Set(['ls -l']),
);
expect(result.allAllowed).toBe(false);
expect(result.blockReason).toContain(
'not on the global or session allowlist',
);
expect(result.disallowedCommands).toEqual(['rm -rf /']);
});
it('should allow a command on the global allowlist even if not on the session allowlist', () => {
config.getCoreTools = () => ['ShellTool(git status)'];
const result = checkCommandPermissions(
'git status',
config,
new Set(['ls -l']),
);
expect(result.allAllowed).toBe(true);
});
it('should allow a chained command if parts are on different allowlists', () => {
config.getCoreTools = () => ['ShellTool(git status)'];
const result = checkCommandPermissions(
'git status && git commit',
config,
new Set(['git commit']),
);
expect(result.allAllowed).toBe(true);
});
it('should block a command on the sessionAllowlist if it is also globally blocked', () => {
config.getExcludeTools = () => ['run_shell_command(rm)'];
const result = checkCommandPermissions(
'rm -rf /',
config,
new Set(['rm -rf /']),
);
expect(result.allAllowed).toBe(false);
expect(result.blockReason).toContain('is blocked by configuration');
});
it('should block a chained command if one part is not on any allowlist', () => {
config.getCoreTools = () => ['run_shell_command(echo)'];
const result = checkCommandPermissions(
'echo "hello" && rm -rf /',
config,
new Set(['echo']),
);
expect(result.allAllowed).toBe(false);
expect(result.disallowedCommands).toEqual(['rm -rf /']);
});
});
});
describe('getCommandRoots', () => {
it('should return a single command', () => {
expect(getCommandRoots('ls -l')).toEqual(['ls']);
});
it('should handle paths and return the binary name', () => {
expect(getCommandRoots('/usr/local/bin/node script.js')).toEqual(['node']);
});
it('should return an empty array for an empty string', () => {
expect(getCommandRoots('')).toEqual([]);
});
it('should handle a mix of operators', () => {
const result = getCommandRoots('a;b|c&&d||e&f');
expect(result).toEqual(['a', 'b', 'c', 'd', 'e', 'f']);
});
it('should correctly parse a chained command with quotes', () => {
const result = getCommandRoots('echo "hello" && git commit -m "feat"');
expect(result).toEqual(['echo', 'git']);
});
});
describe('stripShellWrapper', () => {
it('should strip sh -c with quotes', () => {
expect(stripShellWrapper('sh -c "ls -l"')).toEqual('ls -l');
});
it('should strip bash -c with extra whitespace', () => {
expect(stripShellWrapper(' bash -c "ls -l" ')).toEqual('ls -l');
});
it('should strip zsh -c without quotes', () => {
expect(stripShellWrapper('zsh -c ls -l')).toEqual('ls -l');
});
it('should strip cmd.exe /c', () => {
expect(stripShellWrapper('cmd.exe /c "dir"')).toEqual('dir');
});
it('should not strip anything if no wrapper is present', () => {
expect(stripShellWrapper('ls -l')).toEqual('ls -l');
});
});

359
packages/core/src/utils/shell-utils.ts Normal file
View File

@@ -0,0 +1,359 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { Config } from '../config/config.js';
/**
* Splits a shell command into a list of individual commands, respecting quotes.
* This is used to separate chained commands (e.g., using &&, ||, ;).
* @param command The shell command string to parse
* @returns An array of individual command strings
*/
export function splitCommands(command: string): string[] {
const commands: string[] = [];
let currentCommand = '';
let inSingleQuotes = false;
let inDoubleQuotes = false;
let i = 0;
while (i < command.length) {
const char = command[i];
const nextChar = command[i + 1];
if (char === '\\' && i < command.length - 1) {
currentCommand += char + command[i + 1];
i += 2;
continue;
}
if (char === "'" && !inDoubleQuotes) {
inSingleQuotes = !inSingleQuotes;
} else if (char === '"' && !inSingleQuotes) {
inDoubleQuotes = !inDoubleQuotes;
}
if (!inSingleQuotes && !inDoubleQuotes) {
if (
(char === '&' && nextChar === '&') ||
(char === '|' && nextChar === '|')
) {
commands.push(currentCommand.trim());
currentCommand = '';
i++; // Skip the next character
} else if (char === ';' || char === '&' || char === '|') {
commands.push(currentCommand.trim());
currentCommand = '';
} else {
currentCommand += char;
}
} else {
currentCommand += char;
}
i++;
}
if (currentCommand.trim()) {
commands.push(currentCommand.trim());
}
return commands.filter(Boolean); // Filter out any empty strings
}
/**
* Extracts the root command from a given shell command string.
* This is used to identify the base command for permission checks.
* @param command The shell command string to parse
* @returns The root command name, or undefined if it cannot be determined
* @example getCommandRoot("ls -la /tmp") returns "ls"
* @example getCommandRoot("git status && npm test") returns "git"
*/
export function getCommandRoot(command: string): string | undefined {
const trimmedCommand = command.trim();
if (!trimmedCommand) {
return undefined;
}
// This regex is designed to find the first "word" of a command,
// while respecting quotes. It looks for a sequence of non-whitespace
// characters that are not inside quotes.
const match = trimmedCommand.match(/^"([^"]+)"|^'([^']+)'|^(\S+)/);
if (match) {
// The first element in the match array is the full match.
// The subsequent elements are the capture groups.
// We prefer a captured group because it will be unquoted.
const commandRoot = match[1] || match[2] || match[3];
if (commandRoot) {
// If the command is a path, return the last component.
return commandRoot.split(/[\\/]/).pop();
}
}
return undefined;
}
export function getCommandRoots(command: string): string[] {
if (!command) {
return [];
}
return splitCommands(command)
.map((c) => getCommandRoot(c))
.filter((c): c is string => !!c);
}
export function stripShellWrapper(command: string): string {
const pattern = /^\s*(?:sh|bash|zsh|cmd.exe)\s+(?:\/c|-c)\s+/;
const match = command.match(pattern);
if (match) {
let newCommand = command.substring(match[0].length).trim();
if (
(newCommand.startsWith('"') && newCommand.endsWith('"')) ||
(newCommand.startsWith("'") && newCommand.endsWith("'"))
) {
newCommand = newCommand.substring(1, newCommand.length - 1);
}
return newCommand;
}
return command.trim();
}
/**
* Detects command substitution patterns in a shell command, following bash quoting rules:
* - Single quotes ('): Everything literal, no substitution possible
* - Double quotes ("): Command substitution with $() and backticks unless escaped with \
* - No quotes: Command substitution with $(), <(), and backticks
* @param command The shell command string to check
* @returns true if command substitution would be executed by bash
*/
export function detectCommandSubstitution(command: string): boolean {
let inSingleQuotes = false;
let inDoubleQuotes = false;
let inBackticks = false;
let i = 0;
while (i < command.length) {
const char = command[i];
const nextChar = command[i + 1];
// Handle escaping - only works outside single quotes
if (char === '\\' && !inSingleQuotes) {
i += 2; // Skip the escaped character
continue;
}
// Handle quote state changes
if (char === "'" && !inDoubleQuotes && !inBackticks) {
inSingleQuotes = !inSingleQuotes;
} else if (char === '"' && !inSingleQuotes && !inBackticks) {
inDoubleQuotes = !inDoubleQuotes;
} else if (char === '`' && !inSingleQuotes) {
// Backticks work outside single quotes (including in double quotes)
inBackticks = !inBackticks;
}
// Check for command substitution patterns that would be executed
if (!inSingleQuotes) {
// $(...) command substitution - works in double quotes and unquoted
if (char === '$' && nextChar === '(') {
return true;
}
// <(...) process substitution - works unquoted only (not in double quotes)
if (char === '<' && nextChar === '(' && !inDoubleQuotes && !inBackticks) {
return true;
}
// Backtick command substitution - check for opening backtick
// (We track the state above, so this catches the start of backtick substitution)
if (char === '`' && !inBackticks) {
return true;
}
}
i++;
}
return false;
}
/**
* Checks a shell command against security policies and allowlists.
*
* This function operates in one of two modes depending on the presence of
* the `sessionAllowlist` parameter:
*
* 1. **"Default Deny" Mode (sessionAllowlist is provided):** This is the
* strictest mode, used for user-defined scripts like custom commands.
* A command is only permitted if it is found on the global `coreTools`
* allowlist OR the provided `sessionAllowlist`. It must not be on the
* global `excludeTools` blocklist.
*
* 2. **"Default Allow" Mode (sessionAllowlist is NOT provided):** This mode
* is used for direct tool invocations (e.g., by the model). If a strict
* global `coreTools` allowlist exists, commands must be on it. Otherwise,
* any command is permitted as long as it is not on the `excludeTools`
* blocklist.
*
* @param command The shell command string to validate.
* @param config The application configuration.
* @param sessionAllowlist A session-level list of approved commands. Its
* presence activates "Default Deny" mode.
* @returns An object detailing which commands are not allowed.
*/
export function checkCommandPermissions(
command: string,
config: Config,
sessionAllowlist?: Set<string>,
): {
allAllowed: boolean;
disallowedCommands: string[];
blockReason?: string;
isHardDenial?: boolean;
} {
// Disallow command substitution for security.
if (detectCommandSubstitution(command)) {
return {
allAllowed: false,
disallowedCommands: [command],
blockReason:
'Command substitution using $(), <(), or >() is not allowed for security reasons',
isHardDenial: true,
};
}
const SHELL_TOOL_NAMES = ['run_shell_command', 'ShellTool'];
const normalize = (cmd: string): string => cmd.trim().replace(/\s+/g, ' ');
const isPrefixedBy = (cmd: string, prefix: string): boolean => {
if (!cmd.startsWith(prefix)) {
return false;
}
return cmd.length === prefix.length || cmd[prefix.length] === ' ';
};
const extractCommands = (tools: string[]): string[] =>
tools.flatMap((tool) => {
for (const toolName of SHELL_TOOL_NAMES) {
if (tool.startsWith(`${toolName}(`) && tool.endsWith(')')) {
return [normalize(tool.slice(toolName.length + 1, -1))];
}
}
return [];
});
const coreTools = config.getCoreTools() || [];
const excludeTools = config.getExcludeTools() || [];
const commandsToValidate = splitCommands(command).map(normalize);
// 1. Blocklist Check (Highest Priority)
if (SHELL_TOOL_NAMES.some((name) => excludeTools.includes(name))) {
return {
allAllowed: false,
disallowedCommands: commandsToValidate,
blockReason: 'Shell tool is globally disabled in configuration',
isHardDenial: true,
};
}
const blockedCommands = extractCommands(excludeTools);
for (const cmd of commandsToValidate) {
if (blockedCommands.some((blocked) => isPrefixedBy(cmd, blocked))) {
return {
allAllowed: false,
disallowedCommands: [cmd],
blockReason: `Command '${cmd}' is blocked by configuration`,
isHardDenial: true,
};
}
}
const globallyAllowedCommands = extractCommands(coreTools);
const isWildcardAllowed = SHELL_TOOL_NAMES.some((name) =>
coreTools.includes(name),
);
// If there's a global wildcard, all commands are allowed at this point
// because they have already passed the blocklist check.
if (isWildcardAllowed) {
return { allAllowed: true, disallowedCommands: [] };
}
if (sessionAllowlist) {
// "DEFAULT DENY" MODE: A session allowlist is provided.
// All commands must be in either the session or global allowlist.
const disallowedCommands: string[] = [];
for (const cmd of commandsToValidate) {
const isSessionAllowed = [...sessionAllowlist].some((allowed) =>
isPrefixedBy(cmd, normalize(allowed)),
);
if (isSessionAllowed) continue;
const isGloballyAllowed = globallyAllowedCommands.some((allowed) =>
isPrefixedBy(cmd, allowed),
);
if (isGloballyAllowed) continue;
disallowedCommands.push(cmd);
}
if (disallowedCommands.length > 0) {
return {
allAllowed: false,
disallowedCommands,
blockReason: `Command(s) not on the global or session allowlist.`,
isHardDenial: false, // This is a soft denial; confirmation is possible.
};
}
} else {
// "DEFAULT ALLOW" MODE: No session allowlist.
const hasSpecificAllowedCommands = globallyAllowedCommands.length > 0;
if (hasSpecificAllowedCommands) {
const disallowedCommands: string[] = [];
for (const cmd of commandsToValidate) {
const isGloballyAllowed = globallyAllowedCommands.some((allowed) =>
isPrefixedBy(cmd, allowed),
);
if (!isGloballyAllowed) {
disallowedCommands.push(cmd);
}
}
if (disallowedCommands.length > 0) {
return {
allAllowed: false,
disallowedCommands,
blockReason: `Command(s) not in the allowed commands list.`,
isHardDenial: false, // This is a soft denial.
};
}
}
// If no specific global allowlist exists, and it passed the blocklist,
// the command is allowed by default.
}
// If all checks for the current mode pass, the command is allowed.
return { allAllowed: true, disallowedCommands: [] };
}
/**
* Determines whether a given shell command is allowed to execute based on
* the tool's configuration including allowlists and blocklists.
*
* This function operates in "default allow" mode. It is a wrapper around
* `checkCommandPermissions`.
*
* @param command The shell command string to validate.
* @param config The application configuration.
* @returns An object with 'allowed' boolean and optional 'reason' string if not allowed.
*/
export function isCommandAllowed(
command: string,
config: Config,
): { allowed: boolean; reason?: string } {
// By not providing a sessionAllowlist, we invoke "default allow" behavior.
const { allAllowed, blockReason } = checkCommandPermissions(command, config);
if (allAllowed) {
return { allowed: true };
}
return { allowed: false, reason: blockReason };
}

2
packages/core/src/utils/summarizer.test.ts
View File

@@ -121,7 +121,7 @@ describe('summarizers', () => {
await summarizeToolOutput(longText, mockGeminiClient, abortSignal, 1000);
const expectedPrompt = `Summarize the following tool output to be a maximum of 1000 characters. The summary should be concise and capture the main points of the tool output.
const expectedPrompt = `Summarize the following tool output to be a maximum of 1000 tokens. The summary should be concise and capture the main points of the tool output.
The summarization should be done based on the content that is provided. Here are the basic rules to follow:
1. If the text is a directory listing or any output that is structural, use the history of the conversation to understand the context. Using this context try to understand what information we need from the tool output and return that as a response.

59
packages/core/src/utils/summarizer.ts
View File

@@ -12,7 +12,7 @@ import {
} from '@google/genai';
import { GeminiClient } from '../core/client.js';
import { DEFAULT_GEMINI_FLASH_MODEL } from '../config/models.js';
import { PartListUnion } from '@google/genai';
import { getResponseText, partToString } from './partUtils.js';
/**
* A function that summarizes the result of a tool execution.
@@ -40,46 +40,7 @@ export const defaultSummarizer: Summarizer = (
_abortSignal: AbortSignal,
) => Promise.resolve(JSON.stringify(result.llmContent));
// TODO: Move both these functions to utils
function partToString(part: PartListUnion): string {
if (!part) {
return '';
}
if (typeof part === 'string') {
return part;
}
if (Array.isArray(part)) {
return part.map(partToString).join('');
}
if ('text' in part) {
return part.text ?? '';
}
return '';
}
function getResponseText(response: GenerateContentResponse): string | null {
if (response.candidates && response.candidates.length > 0) {
const candidate = response.candidates[0];
if (
candidate.content &&
candidate.content.parts &&
candidate.content.parts.length > 0
) {
return candidate.content.parts
.filter((part) => part.text)
.map((part) => part.text)
.join('');
}
}
return null;
}
const toolOutputSummarizerModel = DEFAULT_GEMINI_FLASH_MODEL;
const toolOutputSummarizerConfig: GenerateContentConfig = {
maxOutputTokens: 2000,
};
const SUMMARIZE_TOOL_OUTPUT_PROMPT = `Summarize the following tool output to be a maximum of {maxLength} characters. The summary should be concise and capture the main points of the tool output.
const SUMMARIZE_TOOL_OUTPUT_PROMPT = `Summarize the following tool output to be a maximum of {maxOutputTokens} tokens. The summary should be concise and capture the main points of the tool output.
The summarization should be done based on the content that is provided. Here are the basic rules to follow:
1. If the text is a directory listing or any output that is structural, use the history of the conversation to understand the context. Using this context try to understand what information we need from the tool output and return that as a response.
@@ -104,24 +65,28 @@ export async function summarizeToolOutput(
textToSummarize: string,
geminiClient: GeminiClient,
abortSignal: AbortSignal,
maxLength: number = 2000,
maxOutputTokens: number = 2000,
): Promise<string> {
if (!textToSummarize || textToSummarize.length < maxLength) {
// There is going to be a slight difference here since we are comparing length of string with maxOutputTokens.
// This is meant to be a ballpark estimation of if we need to summarize the tool output.
if (!textToSummarize || textToSummarize.length < maxOutputTokens) {
return textToSummarize;
}
const prompt = SUMMARIZE_TOOL_OUTPUT_PROMPT.replace(
'{maxLength}',
String(maxLength),
'{maxOutputTokens}',
String(maxOutputTokens),
).replace('{textToSummarize}', textToSummarize);
const contents: Content[] = [{ role: 'user', parts: [{ text: prompt }] }];
const toolOutputSummarizerConfig: GenerateContentConfig = {
maxOutputTokens,
};
try {
const parsedResponse = (await geminiClient.generateContent(
contents,
toolOutputSummarizerConfig,
abortSignal,
toolOutputSummarizerModel,
DEFAULT_GEMINI_FLASH_MODEL,
)) as unknown as GenerateContentResponse;
return getResponseText(parsedResponse) || textToSummarize;
} catch (error) {

496
packages/core/src/utils/systemEncoding.test.ts Normal file
View File

@@ -0,0 +1,496 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { vi, describe, it, expect, beforeEach, afterEach } from 'vitest';
import { execSync } from 'child_process';
import * as os from 'os';
import { detect as chardetDetect } from 'chardet';
// Mock dependencies
vi.mock('child_process');
vi.mock('os');
vi.mock('chardet');
// Import the functions we want to test after refactoring
import {
getCachedEncodingForBuffer,
getSystemEncoding,
windowsCodePageToEncoding,
detectEncodingFromBuffer,
resetEncodingCache,
} from './systemEncoding.js';
describe('Shell Command Processor - Encoding Functions', () => {
let consoleWarnSpy: ReturnType<typeof vi.spyOn>;
let mockedExecSync: ReturnType<typeof vi.mocked<typeof execSync>>;
let mockedOsPlatform: ReturnType<typeof vi.mocked<() => string>>;
let mockedChardetDetect: ReturnType<typeof vi.mocked<typeof chardetDetect>>;
beforeEach(() => {
consoleWarnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
mockedExecSync = vi.mocked(execSync);
mockedOsPlatform = vi.mocked(os.platform);
mockedChardetDetect = vi.mocked(chardetDetect);
// Reset the encoding cache before each test
resetEncodingCache();
// Clear environment variables that might affect tests
delete process.env.LC_ALL;
delete process.env.LC_CTYPE;
delete process.env.LANG;
});
afterEach(() => {
vi.restoreAllMocks();
resetEncodingCache();
});
describe('windowsCodePageToEncoding', () => {
it('should map common Windows code pages correctly', () => {
expect(windowsCodePageToEncoding(437)).toBe('cp437');
expect(windowsCodePageToEncoding(850)).toBe('cp850');
expect(windowsCodePageToEncoding(65001)).toBe('utf-8');
expect(windowsCodePageToEncoding(1252)).toBe('windows-1252');
expect(windowsCodePageToEncoding(932)).toBe('shift_jis');
expect(windowsCodePageToEncoding(936)).toBe('gb2312');
expect(windowsCodePageToEncoding(949)).toBe('euc-kr');
expect(windowsCodePageToEncoding(950)).toBe('big5');
expect(windowsCodePageToEncoding(1200)).toBe('utf-16le');
expect(windowsCodePageToEncoding(1201)).toBe('utf-16be');
});
it('should return null for unmapped code pages and warn', () => {
expect(windowsCodePageToEncoding(99999)).toBe(null);
expect(consoleWarnSpy).toHaveBeenCalledWith(
'Unable to determine encoding for windows code page 99999.',
);
});
it('should handle all Windows-specific code pages', () => {
expect(windowsCodePageToEncoding(874)).toBe('windows-874');
expect(windowsCodePageToEncoding(1250)).toBe('windows-1250');
expect(windowsCodePageToEncoding(1251)).toBe('windows-1251');
expect(windowsCodePageToEncoding(1253)).toBe('windows-1253');
expect(windowsCodePageToEncoding(1254)).toBe('windows-1254');
expect(windowsCodePageToEncoding(1255)).toBe('windows-1255');
expect(windowsCodePageToEncoding(1256)).toBe('windows-1256');
expect(windowsCodePageToEncoding(1257)).toBe('windows-1257');
expect(windowsCodePageToEncoding(1258)).toBe('windows-1258');
});
});
describe('detectEncodingFromBuffer', () => {
it('should detect encoding using chardet successfully', () => {
const buffer = Buffer.from('test content', 'utf8');
mockedChardetDetect.mockReturnValue('UTF-8');
const result = detectEncodingFromBuffer(buffer);
expect(result).toBe('utf-8');
expect(mockedChardetDetect).toHaveBeenCalledWith(buffer);
});
it('should handle chardet returning mixed case encoding', () => {
const buffer = Buffer.from('test content', 'utf8');
mockedChardetDetect.mockReturnValue('ISO-8859-1');
const result = detectEncodingFromBuffer(buffer);
expect(result).toBe('iso-8859-1');
});
it('should return null when chardet fails', () => {
const buffer = Buffer.from('test content', 'utf8');
mockedChardetDetect.mockImplementation(() => {
throw new Error('Detection failed');
});
const result = detectEncodingFromBuffer(buffer);
expect(result).toBe(null);
expect(consoleWarnSpy).toHaveBeenCalledWith(
'Failed to detect encoding with chardet:',
expect.any(Error),
);
});
it('should return null when chardet returns null', () => {
const buffer = Buffer.from('test content', 'utf8');
mockedChardetDetect.mockReturnValue(null);
const result = detectEncodingFromBuffer(buffer);
expect(result).toBe(null);
});
it('should return null when chardet returns non-string', () => {
const buffer = Buffer.from('test content', 'utf8');
mockedChardetDetect.mockReturnValue([
'utf-8',
'iso-8859-1',
] as unknown as string);
const result = detectEncodingFromBuffer(buffer);
expect(result).toBe(null);
});
});
describe('getSystemEncoding - Windows', () => {
beforeEach(() => {
mockedOsPlatform.mockReturnValue('win32');
});
it('should parse Windows chcp output correctly', () => {
mockedExecSync.mockReturnValue('Active code page: 65001');
const result = getSystemEncoding();
expect(result).toBe('utf-8');
expect(mockedExecSync).toHaveBeenCalledWith('chcp', { encoding: 'utf8' });
});
it('should handle different chcp output formats', () => {
mockedExecSync.mockReturnValue('Current code page: 1252');
const result = getSystemEncoding();
expect(result).toBe('windows-1252');
});
it('should handle chcp output with extra whitespace', () => {
mockedExecSync.mockReturnValue('Active code page: 437 ');
const result = getSystemEncoding();
expect(result).toBe('cp437');
});
it('should return null when chcp command fails', () => {
mockedExecSync.mockImplementation(() => {
throw new Error('Command failed');
});
const result = getSystemEncoding();
expect(result).toBe(null);
expect(consoleWarnSpy).toHaveBeenCalledWith(
expect.stringContaining(
"Failed to get Windows code page using 'chcp' command",
),
);
});
it('should return null when chcp output cannot be parsed', () => {
mockedExecSync.mockReturnValue('Unexpected output format');
const result = getSystemEncoding();
expect(result).toBe(null);
expect(consoleWarnSpy).toHaveBeenCalledWith(
expect.stringContaining(
"Failed to get Windows code page using 'chcp' command",
),
);
});
it('should return null when code page is not a number', () => {
mockedExecSync.mockReturnValue('Active code page: abc');
const result = getSystemEncoding();
expect(result).toBe(null);
expect(consoleWarnSpy).toHaveBeenCalledWith(
expect.stringContaining(
"Failed to get Windows code page using 'chcp' command",
),
);
});
it('should return null when code page maps to null', () => {
mockedExecSync.mockReturnValue('Active code page: 99999');
const result = getSystemEncoding();
expect(result).toBe(null);
// Should warn about unknown code page from windowsCodePageToEncoding
expect(consoleWarnSpy).toHaveBeenCalledWith(
'Unable to determine encoding for windows code page 99999.',
);
});
});
describe('getSystemEncoding - Unix-like', () => {
beforeEach(() => {
mockedOsPlatform.mockReturnValue('linux');
});
it('should parse locale from LC_ALL environment variable', () => {
process.env.LC_ALL = 'en_US.UTF-8';
const result = getSystemEncoding();
expect(result).toBe('utf-8');
});
it('should parse locale from LC_CTYPE when LC_ALL is not set', () => {
process.env.LC_CTYPE = 'fr_FR.ISO-8859-1';
const result = getSystemEncoding();
expect(result).toBe('iso-8859-1');
});
it('should parse locale from LANG when LC_ALL and LC_CTYPE are not set', () => {
process.env.LANG = 'de_DE.UTF-8';
const result = getSystemEncoding();
expect(result).toBe('utf-8');
});
it('should handle locale charmap command when environment variables are empty', () => {
mockedExecSync.mockReturnValue('UTF-8\n');
const result = getSystemEncoding();
expect(result).toBe('utf-8');
expect(mockedExecSync).toHaveBeenCalledWith('locale charmap', {
encoding: 'utf8',
});
});
it('should handle locale charmap with mixed case', () => {
mockedExecSync.mockReturnValue('ISO-8859-1\n');
const result = getSystemEncoding();
expect(result).toBe('iso-8859-1');
});
it('should return null when locale charmap fails', () => {
mockedExecSync.mockImplementation(() => {
throw new Error('Command failed');
});
const result = getSystemEncoding();
expect(result).toBe(null);
expect(consoleWarnSpy).toHaveBeenCalledWith(
'Failed to get locale charmap.',
);
});
it('should handle locale without encoding (no dot)', () => {
process.env.LANG = 'C';
const result = getSystemEncoding();
expect(result).toBe('c');
});
it('should handle empty locale environment variables', () => {
process.env.LC_ALL = '';
process.env.LC_CTYPE = '';
process.env.LANG = '';
mockedExecSync.mockReturnValue('UTF-8');
const result = getSystemEncoding();
expect(result).toBe('utf-8');
});
it('should return locale as-is when locale format has no dot', () => {
process.env.LANG = 'invalid_format';
const result = getSystemEncoding();
expect(result).toBe('invalid_format');
});
it('should prioritize LC_ALL over other environment variables', () => {
process.env.LC_ALL = 'en_US.UTF-8';
process.env.LC_CTYPE = 'fr_FR.ISO-8859-1';
process.env.LANG = 'de_DE.CP1252';
const result = getSystemEncoding();
expect(result).toBe('utf-8');
});
it('should prioritize LC_CTYPE over LANG', () => {
process.env.LC_CTYPE = 'fr_FR.ISO-8859-1';
process.env.LANG = 'de_DE.CP1252';
const result = getSystemEncoding();
expect(result).toBe('iso-8859-1');
});
});
describe('getEncodingForBuffer', () => {
beforeEach(() => {
mockedOsPlatform.mockReturnValue('linux');
});
it('should use cached system encoding on subsequent calls', () => {
process.env.LANG = 'en_US.UTF-8';
const buffer = Buffer.from('test');
// First call
const result1 = getCachedEncodingForBuffer(buffer);
expect(result1).toBe('utf-8');
// Change environment (should not affect cached result)
process.env.LANG = 'fr_FR.ISO-8859-1';
// Second call should use cached value
const result2 = getCachedEncodingForBuffer(buffer);
expect(result2).toBe('utf-8');
});
it('should fall back to buffer detection when system encoding fails', () => {
// No environment variables set
mockedExecSync.mockImplementation(() => {
throw new Error('locale command failed');
});
const buffer = Buffer.from('test');
mockedChardetDetect.mockReturnValue('ISO-8859-1');
const result = getCachedEncodingForBuffer(buffer);
expect(result).toBe('iso-8859-1');
expect(mockedChardetDetect).toHaveBeenCalledWith(buffer);
});
it('should fall back to utf-8 when both system and buffer detection fail', () => {
// System encoding fails
mockedExecSync.mockImplementation(() => {
throw new Error('locale command failed');
});
// Buffer detection fails
mockedChardetDetect.mockImplementation(() => {
throw new Error('chardet failed');
});
const buffer = Buffer.from('test');
const result = getCachedEncodingForBuffer(buffer);
expect(result).toBe('utf-8');
});
it('should not cache buffer detection results', () => {
// System encoding fails initially
mockedExecSync.mockImplementation(() => {
throw new Error('locale command failed');
});
const buffer1 = Buffer.from('test1');
const buffer2 = Buffer.from('test2');
mockedChardetDetect
.mockReturnValueOnce('ISO-8859-1')
.mockReturnValueOnce('UTF-16');
const result1 = getCachedEncodingForBuffer(buffer1);
const result2 = getCachedEncodingForBuffer(buffer2);
expect(result1).toBe('iso-8859-1');
expect(result2).toBe('utf-16');
expect(mockedChardetDetect).toHaveBeenCalledTimes(2);
});
it('should handle Windows system encoding', () => {
mockedOsPlatform.mockReturnValue('win32');
mockedExecSync.mockReturnValue('Active code page: 1252');
const buffer = Buffer.from('test');
const result = getCachedEncodingForBuffer(buffer);
expect(result).toBe('windows-1252');
});
it('should cache null system encoding result', () => {
// Reset the cache specifically for this test
resetEncodingCache();
// Ensure we're on Unix-like for this test
mockedOsPlatform.mockReturnValue('linux');
// System encoding detection returns null
mockedExecSync.mockImplementation(() => {
throw new Error('locale command failed');
});
const buffer1 = Buffer.from('test1');
const buffer2 = Buffer.from('test2');
mockedChardetDetect
.mockReturnValueOnce('ISO-8859-1')
.mockReturnValueOnce('UTF-16');
// Clear any previous calls from beforeEach setup or previous tests
mockedExecSync.mockClear();
const result1 = getCachedEncodingForBuffer(buffer1);
const result2 = getCachedEncodingForBuffer(buffer2);
// Should call execSync only once due to caching (null result is cached)
expect(mockedExecSync).toHaveBeenCalledTimes(1);
expect(result1).toBe('iso-8859-1');
expect(result2).toBe('utf-16');
// Call a third time to verify cache is still used
const buffer3 = Buffer.from('test3');
mockedChardetDetect.mockReturnValueOnce('UTF-32');
const result3 = getCachedEncodingForBuffer(buffer3);
// Still should be only one call to execSync
expect(mockedExecSync).toHaveBeenCalledTimes(1);
expect(result3).toBe('utf-32');
});
});
describe('Cross-platform behavior', () => {
it('should work correctly on macOS', () => {
mockedOsPlatform.mockReturnValue('darwin');
process.env.LANG = 'en_US.UTF-8';
const result = getSystemEncoding();
expect(result).toBe('utf-8');
});
it('should work correctly on other Unix-like systems', () => {
mockedOsPlatform.mockReturnValue('freebsd');
process.env.LANG = 'en_US.UTF-8';
const result = getSystemEncoding();
expect(result).toBe('utf-8');
});
it('should handle unknown platforms as Unix-like', () => {
mockedOsPlatform.mockReturnValue('unknown' as NodeJS.Platform);
process.env.LANG = 'en_US.UTF-8';
const result = getSystemEncoding();
expect(result).toBe('utf-8');
});
});
describe('Edge cases and error handling', () => {
it('should handle empty buffer gracefully', () => {
mockedOsPlatform.mockReturnValue('linux');
process.env.LANG = 'en_US.UTF-8';
const buffer = Buffer.alloc(0);
const result = getCachedEncodingForBuffer(buffer);
expect(result).toBe('utf-8');
});
it('should handle very large buffers', () => {
mockedOsPlatform.mockReturnValue('linux');
process.env.LANG = 'en_US.UTF-8';
const buffer = Buffer.alloc(1024 * 1024, 'a');
const result = getCachedEncodingForBuffer(buffer);
expect(result).toBe('utf-8');
});
it('should handle Unicode content', () => {
mockedOsPlatform.mockReturnValue('linux');
const unicodeText = '你好世界 🌍 ñoño';
// System encoding fails
mockedExecSync.mockImplementation(() => {
throw new Error('locale command failed');
});
mockedChardetDetect.mockReturnValue('UTF-8');
const buffer = Buffer.from(unicodeText, 'utf8');
const result = getCachedEncodingForBuffer(buffer);
expect(result).toBe('utf-8');
});
});
});

166
packages/core/src/utils/systemEncoding.ts Normal file
View File

@@ -0,0 +1,166 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { execSync } from 'child_process';
import os from 'os';
import { detect as chardetDetect } from 'chardet';
// Cache for system encoding to avoid repeated detection
// Use undefined to indicate "not yet checked" vs null meaning "checked but failed"
let cachedSystemEncoding: string | null | undefined = undefined;
/**
* Reset the encoding cache - useful for testing
*/
export function resetEncodingCache(): void {
cachedSystemEncoding = undefined;
}
/**
* Returns the system encoding, caching the result to avoid repeated system calls.
* If system encoding detection fails, falls back to detecting from the provided buffer.
* Note: Only the system encoding is cached - buffer-based detection runs for each buffer
* since different buffers may have different encodings.
* @param buffer A buffer to use for detecting encoding if system detection fails.
*/
export function getCachedEncodingForBuffer(buffer: Buffer): string {
// Cache system encoding detection since it's system-wide
if (cachedSystemEncoding === undefined) {
cachedSystemEncoding = getSystemEncoding();
}
// If we have a cached system encoding, use it
if (cachedSystemEncoding) {
return cachedSystemEncoding;
}
// Otherwise, detect from this specific buffer (don't cache this result)
return detectEncodingFromBuffer(buffer) || 'utf-8';
}
/**
* Detects the system encoding based on the platform.
* For Windows, it uses the 'chcp' command to get the current code page.
* For Unix-like systems, it checks environment variables like LC_ALL, LC_CTYPE, and LANG.
* If those are not set, it tries to run 'locale charmap' to get the encoding.
* If detection fails, it returns null.
* @returns The system encoding as a string, or null if detection fails.
*/
export function getSystemEncoding(): string | null {
// Windows
if (os.platform() === 'win32') {
try {
const output = execSync('chcp', { encoding: 'utf8' });
const match = output.match(/:\s*(\d+)/);
if (match) {
const codePage = parseInt(match[1], 10);
if (!isNaN(codePage)) {
return windowsCodePageToEncoding(codePage);
}
}
// Only warn if we can't parse the output format, not if windowsCodePageToEncoding fails
throw new Error(
`Unable to parse Windows code page from 'chcp' output "${output.trim()}". `,
);
} catch (error) {
console.warn(
`Failed to get Windows code page using 'chcp' command: ${error instanceof Error ? error.message : String(error)}. ` +
`Will attempt to detect encoding from command output instead.`,
);
}
return null;
}
// Unix-like
// Use environment variables LC_ALL, LC_CTYPE, and LANG to determine the
// system encoding. However, these environment variables might not always
// be set or accurate. Handle cases where none of these variables are set.
const env = process.env;
let locale = env.LC_ALL || env.LC_CTYPE || env.LANG || '';
// Fallback to querying the system directly when environment variables are missing
if (!locale) {
try {
locale = execSync('locale charmap', { encoding: 'utf8' })
.toString()
.trim();
} catch (_e) {
console.warn('Failed to get locale charmap.');
return null;
}
}
const match = locale.match(/\.(.+)/); // e.g., "en_US.UTF-8"
if (match && match[1]) {
return match[1].toLowerCase();
}
// Handle cases where locale charmap returns just the encoding name (e.g., "UTF-8")
if (locale && !locale.includes('.')) {
return locale.toLowerCase();
}
return null;
}
/**
* Converts a Windows code page number to a corresponding encoding name.
* @param cp The Windows code page number (e.g., 437, 850, etc.)
* @returns The corresponding encoding name as a string, or null if no mapping exists.
*/
export function windowsCodePageToEncoding(cp: number): string | null {
// Most common mappings; extend as needed
const map: { [key: number]: string } = {
437: 'cp437',
850: 'cp850',
852: 'cp852',
866: 'cp866',
874: 'windows-874',
932: 'shift_jis',
936: 'gb2312',
949: 'euc-kr',
950: 'big5',
1200: 'utf-16le',
1201: 'utf-16be',
1250: 'windows-1250',
1251: 'windows-1251',
1252: 'windows-1252',
1253: 'windows-1253',
1254: 'windows-1254',
1255: 'windows-1255',
1256: 'windows-1256',
1257: 'windows-1257',
1258: 'windows-1258',
65001: 'utf-8',
};
if (map[cp]) {
return map[cp];
}
console.warn(`Unable to determine encoding for windows code page ${cp}.`);
return null; // Return null if no mapping found
}
/**
* Attempts to detect encoding from a buffer using chardet.
* This is useful when system encoding detection fails.
* Returns the detected encoding in lowercase, or null if detection fails.
* @param buffer The buffer to analyze for encoding.
* @return The detected encoding as a lowercase string, or null if detection fails.
*/
export function detectEncodingFromBuffer(buffer: Buffer): string | null {
try {
const detected = chardetDetect(buffer);
if (detected && typeof detected === 'string') {
return detected.toLowerCase();
}
} catch (error) {
console.warn('Failed to detect encoding with chardet:', error);
}
return null;
}

34
packages/core/src/utils/textUtils.ts Normal file
View File

@@ -0,0 +1,34 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
/**
* Checks if a Buffer is likely binary by testing for the presence of a NULL byte.
* The presence of a NULL byte is a strong indicator that the data is not plain text.
* @param data The Buffer to check.
* @param sampleSize The number of bytes from the start of the buffer to test.
* @returns True if a NULL byte is found, false otherwise.
*/
export function isBinary(
data: Buffer | null | undefined,
sampleSize = 512,
): boolean {
if (!data) {
return false;
}
const sample = data.length > sampleSize ? data.subarray(0, sampleSize) : data;
for (const byte of sample) {
// The presence of a NULL byte (0x00) is one of the most reliable
// indicators of a binary file. Text files should not contain them.
if (byte === 0) {
return true;
}
}
// If no NULL bytes were found in the sample, we assume it's text.
return false;
}