abbyswag/qwen-code
1
0
Fork 0
mirror of https://github.com/QwenLM/qwen-code.git synced 2025-12-20 16:57:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

# 🚀 Sync Gemini CLI v0.2.1 - Major Feature Update (#483)

Browse Source
This commit is contained in:
tanzhenxin
2025-09-01 14:48:55 +08:00
committed by GitHub
parent 1610c1586e
commit 2572faf726
292 changed files with 19401 additions and 5941 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
packages/core/src/utils/browser.ts
View File

@@ -15,19 +15,22 @@ export function shouldAttemptBrowserLaunch(): boolean {
// A list of browser names that indicate we should not attempt to open a
// web browser for the user.
const browserBlocklist = ['www-browser'];
const browserEnv = process.env.BROWSER;
const browserEnv = process.env['BROWSER'];
if (browserEnv && browserBlocklist.includes(browserEnv)) {
return false;
}
// Common environment variables used in CI/CD or other non-interactive shells.
if (process.env.CI || process.env.DEBIAN_FRONTEND === 'noninteractive') {
if (
process.env['CI'] ||
process.env['DEBIAN_FRONTEND'] === 'noninteractive'
) {
return false;
}
// The presence of SSH_CONNECTION indicates a remote session.
// We should not attempt to launch a browser unless a display is explicitly available
// (checked below for Linux).
const isSSH = !!process.env.SSH_CONNECTION;
const isSSH = !!process.env['SSH_CONNECTION'];
// On Linux, the presence of a display server is a strong indicator of a GUI.
if (process.platform === 'linux') {

26
packages/core/src/utils/editCorrector.ts
View File

@@ -119,7 +119,7 @@ async function findLastEditTimestamp(
const { response } = part.functionResponse;
if (response && !('error' in response) && 'output' in response) {
id = part.functionResponse.id;
content = response.output;
content = response['output'];
}
}
@@ -411,10 +411,10 @@ Return ONLY the corrected target snippet in the specified JSON format with the k
if (
result &&
typeof result.corrected_target_snippet === 'string' &&
result.corrected_target_snippet.length > 0
typeof result['corrected_target_snippet'] === 'string' &&
result['corrected_target_snippet'].length > 0
) {
return result.corrected_target_snippet;
return result['corrected_target_snippet'];
} else {
return problematicSnippet;
}
@@ -499,10 +499,10 @@ Return ONLY the corrected string in the specified JSON format with the key 'corr
if (
result &&
typeof result.corrected_new_string === 'string' &&
result.corrected_new_string.length > 0
typeof result['corrected_new_string'] === 'string' &&
result['corrected_new_string'].length > 0
) {
return result.corrected_new_string;
return result['corrected_new_string'];
} else {
return originalNewString;
}
@@ -568,10 +568,10 @@ Return ONLY the corrected string in the specified JSON format with the key 'corr
if (
result &&
typeof result.corrected_new_string_escaping === 'string' &&
result.corrected_new_string_escaping.length > 0
typeof result['corrected_new_string_escaping'] === 'string' &&
result['corrected_new_string_escaping'].length > 0
) {
return result.corrected_new_string_escaping;
return result['corrected_new_string_escaping'];
} else {
return potentiallyProblematicNewString;
}
@@ -634,10 +634,10 @@ Return ONLY the corrected string in the specified JSON format with the key 'corr
if (
result &&
typeof result.corrected_string_escaping === 'string' &&
result.corrected_string_escaping.length > 0
typeof result['corrected_string_escaping'] === 'string' &&
result['corrected_string_escaping'].length > 0
) {
return result.corrected_string_escaping;
return result['corrected_string_escaping'];
} else {
return potentiallyProblematicString;
}

20
packages/core/src/utils/editor.test.ts
View File

@@ -33,7 +33,7 @@ const originalPlatform = process.platform;
describe('editor utils', () => {
beforeEach(() => {
vi.clearAllMocks();
delete process.env.SANDBOX;
vi.unstubAllEnvs();
Object.defineProperty(process, 'platform', {
value: originalPlatform,
writable: true,
@@ -42,7 +42,7 @@ describe('editor utils', () => {
afterEach(() => {
vi.restoreAllMocks();
delete process.env.SANDBOX;
vi.unstubAllEnvs();
Object.defineProperty(process, 'platform', {
value: originalPlatform,
writable: true,
@@ -461,7 +461,7 @@ describe('editor utils', () => {
describe('allowEditorTypeInSandbox', () => {
it('should allow vim in sandbox mode', () => {
process.env.SANDBOX = 'sandbox';
vi.stubEnv('SANDBOX', 'sandbox');
expect(allowEditorTypeInSandbox('vim')).toBe(true);
});
@@ -470,7 +470,7 @@ describe('editor utils', () => {
});
it('should allow emacs in sandbox mode', () => {
process.env.SANDBOX = 'sandbox';
vi.stubEnv('SANDBOX', 'sandbox');
expect(allowEditorTypeInSandbox('emacs')).toBe(true);
});
@@ -479,7 +479,7 @@ describe('editor utils', () => {
});
it('should allow neovim in sandbox mode', () => {
process.env.SANDBOX = 'sandbox';
vi.stubEnv('SANDBOX', 'sandbox');
expect(allowEditorTypeInSandbox('neovim')).toBe(true);
});
@@ -496,7 +496,7 @@ describe('editor utils', () => {
];
for (const editor of guiEditors) {
it(`should not allow ${editor} in sandbox mode`, () => {
process.env.SANDBOX = 'sandbox';
vi.stubEnv('SANDBOX', 'sandbox');
expect(allowEditorTypeInSandbox(editor)).toBe(false);
});
@@ -533,25 +533,25 @@ describe('editor utils', () => {
it('should return false for vscode when installed and in sandbox mode', () => {
(execSync as Mock).mockReturnValue(Buffer.from('/usr/bin/code'));
process.env.SANDBOX = 'sandbox';
vi.stubEnv('SANDBOX', 'sandbox');
expect(isEditorAvailable('vscode')).toBe(false);
});
it('should return true for vim when installed and in sandbox mode', () => {
(execSync as Mock).mockReturnValue(Buffer.from('/usr/bin/vim'));
process.env.SANDBOX = 'sandbox';
vi.stubEnv('SANDBOX', 'sandbox');
expect(isEditorAvailable('vim')).toBe(true);
});
it('should return true for emacs when installed and in sandbox mode', () => {
(execSync as Mock).mockReturnValue(Buffer.from('/usr/bin/emacs'));
process.env.SANDBOX = 'sandbox';
vi.stubEnv('SANDBOX', 'sandbox');
expect(isEditorAvailable('emacs')).toBe(true);
});
it('should return true for neovim when installed and in sandbox mode', () => {
(execSync as Mock).mockReturnValue(Buffer.from('/usr/bin/nvim'));
process.env.SANDBOX = 'sandbox';
vi.stubEnv('SANDBOX', 'sandbox');
expect(isEditorAvailable('neovim')).toBe(true);
});
});

2
packages/core/src/utils/editor.ts
View File

@@ -72,7 +72,7 @@ export function checkHasEditorType(editor: EditorType): boolean {
}
export function allowEditorTypeInSandbox(editor: EditorType): boolean {
const notUsingSandbox = !process.env.SANDBOX;
const notUsingSandbox = !process.env['SANDBOX'];
if (['vscode', 'vscodium', 'windsurf', 'cursor', 'zed'].includes(editor)) {
return notUsingSandbox;
}

6
packages/core/src/utils/environmentContext.test.ts
View File

@@ -97,7 +97,7 @@ describe('getEnvironmentContext', () => {
}),
getFileService: vi.fn(),
getFullContext: vi.fn().mockReturnValue(false),
getToolRegistry: vi.fn().mockResolvedValue(mockToolRegistry),
getToolRegistry: vi.fn().mockReturnValue(mockToolRegistry),
};
vi.mocked(getFolderStructure).mockResolvedValue('Mock Folder Structure');
@@ -115,8 +115,8 @@ describe('getEnvironmentContext', () => {
expect(parts.length).toBe(1);
const context = parts[0].text;
// Use a more flexible date assertion that works with different locales
expect(context).toMatch(/Today's date is .*2025.*/);
expect(context).toContain("Today's date is");
expect(context).toContain("(formatted according to the user's locale)");
expect(context).toContain(`My operating system is: ${process.platform}`);
expect(context).toContain(
"I'm currently working in the directory: /test/dir",

4
packages/core/src/utils/environmentContext.ts
View File

@@ -62,13 +62,13 @@ export async function getEnvironmentContext(config: Config): Promise<Part[]> {
const context = `
This is the Qwen Code. We are setting up the context for our chat.
Today's date is ${today}.
Today's date is ${today} (formatted according to the user's locale).
My operating system is: ${platform}
${directoryContext}
`.trim();
const initialParts: Part[] = [{ text: context }];
const toolRegistry = await config.getToolRegistry();
const toolRegistry = config.getToolRegistry();
// Add full file context if the flag is set
if (config.getFullContext()) {

23
packages/core/src/utils/fileUtils.test.ts
View File

@@ -26,6 +26,7 @@ import {
detectFileType,
processSingleFileContent,
} from './fileUtils.js';
import { StandardFileSystemService } from '../services/fileSystemService.js';
vi.mock('mime-types', () => ({
default: { lookup: vi.fn() },
@@ -280,6 +281,7 @@ describe('fileUtils', () => {
const result = await processSingleFileContent(
testTextFilePath,
tempRootDir,
new StandardFileSystemService(),
);
expect(result.llmContent).toBe(content);
expect(result.returnDisplay).toBe('');
@@ -290,6 +292,7 @@ describe('fileUtils', () => {
const result = await processSingleFileContent(
nonexistentFilePath,
tempRootDir,
new StandardFileSystemService(),
);
expect(result.error).toContain('File not found');
expect(result.returnDisplay).toContain('File not found');
@@ -303,6 +306,7 @@ describe('fileUtils', () => {
const result = await processSingleFileContent(
testTextFilePath,
tempRootDir,
new StandardFileSystemService(),
);
expect(result.error).toContain('Simulated read error');
expect(result.returnDisplay).toContain('Simulated read error');
@@ -317,6 +321,7 @@ describe('fileUtils', () => {
const result = await processSingleFileContent(
testImageFilePath,
tempRootDir,
new StandardFileSystemService(),
);
expect(result.error).toContain('Simulated image read error');
expect(result.returnDisplay).toContain('Simulated image read error');
@@ -329,6 +334,7 @@ describe('fileUtils', () => {
const result = await processSingleFileContent(
testImageFilePath,
tempRootDir,
new StandardFileSystemService(),
);
expect(
(result.llmContent as { inlineData: unknown }).inlineData,
@@ -350,6 +356,7 @@ describe('fileUtils', () => {
const result = await processSingleFileContent(
testPdfFilePath,
tempRootDir,
new StandardFileSystemService(),
);
expect(
(result.llmContent as { inlineData: unknown }).inlineData,
@@ -378,6 +385,7 @@ describe('fileUtils', () => {
const result = await processSingleFileContent(
testSvgFilePath,
tempRootDir,
new StandardFileSystemService(),
);
expect(result.llmContent).toBe(svgContent);
@@ -395,6 +403,7 @@ describe('fileUtils', () => {
const result = await processSingleFileContent(
testBinaryFilePath,
tempRootDir,
new StandardFileSystemService(),
);
expect(result.llmContent).toContain(
'Cannot display content of binary file',
@@ -403,7 +412,11 @@ describe('fileUtils', () => {
});
it('should handle path being a directory', async () => {
const result = await processSingleFileContent(directoryPath, tempRootDir);
const result = await processSingleFileContent(
directoryPath,
tempRootDir,
new StandardFileSystemService(),
);
expect(result.error).toContain('Path is a directory');
expect(result.returnDisplay).toContain('Path is a directory');
});
@@ -415,6 +428,7 @@ describe('fileUtils', () => {
const result = await processSingleFileContent(
testTextFilePath,
tempRootDir,
new StandardFileSystemService(),
5,
5,
); // Read lines 6-10
@@ -435,6 +449,7 @@ describe('fileUtils', () => {
const result = await processSingleFileContent(
testTextFilePath,
tempRootDir,
new StandardFileSystemService(),
10,
10,
);
@@ -454,6 +469,7 @@ describe('fileUtils', () => {
const result = await processSingleFileContent(
testTextFilePath,
tempRootDir,
new StandardFileSystemService(),
0,
10,
);
@@ -476,6 +492,7 @@ describe('fileUtils', () => {
const result = await processSingleFileContent(
testTextFilePath,
tempRootDir,
new StandardFileSystemService(),
);
expect(result.llmContent).toContain('Short line');
@@ -497,6 +514,7 @@ describe('fileUtils', () => {
const result = await processSingleFileContent(
testTextFilePath,
tempRootDir,
new StandardFileSystemService(),
0,
5,
);
@@ -515,6 +533,7 @@ describe('fileUtils', () => {
const result = await processSingleFileContent(
testTextFilePath,
tempRootDir,
new StandardFileSystemService(),
0,
11,
);
@@ -540,6 +559,7 @@ describe('fileUtils', () => {
const result = await processSingleFileContent(
testTextFilePath,
tempRootDir,
new StandardFileSystemService(),
0,
10,
);
@@ -558,6 +578,7 @@ describe('fileUtils', () => {
const result = await processSingleFileContent(
testTextFilePath,
tempRootDir,
new StandardFileSystemService(),
);
expect(result.error).toContain('File size exceeds the 20MB limit');

6
packages/core/src/utils/fileUtils.ts
View File

@@ -8,6 +8,7 @@ import fs from 'node:fs';
import path from 'node:path';
import { PartUnion } from '@google/genai';
import mime from 'mime-types';
import { FileSystemService } from '../services/fileSystemService.js';
// Constants for text file processing
const DEFAULT_MAX_LINES_TEXT_FILE = 2000;
@@ -223,6 +224,7 @@ export interface ProcessedFileReadResult {
export async function processSingleFileContent(
filePath: string,
rootDirectory: string,
fileSystemService: FileSystemService,
offset?: number,
limit?: number,
): Promise<ProcessedFileReadResult> {
@@ -279,14 +281,14 @@ export async function processSingleFileContent(
returnDisplay: `Skipped large SVG file (>1MB): ${relativePathForDisplay}`,
};
}
const content = await fs.promises.readFile(filePath, 'utf8');
const content = await fileSystemService.readTextFile(filePath);
return {
llmContent: content,
returnDisplay: `Read SVG as text: ${relativePathForDisplay}`,
};
}
case 'text': {
const content = await fs.promises.readFile(filePath, 'utf8');
const content = await fileSystemService.readTextFile(filePath);
const lines = content.split('\n');
const originalLineCount = lines.length;

573
packages/core/src/utils/filesearch/crawler.test.ts Normal file
View File

@@ -0,0 +1,573 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect, afterEach, vi, beforeEach } from 'vitest';
import * as fs from 'fs/promises';
import * as path from 'path';
import * as cache from './crawlCache.js';
import { crawl } from './crawler.js';
import { createTmpDir, cleanupTmpDir } from '@qwen-code/qwen-code-test-utils';
import { Ignore, loadIgnoreRules } from './ignore.js';
describe('crawler', () => {
let tmpDir: string;
afterEach(async () => {
if (tmpDir) {
await cleanupTmpDir(tmpDir);
}
vi.restoreAllMocks();
});
it('should use .geminiignore rules', async () => {
tmpDir = await createTmpDir({
'.geminiignore': 'dist/',
dist: ['ignored.js'],
src: ['not-ignored.js'],
});
const ignore = loadIgnoreRules({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: true,
ignoreDirs: [],
});
const results = await crawl({
crawlDirectory: tmpDir,
cwd: tmpDir,
ignore,
cache: false,
cacheTtl: 0,
});
expect(results).toEqual(
expect.arrayContaining([
'.',
'src/',
'.geminiignore',
'src/not-ignored.js',
]),
);
});
it('should combine .gitignore and .geminiignore rules', async () => {
tmpDir = await createTmpDir({
'.gitignore': 'dist/',
'.geminiignore': 'build/',
dist: ['ignored-by-git.js'],
build: ['ignored-by-gemini.js'],
src: ['not-ignored.js'],
});
const ignore = loadIgnoreRules({
projectRoot: tmpDir,
useGitignore: true,
useGeminiignore: true,
ignoreDirs: [],
});
const results = await crawl({
crawlDirectory: tmpDir,
cwd: tmpDir,
ignore,
cache: false,
cacheTtl: 0,
});
expect(results).toEqual(
expect.arrayContaining([
'.',
'src/',
'.geminiignore',
'.gitignore',
'src/not-ignored.js',
]),
);
});
it('should use ignoreDirs option', async () => {
tmpDir = await createTmpDir({
logs: ['some.log'],
src: ['main.js'],
});
const ignore = loadIgnoreRules({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: false,
ignoreDirs: ['logs'],
});
const results = await crawl({
crawlDirectory: tmpDir,
cwd: tmpDir,
ignore,
cache: false,
cacheTtl: 0,
});
expect(results).toEqual(
expect.arrayContaining(['.', 'src/', 'src/main.js']),
);
});
it('should handle negated directories', async () => {
tmpDir = await createTmpDir({
'.gitignore': ['build/**', '!build/public', '!build/public/**'].join(
'\n',
),
build: {
'private.js': '',
public: ['index.html'],
},
src: ['main.js'],
});
const ignore = loadIgnoreRules({
projectRoot: tmpDir,
useGitignore: true,
useGeminiignore: false,
ignoreDirs: [],
});
const results = await crawl({
crawlDirectory: tmpDir,
cwd: tmpDir,
ignore,
cache: false,
cacheTtl: 0,
});
expect(results).toEqual(
expect.arrayContaining([
'.',
'build/',
'build/public/',
'src/',
'.gitignore',
'build/public/index.html',
'src/main.js',
]),
);
});
it('should handle root-level file negation', async () => {
tmpDir = await createTmpDir({
'.gitignore': ['*.mk', '!Foo.mk'].join('\n'),
'bar.mk': '',
'Foo.mk': '',
});
const ignore = loadIgnoreRules({
projectRoot: tmpDir,
useGitignore: true,
useGeminiignore: false,
ignoreDirs: [],
});
const results = await crawl({
crawlDirectory: tmpDir,
cwd: tmpDir,
ignore,
cache: false,
cacheTtl: 0,
});
expect(results).toEqual(
expect.arrayContaining(['.', '.gitignore', 'Foo.mk', 'bar.mk']),
);
});
it('should handle directory negation with glob', async () => {
tmpDir = await createTmpDir({
'.gitignore': [
'third_party/**',
'!third_party/foo',
'!third_party/foo/bar',
'!third_party/foo/bar/baz_buffer',
].join('\n'),
third_party: {
foo: {
bar: {
baz_buffer: '',
},
},
ignore_this: '',
},
});
const ignore = loadIgnoreRules({
projectRoot: tmpDir,
useGitignore: true,
useGeminiignore: false,
ignoreDirs: [],
});
const results = await crawl({
crawlDirectory: tmpDir,
cwd: tmpDir,
ignore,
cache: false,
cacheTtl: 0,
});
expect(results).toEqual(
expect.arrayContaining([
'.',
'third_party/',
'third_party/foo/',
'third_party/foo/bar/',
'.gitignore',
'third_party/foo/bar/baz_buffer',
]),
);
});
it('should correctly handle negated patterns in .gitignore', async () => {
tmpDir = await createTmpDir({
'.gitignore': ['dist/**', '!dist/keep.js'].join('\n'),
dist: ['ignore.js', 'keep.js'],
src: ['main.js'],
});
const ignore = loadIgnoreRules({
projectRoot: tmpDir,
useGitignore: true,
useGeminiignore: false,
ignoreDirs: [],
});
const results = await crawl({
crawlDirectory: tmpDir,
cwd: tmpDir,
ignore,
cache: false,
cacheTtl: 0,
});
expect(results).toEqual(
expect.arrayContaining([
'.',
'dist/',
'src/',
'.gitignore',
'dist/keep.js',
'src/main.js',
]),
);
});
it('should initialize correctly when ignore files are missing', async () => {
tmpDir = await createTmpDir({
src: ['file1.js'],
});
const ignore = loadIgnoreRules({
projectRoot: tmpDir,
useGitignore: true,
useGeminiignore: true,
ignoreDirs: [],
});
const results = await crawl({
crawlDirectory: tmpDir,
cwd: tmpDir,
ignore,
cache: false,
cacheTtl: 0,
});
expect(results).toEqual(
expect.arrayContaining(['.', 'src/', 'src/file1.js']),
);
});
it('should handle empty or commented-only ignore files', async () => {
tmpDir = await createTmpDir({
'.gitignore': '# This is a comment\n\n \n',
src: ['main.js'],
});
const ignore = loadIgnoreRules({
projectRoot: tmpDir,
useGitignore: true,
useGeminiignore: false,
ignoreDirs: [],
});
const results = await crawl({
crawlDirectory: tmpDir,
cwd: tmpDir,
ignore,
cache: false,
cacheTtl: 0,
});
expect(results).toEqual(
expect.arrayContaining(['.', 'src/', '.gitignore', 'src/main.js']),
);
});
it('should always ignore the .git directory', async () => {
tmpDir = await createTmpDir({
'.git': ['config', 'HEAD'],
src: ['main.js'],
});
const ignore = loadIgnoreRules({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: false,
ignoreDirs: [],
});
const results = await crawl({
crawlDirectory: tmpDir,
cwd: tmpDir,
ignore,
cache: false,
cacheTtl: 0,
});
expect(results).toEqual(
expect.arrayContaining(['.', 'src/', 'src/main.js']),
);
});
describe('with in-memory cache', () => {
beforeEach(() => {
cache.clear();
vi.useFakeTimers();
});
afterEach(() => {
vi.useRealTimers();
});
it('should hit the cache for subsequent crawls', async () => {
tmpDir = await createTmpDir({ 'file1.js': '' });
const ignore = loadIgnoreRules({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: false,
ignoreDirs: [],
});
const options = {
crawlDirectory: tmpDir,
cwd: tmpDir,
ignore,
cache: true,
cacheTtl: 10,
};
const crawlSpy = vi.spyOn(cache, 'read');
await crawl(options);
expect(crawlSpy).toHaveBeenCalledTimes(1);
await crawl(options);
expect(crawlSpy).toHaveBeenCalledTimes(2);
// fdir should not have been called a second time.
// We can't spy on it directly, but we can check the cache was hit.
const cacheKey = cache.getCacheKey(
options.crawlDirectory,
options.ignore.getFingerprint(),
undefined,
);
expect(cache.read(cacheKey)).toBeDefined();
});
it('should miss the cache when ignore rules change', async () => {
tmpDir = await createTmpDir({
'.gitignore': 'a.txt',
'a.txt': '',
'b.txt': '',
});
const getIgnore = () =>
loadIgnoreRules({
projectRoot: tmpDir,
useGitignore: true,
useGeminiignore: false,
ignoreDirs: [],
});
const getOptions = (ignore: Ignore) => ({
crawlDirectory: tmpDir,
cwd: tmpDir,
ignore,
cache: true,
cacheTtl: 10000,
});
// Initial crawl to populate the cache
const ignore1 = getIgnore();
const results1 = await crawl(getOptions(ignore1));
expect(results1).toEqual(
expect.arrayContaining(['.', '.gitignore', 'b.txt']),
);
// Modify the ignore file
await fs.writeFile(path.join(tmpDir, '.gitignore'), 'b.txt');
// Second crawl should miss the cache and trigger a recrawl
const ignore2 = getIgnore();
const results2 = await crawl(getOptions(ignore2));
expect(results2).toEqual(
expect.arrayContaining(['.', '.gitignore', 'a.txt']),
);
});
it('should miss the cache after TTL expires', async () => {
tmpDir = await createTmpDir({ 'file1.js': '' });
const ignore = loadIgnoreRules({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: false,
ignoreDirs: [],
});
const options = {
crawlDirectory: tmpDir,
cwd: tmpDir,
ignore,
cache: true,
cacheTtl: 10, // 10 seconds
};
const readSpy = vi.spyOn(cache, 'read');
const writeSpy = vi.spyOn(cache, 'write');
await crawl(options);
expect(readSpy).toHaveBeenCalledTimes(1);
expect(writeSpy).toHaveBeenCalledTimes(1);
// Advance time past the TTL
await vi.advanceTimersByTimeAsync(11000);
await crawl(options);
expect(readSpy).toHaveBeenCalledTimes(2);
expect(writeSpy).toHaveBeenCalledTimes(2);
});
it('should miss the cache when maxDepth changes', async () => {
tmpDir = await createTmpDir({ 'file1.js': '' });
const ignore = loadIgnoreRules({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: false,
ignoreDirs: [],
});
const getOptions = (maxDepth?: number) => ({
crawlDirectory: tmpDir,
cwd: tmpDir,
ignore,
cache: true,
cacheTtl: 10000,
maxDepth,
});
const readSpy = vi.spyOn(cache, 'read');
const writeSpy = vi.spyOn(cache, 'write');
// 1. First crawl with maxDepth: 1
await crawl(getOptions(1));
expect(readSpy).toHaveBeenCalledTimes(1);
expect(writeSpy).toHaveBeenCalledTimes(1);
// 2. Second crawl with maxDepth: 2, should be a cache miss
await crawl(getOptions(2));
expect(readSpy).toHaveBeenCalledTimes(2);
expect(writeSpy).toHaveBeenCalledTimes(2);
// 3. Third crawl with maxDepth: 1 again, should be a cache hit.
await crawl(getOptions(1));
expect(readSpy).toHaveBeenCalledTimes(3);
expect(writeSpy).toHaveBeenCalledTimes(2); // No new write
});
});
describe('with maxDepth', () => {
beforeEach(async () => {
tmpDir = await createTmpDir({
'file-root.txt': '',
level1: {
'file-level1.txt': '',
level2: {
'file-level2.txt': '',
level3: {
'file-level3.txt': '',
},
},
},
});
});
const getCrawlResults = (maxDepth?: number) => {
const ignore = loadIgnoreRules({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: false,
ignoreDirs: [],
});
return crawl({
crawlDirectory: tmpDir,
cwd: tmpDir,
ignore,
cache: false,
cacheTtl: 0,
maxDepth,
});
};
it('should only crawl top-level files when maxDepth is 0', async () => {
const results = await getCrawlResults(0);
expect(results).toEqual(
expect.arrayContaining(['.', 'level1/', 'file-root.txt']),
);
});
it('should crawl one level deep when maxDepth is 1', async () => {
const results = await getCrawlResults(1);
expect(results).toEqual(
expect.arrayContaining([
'.',
'level1/',
'level1/level2/',
'file-root.txt',
'level1/file-level1.txt',
]),
);
});
it('should crawl two levels deep when maxDepth is 2', async () => {
const results = await getCrawlResults(2);
expect(results).toEqual(
expect.arrayContaining([
'.',
'level1/',
'level1/level2/',
'level1/level2/level3/',
'file-root.txt',
'level1/file-level1.txt',
'level1/level2/file-level2.txt',
]),
);
});
it('should perform a full recursive crawl when maxDepth is undefined', async () => {
const results = await getCrawlResults(undefined);
expect(results).toEqual(
expect.arrayContaining([
'.',
'level1/',
'level1/level2/',
'level1/level2/level3/',
'file-root.txt',
'level1/file-level1.txt',
'level1/level2/file-level2.txt',
'level1/level2/level3/file-level3.txt',
]),
);
});
});
});

85
packages/core/src/utils/filesearch/crawler.ts Normal file
View File

@@ -0,0 +1,85 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import path from 'node:path';
import { fdir } from 'fdir';
import { Ignore } from './ignore.js';
import * as cache from './crawlCache.js';
export interface CrawlOptions {
// The directory to start the crawl from.
crawlDirectory: string;
// The project's root directory, for path relativity.
cwd: string;
// The fdir maxDepth option.
maxDepth?: number;
// A pre-configured Ignore instance.
ignore: Ignore;
// Caching options.
cache: boolean;
cacheTtl: number;
}
function toPosixPath(p: string) {
return p.split(path.sep).join(path.posix.sep);
}
export async function crawl(options: CrawlOptions): Promise<string[]> {
if (options.cache) {
const cacheKey = cache.getCacheKey(
options.crawlDirectory,
options.ignore.getFingerprint(),
options.maxDepth,
);
const cachedResults = cache.read(cacheKey);
if (cachedResults) {
return cachedResults;
}
}
const posixCwd = toPosixPath(options.cwd);
const posixCrawlDirectory = toPosixPath(options.crawlDirectory);
let results: string[];
try {
const dirFilter = options.ignore.getDirectoryFilter();
const api = new fdir()
.withRelativePaths()
.withDirs()
.withPathSeparator('/') // Always use unix style paths
.exclude((_, dirPath) => {
const relativePath = path.posix.relative(posixCrawlDirectory, dirPath);
return dirFilter(`${relativePath}/`);
});
if (options.maxDepth !== undefined) {
api.withMaxDepth(options.maxDepth);
}
results = await api.crawl(options.crawlDirectory).withPromise();
} catch (_e) {
// The directory probably doesn't exist.
return [];
}
const relativeToCrawlDir = path.posix.relative(posixCwd, posixCrawlDirectory);
const relativeToCwdResults = results.map((p) =>
path.posix.join(relativeToCrawlDir, p),
);
if (options.cache) {
const cacheKey = cache.getCacheKey(
options.crawlDirectory,
options.ignore.getFingerprint(),
options.maxDepth,
);
cache.write(cacheKey, relativeToCwdResults, options.cacheTtl * 1000);
}
return relativeToCwdResults;
}

365
packages/core/src/utils/filesearch/fileSearch.test.ts
View File

@@ -4,17 +4,10 @@
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import * as fs from 'fs/promises';
import * as path from 'path';
import * as cache from './crawlCache.js';
import { FileSearch, AbortError, filter } from './fileSearch.js';
import { describe, it, expect, afterEach, vi } from 'vitest';
import { FileSearchFactory, AbortError, filter } from './fileSearch.js';
import { createTmpDir, cleanupTmpDir } from '@qwen-code/qwen-code-test-utils';
type FileSearchWithPrivateMethods = FileSearch & {
performCrawl: () => Promise<void>;
};
describe('FileSearch', () => {
let tmpDir: string;
afterEach(async () => {
@@ -31,13 +24,14 @@ describe('FileSearch', () => {
src: ['not-ignored.js'],
});
const fileSearch = new FileSearch({
const fileSearch = FileSearchFactory.create({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: true,
ignoreDirs: [],
cache: false,
cacheTtl: 0,
enableRecursiveFileSearch: true,
});
await fileSearch.initialize();
@@ -55,13 +49,14 @@ describe('FileSearch', () => {
src: ['not-ignored.js'],
});
const fileSearch = new FileSearch({
const fileSearch = FileSearchFactory.create({
projectRoot: tmpDir,
useGitignore: true,
useGeminiignore: true,
ignoreDirs: [],
cache: false,
cacheTtl: 0,
enableRecursiveFileSearch: true,
});
await fileSearch.initialize();
@@ -81,13 +76,14 @@ describe('FileSearch', () => {
src: ['main.js'],
});
const fileSearch = new FileSearch({
const fileSearch = FileSearchFactory.create({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: false,
ignoreDirs: ['logs'],
cache: false,
cacheTtl: 0,
enableRecursiveFileSearch: true,
});
await fileSearch.initialize();
@@ -108,13 +104,14 @@ describe('FileSearch', () => {
src: ['main.js'],
});
const fileSearch = new FileSearch({
const fileSearch = FileSearchFactory.create({
projectRoot: tmpDir,
useGitignore: true,
useGeminiignore: false,
ignoreDirs: [],
cache: false,
cacheTtl: 0,
enableRecursiveFileSearch: true,
});
await fileSearch.initialize();
@@ -139,13 +136,14 @@ describe('FileSearch', () => {
},
});
const fileSearch = new FileSearch({
const fileSearch = FileSearchFactory.create({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: false,
ignoreDirs: [],
cache: false,
cacheTtl: 0,
enableRecursiveFileSearch: true,
});
await fileSearch.initialize();
@@ -161,13 +159,14 @@ describe('FileSearch', () => {
'Foo.mk': '',
});
const fileSearch = new FileSearch({
const fileSearch = FileSearchFactory.create({
projectRoot: tmpDir,
useGitignore: true,
useGeminiignore: false,
ignoreDirs: [],
cache: false,
cacheTtl: 0,
enableRecursiveFileSearch: true,
});
await fileSearch.initialize();
@@ -194,13 +193,14 @@ describe('FileSearch', () => {
},
});
const fileSearch = new FileSearch({
const fileSearch = FileSearchFactory.create({
projectRoot: tmpDir,
useGitignore: true,
useGeminiignore: false,
ignoreDirs: [],
cache: false,
cacheTtl: 0,
enableRecursiveFileSearch: true,
});
await fileSearch.initialize();
@@ -222,13 +222,14 @@ describe('FileSearch', () => {
src: ['main.js'],
});
const fileSearch = new FileSearch({
const fileSearch = FileSearchFactory.create({
projectRoot: tmpDir,
useGitignore: true,
useGeminiignore: false,
ignoreDirs: [],
cache: false,
cacheTtl: 0,
enableRecursiveFileSearch: true,
});
await fileSearch.initialize();
@@ -250,13 +251,14 @@ describe('FileSearch', () => {
src: ['file1.js'],
});
const fileSearch = new FileSearch({
const fileSearch = FileSearchFactory.create({
projectRoot: tmpDir,
useGitignore: true,
useGeminiignore: true,
ignoreDirs: [],
cache: false,
cacheTtl: 0,
enableRecursiveFileSearch: true,
});
// Expect no errors to be thrown during initialization
@@ -275,13 +277,14 @@ describe('FileSearch', () => {
},
});
const fileSearch = new FileSearch({
const fileSearch = FileSearchFactory.create({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: false,
ignoreDirs: [],
cache: false,
cacheTtl: 0,
enableRecursiveFileSearch: true,
});
await fileSearch.initialize();
@@ -299,13 +302,14 @@ describe('FileSearch', () => {
},
});
const fileSearch = new FileSearch({
const fileSearch = FileSearchFactory.create({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: false,
ignoreDirs: [],
cache: false,
cacheTtl: 0,
enableRecursiveFileSearch: true,
});
await fileSearch.initialize();
@@ -319,13 +323,14 @@ describe('FileSearch', () => {
src: ['file1.js'],
});
const fileSearch = new FileSearch({
const fileSearch = FileSearchFactory.create({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: false,
ignoreDirs: [],
cache: false,
cacheTtl: 0,
enableRecursiveFileSearch: true,
});
await fileSearch.initialize();
@@ -346,170 +351,21 @@ describe('FileSearch', () => {
await expect(filterPromise).rejects.toThrow(AbortError);
});
describe('with in-memory cache', () => {
beforeEach(() => {
cache.clear();
it('should throw an error if search is called before initialization', async () => {
tmpDir = await createTmpDir({});
const fileSearch = FileSearchFactory.create({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: false,
ignoreDirs: [],
cache: false,
cacheTtl: 0,
enableRecursiveFileSearch: true,
});
afterEach(() => {
vi.useRealTimers();
});
it('should throw an error if search is called before initialization', async () => {
tmpDir = await createTmpDir({});
const fileSearch = new FileSearch({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: false,
ignoreDirs: [],
cache: false,
cacheTtl: 0,
});
await expect(fileSearch.search('')).rejects.toThrow(
'Engine not initialized. Call initialize() first.',
);
});
it('should hit the cache for subsequent searches', async () => {
tmpDir = await createTmpDir({ 'file1.js': '' });
const getOptions = () => ({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: false,
ignoreDirs: [],
cache: true,
cacheTtl: 10,
});
const fs1 = new FileSearch(getOptions());
const crawlSpy1 = vi.spyOn(
fs1 as FileSearchWithPrivateMethods,
'performCrawl',
);
await fs1.initialize();
expect(crawlSpy1).toHaveBeenCalledTimes(1);
// Second search should hit the cache because the options are identical
const fs2 = new FileSearch(getOptions());
const crawlSpy2 = vi.spyOn(
fs2 as FileSearchWithPrivateMethods,
'performCrawl',
);
await fs2.initialize();
expect(crawlSpy2).not.toHaveBeenCalled();
});
it('should miss the cache when ignore rules change', async () => {
tmpDir = await createTmpDir({
'.gitignore': 'a.txt',
'a.txt': '',
'b.txt': '',
});
const options = {
projectRoot: tmpDir,
useGitignore: true,
useGeminiignore: false,
ignoreDirs: [],
cache: true,
cacheTtl: 10000,
};
// Initial search to populate the cache
const fs1 = new FileSearch(options);
const crawlSpy1 = vi.spyOn(
fs1 as FileSearchWithPrivateMethods,
'performCrawl',
);
await fs1.initialize();
const results1 = await fs1.search('');
expect(crawlSpy1).toHaveBeenCalledTimes(1);
expect(results1).toEqual(['.gitignore', 'b.txt']);
// Modify the ignore file
await fs.writeFile(path.join(tmpDir, '.gitignore'), 'b.txt');
// Second search should miss the cache and trigger a recrawl
const fs2 = new FileSearch(options);
const crawlSpy2 = vi.spyOn(
fs2 as FileSearchWithPrivateMethods,
'performCrawl',
);
await fs2.initialize();
const results2 = await fs2.search('');
expect(crawlSpy2).toHaveBeenCalledTimes(1);
expect(results2).toEqual(['.gitignore', 'a.txt']);
});
it('should miss the cache after TTL expires', async () => {
vi.useFakeTimers();
tmpDir = await createTmpDir({ 'file1.js': '' });
const options = {
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: false,
ignoreDirs: [],
cache: true,
cacheTtl: 10, // 10 seconds
};
// Initial search to populate the cache
const fs1 = new FileSearch(options);
await fs1.initialize();
// Advance time past the TTL
await vi.advanceTimersByTimeAsync(11000);
// Second search should miss the cache and trigger a recrawl
const fs2 = new FileSearch(options);
const crawlSpy = vi.spyOn(
fs2 as FileSearchWithPrivateMethods,
'performCrawl',
);
await fs2.initialize();
expect(crawlSpy).toHaveBeenCalledTimes(1);
});
it('should miss the cache when maxDepth changes', async () => {
tmpDir = await createTmpDir({ 'file1.js': '' });
const getOptions = (maxDepth?: number) => ({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: false,
ignoreDirs: [],
cache: true,
cacheTtl: 10000,
maxDepth,
});
// 1. First search with maxDepth: 1, should trigger a crawl.
const fs1 = new FileSearch(getOptions(1));
const crawlSpy1 = vi.spyOn(
fs1 as FileSearchWithPrivateMethods,
'performCrawl',
);
await fs1.initialize();
expect(crawlSpy1).toHaveBeenCalledTimes(1);
// 2. Second search with maxDepth: 2, should be a cache miss and trigger a crawl.
const fs2 = new FileSearch(getOptions(2));
const crawlSpy2 = vi.spyOn(
fs2 as FileSearchWithPrivateMethods,
'performCrawl',
);
await fs2.initialize();
expect(crawlSpy2).toHaveBeenCalledTimes(1);
// 3. Third search with maxDepth: 1 again, should be a cache hit.
const fs3 = new FileSearch(getOptions(1));
const crawlSpy3 = vi.spyOn(
fs3 as FileSearchWithPrivateMethods,
'performCrawl',
);
await fs3.initialize();
expect(crawlSpy3).not.toHaveBeenCalled();
});
await expect(fileSearch.search('')).rejects.toThrow(
'Engine not initialized. Call initialize() first.',
);
});
it('should handle empty or commented-only ignore files', async () => {
@@ -518,13 +374,14 @@ describe('FileSearch', () => {
src: ['main.js'],
});
const fileSearch = new FileSearch({
const fileSearch = FileSearchFactory.create({
projectRoot: tmpDir,
useGitignore: true,
useGeminiignore: false,
ignoreDirs: [],
cache: false,
cacheTtl: 0,
enableRecursiveFileSearch: true,
});
await fileSearch.initialize();
@@ -539,13 +396,14 @@ describe('FileSearch', () => {
src: ['main.js'],
});
const fileSearch = new FileSearch({
const fileSearch = FileSearchFactory.create({
projectRoot: tmpDir,
useGitignore: false, // Explicitly disable .gitignore to isolate this rule
useGeminiignore: false,
ignoreDirs: [],
cache: false,
cacheTtl: 0,
enableRecursiveFileSearch: true,
});
await fileSearch.initialize();
@@ -561,13 +419,14 @@ describe('FileSearch', () => {
}
tmpDir = await createTmpDir(largeDir);
const fileSearch = new FileSearch({
const fileSearch = FileSearchFactory.create({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: false,
ignoreDirs: [],
cache: false,
cacheTtl: 0,
enableRecursiveFileSearch: true,
});
await fileSearch.initialize();
@@ -596,13 +455,14 @@ describe('FileSearch', () => {
},
});
const fileSearch = new FileSearch({
const fileSearch = FileSearchFactory.create({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: false,
ignoreDirs: [],
cache: true, // Enable caching for this test
cacheTtl: 0,
enableRecursiveFileSearch: true,
});
await fileSearch.initialize();
@@ -634,13 +494,14 @@ describe('FileSearch', () => {
'other.txt': '',
});
const fileSearch = new FileSearch({
const fileSearch = FileSearchFactory.create({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: false,
ignoreDirs: [],
cache: false,
cacheTtl: 0,
enableRecursiveFileSearch: true,
});
await fileSearch.initialize();
@@ -676,13 +537,14 @@ describe('FileSearch', () => {
'file5.js': '',
});
const fileSearch = new FileSearch({
const fileSearch = FileSearchFactory.create({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: false,
ignoreDirs: [],
cache: true, // Ensure caching is enabled
cacheTtl: 10000,
enableRecursiveFileSearch: true,
});
await fileSearch.initialize();
@@ -704,108 +566,97 @@ describe('FileSearch', () => {
expect(limitedResults).toEqual(['file1.js', 'file2.js']);
});
describe('with maxDepth', () => {
beforeEach(async () => {
describe('DirectoryFileSearch', () => {
it('should search for files in the current directory', async () => {
tmpDir = await createTmpDir({
'file-root.txt': '',
level1: {
'file-level1.txt': '',
level2: {
'file-level2.txt': '',
level3: {
'file-level3.txt': '',
},
},
'file1.js': '',
'file2.ts': '',
'file3.js': '',
});
const fileSearch = FileSearchFactory.create({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: false,
ignoreDirs: [],
cache: false,
cacheTtl: 0,
enableRecursiveFileSearch: false,
});
await fileSearch.initialize();
const results = await fileSearch.search('*.js');
expect(results).toEqual(['file1.js', 'file3.js']);
});
it('should search for files in a subdirectory', async () => {
tmpDir = await createTmpDir({
'file1.js': '',
src: {
'file2.js': '',
'file3.ts': '',
},
});
});
it('should only search top-level files when maxDepth is 0', async () => {
const fileSearch = new FileSearch({
const fileSearch = FileSearchFactory.create({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: false,
ignoreDirs: [],
cache: false,
cacheTtl: 0,
maxDepth: 0,
enableRecursiveFileSearch: false,
});
await fileSearch.initialize();
const results = await fileSearch.search('');
expect(results).toEqual(['level1/', 'file-root.txt']);
const results = await fileSearch.search('src/*.js');
expect(results).toEqual(['src/file2.js']);
});
it('should search one level deep when maxDepth is 1', async () => {
const fileSearch = new FileSearch({
it('should list all files in a directory', async () => {
tmpDir = await createTmpDir({
'file1.js': '',
src: {
'file2.js': '',
'file3.ts': '',
},
});
const fileSearch = FileSearchFactory.create({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: false,
ignoreDirs: [],
cache: false,
cacheTtl: 0,
maxDepth: 1,
enableRecursiveFileSearch: false,
});
await fileSearch.initialize();
const results = await fileSearch.search('');
expect(results).toEqual([
'level1/',
'level1/level2/',
'file-root.txt',
'level1/file-level1.txt',
]);
const results = await fileSearch.search('src/');
expect(results).toEqual(['src/file2.js', 'src/file3.ts']);
});
it('should search two levels deep when maxDepth is 2', async () => {
const fileSearch = new FileSearch({
it('should respect ignore rules', async () => {
tmpDir = await createTmpDir({
'.gitignore': '*.js',
'file1.js': '',
'file2.ts': '',
});
const fileSearch = FileSearchFactory.create({
projectRoot: tmpDir,
useGitignore: false,
useGitignore: true,
useGeminiignore: false,
ignoreDirs: [],
cache: false,
cacheTtl: 0,
maxDepth: 2,
enableRecursiveFileSearch: false,
});
await fileSearch.initialize();
const results = await fileSearch.search('');
expect(results).toEqual([
'level1/',
'level1/level2/',
'level1/level2/level3/',
'file-root.txt',
'level1/file-level1.txt',
'level1/level2/file-level2.txt',
]);
});
it('should perform a full recursive search when maxDepth is undefined', async () => {
const fileSearch = new FileSearch({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: false,
ignoreDirs: [],
cache: false,
cacheTtl: 0,
maxDepth: undefined, // Explicitly undefined
});
await fileSearch.initialize();
const results = await fileSearch.search('');
expect(results).toEqual([
'level1/',
'level1/level2/',
'level1/level2/level3/',
'file-root.txt',
'level1/file-level1.txt',
'level1/level2/file-level2.txt',
'level1/level2/level3/file-level3.txt',
]);
const results = await fileSearch.search('*');
expect(results).toEqual(['.gitignore', 'file2.ts']);
});
});
});

225
packages/core/src/utils/filesearch/fileSearch.ts
View File

@@ -5,23 +5,22 @@
*/
import path from 'node:path';
import fs from 'node:fs';
import { fdir } from 'fdir';
import picomatch from 'picomatch';
import { Ignore } from './ignore.js';
import { Ignore, loadIgnoreRules } from './ignore.js';
import { ResultCache } from './result-cache.js';
import * as cache from './crawlCache.js';
import { crawl } from './crawler.js';
import { AsyncFzf, FzfResultItem } from 'fzf';
export type FileSearchOptions = {
export interface FileSearchOptions {
projectRoot: string;
ignoreDirs: string[];
useGitignore: boolean;
useGeminiignore: boolean;
cache: boolean;
cacheTtl: number;
enableRecursiveFileSearch: boolean;
maxDepth?: number;
};
}
export class AbortError extends Error {
constructor(message = 'Search aborted') {
@@ -78,54 +77,42 @@ export async function filter(
return results;
}
export type SearchOptions = {
export interface SearchOptions {
signal?: AbortSignal;
maxResults?: number;
};
}
/**
* Provides a fast and efficient way to search for files within a project,
* respecting .gitignore and .geminiignore rules, and utilizing caching
* for improved performance.
*/
export class FileSearch {
private readonly absoluteDir: string;
private readonly ignore: Ignore = new Ignore();
export interface FileSearch {
initialize(): Promise<void>;
search(pattern: string, options?: SearchOptions): Promise<string[]>;
}
class RecursiveFileSearch implements FileSearch {
private ignore: Ignore | undefined;
private resultCache: ResultCache | undefined;
private allFiles: string[] = [];
private fzf: AsyncFzf<string[]> | undefined;
/**
* Constructs a new `FileSearch` instance.
* @param options Configuration options for the file search.
*/
constructor(private readonly options: FileSearchOptions) {
this.absoluteDir = path.resolve(options.projectRoot);
}
constructor(private readonly options: FileSearchOptions) {}
/**
* Initializes the file search engine by loading ignore rules, crawling the
* file system, and building the in-memory cache. This method must be called
* before performing any searches.
*/
async initialize(): Promise<void> {
this.loadIgnoreRules();
await this.crawlFiles();
this.ignore = loadIgnoreRules(this.options);
this.allFiles = await crawl({
crawlDirectory: this.options.projectRoot,
cwd: this.options.projectRoot,
ignore: this.ignore,
cache: this.options.cache,
cacheTtl: this.options.cacheTtl,
maxDepth: this.options.maxDepth,
});
this.buildResultCache();
}
/**
* Searches for files matching a given pattern.
* @param pattern The picomatch pattern to search for (e.g., '*.js', 'src/**').
* @param options Search options, including an AbortSignal and maxResults.
* @returns A promise that resolves to a list of matching file paths, relative
* to the project root.
*/
async search(
pattern: string,
options: SearchOptions = {},
): Promise<string[]> {
if (!this.resultCache || !this.fzf) {
if (!this.resultCache || !this.fzf || !this.ignore) {
throw new Error('Engine not initialized. Call initialize() first.');
}
@@ -159,21 +146,9 @@ export class FileSearch {
}
}
// Trade-off: We apply a two-stage filtering process.
// 1. During the file system crawl (`performCrawl`), we only apply directory-level
// ignore rules (e.g., `node_modules/`, `dist/`). This is because applying
// a full ignore filter (which includes file-specific patterns like `*.log`)
// during the crawl can significantly slow down `fdir`.
// 2. Here, in the `search` method, we apply the full ignore filter
// (including file patterns) to the `filteredCandidates` (which have already
// been filtered by the user's search pattern and sorted). For autocomplete,
// the number of displayed results is small (MAX_SUGGESTIONS_TO_SHOW),
// so applying the full filter to this truncated list is much more efficient
// than applying it to every file during the initial crawl.
const fileFilter = this.ignore.getFileFilter();
const results: string[] = [];
for (const [i, candidate] of filteredCandidates.entries()) {
// Yield to the event loop to avoid blocking on large result sets.
if (i % 1000 === 0) {
await new Promise((resolve) => setImmediate(resolve));
if (options.signal?.aborted) {
@@ -184,7 +159,6 @@ export class FileSearch {
if (results.length >= (options.maxResults ?? Infinity)) {
break;
}
// The `ignore` library throws an error if the path is '.', so we skip it.
if (candidate === '.') {
continue;
}
@@ -195,99 +169,6 @@ export class FileSearch {
return results;
}
/**
* Loads ignore rules from .gitignore and .geminiignore files, and applies
* any additional ignore directories specified in the options.
*/
private loadIgnoreRules(): void {
if (this.options.useGitignore) {
const gitignorePath = path.join(this.absoluteDir, '.gitignore');
if (fs.existsSync(gitignorePath)) {
this.ignore.add(fs.readFileSync(gitignorePath, 'utf8'));
}
}
if (this.options.useGeminiignore) {
const geminiignorePath = path.join(this.absoluteDir, '.geminiignore');
if (fs.existsSync(geminiignorePath)) {
this.ignore.add(fs.readFileSync(geminiignorePath, 'utf8'));
}
}
const ignoreDirs = ['.git', ...this.options.ignoreDirs];
this.ignore.add(
ignoreDirs.map((dir) => {
if (dir.endsWith('/')) {
return dir;
}
return `${dir}/`;
}),
);
}
/**
* Crawls the file system to get a list of all files and directories,
* optionally using a cache for faster initialization.
*/
private async crawlFiles(): Promise<void> {
if (this.options.cache) {
const cacheKey = cache.getCacheKey(
this.absoluteDir,
this.ignore.getFingerprint(),
this.options.maxDepth,
);
const cachedResults = cache.read(cacheKey);
if (cachedResults) {
this.allFiles = cachedResults;
return;
}
}
this.allFiles = await this.performCrawl();
if (this.options.cache) {
const cacheKey = cache.getCacheKey(
this.absoluteDir,
this.ignore.getFingerprint(),
this.options.maxDepth,
);
cache.write(cacheKey, this.allFiles, this.options.cacheTtl * 1000);
}
}
/**
* Performs the actual file system crawl using `fdir`, applying directory
* ignore rules.
* @returns A promise that resolves to a list of all files and directories.
*/
private async performCrawl(): Promise<string[]> {
const dirFilter = this.ignore.getDirectoryFilter();
// We use `fdir` for fast file system traversal. A key performance
// optimization for large workspaces is to exclude entire directories
// early in the traversal process. This is why we apply directory-specific
// ignore rules (e.g., `node_modules/`, `dist/`) directly to `fdir`'s
// exclude filter.
const api = new fdir()
.withRelativePaths()
.withDirs()
.withPathSeparator('/') // Always use unix style paths
.exclude((_, dirPath) => {
const relativePath = path.relative(this.absoluteDir, dirPath);
return dirFilter(`${relativePath}/`);
});
if (this.options.maxDepth !== undefined) {
api.withMaxDepth(this.options.maxDepth);
}
return api.crawl(this.absoluteDir).withPromise();
}
/**
* Builds the in-memory cache for fast pattern matching.
*/
private buildResultCache(): void {
this.resultCache = new ResultCache(this.allFiles);
// The v1 algorithm is much faster since it only looks at the first
@@ -298,3 +179,59 @@ export class FileSearch {
});
}
}
class DirectoryFileSearch implements FileSearch {
private ignore: Ignore | undefined;
constructor(private readonly options: FileSearchOptions) {}
async initialize(): Promise<void> {
this.ignore = loadIgnoreRules(this.options);
}
async search(
pattern: string,
options: SearchOptions = {},
): Promise<string[]> {
if (!this.ignore) {
throw new Error('Engine not initialized. Call initialize() first.');
}
pattern = pattern || '*';
const dir = pattern.endsWith('/') ? pattern : path.dirname(pattern);
const results = await crawl({
crawlDirectory: path.join(this.options.projectRoot, dir),
cwd: this.options.projectRoot,
maxDepth: 0,
ignore: this.ignore,
cache: this.options.cache,
cacheTtl: this.options.cacheTtl,
});
const filteredResults = await filter(results, pattern, options.signal);
const fileFilter = this.ignore.getFileFilter();
const finalResults: string[] = [];
for (const candidate of filteredResults) {
if (finalResults.length >= (options.maxResults ?? Infinity)) {
break;
}
if (candidate === '.') {
continue;
}
if (!fileFilter(candidate)) {
finalResults.push(candidate);
}
}
return finalResults;
}
}
export class FileSearchFactory {
static create(options: FileSearchOptions): FileSearch {
if (options.enableRecursiveFileSearch) {
return new RecursiveFileSearch(options);
}
return new DirectoryFileSearch(options);
}
}

99
packages/core/src/utils/filesearch/ignore.test.ts
View File

@@ -4,8 +4,9 @@
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect } from 'vitest';
import { Ignore } from './ignore.js';
import { describe, it, expect, afterEach } from 'vitest';
import { Ignore, loadIgnoreRules } from './ignore.js';
import { createTmpDir, cleanupTmpDir } from '@qwen-code/qwen-code-test-utils';
describe('Ignore', () => {
describe('getDirectoryFilter', () => {
@@ -63,3 +64,97 @@ describe('Ignore', () => {
expect(ig1.getFingerprint()).not.toBe(ig2.getFingerprint());
});
});
describe('loadIgnoreRules', () => {
let tmpDir: string;
afterEach(async () => {
if (tmpDir) {
await cleanupTmpDir(tmpDir);
}
});
it('should load rules from .gitignore', async () => {
tmpDir = await createTmpDir({
'.gitignore': '*.log',
});
const ignore = loadIgnoreRules({
projectRoot: tmpDir,
useGitignore: true,
useGeminiignore: false,
ignoreDirs: [],
});
const fileFilter = ignore.getFileFilter();
expect(fileFilter('test.log')).toBe(true);
expect(fileFilter('test.txt')).toBe(false);
});
it('should load rules from .geminiignore', async () => {
tmpDir = await createTmpDir({
'.geminiignore': '*.log',
});
const ignore = loadIgnoreRules({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: true,
ignoreDirs: [],
});
const fileFilter = ignore.getFileFilter();
expect(fileFilter('test.log')).toBe(true);
expect(fileFilter('test.txt')).toBe(false);
});
it('should combine rules from .gitignore and .geminiignore', async () => {
tmpDir = await createTmpDir({
'.gitignore': '*.log',
'.geminiignore': '*.txt',
});
const ignore = loadIgnoreRules({
projectRoot: tmpDir,
useGitignore: true,
useGeminiignore: true,
ignoreDirs: [],
});
const fileFilter = ignore.getFileFilter();
expect(fileFilter('test.log')).toBe(true);
expect(fileFilter('test.txt')).toBe(true);
expect(fileFilter('test.md')).toBe(false);
});
it('should add ignoreDirs', async () => {
tmpDir = await createTmpDir({});
const ignore = loadIgnoreRules({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: false,
ignoreDirs: ['logs/'],
});
const dirFilter = ignore.getDirectoryFilter();
expect(dirFilter('logs/')).toBe(true);
expect(dirFilter('src/')).toBe(false);
});
it('should handle missing ignore files gracefully', async () => {
tmpDir = await createTmpDir({});
const ignore = loadIgnoreRules({
projectRoot: tmpDir,
useGitignore: true,
useGeminiignore: true,
ignoreDirs: [],
});
const fileFilter = ignore.getFileFilter();
expect(fileFilter('anyfile.txt')).toBe(false);
});
it('should always add .git to the ignore list', async () => {
tmpDir = await createTmpDir({});
const ignore = loadIgnoreRules({
projectRoot: tmpDir,
useGitignore: false,
useGeminiignore: false,
ignoreDirs: [],
});
const dirFilter = ignore.getDirectoryFilter();
expect(dirFilter('.git/')).toBe(true);
});
});

38
packages/core/src/utils/filesearch/ignore.ts
View File

@@ -4,11 +4,49 @@
* SPDX-License-Identifier: Apache-2.0
*/
import fs from 'node:fs';
import path from 'node:path';
import ignore from 'ignore';
import picomatch from 'picomatch';
const hasFileExtension = picomatch('**/*[*.]*');
export interface LoadIgnoreRulesOptions {
projectRoot: string;
useGitignore: boolean;
useGeminiignore: boolean;
ignoreDirs: string[];
}
export function loadIgnoreRules(options: LoadIgnoreRulesOptions): Ignore {
const ignorer = new Ignore();
if (options.useGitignore) {
const gitignorePath = path.join(options.projectRoot, '.gitignore');
if (fs.existsSync(gitignorePath)) {
ignorer.add(fs.readFileSync(gitignorePath, 'utf8'));
}
}
if (options.useGeminiignore) {
const geminiignorePath = path.join(options.projectRoot, '.geminiignore');
if (fs.existsSync(geminiignorePath)) {
ignorer.add(fs.readFileSync(geminiignorePath, 'utf8'));
}
}
const ignoreDirs = ['.git', ...options.ignoreDirs];
ignorer.add(
ignoreDirs.map((dir) => {
if (dir.endsWith('/')) {
return dir;
}
return `${dir}/`;
}),
);
return ignorer;
}
export class Ignore {
private readonly allPatterns: string[] = [];
private dirIgnorer = ignore();

34
packages/core/src/utils/getPty.ts Normal file
View File

@@ -0,0 +1,34 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
export type PtyImplementation = {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
module: any;
name: 'lydell-node-pty' | 'node-pty';
} | null;
export interface PtyProcess {
readonly pid: number;
onData(callback: (data: string) => void): void;
onExit(callback: (e: { exitCode: number; signal?: number }) => void): void;
kill(signal?: string): void;
}
export const getPty = async (): Promise<PtyImplementation> => {
try {
const lydell = '@lydell/node-pty';
const module = await import(lydell);
return { module, name: 'lydell-node-pty' };
} catch (_e) {
try {
const nodePty = 'node-pty';
const module = await import(nodePty);
return { module, name: 'node-pty' };
} catch (_e2) {
return null;
}
}
};

5
packages/core/src/utils/memoryDiscovery.test.ts
View File

@@ -48,8 +48,8 @@ describe('loadServerHierarchicalMemory', () => {
vi.resetAllMocks();
// Set environment variables to indicate test environment
process.env.NODE_ENV = 'test';
process.env.VITEST = 'true';
vi.stubEnv('NODE_ENV', 'test');
vi.stubEnv('VITEST', 'true');
projectRoot = await createEmptyDir(path.join(testRootDir, 'project'));
cwd = await createEmptyDir(path.join(projectRoot, 'src'));
@@ -58,6 +58,7 @@ describe('loadServerHierarchicalMemory', () => {
});
afterEach(async () => {
vi.unstubAllEnvs();
// Some tests set this to a different value.
setGeminiMdFilename(DEFAULT_CONTEXT_FILENAME);
// Clean up the temporary directory to prevent resource leaks.

8
packages/core/src/utils/memoryDiscovery.ts
View File

@@ -57,8 +57,9 @@ async function findProjectRoot(startDir: string): Promise<string | null> {
(error as { code: string }).code === 'ENOENT';
// Only log unexpected errors in non-test environments
// process.env.NODE_ENV === 'test' or VITEST are common test indicators
const isTestEnv = process.env.NODE_ENV === 'test' || process.env.VITEST;
// process.env['NODE_ENV'] === 'test' or VITEST are common test indicators
const isTestEnv =
process.env['NODE_ENV'] === 'test' || process.env['VITEST'];
if (!isENOENT && !isTestEnv) {
if (typeof error === 'object' && error !== null && 'code' in error) {
@@ -265,7 +266,8 @@ async function readGeminiMdFiles(
`Successfully read and processed imports: ${filePath} (Length: ${processedResult.content.length})`,
);
} catch (error: unknown) {
const isTestEnv = process.env.NODE_ENV === 'test' || process.env.VITEST;
const isTestEnv =
process.env['NODE_ENV'] === 'test' || process.env['VITEST'];
if (!isTestEnv) {
const message = error instanceof Error ? error.message : String(error);
logger.warn(

10
packages/core/src/utils/memoryImportProcessor.ts
View File

@@ -6,6 +6,7 @@
import * as fs from 'fs/promises';
import * as path from 'path';
import { isSubpath } from './paths.js';
import { marked } from 'marked';
// Simple console logger for import processing
@@ -411,10 +412,7 @@ export function validateImportPath(
const resolvedPath = path.resolve(basePath, importPath);
return allowedDirectories.some((allowedDir) => {
const normalizedAllowedDir = path.resolve(allowedDir);
const isSamePath = resolvedPath === normalizedAllowedDir;
const isSubPath = resolvedPath.startsWith(normalizedAllowedDir + path.sep);
return isSamePath || isSubPath;
});
return allowedDirectories.some((allowedDir) =>
isSubpath(allowedDir, resolvedPath),
);
}

106
packages/core/src/utils/paths.test.ts
View File

@@ -4,8 +4,8 @@
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect } from 'vitest';
import { escapePath, unescapePath } from './paths.js';
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
import { escapePath, unescapePath, isSubpath } from './paths.js';
describe('escapePath', () => {
it('should escape spaces', () => {
@@ -212,3 +212,105 @@ describe('unescapePath', () => {
expect(unescapePath('file\\\\\\(test\\).txt')).toBe('file\\\\(test).txt');
});
});
describe('isSubpath', () => {
it('should return true for a direct subpath', () => {
expect(isSubpath('/a/b', '/a/b/c')).toBe(true);
});
it('should return true for the same path', () => {
expect(isSubpath('/a/b', '/a/b')).toBe(true);
});
it('should return false for a parent path', () => {
expect(isSubpath('/a/b/c', '/a/b')).toBe(false);
});
it('should return false for a completely different path', () => {
expect(isSubpath('/a/b', '/x/y')).toBe(false);
});
it('should handle relative paths', () => {
expect(isSubpath('a/b', 'a/b/c')).toBe(true);
expect(isSubpath('a/b', 'a/c')).toBe(false);
});
it('should handle paths with ..', () => {
expect(isSubpath('/a/b', '/a/b/../b/c')).toBe(true);
expect(isSubpath('/a/b', '/a/c/../b')).toBe(true);
});
it('should handle root paths', () => {
expect(isSubpath('/', '/a')).toBe(true);
expect(isSubpath('/a', '/')).toBe(false);
});
it('should handle trailing slashes', () => {
expect(isSubpath('/a/b/', '/a/b/c')).toBe(true);
expect(isSubpath('/a/b', '/a/b/c/')).toBe(true);
expect(isSubpath('/a/b/', '/a/b/c/')).toBe(true);
});
});
describe('isSubpath on Windows', () => {
const originalPlatform = process.platform;
beforeAll(() => {
Object.defineProperty(process, 'platform', {
value: 'win32',
});
});
afterAll(() => {
Object.defineProperty(process, 'platform', {
value: originalPlatform,
});
});
it('should return true for a direct subpath on Windows', () => {
expect(isSubpath('C:\\Users\\Test', 'C:\\Users\\Test\\file.txt')).toBe(
true,
);
});
it('should return true for the same path on Windows', () => {
expect(isSubpath('C:\\Users\\Test', 'C:\\Users\\Test')).toBe(true);
});
it('should return false for a parent path on Windows', () => {
expect(isSubpath('C:\\Users\\Test\\file.txt', 'C:\\Users\\Test')).toBe(
false,
);
});
it('should return false for a different drive on Windows', () => {
expect(isSubpath('C:\\Users\\Test', 'D:\\Users\\Test')).toBe(false);
});
it('should be case-insensitive for drive letters on Windows', () => {
expect(isSubpath('c:\\Users\\Test', 'C:\\Users\\Test\\file.txt')).toBe(
true,
);
});
it('should be case-insensitive for path components on Windows', () => {
expect(isSubpath('C:\\Users\\Test', 'c:\\users\\test\\file.txt')).toBe(
true,
);
});
it('should handle mixed slashes on Windows', () => {
expect(isSubpath('C:/Users/Test', 'C:\\Users\\Test\\file.txt')).toBe(true);
});
it('should handle trailing slashes on Windows', () => {
expect(isSubpath('C:\\Users\\Test\\', 'C:\\Users\\Test\\file.txt')).toBe(
true,
);
});
it('should handle relative paths correctly on Windows', () => {
expect(isSubpath('Users\\Test', 'Users\\Test\\file.txt')).toBe(true);
expect(isSubpath('Users\\Test\\file.txt', 'Users\\Test')).toBe(false);
});
});

20
packages/core/src/utils/paths.ts
View File

@@ -200,3 +200,23 @@ export function getUserCommandsDir(): string {
export function getProjectCommandsDir(projectRoot: string): string {
return path.join(projectRoot, QWEN_DIR, COMMANDS_DIR_NAME);
}
/**
* Checks if a path is a subpath of another path.
* @param parentPath The parent path.
* @param childPath The child path.
* @returns True if childPath is a subpath of parentPath, false otherwise.
*/
export function isSubpath(parentPath: string, childPath: string): boolean {
const isWindows = os.platform() === 'win32';
const pathModule = isWindows ? path.win32 : path;
// On Windows, path.relative is case-insensitive. On POSIX, it's case-sensitive.
const relative = pathModule.relative(parentPath, childPath);
return (
!relative.startsWith(`..${pathModule.sep}`) &&
relative !== '..' &&
!pathModule.isAbsolute(relative)
);
}

9
packages/core/src/utils/secure-browser-launcher.ts
View File

@@ -151,20 +151,23 @@ export function shouldLaunchBrowser(): boolean {
// A list of browser names that indicate we should not attempt to open a
// web browser for the user.
const browserBlocklist = ['www-browser'];
const browserEnv = process.env.BROWSER;
const browserEnv = process.env['BROWSER'];
if (browserEnv && browserBlocklist.includes(browserEnv)) {
return false;
}
// Common environment variables used in CI/CD or other non-interactive shells.
if (process.env.CI || process.env.DEBIAN_FRONTEND === 'noninteractive') {
if (
process.env['CI'] ||
process.env['DEBIAN_FRONTEND'] === 'noninteractive'
) {
return false;
}
// The presence of SSH_CONNECTION indicates a remote session.
// We should not attempt to launch a browser unless a display is explicitly available
// (checked below for Linux).
const isSSH = !!process.env.SSH_CONNECTION;
const isSSH = !!process.env['SSH_CONNECTION'];
// On Linux, the presence of a display server is a strong indicator of a GUI.
if (platform() === 'linux') {

163
packages/core/src/utils/shell-utils.test.ts
View File

@@ -4,24 +4,47 @@
* SPDX-License-Identifier: Apache-2.0
*/
import { expect, describe, it, beforeEach } from 'vitest';
import { expect, describe, it, beforeEach, vi, afterEach } from 'vitest';
import {
checkCommandPermissions,
escapeShellArg,
getCommandRoots,
getShellConfiguration,
isCommandAllowed,
stripShellWrapper,
} from './shell-utils.js';
import { Config } from '../config/config.js';
const mockPlatform = vi.hoisted(() => vi.fn());
vi.mock('os', () => ({
default: {
platform: mockPlatform,
},
platform: mockPlatform,
}));
const mockQuote = vi.hoisted(() => vi.fn());
vi.mock('shell-quote', () => ({
quote: mockQuote,
}));
let config: Config;
beforeEach(() => {
mockPlatform.mockReturnValue('linux');
mockQuote.mockImplementation((args: string[]) =>
args.map((arg) => `'${arg}'`).join(' '),
);
config = {
getCoreTools: () => [],
getExcludeTools: () => [],
} as unknown as Config;
});
afterEach(() => {
vi.clearAllMocks();
});
describe('isCommandAllowed', () => {
it('should allow a command if no restrictions are provided', () => {
const result = isCommandAllowed('ls -l', config);
@@ -38,7 +61,9 @@ describe('isCommandAllowed', () => {
config.getCoreTools = () => ['ShellTool(ls -l)'];
const result = isCommandAllowed('rm -rf /', config);
expect(result.allowed).toBe(false);
expect(result.reason).toBe(`Command(s) not in the allowed commands list.`);
expect(result.reason).toBe(
`Command(s) not in the allowed commands list. Disallowed commands: "rm -rf /"`,
);
});
it('should block a command if it is in the blocked list', () => {
@@ -157,7 +182,7 @@ describe('checkCommandPermissions', () => {
expect(result).toEqual({
allAllowed: false,
disallowedCommands: ['git status'],
blockReason: `Command(s) not in the allowed commands list.`,
blockReason: `Command(s) not in the allowed commands list. Disallowed commands: "git status"`,
isHardDenial: false,
});
});
@@ -275,3 +300,135 @@ describe('stripShellWrapper', () => {
expect(stripShellWrapper('ls -l')).toEqual('ls -l');
});
});
describe('escapeShellArg', () => {
describe('POSIX (bash)', () => {
it('should use shell-quote for escaping', () => {
mockQuote.mockReturnValueOnce("'escaped value'");
const result = escapeShellArg('raw value', 'bash');
expect(mockQuote).toHaveBeenCalledWith(['raw value']);
expect(result).toBe("'escaped value'");
});
it('should handle empty strings', () => {
const result = escapeShellArg('', 'bash');
expect(result).toBe('');
expect(mockQuote).not.toHaveBeenCalled();
});
});
describe('Windows', () => {
describe('when shell is cmd.exe', () => {
it('should wrap simple arguments in double quotes', () => {
const result = escapeShellArg('search term', 'cmd');
expect(result).toBe('"search term"');
});
it('should escape internal double quotes by doubling them', () => {
const result = escapeShellArg('He said "Hello"', 'cmd');
expect(result).toBe('"He said ""Hello"""');
});
it('should handle empty strings', () => {
const result = escapeShellArg('', 'cmd');
expect(result).toBe('');
});
});
describe('when shell is PowerShell', () => {
it('should wrap simple arguments in single quotes', () => {
const result = escapeShellArg('search term', 'powershell');
expect(result).toBe("'search term'");
});
it('should escape internal single quotes by doubling them', () => {
const result = escapeShellArg("It's a test", 'powershell');
expect(result).toBe("'It''s a test'");
});
it('should handle double quotes without escaping them', () => {
const result = escapeShellArg('He said "Hello"', 'powershell');
expect(result).toBe('\'He said "Hello"\'');
});
it('should handle empty strings', () => {
const result = escapeShellArg('', 'powershell');
expect(result).toBe('');
});
});
});
});
describe('getShellConfiguration', () => {
const originalEnv = { ...process.env };
afterEach(() => {
process.env = originalEnv;
});
it('should return bash configuration on Linux', () => {
mockPlatform.mockReturnValue('linux');
const config = getShellConfiguration();
expect(config.executable).toBe('bash');
expect(config.argsPrefix).toEqual(['-c']);
expect(config.shell).toBe('bash');
});
it('should return bash configuration on macOS (darwin)', () => {
mockPlatform.mockReturnValue('darwin');
const config = getShellConfiguration();
expect(config.executable).toBe('bash');
expect(config.argsPrefix).toEqual(['-c']);
expect(config.shell).toBe('bash');
});
describe('on Windows', () => {
beforeEach(() => {
mockPlatform.mockReturnValue('win32');
});
it('should return cmd.exe configuration by default', () => {
delete process.env['ComSpec'];
const config = getShellConfiguration();
expect(config.executable).toBe('cmd.exe');
expect(config.argsPrefix).toEqual(['/d', '/s', '/c']);
expect(config.shell).toBe('cmd');
});
it('should respect ComSpec for cmd.exe', () => {
const cmdPath = 'C:\\WINDOWS\\system32\\cmd.exe';
process.env['ComSpec'] = cmdPath;
const config = getShellConfiguration();
expect(config.executable).toBe(cmdPath);
expect(config.argsPrefix).toEqual(['/d', '/s', '/c']);
expect(config.shell).toBe('cmd');
});
it('should return PowerShell configuration if ComSpec points to powershell.exe', () => {
const psPath =
'C:\\WINDOWS\\System32\\WindowsPowerShell\\v1.0\\powershell.exe';
process.env['ComSpec'] = psPath;
const config = getShellConfiguration();
expect(config.executable).toBe(psPath);
expect(config.argsPrefix).toEqual(['-NoProfile', '-Command']);
expect(config.shell).toBe('powershell');
});
it('should return PowerShell configuration if ComSpec points to pwsh.exe', () => {
const pwshPath = 'C:\\Program Files\\PowerShell\\7\\pwsh.exe';
process.env['ComSpec'] = pwshPath;
const config = getShellConfiguration();
expect(config.executable).toBe(pwshPath);
expect(config.argsPrefix).toEqual(['-NoProfile', '-Command']);
expect(config.shell).toBe('powershell');
});
it('should be case-insensitive when checking ComSpec', () => {
process.env['ComSpec'] = 'C:\\Path\\To\\POWERSHELL.EXE';
const config = getShellConfiguration();
expect(config.executable).toBe('C:\\Path\\To\\POWERSHELL.EXE');
expect(config.argsPrefix).toEqual(['-NoProfile', '-Command']);
expect(config.shell).toBe('powershell');
});
});
});

102
packages/core/src/utils/shell-utils.ts
View File

@@ -5,6 +5,102 @@
*/
import { Config } from '../config/config.js';
import os from 'os';
import { quote } from 'shell-quote';
/**
* An identifier for the shell type.
*/
export type ShellType = 'cmd' | 'powershell' | 'bash';
/**
* Defines the configuration required to execute a command string within a specific shell.
*/
export interface ShellConfiguration {
/** The path or name of the shell executable (e.g., 'bash', 'cmd.exe'). */
executable: string;
/**
* The arguments required by the shell to execute a subsequent string argument.
*/
argsPrefix: string[];
/** An identifier for the shell type. */
shell: ShellType;
}
/**
* Determines the appropriate shell configuration for the current platform.
*
* This ensures we can execute command strings predictably and securely across platforms
* using the `spawn(executable, [...argsPrefix, commandString], { shell: false })` pattern.
*
* @returns The ShellConfiguration for the current environment.
*/
export function getShellConfiguration(): ShellConfiguration {
if (isWindows()) {
const comSpec = process.env['ComSpec'] || 'cmd.exe';
const executable = comSpec.toLowerCase();
if (
executable.endsWith('powershell.exe') ||
executable.endsWith('pwsh.exe')
) {
// For PowerShell, the arguments are different.
// -NoProfile: Speeds up startup.
// -Command: Executes the following command.
return {
executable: comSpec,
argsPrefix: ['-NoProfile', '-Command'],
shell: 'powershell',
};
}
// Default to cmd.exe for anything else on Windows.
// Flags for CMD:
// /d: Skip execution of AutoRun commands.
// /s: Modifies the treatment of the command string (important for quoting).
// /c: Carries out the command specified by the string and then terminates.
return {
executable: comSpec,
argsPrefix: ['/d', '/s', '/c'],
shell: 'cmd',
};
}
// Unix-like systems (Linux, macOS)
return { executable: 'bash', argsPrefix: ['-c'], shell: 'bash' };
}
/**
* Export the platform detection constant for use in process management (e.g., killing processes).
*/
export const isWindows = () => os.platform() === 'win32';
/**
* Escapes a string so that it can be safely used as a single argument
* in a shell command, preventing command injection.
*
* @param arg The argument string to escape.
* @param shell The type of shell the argument is for.
* @returns The shell-escaped string.
*/
export function escapeShellArg(arg: string, shell: ShellType): string {
if (!arg) {
return '';
}
switch (shell) {
case 'powershell':
// For PowerShell, wrap in single quotes and escape internal single quotes by doubling them.
return `'${arg.replace(/'/g, "''")}'`;
case 'cmd':
// Simple Windows escaping for cmd.exe: wrap in double quotes and escape inner double quotes.
return `"${arg.replace(/"/g, '""')}"`;
case 'bash':
default:
// POSIX shell escaping using shell-quote.
return quote([arg]);
}
}
/**
* Splits a shell command into a list of individual commands, respecting quotes.
@@ -301,7 +397,9 @@ export function checkCommandPermissions(
return {
allAllowed: false,
disallowedCommands,
blockReason: `Command(s) not on the global or session allowlist.`,
blockReason: `Command(s) not on the global or session allowlist. Disallowed commands: ${disallowedCommands
.map((c) => JSON.stringify(c))
.join(', ')}`,
isHardDenial: false, // This is a soft denial; confirmation is possible.
};
}
@@ -322,7 +420,7 @@ export function checkCommandPermissions(
return {
allAllowed: false,
disallowedCommands,
blockReason: `Command(s) not in the allowed commands list.`,
blockReason: `Command(s) not in the allowed commands list. Disallowed commands: ${disallowedCommands.map((c) => JSON.stringify(c)).join(', ')}`,
isHardDenial: false, // This is a soft denial.
};
}

46
packages/core/src/utils/systemEncoding.test.ts
View File

@@ -39,9 +39,9 @@ describe('Shell Command Processor - Encoding Functions', () => {
resetEncodingCache();
// Clear environment variables that might affect tests
delete process.env.LC_ALL;
delete process.env.LC_CTYPE;
delete process.env.LANG;
delete process.env['LC_ALL'];
delete process.env['LC_CTYPE'];
delete process.env['LANG'];
});
afterEach(() => {
@@ -218,21 +218,21 @@ describe('Shell Command Processor - Encoding Functions', () => {
});
it('should parse locale from LC_ALL environment variable', () => {
process.env.LC_ALL = 'en_US.UTF-8';
process.env['LC_ALL'] = 'en_US.UTF-8';
const result = getSystemEncoding();
expect(result).toBe('utf-8');
});
it('should parse locale from LC_CTYPE when LC_ALL is not set', () => {
process.env.LC_CTYPE = 'fr_FR.ISO-8859-1';
process.env['LC_CTYPE'] = 'fr_FR.ISO-8859-1';
const result = getSystemEncoding();
expect(result).toBe('iso-8859-1');
});
it('should parse locale from LANG when LC_ALL and LC_CTYPE are not set', () => {
process.env.LANG = 'de_DE.UTF-8';
process.env['LANG'] = 'de_DE.UTF-8';
const result = getSystemEncoding();
expect(result).toBe('utf-8');
@@ -268,16 +268,16 @@ describe('Shell Command Processor - Encoding Functions', () => {
});
it('should handle locale without encoding (no dot)', () => {
process.env.LANG = 'C';
process.env['LANG'] = 'C';
const result = getSystemEncoding();
expect(result).toBe('c');
});
it('should handle empty locale environment variables', () => {
process.env.LC_ALL = '';
process.env.LC_CTYPE = '';
process.env.LANG = '';
process.env['LC_ALL'] = '';
process.env['LC_CTYPE'] = '';
process.env['LANG'] = '';
mockedExecSync.mockReturnValue('UTF-8');
const result = getSystemEncoding();
@@ -285,24 +285,24 @@ describe('Shell Command Processor - Encoding Functions', () => {
});
it('should return locale as-is when locale format has no dot', () => {
process.env.LANG = 'invalid_format';
process.env['LANG'] = 'invalid_format';
const result = getSystemEncoding();
expect(result).toBe('invalid_format');
});
it('should prioritize LC_ALL over other environment variables', () => {
process.env.LC_ALL = 'en_US.UTF-8';
process.env.LC_CTYPE = 'fr_FR.ISO-8859-1';
process.env.LANG = 'de_DE.CP1252';
process.env['LC_ALL'] = 'en_US.UTF-8';
process.env['LC_CTYPE'] = 'fr_FR.ISO-8859-1';
process.env['LANG'] = 'de_DE.CP1252';
const result = getSystemEncoding();
expect(result).toBe('utf-8');
});
it('should prioritize LC_CTYPE over LANG', () => {
process.env.LC_CTYPE = 'fr_FR.ISO-8859-1';
process.env.LANG = 'de_DE.CP1252';
process.env['LC_CTYPE'] = 'fr_FR.ISO-8859-1';
process.env['LANG'] = 'de_DE.CP1252';
const result = getSystemEncoding();
expect(result).toBe('iso-8859-1');
@@ -315,7 +315,7 @@ describe('Shell Command Processor - Encoding Functions', () => {
});
it('should use cached system encoding on subsequent calls', () => {
process.env.LANG = 'en_US.UTF-8';
process.env['LANG'] = 'en_US.UTF-8';
const buffer = Buffer.from('test');
// First call
@@ -323,7 +323,7 @@ describe('Shell Command Processor - Encoding Functions', () => {
expect(result1).toBe('utf-8');
// Change environment (should not affect cached result)
process.env.LANG = 'fr_FR.ISO-8859-1';
process.env['LANG'] = 'fr_FR.ISO-8859-1';
// Second call should use cached value
const result2 = getCachedEncodingForBuffer(buffer);
@@ -435,7 +435,7 @@ describe('Shell Command Processor - Encoding Functions', () => {
describe('Cross-platform behavior', () => {
it('should work correctly on macOS', () => {
mockedOsPlatform.mockReturnValue('darwin');
process.env.LANG = 'en_US.UTF-8';
process.env['LANG'] = 'en_US.UTF-8';
const result = getSystemEncoding();
expect(result).toBe('utf-8');
@@ -443,7 +443,7 @@ describe('Shell Command Processor - Encoding Functions', () => {
it('should work correctly on other Unix-like systems', () => {
mockedOsPlatform.mockReturnValue('freebsd');
process.env.LANG = 'en_US.UTF-8';
process.env['LANG'] = 'en_US.UTF-8';
const result = getSystemEncoding();
expect(result).toBe('utf-8');
@@ -451,7 +451,7 @@ describe('Shell Command Processor - Encoding Functions', () => {
it('should handle unknown platforms as Unix-like', () => {
mockedOsPlatform.mockReturnValue('unknown' as NodeJS.Platform);
process.env.LANG = 'en_US.UTF-8';
process.env['LANG'] = 'en_US.UTF-8';
const result = getSystemEncoding();
expect(result).toBe('utf-8');
@@ -461,7 +461,7 @@ describe('Shell Command Processor - Encoding Functions', () => {
describe('Edge cases and error handling', () => {
it('should handle empty buffer gracefully', () => {
mockedOsPlatform.mockReturnValue('linux');
process.env.LANG = 'en_US.UTF-8';
process.env['LANG'] = 'en_US.UTF-8';
const buffer = Buffer.alloc(0);
const result = getCachedEncodingForBuffer(buffer);
@@ -470,7 +470,7 @@ describe('Shell Command Processor - Encoding Functions', () => {
it('should handle very large buffers', () => {
mockedOsPlatform.mockReturnValue('linux');
process.env.LANG = 'en_US.UTF-8';
process.env['LANG'] = 'en_US.UTF-8';
const buffer = Buffer.alloc(1024 * 1024, 'a');
const result = getCachedEncodingForBuffer(buffer);

2
packages/core/src/utils/systemEncoding.ts
View File

@@ -79,7 +79,7 @@ export function getSystemEncoding(): string | null {
// system encoding. However, these environment variables might not always
// be set or accurate. Handle cases where none of these variables are set.
const env = process.env;
let locale = env.LC_ALL || env.LC_CTYPE || env.LANG || '';
let locale = env['LC_ALL'] || env['LC_CTYPE'] || env['LANG'] || '';
// Fallback to querying the system directly when environment variables are missing
if (!locale) {

120
packages/core/src/utils/user_account.test.ts
View File

@@ -99,18 +99,37 @@ describe('user_account', () => {
it('should handle corrupted JSON by starting fresh', async () => {
fs.mkdirSync(path.dirname(accountsFile()), { recursive: true });
fs.writeFileSync(accountsFile(), 'not valid json');
const consoleDebugSpy = vi
.spyOn(console, 'debug')
const consoleLogSpy = vi
.spyOn(console, 'log')
.mockImplementation(() => {});
await cacheGoogleAccount('test1@google.com');
expect(consoleDebugSpy).toHaveBeenCalled();
expect(consoleLogSpy).toHaveBeenCalled();
expect(JSON.parse(fs.readFileSync(accountsFile(), 'utf-8'))).toEqual({
active: 'test1@google.com',
old: [],
});
});
it('should handle valid JSON with incorrect schema by starting fresh', async () => {
fs.mkdirSync(path.dirname(accountsFile()), { recursive: true });
fs.writeFileSync(
accountsFile(),
JSON.stringify({ active: 'test1@google.com', old: 'not-an-array' }),
);
const consoleLogSpy = vi
.spyOn(console, 'log')
.mockImplementation(() => {});
await cacheGoogleAccount('test2@google.com');
expect(consoleLogSpy).toHaveBeenCalled();
expect(JSON.parse(fs.readFileSync(accountsFile(), 'utf-8'))).toEqual({
active: 'test2@google.com',
old: [],
});
});
});
describe('getCachedGoogleAccount', () => {
@@ -139,14 +158,21 @@ describe('user_account', () => {
it('should return null and log if file is corrupted', () => {
fs.mkdirSync(path.dirname(accountsFile()), { recursive: true });
fs.writeFileSync(accountsFile(), '{ "active": "test@google.com"'); // Invalid JSON
const consoleDebugSpy = vi
.spyOn(console, 'debug')
const consoleLogSpy = vi
.spyOn(console, 'log')
.mockImplementation(() => {});
const account = getCachedGoogleAccount();
expect(account).toBeNull();
expect(consoleDebugSpy).toHaveBeenCalled();
expect(consoleLogSpy).toHaveBeenCalled();
});
it('should return null if active key is missing', () => {
fs.mkdirSync(path.dirname(accountsFile()), { recursive: true });
fs.writeFileSync(accountsFile(), JSON.stringify({ old: [] }));
const account = getCachedGoogleAccount();
expect(account).toBeNull();
});
});
@@ -177,6 +203,56 @@ describe('user_account', () => {
expect(stored.active).toBeNull();
expect(stored.old).toEqual([]);
});
it('should handle corrupted JSON by creating a fresh file', async () => {
fs.mkdirSync(path.dirname(accountsFile()), { recursive: true });
fs.writeFileSync(accountsFile(), 'not valid json');
const consoleLogSpy = vi
.spyOn(console, 'log')
.mockImplementation(() => {});
await clearCachedGoogleAccount();
expect(consoleLogSpy).toHaveBeenCalled();
const stored = JSON.parse(fs.readFileSync(accountsFile(), 'utf-8'));
expect(stored.active).toBeNull();
expect(stored.old).toEqual([]);
});
it('should be idempotent if active account is already null', async () => {
fs.mkdirSync(path.dirname(accountsFile()), { recursive: true });
fs.writeFileSync(
accountsFile(),
JSON.stringify({ active: null, old: ['old1@google.com'] }, null, 2),
);
await clearCachedGoogleAccount();
const stored = JSON.parse(fs.readFileSync(accountsFile(), 'utf-8'));
expect(stored.active).toBeNull();
expect(stored.old).toEqual(['old1@google.com']);
});
it('should not add a duplicate to the old list', async () => {
fs.mkdirSync(path.dirname(accountsFile()), { recursive: true });
fs.writeFileSync(
accountsFile(),
JSON.stringify(
{
active: 'active@google.com',
old: ['active@google.com'],
},
null,
2,
),
);
await clearCachedGoogleAccount();
const stored = JSON.parse(fs.readFileSync(accountsFile(), 'utf-8'));
expect(stored.active).toBeNull();
expect(stored.old).toEqual(['active@google.com']);
});
});
describe('getLifetimeGoogleAccounts', () => {
@@ -193,12 +269,12 @@ describe('user_account', () => {
it('should return 0 if the file is corrupted', () => {
fs.mkdirSync(path.dirname(accountsFile()), { recursive: true });
fs.writeFileSync(accountsFile(), 'invalid json');
const consoleDebugSpy = vi
.spyOn(console, 'debug')
const consoleLogSpy = vi
.spyOn(console, 'log')
.mockImplementation(() => {});
expect(getLifetimeGoogleAccounts()).toBe(0);
expect(consoleDebugSpy).toHaveBeenCalled();
expect(consoleLogSpy).toHaveBeenCalled();
});
it('should return 1 if there is only an active account', () => {
@@ -233,5 +309,31 @@ describe('user_account', () => {
);
expect(getLifetimeGoogleAccounts()).toBe(3);
});
it('should handle valid JSON with incorrect schema by returning 0', () => {
fs.mkdirSync(path.dirname(accountsFile()), { recursive: true });
fs.writeFileSync(
accountsFile(),
JSON.stringify({ active: null, old: 1 }),
);
const consoleLogSpy = vi
.spyOn(console, 'log')
.mockImplementation(() => {});
expect(getLifetimeGoogleAccounts()).toBe(0);
expect(consoleLogSpy).toHaveBeenCalled();
});
it('should not double count if active account is also in old list', () => {
fs.mkdirSync(path.dirname(accountsFile()), { recursive: true });
fs.writeFileSync(
accountsFile(),
JSON.stringify({
active: 'test1@google.com',
old: ['test1@google.com', 'test2@google.com'],
}),
);
expect(getLifetimeGoogleAccounts()).toBe(2);
});
});
});

116
packages/core/src/utils/user_account.ts
View File

@@ -5,7 +5,7 @@
*/
import path from 'node:path';
import { promises as fsp, existsSync, readFileSync } from 'node:fs';
import { promises as fsp, readFileSync } from 'node:fs';
import * as os from 'os';
import { QWEN_DIR, GOOGLE_ACCOUNTS_FILENAME } from './paths.js';
@@ -18,21 +18,66 @@ function getGoogleAccountsCachePath(): string {
return path.join(os.homedir(), QWEN_DIR, GOOGLE_ACCOUNTS_FILENAME);
}
async function readAccounts(filePath: string): Promise<UserAccounts> {
/**
* Parses and validates the string content of an accounts file.
* @param content The raw string content from the file.
* @returns A valid UserAccounts object.
*/
function parseAndValidateAccounts(content: string): UserAccounts {
const defaultState = { active: null, old: [] };
if (!content.trim()) {
return defaultState;
}
const parsed = JSON.parse(content);
// Inlined validation logic
if (typeof parsed !== 'object' || parsed === null) {
console.log('Invalid accounts file schema, starting fresh.');
return defaultState;
}
const { active, old } = parsed as Partial<UserAccounts>;
const isValid =
(active === undefined || active === null || typeof active === 'string') &&
(old === undefined ||
(Array.isArray(old) && old.every((i) => typeof i === 'string')));
if (!isValid) {
console.log('Invalid accounts file schema, starting fresh.');
return defaultState;
}
return {
active: parsed.active ?? null,
old: parsed.old ?? [],
};
}
function readAccountsSync(filePath: string): UserAccounts {
const defaultState = { active: null, old: [] };
try {
const content = await fsp.readFile(filePath, 'utf-8');
if (!content.trim()) {
return { active: null, old: [] };
}
return JSON.parse(content) as UserAccounts;
const content = readFileSync(filePath, 'utf-8');
return parseAndValidateAccounts(content);
} catch (error) {
if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
// File doesn't exist, which is fine.
return { active: null, old: [] };
return defaultState;
}
// File is corrupted or not valid JSON, start with a fresh object.
console.debug('Could not parse accounts file, starting fresh.', error);
return { active: null, old: [] };
console.log('Error during sync read of accounts, starting fresh.', error);
return defaultState;
}
}
async function readAccounts(filePath: string): Promise<UserAccounts> {
const defaultState = { active: null, old: [] };
try {
const content = await fsp.readFile(filePath, 'utf-8');
return parseAndValidateAccounts(content);
} catch (error) {
if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
return defaultState;
}
console.log('Could not parse accounts file, starting fresh.', error);
return defaultState;
}
}
@@ -56,52 +101,23 @@ export async function cacheGoogleAccount(email: string): Promise<void> {
}
export function getCachedGoogleAccount(): string | null {
try {
const filePath = getGoogleAccountsCachePath();
if (existsSync(filePath)) {
const content = readFileSync(filePath, 'utf-8').trim();
if (!content) {
return null;
}
const accounts: UserAccounts = JSON.parse(content);
return accounts.active;
}
return null;
} catch (error) {
console.debug('Error reading cached Google Account:', error);
return null;
}
const filePath = getGoogleAccountsCachePath();
const accounts = readAccountsSync(filePath);
return accounts.active;
}
export function getLifetimeGoogleAccounts(): number {
try {
const filePath = getGoogleAccountsCachePath();
if (!existsSync(filePath)) {
return 0;
}
const content = readFileSync(filePath, 'utf-8').trim();
if (!content) {
return 0;
}
const accounts: UserAccounts = JSON.parse(content);
let count = accounts.old.length;
if (accounts.active) {
count++;
}
return count;
} catch (error) {
console.debug('Error reading lifetime Google Accounts:', error);
return 0;
const filePath = getGoogleAccountsCachePath();
const accounts = readAccountsSync(filePath);
const allAccounts = new Set(accounts.old);
if (accounts.active) {
allAccounts.add(accounts.active);
}
return allAccounts.size;
}
export async function clearCachedGoogleAccount(): Promise<void> {
const filePath = getGoogleAccountsCachePath();
if (!existsSync(filePath)) {
return;
}
const accounts = await readAccounts(filePath);
if (accounts.active) {

472
packages/core/src/utils/workspaceContext.test.ts
View File

@@ -4,280 +4,386 @@
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect, beforeEach, vi } from 'vitest';
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { WorkspaceContext } from './workspaceContext.js';
vi.mock('fs');
describe('WorkspaceContext', () => {
let workspaceContext: WorkspaceContext;
// Use path module to create platform-agnostic paths
const mockCwd = path.resolve(path.sep, 'home', 'user', 'project');
const mockExistingDir = path.resolve(
path.sep,
'home',
'user',
'other-project',
);
const mockNonExistentDir = path.resolve(
path.sep,
'home',
'user',
'does-not-exist',
);
const mockSymlinkDir = path.resolve(path.sep, 'home', 'user', 'symlink');
const mockRealPath = path.resolve(path.sep, 'home', 'user', 'real-directory');
describe('WorkspaceContext with real filesystem', () => {
let tempDir: string;
let cwd: string;
let otherDir: string;
beforeEach(() => {
vi.resetAllMocks();
// os.tmpdir() can return a path using a symlink (this is standard on macOS)
// Use fs.realpathSync to fully resolve the absolute path.
tempDir = fs.realpathSync(
fs.mkdtempSync(path.join(os.tmpdir(), 'workspace-context-test-')),
);
// Mock fs.existsSync
vi.mocked(fs.existsSync).mockImplementation((path) => {
const pathStr = path.toString();
return (
pathStr === mockCwd ||
pathStr === mockExistingDir ||
pathStr === mockSymlinkDir ||
pathStr === mockRealPath
);
});
cwd = path.join(tempDir, 'project');
otherDir = path.join(tempDir, 'other-project');
// Mock fs.statSync
vi.mocked(fs.statSync).mockImplementation((path) => {
const pathStr = path.toString();
if (pathStr === mockNonExistentDir) {
throw new Error('ENOENT');
}
return {
isDirectory: () => true,
} as fs.Stats;
});
fs.mkdirSync(cwd, { recursive: true });
fs.mkdirSync(otherDir, { recursive: true });
});
// Mock fs.realpathSync
vi.mocked(fs.realpathSync).mockImplementation((path) => {
const pathStr = path.toString();
if (pathStr === mockSymlinkDir) {
return mockRealPath;
}
return pathStr;
});
afterEach(() => {
fs.rmSync(tempDir, { recursive: true, force: true });
});
describe('initialization', () => {
it('should initialize with a single directory (cwd)', () => {
workspaceContext = new WorkspaceContext(mockCwd);
const workspaceContext = new WorkspaceContext(cwd);
const directories = workspaceContext.getDirectories();
expect(directories).toHaveLength(1);
expect(directories[0]).toBe(mockCwd);
expect(directories).toEqual([cwd]);
});
it('should validate and resolve directories to absolute paths', () => {
const absolutePath = path.join(mockCwd, 'subdir');
vi.mocked(fs.existsSync).mockImplementation(
(p) => p === mockCwd || p === absolutePath,
);
vi.mocked(fs.realpathSync).mockImplementation((p) => p.toString());
workspaceContext = new WorkspaceContext(mockCwd, [absolutePath]);
const workspaceContext = new WorkspaceContext(cwd, [otherDir]);
const directories = workspaceContext.getDirectories();
expect(directories).toContain(absolutePath);
expect(directories).toEqual([cwd, otherDir]);
});
it('should reject non-existent directories', () => {
const nonExistentDir = path.join(tempDir, 'does-not-exist');
expect(() => {
new WorkspaceContext(mockCwd, [mockNonExistentDir]);
new WorkspaceContext(cwd, [nonExistentDir]);
}).toThrow('Directory does not exist');
});
it('should handle empty initialization', () => {
workspaceContext = new WorkspaceContext(mockCwd, []);
const workspaceContext = new WorkspaceContext(cwd, []);
const directories = workspaceContext.getDirectories();
expect(directories).toHaveLength(1);
expect(directories[0]).toBe(mockCwd);
expect(fs.realpathSync(directories[0])).toBe(cwd);
});
});
describe('adding directories', () => {
beforeEach(() => {
workspaceContext = new WorkspaceContext(mockCwd);
});
it('should add valid directories', () => {
workspaceContext.addDirectory(mockExistingDir);
const workspaceContext = new WorkspaceContext(cwd);
workspaceContext.addDirectory(otherDir);
const directories = workspaceContext.getDirectories();
expect(directories).toHaveLength(2);
expect(directories).toContain(mockExistingDir);
expect(directories).toEqual([cwd, otherDir]);
});
it('should resolve relative paths to absolute', () => {
// Since we can't mock path.resolve, we'll test with absolute paths
workspaceContext.addDirectory(mockExistingDir);
const workspaceContext = new WorkspaceContext(cwd);
const relativePath = path.relative(cwd, otherDir);
workspaceContext.addDirectory(relativePath, cwd);
const directories = workspaceContext.getDirectories();
expect(directories).toContain(mockExistingDir);
expect(directories).toEqual([cwd, otherDir]);
});
it('should reject non-existent directories', () => {
const nonExistentDir = path.join(tempDir, 'does-not-exist');
const workspaceContext = new WorkspaceContext(cwd);
expect(() => {
workspaceContext.addDirectory(mockNonExistentDir);
workspaceContext.addDirectory(nonExistentDir);
}).toThrow('Directory does not exist');
});
it('should prevent duplicate directories', () => {
workspaceContext.addDirectory(mockExistingDir);
workspaceContext.addDirectory(mockExistingDir);
const workspaceContext = new WorkspaceContext(cwd);
workspaceContext.addDirectory(otherDir);
workspaceContext.addDirectory(otherDir);
const directories = workspaceContext.getDirectories();
expect(directories.filter((d) => d === mockExistingDir)).toHaveLength(1);
expect(directories).toHaveLength(2);
});
it('should handle symbolic links correctly', () => {
workspaceContext.addDirectory(mockSymlinkDir);
const realDir = path.join(tempDir, 'real');
fs.mkdirSync(realDir, { recursive: true });
const symlinkDir = path.join(tempDir, 'symlink-to-real');
fs.symlinkSync(realDir, symlinkDir, 'dir');
const workspaceContext = new WorkspaceContext(cwd);
workspaceContext.addDirectory(symlinkDir);
const directories = workspaceContext.getDirectories();
expect(directories).toContain(mockRealPath);
expect(directories).not.toContain(mockSymlinkDir);
expect(directories).toEqual([cwd, realDir]);
});
});
describe('path validation', () => {
beforeEach(() => {
workspaceContext = new WorkspaceContext(mockCwd, [mockExistingDir]);
it('should accept paths within workspace directories', () => {
const workspaceContext = new WorkspaceContext(cwd, [otherDir]);
const validPath1 = path.join(cwd, 'src', 'file.ts');
const validPath2 = path.join(otherDir, 'lib', 'module.js');
fs.mkdirSync(path.dirname(validPath1), { recursive: true });
fs.writeFileSync(validPath1, 'content');
fs.mkdirSync(path.dirname(validPath2), { recursive: true });
fs.writeFileSync(validPath2, 'content');
expect(workspaceContext.isPathWithinWorkspace(validPath1)).toBe(true);
expect(workspaceContext.isPathWithinWorkspace(validPath2)).toBe(true);
});
it('should accept paths within workspace directories', () => {
const validPath1 = path.join(mockCwd, 'src', 'file.ts');
const validPath2 = path.join(mockExistingDir, 'lib', 'module.js');
it('should accept non-existent paths within workspace directories', () => {
const workspaceContext = new WorkspaceContext(cwd, [otherDir]);
const validPath1 = path.join(cwd, 'src', 'file.ts');
const validPath2 = path.join(otherDir, 'lib', 'module.js');
expect(workspaceContext.isPathWithinWorkspace(validPath1)).toBe(true);
expect(workspaceContext.isPathWithinWorkspace(validPath2)).toBe(true);
});
it('should reject paths outside workspace', () => {
const invalidPath = path.resolve(
path.dirname(mockCwd),
'outside-workspace',
'file.txt',
);
const workspaceContext = new WorkspaceContext(cwd, [otherDir]);
const invalidPath = path.join(tempDir, 'outside-workspace', 'file.txt');
expect(workspaceContext.isPathWithinWorkspace(invalidPath)).toBe(false);
});
it('should resolve symbolic links before validation', () => {
const symlinkPath = path.join(mockCwd, 'symlink-file');
const realPath = path.join(mockCwd, 'real-file');
it('should reject non-existent paths outside workspace', () => {
const workspaceContext = new WorkspaceContext(cwd, [otherDir]);
const invalidPath = path.join(tempDir, 'outside-workspace', 'file.txt');
vi.mocked(fs.existsSync).mockReturnValue(true);
vi.mocked(fs.realpathSync).mockImplementation((p) => {
if (p === symlinkPath) {
return realPath;
}
return p.toString();
});
expect(workspaceContext.isPathWithinWorkspace(symlinkPath)).toBe(true);
expect(workspaceContext.isPathWithinWorkspace(invalidPath)).toBe(false);
});
it('should handle nested directories correctly', () => {
const nestedPath = path.join(
mockCwd,
'deeply',
'nested',
'path',
'file.txt',
);
const workspaceContext = new WorkspaceContext(cwd, [otherDir]);
const nestedPath = path.join(cwd, 'deeply', 'nested', 'path', 'file.txt');
expect(workspaceContext.isPathWithinWorkspace(nestedPath)).toBe(true);
});
it('should handle edge cases (root, parent references)', () => {
const rootPath = '/';
const parentPath = path.dirname(mockCwd);
const workspaceContext = new WorkspaceContext(cwd, [otherDir]);
const rootPath = path.parse(tempDir).root;
const parentPath = path.dirname(cwd);
expect(workspaceContext.isPathWithinWorkspace(rootPath)).toBe(false);
expect(workspaceContext.isPathWithinWorkspace(parentPath)).toBe(false);
});
it('should handle non-existent paths correctly', () => {
const nonExistentPath = path.join(mockCwd, 'does-not-exist.txt');
vi.mocked(fs.existsSync).mockImplementation((p) => p !== nonExistentPath);
// Should still validate based on path structure
const workspaceContext = new WorkspaceContext(cwd, [otherDir]);
const nonExistentPath = path.join(cwd, 'does-not-exist.txt');
expect(workspaceContext.isPathWithinWorkspace(nonExistentPath)).toBe(
true,
);
});
describe('with symbolic link', () => {
describe('in the workspace', () => {
let realDir: string;
let symlinkDir: string;
beforeEach(() => {
realDir = path.join(cwd, 'real-dir');
fs.mkdirSync(realDir, { recursive: true });
symlinkDir = path.join(cwd, 'symlink-file');
fs.symlinkSync(realDir, symlinkDir, 'dir');
});
it('should accept dir paths', () => {
const workspaceContext = new WorkspaceContext(cwd);
expect(workspaceContext.isPathWithinWorkspace(symlinkDir)).toBe(true);
});
it('should accept non-existent paths', () => {
const filePath = path.join(symlinkDir, 'does-not-exist.txt');
const workspaceContext = new WorkspaceContext(cwd);
expect(workspaceContext.isPathWithinWorkspace(filePath)).toBe(true);
});
it('should accept non-existent deep paths', () => {
const filePath = path.join(symlinkDir, 'deep', 'does-not-exist.txt');
const workspaceContext = new WorkspaceContext(cwd);
expect(workspaceContext.isPathWithinWorkspace(filePath)).toBe(true);
});
});
describe('outside the workspace', () => {
let realDir: string;
let symlinkDir: string;
beforeEach(() => {
realDir = path.join(tempDir, 'real-dir');
fs.mkdirSync(realDir, { recursive: true });
symlinkDir = path.join(cwd, 'symlink-file');
fs.symlinkSync(realDir, symlinkDir, 'dir');
});
it('should reject dir paths', () => {
const workspaceContext = new WorkspaceContext(cwd);
expect(workspaceContext.isPathWithinWorkspace(symlinkDir)).toBe(
false,
);
});
it('should reject non-existent paths', () => {
const filePath = path.join(symlinkDir, 'does-not-exist.txt');
const workspaceContext = new WorkspaceContext(cwd);
expect(workspaceContext.isPathWithinWorkspace(filePath)).toBe(false);
});
it('should reject non-existent deep paths', () => {
const filePath = path.join(symlinkDir, 'deep', 'does-not-exist.txt');
const workspaceContext = new WorkspaceContext(cwd);
expect(workspaceContext.isPathWithinWorkspace(filePath)).toBe(false);
});
it('should reject partially non-existent deep paths', () => {
const deepDir = path.join(symlinkDir, 'deep');
fs.mkdirSync(deepDir, { recursive: true });
const filePath = path.join(deepDir, 'does-not-exist.txt');
const workspaceContext = new WorkspaceContext(cwd);
expect(workspaceContext.isPathWithinWorkspace(filePath)).toBe(false);
});
});
it('should reject symbolic file links outside the workspace', () => {
const realFile = path.join(tempDir, 'real-file.txt');
fs.writeFileSync(realFile, 'content');
const symlinkFile = path.join(cwd, 'symlink-to-real-file');
fs.symlinkSync(realFile, symlinkFile, 'file');
const workspaceContext = new WorkspaceContext(cwd);
expect(workspaceContext.isPathWithinWorkspace(symlinkFile)).toBe(false);
});
it('should reject non-existent symbolic file links outside the workspace', () => {
const realFile = path.join(tempDir, 'real-file.txt');
const symlinkFile = path.join(cwd, 'symlink-to-real-file');
fs.symlinkSync(realFile, symlinkFile, 'file');
const workspaceContext = new WorkspaceContext(cwd);
expect(workspaceContext.isPathWithinWorkspace(symlinkFile)).toBe(false);
});
it('should handle circular symlinks gracefully', () => {
const workspaceContext = new WorkspaceContext(cwd);
const linkA = path.join(cwd, 'link-a');
const linkB = path.join(cwd, 'link-b');
// Create a circular dependency: linkA -> linkB -> linkA
fs.symlinkSync(linkB, linkA, 'dir');
fs.symlinkSync(linkA, linkB, 'dir');
// fs.realpathSync should throw ELOOP, and isPathWithinWorkspace should
// handle it gracefully and return false.
expect(workspaceContext.isPathWithinWorkspace(linkA)).toBe(false);
expect(workspaceContext.isPathWithinWorkspace(linkB)).toBe(false);
});
});
});
describe('onDirectoriesChanged', () => {
it('should call listener when adding a directory', () => {
const workspaceContext = new WorkspaceContext(cwd);
const listener = vi.fn();
workspaceContext.onDirectoriesChanged(listener);
workspaceContext.addDirectory(otherDir);
expect(listener).toHaveBeenCalledOnce();
});
it('should not call listener when adding a duplicate directory', () => {
const workspaceContext = new WorkspaceContext(cwd);
workspaceContext.addDirectory(otherDir);
const listener = vi.fn();
workspaceContext.onDirectoriesChanged(listener);
workspaceContext.addDirectory(otherDir);
expect(listener).not.toHaveBeenCalled();
});
it('should call listener when setting different directories', () => {
const workspaceContext = new WorkspaceContext(cwd);
const listener = vi.fn();
workspaceContext.onDirectoriesChanged(listener);
workspaceContext.setDirectories([otherDir]);
expect(listener).toHaveBeenCalledOnce();
});
it('should not call listener when setting same directories', () => {
const workspaceContext = new WorkspaceContext(cwd);
const listener = vi.fn();
workspaceContext.onDirectoriesChanged(listener);
workspaceContext.setDirectories([cwd]);
expect(listener).not.toHaveBeenCalled();
});
it('should support multiple listeners', () => {
const workspaceContext = new WorkspaceContext(cwd);
const listener1 = vi.fn();
const listener2 = vi.fn();
workspaceContext.onDirectoriesChanged(listener1);
workspaceContext.onDirectoriesChanged(listener2);
workspaceContext.addDirectory(otherDir);
expect(listener1).toHaveBeenCalledOnce();
expect(listener2).toHaveBeenCalledOnce();
});
it('should allow unsubscribing a listener', () => {
const workspaceContext = new WorkspaceContext(cwd);
const listener = vi.fn();
const unsubscribe = workspaceContext.onDirectoriesChanged(listener);
unsubscribe();
workspaceContext.addDirectory(otherDir);
expect(listener).not.toHaveBeenCalled();
});
it('should not fail if a listener throws an error', () => {
const workspaceContext = new WorkspaceContext(cwd);
const errorListener = () => {
throw new Error('test error');
};
const listener = vi.fn();
workspaceContext.onDirectoriesChanged(errorListener);
workspaceContext.onDirectoriesChanged(listener);
expect(() => {
workspaceContext.addDirectory(otherDir);
}).not.toThrow();
expect(listener).toHaveBeenCalledOnce();
});
});
describe('getDirectories', () => {
it('should return a copy of directories array', () => {
workspaceContext = new WorkspaceContext(mockCwd);
const workspaceContext = new WorkspaceContext(cwd);
const dirs1 = workspaceContext.getDirectories();
const dirs2 = workspaceContext.getDirectories();
expect(dirs1).not.toBe(dirs2); // Different array instances
expect(dirs1).toEqual(dirs2); // Same content
});
});
describe('symbolic link security', () => {
beforeEach(() => {
workspaceContext = new WorkspaceContext(mockCwd);
});
it('should follow symlinks but validate resolved path', () => {
const symlinkInsideWorkspace = path.join(mockCwd, 'link-to-subdir');
const resolvedInsideWorkspace = path.join(mockCwd, 'subdir');
vi.mocked(fs.existsSync).mockReturnValue(true);
vi.mocked(fs.realpathSync).mockImplementation((p) => {
if (p === symlinkInsideWorkspace) {
return resolvedInsideWorkspace;
}
return p.toString();
});
expect(
workspaceContext.isPathWithinWorkspace(symlinkInsideWorkspace),
).toBe(true);
});
it('should prevent sandbox escape via symlinks', () => {
const symlinkEscape = path.join(mockCwd, 'escape-link');
const resolvedOutside = path.resolve(mockCwd, '..', 'outside-file');
vi.mocked(fs.existsSync).mockImplementation((p) => {
const pathStr = p.toString();
return (
pathStr === symlinkEscape ||
pathStr === resolvedOutside ||
pathStr === mockCwd
);
});
vi.mocked(fs.realpathSync).mockImplementation((p) => {
if (p.toString() === symlinkEscape) {
return resolvedOutside;
}
return p.toString();
});
vi.mocked(fs.statSync).mockImplementation(
(p) =>
({
isDirectory: () => p.toString() !== resolvedOutside,
}) as fs.Stats,
);
workspaceContext = new WorkspaceContext(mockCwd);
expect(workspaceContext.isPathWithinWorkspace(symlinkEscape)).toBe(false);
});
it('should handle circular symlinks', () => {
const circularLink = path.join(mockCwd, 'circular');
vi.mocked(fs.existsSync).mockReturnValue(true);
vi.mocked(fs.realpathSync).mockImplementation(() => {
throw new Error('ELOOP: too many symbolic links encountered');
});
// Should handle the error gracefully
expect(workspaceContext.isPathWithinWorkspace(circularLink)).toBe(false);
expect(dirs1).not.toBe(dirs2);
expect(dirs1).toEqual(dirs2);
});
});
});

157
packages/core/src/utils/workspaceContext.ts
View File

@@ -4,34 +4,57 @@
* SPDX-License-Identifier: Apache-2.0
*/
import { isNodeError } from '../utils/errors.js';
import * as fs from 'fs';
import * as path from 'path';
export type Unsubscribe = () => void;
/**
* WorkspaceContext manages multiple workspace directories and validates paths
* against them. This allows the CLI to operate on files from multiple directories
* in a single session.
*/
export class WorkspaceContext {
private directories: Set<string>;
private directories = new Set<string>();
private initialDirectories: Set<string>;
private onDirectoriesChangedListeners = new Set<() => void>();
/**
* Creates a new WorkspaceContext with the given initial directory and optional additional directories.
* @param initialDirectory The initial working directory (usually cwd)
* @param directory The initial working directory (usually cwd)
* @param additionalDirectories Optional array of additional directories to include
*/
constructor(initialDirectory: string, additionalDirectories: string[] = []) {
this.directories = new Set<string>();
this.initialDirectories = new Set<string>();
constructor(directory: string, additionalDirectories: string[] = []) {
this.addDirectory(directory);
for (const additionalDirectory of additionalDirectories) {
this.addDirectory(additionalDirectory);
}
this.addDirectoryInternal(initialDirectory);
this.addInitialDirectoryInternal(initialDirectory);
this.initialDirectories = new Set(this.directories);
}
for (const dir of additionalDirectories) {
this.addDirectoryInternal(dir);
this.addInitialDirectoryInternal(dir);
/**
* Registers a listener that is called when the workspace directories change.
* @param listener The listener to call.
* @returns A function to unsubscribe the listener.
*/
onDirectoriesChanged(listener: () => void): Unsubscribe {
this.onDirectoriesChangedListeners.add(listener);
return () => {
this.onDirectoriesChangedListeners.delete(listener);
};
}
private notifyDirectoriesChanged() {
// Iterate over a copy of the set in case a listener unsubscribes itself or others.
for (const listener of [...this.onDirectoriesChangedListeners]) {
try {
listener();
} catch (e) {
// Don't let one listener break others.
console.error('Error in WorkspaceContext listener:', e);
}
}
}
@@ -41,16 +64,18 @@ export class WorkspaceContext {
* @param basePath Optional base path for resolving relative paths (defaults to cwd)
*/
addDirectory(directory: string, basePath: string = process.cwd()): void {
this.addDirectoryInternal(directory, basePath);
const resolved = this.resolveAndValidateDir(directory, basePath);
if (this.directories.has(resolved)) {
return;
}
this.directories.add(resolved);
this.notifyDirectoriesChanged();
}
/**
* Internal method to add a directory with validation.
*/
private addDirectoryInternal(
private resolveAndValidateDir(
directory: string,
basePath: string = process.cwd(),
): void {
): string {
const absolutePath = path.isAbsolute(directory)
? directory
: path.resolve(basePath, directory);
@@ -58,47 +83,12 @@ export class WorkspaceContext {
if (!fs.existsSync(absolutePath)) {
throw new Error(`Directory does not exist: ${absolutePath}`);
}
const stats = fs.statSync(absolutePath);
if (!stats.isDirectory()) {
throw new Error(`Path is not a directory: ${absolutePath}`);
}
let realPath: string;
try {
realPath = fs.realpathSync(absolutePath);
} catch (_error) {
throw new Error(`Failed to resolve path: ${absolutePath}`);
}
this.directories.add(realPath);
}
private addInitialDirectoryInternal(
directory: string,
basePath: string = process.cwd(),
): void {
const absolutePath = path.isAbsolute(directory)
? directory
: path.resolve(basePath, directory);
if (!fs.existsSync(absolutePath)) {
throw new Error(`Directory does not exist: ${absolutePath}`);
}
const stats = fs.statSync(absolutePath);
if (!stats.isDirectory()) {
throw new Error(`Path is not a directory: ${absolutePath}`);
}
let realPath: string;
try {
realPath = fs.realpathSync(absolutePath);
} catch (_error) {
throw new Error(`Failed to resolve path: ${absolutePath}`);
}
this.initialDirectories.add(realPath);
return fs.realpathSync(absolutePath);
}
/**
@@ -114,9 +104,17 @@ export class WorkspaceContext {
}
setDirectories(directories: readonly string[]): void {
this.directories.clear();
const newDirectories = new Set<string>();
for (const dir of directories) {
this.addDirectoryInternal(dir);
newDirectories.add(this.resolveAndValidateDir(dir));
}
if (
newDirectories.size !== this.directories.size ||
![...newDirectories].every((d) => this.directories.has(d))
) {
this.directories = newDirectories;
this.notifyDirectoriesChanged();
}
}
@@ -127,29 +125,43 @@ export class WorkspaceContext {
*/
isPathWithinWorkspace(pathToCheck: string): boolean {
try {
const absolutePath = path.resolve(pathToCheck);
let resolvedPath = absolutePath;
if (fs.existsSync(absolutePath)) {
try {
resolvedPath = fs.realpathSync(absolutePath);
} catch (_error) {
return false;
}
}
const fullyResolvedPath = this.fullyResolvedPath(pathToCheck);
for (const dir of this.directories) {
if (this.isPathWithinRoot(resolvedPath, dir)) {
if (this.isPathWithinRoot(fullyResolvedPath, dir)) {
return true;
}
}
return false;
} catch (_error) {
return false;
}
}
/**
* Fully resolves a path, including symbolic links.
* If the path does not exist, it returns the fully resolved path as it would be
* if it did exist.
*/
private fullyResolvedPath(pathToCheck: string): string {
try {
return fs.realpathSync(pathToCheck);
} catch (e: unknown) {
if (
isNodeError(e) &&
e.code === 'ENOENT' &&
e.path &&
// realpathSync does not set e.path correctly for symlinks to
// non-existent files.
!this.isFileSymlink(e.path)
) {
// If it doesn't exist, e.path contains the fully resolved path.
return e.path;
}
throw e;
}
}
/**
* Checks if a path is within a given root directory.
* @param pathToCheck The absolute path to check
@@ -167,4 +179,15 @@ export class WorkspaceContext {
!path.isAbsolute(relative)
);
}
/**
* Checks if a file path is a symbolic link that points to a file.
*/
private isFileSymlink(filePath: string): boolean {
try {
return !fs.readlinkSync(filePath).endsWith('/');
} catch (_error) {
return false;
}
}
}