abbyswag/qwen-code
1
0
Fork 0
mirror of https://github.com/QwenLM/qwen-code.git synced 2025-12-20 16:57:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

centralize file filtering in FileDiscoveryService (#1039)

Browse Source
This commit is contained in:
Anas H. Sulaiman
2025-06-14 10:25:34 -04:00
committed by GitHub
parent e6d5477168
commit 4873fce791
27 changed files with 221 additions and 567 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
packages/cli/src/config/config.ts
View File

@@ -136,7 +136,6 @@ export async function loadHierarchicalGeminiMemory(
export async function loadCliConfig(
settings: Settings,
extensions: Extension[],
geminiIgnorePatterns: string[],
sessionId: string,
): Promise<Config> {
loadEnvironment();
@@ -158,9 +157,6 @@ export async function loadCliConfig(
const extensionContextFilePaths = extensions.flatMap((e) => e.contextFiles);
const fileService = new FileDiscoveryService(process.cwd());
await fileService.initialize({
respectGitIgnore: settings.fileFiltering?.respectGitIgnore,
});
// Call the (now wrapper) loadHierarchicalGeminiMemory which calls the server's version
const { memoryContent, fileCount } = await loadHierarchicalGeminiMemory(
process.cwd(),
@@ -193,7 +189,6 @@ export async function loadCliConfig(
approvalMode: argv.yolo || false ? ApprovalMode.YOLO : ApprovalMode.DEFAULT,
showMemoryUsage:
argv.show_memory_usage || settings.showMemoryUsage || false,
geminiIgnorePatterns,
accessibility: settings.accessibility,
telemetry:
argv.telemetry !== undefined

12
packages/cli/src/gemini.tsx
View File

@@ -15,7 +15,6 @@ import { LoadedSettings, loadSettings } from './config/settings.js';
import { themeManager } from './ui/themes/theme-manager.js';
import { getStartupWarnings } from './utils/startupWarnings.js';
import { runNonInteractive } from './nonInteractiveCli.js';
import { loadGeminiIgnorePatterns } from './utils/loadIgnorePatterns.js';
import { loadExtensions, Extension } from './config/extension.js';
import { cleanupCheckpoints } from './utils/cleanup.js';
import {
@@ -41,7 +40,6 @@ export async function main() {
const settings = loadSettings(workspaceRoot);
setWindowTitle(basename(workspaceRoot), settings);
const geminiIgnorePatterns = await loadGeminiIgnorePatterns(workspaceRoot);
await cleanupCheckpoints();
if (settings.errors.length > 0) {
for (const error of settings.errors) {
@@ -56,15 +54,10 @@ export async function main() {
}
const extensions = loadExtensions(workspaceRoot);
const config = await loadCliConfig(
settings.merged,
extensions,
geminiIgnorePatterns,
sessionId,
);
const config = await loadCliConfig(settings.merged, extensions, sessionId);
// Initialize centralized FileDiscoveryService
await config.getFileService();
config.getFileService();
if (config.getCheckpointEnabled()) {
try {
await config.getGitService();
@@ -199,7 +192,6 @@ async function loadNonInteractiveConfig(
return await loadCliConfig(
nonInteractiveSettings,
extensions,
config.getGeminiIgnorePatterns(),
config.getSessionId(),
);
}

2
packages/cli/src/ui/App.tsx
View File

@@ -139,7 +139,7 @@ const App = ({ config, settings, startupWarnings = [] }: AppProps) => {
const { memoryContent, fileCount } = await loadHierarchicalGeminiMemory(
process.cwd(),
config.getDebugMode(),
await config.getFileService(),
config.getFileService(),
);
config.setUserMemory(memoryContent);
config.setGeminiMdFileCount(fileCount);

22
packages/cli/src/ui/hooks/atCommandProcessor.test.ts
View File

@@ -89,7 +89,7 @@ describe('handleAtCommand', () => {
// Mock FileDiscoveryService
mockFileDiscoveryService = {
initialize: vi.fn(),
shouldIgnoreFile: vi.fn(() => false),
shouldGitIgnoreFile: vi.fn(() => false),
filterFiles: vi.fn((files) => files),
getIgnoreInfo: vi.fn(() => ({ gitIgnored: [] })),
isGitRepository: vi.fn(() => true),
@@ -101,7 +101,7 @@ describe('handleAtCommand', () => {
// Mock getFileService to return the mocked FileDiscoveryService
mockConfig.getFileService = vi
.fn()
.mockResolvedValue(mockFileDiscoveryService);
.mockReturnValue(mockFileDiscoveryService);
});
afterEach(() => {
@@ -581,7 +581,7 @@ describe('handleAtCommand', () => {
const query = `@${gitIgnoredFile}`;
// Mock the file discovery service to report this file as git-ignored
mockFileDiscoveryService.shouldIgnoreFile.mockImplementation(
mockFileDiscoveryService.shouldGitIgnoreFile.mockImplementation(
(path: string) => path === gitIgnoredFile,
);
@@ -594,7 +594,7 @@ describe('handleAtCommand', () => {
signal: abortController.signal,
});
expect(mockFileDiscoveryService.shouldIgnoreFile).toHaveBeenCalledWith(
expect(mockFileDiscoveryService.shouldGitIgnoreFile).toHaveBeenCalledWith(
gitIgnoredFile,
);
expect(mockOnDebugMessage).toHaveBeenCalledWith(
@@ -613,7 +613,7 @@ describe('handleAtCommand', () => {
const query = `@${validFile}`;
const fileContent = 'console.log("Hello world");';
mockFileDiscoveryService.shouldIgnoreFile.mockReturnValue(false);
mockFileDiscoveryService.shouldGitIgnoreFile.mockReturnValue(false);
mockReadManyFilesExecute.mockResolvedValue({
llmContent: [`--- ${validFile} ---\n\n${fileContent}\n\n`],
returnDisplay: 'Read 1 file.',
@@ -628,7 +628,7 @@ describe('handleAtCommand', () => {
signal: abortController.signal,
});
expect(mockFileDiscoveryService.shouldIgnoreFile).toHaveBeenCalledWith(
expect(mockFileDiscoveryService.shouldGitIgnoreFile).toHaveBeenCalledWith(
validFile,
);
expect(mockReadManyFilesExecute).toHaveBeenCalledWith(
@@ -651,7 +651,7 @@ describe('handleAtCommand', () => {
const query = `@${validFile} @${gitIgnoredFile}`;
const fileContent = '# Project README';
mockFileDiscoveryService.shouldIgnoreFile.mockImplementation(
mockFileDiscoveryService.shouldGitIgnoreFile.mockImplementation(
(path: string) => path === gitIgnoredFile,
);
mockReadManyFilesExecute.mockResolvedValue({
@@ -668,10 +668,10 @@ describe('handleAtCommand', () => {
signal: abortController.signal,
});
expect(mockFileDiscoveryService.shouldIgnoreFile).toHaveBeenCalledWith(
expect(mockFileDiscoveryService.shouldGitIgnoreFile).toHaveBeenCalledWith(
validFile,
);
expect(mockFileDiscoveryService.shouldIgnoreFile).toHaveBeenCalledWith(
expect(mockFileDiscoveryService.shouldGitIgnoreFile).toHaveBeenCalledWith(
gitIgnoredFile,
);
expect(mockOnDebugMessage).toHaveBeenCalledWith(
@@ -698,7 +698,7 @@ describe('handleAtCommand', () => {
const gitFile = '.git/config';
const query = `@${gitFile}`;
mockFileDiscoveryService.shouldIgnoreFile.mockReturnValue(true);
mockFileDiscoveryService.shouldGitIgnoreFile.mockReturnValue(true);
const result = await handleAtCommand({
query,
@@ -709,7 +709,7 @@ describe('handleAtCommand', () => {
signal: abortController.signal,
});
expect(mockFileDiscoveryService.shouldIgnoreFile).toHaveBeenCalledWith(
expect(mockFileDiscoveryService.shouldGitIgnoreFile).toHaveBeenCalledWith(
gitFile,
);
expect(mockOnDebugMessage).toHaveBeenCalledWith(

4
packages/cli/src/ui/hooks/atCommandProcessor.ts
View File

@@ -135,7 +135,7 @@ export async function handleAtCommand({
addItem({ type: 'user', text: query }, userMessageTimestamp);
// Get centralized file discovery service
const fileDiscovery = await config.getFileService();
const fileDiscovery = config.getFileService();
const respectGitIgnore = config.getFileFilteringRespectGitIgnore();
const pathSpecsToRead: string[] = [];
@@ -182,7 +182,7 @@ export async function handleAtCommand({
}
// Check if path should be ignored by git
if (fileDiscovery.shouldIgnoreFile(pathName)) {
if (fileDiscovery.shouldGitIgnoreFile(pathName)) {
const reason = respectGitIgnore
? 'git-ignored and will be skipped'
: 'ignored by custom patterns';

29
packages/cli/src/ui/hooks/useCompletion.integration.test.ts
View File

@@ -10,6 +10,7 @@ import { renderHook, act } from '@testing-library/react';
import { useCompletion } from './useCompletion.js';
import * as fs from 'fs/promises';
import { FileDiscoveryService } from '@gemini-cli/core';
import { glob } from 'glob';
// Mock dependencies
vi.mock('fs/promises');
@@ -24,6 +25,7 @@ vi.mock('@gemini-cli/core', async () => {
getErrorMessage: vi.fn((error) => error.message),
};
});
vi.mock('glob');
describe('useCompletion git-aware filtering integration', () => {
let mockFileDiscoveryService: Mocked<FileDiscoveryService>;
@@ -38,16 +40,13 @@ describe('useCompletion git-aware filtering integration', () => {
beforeEach(() => {
mockFileDiscoveryService = {
initialize: vi.fn(),
shouldIgnoreFile: vi.fn(),
shouldGitIgnoreFile: vi.fn(),
filterFiles: vi.fn(),
getIgnoreInfo: vi.fn(() => ({ gitIgnored: [] })),
glob: vi.fn().mockResolvedValue([]),
};
mockConfig = {
getFileFilteringRespectGitIgnore: vi.fn(() => true),
getFileService: vi.fn().mockResolvedValue(mockFileDiscoveryService),
getFileService: vi.fn().mockReturnValue(mockFileDiscoveryService),
};
vi.mocked(FileDiscoveryService).mockImplementation(
@@ -71,7 +70,7 @@ describe('useCompletion git-aware filtering integration', () => {
] as Array<{ name: string; isDirectory: () => boolean }>);
// Mock git ignore service to ignore certain files
mockFileDiscoveryService.shouldIgnoreFile.mockImplementation(
mockFileDiscoveryService.shouldGitIgnoreFile.mockImplementation(
(path: string) =>
path.includes('node_modules') ||
path.includes('dist') ||
@@ -125,7 +124,7 @@ describe('useCompletion git-aware filtering integration', () => {
);
// Mock git ignore service
mockFileDiscoveryService.shouldIgnoreFile.mockImplementation(
mockFileDiscoveryService.shouldGitIgnoreFile.mockImplementation(
(path: string) => path.includes('node_modules') || path.includes('temp'),
);
@@ -173,10 +172,6 @@ describe('useCompletion git-aware filtering integration', () => {
});
it('should handle git discovery service initialization failure gracefully', async () => {
mockFileDiscoveryService.initialize.mockRejectedValue(
new Error('Git not found'),
);
vi.mocked(fs.readdir).mockResolvedValue([
{ name: 'src', isDirectory: () => true },
{ name: 'README.md', isDirectory: () => false },
@@ -208,7 +203,7 @@ describe('useCompletion git-aware filtering integration', () => {
{ name: 'index.ts', isDirectory: () => false },
] as Array<{ name: string; isDirectory: () => boolean }>);
mockFileDiscoveryService.shouldIgnoreFile.mockImplementation(
mockFileDiscoveryService.shouldGitIgnoreFile.mockImplementation(
(path: string) => path.includes('.log'),
);
@@ -228,7 +223,7 @@ describe('useCompletion git-aware filtering integration', () => {
it('should use glob for top-level @ completions when available', async () => {
const globResults = [`${testCwd}/src/index.ts`, `${testCwd}/README.md`];
mockFileDiscoveryService.glob.mockResolvedValue(globResults);
vi.mocked(glob).mockResolvedValue(globResults);
const { result } = renderHook(() =>
useCompletion('@s', testCwd, true, slashCommands, mockConfig),
@@ -238,9 +233,10 @@ describe('useCompletion git-aware filtering integration', () => {
await new Promise((resolve) => setTimeout(resolve, 150));
});
expect(mockFileDiscoveryService.glob).toHaveBeenCalledWith('**/s*', {
expect(glob).toHaveBeenCalledWith('**/s*', {
cwd: testCwd,
dot: false,
nocase: true,
});
expect(fs.readdir).not.toHaveBeenCalled(); // Ensure glob is used instead of readdir
expect(result.current.suggestions).toEqual([
@@ -255,7 +251,7 @@ describe('useCompletion git-aware filtering integration', () => {
`${testCwd}/.gitignore`,
`${testCwd}/src/index.ts`,
];
mockFileDiscoveryService.glob.mockResolvedValue(globResults);
vi.mocked(glob).mockResolvedValue(globResults);
const { result } = renderHook(() =>
useCompletion('@.', testCwd, true, slashCommands, mockConfig),
@@ -265,9 +261,10 @@ describe('useCompletion git-aware filtering integration', () => {
await new Promise((resolve) => setTimeout(resolve, 150));
});
expect(mockFileDiscoveryService.glob).toHaveBeenCalledWith('**/.*', {
expect(glob).toHaveBeenCalledWith('**/.*', {
cwd: testCwd,
dot: true,
nocase: true,
});
expect(fs.readdir).not.toHaveBeenCalled();
expect(result.current.suggestions).toEqual([

14
packages/cli/src/ui/hooks/useCompletion.ts
View File

@@ -7,6 +7,7 @@
import { useState, useEffect, useCallback } from 'react';
import * as fs from 'fs/promises';
import * as path from 'path';
import { glob } from 'glob';
import {
isNodeError,
escapePath,
@@ -187,7 +188,7 @@ export function useCompletion(
const findFilesRecursively = async (
startDir: string,
searchPrefix: string,
fileDiscovery: { shouldIgnoreFile: (path: string) => boolean } | null,
fileDiscovery: { shouldGitIgnoreFile: (path: string) => boolean } | null,
currentRelativePath = '',
depth = 0,
maxDepth = 10, // Limit recursion depth
@@ -218,7 +219,7 @@ export function useCompletion(
// Check if this entry should be ignored by git-aware filtering
if (
fileDiscovery &&
fileDiscovery.shouldIgnoreFile(entryPathFromRoot)
fileDiscovery.shouldGitIgnoreFile(entryPathFromRoot)
) {
continue;
}
@@ -263,9 +264,10 @@ export function useCompletion(
maxResults = 50,
): Promise<Suggestion[]> => {
const globPattern = `**/${searchPrefix}*`;
const files = await fileDiscoveryService.glob(globPattern, {
const files = await glob(globPattern, {
cwd,
dot: searchPrefix.startsWith('.'),
nocase: true,
});
const suggestions: Suggestion[] = files
@@ -285,9 +287,7 @@ export function useCompletion(
setIsLoadingSuggestions(true);
let fetchedSuggestions: Suggestion[] = [];
const fileDiscoveryService = config
? await config.getFileService()
: null;
const fileDiscoveryService = config ? config.getFileService() : null;
try {
// If there's no slash, or it's the root, do a recursive search from cwd
@@ -326,7 +326,7 @@ export function useCompletion(
);
if (
fileDiscoveryService &&
fileDiscoveryService.shouldIgnoreFile(relativePath)
fileDiscoveryService.shouldGitIgnoreFile(relativePath)
) {
continue;
}

150
packages/cli/src/utils/loadIgnorePatterns.test.ts
View File

@@ -1,150 +0,0 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import {
vi,
describe,
it,
expect,
beforeEach,
afterEach,
Mock,
beforeAll,
} from 'vitest';
import * as path from 'node:path';
import { loadGeminiIgnorePatterns } from './loadIgnorePatterns.js';
import os from 'node:os';
// Define the type for our mock function explicitly.
type ReadFileSyncMockType = Mock<
(path: string, encoding: string) => string | Buffer
>;
// Declare a variable to hold our mock function instance.
let mockedFsReadFileSync: ReadFileSyncMockType;
vi.mock('node:fs', async () => {
const actualFsModule =
await vi.importActual<typeof import('node:fs')>('node:fs');
return {
...actualFsModule,
readFileSync: vi.fn(), // The factory creates and returns the vi.fn() instance.
};
});
let actualFs: typeof import('node:fs');
describe('loadGeminiIgnorePatterns', () => {
let tempDir: string;
let consoleLogSpy: Mock<
(message?: unknown, ...optionalParams: unknown[]) => void
>;
beforeAll(async () => {
actualFs = await vi.importActual<typeof import('node:fs')>('node:fs');
const mockedFsModule = await import('node:fs');
mockedFsReadFileSync =
mockedFsModule.readFileSync as unknown as ReadFileSyncMockType;
});
beforeEach(() => {
tempDir = actualFs.mkdtempSync(
path.join(os.tmpdir(), 'gemini-ignore-test-'),
);
consoleLogSpy = vi
.spyOn(console, 'log')
.mockImplementation(() => {}) as Mock<
(message?: unknown, ...optionalParams: unknown[]) => void
>;
mockedFsReadFileSync.mockReset();
});
afterEach(() => {
if (actualFs.existsSync(tempDir)) {
actualFs.rmSync(tempDir, { recursive: true, force: true });
}
vi.restoreAllMocks();
});
it('should load and parse patterns from .geminiignore, ignoring comments and empty lines', async () => {
const ignoreContent = [
'# This is a comment',
'pattern1',
' pattern2 ', // Should be trimmed
'', // Empty line
'pattern3 # Inline comment', // Handled by trim
'*.log',
'!important.file',
].join('\n');
const ignoreFilePath = path.join(tempDir, '.geminiignore');
actualFs.writeFileSync(ignoreFilePath, ignoreContent);
const patterns = await loadGeminiIgnorePatterns(tempDir);
expect(patterns).toEqual([
'pattern1',
'pattern2',
'pattern3 # Inline comment',
'*.log',
'!important.file',
]);
expect(consoleLogSpy).toHaveBeenCalledWith(
expect.stringContaining('Loaded 5 patterns from .geminiignore'),
);
});
it('should return an empty array and log info if .geminiignore is not found', async () => {
const patterns = await loadGeminiIgnorePatterns(tempDir);
expect(patterns).toEqual([]);
expect(consoleLogSpy).not.toHaveBeenCalled();
});
it('should return an empty array if .geminiignore is empty', async () => {
const ignoreFilePath = path.join(tempDir, '.geminiignore');
actualFs.writeFileSync(ignoreFilePath, '');
const patterns = await loadGeminiIgnorePatterns(tempDir);
expect(patterns).toEqual([]);
expect(consoleLogSpy).not.toHaveBeenCalledWith(
expect.stringContaining('Loaded 0 patterns from .geminiignore'),
);
expect(consoleLogSpy).not.toHaveBeenCalledWith(
expect.stringContaining('No .geminiignore file found'),
);
});
it('should return an empty array if .geminiignore contains only comments and empty lines', async () => {
const ignoreContent = [
'# Comment 1',
' # Comment 2 with leading spaces',
'',
' ', // Whitespace only line
].join('\n');
const ignoreFilePath = path.join(tempDir, '.geminiignore');
actualFs.writeFileSync(ignoreFilePath, ignoreContent);
const patterns = await loadGeminiIgnorePatterns(tempDir);
expect(patterns).toEqual([]);
expect(consoleLogSpy).not.toHaveBeenCalledWith(
expect.stringContaining('Loaded 0 patterns from .geminiignore'),
);
expect(consoleLogSpy).not.toHaveBeenCalledWith(
expect.stringContaining('No .geminiignore file found'),
);
});
it('should correctly handle patterns with inline comments if not starting with #', async () => {
const ignoreContent = 'src/important # but not this part';
const ignoreFilePath = path.join(tempDir, '.geminiignore');
actualFs.writeFileSync(ignoreFilePath, ignoreContent);
const patterns = await loadGeminiIgnorePatterns(tempDir);
expect(patterns).toEqual(['src/important # but not this part']);
expect(consoleLogSpy).toHaveBeenCalledWith(
expect.stringContaining('Loaded 1 patterns from .geminiignore'),
);
});
});

56
packages/cli/src/utils/loadIgnorePatterns.ts
View File

@@ -1,56 +0,0 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import * as path from 'node:path';
import { GitIgnoreParser } from '@gemini-cli/core';
const GEMINI_IGNORE_FILE_NAME = '.geminiignore';
/**
* Loads and parses a .geminiignore file from the given workspace root.
* The .geminiignore file follows a format similar to .gitignore.
*
* @param workspaceRoot The absolute path to the workspace root where the .geminiignore file is expected.
* @returns An array of glob patterns extracted from the .geminiignore file. Returns an empty array
* if the file does not exist or contains no valid patterns.
*/
export async function loadGeminiIgnorePatterns(
workspaceRoot: string,
): Promise<string[]> {
const parser = new GitIgnoreParser(workspaceRoot);
try {
await parser.loadPatterns(GEMINI_IGNORE_FILE_NAME);
} catch (error: unknown) {
const ignoreFilePath = path.join(workspaceRoot, GEMINI_IGNORE_FILE_NAME);
if (
error instanceof Error &&
'code' in error &&
typeof error.code === 'string'
) {
if (error.code === 'ENOENT') {
// .geminiignore not found, which is fine.
} else {
// Other error reading the file (e.g., permissions)
console.warn(
`[WARN] Could not read .geminiignore file at ${ignoreFilePath}: ${error.message}`,
);
}
} else {
// For other types of errors, or if code is not available
console.warn(
`[WARN] An unexpected error occurred while trying to read ${ignoreFilePath}: ${String(error)}`,
);
}
}
const loadedPatterns = parser.getPatterns();
if (loadedPatterns.length > 0) {
console.log(
`[INFO] Loaded ${loadedPatterns.length} patterns from .geminiignore`,
);
}
return loadedPatterns;
}