mirror of
https://github.com/QwenLM/qwen-code.git
synced 2025-12-19 09:33:53 +00:00
Merge branch 'main' into feature/stream-json-migration
This commit is contained in:
mingholy.lmh
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@qwen-code/qwen-code-core",
|
||||
"version": "0.1.5",
|
||||
"version": "0.2.1",
|
||||
"description": "Qwen Code Core",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
|
||||
@@ -738,13 +738,13 @@ describe('Server Config (config.ts)', () => {
|
||||
|
||||
it('should return the calculated threshold when it is smaller than the default', () => {
|
||||
const config = new Config(baseParams);
|
||||
vi.mocked(tokenLimit).mockReturnValue(32000);
|
||||
vi.mocked(tokenLimit).mockReturnValue(8000);
|
||||
vi.mocked(uiTelemetryService.getLastPromptTokenCount).mockReturnValue(
|
||||
1000,
|
||||
2000,
|
||||
);
|
||||
// 4 * (32000 - 1000) = 4 * 31000 = 124000
|
||||
// default is 4_000_000
|
||||
expect(config.getTruncateToolOutputThreshold()).toBe(124000);
|
||||
// 4 * (8000 - 2000) = 4 * 6000 = 24000
|
||||
// default is 25_000
|
||||
expect(config.getTruncateToolOutputThreshold()).toBe(24000);
|
||||
});
|
||||
|
||||
it('should return the default threshold when the calculated value is larger', () => {
|
||||
@@ -754,8 +754,8 @@ describe('Server Config (config.ts)', () => {
|
||||
500_000,
|
||||
);
|
||||
// 4 * (2_000_000 - 500_000) = 4 * 1_500_000 = 6_000_000
|
||||
// default is 4_000_000
|
||||
expect(config.getTruncateToolOutputThreshold()).toBe(4_000_000);
|
||||
// default is 25_000
|
||||
expect(config.getTruncateToolOutputThreshold()).toBe(25_000);
|
||||
});
|
||||
|
||||
it('should use a custom truncateToolOutputThreshold if provided', () => {
|
||||
|
||||
@@ -161,7 +161,7 @@ export interface ExtensionInstallMetadata {
|
||||
autoUpdate?: boolean;
|
||||
}
|
||||
|
||||
export const DEFAULT_TRUNCATE_TOOL_OUTPUT_THRESHOLD = 4_000_000;
|
||||
export const DEFAULT_TRUNCATE_TOOL_OUTPUT_THRESHOLD = 25_000;
|
||||
export const DEFAULT_TRUNCATE_TOOL_OUTPUT_LINES = 1000;
|
||||
|
||||
export class MCPServerConfig {
|
||||
@@ -289,6 +289,7 @@ export interface ConfigParameters {
|
||||
eventEmitter?: EventEmitter;
|
||||
useSmartEdit?: boolean;
|
||||
output?: OutputSettings;
|
||||
skipStartupContext?: boolean;
|
||||
inputFormat?: InputFormat;
|
||||
outputFormat?: OutputFormat;
|
||||
}
|
||||
@@ -402,6 +403,7 @@ export class Config {
|
||||
private readonly extensionManagement: boolean = true;
|
||||
private readonly enablePromptCompletion: boolean = false;
|
||||
private readonly skipLoopDetection: boolean;
|
||||
private readonly skipStartupContext: boolean;
|
||||
private readonly vlmSwitchMode: string | undefined;
|
||||
private initialized: boolean = false;
|
||||
readonly storage: Storage;
|
||||
@@ -499,6 +501,7 @@ export class Config {
|
||||
this.interactive = params.interactive ?? false;
|
||||
this.trustedFolder = params.trustedFolder;
|
||||
this.skipLoopDetection = params.skipLoopDetection ?? false;
|
||||
this.skipStartupContext = params.skipStartupContext ?? false;
|
||||
|
||||
// Web search
|
||||
this.webSearch = params.webSearch;
|
||||
@@ -1076,6 +1079,10 @@ export class Config {
|
||||
return this.skipLoopDetection;
|
||||
}
|
||||
|
||||
getSkipStartupContext(): boolean {
|
||||
return this.skipStartupContext;
|
||||
}
|
||||
|
||||
getVlmSwitchMode(): string | undefined {
|
||||
return this.vlmSwitchMode;
|
||||
}
|
||||
@@ -1085,6 +1092,13 @@ export class Config {
|
||||
}
|
||||
|
||||
getTruncateToolOutputThreshold(): number {
|
||||
if (
|
||||
!this.enableToolOutputTruncation ||
|
||||
this.truncateToolOutputThreshold <= 0
|
||||
) {
|
||||
return Number.POSITIVE_INFINITY;
|
||||
}
|
||||
|
||||
return Math.min(
|
||||
// Estimate remaining context window in characters (1 token ~= 4 chars).
|
||||
4 *
|
||||
@@ -1095,6 +1109,10 @@ export class Config {
|
||||
}
|
||||
|
||||
getTruncateToolOutputLines(): number {
|
||||
if (!this.enableToolOutputTruncation || this.truncateToolOutputLines <= 0) {
|
||||
return Number.POSITIVE_INFINITY;
|
||||
}
|
||||
|
||||
return this.truncateToolOutputLines;
|
||||
}
|
||||
|
||||
|
||||
@@ -1789,6 +1789,268 @@ describe('CoreToolScheduler request queueing', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('CoreToolScheduler Sequential Execution', () => {
|
||||
it('should execute tool calls in a batch sequentially', async () => {
|
||||
// Arrange
|
||||
let firstCallFinished = false;
|
||||
const executeFn = vi
|
||||
.fn()
|
||||
.mockImplementation(async (args: { call: number }) => {
|
||||
if (args.call === 1) {
|
||||
// First call, wait for a bit to simulate work
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
firstCallFinished = true;
|
||||
return { llmContent: 'First call done' };
|
||||
}
|
||||
if (args.call === 2) {
|
||||
// Second call, should only happen after the first is finished
|
||||
if (!firstCallFinished) {
|
||||
throw new Error(
|
||||
'Second tool call started before the first one finished!',
|
||||
);
|
||||
}
|
||||
return { llmContent: 'Second call done' };
|
||||
}
|
||||
return { llmContent: 'default' };
|
||||
});
|
||||
|
||||
const mockTool = new MockTool({ name: 'mockTool', execute: executeFn });
|
||||
const declarativeTool = mockTool;
|
||||
|
||||
const mockToolRegistry = {
|
||||
getTool: () => declarativeTool,
|
||||
getToolByName: () => declarativeTool,
|
||||
getFunctionDeclarations: () => [],
|
||||
tools: new Map(),
|
||||
discovery: {},
|
||||
registerTool: () => {},
|
||||
getToolByDisplayName: () => declarativeTool,
|
||||
getTools: () => [],
|
||||
discoverTools: async () => {},
|
||||
getAllTools: () => [],
|
||||
getToolsByServer: () => [],
|
||||
} as unknown as ToolRegistry;
|
||||
|
||||
const onAllToolCallsComplete = vi.fn();
|
||||
const onToolCallsUpdate = vi.fn();
|
||||
|
||||
const mockConfig = {
|
||||
getSessionId: () => 'test-session-id',
|
||||
getUsageStatisticsEnabled: () => true,
|
||||
getDebugMode: () => false,
|
||||
getApprovalMode: () => ApprovalMode.YOLO, // Use YOLO to avoid confirmation prompts
|
||||
getAllowedTools: () => [],
|
||||
getContentGeneratorConfig: () => ({
|
||||
model: 'test-model',
|
||||
authType: 'oauth-personal',
|
||||
}),
|
||||
getShellExecutionConfig: () => ({
|
||||
terminalWidth: 90,
|
||||
terminalHeight: 30,
|
||||
}),
|
||||
storage: {
|
||||
getProjectTempDir: () => '/tmp',
|
||||
},
|
||||
getToolRegistry: () => mockToolRegistry,
|
||||
getTruncateToolOutputThreshold: () =>
|
||||
DEFAULT_TRUNCATE_TOOL_OUTPUT_THRESHOLD,
|
||||
getTruncateToolOutputLines: () => DEFAULT_TRUNCATE_TOOL_OUTPUT_LINES,
|
||||
getUseSmartEdit: () => false,
|
||||
getUseModelRouter: () => false,
|
||||
getGeminiClient: () => null,
|
||||
} as unknown as Config;
|
||||
|
||||
const scheduler = new CoreToolScheduler({
|
||||
config: mockConfig,
|
||||
onAllToolCallsComplete,
|
||||
onToolCallsUpdate,
|
||||
getPreferredEditor: () => 'vscode',
|
||||
onEditorClose: vi.fn(),
|
||||
});
|
||||
|
||||
const abortController = new AbortController();
|
||||
const requests = [
|
||||
{
|
||||
callId: '1',
|
||||
name: 'mockTool',
|
||||
args: { call: 1 },
|
||||
isClientInitiated: false,
|
||||
prompt_id: 'prompt-1',
|
||||
},
|
||||
{
|
||||
callId: '2',
|
||||
name: 'mockTool',
|
||||
args: { call: 2 },
|
||||
isClientInitiated: false,
|
||||
prompt_id: 'prompt-1',
|
||||
},
|
||||
];
|
||||
|
||||
// Act
|
||||
await scheduler.schedule(requests, abortController.signal);
|
||||
|
||||
// Assert
|
||||
await vi.waitFor(() => {
|
||||
expect(onAllToolCallsComplete).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// Check that execute was called twice
|
||||
expect(executeFn).toHaveBeenCalledTimes(2);
|
||||
|
||||
// Check the order of calls
|
||||
const calls = executeFn.mock.calls;
|
||||
expect(calls[0][0]).toEqual({ call: 1 });
|
||||
expect(calls[1][0]).toEqual({ call: 2 });
|
||||
|
||||
// The onAllToolCallsComplete should be called once with both results
|
||||
const completedCalls = onAllToolCallsComplete.mock
|
||||
.calls[0][0] as ToolCall[];
|
||||
expect(completedCalls).toHaveLength(2);
|
||||
expect(completedCalls[0].status).toBe('success');
|
||||
expect(completedCalls[1].status).toBe('success');
|
||||
});
|
||||
|
||||
it('should cancel subsequent tools when the signal is aborted.', async () => {
|
||||
// Arrange
|
||||
const abortController = new AbortController();
|
||||
let secondCallStarted = false;
|
||||
|
||||
const executeFn = vi
|
||||
.fn()
|
||||
.mockImplementation(async (args: { call: number }) => {
|
||||
if (args.call === 1) {
|
||||
return { llmContent: 'First call done' };
|
||||
}
|
||||
if (args.call === 2) {
|
||||
secondCallStarted = true;
|
||||
// This call will be cancelled while it's "running".
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
// It should not return a value because it will be cancelled.
|
||||
return { llmContent: 'Second call should not complete' };
|
||||
}
|
||||
if (args.call === 3) {
|
||||
return { llmContent: 'Third call done' };
|
||||
}
|
||||
return { llmContent: 'default' };
|
||||
});
|
||||
|
||||
const mockTool = new MockTool({ name: 'mockTool', execute: executeFn });
|
||||
const declarativeTool = mockTool;
|
||||
|
||||
const mockToolRegistry = {
|
||||
getTool: () => declarativeTool,
|
||||
getToolByName: () => declarativeTool,
|
||||
getFunctionDeclarations: () => [],
|
||||
tools: new Map(),
|
||||
discovery: {},
|
||||
registerTool: () => {},
|
||||
getToolByDisplayName: () => declarativeTool,
|
||||
getTools: () => [],
|
||||
discoverTools: async () => {},
|
||||
getAllTools: () => [],
|
||||
getToolsByServer: () => [],
|
||||
} as unknown as ToolRegistry;
|
||||
|
||||
const onAllToolCallsComplete = vi.fn();
|
||||
const onToolCallsUpdate = vi.fn();
|
||||
|
||||
const mockConfig = {
|
||||
getSessionId: () => 'test-session-id',
|
||||
getUsageStatisticsEnabled: () => true,
|
||||
getDebugMode: () => false,
|
||||
getApprovalMode: () => ApprovalMode.YOLO,
|
||||
getAllowedTools: () => [],
|
||||
getContentGeneratorConfig: () => ({
|
||||
model: 'test-model',
|
||||
authType: 'oauth-personal',
|
||||
}),
|
||||
getShellExecutionConfig: () => ({
|
||||
terminalWidth: 90,
|
||||
terminalHeight: 30,
|
||||
}),
|
||||
storage: {
|
||||
getProjectTempDir: () => '/tmp',
|
||||
},
|
||||
getToolRegistry: () => mockToolRegistry,
|
||||
getTruncateToolOutputThreshold: () =>
|
||||
DEFAULT_TRUNCATE_TOOL_OUTPUT_THRESHOLD,
|
||||
getTruncateToolOutputLines: () => DEFAULT_TRUNCATE_TOOL_OUTPUT_LINES,
|
||||
getUseSmartEdit: () => false,
|
||||
getUseModelRouter: () => false,
|
||||
getGeminiClient: () => null,
|
||||
} as unknown as Config;
|
||||
|
||||
const scheduler = new CoreToolScheduler({
|
||||
config: mockConfig,
|
||||
onAllToolCallsComplete,
|
||||
onToolCallsUpdate,
|
||||
getPreferredEditor: () => 'vscode',
|
||||
onEditorClose: vi.fn(),
|
||||
});
|
||||
|
||||
const requests = [
|
||||
{
|
||||
callId: '1',
|
||||
name: 'mockTool',
|
||||
args: { call: 1 },
|
||||
isClientInitiated: false,
|
||||
prompt_id: 'prompt-1',
|
||||
},
|
||||
{
|
||||
callId: '2',
|
||||
name: 'mockTool',
|
||||
args: { call: 2 },
|
||||
isClientInitiated: false,
|
||||
prompt_id: 'prompt-1',
|
||||
},
|
||||
{
|
||||
callId: '3',
|
||||
name: 'mockTool',
|
||||
args: { call: 3 },
|
||||
isClientInitiated: false,
|
||||
prompt_id: 'prompt-1',
|
||||
},
|
||||
];
|
||||
|
||||
// Act
|
||||
const schedulePromise = scheduler.schedule(
|
||||
requests,
|
||||
abortController.signal,
|
||||
);
|
||||
|
||||
// Wait for the second call to start, then abort.
|
||||
await vi.waitFor(() => {
|
||||
expect(secondCallStarted).toBe(true);
|
||||
});
|
||||
abortController.abort();
|
||||
|
||||
await schedulePromise;
|
||||
|
||||
// Assert
|
||||
await vi.waitFor(() => {
|
||||
expect(onAllToolCallsComplete).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// Check that execute was called for all three tools initially
|
||||
expect(executeFn).toHaveBeenCalledTimes(3);
|
||||
expect(executeFn).toHaveBeenCalledWith({ call: 1 });
|
||||
expect(executeFn).toHaveBeenCalledWith({ call: 2 });
|
||||
expect(executeFn).toHaveBeenCalledWith({ call: 3 });
|
||||
|
||||
const completedCalls = onAllToolCallsComplete.mock
|
||||
.calls[0][0] as ToolCall[];
|
||||
expect(completedCalls).toHaveLength(3);
|
||||
|
||||
const call1 = completedCalls.find((c) => c.request.callId === '1');
|
||||
const call2 = completedCalls.find((c) => c.request.callId === '2');
|
||||
const call3 = completedCalls.find((c) => c.request.callId === '3');
|
||||
|
||||
expect(call1?.status).toBe('success');
|
||||
expect(call2?.status).toBe('cancelled');
|
||||
expect(call3?.status).toBe('cancelled');
|
||||
});
|
||||
});
|
||||
|
||||
describe('truncateAndSaveToFile', () => {
|
||||
const mockWriteFile = vi.mocked(fs.writeFile);
|
||||
const THRESHOLD = 40_000;
|
||||
@@ -1968,14 +2230,14 @@ describe('truncateAndSaveToFile', () => {
|
||||
);
|
||||
|
||||
expect(result.content).toContain(
|
||||
'read_file tool with the absolute file path above',
|
||||
'Tool output was too large and has been truncated',
|
||||
);
|
||||
expect(result.content).toContain('read_file tool with offset=0, limit=100');
|
||||
expect(result.content).toContain('The full output has been saved to:');
|
||||
expect(result.content).toContain(
|
||||
'read_file tool with offset=N to skip N lines',
|
||||
'To read the complete output, use the read_file tool with the absolute file path above',
|
||||
);
|
||||
expect(result.content).toContain(
|
||||
'read_file tool with limit=M to read only M lines',
|
||||
'The truncated output below shows the beginning and end of the content',
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
@@ -299,10 +299,7 @@ export async function truncateAndSaveToFile(
|
||||
return {
|
||||
content: `Tool output was too large and has been truncated.
|
||||
The full output has been saved to: ${outputFile}
|
||||
To read the complete output, use the ${ReadFileTool.Name} tool with the absolute file path above. For large files, you can use the offset and limit parameters to read specific sections:
|
||||
- ${ReadFileTool.Name} tool with offset=0, limit=100 to see the first 100 lines
|
||||
- ${ReadFileTool.Name} tool with offset=N to skip N lines from the beginning
|
||||
- ${ReadFileTool.Name} tool with limit=M to read only M lines at a time
|
||||
To read the complete output, use the ${ReadFileTool.Name} tool with the absolute file path above.
|
||||
The truncated output below shows the beginning and end of the content. The marker '... [CONTENT TRUNCATED] ...' indicates where content was removed.
|
||||
This allows you to efficiently examine different parts of the output without loading the entire file.
|
||||
Truncated part of the output:
|
||||
@@ -903,7 +900,7 @@ export class CoreToolScheduler {
|
||||
);
|
||||
}
|
||||
}
|
||||
this.attemptExecutionOfScheduledCalls(signal);
|
||||
await this.attemptExecutionOfScheduledCalls(signal);
|
||||
void this.checkAndNotifyCompletion();
|
||||
} finally {
|
||||
this.isScheduling = false;
|
||||
@@ -978,7 +975,7 @@ export class CoreToolScheduler {
|
||||
}
|
||||
this.setStatusInternal(callId, 'scheduled');
|
||||
}
|
||||
this.attemptExecutionOfScheduledCalls(signal);
|
||||
await this.attemptExecutionOfScheduledCalls(signal);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1024,7 +1021,9 @@ export class CoreToolScheduler {
|
||||
});
|
||||
}
|
||||
|
||||
private attemptExecutionOfScheduledCalls(signal: AbortSignal): void {
|
||||
private async attemptExecutionOfScheduledCalls(
|
||||
signal: AbortSignal,
|
||||
): Promise<void> {
|
||||
const allCallsFinalOrScheduled = this.toolCalls.every(
|
||||
(call) =>
|
||||
call.status === 'scheduled' ||
|
||||
@@ -1038,8 +1037,8 @@ export class CoreToolScheduler {
|
||||
(call) => call.status === 'scheduled',
|
||||
);
|
||||
|
||||
callsToExecute.forEach((toolCall) => {
|
||||
if (toolCall.status !== 'scheduled') return;
|
||||
for (const toolCall of callsToExecute) {
|
||||
if (toolCall.status !== 'scheduled') continue;
|
||||
|
||||
const scheduledCall = toolCall;
|
||||
const { callId, name: toolName } = scheduledCall.request;
|
||||
@@ -1090,107 +1089,106 @@ export class CoreToolScheduler {
|
||||
);
|
||||
}
|
||||
|
||||
promise
|
||||
.then(async (toolResult: ToolResult) => {
|
||||
if (signal.aborted) {
|
||||
this.setStatusInternal(
|
||||
callId,
|
||||
'cancelled',
|
||||
'User cancelled tool execution.',
|
||||
);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const toolResult: ToolResult = await promise;
|
||||
if (signal.aborted) {
|
||||
this.setStatusInternal(
|
||||
callId,
|
||||
'cancelled',
|
||||
'User cancelled tool execution.',
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (toolResult.error === undefined) {
|
||||
let content = toolResult.llmContent;
|
||||
let outputFile: string | undefined = undefined;
|
||||
const contentLength =
|
||||
typeof content === 'string' ? content.length : undefined;
|
||||
if (
|
||||
typeof content === 'string' &&
|
||||
toolName === ShellTool.Name &&
|
||||
this.config.getEnableToolOutputTruncation() &&
|
||||
this.config.getTruncateToolOutputThreshold() > 0 &&
|
||||
this.config.getTruncateToolOutputLines() > 0
|
||||
) {
|
||||
const originalContentLength = content.length;
|
||||
const threshold = this.config.getTruncateToolOutputThreshold();
|
||||
const lines = this.config.getTruncateToolOutputLines();
|
||||
const truncatedResult = await truncateAndSaveToFile(
|
||||
content,
|
||||
callId,
|
||||
this.config.storage.getProjectTempDir(),
|
||||
threshold,
|
||||
lines,
|
||||
);
|
||||
content = truncatedResult.content;
|
||||
outputFile = truncatedResult.outputFile;
|
||||
|
||||
if (outputFile) {
|
||||
logToolOutputTruncated(
|
||||
this.config,
|
||||
new ToolOutputTruncatedEvent(
|
||||
scheduledCall.request.prompt_id,
|
||||
{
|
||||
toolName,
|
||||
originalContentLength,
|
||||
truncatedContentLength: content.length,
|
||||
threshold,
|
||||
lines,
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const response = convertToFunctionResponse(
|
||||
toolName,
|
||||
callId,
|
||||
if (toolResult.error === undefined) {
|
||||
let content = toolResult.llmContent;
|
||||
let outputFile: string | undefined = undefined;
|
||||
const contentLength =
|
||||
typeof content === 'string' ? content.length : undefined;
|
||||
if (
|
||||
typeof content === 'string' &&
|
||||
toolName === ShellTool.Name &&
|
||||
this.config.getEnableToolOutputTruncation() &&
|
||||
this.config.getTruncateToolOutputThreshold() > 0 &&
|
||||
this.config.getTruncateToolOutputLines() > 0
|
||||
) {
|
||||
const originalContentLength = content.length;
|
||||
const threshold = this.config.getTruncateToolOutputThreshold();
|
||||
const lines = this.config.getTruncateToolOutputLines();
|
||||
const truncatedResult = await truncateAndSaveToFile(
|
||||
content,
|
||||
);
|
||||
const successResponse: ToolCallResponseInfo = {
|
||||
callId,
|
||||
responseParts: response,
|
||||
resultDisplay: toolResult.returnDisplay,
|
||||
error: undefined,
|
||||
errorType: undefined,
|
||||
outputFile,
|
||||
contentLength,
|
||||
};
|
||||
this.setStatusInternal(callId, 'success', successResponse);
|
||||
} else {
|
||||
// It is a failure
|
||||
const error = new Error(toolResult.error.message);
|
||||
const errorResponse = createErrorResponse(
|
||||
this.config.storage.getProjectTempDir(),
|
||||
threshold,
|
||||
lines,
|
||||
);
|
||||
content = truncatedResult.content;
|
||||
outputFile = truncatedResult.outputFile;
|
||||
|
||||
if (outputFile) {
|
||||
logToolOutputTruncated(
|
||||
this.config,
|
||||
new ToolOutputTruncatedEvent(
|
||||
scheduledCall.request.prompt_id,
|
||||
{
|
||||
toolName,
|
||||
originalContentLength,
|
||||
truncatedContentLength: content.length,
|
||||
threshold,
|
||||
lines,
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const response = convertToFunctionResponse(
|
||||
toolName,
|
||||
callId,
|
||||
content,
|
||||
);
|
||||
const successResponse: ToolCallResponseInfo = {
|
||||
callId,
|
||||
responseParts: response,
|
||||
resultDisplay: toolResult.returnDisplay,
|
||||
error: undefined,
|
||||
errorType: undefined,
|
||||
outputFile,
|
||||
contentLength,
|
||||
};
|
||||
this.setStatusInternal(callId, 'success', successResponse);
|
||||
} else {
|
||||
// It is a failure
|
||||
const error = new Error(toolResult.error.message);
|
||||
const errorResponse = createErrorResponse(
|
||||
scheduledCall.request,
|
||||
error,
|
||||
toolResult.error.type,
|
||||
);
|
||||
this.setStatusInternal(callId, 'error', errorResponse);
|
||||
}
|
||||
} catch (executionError: unknown) {
|
||||
if (signal.aborted) {
|
||||
this.setStatusInternal(
|
||||
callId,
|
||||
'cancelled',
|
||||
'User cancelled tool execution.',
|
||||
);
|
||||
} else {
|
||||
this.setStatusInternal(
|
||||
callId,
|
||||
'error',
|
||||
createErrorResponse(
|
||||
scheduledCall.request,
|
||||
error,
|
||||
toolResult.error.type,
|
||||
);
|
||||
this.setStatusInternal(callId, 'error', errorResponse);
|
||||
}
|
||||
})
|
||||
.catch((executionError: Error) => {
|
||||
if (signal.aborted) {
|
||||
this.setStatusInternal(
|
||||
callId,
|
||||
'cancelled',
|
||||
'User cancelled tool execution.',
|
||||
);
|
||||
} else {
|
||||
this.setStatusInternal(
|
||||
callId,
|
||||
'error',
|
||||
createErrorResponse(
|
||||
scheduledCall.request,
|
||||
executionError instanceof Error
|
||||
? executionError
|
||||
: new Error(String(executionError)),
|
||||
ToolErrorType.UNHANDLED_EXCEPTION,
|
||||
),
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
executionError instanceof Error
|
||||
? executionError
|
||||
: new Error(String(executionError)),
|
||||
ToolErrorType.UNHANDLED_EXCEPTION,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -23,8 +23,6 @@ import { setSimulate429 } from '../utils/testUtils.js';
|
||||
import { DEFAULT_GEMINI_FLASH_MODEL } from '../config/models.js';
|
||||
import { AuthType } from './contentGenerator.js';
|
||||
import { type RetryOptions } from '../utils/retry.js';
|
||||
import type { ToolRegistry } from '../tools/tool-registry.js';
|
||||
import { Kind } from '../tools/tools.js';
|
||||
import { uiTelemetryService } from '../telemetry/uiTelemetry.js';
|
||||
|
||||
// Mock fs module to prevent actual file system operations during tests
|
||||
@@ -1305,259 +1303,6 @@ describe('GeminiChat', () => {
|
||||
expect(turn4.parts[0].text).toBe('second response');
|
||||
});
|
||||
|
||||
describe('stopBeforeSecondMutator', () => {
|
||||
beforeEach(() => {
|
||||
// Common setup for these tests: mock the tool registry.
|
||||
const mockToolRegistry = {
|
||||
getTool: vi.fn((toolName: string) => {
|
||||
if (toolName === 'edit') {
|
||||
return { kind: Kind.Edit };
|
||||
}
|
||||
return { kind: Kind.Other };
|
||||
}),
|
||||
} as unknown as ToolRegistry;
|
||||
vi.mocked(mockConfig.getToolRegistry).mockReturnValue(mockToolRegistry);
|
||||
});
|
||||
|
||||
it('should stop streaming before a second mutator tool call', async () => {
|
||||
const responses = [
|
||||
{
|
||||
candidates: [
|
||||
{ content: { role: 'model', parts: [{ text: 'First part. ' }] } },
|
||||
],
|
||||
},
|
||||
{
|
||||
candidates: [
|
||||
{
|
||||
content: {
|
||||
role: 'model',
|
||||
parts: [{ functionCall: { name: 'edit', args: {} } }],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
candidates: [
|
||||
{
|
||||
content: {
|
||||
role: 'model',
|
||||
parts: [{ functionCall: { name: 'fetch', args: {} } }],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
// This chunk contains the second mutator and should be clipped.
|
||||
{
|
||||
candidates: [
|
||||
{
|
||||
content: {
|
||||
role: 'model',
|
||||
parts: [
|
||||
{ functionCall: { name: 'edit', args: {} } },
|
||||
{ text: 'some trailing text' },
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
// This chunk should never be reached.
|
||||
{
|
||||
candidates: [
|
||||
{
|
||||
content: {
|
||||
role: 'model',
|
||||
parts: [{ text: 'This should not appear.' }],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
] as unknown as GenerateContentResponse[];
|
||||
|
||||
vi.mocked(mockContentGenerator.generateContentStream).mockResolvedValue(
|
||||
(async function* () {
|
||||
for (const response of responses) {
|
||||
yield response;
|
||||
}
|
||||
})(),
|
||||
);
|
||||
|
||||
const stream = await chat.sendMessageStream(
|
||||
'test-model',
|
||||
{ message: 'test message' },
|
||||
'prompt-id-mutator-test',
|
||||
);
|
||||
for await (const _ of stream) {
|
||||
// Consume the stream to trigger history recording.
|
||||
}
|
||||
|
||||
const history = chat.getHistory();
|
||||
expect(history.length).toBe(2);
|
||||
|
||||
const modelTurn = history[1]!;
|
||||
expect(modelTurn.role).toBe('model');
|
||||
expect(modelTurn?.parts?.length).toBe(3);
|
||||
expect(modelTurn?.parts![0]!.text).toBe('First part. ');
|
||||
expect(modelTurn.parts![1]!.functionCall?.name).toBe('edit');
|
||||
expect(modelTurn.parts![2]!.functionCall?.name).toBe('fetch');
|
||||
});
|
||||
|
||||
it('should not stop streaming if only one mutator is present', async () => {
|
||||
const responses = [
|
||||
{
|
||||
candidates: [
|
||||
{ content: { role: 'model', parts: [{ text: 'Part 1. ' }] } },
|
||||
],
|
||||
},
|
||||
{
|
||||
candidates: [
|
||||
{
|
||||
content: {
|
||||
role: 'model',
|
||||
parts: [{ functionCall: { name: 'edit', args: {} } }],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
candidates: [
|
||||
{
|
||||
content: {
|
||||
role: 'model',
|
||||
parts: [{ text: 'Part 2.' }],
|
||||
},
|
||||
finishReason: 'STOP',
|
||||
},
|
||||
],
|
||||
},
|
||||
] as unknown as GenerateContentResponse[];
|
||||
|
||||
vi.mocked(mockContentGenerator.generateContentStream).mockResolvedValue(
|
||||
(async function* () {
|
||||
for (const response of responses) {
|
||||
yield response;
|
||||
}
|
||||
})(),
|
||||
);
|
||||
|
||||
const stream = await chat.sendMessageStream(
|
||||
'test-model',
|
||||
{ message: 'test message' },
|
||||
'prompt-id-one-mutator',
|
||||
);
|
||||
for await (const _ of stream) {
|
||||
/* consume */
|
||||
}
|
||||
|
||||
const history = chat.getHistory();
|
||||
const modelTurn = history[1]!;
|
||||
expect(modelTurn?.parts?.length).toBe(3);
|
||||
expect(modelTurn.parts![1]!.functionCall?.name).toBe('edit');
|
||||
expect(modelTurn.parts![2]!.text).toBe('Part 2.');
|
||||
});
|
||||
|
||||
it('should clip the chunk containing the second mutator, preserving prior parts', async () => {
|
||||
const responses = [
|
||||
{
|
||||
candidates: [
|
||||
{
|
||||
content: {
|
||||
role: 'model',
|
||||
parts: [{ functionCall: { name: 'edit', args: {} } }],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
// This chunk has a valid part before the second mutator.
|
||||
// The valid part should be kept, the rest of the chunk discarded.
|
||||
{
|
||||
candidates: [
|
||||
{
|
||||
content: {
|
||||
role: 'model',
|
||||
parts: [
|
||||
{ text: 'Keep this text. ' },
|
||||
{ functionCall: { name: 'edit', args: {} } },
|
||||
{ text: 'Discard this text.' },
|
||||
],
|
||||
},
|
||||
finishReason: 'STOP',
|
||||
},
|
||||
],
|
||||
},
|
||||
] as unknown as GenerateContentResponse[];
|
||||
|
||||
const stream = (async function* () {
|
||||
for (const response of responses) {
|
||||
yield response;
|
||||
}
|
||||
})();
|
||||
|
||||
vi.mocked(mockContentGenerator.generateContentStream).mockResolvedValue(
|
||||
stream,
|
||||
);
|
||||
|
||||
const resultStream = await chat.sendMessageStream(
|
||||
'test-model',
|
||||
{ message: 'test' },
|
||||
'prompt-id-clip-chunk',
|
||||
);
|
||||
for await (const _ of resultStream) {
|
||||
/* consume */
|
||||
}
|
||||
|
||||
const history = chat.getHistory();
|
||||
const modelTurn = history[1]!;
|
||||
expect(modelTurn?.parts?.length).toBe(2);
|
||||
expect(modelTurn.parts![0]!.functionCall?.name).toBe('edit');
|
||||
expect(modelTurn.parts![1]!.text).toBe('Keep this text. ');
|
||||
});
|
||||
|
||||
it('should handle two mutators in the same chunk (parallel call scenario)', async () => {
|
||||
const responses = [
|
||||
{
|
||||
candidates: [
|
||||
{
|
||||
content: {
|
||||
role: 'model',
|
||||
parts: [
|
||||
{ text: 'Some text. ' },
|
||||
{ functionCall: { name: 'edit', args: {} } },
|
||||
{ functionCall: { name: 'edit', args: {} } },
|
||||
],
|
||||
},
|
||||
finishReason: 'STOP',
|
||||
},
|
||||
],
|
||||
},
|
||||
] as unknown as GenerateContentResponse[];
|
||||
|
||||
const stream = (async function* () {
|
||||
for (const response of responses) {
|
||||
yield response;
|
||||
}
|
||||
})();
|
||||
|
||||
vi.mocked(mockContentGenerator.generateContentStream).mockResolvedValue(
|
||||
stream,
|
||||
);
|
||||
|
||||
const resultStream = await chat.sendMessageStream(
|
||||
'test-model',
|
||||
{ message: 'test' },
|
||||
'prompt-id-parallel-mutators',
|
||||
);
|
||||
for await (const _ of resultStream) {
|
||||
/* consume */
|
||||
}
|
||||
|
||||
const history = chat.getHistory();
|
||||
const modelTurn = history[1]!;
|
||||
expect(modelTurn?.parts?.length).toBe(2);
|
||||
expect(modelTurn.parts![0]!.text).toBe('Some text. ');
|
||||
expect(modelTurn.parts![1]!.functionCall?.name).toBe('edit');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Model Resolution', () => {
|
||||
const mockResponse = {
|
||||
candidates: [
|
||||
|
||||
@@ -7,16 +7,15 @@
|
||||
// DISCLAIMER: This is a copied version of https://github.com/googleapis/js-genai/blob/main/src/chats.ts with the intention of working around a key bug
|
||||
// where function responses are not treated as "valid" responses: https://b.corp.google.com/issues/420354090
|
||||
|
||||
import {
|
||||
import type {
|
||||
GenerateContentResponse,
|
||||
type Content,
|
||||
type GenerateContentConfig,
|
||||
type SendMessageParameters,
|
||||
type Part,
|
||||
type Tool,
|
||||
FinishReason,
|
||||
ApiError,
|
||||
Content,
|
||||
GenerateContentConfig,
|
||||
SendMessageParameters,
|
||||
Part,
|
||||
Tool,
|
||||
} from '@google/genai';
|
||||
import { ApiError } from '@google/genai';
|
||||
import { toParts } from '../code_assist/converter.js';
|
||||
import { createUserContent } from '@google/genai';
|
||||
import { retryWithBackoff } from '../utils/retry.js';
|
||||
@@ -25,7 +24,7 @@ import {
|
||||
DEFAULT_GEMINI_FLASH_MODEL,
|
||||
getEffectiveModel,
|
||||
} from '../config/models.js';
|
||||
import { hasCycleInSchema, MUTATOR_KINDS } from '../tools/tools.js';
|
||||
import { hasCycleInSchema } from '../tools/tools.js';
|
||||
import type { StructuredError } from './turn.js';
|
||||
import {
|
||||
logContentRetry,
|
||||
@@ -511,7 +510,7 @@ export class GeminiChat {
|
||||
let hasToolCall = false;
|
||||
let hasFinishReason = false;
|
||||
|
||||
for await (const chunk of this.stopBeforeSecondMutator(streamResponse)) {
|
||||
for await (const chunk of streamResponse) {
|
||||
hasFinishReason =
|
||||
chunk?.candidates?.some((candidate) => candidate.finishReason) ?? false;
|
||||
if (isValidResponse(chunk)) {
|
||||
@@ -629,64 +628,6 @@ export class GeminiChat {
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncates the chunkStream right before the second function call to a
|
||||
* function that mutates state. This may involve trimming parts from a chunk
|
||||
* as well as omtting some chunks altogether.
|
||||
*
|
||||
* We do this because it improves tool call quality if the model gets
|
||||
* feedback from one mutating function call before it makes the next one.
|
||||
*/
|
||||
private async *stopBeforeSecondMutator(
|
||||
chunkStream: AsyncGenerator<GenerateContentResponse>,
|
||||
): AsyncGenerator<GenerateContentResponse> {
|
||||
let foundMutatorFunctionCall = false;
|
||||
|
||||
for await (const chunk of chunkStream) {
|
||||
const candidate = chunk.candidates?.[0];
|
||||
const content = candidate?.content;
|
||||
if (!candidate || !content?.parts) {
|
||||
yield chunk;
|
||||
continue;
|
||||
}
|
||||
|
||||
const truncatedParts: Part[] = [];
|
||||
for (const part of content.parts) {
|
||||
if (this.isMutatorFunctionCall(part)) {
|
||||
if (foundMutatorFunctionCall) {
|
||||
// This is the second mutator call.
|
||||
// Truncate and return immedaitely.
|
||||
const newChunk = new GenerateContentResponse();
|
||||
newChunk.candidates = [
|
||||
{
|
||||
...candidate,
|
||||
content: {
|
||||
...content,
|
||||
parts: truncatedParts,
|
||||
},
|
||||
finishReason: FinishReason.STOP,
|
||||
},
|
||||
];
|
||||
yield newChunk;
|
||||
return;
|
||||
}
|
||||
foundMutatorFunctionCall = true;
|
||||
}
|
||||
truncatedParts.push(part);
|
||||
}
|
||||
|
||||
yield chunk;
|
||||
}
|
||||
}
|
||||
|
||||
private isMutatorFunctionCall(part: Part): boolean {
|
||||
if (!part?.functionCall?.name) {
|
||||
return false;
|
||||
}
|
||||
const tool = this.config.getToolRegistry().getTool(part.functionCall.name);
|
||||
return !!tool && MUTATOR_KINDS.includes(tool.kind);
|
||||
}
|
||||
}
|
||||
|
||||
/** Visible for Testing */
|
||||
|
||||
@@ -60,7 +60,10 @@ function verifyVSCode(
|
||||
if (ide.name !== IDE_DEFINITIONS.vscode.name) {
|
||||
return ide;
|
||||
}
|
||||
if (ideProcessInfo.command.toLowerCase().includes('code')) {
|
||||
if (
|
||||
ideProcessInfo.command &&
|
||||
ideProcessInfo.command.toLowerCase().includes('code')
|
||||
) {
|
||||
return IDE_DEFINITIONS.vscode;
|
||||
}
|
||||
return IDE_DEFINITIONS.vscodefork;
|
||||
|
||||
@@ -102,6 +102,8 @@ export * from './tools/web-search/index.js';
|
||||
export * from './tools/read-many-files.js';
|
||||
export * from './tools/mcp-client.js';
|
||||
export * from './tools/mcp-tool.js';
|
||||
export * from './tools/task.js';
|
||||
export * from './tools/todoWrite.js';
|
||||
|
||||
// MCP OAuth
|
||||
export { MCPOAuthProvider } from './mcp/oauth-provider.js';
|
||||
|
||||
@@ -181,6 +181,56 @@ describe('ChatCompressionService', () => {
|
||||
expect(result.newHistory).toBeNull();
|
||||
});
|
||||
|
||||
it('should return NOOP when contextPercentageThreshold is 0', async () => {
|
||||
const history: Content[] = [
|
||||
{ role: 'user', parts: [{ text: 'msg1' }] },
|
||||
{ role: 'model', parts: [{ text: 'msg2' }] },
|
||||
];
|
||||
vi.mocked(mockChat.getHistory).mockReturnValue(history);
|
||||
vi.mocked(uiTelemetryService.getLastPromptTokenCount).mockReturnValue(800);
|
||||
vi.mocked(mockConfig.getChatCompression).mockReturnValue({
|
||||
contextPercentageThreshold: 0,
|
||||
});
|
||||
|
||||
const mockGenerateContent = vi.fn();
|
||||
vi.mocked(mockConfig.getContentGenerator).mockReturnValue({
|
||||
generateContent: mockGenerateContent,
|
||||
} as unknown as ContentGenerator);
|
||||
|
||||
const result = await service.compress(
|
||||
mockChat,
|
||||
mockPromptId,
|
||||
false,
|
||||
mockModel,
|
||||
mockConfig,
|
||||
false,
|
||||
);
|
||||
|
||||
expect(result.info).toMatchObject({
|
||||
compressionStatus: CompressionStatus.NOOP,
|
||||
originalTokenCount: 0,
|
||||
newTokenCount: 0,
|
||||
});
|
||||
expect(mockGenerateContent).not.toHaveBeenCalled();
|
||||
expect(tokenLimit).not.toHaveBeenCalled();
|
||||
|
||||
const forcedResult = await service.compress(
|
||||
mockChat,
|
||||
mockPromptId,
|
||||
true,
|
||||
mockModel,
|
||||
mockConfig,
|
||||
false,
|
||||
);
|
||||
expect(forcedResult.info).toMatchObject({
|
||||
compressionStatus: CompressionStatus.NOOP,
|
||||
originalTokenCount: 0,
|
||||
newTokenCount: 0,
|
||||
});
|
||||
expect(mockGenerateContent).not.toHaveBeenCalled();
|
||||
expect(tokenLimit).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should compress if over token threshold', async () => {
|
||||
const history: Content[] = [
|
||||
{ role: 'user', parts: [{ text: 'msg1' }] },
|
||||
|
||||
@@ -86,10 +86,14 @@ export class ChatCompressionService {
|
||||
hasFailedCompressionAttempt: boolean,
|
||||
): Promise<{ newHistory: Content[] | null; info: ChatCompressionInfo }> {
|
||||
const curatedHistory = chat.getHistory(true);
|
||||
const threshold =
|
||||
config.getChatCompression()?.contextPercentageThreshold ??
|
||||
COMPRESSION_TOKEN_THRESHOLD;
|
||||
|
||||
// Regardless of `force`, don't do anything if the history is empty.
|
||||
if (
|
||||
curatedHistory.length === 0 ||
|
||||
threshold <= 0 ||
|
||||
(hasFailedCompressionAttempt && !force)
|
||||
) {
|
||||
return {
|
||||
@@ -104,13 +108,8 @@ export class ChatCompressionService {
|
||||
|
||||
const originalTokenCount = uiTelemetryService.getLastPromptTokenCount();
|
||||
|
||||
const contextPercentageThreshold =
|
||||
config.getChatCompression()?.contextPercentageThreshold;
|
||||
|
||||
// Don't compress if not forced and we are under the limit.
|
||||
if (!force) {
|
||||
const threshold =
|
||||
contextPercentageThreshold ?? COMPRESSION_TOKEN_THRESHOLD;
|
||||
if (originalTokenCount < threshold * tokenLimit(model)) {
|
||||
return {
|
||||
newHistory: null,
|
||||
|
||||
@@ -62,9 +62,10 @@ export type {
|
||||
SubAgentToolResultEvent,
|
||||
SubAgentFinishEvent,
|
||||
SubAgentErrorEvent,
|
||||
SubAgentApprovalRequestEvent,
|
||||
} from './subagent-events.js';
|
||||
|
||||
export { SubAgentEventEmitter } from './subagent-events.js';
|
||||
export { SubAgentEventEmitter, SubAgentEventType } from './subagent-events.js';
|
||||
|
||||
// Statistics and formatting
|
||||
export type {
|
||||
|
||||
@@ -37,6 +37,7 @@ describe('GlobTool', () => {
|
||||
getFileExclusions: () => ({
|
||||
getGlobExcludes: () => [],
|
||||
}),
|
||||
getTruncateToolOutputLines: () => 1000,
|
||||
} as unknown as Config;
|
||||
|
||||
beforeEach(async () => {
|
||||
@@ -88,17 +89,6 @@ describe('GlobTool', () => {
|
||||
expect(result.returnDisplay).toBe('Found 2 matching file(s)');
|
||||
});
|
||||
|
||||
it('should find files case-sensitively when case_sensitive is true', async () => {
|
||||
const params: GlobToolParams = { pattern: '*.txt', case_sensitive: true };
|
||||
const invocation = globTool.build(params);
|
||||
const result = await invocation.execute(abortSignal);
|
||||
expect(result.llmContent).toContain('Found 1 file(s)');
|
||||
expect(result.llmContent).toContain(path.join(tempRootDir, 'fileA.txt'));
|
||||
expect(result.llmContent).not.toContain(
|
||||
path.join(tempRootDir, 'FileB.TXT'),
|
||||
);
|
||||
});
|
||||
|
||||
it('should find files case-insensitively by default (pattern: *.TXT)', async () => {
|
||||
const params: GlobToolParams = { pattern: '*.TXT' };
|
||||
const invocation = globTool.build(params);
|
||||
@@ -108,18 +98,6 @@ describe('GlobTool', () => {
|
||||
expect(result.llmContent).toContain(path.join(tempRootDir, 'FileB.TXT'));
|
||||
});
|
||||
|
||||
it('should find files case-insensitively when case_sensitive is false (pattern: *.TXT)', async () => {
|
||||
const params: GlobToolParams = {
|
||||
pattern: '*.TXT',
|
||||
case_sensitive: false,
|
||||
};
|
||||
const invocation = globTool.build(params);
|
||||
const result = await invocation.execute(abortSignal);
|
||||
expect(result.llmContent).toContain('Found 2 file(s)');
|
||||
expect(result.llmContent).toContain(path.join(tempRootDir, 'fileA.txt'));
|
||||
expect(result.llmContent).toContain(path.join(tempRootDir, 'FileB.TXT'));
|
||||
});
|
||||
|
||||
it('should find files using a pattern that includes a subdirectory', async () => {
|
||||
const params: GlobToolParams = { pattern: 'sub/*.md' };
|
||||
const invocation = globTool.build(params);
|
||||
@@ -207,7 +185,7 @@ describe('GlobTool', () => {
|
||||
const filesListed = llmContent
|
||||
.trim()
|
||||
.split(/\r?\n/)
|
||||
.slice(1)
|
||||
.slice(2)
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
@@ -220,14 +198,13 @@ describe('GlobTool', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('should return a PATH_NOT_IN_WORKSPACE error if path is outside workspace', async () => {
|
||||
it('should return error if path is outside workspace', async () => {
|
||||
// Bypassing validation to test execute method directly
|
||||
vi.spyOn(globTool, 'validateToolParams').mockReturnValue(null);
|
||||
const params: GlobToolParams = { pattern: '*.txt', path: '/etc' };
|
||||
const invocation = globTool.build(params);
|
||||
const result = await invocation.execute(abortSignal);
|
||||
expect(result.error?.type).toBe(ToolErrorType.PATH_NOT_IN_WORKSPACE);
|
||||
expect(result.returnDisplay).toBe('Path is not within workspace');
|
||||
expect(result.returnDisplay).toBe('Error: Path is not within workspace');
|
||||
});
|
||||
|
||||
it('should return a GLOB_EXECUTION_ERROR on glob failure', async () => {
|
||||
@@ -255,15 +232,6 @@ describe('GlobTool', () => {
|
||||
expect(globTool.validateToolParams(params)).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for valid parameters (pattern, path, and case_sensitive)', () => {
|
||||
const params: GlobToolParams = {
|
||||
pattern: '*.js',
|
||||
path: 'sub',
|
||||
case_sensitive: true,
|
||||
};
|
||||
expect(globTool.validateToolParams(params)).toBeNull();
|
||||
});
|
||||
|
||||
it('should return error if pattern is missing (schema validation)', () => {
|
||||
// Need to correctly define this as an object without pattern
|
||||
const params = { path: '.' };
|
||||
@@ -297,16 +265,6 @@ describe('GlobTool', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('should return error if case_sensitive is provided but is not a boolean', () => {
|
||||
const params = {
|
||||
pattern: '*.ts',
|
||||
case_sensitive: 'true',
|
||||
} as unknown as GlobToolParams; // Force incorrect type
|
||||
expect(globTool.validateToolParams(params)).toBe(
|
||||
'params/case_sensitive must be boolean',
|
||||
);
|
||||
});
|
||||
|
||||
it("should return error if search path resolves outside the tool's root directory", () => {
|
||||
// Create a globTool instance specifically for this test, with a deeper root
|
||||
tempRootDir = path.join(tempRootDir, 'sub');
|
||||
@@ -319,7 +277,7 @@ describe('GlobTool', () => {
|
||||
path: '../../../../../../../../../../tmp', // Definitely outside
|
||||
};
|
||||
expect(specificGlobTool.validateToolParams(paramsOutside)).toContain(
|
||||
'resolves outside the allowed workspace directories',
|
||||
'Path is not within workspace',
|
||||
);
|
||||
});
|
||||
|
||||
@@ -329,14 +287,14 @@ describe('GlobTool', () => {
|
||||
path: 'nonexistent_subdir',
|
||||
};
|
||||
expect(globTool.validateToolParams(params)).toContain(
|
||||
'Search path does not exist',
|
||||
'Path does not exist',
|
||||
);
|
||||
});
|
||||
|
||||
it('should return error if specified search path is a file, not a directory', async () => {
|
||||
const params: GlobToolParams = { pattern: '*.txt', path: 'fileA.txt' };
|
||||
expect(globTool.validateToolParams(params)).toContain(
|
||||
'Search path is not a directory',
|
||||
'Path is not a directory',
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -348,20 +306,10 @@ describe('GlobTool', () => {
|
||||
|
||||
expect(globTool.validateToolParams(validPath)).toBeNull();
|
||||
expect(globTool.validateToolParams(invalidPath)).toContain(
|
||||
'resolves outside the allowed workspace directories',
|
||||
'Path is not within workspace',
|
||||
);
|
||||
});
|
||||
|
||||
it('should provide clear error messages when path is outside workspace', () => {
|
||||
const invalidPath = { pattern: '*.ts', path: '/etc' };
|
||||
const error = globTool.validateToolParams(invalidPath);
|
||||
|
||||
expect(error).toContain(
|
||||
'resolves outside the allowed workspace directories',
|
||||
);
|
||||
expect(error).toContain(tempRootDir);
|
||||
});
|
||||
|
||||
it('should work with paths in workspace subdirectories', async () => {
|
||||
const params: GlobToolParams = { pattern: '*.md', path: 'sub' };
|
||||
const invocation = globTool.build(params);
|
||||
@@ -417,47 +365,123 @@ describe('GlobTool', () => {
|
||||
expect(result.llmContent).toContain('Found 3 file(s)'); // fileA.txt, FileB.TXT, b.notignored.txt
|
||||
expect(result.llmContent).not.toContain('a.qwenignored.txt');
|
||||
});
|
||||
});
|
||||
|
||||
it('should not respect .gitignore when respect_git_ignore is false', async () => {
|
||||
await fs.writeFile(path.join(tempRootDir, '.gitignore'), '*.ignored.txt');
|
||||
await fs.writeFile(
|
||||
path.join(tempRootDir, 'a.ignored.txt'),
|
||||
'ignored content',
|
||||
);
|
||||
describe('file count truncation', () => {
|
||||
it('should truncate results when more than 100 files are found', async () => {
|
||||
// Create 150 test files
|
||||
for (let i = 1; i <= 150; i++) {
|
||||
await fs.writeFile(
|
||||
path.join(tempRootDir, `file${i}.trunctest`),
|
||||
`content${i}`,
|
||||
);
|
||||
}
|
||||
|
||||
const params: GlobToolParams = {
|
||||
pattern: '*.txt',
|
||||
respect_git_ignore: false,
|
||||
};
|
||||
const params: GlobToolParams = { pattern: '*.trunctest' };
|
||||
const invocation = globTool.build(params);
|
||||
const result = await invocation.execute(abortSignal);
|
||||
const llmContent = partListUnionToString(result.llmContent);
|
||||
|
||||
expect(result.llmContent).toContain('Found 3 file(s)'); // fileA.txt, FileB.TXT, a.ignored.txt
|
||||
expect(result.llmContent).toContain('a.ignored.txt');
|
||||
// Should report all 150 files found
|
||||
expect(llmContent).toContain('Found 150 file(s)');
|
||||
|
||||
// Should include truncation notice
|
||||
expect(llmContent).toContain('[50 files truncated] ...');
|
||||
|
||||
// Count the number of .trunctest files mentioned in the output
|
||||
const fileMatches = llmContent.match(/file\d+\.trunctest/g);
|
||||
expect(fileMatches).toBeDefined();
|
||||
expect(fileMatches?.length).toBe(100);
|
||||
|
||||
// returnDisplay should indicate truncation
|
||||
expect(result.returnDisplay).toBe(
|
||||
'Found 150 matching file(s) (truncated)',
|
||||
);
|
||||
});
|
||||
|
||||
it('should not respect .qwenignore when respect_qwen_ignore is false', async () => {
|
||||
await fs.writeFile(
|
||||
path.join(tempRootDir, '.qwenignore'),
|
||||
'*.qwenignored.txt',
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(tempRootDir, 'a.qwenignored.txt'),
|
||||
'ignored content',
|
||||
);
|
||||
it('should not truncate when exactly 100 files are found', async () => {
|
||||
// Create exactly 100 test files
|
||||
for (let i = 1; i <= 100; i++) {
|
||||
await fs.writeFile(
|
||||
path.join(tempRootDir, `exact${i}.trunctest`),
|
||||
`content${i}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Recreate the tool to pick up the new .qwenignore file
|
||||
globTool = new GlobTool(mockConfig);
|
||||
|
||||
const params: GlobToolParams = {
|
||||
pattern: '*.txt',
|
||||
respect_qwen_ignore: false,
|
||||
};
|
||||
const params: GlobToolParams = { pattern: '*.trunctest' };
|
||||
const invocation = globTool.build(params);
|
||||
const result = await invocation.execute(abortSignal);
|
||||
|
||||
expect(result.llmContent).toContain('Found 3 file(s)'); // fileA.txt, FileB.TXT, a.qwenignored.txt
|
||||
expect(result.llmContent).toContain('a.qwenignored.txt');
|
||||
// Should report all 100 files found
|
||||
expect(result.llmContent).toContain('Found 100 file(s)');
|
||||
|
||||
// Should NOT include truncation notice
|
||||
expect(result.llmContent).not.toContain('truncated');
|
||||
|
||||
// Should show all 100 files
|
||||
expect(result.llmContent).toContain('exact1.trunctest');
|
||||
expect(result.llmContent).toContain('exact100.trunctest');
|
||||
|
||||
// returnDisplay should NOT indicate truncation
|
||||
expect(result.returnDisplay).toBe('Found 100 matching file(s)');
|
||||
});
|
||||
|
||||
it('should not truncate when fewer than 100 files are found', async () => {
|
||||
// Create 50 test files
|
||||
for (let i = 1; i <= 50; i++) {
|
||||
await fs.writeFile(
|
||||
path.join(tempRootDir, `small${i}.trunctest`),
|
||||
`content${i}`,
|
||||
);
|
||||
}
|
||||
|
||||
const params: GlobToolParams = { pattern: '*.trunctest' };
|
||||
const invocation = globTool.build(params);
|
||||
const result = await invocation.execute(abortSignal);
|
||||
|
||||
// Should report all 50 files found
|
||||
expect(result.llmContent).toContain('Found 50 file(s)');
|
||||
|
||||
// Should NOT include truncation notice
|
||||
expect(result.llmContent).not.toContain('truncated');
|
||||
|
||||
// returnDisplay should NOT indicate truncation
|
||||
expect(result.returnDisplay).toBe('Found 50 matching file(s)');
|
||||
});
|
||||
|
||||
it('should use correct singular/plural in truncation message for 1 file truncated', async () => {
|
||||
// Create 101 test files (will truncate 1 file)
|
||||
for (let i = 1; i <= 101; i++) {
|
||||
await fs.writeFile(
|
||||
path.join(tempRootDir, `singular${i}.trunctest`),
|
||||
`content${i}`,
|
||||
);
|
||||
}
|
||||
|
||||
const params: GlobToolParams = { pattern: '*.trunctest' };
|
||||
const invocation = globTool.build(params);
|
||||
const result = await invocation.execute(abortSignal);
|
||||
|
||||
// Should use singular "file" for 1 truncated file
|
||||
expect(result.llmContent).toContain('[1 file truncated] ...');
|
||||
expect(result.llmContent).not.toContain('[1 files truncated]');
|
||||
});
|
||||
|
||||
it('should use correct plural in truncation message for multiple files truncated', async () => {
|
||||
// Create 105 test files (will truncate 5 files)
|
||||
for (let i = 1; i <= 105; i++) {
|
||||
await fs.writeFile(
|
||||
path.join(tempRootDir, `plural${i}.trunctest`),
|
||||
`content${i}`,
|
||||
);
|
||||
}
|
||||
|
||||
const params: GlobToolParams = { pattern: '*.trunctest' };
|
||||
const invocation = globTool.build(params);
|
||||
const result = await invocation.execute(abortSignal);
|
||||
|
||||
// Should use plural "files" for multiple truncated files
|
||||
expect(result.llmContent).toContain('[5 files truncated] ...');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -10,10 +10,17 @@ import { glob, escape } from 'glob';
|
||||
import type { ToolInvocation, ToolResult } from './tools.js';
|
||||
import { BaseDeclarativeTool, BaseToolInvocation, Kind } from './tools.js';
|
||||
import { ToolNames } from './tool-names.js';
|
||||
import { shortenPath, makeRelative } from '../utils/paths.js';
|
||||
import { resolveAndValidatePath } from '../utils/paths.js';
|
||||
import { type Config } from '../config/config.js';
|
||||
import { DEFAULT_FILE_FILTERING_OPTIONS } from '../config/constants.js';
|
||||
import {
|
||||
DEFAULT_FILE_FILTERING_OPTIONS,
|
||||
type FileFilteringOptions,
|
||||
} from '../config/constants.js';
|
||||
import { ToolErrorType } from './tool-error.js';
|
||||
import { getErrorMessage } from '../utils/errors.js';
|
||||
import type { FileDiscoveryService } from '../services/fileDiscoveryService.js';
|
||||
|
||||
const MAX_FILE_COUNT = 100;
|
||||
|
||||
// Subset of 'Path' interface provided by 'glob' that we can implement for testing
|
||||
export interface GlobPath {
|
||||
@@ -64,118 +71,68 @@ export interface GlobToolParams {
|
||||
* The directory to search in (optional, defaults to current directory)
|
||||
*/
|
||||
path?: string;
|
||||
|
||||
/**
|
||||
* Whether the search should be case-sensitive (optional, defaults to false)
|
||||
*/
|
||||
case_sensitive?: boolean;
|
||||
|
||||
/**
|
||||
* Whether to respect .gitignore patterns (optional, defaults to true)
|
||||
*/
|
||||
respect_git_ignore?: boolean;
|
||||
|
||||
/**
|
||||
* Whether to respect .qwenignore patterns (optional, defaults to true)
|
||||
*/
|
||||
respect_qwen_ignore?: boolean;
|
||||
}
|
||||
|
||||
class GlobToolInvocation extends BaseToolInvocation<
|
||||
GlobToolParams,
|
||||
ToolResult
|
||||
> {
|
||||
private fileService: FileDiscoveryService;
|
||||
|
||||
constructor(
|
||||
private config: Config,
|
||||
params: GlobToolParams,
|
||||
) {
|
||||
super(params);
|
||||
this.fileService = config.getFileService();
|
||||
}
|
||||
|
||||
getDescription(): string {
|
||||
let description = `'${this.params.pattern}'`;
|
||||
if (this.params.path) {
|
||||
const searchDir = path.resolve(
|
||||
this.config.getTargetDir(),
|
||||
this.params.path || '.',
|
||||
);
|
||||
const relativePath = makeRelative(searchDir, this.config.getTargetDir());
|
||||
description += ` within ${shortenPath(relativePath)}`;
|
||||
description += ` in path '${this.params.path}'`;
|
||||
}
|
||||
|
||||
return description;
|
||||
}
|
||||
|
||||
async execute(signal: AbortSignal): Promise<ToolResult> {
|
||||
try {
|
||||
const workspaceContext = this.config.getWorkspaceContext();
|
||||
const workspaceDirectories = workspaceContext.getDirectories();
|
||||
// Default to target directory if no path is provided
|
||||
const searchDirAbs = resolveAndValidatePath(
|
||||
this.config,
|
||||
this.params.path,
|
||||
);
|
||||
const searchLocationDescription = this.params.path
|
||||
? `within ${searchDirAbs}`
|
||||
: `in the workspace directory`;
|
||||
|
||||
// If a specific path is provided, resolve it and check if it's within workspace
|
||||
let searchDirectories: readonly string[];
|
||||
if (this.params.path) {
|
||||
const searchDirAbsolute = path.resolve(
|
||||
this.config.getTargetDir(),
|
||||
this.params.path,
|
||||
);
|
||||
if (!workspaceContext.isPathWithinWorkspace(searchDirAbsolute)) {
|
||||
const rawError = `Error: Path "${this.params.path}" is not within any workspace directory`;
|
||||
return {
|
||||
llmContent: rawError,
|
||||
returnDisplay: `Path is not within workspace`,
|
||||
error: {
|
||||
message: rawError,
|
||||
type: ToolErrorType.PATH_NOT_IN_WORKSPACE,
|
||||
},
|
||||
};
|
||||
}
|
||||
searchDirectories = [searchDirAbsolute];
|
||||
} else {
|
||||
// Search across all workspace directories
|
||||
searchDirectories = workspaceDirectories;
|
||||
// Collect entries from the search directory
|
||||
let pattern = this.params.pattern;
|
||||
const fullPath = path.join(searchDirAbs, pattern);
|
||||
if (fs.existsSync(fullPath)) {
|
||||
pattern = escape(pattern);
|
||||
}
|
||||
|
||||
// Get centralized file discovery service
|
||||
const fileDiscovery = this.config.getFileService();
|
||||
|
||||
// Collect entries from all search directories
|
||||
const allEntries: GlobPath[] = [];
|
||||
for (const searchDir of searchDirectories) {
|
||||
let pattern = this.params.pattern;
|
||||
const fullPath = path.join(searchDir, pattern);
|
||||
if (fs.existsSync(fullPath)) {
|
||||
pattern = escape(pattern);
|
||||
}
|
||||
|
||||
const entries = (await glob(pattern, {
|
||||
cwd: searchDir,
|
||||
withFileTypes: true,
|
||||
nodir: true,
|
||||
stat: true,
|
||||
nocase: !this.params.case_sensitive,
|
||||
dot: true,
|
||||
ignore: this.config.getFileExclusions().getGlobExcludes(),
|
||||
follow: false,
|
||||
signal,
|
||||
})) as GlobPath[];
|
||||
|
||||
allEntries.push(...entries);
|
||||
}
|
||||
const allEntries = (await glob(pattern, {
|
||||
cwd: searchDirAbs,
|
||||
withFileTypes: true,
|
||||
nodir: true,
|
||||
stat: true,
|
||||
nocase: true,
|
||||
dot: true,
|
||||
follow: false,
|
||||
signal,
|
||||
})) as GlobPath[];
|
||||
|
||||
const relativePaths = allEntries.map((p) =>
|
||||
path.relative(this.config.getTargetDir(), p.fullpath()),
|
||||
);
|
||||
|
||||
const { filteredPaths, gitIgnoredCount, qwenIgnoredCount } =
|
||||
fileDiscovery.filterFilesWithReport(relativePaths, {
|
||||
respectGitIgnore:
|
||||
this.params?.respect_git_ignore ??
|
||||
this.config.getFileFilteringOptions().respectGitIgnore ??
|
||||
DEFAULT_FILE_FILTERING_OPTIONS.respectGitIgnore,
|
||||
respectQwenIgnore:
|
||||
this.params?.respect_qwen_ignore ??
|
||||
this.config.getFileFilteringOptions().respectQwenIgnore ??
|
||||
DEFAULT_FILE_FILTERING_OPTIONS.respectQwenIgnore,
|
||||
});
|
||||
const { filteredPaths } = this.fileService.filterFilesWithReport(
|
||||
relativePaths,
|
||||
this.getFileFilteringOptions(),
|
||||
);
|
||||
|
||||
const filteredAbsolutePaths = new Set(
|
||||
filteredPaths.map((p) => path.resolve(this.config.getTargetDir(), p)),
|
||||
@@ -186,20 +143,8 @@ class GlobToolInvocation extends BaseToolInvocation<
|
||||
);
|
||||
|
||||
if (!filteredEntries || filteredEntries.length === 0) {
|
||||
let message = `No files found matching pattern "${this.params.pattern}"`;
|
||||
if (searchDirectories.length === 1) {
|
||||
message += ` within ${searchDirectories[0]}`;
|
||||
} else {
|
||||
message += ` within ${searchDirectories.length} workspace directories`;
|
||||
}
|
||||
if (gitIgnoredCount > 0) {
|
||||
message += ` (${gitIgnoredCount} files were git-ignored)`;
|
||||
}
|
||||
if (qwenIgnoredCount > 0) {
|
||||
message += ` (${qwenIgnoredCount} files were qwen-ignored)`;
|
||||
}
|
||||
return {
|
||||
llmContent: message,
|
||||
llmContent: `No files found matching pattern "${this.params.pattern}" ${searchLocationDescription}`,
|
||||
returnDisplay: `No files found`,
|
||||
};
|
||||
}
|
||||
@@ -215,29 +160,36 @@ class GlobToolInvocation extends BaseToolInvocation<
|
||||
oneDayInMs,
|
||||
);
|
||||
|
||||
const sortedAbsolutePaths = sortedEntries.map((entry) =>
|
||||
const totalFileCount = sortedEntries.length;
|
||||
const fileLimit = Math.min(
|
||||
MAX_FILE_COUNT,
|
||||
this.config.getTruncateToolOutputLines(),
|
||||
);
|
||||
const truncated = totalFileCount > fileLimit;
|
||||
|
||||
// Limit to fileLimit if needed
|
||||
const entriesToShow = truncated
|
||||
? sortedEntries.slice(0, fileLimit)
|
||||
: sortedEntries;
|
||||
|
||||
const sortedAbsolutePaths = entriesToShow.map((entry) =>
|
||||
entry.fullpath(),
|
||||
);
|
||||
const fileListDescription = sortedAbsolutePaths.join('\n');
|
||||
const fileCount = sortedAbsolutePaths.length;
|
||||
|
||||
let resultMessage = `Found ${fileCount} file(s) matching "${this.params.pattern}"`;
|
||||
if (searchDirectories.length === 1) {
|
||||
resultMessage += ` within ${searchDirectories[0]}`;
|
||||
} else {
|
||||
resultMessage += ` across ${searchDirectories.length} workspace directories`;
|
||||
let resultMessage = `Found ${totalFileCount} file(s) matching "${this.params.pattern}" ${searchLocationDescription}`;
|
||||
resultMessage += `, sorted by modification time (newest first):\n---\n${fileListDescription}`;
|
||||
|
||||
// Add truncation notice if needed
|
||||
if (truncated) {
|
||||
const omittedFiles = totalFileCount - fileLimit;
|
||||
const fileTerm = omittedFiles === 1 ? 'file' : 'files';
|
||||
resultMessage += `\n---\n[${omittedFiles} ${fileTerm} truncated] ...`;
|
||||
}
|
||||
if (gitIgnoredCount > 0) {
|
||||
resultMessage += ` (${gitIgnoredCount} additional files were git-ignored)`;
|
||||
}
|
||||
if (qwenIgnoredCount > 0) {
|
||||
resultMessage += ` (${qwenIgnoredCount} additional files were qwen-ignored)`;
|
||||
}
|
||||
resultMessage += `, sorted by modification time (newest first):\n${fileListDescription}`;
|
||||
|
||||
return {
|
||||
llmContent: resultMessage,
|
||||
returnDisplay: `Found ${fileCount} matching file(s)`,
|
||||
returnDisplay: `Found ${totalFileCount} matching file(s)${truncated ? ' (truncated)' : ''}`,
|
||||
};
|
||||
} catch (error) {
|
||||
const errorMessage =
|
||||
@@ -246,7 +198,7 @@ class GlobToolInvocation extends BaseToolInvocation<
|
||||
const rawError = `Error during glob search operation: ${errorMessage}`;
|
||||
return {
|
||||
llmContent: rawError,
|
||||
returnDisplay: `Error: An unexpected error occurred.`,
|
||||
returnDisplay: `Error: ${errorMessage || 'An unexpected error occurred.'}`,
|
||||
error: {
|
||||
message: rawError,
|
||||
type: ToolErrorType.GLOB_EXECUTION_ERROR,
|
||||
@@ -254,6 +206,18 @@ class GlobToolInvocation extends BaseToolInvocation<
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
private getFileFilteringOptions(): FileFilteringOptions {
|
||||
const options = this.config.getFileFilteringOptions?.();
|
||||
return {
|
||||
respectGitIgnore:
|
||||
options?.respectGitIgnore ??
|
||||
DEFAULT_FILE_FILTERING_OPTIONS.respectGitIgnore,
|
||||
respectQwenIgnore:
|
||||
options?.respectQwenIgnore ??
|
||||
DEFAULT_FILE_FILTERING_OPTIONS.respectQwenIgnore,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -266,35 +230,19 @@ export class GlobTool extends BaseDeclarativeTool<GlobToolParams, ToolResult> {
|
||||
super(
|
||||
GlobTool.Name,
|
||||
'FindFiles',
|
||||
'Efficiently finds files matching specific glob patterns (e.g., `src/**/*.ts`, `**/*.md`), returning absolute paths sorted by modification time (newest first). Ideal for quickly locating files based on their name or path structure, especially in large codebases.',
|
||||
'Fast file pattern matching tool that works with any codebase size\n- Supports glob patterns like "**/*.js" or "src/**/*.ts"\n- Returns matching file paths sorted by modification time\n- Use this tool when you need to find files by name patterns\n- When you are doing an open ended search that may require multiple rounds of globbing and grepping, use the Agent tool instead\n- You have the capability to call multiple tools in a single response. It is always better to speculatively perform multiple searches as a batch that are potentially useful.',
|
||||
Kind.Search,
|
||||
{
|
||||
properties: {
|
||||
pattern: {
|
||||
description:
|
||||
"The glob pattern to match against (e.g., '**/*.py', 'docs/*.md').",
|
||||
description: 'The glob pattern to match files against',
|
||||
type: 'string',
|
||||
},
|
||||
path: {
|
||||
description:
|
||||
'Optional: The absolute path to the directory to search within. If omitted, searches the root directory.',
|
||||
'The directory to search in. If not specified, the current working directory will be used. IMPORTANT: Omit this field to use the default directory. DO NOT enter "undefined" or "null" - simply omit it for the default behavior. Must be a valid directory path if provided.',
|
||||
type: 'string',
|
||||
},
|
||||
case_sensitive: {
|
||||
description:
|
||||
'Optional: Whether the search should be case-sensitive. Defaults to false.',
|
||||
type: 'boolean',
|
||||
},
|
||||
respect_git_ignore: {
|
||||
description:
|
||||
'Optional: Whether to respect .gitignore patterns when finding files. Only available in git repositories. Defaults to true.',
|
||||
type: 'boolean',
|
||||
},
|
||||
respect_qwen_ignore: {
|
||||
description:
|
||||
'Optional: Whether to respect .qwenignore patterns when finding files. Defaults to true.',
|
||||
type: 'boolean',
|
||||
},
|
||||
},
|
||||
required: ['pattern'],
|
||||
type: 'object',
|
||||
@@ -308,29 +256,6 @@ export class GlobTool extends BaseDeclarativeTool<GlobToolParams, ToolResult> {
|
||||
protected override validateToolParamValues(
|
||||
params: GlobToolParams,
|
||||
): string | null {
|
||||
const searchDirAbsolute = path.resolve(
|
||||
this.config.getTargetDir(),
|
||||
params.path || '.',
|
||||
);
|
||||
|
||||
const workspaceContext = this.config.getWorkspaceContext();
|
||||
if (!workspaceContext.isPathWithinWorkspace(searchDirAbsolute)) {
|
||||
const directories = workspaceContext.getDirectories();
|
||||
return `Search path ("${searchDirAbsolute}") resolves outside the allowed workspace directories: ${directories.join(', ')}`;
|
||||
}
|
||||
|
||||
const targetDir = searchDirAbsolute || this.config.getTargetDir();
|
||||
try {
|
||||
if (!fs.existsSync(targetDir)) {
|
||||
return `Search path does not exist ${targetDir}`;
|
||||
}
|
||||
if (!fs.statSync(targetDir).isDirectory()) {
|
||||
return `Search path is not a directory: ${targetDir}`;
|
||||
}
|
||||
} catch (e: unknown) {
|
||||
return `Error accessing search path: ${e}`;
|
||||
}
|
||||
|
||||
if (
|
||||
!params.pattern ||
|
||||
typeof params.pattern !== 'string' ||
|
||||
@@ -339,6 +264,15 @@ export class GlobTool extends BaseDeclarativeTool<GlobToolParams, ToolResult> {
|
||||
return "The 'pattern' parameter cannot be empty.";
|
||||
}
|
||||
|
||||
// Only validate path if one is provided
|
||||
if (params.path) {
|
||||
try {
|
||||
resolveAndValidatePath(this.config, params.path);
|
||||
} catch (error) {
|
||||
return getErrorMessage(error);
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
@@ -43,6 +43,8 @@ describe('GrepTool', () => {
|
||||
getFileExclusions: () => ({
|
||||
getGlobExcludes: () => [],
|
||||
}),
|
||||
getTruncateToolOutputThreshold: () => 25000,
|
||||
getTruncateToolOutputLines: () => 1000,
|
||||
} as unknown as Config;
|
||||
|
||||
beforeEach(async () => {
|
||||
@@ -84,11 +86,11 @@ describe('GrepTool', () => {
|
||||
expect(grepTool.validateToolParams(params)).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for valid params (pattern, path, and include)', () => {
|
||||
it('should return null for valid params (pattern, path, and glob)', () => {
|
||||
const params: GrepToolParams = {
|
||||
pattern: 'hello',
|
||||
path: '.',
|
||||
include: '*.txt',
|
||||
glob: '*.txt',
|
||||
};
|
||||
expect(grepTool.validateToolParams(params)).toBeNull();
|
||||
});
|
||||
@@ -111,7 +113,7 @@ describe('GrepTool', () => {
|
||||
const params: GrepToolParams = { pattern: 'hello', path: 'nonexistent' };
|
||||
// Check for the core error message, as the full path might vary
|
||||
expect(grepTool.validateToolParams(params)).toContain(
|
||||
'Failed to access path stats for',
|
||||
'Path does not exist:',
|
||||
);
|
||||
expect(grepTool.validateToolParams(params)).toContain('nonexistent');
|
||||
});
|
||||
@@ -155,8 +157,8 @@ describe('GrepTool', () => {
|
||||
expect(result.returnDisplay).toBe('Found 1 match');
|
||||
});
|
||||
|
||||
it('should find matches with an include glob', async () => {
|
||||
const params: GrepToolParams = { pattern: 'hello', include: '*.js' };
|
||||
it('should find matches with a glob filter', async () => {
|
||||
const params: GrepToolParams = { pattern: 'hello', glob: '*.js' };
|
||||
const invocation = grepTool.build(params);
|
||||
const result = await invocation.execute(abortSignal);
|
||||
expect(result.llmContent).toContain(
|
||||
@@ -169,7 +171,7 @@ describe('GrepTool', () => {
|
||||
expect(result.returnDisplay).toBe('Found 1 match');
|
||||
});
|
||||
|
||||
it('should find matches with an include glob and path', async () => {
|
||||
it('should find matches with a glob filter and path', async () => {
|
||||
await fs.writeFile(
|
||||
path.join(tempRootDir, 'sub', 'another.js'),
|
||||
'const greeting = "hello";',
|
||||
@@ -177,7 +179,7 @@ describe('GrepTool', () => {
|
||||
const params: GrepToolParams = {
|
||||
pattern: 'hello',
|
||||
path: 'sub',
|
||||
include: '*.js',
|
||||
glob: '*.js',
|
||||
};
|
||||
const invocation = grepTool.build(params);
|
||||
const result = await invocation.execute(abortSignal);
|
||||
@@ -244,59 +246,23 @@ describe('GrepTool', () => {
|
||||
|
||||
describe('multi-directory workspace', () => {
|
||||
it('should search across all workspace directories when no path is specified', async () => {
|
||||
// Create additional directory with test files
|
||||
const secondDir = await fs.mkdtemp(
|
||||
path.join(os.tmpdir(), 'grep-tool-second-'),
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(secondDir, 'other.txt'),
|
||||
'hello from second directory\nworld in second',
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(secondDir, 'another.js'),
|
||||
'function world() { return "test"; }',
|
||||
);
|
||||
|
||||
// Create a mock config with multiple directories
|
||||
const multiDirConfig = {
|
||||
getTargetDir: () => tempRootDir,
|
||||
getWorkspaceContext: () =>
|
||||
createMockWorkspaceContext(tempRootDir, [secondDir]),
|
||||
getFileExclusions: () => ({
|
||||
getGlobExcludes: () => [],
|
||||
}),
|
||||
} as unknown as Config;
|
||||
|
||||
const multiDirGrepTool = new GrepTool(multiDirConfig);
|
||||
// The new implementation searches only in the target directory (first workspace directory)
|
||||
// when no path is specified, not across all workspace directories
|
||||
const params: GrepToolParams = { pattern: 'world' };
|
||||
const invocation = multiDirGrepTool.build(params);
|
||||
const invocation = grepTool.build(params);
|
||||
const result = await invocation.execute(abortSignal);
|
||||
|
||||
// Should find matches in both directories
|
||||
// Should find matches in the target directory only
|
||||
expect(result.llmContent).toContain(
|
||||
'Found 5 matches for pattern "world"',
|
||||
'Found 3 matches for pattern "world" in the workspace directory',
|
||||
);
|
||||
|
||||
// Matches from first directory
|
||||
// Matches from target directory
|
||||
expect(result.llmContent).toContain('fileA.txt');
|
||||
expect(result.llmContent).toContain('L1: hello world');
|
||||
expect(result.llmContent).toContain('L2: second line with world');
|
||||
expect(result.llmContent).toContain('fileC.txt');
|
||||
expect(result.llmContent).toContain('L1: another world in sub dir');
|
||||
|
||||
// Matches from second directory (with directory name prefix)
|
||||
const secondDirName = path.basename(secondDir);
|
||||
expect(result.llmContent).toContain(
|
||||
`File: ${path.join(secondDirName, 'other.txt')}`,
|
||||
);
|
||||
expect(result.llmContent).toContain('L2: world in second');
|
||||
expect(result.llmContent).toContain(
|
||||
`File: ${path.join(secondDirName, 'another.js')}`,
|
||||
);
|
||||
expect(result.llmContent).toContain('L1: function world()');
|
||||
|
||||
// Clean up
|
||||
await fs.rm(secondDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('should search only specified path within workspace directories', async () => {
|
||||
@@ -318,6 +284,8 @@ describe('GrepTool', () => {
|
||||
getFileExclusions: () => ({
|
||||
getGlobExcludes: () => [],
|
||||
}),
|
||||
getTruncateToolOutputThreshold: () => 25000,
|
||||
getTruncateToolOutputLines: () => 1000,
|
||||
} as unknown as Config;
|
||||
|
||||
const multiDirGrepTool = new GrepTool(multiDirConfig);
|
||||
@@ -346,16 +314,18 @@ describe('GrepTool', () => {
|
||||
it('should generate correct description with pattern only', () => {
|
||||
const params: GrepToolParams = { pattern: 'testPattern' };
|
||||
const invocation = grepTool.build(params);
|
||||
expect(invocation.getDescription()).toBe("'testPattern'");
|
||||
expect(invocation.getDescription()).toBe("'testPattern' in path './'");
|
||||
});
|
||||
|
||||
it('should generate correct description with pattern and include', () => {
|
||||
it('should generate correct description with pattern and glob', () => {
|
||||
const params: GrepToolParams = {
|
||||
pattern: 'testPattern',
|
||||
include: '*.ts',
|
||||
glob: '*.ts',
|
||||
};
|
||||
const invocation = grepTool.build(params);
|
||||
expect(invocation.getDescription()).toBe("'testPattern' in *.ts");
|
||||
expect(invocation.getDescription()).toBe(
|
||||
"'testPattern' in path './' (filter: '*.ts')",
|
||||
);
|
||||
});
|
||||
|
||||
it('should generate correct description with pattern and path', async () => {
|
||||
@@ -366,49 +336,37 @@ describe('GrepTool', () => {
|
||||
path: path.join('src', 'app'),
|
||||
};
|
||||
const invocation = grepTool.build(params);
|
||||
// The path will be relative to the tempRootDir, so we check for containment.
|
||||
expect(invocation.getDescription()).toContain("'testPattern' within");
|
||||
expect(invocation.getDescription()).toContain(path.join('src', 'app'));
|
||||
});
|
||||
|
||||
it('should indicate searching across all workspace directories when no path specified', () => {
|
||||
// Create a mock config with multiple directories
|
||||
const multiDirConfig = {
|
||||
getTargetDir: () => tempRootDir,
|
||||
getWorkspaceContext: () =>
|
||||
createMockWorkspaceContext(tempRootDir, ['/another/dir']),
|
||||
getFileExclusions: () => ({
|
||||
getGlobExcludes: () => [],
|
||||
}),
|
||||
} as unknown as Config;
|
||||
|
||||
const multiDirGrepTool = new GrepTool(multiDirConfig);
|
||||
const params: GrepToolParams = { pattern: 'testPattern' };
|
||||
const invocation = multiDirGrepTool.build(params);
|
||||
expect(invocation.getDescription()).toBe(
|
||||
"'testPattern' across all workspace directories",
|
||||
expect(invocation.getDescription()).toContain(
|
||||
"'testPattern' in path 'src",
|
||||
);
|
||||
expect(invocation.getDescription()).toContain("app'");
|
||||
});
|
||||
|
||||
it('should generate correct description with pattern, include, and path', async () => {
|
||||
it('should indicate searching workspace directory when no path specified', () => {
|
||||
const params: GrepToolParams = { pattern: 'testPattern' };
|
||||
const invocation = grepTool.build(params);
|
||||
expect(invocation.getDescription()).toBe("'testPattern' in path './'");
|
||||
});
|
||||
|
||||
it('should generate correct description with pattern, glob, and path', async () => {
|
||||
const dirPath = path.join(tempRootDir, 'src', 'app');
|
||||
await fs.mkdir(dirPath, { recursive: true });
|
||||
const params: GrepToolParams = {
|
||||
pattern: 'testPattern',
|
||||
include: '*.ts',
|
||||
glob: '*.ts',
|
||||
path: path.join('src', 'app'),
|
||||
};
|
||||
const invocation = grepTool.build(params);
|
||||
expect(invocation.getDescription()).toContain(
|
||||
"'testPattern' in *.ts within",
|
||||
"'testPattern' in path 'src",
|
||||
);
|
||||
expect(invocation.getDescription()).toContain(path.join('src', 'app'));
|
||||
expect(invocation.getDescription()).toContain("(filter: '*.ts')");
|
||||
});
|
||||
|
||||
it('should use ./ for root path in description', () => {
|
||||
const params: GrepToolParams = { pattern: 'testPattern', path: '.' };
|
||||
const invocation = grepTool.build(params);
|
||||
expect(invocation.getDescription()).toBe("'testPattern' within ./");
|
||||
expect(invocation.getDescription()).toBe("'testPattern' in path '.'");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -422,67 +380,50 @@ describe('GrepTool', () => {
|
||||
}
|
||||
});
|
||||
|
||||
it('should limit results to default 20 matches', async () => {
|
||||
it('should show all results when no limit is specified', async () => {
|
||||
const params: GrepToolParams = { pattern: 'testword' };
|
||||
const invocation = grepTool.build(params);
|
||||
const result = await invocation.execute(abortSignal);
|
||||
|
||||
expect(result.llmContent).toContain('Found 20 matches');
|
||||
expect(result.llmContent).toContain(
|
||||
'showing first 20 of 30+ total matches',
|
||||
);
|
||||
expect(result.llmContent).toContain('WARNING: Results truncated');
|
||||
expect(result.returnDisplay).toContain(
|
||||
'Found 20 matches (truncated from 30+)',
|
||||
);
|
||||
// New implementation shows all matches when limit is not specified
|
||||
expect(result.llmContent).toContain('Found 30 matches');
|
||||
expect(result.llmContent).not.toContain('truncated');
|
||||
expect(result.returnDisplay).toBe('Found 30 matches');
|
||||
});
|
||||
|
||||
it('should respect custom maxResults parameter', async () => {
|
||||
const params: GrepToolParams = { pattern: 'testword', maxResults: 5 };
|
||||
it('should respect custom limit parameter', async () => {
|
||||
const params: GrepToolParams = { pattern: 'testword', limit: 5 };
|
||||
const invocation = grepTool.build(params);
|
||||
const result = await invocation.execute(abortSignal);
|
||||
|
||||
expect(result.llmContent).toContain('Found 5 matches');
|
||||
expect(result.llmContent).toContain(
|
||||
'showing first 5 of 30+ total matches',
|
||||
);
|
||||
expect(result.llmContent).toContain('current: 5');
|
||||
expect(result.returnDisplay).toContain(
|
||||
'Found 5 matches (truncated from 30+)',
|
||||
);
|
||||
// Should find 30 total but limit to 5
|
||||
expect(result.llmContent).toContain('Found 30 matches');
|
||||
expect(result.llmContent).toContain('25 lines truncated');
|
||||
expect(result.returnDisplay).toContain('Found 30 matches (truncated)');
|
||||
});
|
||||
|
||||
it('should not show truncation warning when all results fit', async () => {
|
||||
const params: GrepToolParams = { pattern: 'testword', maxResults: 50 };
|
||||
const params: GrepToolParams = { pattern: 'testword', limit: 50 };
|
||||
const invocation = grepTool.build(params);
|
||||
const result = await invocation.execute(abortSignal);
|
||||
|
||||
expect(result.llmContent).toContain('Found 30 matches');
|
||||
expect(result.llmContent).not.toContain('WARNING: Results truncated');
|
||||
expect(result.llmContent).not.toContain('showing first');
|
||||
expect(result.llmContent).not.toContain('truncated');
|
||||
expect(result.returnDisplay).toBe('Found 30 matches');
|
||||
});
|
||||
|
||||
it('should validate maxResults parameter', () => {
|
||||
const invalidParams = [
|
||||
{ pattern: 'test', maxResults: 0 },
|
||||
{ pattern: 'test', maxResults: 101 },
|
||||
{ pattern: 'test', maxResults: -1 },
|
||||
{ pattern: 'test', maxResults: 1.5 },
|
||||
];
|
||||
|
||||
invalidParams.forEach((params) => {
|
||||
const error = grepTool.validateToolParams(params as GrepToolParams);
|
||||
expect(error).toBeTruthy(); // Just check that validation fails
|
||||
expect(error).toMatch(/maxResults|must be/); // Check it's about maxResults validation
|
||||
});
|
||||
it('should not validate limit parameter', () => {
|
||||
// limit parameter has no validation constraints in the new implementation
|
||||
const params = { pattern: 'test', limit: 5 };
|
||||
const error = grepTool.validateToolParams(params as GrepToolParams);
|
||||
expect(error).toBeNull();
|
||||
});
|
||||
|
||||
it('should accept valid maxResults parameter', () => {
|
||||
it('should accept valid limit parameter', () => {
|
||||
const validParams = [
|
||||
{ pattern: 'test', maxResults: 1 },
|
||||
{ pattern: 'test', maxResults: 50 },
|
||||
{ pattern: 'test', maxResults: 100 },
|
||||
{ pattern: 'test', limit: 1 },
|
||||
{ pattern: 'test', limit: 50 },
|
||||
{ pattern: 'test', limit: 100 },
|
||||
];
|
||||
|
||||
validParams.forEach((params) => {
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import fsPromises from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { EOL } from 'node:os';
|
||||
@@ -13,7 +12,7 @@ import { globStream } from 'glob';
|
||||
import type { ToolInvocation, ToolResult } from './tools.js';
|
||||
import { BaseDeclarativeTool, BaseToolInvocation, Kind } from './tools.js';
|
||||
import { ToolNames } from './tool-names.js';
|
||||
import { makeRelative, shortenPath } from '../utils/paths.js';
|
||||
import { resolveAndValidatePath } from '../utils/paths.js';
|
||||
import { getErrorMessage, isNodeError } from '../utils/errors.js';
|
||||
import { isGitRepository } from '../utils/gitUtils.js';
|
||||
import type { Config } from '../config/config.js';
|
||||
@@ -37,14 +36,14 @@ export interface GrepToolParams {
|
||||
path?: string;
|
||||
|
||||
/**
|
||||
* File pattern to include in the search (e.g. "*.js", "*.{ts,tsx}")
|
||||
* Glob pattern to filter files (e.g. "*.js", "*.{ts,tsx}")
|
||||
*/
|
||||
include?: string;
|
||||
glob?: string;
|
||||
|
||||
/**
|
||||
* Maximum number of matches to return (optional, defaults to 20)
|
||||
* Maximum number of matching lines to return (optional, shows all if not specified)
|
||||
*/
|
||||
maxResults?: number;
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -70,121 +69,60 @@ class GrepToolInvocation extends BaseToolInvocation<
|
||||
this.fileExclusions = config.getFileExclusions();
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a path is within the root directory and resolves it.
|
||||
* @param relativePath Path relative to the root directory (or undefined for root).
|
||||
* @returns The absolute path if valid and exists, or null if no path specified (to search all directories).
|
||||
* @throws {Error} If path is outside root, doesn't exist, or isn't a directory.
|
||||
*/
|
||||
private resolveAndValidatePath(relativePath?: string): string | null {
|
||||
// If no path specified, return null to indicate searching all workspace directories
|
||||
if (!relativePath) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const targetPath = path.resolve(this.config.getTargetDir(), relativePath);
|
||||
|
||||
// Security Check: Ensure the resolved path is within workspace boundaries
|
||||
const workspaceContext = this.config.getWorkspaceContext();
|
||||
if (!workspaceContext.isPathWithinWorkspace(targetPath)) {
|
||||
const directories = workspaceContext.getDirectories();
|
||||
throw new Error(
|
||||
`Path validation failed: Attempted path "${relativePath}" resolves outside the allowed workspace directories: ${directories.join(', ')}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Check existence and type after resolving
|
||||
try {
|
||||
const stats = fs.statSync(targetPath);
|
||||
if (!stats.isDirectory()) {
|
||||
throw new Error(`Path is not a directory: ${targetPath}`);
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
if (isNodeError(error) && error.code !== 'ENOENT') {
|
||||
throw new Error(`Path does not exist: ${targetPath}`);
|
||||
}
|
||||
throw new Error(
|
||||
`Failed to access path stats for ${targetPath}: ${error}`,
|
||||
);
|
||||
}
|
||||
|
||||
return targetPath;
|
||||
}
|
||||
|
||||
async execute(signal: AbortSignal): Promise<ToolResult> {
|
||||
try {
|
||||
const workspaceContext = this.config.getWorkspaceContext();
|
||||
const searchDirAbs = this.resolveAndValidatePath(this.params.path);
|
||||
// Default to target directory if no path is provided
|
||||
const searchDirAbs = resolveAndValidatePath(
|
||||
this.config,
|
||||
this.params.path,
|
||||
);
|
||||
const searchDirDisplay = this.params.path || '.';
|
||||
|
||||
// Determine which directories to search
|
||||
let searchDirectories: readonly string[];
|
||||
if (searchDirAbs === null) {
|
||||
// No path specified - search all workspace directories
|
||||
searchDirectories = workspaceContext.getDirectories();
|
||||
} else {
|
||||
// Specific path provided - search only that directory
|
||||
searchDirectories = [searchDirAbs];
|
||||
}
|
||||
// Perform grep search
|
||||
const rawMatches = await this.performGrepSearch({
|
||||
pattern: this.params.pattern,
|
||||
path: searchDirAbs,
|
||||
glob: this.params.glob,
|
||||
signal,
|
||||
});
|
||||
|
||||
// Collect matches from all search directories
|
||||
let allMatches: GrepMatch[] = [];
|
||||
const maxResults = this.params.maxResults ?? 20; // Default to 20 results
|
||||
let totalMatchesFound = 0;
|
||||
let searchTruncated = false;
|
||||
// Build search description
|
||||
const searchLocationDescription = this.params.path
|
||||
? `in path "${searchDirDisplay}"`
|
||||
: `in the workspace directory`;
|
||||
|
||||
for (const searchDir of searchDirectories) {
|
||||
const matches = await this.performGrepSearch({
|
||||
pattern: this.params.pattern,
|
||||
path: searchDir,
|
||||
include: this.params.include,
|
||||
signal,
|
||||
});
|
||||
const filterDescription = this.params.glob
|
||||
? ` (filter: "${this.params.glob}")`
|
||||
: '';
|
||||
|
||||
totalMatchesFound += matches.length;
|
||||
|
||||
// Add directory prefix if searching multiple directories
|
||||
if (searchDirectories.length > 1) {
|
||||
const dirName = path.basename(searchDir);
|
||||
matches.forEach((match) => {
|
||||
match.filePath = path.join(dirName, match.filePath);
|
||||
});
|
||||
}
|
||||
|
||||
// Apply result limiting
|
||||
const remainingSlots = maxResults - allMatches.length;
|
||||
if (remainingSlots <= 0) {
|
||||
searchTruncated = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if (matches.length > remainingSlots) {
|
||||
allMatches = allMatches.concat(matches.slice(0, remainingSlots));
|
||||
searchTruncated = true;
|
||||
break;
|
||||
} else {
|
||||
allMatches = allMatches.concat(matches);
|
||||
}
|
||||
}
|
||||
|
||||
let searchLocationDescription: string;
|
||||
if (searchDirAbs === null) {
|
||||
const numDirs = workspaceContext.getDirectories().length;
|
||||
searchLocationDescription =
|
||||
numDirs > 1
|
||||
? `across ${numDirs} workspace directories`
|
||||
: `in the workspace directory`;
|
||||
} else {
|
||||
searchLocationDescription = `in path "${searchDirDisplay}"`;
|
||||
}
|
||||
|
||||
if (allMatches.length === 0) {
|
||||
const noMatchMsg = `No matches found for pattern "${this.params.pattern}" ${searchLocationDescription}${this.params.include ? ` (filter: "${this.params.include}")` : ''}.`;
|
||||
// Check if we have any matches
|
||||
if (rawMatches.length === 0) {
|
||||
const noMatchMsg = `No matches found for pattern "${this.params.pattern}" ${searchLocationDescription}${filterDescription}.`;
|
||||
return { llmContent: noMatchMsg, returnDisplay: `No matches found` };
|
||||
}
|
||||
|
||||
const charLimit = this.config.getTruncateToolOutputThreshold();
|
||||
const lineLimit = Math.min(
|
||||
this.config.getTruncateToolOutputLines(),
|
||||
this.params.limit ?? Number.POSITIVE_INFINITY,
|
||||
);
|
||||
|
||||
// Apply line limit if specified
|
||||
let truncatedByLineLimit = false;
|
||||
let matchesToInclude = rawMatches;
|
||||
if (rawMatches.length > lineLimit) {
|
||||
matchesToInclude = rawMatches.slice(0, lineLimit);
|
||||
truncatedByLineLimit = true;
|
||||
}
|
||||
|
||||
const totalMatches = rawMatches.length;
|
||||
const matchTerm = totalMatches === 1 ? 'match' : 'matches';
|
||||
|
||||
// Build header
|
||||
const header = `Found ${totalMatches} ${matchTerm} for pattern "${this.params.pattern}" ${searchLocationDescription}${filterDescription}:\n---\n`;
|
||||
|
||||
// Group matches by file
|
||||
const matchesByFile = allMatches.reduce(
|
||||
const matchesByFile = matchesToInclude.reduce(
|
||||
(acc, match) => {
|
||||
const fileKey = match.filePath;
|
||||
if (!acc[fileKey]) {
|
||||
@@ -197,46 +135,51 @@ class GrepToolInvocation extends BaseToolInvocation<
|
||||
{} as Record<string, GrepMatch[]>,
|
||||
);
|
||||
|
||||
const matchCount = allMatches.length;
|
||||
const matchTerm = matchCount === 1 ? 'match' : 'matches';
|
||||
|
||||
// Build the header with truncation info if needed
|
||||
let headerText = `Found ${matchCount} ${matchTerm} for pattern "${this.params.pattern}" ${searchLocationDescription}${this.params.include ? ` (filter: "${this.params.include}")` : ''}`;
|
||||
|
||||
if (searchTruncated) {
|
||||
headerText += ` (showing first ${matchCount} of ${totalMatchesFound}+ total matches)`;
|
||||
}
|
||||
|
||||
let llmContent = `${headerText}:
|
||||
---
|
||||
`;
|
||||
|
||||
// Build grep output
|
||||
let grepOutput = '';
|
||||
for (const filePath in matchesByFile) {
|
||||
llmContent += `File: ${filePath}\n`;
|
||||
grepOutput += `File: ${filePath}\n`;
|
||||
matchesByFile[filePath].forEach((match) => {
|
||||
const trimmedLine = match.line.trim();
|
||||
llmContent += `L${match.lineNumber}: ${trimmedLine}\n`;
|
||||
grepOutput += `L${match.lineNumber}: ${trimmedLine}\n`;
|
||||
});
|
||||
llmContent += '---\n';
|
||||
grepOutput += '---\n';
|
||||
}
|
||||
|
||||
// Add truncation guidance if results were limited
|
||||
if (searchTruncated) {
|
||||
llmContent += `\nWARNING: Results truncated to prevent context overflow. To see more results:
|
||||
- Use a more specific pattern to reduce matches
|
||||
- Add file filters with the 'include' parameter (e.g., "*.js", "src/**")
|
||||
- Specify a narrower 'path' to search in a subdirectory
|
||||
- Increase 'maxResults' parameter if you need more matches (current: ${maxResults})`;
|
||||
// Apply character limit as safety net
|
||||
let truncatedByCharLimit = false;
|
||||
if (Number.isFinite(charLimit) && grepOutput.length > charLimit) {
|
||||
grepOutput = grepOutput.slice(0, charLimit) + '...';
|
||||
truncatedByCharLimit = true;
|
||||
}
|
||||
|
||||
let displayText = `Found ${matchCount} ${matchTerm}`;
|
||||
if (searchTruncated) {
|
||||
displayText += ` (truncated from ${totalMatchesFound}+)`;
|
||||
// Count how many lines we actually included after character truncation
|
||||
const finalLines = grepOutput
|
||||
.split('\n')
|
||||
.filter(
|
||||
(line) =>
|
||||
line.trim() && !line.startsWith('File:') && !line.startsWith('---'),
|
||||
);
|
||||
const includedLines = finalLines.length;
|
||||
|
||||
// Build result
|
||||
let llmContent = header + grepOutput;
|
||||
|
||||
// Add truncation notice if needed
|
||||
if (truncatedByLineLimit || truncatedByCharLimit) {
|
||||
const omittedMatches = totalMatches - includedLines;
|
||||
llmContent += ` [${omittedMatches} ${omittedMatches === 1 ? 'line' : 'lines'} truncated] ...`;
|
||||
}
|
||||
|
||||
// Build display message
|
||||
let displayMessage = `Found ${totalMatches} ${matchTerm}`;
|
||||
if (truncatedByLineLimit || truncatedByCharLimit) {
|
||||
displayMessage += ` (truncated)`;
|
||||
}
|
||||
|
||||
return {
|
||||
llmContent: llmContent.trim(),
|
||||
returnDisplay: displayText,
|
||||
returnDisplay: displayMessage,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error(`Error during GrepLogic execution: ${error}`);
|
||||
@@ -329,50 +272,26 @@ class GrepToolInvocation extends BaseToolInvocation<
|
||||
* @returns A string describing the grep
|
||||
*/
|
||||
getDescription(): string {
|
||||
let description = `'${this.params.pattern}'`;
|
||||
if (this.params.include) {
|
||||
description += ` in ${this.params.include}`;
|
||||
}
|
||||
if (this.params.path) {
|
||||
const resolvedPath = path.resolve(
|
||||
this.config.getTargetDir(),
|
||||
this.params.path,
|
||||
);
|
||||
if (
|
||||
resolvedPath === this.config.getTargetDir() ||
|
||||
this.params.path === '.'
|
||||
) {
|
||||
description += ` within ./`;
|
||||
} else {
|
||||
const relativePath = makeRelative(
|
||||
resolvedPath,
|
||||
this.config.getTargetDir(),
|
||||
);
|
||||
description += ` within ${shortenPath(relativePath)}`;
|
||||
}
|
||||
} else {
|
||||
// When no path is specified, indicate searching all workspace directories
|
||||
const workspaceContext = this.config.getWorkspaceContext();
|
||||
const directories = workspaceContext.getDirectories();
|
||||
if (directories.length > 1) {
|
||||
description += ` across all workspace directories`;
|
||||
}
|
||||
let description = `'${this.params.pattern}' in path '${this.params.path || './'}'`;
|
||||
if (this.params.glob) {
|
||||
description += ` (filter: '${this.params.glob}')`;
|
||||
}
|
||||
|
||||
return description;
|
||||
}
|
||||
|
||||
/**
|
||||
* Performs the actual search using the prioritized strategies.
|
||||
* @param options Search options including pattern, absolute path, and include glob.
|
||||
* @param options Search options including pattern, absolute path, and glob filter.
|
||||
* @returns A promise resolving to an array of match objects.
|
||||
*/
|
||||
private async performGrepSearch(options: {
|
||||
pattern: string;
|
||||
path: string; // Expects absolute path
|
||||
include?: string;
|
||||
glob?: string;
|
||||
signal: AbortSignal;
|
||||
}): Promise<GrepMatch[]> {
|
||||
const { pattern, path: absolutePath, include } = options;
|
||||
const { pattern, path: absolutePath, glob } = options;
|
||||
let strategyUsed = 'none';
|
||||
|
||||
try {
|
||||
@@ -390,8 +309,8 @@ class GrepToolInvocation extends BaseToolInvocation<
|
||||
'--ignore-case',
|
||||
pattern,
|
||||
];
|
||||
if (include) {
|
||||
gitArgs.push('--', include);
|
||||
if (glob) {
|
||||
gitArgs.push('--', glob);
|
||||
}
|
||||
|
||||
try {
|
||||
@@ -457,8 +376,8 @@ class GrepToolInvocation extends BaseToolInvocation<
|
||||
})
|
||||
.filter((dir): dir is string => !!dir);
|
||||
commonExcludes.forEach((dir) => grepArgs.push(`--exclude-dir=${dir}`));
|
||||
if (include) {
|
||||
grepArgs.push(`--include=${include}`);
|
||||
if (glob) {
|
||||
grepArgs.push(`--include=${glob}`);
|
||||
}
|
||||
grepArgs.push(pattern);
|
||||
grepArgs.push('.');
|
||||
@@ -537,7 +456,7 @@ class GrepToolInvocation extends BaseToolInvocation<
|
||||
'GrepLogic: Falling back to JavaScript grep implementation.',
|
||||
);
|
||||
strategyUsed = 'javascript fallback';
|
||||
const globPattern = include ? include : '**/*';
|
||||
const globPattern = glob ? glob : '**/*';
|
||||
const ignorePatterns = this.fileExclusions.getGlobExcludes();
|
||||
|
||||
const filesIterator = globStream(globPattern, {
|
||||
@@ -603,32 +522,30 @@ export class GrepTool extends BaseDeclarativeTool<GrepToolParams, ToolResult> {
|
||||
constructor(private readonly config: Config) {
|
||||
super(
|
||||
GrepTool.Name,
|
||||
'SearchText',
|
||||
'Searches for a regular expression pattern within the content of files in a specified directory (or current working directory). Can filter files by a glob pattern. Returns the lines containing matches, along with their file paths and line numbers.',
|
||||
'Grep',
|
||||
'A powerful search tool for finding patterns in files\n\n Usage:\n - ALWAYS use Grep for search tasks. NEVER invoke `grep` or `rg` as a Bash command. The Grep tool has been optimized for correct permissions and access.\n - Supports full regex syntax (e.g., "log.*Error", "function\\s+\\w+")\n - Filter files with glob parameter (e.g., "*.js", "**/*.tsx")\n - Case-insensitive by default\n - Use Task tool for open-ended searches requiring multiple rounds\n',
|
||||
Kind.Search,
|
||||
{
|
||||
properties: {
|
||||
pattern: {
|
||||
description:
|
||||
"The regular expression (regex) pattern to search for within file contents (e.g., 'function\\s+myFunction', 'import\\s+\\{.*\\}\\s+from\\s+.*').",
|
||||
type: 'string',
|
||||
description:
|
||||
'The regular expression pattern to search for in file contents',
|
||||
},
|
||||
glob: {
|
||||
type: 'string',
|
||||
description:
|
||||
'Glob pattern to filter files (e.g. "*.js", "*.{ts,tsx}")',
|
||||
},
|
||||
path: {
|
||||
description:
|
||||
'Optional: The absolute path to the directory to search within. If omitted, searches the current working directory.',
|
||||
type: 'string',
|
||||
},
|
||||
include: {
|
||||
description:
|
||||
"Optional: A glob pattern to filter which files are searched (e.g., '*.js', '*.{ts,tsx}', 'src/**'). If omitted, searches all files (respecting potential global ignores).",
|
||||
type: 'string',
|
||||
'File or directory to search in. Defaults to current working directory.',
|
||||
},
|
||||
maxResults: {
|
||||
description:
|
||||
'Optional: Maximum number of matches to return to prevent context overflow (default: 20, max: 100). Use lower values for broad searches, higher for specific searches.',
|
||||
limit: {
|
||||
type: 'number',
|
||||
minimum: 1,
|
||||
maximum: 100,
|
||||
description:
|
||||
'Limit output to first N matching lines. Optional - shows all matches if not specified.',
|
||||
},
|
||||
},
|
||||
required: ['pattern'],
|
||||
@@ -637,47 +554,6 @@ export class GrepTool extends BaseDeclarativeTool<GrepToolParams, ToolResult> {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a path is within the root directory and resolves it.
|
||||
* @param relativePath Path relative to the root directory (or undefined for root).
|
||||
* @returns The absolute path if valid and exists, or null if no path specified (to search all directories).
|
||||
* @throws {Error} If path is outside root, doesn't exist, or isn't a directory.
|
||||
*/
|
||||
private resolveAndValidatePath(relativePath?: string): string | null {
|
||||
// If no path specified, return null to indicate searching all workspace directories
|
||||
if (!relativePath) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const targetPath = path.resolve(this.config.getTargetDir(), relativePath);
|
||||
|
||||
// Security Check: Ensure the resolved path is within workspace boundaries
|
||||
const workspaceContext = this.config.getWorkspaceContext();
|
||||
if (!workspaceContext.isPathWithinWorkspace(targetPath)) {
|
||||
const directories = workspaceContext.getDirectories();
|
||||
throw new Error(
|
||||
`Path validation failed: Attempted path "${relativePath}" resolves outside the allowed workspace directories: ${directories.join(', ')}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Check existence and type after resolving
|
||||
try {
|
||||
const stats = fs.statSync(targetPath);
|
||||
if (!stats.isDirectory()) {
|
||||
throw new Error(`Path is not a directory: ${targetPath}`);
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
if (isNodeError(error) && error.code !== 'ENOENT') {
|
||||
throw new Error(`Path does not exist: ${targetPath}`);
|
||||
}
|
||||
throw new Error(
|
||||
`Failed to access path stats for ${targetPath}: ${error}`,
|
||||
);
|
||||
}
|
||||
|
||||
return targetPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates the parameters for the tool
|
||||
* @param params Parameters to validate
|
||||
@@ -686,27 +562,17 @@ export class GrepTool extends BaseDeclarativeTool<GrepToolParams, ToolResult> {
|
||||
protected override validateToolParamValues(
|
||||
params: GrepToolParams,
|
||||
): string | null {
|
||||
// Validate pattern is a valid regex
|
||||
try {
|
||||
new RegExp(params.pattern);
|
||||
} catch (error) {
|
||||
return `Invalid regular expression pattern provided: ${params.pattern}. Error: ${getErrorMessage(error)}`;
|
||||
}
|
||||
|
||||
// Validate maxResults if provided
|
||||
if (params.maxResults !== undefined) {
|
||||
if (
|
||||
!Number.isInteger(params.maxResults) ||
|
||||
params.maxResults < 1 ||
|
||||
params.maxResults > 100
|
||||
) {
|
||||
return `maxResults must be an integer between 1 and 100, got: ${params.maxResults}`;
|
||||
}
|
||||
return `Invalid regular expression pattern: ${params.pattern}. Error: ${getErrorMessage(error)}`;
|
||||
}
|
||||
|
||||
// Only validate path if one is provided
|
||||
if (params.path) {
|
||||
try {
|
||||
this.resolveAndValidatePath(params.path);
|
||||
resolveAndValidatePath(this.config, params.path);
|
||||
} catch (error) {
|
||||
return getErrorMessage(error);
|
||||
}
|
||||
|
||||
@@ -41,6 +41,8 @@ describe('ReadFileTool', () => {
|
||||
storage: {
|
||||
getProjectTempDir: () => path.join(tempRootDir, '.temp'),
|
||||
},
|
||||
getTruncateToolOutputThreshold: () => 2500,
|
||||
getTruncateToolOutputLines: () => 500,
|
||||
} as unknown as Config;
|
||||
tool = new ReadFileTool(mockConfigInstance);
|
||||
});
|
||||
@@ -281,11 +283,9 @@ describe('ReadFileTool', () => {
|
||||
>;
|
||||
|
||||
const result = await invocation.execute(abortSignal);
|
||||
expect(result.llmContent).toContain(
|
||||
'IMPORTANT: The file content has been truncated',
|
||||
expect(result.returnDisplay).toContain(
|
||||
'Read lines 1-2 of 3 from longlines.txt (truncated)',
|
||||
);
|
||||
expect(result.llmContent).toContain('--- FILE CONTENT (truncated) ---');
|
||||
expect(result.returnDisplay).toContain('some lines were shortened');
|
||||
});
|
||||
|
||||
it('should handle image file and return appropriate content', async () => {
|
||||
@@ -417,10 +417,7 @@ describe('ReadFileTool', () => {
|
||||
|
||||
const result = await invocation.execute(abortSignal);
|
||||
expect(result.llmContent).toContain(
|
||||
'IMPORTANT: The file content has been truncated',
|
||||
);
|
||||
expect(result.llmContent).toContain(
|
||||
'Status: Showing lines 6-8 of 20 total lines',
|
||||
'Showing lines 6-8 of 20 total lines',
|
||||
);
|
||||
expect(result.llmContent).toContain('Line 6');
|
||||
expect(result.llmContent).toContain('Line 7');
|
||||
|
||||
@@ -67,8 +67,7 @@ class ReadFileToolInvocation extends BaseToolInvocation<
|
||||
async execute(): Promise<ToolResult> {
|
||||
const result = await processSingleFileContent(
|
||||
this.params.absolute_path,
|
||||
this.config.getTargetDir(),
|
||||
this.config.getFileSystemService(),
|
||||
this.config,
|
||||
this.params.offset,
|
||||
this.params.limit,
|
||||
);
|
||||
@@ -88,16 +87,7 @@ class ReadFileToolInvocation extends BaseToolInvocation<
|
||||
if (result.isTruncated) {
|
||||
const [start, end] = result.linesShown!;
|
||||
const total = result.originalLineCount!;
|
||||
const nextOffset = this.params.offset
|
||||
? this.params.offset + end - start + 1
|
||||
: end;
|
||||
llmContent = `
|
||||
IMPORTANT: The file content has been truncated.
|
||||
Status: Showing lines ${start}-${end} of ${total} total lines.
|
||||
Action: To read more of the file, you can use the 'offset' and 'limit' parameters in a subsequent 'read_file' call. For example, to read the next section of the file, use offset: ${nextOffset}.
|
||||
|
||||
--- FILE CONTENT (truncated) ---
|
||||
${result.llmContent}`;
|
||||
llmContent = `Showing lines ${start}-${end} of ${total} total lines.\n\n---\n\n${result.llmContent}`;
|
||||
} else {
|
||||
llmContent = result.llmContent || '';
|
||||
}
|
||||
|
||||
@@ -88,6 +88,8 @@ describe('ReadManyFilesTool', () => {
|
||||
buildExcludePatterns: () => DEFAULT_FILE_EXCLUDES,
|
||||
getReadManyFilesExcludes: () => DEFAULT_FILE_EXCLUDES,
|
||||
}),
|
||||
getTruncateToolOutputThreshold: () => 2500,
|
||||
getTruncateToolOutputLines: () => 500,
|
||||
} as Partial<Config> as Config;
|
||||
tool = new ReadManyFilesTool(mockConfig);
|
||||
|
||||
@@ -500,6 +502,8 @@ describe('ReadManyFilesTool', () => {
|
||||
buildExcludePatterns: () => [],
|
||||
getReadManyFilesExcludes: () => [],
|
||||
}),
|
||||
getTruncateToolOutputThreshold: () => 2500,
|
||||
getTruncateToolOutputLines: () => 500,
|
||||
} as Partial<Config> as Config;
|
||||
tool = new ReadManyFilesTool(mockConfig);
|
||||
|
||||
@@ -552,15 +556,10 @@ describe('ReadManyFilesTool', () => {
|
||||
c.includes('large-file.txt'),
|
||||
);
|
||||
|
||||
expect(normalFileContent).not.toContain(
|
||||
'[WARNING: This file was truncated.',
|
||||
);
|
||||
expect(normalFileContent).not.toContain('Showing lines');
|
||||
expect(truncatedFileContent).toContain(
|
||||
"[WARNING: This file was truncated. To view the full content, use the 'read_file' tool on this specific file.]",
|
||||
'Showing lines 1-250 of 2500 total lines.',
|
||||
);
|
||||
// Check that the actual content is still there but truncated
|
||||
expect(truncatedFileContent).toContain('L200');
|
||||
expect(truncatedFileContent).not.toContain('L2400');
|
||||
});
|
||||
|
||||
it('should read files with special characters like [] and () in the path', async () => {
|
||||
|
||||
@@ -17,7 +17,6 @@ import {
|
||||
processSingleFileContent,
|
||||
DEFAULT_ENCODING,
|
||||
getSpecificMimeType,
|
||||
DEFAULT_MAX_LINES_TEXT_FILE,
|
||||
} from '../utils/fileUtils.js';
|
||||
import type { PartListUnion } from '@google/genai';
|
||||
import {
|
||||
@@ -278,8 +277,10 @@ ${finalExclusionPatternsForDescription
|
||||
}
|
||||
|
||||
const sortedFiles = Array.from(filesToConsider).sort();
|
||||
const file_line_limit =
|
||||
DEFAULT_MAX_LINES_TEXT_FILE / Math.max(1, sortedFiles.length);
|
||||
const truncateToolOutputLines = this.config.getTruncateToolOutputLines();
|
||||
const file_line_limit = Number.isFinite(truncateToolOutputLines)
|
||||
? Math.floor(truncateToolOutputLines / Math.max(1, sortedFiles.length))
|
||||
: undefined;
|
||||
|
||||
const fileProcessingPromises = sortedFiles.map(
|
||||
async (filePath): Promise<FileProcessingResult> => {
|
||||
@@ -316,8 +317,7 @@ ${finalExclusionPatternsForDescription
|
||||
// Use processSingleFileContent for all file types now
|
||||
const fileReadResult = await processSingleFileContent(
|
||||
filePath,
|
||||
this.config.getTargetDir(),
|
||||
this.config.getFileSystemService(),
|
||||
this.config,
|
||||
0,
|
||||
file_line_limit,
|
||||
);
|
||||
@@ -376,9 +376,12 @@ ${finalExclusionPatternsForDescription
|
||||
);
|
||||
let fileContentForLlm = '';
|
||||
if (fileReadResult.isTruncated) {
|
||||
fileContentForLlm += `[WARNING: This file was truncated. To view the full content, use the 'read_file' tool on this specific file.]\n\n`;
|
||||
const [start, end] = fileReadResult.linesShown!;
|
||||
const total = fileReadResult.originalLineCount!;
|
||||
fileContentForLlm = `Showing lines ${start}-${end} of ${total} total lines.\n---\n${fileReadResult.llmContent}`;
|
||||
} else {
|
||||
fileContentForLlm = fileReadResult.llmContent;
|
||||
}
|
||||
fileContentForLlm += fileReadResult.llmContent;
|
||||
contentParts.push(`${separator}\n\n${fileContentForLlm}\n\n`);
|
||||
} else {
|
||||
// This is a Part for image/pdf, which we don't add the separator to.
|
||||
|
||||
@@ -103,6 +103,8 @@ describe('RipGrepTool', () => {
|
||||
getWorkingDir: () => tempRootDir,
|
||||
getDebugMode: () => false,
|
||||
getUseBuiltinRipgrep: () => true,
|
||||
getTruncateToolOutputThreshold: () => 25000,
|
||||
getTruncateToolOutputLines: () => 1000,
|
||||
} as unknown as Config;
|
||||
|
||||
beforeEach(async () => {
|
||||
@@ -184,17 +186,15 @@ describe('RipGrepTool', () => {
|
||||
};
|
||||
// Check for the core error message, as the full path might vary
|
||||
expect(grepTool.validateToolParams(params)).toContain(
|
||||
'Failed to access path stats for',
|
||||
'Path does not exist:',
|
||||
);
|
||||
expect(grepTool.validateToolParams(params)).toContain('nonexistent');
|
||||
});
|
||||
|
||||
it('should return error if path is a file, not a directory', async () => {
|
||||
it('should allow path to be a file', () => {
|
||||
const filePath = path.join(tempRootDir, 'fileA.txt');
|
||||
const params: RipGrepToolParams = { pattern: 'hello', path: filePath };
|
||||
expect(grepTool.validateToolParams(params)).toContain(
|
||||
`Path is not a directory: ${filePath}`,
|
||||
);
|
||||
expect(grepTool.validateToolParams(params)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -419,7 +419,7 @@ describe('RipGrepTool', () => {
|
||||
});
|
||||
|
||||
it('should truncate llm content when exceeding maximum length', async () => {
|
||||
const longMatch = 'fileA.txt:1:' + 'a'.repeat(25_000);
|
||||
const longMatch = 'fileA.txt:1:' + 'a'.repeat(30_000);
|
||||
|
||||
mockSpawn.mockImplementationOnce(
|
||||
createMockSpawn({
|
||||
@@ -432,7 +432,7 @@ describe('RipGrepTool', () => {
|
||||
const invocation = grepTool.build(params);
|
||||
const result = await invocation.execute(abortSignal);
|
||||
|
||||
expect(String(result.llmContent).length).toBeLessThanOrEqual(20_000);
|
||||
expect(String(result.llmContent).length).toBeLessThanOrEqual(26_000);
|
||||
expect(result.llmContent).toMatch(/\[\d+ lines? truncated\] \.\.\./);
|
||||
expect(result.returnDisplay).toContain('truncated');
|
||||
});
|
||||
@@ -567,6 +567,26 @@ describe('RipGrepTool', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('should search within a single file when path is a file', async () => {
|
||||
mockSpawn.mockImplementationOnce(
|
||||
createMockSpawn({
|
||||
outputData: `fileA.txt:1:hello world${EOL}fileA.txt:2:second line with world${EOL}`,
|
||||
exitCode: 0,
|
||||
}),
|
||||
);
|
||||
|
||||
const params: RipGrepToolParams = {
|
||||
pattern: 'world',
|
||||
path: path.join(tempRootDir, 'fileA.txt'),
|
||||
};
|
||||
const invocation = grepTool.build(params);
|
||||
const result = await invocation.execute(abortSignal);
|
||||
expect(result.llmContent).toContain('Found 2 matches');
|
||||
expect(result.llmContent).toContain('fileA.txt:1:hello world');
|
||||
expect(result.llmContent).toContain('fileA.txt:2:second line with world');
|
||||
expect(result.returnDisplay).toBe('Found 2 matches');
|
||||
});
|
||||
|
||||
it('should throw an error if ripgrep is not available', async () => {
|
||||
// Make ensureRipgrepBinary throw
|
||||
(ensureRipgrepPath as Mock).mockRejectedValue(
|
||||
@@ -648,7 +668,9 @@ describe('RipGrepTool', () => {
|
||||
describe('error handling and edge cases', () => {
|
||||
it('should handle workspace boundary violations', () => {
|
||||
const params: RipGrepToolParams = { pattern: 'test', path: '../outside' };
|
||||
expect(() => grepTool.build(params)).toThrow(/Path validation failed/);
|
||||
expect(() => grepTool.build(params)).toThrow(
|
||||
/Path is not within workspace/,
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle empty directories gracefully', async () => {
|
||||
@@ -1132,7 +1154,9 @@ describe('RipGrepTool', () => {
|
||||
glob: '*.ts',
|
||||
};
|
||||
const invocation = grepTool.build(params);
|
||||
expect(invocation.getDescription()).toBe("'testPattern' in *.ts");
|
||||
expect(invocation.getDescription()).toBe(
|
||||
"'testPattern' (filter: '*.ts')",
|
||||
);
|
||||
});
|
||||
|
||||
it('should generate correct description with pattern and path', async () => {
|
||||
@@ -1143,9 +1167,10 @@ describe('RipGrepTool', () => {
|
||||
path: path.join('src', 'app'),
|
||||
};
|
||||
const invocation = grepTool.build(params);
|
||||
// The path will be relative to the tempRootDir, so we check for containment.
|
||||
expect(invocation.getDescription()).toContain("'testPattern' within");
|
||||
expect(invocation.getDescription()).toContain(path.join('src', 'app'));
|
||||
expect(invocation.getDescription()).toContain(
|
||||
"'testPattern' in path 'src",
|
||||
);
|
||||
expect(invocation.getDescription()).toContain("app'");
|
||||
});
|
||||
|
||||
it('should generate correct description with default search path', () => {
|
||||
@@ -1164,15 +1189,15 @@ describe('RipGrepTool', () => {
|
||||
};
|
||||
const invocation = grepTool.build(params);
|
||||
expect(invocation.getDescription()).toContain(
|
||||
"'testPattern' in *.ts within",
|
||||
"'testPattern' in path 'src",
|
||||
);
|
||||
expect(invocation.getDescription()).toContain(path.join('src', 'app'));
|
||||
expect(invocation.getDescription()).toContain("(filter: '*.ts')");
|
||||
});
|
||||
|
||||
it('should use ./ for root path in description', () => {
|
||||
it('should use path when specified in description', () => {
|
||||
const params: RipGrepToolParams = { pattern: 'testPattern', path: '.' };
|
||||
const invocation = grepTool.build(params);
|
||||
expect(invocation.getDescription()).toBe("'testPattern' within ./");
|
||||
expect(invocation.getDescription()).toBe("'testPattern' in path '.'");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -11,16 +11,14 @@ import { spawn } from 'node:child_process';
|
||||
import type { ToolInvocation, ToolResult } from './tools.js';
|
||||
import { BaseDeclarativeTool, BaseToolInvocation, Kind } from './tools.js';
|
||||
import { ToolNames } from './tool-names.js';
|
||||
import { makeRelative, shortenPath } from '../utils/paths.js';
|
||||
import { getErrorMessage, isNodeError } from '../utils/errors.js';
|
||||
import { resolveAndValidatePath } from '../utils/paths.js';
|
||||
import { getErrorMessage } from '../utils/errors.js';
|
||||
import type { Config } from '../config/config.js';
|
||||
import { ensureRipgrepPath } from '../utils/ripgrepUtils.js';
|
||||
import { SchemaValidator } from '../utils/schemaValidator.js';
|
||||
import type { FileFilteringOptions } from '../config/constants.js';
|
||||
import { DEFAULT_FILE_FILTERING_OPTIONS } from '../config/constants.js';
|
||||
|
||||
const MAX_LLM_CONTENT_LENGTH = 20_000;
|
||||
|
||||
/**
|
||||
* Parameters for the GrepTool (Simplified)
|
||||
*/
|
||||
@@ -57,50 +55,13 @@ class GrepToolInvocation extends BaseToolInvocation<
|
||||
super(params);
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a path is within the root directory and resolves it.
|
||||
* @param relativePath Path relative to the root directory (or undefined for root).
|
||||
* @returns The absolute path to search within.
|
||||
* @throws {Error} If path is outside root, doesn't exist, or isn't a directory.
|
||||
*/
|
||||
private resolveAndValidatePath(relativePath?: string): string {
|
||||
const targetDir = this.config.getTargetDir();
|
||||
const targetPath = relativePath
|
||||
? path.resolve(targetDir, relativePath)
|
||||
: targetDir;
|
||||
|
||||
const workspaceContext = this.config.getWorkspaceContext();
|
||||
if (!workspaceContext.isPathWithinWorkspace(targetPath)) {
|
||||
const directories = workspaceContext.getDirectories();
|
||||
throw new Error(
|
||||
`Path validation failed: Attempted path "${relativePath}" resolves outside the allowed workspace directories: ${directories.join(', ')}`,
|
||||
);
|
||||
}
|
||||
|
||||
return this.ensureDirectory(targetPath);
|
||||
}
|
||||
|
||||
private ensureDirectory(targetPath: string): string {
|
||||
try {
|
||||
const stats = fs.statSync(targetPath);
|
||||
if (!stats.isDirectory()) {
|
||||
throw new Error(`Path is not a directory: ${targetPath}`);
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
if (isNodeError(error) && error.code !== 'ENOENT') {
|
||||
throw new Error(`Path does not exist: ${targetPath}`);
|
||||
}
|
||||
throw new Error(
|
||||
`Failed to access path stats for ${targetPath}: ${error}`,
|
||||
);
|
||||
}
|
||||
|
||||
return targetPath;
|
||||
}
|
||||
|
||||
async execute(signal: AbortSignal): Promise<ToolResult> {
|
||||
try {
|
||||
const searchDirAbs = this.resolveAndValidatePath(this.params.path);
|
||||
const searchDirAbs = resolveAndValidatePath(
|
||||
this.config,
|
||||
this.params.path,
|
||||
{ allowFiles: true },
|
||||
);
|
||||
const searchDirDisplay = this.params.path || '.';
|
||||
|
||||
// Get raw ripgrep output
|
||||
@@ -133,34 +94,50 @@ class GrepToolInvocation extends BaseToolInvocation<
|
||||
|
||||
// Build header early to calculate available space
|
||||
const header = `Found ${totalMatches} ${matchTerm} for pattern "${this.params.pattern}" ${searchLocationDescription}${filterDescription}:\n---\n`;
|
||||
const maxTruncationNoticeLength = 100; // "[... N more matches truncated]"
|
||||
const maxGrepOutputLength =
|
||||
MAX_LLM_CONTENT_LENGTH - header.length - maxTruncationNoticeLength;
|
||||
|
||||
const charLimit = this.config.getTruncateToolOutputThreshold();
|
||||
const lineLimit = Math.min(
|
||||
this.config.getTruncateToolOutputLines(),
|
||||
this.params.limit ?? Number.POSITIVE_INFINITY,
|
||||
);
|
||||
|
||||
// Apply line limit first (if specified)
|
||||
let truncatedByLineLimit = false;
|
||||
let linesToInclude = allLines;
|
||||
if (
|
||||
this.params.limit !== undefined &&
|
||||
allLines.length > this.params.limit
|
||||
) {
|
||||
linesToInclude = allLines.slice(0, this.params.limit);
|
||||
if (allLines.length > lineLimit) {
|
||||
linesToInclude = allLines.slice(0, lineLimit);
|
||||
truncatedByLineLimit = true;
|
||||
}
|
||||
|
||||
// Join lines back into grep output
|
||||
let grepOutput = linesToInclude.join(EOL);
|
||||
|
||||
// Apply character limit as safety net
|
||||
// Build output and track how many lines we include, respecting character limit
|
||||
let grepOutput = '';
|
||||
let truncatedByCharLimit = false;
|
||||
if (grepOutput.length > maxGrepOutputLength) {
|
||||
grepOutput = grepOutput.slice(0, maxGrepOutputLength) + '...';
|
||||
truncatedByCharLimit = true;
|
||||
}
|
||||
let includedLines = 0;
|
||||
if (Number.isFinite(charLimit)) {
|
||||
const parts: string[] = [];
|
||||
let currentLength = 0;
|
||||
|
||||
// Count how many lines we actually included after character truncation
|
||||
const finalLines = grepOutput.split(EOL).filter((line) => line.trim());
|
||||
const includedLines = finalLines.length;
|
||||
for (const line of linesToInclude) {
|
||||
const sep = includedLines > 0 ? 1 : 0;
|
||||
includedLines++;
|
||||
|
||||
const projectedLength = currentLength + line.length + sep;
|
||||
if (projectedLength <= charLimit) {
|
||||
parts.push(line);
|
||||
currentLength = projectedLength;
|
||||
} else {
|
||||
const remaining = Math.max(charLimit - currentLength - sep, 10);
|
||||
parts.push(line.slice(0, remaining) + '...');
|
||||
truncatedByCharLimit = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
grepOutput = parts.join('\n');
|
||||
} else {
|
||||
grepOutput = linesToInclude.join('\n');
|
||||
includedLines = linesToInclude.length;
|
||||
}
|
||||
|
||||
// Build result
|
||||
let llmContent = header + grepOutput;
|
||||
@@ -168,7 +145,7 @@ class GrepToolInvocation extends BaseToolInvocation<
|
||||
// Add truncation notice if needed
|
||||
if (truncatedByLineLimit || truncatedByCharLimit) {
|
||||
const omittedMatches = totalMatches - includedLines;
|
||||
llmContent += ` [${omittedMatches} ${omittedMatches === 1 ? 'line' : 'lines'} truncated] ...`;
|
||||
llmContent += `\n---\n[${omittedMatches} ${omittedMatches === 1 ? 'line' : 'lines'} truncated] ...`;
|
||||
}
|
||||
|
||||
// Build display message (show real count, not truncated)
|
||||
@@ -193,7 +170,7 @@ class GrepToolInvocation extends BaseToolInvocation<
|
||||
|
||||
private async performRipgrepSearch(options: {
|
||||
pattern: string;
|
||||
path: string;
|
||||
path: string; // Can be a file or directory
|
||||
glob?: string;
|
||||
signal: AbortSignal;
|
||||
}): Promise<string> {
|
||||
@@ -302,34 +279,13 @@ class GrepToolInvocation extends BaseToolInvocation<
|
||||
*/
|
||||
getDescription(): string {
|
||||
let description = `'${this.params.pattern}'`;
|
||||
if (this.params.glob) {
|
||||
description += ` in ${this.params.glob}`;
|
||||
}
|
||||
if (this.params.path) {
|
||||
const resolvedPath = path.resolve(
|
||||
this.config.getTargetDir(),
|
||||
this.params.path,
|
||||
);
|
||||
if (
|
||||
resolvedPath === this.config.getTargetDir() ||
|
||||
this.params.path === '.'
|
||||
) {
|
||||
description += ` within ./`;
|
||||
} else {
|
||||
const relativePath = makeRelative(
|
||||
resolvedPath,
|
||||
this.config.getTargetDir(),
|
||||
);
|
||||
description += ` within ${shortenPath(relativePath)}`;
|
||||
}
|
||||
} else {
|
||||
// When no path is specified, indicate searching all workspace directories
|
||||
const workspaceContext = this.config.getWorkspaceContext();
|
||||
const directories = workspaceContext.getDirectories();
|
||||
if (directories.length > 1) {
|
||||
description += ` across all workspace directories`;
|
||||
}
|
||||
description += ` in path '${this.params.path}'`;
|
||||
}
|
||||
if (this.params.glob) {
|
||||
description += ` (filter: '${this.params.glob}')`;
|
||||
}
|
||||
|
||||
return description;
|
||||
}
|
||||
}
|
||||
@@ -378,47 +334,6 @@ export class RipGrepTool extends BaseDeclarativeTool<
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a path is within the root directory and resolves it.
|
||||
* @param relativePath Path relative to the root directory (or undefined for root).
|
||||
* @returns The absolute path to search within.
|
||||
* @throws {Error} If path is outside root, doesn't exist, or isn't a directory.
|
||||
*/
|
||||
private resolveAndValidatePath(relativePath?: string): string {
|
||||
// If no path specified, search within the workspace root directory
|
||||
if (!relativePath) {
|
||||
return this.config.getTargetDir();
|
||||
}
|
||||
|
||||
const targetPath = path.resolve(this.config.getTargetDir(), relativePath);
|
||||
|
||||
// Security Check: Ensure the resolved path is within workspace boundaries
|
||||
const workspaceContext = this.config.getWorkspaceContext();
|
||||
if (!workspaceContext.isPathWithinWorkspace(targetPath)) {
|
||||
const directories = workspaceContext.getDirectories();
|
||||
throw new Error(
|
||||
`Path validation failed: Attempted path "${relativePath}" resolves outside the allowed workspace directories: ${directories.join(', ')}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Check existence and type after resolving
|
||||
try {
|
||||
const stats = fs.statSync(targetPath);
|
||||
if (!stats.isDirectory()) {
|
||||
throw new Error(`Path is not a directory: ${targetPath}`);
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
if (isNodeError(error) && error.code !== 'ENOENT') {
|
||||
throw new Error(`Path does not exist: ${targetPath}`);
|
||||
}
|
||||
throw new Error(
|
||||
`Failed to access path stats for ${targetPath}: ${error}`,
|
||||
);
|
||||
}
|
||||
|
||||
return targetPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates the parameters for the tool
|
||||
* @param params Parameters to validate
|
||||
@@ -445,7 +360,7 @@ export class RipGrepTool extends BaseDeclarativeTool<
|
||||
// Only validate path if one is provided
|
||||
if (params.path) {
|
||||
try {
|
||||
this.resolveAndValidatePath(params.path);
|
||||
resolveAndValidatePath(this.config, params.path, { allowFiles: true });
|
||||
} catch (error) {
|
||||
return getErrorMessage(error);
|
||||
}
|
||||
|
||||
@@ -21,4 +21,6 @@ export const ToolNames = {
|
||||
MEMORY: 'save_memory',
|
||||
TASK: 'task',
|
||||
EXIT_PLAN_MODE: 'exit_plan_mode',
|
||||
WEB_FETCH: 'web_fetch',
|
||||
WEB_SEARCH: 'web_search',
|
||||
} as const;
|
||||
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
ToolConfirmationOutcome,
|
||||
} from './tools.js';
|
||||
import { DEFAULT_QWEN_MODEL } from '../config/models.js';
|
||||
import { ToolNames } from './tool-names.js';
|
||||
|
||||
const URL_FETCH_TIMEOUT_MS = 10000;
|
||||
const MAX_CONTENT_LENGTH = 100000;
|
||||
@@ -190,7 +191,7 @@ export class WebFetchTool extends BaseDeclarativeTool<
|
||||
WebFetchToolParams,
|
||||
ToolResult
|
||||
> {
|
||||
static readonly Name: string = 'web_fetch';
|
||||
static readonly Name: string = ToolNames.WEB_FETCH;
|
||||
|
||||
constructor(private readonly config: Config) {
|
||||
super(
|
||||
|
||||
@@ -30,6 +30,7 @@ import type {
|
||||
WebSearchProviderConfig,
|
||||
DashScopeProviderConfig,
|
||||
} from './types.js';
|
||||
import { ToolNames } from '../tool-names.js';
|
||||
|
||||
class WebSearchToolInvocation extends BaseToolInvocation<
|
||||
WebSearchToolParams,
|
||||
@@ -274,7 +275,7 @@ export class WebSearchTool extends BaseDeclarativeTool<
|
||||
WebSearchToolParams,
|
||||
WebSearchToolResult
|
||||
> {
|
||||
static readonly Name: string = 'web_search';
|
||||
static readonly Name: string = ToolNames.WEB_SEARCH;
|
||||
|
||||
constructor(private readonly config: Config) {
|
||||
super(
|
||||
|
||||
@@ -13,9 +13,11 @@ import {
|
||||
afterEach,
|
||||
type Mock,
|
||||
} from 'vitest';
|
||||
import type { Content } from '@google/genai';
|
||||
import {
|
||||
getEnvironmentContext,
|
||||
getDirectoryContextString,
|
||||
getInitialChatHistory,
|
||||
} from './environmentContext.js';
|
||||
import type { Config } from '../config/config.js';
|
||||
import { getFolderStructure } from './getFolderStructure.js';
|
||||
@@ -213,3 +215,102 @@ describe('getEnvironmentContext', () => {
|
||||
expect(parts[1].text).toBe('\n--- Error reading full file context ---');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getInitialChatHistory', () => {
|
||||
let mockConfig: Partial<Config>;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.mocked(getFolderStructure).mockResolvedValue('Mock Folder Structure');
|
||||
mockConfig = {
|
||||
getSkipStartupContext: vi.fn().mockReturnValue(false),
|
||||
getWorkspaceContext: vi.fn().mockReturnValue({
|
||||
getDirectories: vi.fn().mockReturnValue(['/test/dir']),
|
||||
}),
|
||||
getFileService: vi.fn(),
|
||||
getFullContext: vi.fn().mockReturnValue(false),
|
||||
getToolRegistry: vi.fn().mockReturnValue({ getTool: vi.fn() }),
|
||||
};
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.clearAllMocks();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('includes startup context when skipStartupContext is false', async () => {
|
||||
const history = await getInitialChatHistory(mockConfig as Config);
|
||||
|
||||
expect(mockConfig.getSkipStartupContext).toHaveBeenCalled();
|
||||
expect(history).toHaveLength(2);
|
||||
expect(history).toEqual([
|
||||
expect.objectContaining({
|
||||
role: 'user',
|
||||
parts: [
|
||||
expect.objectContaining({
|
||||
text: expect.stringContaining(
|
||||
"I'm currently working in the directory",
|
||||
),
|
||||
}),
|
||||
],
|
||||
}),
|
||||
{
|
||||
role: 'model',
|
||||
parts: [{ text: 'Got it. Thanks for the context!' }],
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('returns only extra history when skipStartupContext is true', async () => {
|
||||
mockConfig.getSkipStartupContext = vi.fn().mockReturnValue(true);
|
||||
mockConfig.getWorkspaceContext = vi.fn(() => {
|
||||
throw new Error(
|
||||
'getWorkspaceContext should not be called when skipping startup context',
|
||||
);
|
||||
});
|
||||
mockConfig.getFullContext = vi.fn(() => {
|
||||
throw new Error(
|
||||
'getFullContext should not be called when skipping startup context',
|
||||
);
|
||||
});
|
||||
mockConfig.getToolRegistry = vi.fn(() => {
|
||||
throw new Error(
|
||||
'getToolRegistry should not be called when skipping startup context',
|
||||
);
|
||||
});
|
||||
const extraHistory: Content[] = [
|
||||
{ role: 'user', parts: [{ text: 'custom context' }] },
|
||||
];
|
||||
|
||||
const history = await getInitialChatHistory(
|
||||
mockConfig as Config,
|
||||
extraHistory,
|
||||
);
|
||||
|
||||
expect(mockConfig.getSkipStartupContext).toHaveBeenCalled();
|
||||
expect(history).toEqual(extraHistory);
|
||||
expect(history).not.toBe(extraHistory);
|
||||
});
|
||||
|
||||
it('returns empty history when skipping startup context without extras', async () => {
|
||||
mockConfig.getSkipStartupContext = vi.fn().mockReturnValue(true);
|
||||
mockConfig.getWorkspaceContext = vi.fn(() => {
|
||||
throw new Error(
|
||||
'getWorkspaceContext should not be called when skipping startup context',
|
||||
);
|
||||
});
|
||||
mockConfig.getFullContext = vi.fn(() => {
|
||||
throw new Error(
|
||||
'getFullContext should not be called when skipping startup context',
|
||||
);
|
||||
});
|
||||
mockConfig.getToolRegistry = vi.fn(() => {
|
||||
throw new Error(
|
||||
'getToolRegistry should not be called when skipping startup context',
|
||||
);
|
||||
});
|
||||
|
||||
const history = await getInitialChatHistory(mockConfig as Config);
|
||||
|
||||
expect(history).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -112,6 +112,10 @@ export async function getInitialChatHistory(
|
||||
config: Config,
|
||||
extraHistory?: Content[],
|
||||
): Promise<Content[]> {
|
||||
if (config.getSkipStartupContext()) {
|
||||
return extraHistory ? [...extraHistory] : [];
|
||||
}
|
||||
|
||||
const envParts = await getEnvironmentContext(config);
|
||||
const envContextString = envParts.map((part) => part.text || '').join('\n\n');
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ import {
|
||||
readFileWithEncoding,
|
||||
fileExists,
|
||||
} from './fileUtils.js';
|
||||
import { StandardFileSystemService } from '../services/fileSystemService.js';
|
||||
import type { Config } from '../config/config.js';
|
||||
|
||||
vi.mock('mime/lite', () => ({
|
||||
default: { getType: vi.fn() },
|
||||
@@ -50,6 +50,12 @@ describe('fileUtils', () => {
|
||||
let nonexistentFilePath: string;
|
||||
let directoryPath: string;
|
||||
|
||||
const mockConfig = {
|
||||
getTruncateToolOutputThreshold: () => 2500,
|
||||
getTruncateToolOutputLines: () => 500,
|
||||
getTargetDir: () => tempRootDir,
|
||||
} as unknown as Config;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks(); // Reset all mocks, including mime.getType
|
||||
|
||||
@@ -664,8 +670,7 @@ describe('fileUtils', () => {
|
||||
actualNodeFs.writeFileSync(testTextFilePath, content);
|
||||
const result = await processSingleFileContent(
|
||||
testTextFilePath,
|
||||
tempRootDir,
|
||||
new StandardFileSystemService(),
|
||||
mockConfig,
|
||||
);
|
||||
expect(result.llmContent).toBe(content);
|
||||
expect(result.returnDisplay).toBe('');
|
||||
@@ -675,8 +680,7 @@ describe('fileUtils', () => {
|
||||
it('should handle file not found', async () => {
|
||||
const result = await processSingleFileContent(
|
||||
nonexistentFilePath,
|
||||
tempRootDir,
|
||||
new StandardFileSystemService(),
|
||||
mockConfig,
|
||||
);
|
||||
expect(result.error).toContain('File not found');
|
||||
expect(result.returnDisplay).toContain('File not found');
|
||||
@@ -689,8 +693,7 @@ describe('fileUtils', () => {
|
||||
|
||||
const result = await processSingleFileContent(
|
||||
testTextFilePath,
|
||||
tempRootDir,
|
||||
new StandardFileSystemService(),
|
||||
mockConfig,
|
||||
);
|
||||
expect(result.error).toContain('Simulated read error');
|
||||
expect(result.returnDisplay).toContain('Simulated read error');
|
||||
@@ -704,8 +707,7 @@ describe('fileUtils', () => {
|
||||
|
||||
const result = await processSingleFileContent(
|
||||
testImageFilePath,
|
||||
tempRootDir,
|
||||
new StandardFileSystemService(),
|
||||
mockConfig,
|
||||
);
|
||||
expect(result.error).toContain('Simulated image read error');
|
||||
expect(result.returnDisplay).toContain('Simulated image read error');
|
||||
@@ -717,8 +719,7 @@ describe('fileUtils', () => {
|
||||
mockMimeGetType.mockReturnValue('image/png');
|
||||
const result = await processSingleFileContent(
|
||||
testImageFilePath,
|
||||
tempRootDir,
|
||||
new StandardFileSystemService(),
|
||||
mockConfig,
|
||||
);
|
||||
expect(
|
||||
(result.llmContent as { inlineData: unknown }).inlineData,
|
||||
@@ -739,8 +740,7 @@ describe('fileUtils', () => {
|
||||
mockMimeGetType.mockReturnValue('application/pdf');
|
||||
const result = await processSingleFileContent(
|
||||
testPdfFilePath,
|
||||
tempRootDir,
|
||||
new StandardFileSystemService(),
|
||||
mockConfig,
|
||||
);
|
||||
expect(
|
||||
(result.llmContent as { inlineData: unknown }).inlineData,
|
||||
@@ -768,8 +768,7 @@ describe('fileUtils', () => {
|
||||
|
||||
const result = await processSingleFileContent(
|
||||
testSvgFilePath,
|
||||
tempRootDir,
|
||||
new StandardFileSystemService(),
|
||||
mockConfig,
|
||||
);
|
||||
|
||||
expect(result.llmContent).toBe(svgContent);
|
||||
@@ -786,8 +785,7 @@ describe('fileUtils', () => {
|
||||
|
||||
const result = await processSingleFileContent(
|
||||
testBinaryFilePath,
|
||||
tempRootDir,
|
||||
new StandardFileSystemService(),
|
||||
mockConfig,
|
||||
);
|
||||
expect(result.llmContent).toContain(
|
||||
'Cannot display content of binary file',
|
||||
@@ -796,11 +794,7 @@ describe('fileUtils', () => {
|
||||
});
|
||||
|
||||
it('should handle path being a directory', async () => {
|
||||
const result = await processSingleFileContent(
|
||||
directoryPath,
|
||||
tempRootDir,
|
||||
new StandardFileSystemService(),
|
||||
);
|
||||
const result = await processSingleFileContent(directoryPath, mockConfig);
|
||||
expect(result.error).toContain('Path is a directory');
|
||||
expect(result.returnDisplay).toContain('Path is a directory');
|
||||
});
|
||||
@@ -811,8 +805,7 @@ describe('fileUtils', () => {
|
||||
|
||||
const result = await processSingleFileContent(
|
||||
testTextFilePath,
|
||||
tempRootDir,
|
||||
new StandardFileSystemService(),
|
||||
mockConfig,
|
||||
5,
|
||||
5,
|
||||
); // Read lines 6-10
|
||||
@@ -832,8 +825,7 @@ describe('fileUtils', () => {
|
||||
// Read from line 11 to 20. The start is not 0, so it's truncated.
|
||||
const result = await processSingleFileContent(
|
||||
testTextFilePath,
|
||||
tempRootDir,
|
||||
new StandardFileSystemService(),
|
||||
mockConfig,
|
||||
10,
|
||||
10,
|
||||
);
|
||||
@@ -852,8 +844,7 @@ describe('fileUtils', () => {
|
||||
|
||||
const result = await processSingleFileContent(
|
||||
testTextFilePath,
|
||||
tempRootDir,
|
||||
new StandardFileSystemService(),
|
||||
mockConfig,
|
||||
0,
|
||||
10,
|
||||
);
|
||||
@@ -875,17 +866,16 @@ describe('fileUtils', () => {
|
||||
|
||||
const result = await processSingleFileContent(
|
||||
testTextFilePath,
|
||||
tempRootDir,
|
||||
new StandardFileSystemService(),
|
||||
mockConfig,
|
||||
);
|
||||
|
||||
expect(result.llmContent).toContain('Short line');
|
||||
expect(result.llmContent).toContain(
|
||||
longLine.substring(0, 2000) + '... [truncated]',
|
||||
);
|
||||
expect(result.llmContent).toContain('Another short line');
|
||||
expect(result.llmContent).not.toContain('Another short line');
|
||||
expect(result.returnDisplay).toBe(
|
||||
'Read all 3 lines from test.txt (some lines were shortened)',
|
||||
'Read lines 1-2 of 3 from test.txt (truncated)',
|
||||
);
|
||||
expect(result.isTruncated).toBe(true);
|
||||
});
|
||||
@@ -897,8 +887,7 @@ describe('fileUtils', () => {
|
||||
// Read 5 lines, but there are 11 total
|
||||
const result = await processSingleFileContent(
|
||||
testTextFilePath,
|
||||
tempRootDir,
|
||||
new StandardFileSystemService(),
|
||||
mockConfig,
|
||||
0,
|
||||
5,
|
||||
);
|
||||
@@ -916,15 +905,14 @@ describe('fileUtils', () => {
|
||||
// Read all 11 lines, including the long one
|
||||
const result = await processSingleFileContent(
|
||||
testTextFilePath,
|
||||
tempRootDir,
|
||||
new StandardFileSystemService(),
|
||||
mockConfig,
|
||||
0,
|
||||
11,
|
||||
);
|
||||
|
||||
expect(result.isTruncated).toBe(true);
|
||||
expect(result.returnDisplay).toBe(
|
||||
'Read all 11 lines from test.txt (some lines were shortened)',
|
||||
'Read lines 1-11 of 11 from test.txt (truncated)',
|
||||
);
|
||||
});
|
||||
|
||||
@@ -942,14 +930,13 @@ describe('fileUtils', () => {
|
||||
// Read 10 lines out of 20, including the long line
|
||||
const result = await processSingleFileContent(
|
||||
testTextFilePath,
|
||||
tempRootDir,
|
||||
new StandardFileSystemService(),
|
||||
mockConfig,
|
||||
0,
|
||||
10,
|
||||
);
|
||||
expect(result.isTruncated).toBe(true);
|
||||
expect(result.returnDisplay).toBe(
|
||||
'Read lines 1-10 of 20 from test.txt (some lines were shortened)',
|
||||
'Read lines 1-5 of 20 from test.txt (truncated)',
|
||||
);
|
||||
});
|
||||
|
||||
@@ -966,8 +953,7 @@ describe('fileUtils', () => {
|
||||
try {
|
||||
const result = await processSingleFileContent(
|
||||
testTextFilePath,
|
||||
tempRootDir,
|
||||
new StandardFileSystemService(),
|
||||
mockConfig,
|
||||
);
|
||||
|
||||
expect(result.error).toContain('File size exceeds the 20MB limit');
|
||||
|
||||
@@ -9,13 +9,9 @@ import fsPromises from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import type { PartUnion } from '@google/genai';
|
||||
import mime from 'mime/lite';
|
||||
import type { FileSystemService } from '../services/fileSystemService.js';
|
||||
import { ToolErrorType } from '../tools/tool-error.js';
|
||||
import { BINARY_EXTENSIONS } from './ignorePatterns.js';
|
||||
|
||||
// Constants for text file processing
|
||||
export const DEFAULT_MAX_LINES_TEXT_FILE = 2000;
|
||||
const MAX_LINE_LENGTH_TEXT_FILE = 2000;
|
||||
import type { Config } from '../config/config.js';
|
||||
|
||||
// Default values for encoding and separator format
|
||||
export const DEFAULT_ENCODING: BufferEncoding = 'utf-8';
|
||||
@@ -306,18 +302,18 @@ export interface ProcessedFileReadResult {
|
||||
/**
|
||||
* Reads and processes a single file, handling text, images, and PDFs.
|
||||
* @param filePath Absolute path to the file.
|
||||
* @param rootDirectory Absolute path to the project root for relative path display.
|
||||
* @param config Config instance for truncation settings.
|
||||
* @param offset Optional offset for text files (0-based line number).
|
||||
* @param limit Optional limit for text files (number of lines to read).
|
||||
* @returns ProcessedFileReadResult object.
|
||||
*/
|
||||
export async function processSingleFileContent(
|
||||
filePath: string,
|
||||
rootDirectory: string,
|
||||
fileSystemService: FileSystemService,
|
||||
config: Config,
|
||||
offset?: number,
|
||||
limit?: number,
|
||||
): Promise<ProcessedFileReadResult> {
|
||||
const rootDirectory = config.getTargetDir();
|
||||
try {
|
||||
if (!fs.existsSync(filePath)) {
|
||||
// Sync check is acceptable before async read
|
||||
@@ -379,45 +375,76 @@ export async function processSingleFileContent(
|
||||
case 'text': {
|
||||
// Use BOM-aware reader to avoid leaving a BOM character in content and to support UTF-16/32 transparently
|
||||
const content = await readFileWithEncoding(filePath);
|
||||
const lines = content.split('\n');
|
||||
const lines = content.split('\n').map((line) => line.trimEnd());
|
||||
const originalLineCount = lines.length;
|
||||
|
||||
const startLine = offset || 0;
|
||||
const effectiveLimit =
|
||||
limit === undefined ? DEFAULT_MAX_LINES_TEXT_FILE : limit;
|
||||
const configLineLimit = config.getTruncateToolOutputLines();
|
||||
const configCharLimit = config.getTruncateToolOutputThreshold();
|
||||
const effectiveLimit = limit === undefined ? configLineLimit : limit;
|
||||
|
||||
// Ensure endLine does not exceed originalLineCount
|
||||
const endLine = Math.min(startLine + effectiveLimit, originalLineCount);
|
||||
// Ensure selectedLines doesn't try to slice beyond array bounds if startLine is too high
|
||||
const actualStartLine = Math.min(startLine, originalLineCount);
|
||||
const selectedLines = lines.slice(actualStartLine, endLine);
|
||||
|
||||
let linesWereTruncatedInLength = false;
|
||||
const formattedLines = selectedLines.map((line) => {
|
||||
if (line.length > MAX_LINE_LENGTH_TEXT_FILE) {
|
||||
linesWereTruncatedInLength = true;
|
||||
return (
|
||||
line.substring(0, MAX_LINE_LENGTH_TEXT_FILE) + '... [truncated]'
|
||||
);
|
||||
// Apply character limit truncation
|
||||
let llmContent = '';
|
||||
let contentLengthTruncated = false;
|
||||
let linesIncluded = 0;
|
||||
|
||||
if (Number.isFinite(configCharLimit)) {
|
||||
const formattedLines: string[] = [];
|
||||
let currentLength = 0;
|
||||
|
||||
for (const line of selectedLines) {
|
||||
const sep = linesIncluded > 0 ? 1 : 0; // newline separator
|
||||
linesIncluded++;
|
||||
|
||||
const projectedLength = currentLength + line.length + sep;
|
||||
if (projectedLength <= configCharLimit) {
|
||||
formattedLines.push(line);
|
||||
currentLength = projectedLength;
|
||||
} else {
|
||||
// Truncate the current line to fit
|
||||
const remaining = Math.max(
|
||||
configCharLimit - currentLength - sep,
|
||||
10,
|
||||
);
|
||||
formattedLines.push(
|
||||
line.substring(0, remaining) + '... [truncated]',
|
||||
);
|
||||
contentLengthTruncated = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return line;
|
||||
});
|
||||
|
||||
llmContent = formattedLines.join('\n');
|
||||
} else {
|
||||
// No character limit, use all selected lines
|
||||
llmContent = selectedLines.join('\n');
|
||||
linesIncluded = selectedLines.length;
|
||||
}
|
||||
|
||||
// Calculate actual end line shown
|
||||
const actualEndLine = contentLengthTruncated
|
||||
? actualStartLine + linesIncluded
|
||||
: endLine;
|
||||
|
||||
const contentRangeTruncated =
|
||||
startLine > 0 || endLine < originalLineCount;
|
||||
const isTruncated = contentRangeTruncated || linesWereTruncatedInLength;
|
||||
const llmContent = formattedLines.join('\n');
|
||||
startLine > 0 || actualEndLine < originalLineCount;
|
||||
const isTruncated = contentRangeTruncated || contentLengthTruncated;
|
||||
|
||||
// By default, return nothing to streamline the common case of a successful read_file.
|
||||
let returnDisplay = '';
|
||||
if (contentRangeTruncated) {
|
||||
if (isTruncated) {
|
||||
returnDisplay = `Read lines ${
|
||||
actualStartLine + 1
|
||||
}-${endLine} of ${originalLineCount} from ${relativePathForDisplay}`;
|
||||
if (linesWereTruncatedInLength) {
|
||||
returnDisplay += ' (some lines were shortened)';
|
||||
}-${actualEndLine} of ${originalLineCount} from ${relativePathForDisplay}`;
|
||||
if (contentLengthTruncated) {
|
||||
returnDisplay += ' (truncated)';
|
||||
}
|
||||
} else if (linesWereTruncatedInLength) {
|
||||
returnDisplay = `Read all ${originalLineCount} lines from ${relativePathForDisplay} (some lines were shortened)`;
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -425,7 +452,7 @@ export async function processSingleFileContent(
|
||||
returnDisplay,
|
||||
isTruncated,
|
||||
originalLineCount,
|
||||
linesShown: [actualStartLine + 1, endLine],
|
||||
linesShown: [actualStartLine + 1, actualEndLine],
|
||||
};
|
||||
}
|
||||
case 'image':
|
||||
|
||||
@@ -29,6 +29,8 @@ const createMockConfig = (
|
||||
getTargetDir: () => cwd,
|
||||
getFileSystemService: () => fileSystemService,
|
||||
getFileService: () => mockFileService,
|
||||
getTruncateToolOutputThreshold: () => 2500,
|
||||
getTruncateToolOutputLines: () => 500,
|
||||
} as unknown as Config;
|
||||
};
|
||||
|
||||
|
||||
@@ -83,11 +83,7 @@ export async function readPathFromWorkspace(
|
||||
for (const filePath of finalFiles) {
|
||||
const relativePathForDisplay = path.relative(absolutePath, filePath);
|
||||
allParts.push({ text: `--- ${relativePathForDisplay} ---\n` });
|
||||
const result = await processSingleFileContent(
|
||||
filePath,
|
||||
config.getTargetDir(),
|
||||
config.getFileSystemService(),
|
||||
);
|
||||
const result = await processSingleFileContent(filePath, config);
|
||||
allParts.push(result.llmContent);
|
||||
allParts.push({ text: '\n' }); // Add a newline for separation
|
||||
}
|
||||
@@ -108,11 +104,7 @@ export async function readPathFromWorkspace(
|
||||
}
|
||||
|
||||
// It's a single file, process it directly.
|
||||
const result = await processSingleFileContent(
|
||||
absolutePath,
|
||||
config.getTargetDir(),
|
||||
config.getFileSystemService(),
|
||||
);
|
||||
const result = await processSingleFileContent(absolutePath, config);
|
||||
return [result.llmContent];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,8 +4,53 @@
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import { escapePath, unescapePath, isSubpath } from './paths.js';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
|
||||
import {
|
||||
escapePath,
|
||||
resolvePath,
|
||||
validatePath,
|
||||
resolveAndValidatePath,
|
||||
unescapePath,
|
||||
isSubpath,
|
||||
} from './paths.js';
|
||||
import type { Config } from '../config/config.js';
|
||||
|
||||
function createConfigStub({
|
||||
targetDir,
|
||||
allowedDirectories,
|
||||
}: {
|
||||
targetDir: string;
|
||||
allowedDirectories: string[];
|
||||
}): Config {
|
||||
const resolvedTargetDir = path.resolve(targetDir);
|
||||
const resolvedDirectories = allowedDirectories.map((dir) =>
|
||||
path.resolve(dir),
|
||||
);
|
||||
|
||||
const workspaceContext = {
|
||||
isPathWithinWorkspace(testPath: string) {
|
||||
const resolvedPath = path.resolve(testPath);
|
||||
return resolvedDirectories.some((dir) => {
|
||||
const relative = path.relative(dir, resolvedPath);
|
||||
return (
|
||||
relative === '' ||
|
||||
(!relative.startsWith('..') && !path.isAbsolute(relative))
|
||||
);
|
||||
});
|
||||
},
|
||||
getDirectories() {
|
||||
return resolvedDirectories;
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
getTargetDir: () => resolvedTargetDir,
|
||||
getWorkspaceContext: () => workspaceContext,
|
||||
} as unknown as Config;
|
||||
}
|
||||
|
||||
describe('escapePath', () => {
|
||||
it('should escape spaces', () => {
|
||||
@@ -314,3 +359,240 @@ describe('isSubpath on Windows', () => {
|
||||
expect(isSubpath('Users\\Test\\file.txt', 'Users\\Test')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolvePath', () => {
|
||||
it('resolves relative paths against the provided base directory', () => {
|
||||
const result = resolvePath('/home/user/project', 'src/main.ts');
|
||||
expect(result).toBe(path.resolve('/home/user/project', 'src/main.ts'));
|
||||
});
|
||||
|
||||
it('resolves relative paths against cwd when baseDir is undefined', () => {
|
||||
const cwd = process.cwd();
|
||||
const result = resolvePath(undefined, 'src/main.ts');
|
||||
expect(result).toBe(path.resolve(cwd, 'src/main.ts'));
|
||||
});
|
||||
|
||||
it('returns absolute paths unchanged', () => {
|
||||
const absolutePath = '/absolute/path/to/file.ts';
|
||||
const result = resolvePath('/some/base', absolutePath);
|
||||
expect(result).toBe(absolutePath);
|
||||
});
|
||||
|
||||
it('expands tilde to home directory', () => {
|
||||
const homeDir = os.homedir();
|
||||
const result = resolvePath(undefined, '~');
|
||||
expect(result).toBe(homeDir);
|
||||
});
|
||||
|
||||
it('expands tilde-prefixed paths to home directory', () => {
|
||||
const homeDir = os.homedir();
|
||||
const result = resolvePath(undefined, '~/documents/file.txt');
|
||||
expect(result).toBe(path.join(homeDir, 'documents/file.txt'));
|
||||
});
|
||||
|
||||
it('uses baseDir when provided for relative paths', () => {
|
||||
const baseDir = '/custom/base';
|
||||
const result = resolvePath(baseDir, './relative/path');
|
||||
expect(result).toBe(path.resolve(baseDir, './relative/path'));
|
||||
});
|
||||
|
||||
it('handles tilde expansion regardless of baseDir', () => {
|
||||
const homeDir = os.homedir();
|
||||
const result = resolvePath('/some/base', '~/file.txt');
|
||||
expect(result).toBe(path.join(homeDir, 'file.txt'));
|
||||
});
|
||||
|
||||
it('handles dot paths correctly', () => {
|
||||
const result = resolvePath('/base/dir', '.');
|
||||
expect(result).toBe(path.resolve('/base/dir', '.'));
|
||||
});
|
||||
|
||||
it('handles parent directory references', () => {
|
||||
const result = resolvePath('/base/dir/subdir', '..');
|
||||
expect(result).toBe(path.resolve('/base/dir/subdir', '..'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('validatePath', () => {
|
||||
let workspaceRoot: string;
|
||||
let config: Config;
|
||||
|
||||
beforeAll(() => {
|
||||
workspaceRoot = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), 'validate-path-test-'),
|
||||
);
|
||||
fs.mkdirSync(path.join(workspaceRoot, 'subdir'));
|
||||
config = createConfigStub({
|
||||
targetDir: workspaceRoot,
|
||||
allowedDirectories: [workspaceRoot],
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
fs.rmSync(workspaceRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('validates paths within workspace boundaries', () => {
|
||||
const validPath = path.join(workspaceRoot, 'subdir');
|
||||
expect(() => validatePath(config, validPath)).not.toThrow();
|
||||
});
|
||||
|
||||
it('throws when path is outside workspace boundaries', () => {
|
||||
const outsidePath = path.join(os.tmpdir(), 'outside');
|
||||
expect(() => validatePath(config, outsidePath)).toThrowError(
|
||||
/Path is not within workspace/,
|
||||
);
|
||||
});
|
||||
|
||||
it('throws when path does not exist', () => {
|
||||
const nonExistentPath = path.join(workspaceRoot, 'nonexistent');
|
||||
expect(() => validatePath(config, nonExistentPath)).toThrowError(
|
||||
/Path does not exist:/,
|
||||
);
|
||||
});
|
||||
|
||||
it('throws when path is a file, not a directory (default behavior)', () => {
|
||||
const filePath = path.join(workspaceRoot, 'test-file.txt');
|
||||
fs.writeFileSync(filePath, 'content');
|
||||
try {
|
||||
expect(() => validatePath(config, filePath)).toThrowError(
|
||||
/Path is not a directory/,
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(filePath);
|
||||
}
|
||||
});
|
||||
|
||||
it('allows files when allowFiles option is true', () => {
|
||||
const filePath = path.join(workspaceRoot, 'test-file.txt');
|
||||
fs.writeFileSync(filePath, 'content');
|
||||
try {
|
||||
expect(() =>
|
||||
validatePath(config, filePath, { allowFiles: true }),
|
||||
).not.toThrow();
|
||||
} finally {
|
||||
fs.rmSync(filePath);
|
||||
}
|
||||
});
|
||||
|
||||
it('validates paths at workspace root', () => {
|
||||
expect(() => validatePath(config, workspaceRoot)).not.toThrow();
|
||||
});
|
||||
|
||||
it('validates paths in allowed directories', () => {
|
||||
const extraDir = fs.mkdtempSync(path.join(os.tmpdir(), 'validate-extra-'));
|
||||
try {
|
||||
const configWithExtra = createConfigStub({
|
||||
targetDir: workspaceRoot,
|
||||
allowedDirectories: [workspaceRoot, extraDir],
|
||||
});
|
||||
expect(() => validatePath(configWithExtra, extraDir)).not.toThrow();
|
||||
} finally {
|
||||
fs.rmSync(extraDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveAndValidatePath', () => {
|
||||
let workspaceRoot: string;
|
||||
let config: Config;
|
||||
|
||||
beforeAll(() => {
|
||||
workspaceRoot = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), 'resolve-and-validate-'),
|
||||
);
|
||||
fs.mkdirSync(path.join(workspaceRoot, 'subdir'));
|
||||
config = createConfigStub({
|
||||
targetDir: workspaceRoot,
|
||||
allowedDirectories: [workspaceRoot],
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
fs.rmSync(workspaceRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('returns the target directory when no path is provided', () => {
|
||||
expect(resolveAndValidatePath(config)).toBe(workspaceRoot);
|
||||
});
|
||||
|
||||
it('resolves relative paths within the workspace', () => {
|
||||
const expected = path.join(workspaceRoot, 'subdir');
|
||||
expect(resolveAndValidatePath(config, 'subdir')).toBe(expected);
|
||||
});
|
||||
|
||||
it('allows absolute paths that are permitted by the workspace context', () => {
|
||||
const extraDir = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), 'resolve-and-validate-extra-'),
|
||||
);
|
||||
try {
|
||||
const configWithExtra = createConfigStub({
|
||||
targetDir: workspaceRoot,
|
||||
allowedDirectories: [workspaceRoot, extraDir],
|
||||
});
|
||||
expect(resolveAndValidatePath(configWithExtra, extraDir)).toBe(extraDir);
|
||||
} finally {
|
||||
fs.rmSync(extraDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('expands tilde-prefixed paths using the home directory', () => {
|
||||
const fakeHome = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), 'resolve-and-validate-home-'),
|
||||
);
|
||||
const homeSubdir = path.join(fakeHome, 'project');
|
||||
fs.mkdirSync(homeSubdir);
|
||||
|
||||
const homedirSpy = vi.spyOn(os, 'homedir').mockReturnValue(fakeHome);
|
||||
try {
|
||||
const configWithHome = createConfigStub({
|
||||
targetDir: workspaceRoot,
|
||||
allowedDirectories: [workspaceRoot, fakeHome],
|
||||
});
|
||||
expect(resolveAndValidatePath(configWithHome, '~/project')).toBe(
|
||||
homeSubdir,
|
||||
);
|
||||
expect(resolveAndValidatePath(configWithHome, '~')).toBe(fakeHome);
|
||||
} finally {
|
||||
homedirSpy.mockRestore();
|
||||
fs.rmSync(fakeHome, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('throws when the path resolves outside of the workspace', () => {
|
||||
expect(() => resolveAndValidatePath(config, '../outside')).toThrowError(
|
||||
/Path is not within workspace/,
|
||||
);
|
||||
});
|
||||
|
||||
it('throws when the path does not exist', () => {
|
||||
expect(() => resolveAndValidatePath(config, 'missing')).toThrowError(
|
||||
/Path does not exist:/,
|
||||
);
|
||||
});
|
||||
|
||||
it('throws when the path points to a file (default behavior)', () => {
|
||||
const filePath = path.join(workspaceRoot, 'file.txt');
|
||||
fs.writeFileSync(filePath, 'content');
|
||||
try {
|
||||
expect(() => resolveAndValidatePath(config, 'file.txt')).toThrowError(
|
||||
`Path is not a directory: ${filePath}`,
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(filePath);
|
||||
}
|
||||
});
|
||||
|
||||
it('allows file paths when allowFiles option is true', () => {
|
||||
const filePath = path.join(workspaceRoot, 'file.txt');
|
||||
fs.writeFileSync(filePath, 'content');
|
||||
try {
|
||||
const result = resolveAndValidatePath(config, 'file.txt', {
|
||||
allowFiles: true,
|
||||
});
|
||||
expect(result).toBe(filePath);
|
||||
} finally {
|
||||
fs.rmSync(filePath);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -4,9 +4,12 @@
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import * as crypto from 'node:crypto';
|
||||
import type { Config } from '../config/config.js';
|
||||
import { isNodeError } from './errors.js';
|
||||
|
||||
export const QWEN_DIR = '.qwen';
|
||||
export const GOOGLE_ACCOUNTS_FILENAME = 'google_accounts.json';
|
||||
@@ -191,3 +194,93 @@ export function isSubpath(parentPath: string, childPath: string): boolean {
|
||||
!pathModule.isAbsolute(relative)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves a path with tilde (~) expansion and relative path resolution.
|
||||
* Handles tilde expansion for home directory and resolves relative paths
|
||||
* against the provided base directory or current working directory.
|
||||
*
|
||||
* @param baseDir The base directory to resolve relative paths against (defaults to current working directory)
|
||||
* @param relativePath The path to resolve (can be relative, absolute, or tilde-prefixed)
|
||||
* @returns The resolved absolute path
|
||||
*/
|
||||
export function resolvePath(
|
||||
baseDir: string | undefined = process.cwd(),
|
||||
relativePath: string,
|
||||
): string {
|
||||
const homeDir = os.homedir();
|
||||
|
||||
if (relativePath === '~') {
|
||||
return homeDir;
|
||||
} else if (relativePath.startsWith('~/')) {
|
||||
return path.join(homeDir, relativePath.slice(2));
|
||||
} else if (path.isAbsolute(relativePath)) {
|
||||
return relativePath;
|
||||
} else {
|
||||
return path.resolve(baseDir, relativePath);
|
||||
}
|
||||
}
|
||||
|
||||
export interface PathValidationOptions {
|
||||
/**
|
||||
* If true, allows both files and directories. If false (default), only allows directories.
|
||||
*/
|
||||
allowFiles?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates that a resolved path exists within the workspace boundaries.
|
||||
*
|
||||
* @param config The configuration object containing workspace context
|
||||
* @param resolvedPath The absolute path to validate
|
||||
* @param options Validation options
|
||||
* @throws Error if the path is outside workspace boundaries, doesn't exist, or is not a directory (when allowFiles is false)
|
||||
*/
|
||||
export function validatePath(
|
||||
config: Config,
|
||||
resolvedPath: string,
|
||||
options: PathValidationOptions = {},
|
||||
): void {
|
||||
const { allowFiles = false } = options;
|
||||
const workspaceContext = config.getWorkspaceContext();
|
||||
|
||||
if (!workspaceContext.isPathWithinWorkspace(resolvedPath)) {
|
||||
throw new Error('Path is not within workspace');
|
||||
}
|
||||
|
||||
try {
|
||||
const stats = fs.statSync(resolvedPath);
|
||||
if (!allowFiles && !stats.isDirectory()) {
|
||||
throw new Error(`Path is not a directory: ${resolvedPath}`);
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
if (isNodeError(error) && error.code === 'ENOENT') {
|
||||
throw new Error(`Path does not exist: ${resolvedPath}`);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves a path relative to the workspace root and verifies that it exists
|
||||
* within the workspace boundaries defined in the config.
|
||||
*
|
||||
* @param config The configuration object
|
||||
* @param relativePath The relative path to resolve (optional, defaults to target directory)
|
||||
* @param options Validation options (e.g., allowFiles to permit file paths)
|
||||
*/
|
||||
export function resolveAndValidatePath(
|
||||
config: Config,
|
||||
relativePath?: string,
|
||||
options: PathValidationOptions = {},
|
||||
): string {
|
||||
const targetDir = config.getTargetDir();
|
||||
|
||||
if (!relativePath) {
|
||||
return targetDir;
|
||||
}
|
||||
|
||||
const resolvedPath = resolvePath(targetDir, relativePath);
|
||||
validatePath(config, resolvedPath, options);
|
||||
return resolvedPath;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user