abbyswag/qwen-code
1
0
Fork 0
mirror of https://github.com/QwenLM/qwen-code.git synced 2025-12-21 01:07:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:QwenLM/qwen-code into mingholy/feat/cli-sdk

Browse Source
This commit is contained in:
mingholy.lmh
2025-11-21 16:19:55 +08:00
parent e7f61d879e 83fc321e15
commit 2fe6ba7c56
134 changed files with 5644 additions and 2393 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/core/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "@qwen-code/qwen-code-core",
"version": "0.2.2",
"version": "0.3.0",
"description": "Qwen Code Core",
"repository": {
"type": "git",

6
packages/core/src/config/config.test.ts
View File

@@ -1085,7 +1085,7 @@ describe('setApprovalMode with folder trust', () => {
expect.any(RipgrepFallbackEvent),
);
const event = (logRipgrepFallback as Mock).mock.calls[0][1];
expect(event.error).toContain('Ripgrep is not available');
expect(event.error).toContain('ripgrep is not available');
});
it('should fall back to GrepTool and log error when useRipgrep is true and builtin ripgrep is not available', async () => {
@@ -1109,7 +1109,7 @@ describe('setApprovalMode with folder trust', () => {
expect.any(RipgrepFallbackEvent),
);
const event = (logRipgrepFallback as Mock).mock.calls[0][1];
expect(event.error).toContain('Ripgrep is not available');
expect(event.error).toContain('ripgrep is not available');
});
it('should fall back to GrepTool and log error when canUseRipgrep throws an error', async () => {
@@ -1133,7 +1133,7 @@ describe('setApprovalMode with folder trust', () => {
expect.any(RipgrepFallbackEvent),
);
const event = (logRipgrepFallback as Mock).mock.calls[0][1];
expect(event.error).toBe(String(error));
expect(event.error).toBe(`ripGrep check failed`);
});
it('should register GrepTool when useRipgrep is false', async () => {

44
packages/core/src/config/config.ts
View File

@@ -82,6 +82,7 @@ import { shouldAttemptBrowserLaunch } from '../utils/browser.js';
import { FileExclusions } from '../utils/ignorePatterns.js';
import { WorkspaceContext } from '../utils/workspaceContext.js';
import { isToolEnabled, type ToolName } from '../utils/tool-utils.js';
import { getErrorMessage } from '../utils/errors.js';
// Local config modules
import type { FileFilteringOptions } from './constants.js';
@@ -281,7 +282,6 @@ export interface ConfigParameters {
skipNextSpeakerCheck?: boolean;
shellExecutionConfig?: ShellExecutionConfig;
extensionManagement?: boolean;
enablePromptCompletion?: boolean;
skipLoopDetection?: boolean;
vlmSwitchMode?: string;
truncateToolOutputThreshold?: number;
@@ -293,6 +293,27 @@ export interface ConfigParameters {
inputFormat?: InputFormat;
outputFormat?: OutputFormat;
skipStartupContext?: boolean;
inputFormat?: InputFormat;
outputFormat?: OutputFormat;
}
function normalizeConfigOutputFormat(
format: OutputFormat | undefined,
): OutputFormat | undefined {
if (!format) {
return undefined;
}
switch (format) {
case 'stream-json':
return OutputFormat.STREAM_JSON;
case 'json':
case OutputFormat.JSON:
return OutputFormat.JSON;
case 'text':
case OutputFormat.TEXT:
default:
return OutputFormat.TEXT;
}
}
function normalizeConfigOutputFormat(
@@ -402,7 +423,6 @@ export class Config {
private readonly skipNextSpeakerCheck: boolean;
private shellExecutionConfig: ShellExecutionConfig;
private readonly extensionManagement: boolean = true;
private readonly enablePromptCompletion: boolean = false;
private readonly skipLoopDetection: boolean;
private readonly skipStartupContext: boolean;
private readonly vlmSwitchMode: string | undefined;
@@ -525,7 +545,6 @@ export class Config {
this.useSmartEdit = params.useSmartEdit ?? false;
this.extensionManagement = params.extensionManagement ?? true;
this.storage = new Storage(this.targetDir);
this.enablePromptCompletion = params.enablePromptCompletion ?? false;
this.vlmSwitchMode = params.vlmSwitchMode;
this.inputFormat = params.inputFormat ?? InputFormat.TEXT;
this.fileExclusions = new FileExclusions(this);
@@ -1073,10 +1092,6 @@ export class Config {
return this.accessibility.screenReader ?? false;
}
getEnablePromptCompletion(): boolean {
return this.enablePromptCompletion;
}
getSkipLoopDetection(): boolean {
return this.skipLoopDetection;
}
@@ -1187,17 +1202,20 @@ export class Config {
try {
useRipgrep = await canUseRipgrep(this.getUseBuiltinRipgrep());
} catch (error: unknown) {
errorString = String(error);
errorString = getErrorMessage(error);
}
if (useRipgrep) {
registerCoreTool(RipGrepTool, this);
} else {
errorString =
errorString ||
'Ripgrep is not available. Please install ripgrep globally.';
// Log for telemetry
logRipgrepFallback(this, new RipgrepFallbackEvent(errorString));
logRipgrepFallback(
this,
new RipgrepFallbackEvent(
this.getUseRipgrep(),
this.getUseBuiltinRipgrep(),
errorString || 'ripgrep is not available',
),
);
registerCoreTool(GrepTool, this);
}
} else {

30
packages/core/src/qwen/qwenOAuth2.test.ts
View File

@@ -623,14 +623,16 @@ describe('QwenOAuth2Client', () => {
});
it('should handle authorization_pending with HTTP 400 according to RFC 8628', async () => {
const errorData = {
error: 'authorization_pending',
error_description: 'The authorization request is still pending',
};
const mockResponse = {
ok: false,
status: 400,
statusText: 'Bad Request',
json: async () => ({
error: 'authorization_pending',
error_description: 'The authorization request is still pending',
}),
text: async () => JSON.stringify(errorData),
json: async () => errorData,
};
vi.mocked(global.fetch).mockResolvedValue(mockResponse as Response);
@@ -646,14 +648,16 @@ describe('QwenOAuth2Client', () => {
});
it('should handle slow_down with HTTP 429 according to RFC 8628', async () => {
const errorData = {
error: 'slow_down',
error_description: 'The client is polling too frequently',
};
const mockResponse = {
ok: false,
status: 429,
statusText: 'Too Many Requests',
json: async () => ({
error: 'slow_down',
error_description: 'The client is polling too frequently',
}),
text: async () => JSON.stringify(errorData),
json: async () => errorData,
};
vi.mocked(global.fetch).mockResolvedValue(mockResponse as Response);
@@ -1993,14 +1997,16 @@ describe('Enhanced Error Handling and Edge Cases', () => {
});
it('should handle authorization_pending with correct status', async () => {
const errorData = {
error: 'authorization_pending',
error_description: 'Authorization request is pending',
};
const mockResponse = {
ok: false,
status: 400,
statusText: 'Bad Request',
json: vi.fn().mockResolvedValue({
error: 'authorization_pending',
error_description: 'Authorization request is pending',
}),
text: vi.fn().mockResolvedValue(JSON.stringify(errorData)),
json: vi.fn().mockResolvedValue(errorData),
};
vi.mocked(global.fetch).mockResolvedValue(

65
packages/core/src/qwen/qwenOAuth2.ts
View File

@@ -345,44 +345,47 @@ export class QwenOAuth2Client implements IQwenOAuth2Client {
});
if (!response.ok) {
// Parse the response as JSON to check for OAuth RFC 8628 standard errors
// Read response body as text first (can only be read once)
const responseText = await response.text();
// Try to parse as JSON to check for OAuth RFC 8628 standard errors
let errorData: ErrorData | null = null;
try {
const errorData = (await response.json()) as ErrorData;
// According to OAuth RFC 8628, handle standard polling responses
if (
response.status === 400 &&
errorData.error === 'authorization_pending'
) {
// User has not yet approved the authorization request. Continue polling.
return { status: 'pending' } as DeviceTokenPendingData;
}
if (response.status === 429 && errorData.error === 'slow_down') {
// Client is polling too frequently. Return pending with slowDown flag.
return {
status: 'pending',
slowDown: true,
} as DeviceTokenPendingData;
}
// Handle other 400 errors (access_denied, expired_token, etc.) as real errors
// For other errors, throw with proper error information
const error = new Error(
`Device token poll failed: ${errorData.error || 'Unknown error'} - ${errorData.error_description || 'No details provided'}`,
);
(error as Error & { status?: number }).status = response.status;
throw error;
errorData = JSON.parse(responseText) as ErrorData;
} catch (_parseError) {
// If JSON parsing fails, fall back to text response
const errorData = await response.text();
// If JSON parsing fails, use text response
const error = new Error(
`Device token poll failed: ${response.status} ${response.statusText}. Response: ${errorData}`,
`Device token poll failed: ${response.status} ${response.statusText}. Response: ${responseText}`,
);
(error as Error & { status?: number }).status = response.status;
throw error;
}
// According to OAuth RFC 8628, handle standard polling responses
if (
response.status === 400 &&
errorData.error === 'authorization_pending'
) {
// User has not yet approved the authorization request. Continue polling.
return { status: 'pending' } as DeviceTokenPendingData;
}
if (response.status === 429 && errorData.error === 'slow_down') {
// Client is polling too frequently. Return pending with slowDown flag.
return {
status: 'pending',
slowDown: true,
} as DeviceTokenPendingData;
}
// Handle other 400 errors (access_denied, expired_token, etc.) as real errors
// For other errors, throw with proper error information
const error = new Error(
`Device token poll failed: ${errorData.error || 'Unknown error'} - ${errorData.error_description}`,
);
(error as Error & { status?: number }).status = response.status;
throw error;
}
return (await response.json()) as DeviceTokenResponse;

25
packages/core/src/services/gitService.test.ts
View File

@@ -19,10 +19,10 @@ import * as path from 'node:path';
import * as fs from 'node:fs/promises';
import * as os from 'node:os';
import { getProjectHash, QWEN_DIR } from '../utils/paths.js';
import { spawnAsync } from '../utils/shell-utils.js';
import { isCommandAvailable } from '../utils/shell-utils.js';
vi.mock('../utils/shell-utils.js', () => ({
spawnAsync: vi.fn(),
isCommandAvailable: vi.fn(),
}));
const hoistedMockEnv = vi.hoisted(() => vi.fn());
@@ -76,10 +76,7 @@ describe('GitService', () => {
vi.clearAllMocks();
hoistedIsGitRepositoryMock.mockReturnValue(true);
(spawnAsync as Mock).mockResolvedValue({
stdout: 'git version 2.0.0',
stderr: '',
});
(isCommandAvailable as Mock).mockReturnValue({ available: true });
hoistedMockHomedir.mockReturnValue(homedir);
@@ -119,23 +116,9 @@ describe('GitService', () => {
});
});
describe('verifyGitAvailability', () => {
it('should resolve true if git --version command succeeds', async () => {
const service = new GitService(projectRoot, storage);
await expect(service.verifyGitAvailability()).resolves.toBe(true);
expect(spawnAsync).toHaveBeenCalledWith('git', ['--version']);
});
it('should resolve false if git --version command fails', async () => {
(spawnAsync as Mock).mockRejectedValue(new Error('git not found'));
const service = new GitService(projectRoot, storage);
await expect(service.verifyGitAvailability()).resolves.toBe(false);
});
});
describe('initialize', () => {
it('should throw an error if Git is not available', async () => {
(spawnAsync as Mock).mockRejectedValue(new Error('git not found'));
(isCommandAvailable as Mock).mockReturnValue({ available: false });
const service = new GitService(projectRoot, storage);
await expect(service.initialize()).rejects.toThrow(
'Checkpointing is enabled, but Git is not installed. Please install Git or disable checkpointing to continue.',

13
packages/core/src/services/gitService.ts
View File

@@ -6,7 +6,7 @@
import * as fs from 'node:fs/promises';
import * as path from 'node:path';
import { spawnAsync } from '../utils/shell-utils.js';
import { isCommandAvailable } from '../utils/shell-utils.js';
import type { SimpleGit } from 'simple-git';
import { simpleGit, CheckRepoActions } from 'simple-git';
import type { Storage } from '../config/storage.js';
@@ -26,7 +26,7 @@ export class GitService {
}
async initialize(): Promise<void> {
const gitAvailable = await this.verifyGitAvailability();
const { available: gitAvailable } = isCommandAvailable('git');
if (!gitAvailable) {
throw new Error(
'Checkpointing is enabled, but Git is not installed. Please install Git or disable checkpointing to continue.',
@@ -41,15 +41,6 @@ export class GitService {
}
}
async verifyGitAvailability(): Promise<boolean> {
try {
await spawnAsync('git', ['--version']);
return true;
} catch (_error) {
return false;
}
}
/**
* Creates a hidden git repository in the project root.
* The Git repository is used to support checkpointing.

10
packages/core/src/telemetry/loggers.test.ts
View File

@@ -447,7 +447,11 @@ describe('loggers', () => {
});
it('should log ripgrep fallback event', () => {
const event = new RipgrepFallbackEvent();
const event = new RipgrepFallbackEvent(
false,
false,
'ripgrep is not available',
);
logRipgrepFallback(mockConfig, event);
@@ -460,13 +464,13 @@ describe('loggers', () => {
'session.id': 'test-session-id',
'user.email': 'test-user@example.com',
'event.name': EVENT_RIPGREP_FALLBACK,
error: undefined,
error: 'ripgrep is not available',
}),
);
});
it('should log ripgrep fallback event with an error', () => {
const event = new RipgrepFallbackEvent('rg not found');
const event = new RipgrepFallbackEvent(false, false, 'rg not found');
logRipgrepFallback(mockConfig, event);

2
packages/core/src/telemetry/loggers.ts
View File

@@ -314,7 +314,7 @@ export function logRipgrepFallback(
config: Config,
event: RipgrepFallbackEvent,
): void {
QwenLogger.getInstance(config)?.logRipgrepFallbackEvent();
QwenLogger.getInstance(config)?.logRipgrepFallbackEvent(event);
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {

8
packages/core/src/telemetry/qwen-logger/qwen-logger.test.ts
View File

@@ -286,9 +286,9 @@ describe('QwenLogger', () => {
event_type: 'action',
type: 'ide',
name: 'ide_connection',
snapshots: JSON.stringify({
properties: {
connection_type: IdeConnectionType.SESSION,
}),
},
}),
);
});
@@ -307,8 +307,10 @@ describe('QwenLogger', () => {
type: 'overflow',
name: 'kitty_sequence_overflow',
subtype: 'kitty_sequence_overflow',
snapshots: JSON.stringify({
properties: {
sequence_length: 1024,
},
snapshots: JSON.stringify({
truncated_sequence: 'truncated...',
}),
}),

138
packages/core/src/telemetry/qwen-logger/qwen-logger.ts
View File

@@ -38,6 +38,7 @@ import type {
ModelSlashCommandEvent,
ExtensionDisableEvent,
AuthEvent,
RipgrepFallbackEvent,
} from '../types.js';
import { EndSessionEvent } from '../types.js';
import type {
@@ -258,7 +259,7 @@ export class QwenLogger {
: '',
},
_v: `qwen-code@${version}`,
};
} as RumPayload;
}
flushIfNeeded(): void {
@@ -367,12 +368,10 @@ export class QwenLogger {
const applicationEvent = this.createViewEvent('session', 'session_start', {
properties: {
model: event.model,
},
snapshots: JSON.stringify({
approval_mode: event.approval_mode,
embedding_model: event.embedding_model,
sandbox_enabled: event.sandbox_enabled,
core_tools_enabled: event.core_tools_enabled,
approval_mode: event.approval_mode,
api_key_enabled: event.api_key_enabled,
vertex_ai_enabled: event.vertex_ai_enabled,
debug_enabled: event.debug_enabled,
@@ -380,7 +379,7 @@ export class QwenLogger {
telemetry_enabled: event.telemetry_enabled,
telemetry_log_user_prompts_enabled:
event.telemetry_log_user_prompts_enabled,
}),
},
});
// Flush start event immediately
@@ -409,10 +408,10 @@ export class QwenLogger {
'conversation',
'conversation_finished',
{
snapshots: JSON.stringify({
properties: {
approval_mode: event.approvalMode,
turn_count: event.turnCount,
}),
},
},
);
@@ -426,10 +425,8 @@ export class QwenLogger {
properties: {
auth_type: event.auth_type,
prompt_id: event.prompt_id,
},
snapshots: JSON.stringify({
prompt_length: event.prompt_length,
}),
},
});
this.enqueueLogEvent(rumEvent);
@@ -438,10 +435,10 @@ export class QwenLogger {
logSlashCommandEvent(event: SlashCommandEvent): void {
const rumEvent = this.createActionEvent('user', 'slash_command', {
snapshots: JSON.stringify({
properties: {
command: event.command,
subcommand: event.subcommand,
}),
},
});
this.enqueueLogEvent(rumEvent);
@@ -450,9 +447,9 @@ export class QwenLogger {
logModelSlashCommandEvent(event: ModelSlashCommandEvent): void {
const rumEvent = this.createActionEvent('user', 'model_slash_command', {
snapshots: JSON.stringify({
model_name: event.model_name,
}),
properties: {
model: event.model_name,
},
});
this.enqueueLogEvent(rumEvent);
@@ -468,15 +465,13 @@ export class QwenLogger {
properties: {
prompt_id: event.prompt_id,
response_id: event.response_id,
},
snapshots: JSON.stringify({
function_name: event.function_name,
decision: event.decision,
success: event.success,
tool_name: event.function_name,
permission: event.decision,
success: event.success ? 1 : 0,
duration_ms: event.duration_ms,
error: event.error,
error_type: event.error_type,
}),
error_message: event.error,
},
},
);
@@ -489,14 +484,14 @@ export class QwenLogger {
'tool',
`file_operation#${event.tool_name}`,
{
snapshots: JSON.stringify({
properties: {
tool_name: event.tool_name,
operation: event.operation,
lines: event.lines,
mimetype: event.mimetype,
extension: event.extension,
programming_language: event.programming_language,
}),
},
},
);
@@ -506,11 +501,15 @@ export class QwenLogger {
logSubagentExecutionEvent(event: SubagentExecutionEvent): void {
const rumEvent = this.createActionEvent('tool', 'subagent_execution', {
snapshots: JSON.stringify({
properties: {
subagent_name: event.subagent_name,
status: event.status,
terminate_reason: event.terminate_reason,
execution_summary: event.execution_summary,
},
snapshots: JSON.stringify({
...(event.execution_summary
? { execution_summary: event.execution_summary }
: {}),
}),
});
@@ -520,8 +519,10 @@ export class QwenLogger {
logToolOutputTruncatedEvent(event: ToolOutputTruncatedEvent): void {
const rumEvent = this.createActionEvent('tool', 'tool_output_truncated', {
snapshots: JSON.stringify({
properties: {
tool_name: event.tool_name,
},
snapshots: JSON.stringify({
original_content_length: event.original_content_length,
truncated_content_length: event.truncated_content_length,
threshold: event.threshold,
@@ -594,10 +595,8 @@ export class QwenLogger {
auth_type: event.auth_type,
model: event.model,
prompt_id: event.prompt_id,
},
snapshots: JSON.stringify({
error_type: event.error_type,
}),
},
});
this.enqueueLogEvent(rumEvent);
@@ -622,11 +621,11 @@ export class QwenLogger {
{
subtype: 'content_retry_failure',
message: `Content retry failed after ${event.total_attempts} attempts`,
snapshots: JSON.stringify({
properties: {
error_type: event.final_error_type,
total_attempts: event.total_attempts,
final_error_type: event.final_error_type,
total_duration_ms: event.total_duration_ms,
}),
},
},
);
@@ -655,10 +654,8 @@ export class QwenLogger {
subtype: 'loop_detected',
properties: {
prompt_id: event.prompt_id,
error_type: event.loop_type,
},
snapshots: JSON.stringify({
loop_type: event.loop_type,
}),
});
this.enqueueLogEvent(rumEvent);
@@ -671,8 +668,10 @@ export class QwenLogger {
'kitty_sequence_overflow',
{
subtype: 'kitty_sequence_overflow',
snapshots: JSON.stringify({
properties: {
sequence_length: event.sequence_length,
},
snapshots: JSON.stringify({
truncated_sequence: event.truncated_sequence,
}),
},
@@ -685,7 +684,9 @@ export class QwenLogger {
// ide events
logIdeConnectionEvent(event: IdeConnectionEvent): void {
const rumEvent = this.createActionEvent('ide', 'ide_connection', {
snapshots: JSON.stringify({ connection_type: event.connection_type }),
properties: {
connection_type: event.connection_type,
},
});
this.enqueueLogEvent(rumEvent);
@@ -695,12 +696,12 @@ export class QwenLogger {
// extension events
logExtensionInstallEvent(event: ExtensionInstallEvent): void {
const rumEvent = this.createActionEvent('extension', 'extension_install', {
snapshots: JSON.stringify({
properties: {
extension_name: event.extension_name,
extension_version: event.extension_version,
extension_source: event.extension_source,
status: event.status,
}),
},
});
this.enqueueLogEvent(rumEvent);
@@ -712,10 +713,10 @@ export class QwenLogger {
'extension',
'extension_uninstall',
{
snapshots: JSON.stringify({
properties: {
extension_name: event.extension_name,
status: event.status,
}),
},
},
);
@@ -725,10 +726,10 @@ export class QwenLogger {
logExtensionEnableEvent(event: ExtensionEnableEvent): void {
const rumEvent = this.createActionEvent('extension', 'extension_enable', {
snapshots: JSON.stringify({
properties: {
extension_name: event.extension_name,
setting_scope: event.setting_scope,
}),
},
});
this.enqueueLogEvent(rumEvent);
@@ -737,10 +738,10 @@ export class QwenLogger {
logExtensionDisableEvent(event: ExtensionDisableEvent): void {
const rumEvent = this.createActionEvent('extension', 'extension_disable', {
snapshots: JSON.stringify({
properties: {
extension_name: event.extension_name,
setting_scope: event.setting_scope,
}),
},
});
this.enqueueLogEvent(rumEvent);
@@ -748,18 +749,15 @@ export class QwenLogger {
}
logAuthEvent(event: AuthEvent): void {
const snapshots: Record<string, unknown> = {
auth_type: event.auth_type,
action_type: event.action_type,
status: event.status,
};
if (event.error_message) {
snapshots['error_message'] = event.error_message;
}
const rumEvent = this.createActionEvent('auth', 'auth', {
snapshots: JSON.stringify(snapshots),
properties: {
auth_type: event.auth_type,
action_type: event.action_type,
success: event.status === 'success' ? 1 : 0,
error_type: event.status !== 'success' ? event.status : undefined,
error_message:
event.status === 'error' ? event.error_message : undefined,
},
});
this.enqueueLogEvent(rumEvent);
@@ -778,8 +776,16 @@ export class QwenLogger {
this.flushIfNeeded();
}
logRipgrepFallbackEvent(): void {
const rumEvent = this.createActionEvent('misc', 'ripgrep_fallback', {});
logRipgrepFallbackEvent(event: RipgrepFallbackEvent): void {
const rumEvent = this.createActionEvent('misc', 'ripgrep_fallback', {
properties: {
platform: process.platform,
arch: process.arch,
use_ripgrep: event.use_ripgrep,
use_builtin_ripgrep: event.use_builtin_ripgrep,
error_message: event.error,
},
});
this.enqueueLogEvent(rumEvent);
this.flushIfNeeded();
@@ -800,11 +806,9 @@ export class QwenLogger {
const rumEvent = this.createActionEvent('misc', 'next_speaker_check', {
properties: {
prompt_id: event.prompt_id,
},
snapshots: JSON.stringify({
finish_reason: event.finish_reason,
result: event.result,
}),
},
});
this.enqueueLogEvent(rumEvent);
@@ -813,10 +817,10 @@ export class QwenLogger {
logChatCompressionEvent(event: ChatCompressionEvent): void {
const rumEvent = this.createActionEvent('misc', 'chat_compression', {
snapshots: JSON.stringify({
properties: {
tokens_before: event.tokens_before,
tokens_after: event.tokens_after,
}),
},
});
this.enqueueLogEvent(rumEvent);
@@ -825,11 +829,11 @@ export class QwenLogger {
logContentRetryEvent(event: ContentRetryEvent): void {
const rumEvent = this.createActionEvent('misc', 'content_retry', {
snapshots: JSON.stringify({
attempt_number: event.attempt_number,
properties: {
error_type: event.error_type,
attempt_number: event.attempt_number,
retry_delay_ms: event.retry_delay_ms,
}),
},
});
this.enqueueLogEvent(rumEvent);

12
packages/core/src/telemetry/types.ts
View File

@@ -318,10 +318,20 @@ export class FlashFallbackEvent implements BaseTelemetryEvent {
export class RipgrepFallbackEvent implements BaseTelemetryEvent {
'event.name': 'ripgrep_fallback';
'event.timestamp': string;
use_ripgrep: boolean;
use_builtin_ripgrep: boolean;
error?: string;
constructor(public error?: string) {
constructor(
use_ripgrep: boolean,
use_builtin_ripgrep: boolean,
error?: string,
) {
this['event.name'] = 'ripgrep_fallback';
this['event.timestamp'] = new Date().toISOString();
this.use_ripgrep = use_ripgrep;
this.use_builtin_ripgrep = use_builtin_ripgrep;
this.error = error;
}
}

75
packages/core/src/tools/grep.test.ts
View File

@@ -18,19 +18,68 @@ import * as glob from 'glob';
vi.mock('glob', { spy: true });
// Mock the child_process module to control grep/git grep behavior
vi.mock('child_process', () => ({
spawn: vi.fn(() => ({
on: (event: string, cb: (...args: unknown[]) => void) => {
if (event === 'error' || event === 'close') {
// Simulate command not found or error for git grep and system grep
// to force it to fall back to JS implementation.
setTimeout(() => cb(1), 0); // cb(1) for error/close
}
},
stdout: { on: vi.fn() },
stderr: { on: vi.fn() },
})),
}));
vi.mock('child_process', async (importOriginal) => {
const actual = await importOriginal<typeof import('child_process')>();
return {
...actual,
spawn: vi.fn(() => {
// Create a proper mock EventEmitter-like child process
const listeners: Map<
string,
Set<(...args: unknown[]) => void>
> = new Map();
const createStream = () => ({
on: vi.fn((event: string, cb: (...args: unknown[]) => void) => {
const key = `stream:${event}`;
if (!listeners.has(key)) listeners.set(key, new Set());
listeners.get(key)!.add(cb);
}),
removeListener: vi.fn(
(event: string, cb: (...args: unknown[]) => void) => {
const key = `stream:${event}`;
listeners.get(key)?.delete(cb);
},
),
});
return {
on: vi.fn((event: string, cb: (...args: unknown[]) => void) => {
const key = `child:${event}`;
if (!listeners.has(key)) listeners.set(key, new Set());
listeners.get(key)!.add(cb);
// Simulate command not found or error for git grep and system grep
// to force it to fall back to JS implementation.
if (event === 'error') {
setTimeout(() => cb(new Error('Command not found')), 0);
} else if (event === 'close') {
setTimeout(() => cb(1), 0); // Exit code 1 for error
}
}),
removeListener: vi.fn(
(event: string, cb: (...args: unknown[]) => void) => {
const key = `child:${event}`;
listeners.get(key)?.delete(cb);
},
),
stdout: createStream(),
stderr: createStream(),
connected: false,
disconnect: vi.fn(),
};
}),
exec: vi.fn(
(
cmd: string,
callback: (error: Error | null, stdout: string, stderr: string) => void,
) => {
// Mock exec to fail for git grep commands
callback(new Error('Command not found'), '', '');
},
),
};
});
describe('GrepTool', () => {
let tempRootDir: string;

28
packages/core/src/tools/grep.ts
View File

@@ -18,6 +18,7 @@ import { isGitRepository } from '../utils/gitUtils.js';
import type { Config } from '../config/config.js';
import type { FileExclusions } from '../utils/ignorePatterns.js';
import { ToolErrorType } from './tool-error.js';
import { isCommandAvailable } from '../utils/shell-utils.js';
// --- Interfaces ---
@@ -195,29 +196,6 @@ class GrepToolInvocation extends BaseToolInvocation<
}
}
/**
* Checks if a command is available in the system's PATH.
* @param {string} command The command name (e.g., 'git', 'grep').
* @returns {Promise<boolean>} True if the command is available, false otherwise.
*/
private isCommandAvailable(command: string): Promise<boolean> {
return new Promise((resolve) => {
const checkCommand = process.platform === 'win32' ? 'where' : 'command';
const checkArgs =
process.platform === 'win32' ? [command] : ['-v', command];
try {
const child = spawn(checkCommand, checkArgs, {
stdio: 'ignore',
shell: process.platform === 'win32',
});
child.on('close', (code) => resolve(code === 0));
child.on('error', () => resolve(false));
} catch {
resolve(false);
}
});
}
/**
* Parses the standard output of grep-like commands (git grep, system grep).
* Expects format: filePath:lineNumber:lineContent
@@ -297,7 +275,7 @@ class GrepToolInvocation extends BaseToolInvocation<
try {
// --- Strategy 1: git grep ---
const isGit = isGitRepository(absolutePath);
const gitAvailable = isGit && (await this.isCommandAvailable('git'));
const gitAvailable = isGit && isCommandAvailable('git').available;
if (gitAvailable) {
strategyUsed = 'git grep';
@@ -350,7 +328,7 @@ class GrepToolInvocation extends BaseToolInvocation<
}
// --- Strategy 2: System grep ---
const grepAvailable = await this.isCommandAvailable('grep');
const { available: grepAvailable } = isCommandAvailable('grep');
if (grepAvailable) {
strategyUsed = 'system grep';
const grepArgs = ['-r', '-n', '-H', '-E'];

683
packages/core/src/tools/ripGrep.test.ts
View File

@@ -20,14 +20,13 @@ import fs from 'node:fs/promises';
import os, { EOL } from 'node:os';
import type { Config } from '../config/config.js';
import { createMockWorkspaceContext } from '../test-utils/mockWorkspaceContext.js';
import type { ChildProcess } from 'node:child_process';
import { spawn } from 'node:child_process';
import { getRipgrepCommand } from '../utils/ripgrepUtils.js';
import { runRipgrep } from '../utils/ripgrepUtils.js';
import { DEFAULT_FILE_FILTERING_OPTIONS } from '../config/constants.js';
// Mock ripgrepUtils
vi.mock('../utils/ripgrepUtils.js', () => ({
getRipgrepCommand: vi.fn(),
runRipgrep: vi.fn(),
}));
// Mock child_process for ripgrep calls
@@ -37,60 +36,6 @@ vi.mock('child_process', () => ({
const mockSpawn = vi.mocked(spawn);
// Helper function to create mock spawn implementations
function createMockSpawn(
options: {
outputData?: string;
exitCode?: number;
signal?: string;
onCall?: (
command: string,
args: readonly string[],
spawnOptions?: unknown,
) => void;
} = {},
) {
const { outputData, exitCode = 0, signal, onCall } = options;
return (command: string, args: readonly string[], spawnOptions?: unknown) => {
onCall?.(command, args, spawnOptions);
const mockProcess = {
stdout: {
on: vi.fn(),
removeListener: vi.fn(),
},
stderr: {
on: vi.fn(),
removeListener: vi.fn(),
},
on: vi.fn(),
removeListener: vi.fn(),
kill: vi.fn(),
};
// Set up event listeners immediately
setTimeout(() => {
const stdoutDataHandler = mockProcess.stdout.on.mock.calls.find(
(call) => call[0] === 'data',
)?.[1];
const closeHandler = mockProcess.on.mock.calls.find(
(call) => call[0] === 'close',
)?.[1];
if (stdoutDataHandler && outputData) {
stdoutDataHandler(Buffer.from(outputData));
}
if (closeHandler) {
closeHandler(exitCode, signal);
}
}, 0);
return mockProcess as unknown as ChildProcess;
};
}
describe('RipGrepTool', () => {
let tempRootDir: string;
let grepTool: RipGrepTool;
@@ -109,7 +54,6 @@ describe('RipGrepTool', () => {
beforeEach(async () => {
vi.clearAllMocks();
(getRipgrepCommand as Mock).mockResolvedValue('/mock/path/to/rg');
mockSpawn.mockReset();
tempRootDir = await fs.mkdtemp(path.join(os.tmpdir(), 'grep-tool-root-'));
fileExclusionsMock = {
@@ -200,12 +144,11 @@ describe('RipGrepTool', () => {
describe('execute', () => {
it('should find matches for a simple pattern in all files', async () => {
mockSpawn.mockImplementationOnce(
createMockSpawn({
outputData: `fileA.txt:1:hello world${EOL}fileA.txt:2:second line with world${EOL}sub/fileC.txt:1:another world in sub dir${EOL}`,
exitCode: 0,
}),
);
(runRipgrep as Mock).mockResolvedValue({
stdout: `fileA.txt:1:hello world${EOL}fileA.txt:2:second line with world${EOL}sub/fileC.txt:1:another world in sub dir${EOL}`,
truncated: false,
error: undefined,
});
const params: RipGrepToolParams = { pattern: 'world' };
const invocation = grepTool.build(params);
@@ -223,12 +166,11 @@ describe('RipGrepTool', () => {
it('should find matches in a specific path', async () => {
// Setup specific mock for this test - searching in 'sub' should only return matches from that directory
mockSpawn.mockImplementationOnce(
createMockSpawn({
outputData: `fileC.txt:1:another world in sub dir${EOL}`,
exitCode: 0,
}),
);
(runRipgrep as Mock).mockResolvedValue({
stdout: `fileC.txt:1:another world in sub dir${EOL}`,
truncated: false,
error: undefined,
});
const params: RipGrepToolParams = { pattern: 'world', path: 'sub' };
const invocation = grepTool.build(params);
@@ -243,16 +185,11 @@ describe('RipGrepTool', () => {
});
it('should use target directory when path is not provided', async () => {
mockSpawn.mockImplementationOnce(
createMockSpawn({
outputData: `fileA.txt:1:hello world${EOL}`,
exitCode: 0,
onCall: (_, args) => {
// Should search in the target directory (tempRootDir)
expect(args[args.length - 1]).toBe(tempRootDir);
},
}),
);
(runRipgrep as Mock).mockResolvedValue({
stdout: `fileA.txt:1:hello world${EOL}`,
truncated: false,
error: undefined,
});
const params: RipGrepToolParams = { pattern: 'world' };
const invocation = grepTool.build(params);
@@ -264,12 +201,11 @@ describe('RipGrepTool', () => {
it('should find matches with a glob filter', async () => {
// Setup specific mock for this test
mockSpawn.mockImplementationOnce(
createMockSpawn({
outputData: `fileB.js:2:function baz() { return "hello"; }${EOL}`,
exitCode: 0,
}),
);
(runRipgrep as Mock).mockResolvedValue({
stdout: `fileB.js:2:function baz() { return "hello"; }${EOL}`,
truncated: false,
error: undefined,
});
const params: RipGrepToolParams = { pattern: 'hello', glob: '*.js' };
const invocation = grepTool.build(params);
@@ -290,39 +226,10 @@ describe('RipGrepTool', () => {
);
// Setup specific mock for this test - searching for 'hello' in 'sub' with '*.js' filter
mockSpawn.mockImplementationOnce(() => {
const mockProcess = {
stdout: {
on: vi.fn(),
removeListener: vi.fn(),
},
stderr: {
on: vi.fn(),
removeListener: vi.fn(),
},
on: vi.fn(),
removeListener: vi.fn(),
kill: vi.fn(),
};
setTimeout(() => {
const onData = mockProcess.stdout.on.mock.calls.find(
(call) => call[0] === 'data',
)?.[1];
const onClose = mockProcess.on.mock.calls.find(
(call) => call[0] === 'close',
)?.[1];
if (onData) {
// Only return match from the .js file in sub directory
onData(Buffer.from(`another.js:1:const greeting = "hello";${EOL}`));
}
if (onClose) {
onClose(0);
}
}, 0);
return mockProcess as unknown as ChildProcess;
(runRipgrep as Mock).mockResolvedValue({
stdout: `another.js:1:const greeting = "hello";${EOL}`,
truncated: false,
error: undefined,
});
const params: RipGrepToolParams = {
@@ -346,15 +253,11 @@ describe('RipGrepTool', () => {
path.join(tempRootDir, '.qwenignore'),
'ignored.txt\n',
);
mockSpawn.mockImplementationOnce(
createMockSpawn({
exitCode: 1,
onCall: (_, args) => {
expect(args).toContain('--ignore-file');
expect(args).toContain(path.join(tempRootDir, '.qwenignore'));
},
}),
);
(runRipgrep as Mock).mockResolvedValue({
stdout: '',
truncated: false,
error: undefined,
});
const params: RipGrepToolParams = { pattern: 'secret' };
const invocation = grepTool.build(params);
@@ -375,16 +278,11 @@ describe('RipGrepTool', () => {
}),
});
mockSpawn.mockImplementationOnce(
createMockSpawn({
outputData: `kept.txt:1:keep me${EOL}`,
exitCode: 0,
onCall: (_, args) => {
expect(args).not.toContain('--ignore-file');
expect(args).not.toContain(path.join(tempRootDir, '.qwenignore'));
},
}),
);
(runRipgrep as Mock).mockResolvedValue({
stdout: `kept.txt:1:keep me${EOL}`,
truncated: false,
error: undefined,
});
const params: RipGrepToolParams = { pattern: 'keep' };
const invocation = grepTool.build(params);
@@ -404,14 +302,11 @@ describe('RipGrepTool', () => {
}),
});
mockSpawn.mockImplementationOnce(
createMockSpawn({
exitCode: 1,
onCall: (_, args) => {
expect(args).toContain('--no-ignore-vcs');
},
}),
);
(runRipgrep as Mock).mockResolvedValue({
stdout: '',
truncated: false,
error: undefined,
});
const params: RipGrepToolParams = { pattern: 'ignored' };
const invocation = grepTool.build(params);
@@ -421,12 +316,11 @@ describe('RipGrepTool', () => {
it('should truncate llm content when exceeding maximum length', async () => {
const longMatch = 'fileA.txt:1:' + 'a'.repeat(30_000);
mockSpawn.mockImplementationOnce(
createMockSpawn({
outputData: `${longMatch}${EOL}`,
exitCode: 0,
}),
);
(runRipgrep as Mock).mockResolvedValue({
stdout: `${longMatch}${EOL}`,
truncated: false,
error: undefined,
});
const params: RipGrepToolParams = { pattern: 'a+' };
const invocation = grepTool.build(params);
@@ -439,11 +333,11 @@ describe('RipGrepTool', () => {
it('should return "No matches found" when pattern does not exist', async () => {
// Setup specific mock for no matches
mockSpawn.mockImplementationOnce(
createMockSpawn({
exitCode: 1, // No matches found
}),
);
(runRipgrep as Mock).mockResolvedValue({
stdout: '',
truncated: false,
error: undefined,
});
const params: RipGrepToolParams = { pattern: 'nonexistentpattern' };
const invocation = grepTool.build(params);
@@ -463,39 +357,10 @@ describe('RipGrepTool', () => {
it('should handle regex special characters correctly', async () => {
// Setup specific mock for this test - regex pattern 'foo.*bar' should match 'const foo = "bar";'
mockSpawn.mockImplementationOnce(() => {
const mockProcess = {
stdout: {
on: vi.fn(),
removeListener: vi.fn(),
},
stderr: {
on: vi.fn(),
removeListener: vi.fn(),
},
on: vi.fn(),
removeListener: vi.fn(),
kill: vi.fn(),
};
setTimeout(() => {
const onData = mockProcess.stdout.on.mock.calls.find(
(call) => call[0] === 'data',
)?.[1];
const onClose = mockProcess.on.mock.calls.find(
(call) => call[0] === 'close',
)?.[1];
if (onData) {
// Return match for the regex pattern
onData(Buffer.from(`fileB.js:1:const foo = "bar";${EOL}`));
}
if (onClose) {
onClose(0);
}
}, 0);
return mockProcess as unknown as ChildProcess;
(runRipgrep as Mock).mockResolvedValue({
stdout: `fileB.js:1:const foo = "bar";${EOL}`,
truncated: false,
error: undefined,
});
const params: RipGrepToolParams = { pattern: 'foo.*bar' }; // Matches 'const foo = "bar";'
@@ -509,43 +374,10 @@ describe('RipGrepTool', () => {
it('should be case-insensitive by default (JS fallback)', async () => {
// Setup specific mock for this test - case insensitive search for 'HELLO'
mockSpawn.mockImplementationOnce(() => {
const mockProcess = {
stdout: {
on: vi.fn(),
removeListener: vi.fn(),
},
stderr: {
on: vi.fn(),
removeListener: vi.fn(),
},
on: vi.fn(),
removeListener: vi.fn(),
kill: vi.fn(),
};
setTimeout(() => {
const onData = mockProcess.stdout.on.mock.calls.find(
(call) => call[0] === 'data',
)?.[1];
const onClose = mockProcess.on.mock.calls.find(
(call) => call[0] === 'close',
)?.[1];
if (onData) {
// Return case-insensitive matches for 'HELLO'
onData(
Buffer.from(
`fileA.txt:1:hello world${EOL}fileB.js:2:function baz() { return "hello"; }${EOL}`,
),
);
}
if (onClose) {
onClose(0);
}
}, 0);
return mockProcess as unknown as ChildProcess;
(runRipgrep as Mock).mockResolvedValue({
stdout: `fileA.txt:1:hello world${EOL}fileB.js:2:function baz() { return "hello"; }${EOL}`,
truncated: false,
error: undefined,
});
const params: RipGrepToolParams = { pattern: 'HELLO' };
@@ -568,12 +400,11 @@ describe('RipGrepTool', () => {
});
it('should search within a single file when path is a file', async () => {
mockSpawn.mockImplementationOnce(
createMockSpawn({
outputData: `fileA.txt:1:hello world${EOL}fileA.txt:2:second line with world${EOL}`,
exitCode: 0,
}),
);
(runRipgrep as Mock).mockResolvedValue({
stdout: `fileA.txt:1:hello world${EOL}fileA.txt:2:second line with world${EOL}`,
truncated: false,
error: undefined,
});
const params: RipGrepToolParams = {
pattern: 'world',
@@ -588,7 +419,11 @@ describe('RipGrepTool', () => {
});
it('should throw an error if ripgrep is not available', async () => {
(getRipgrepCommand as Mock).mockResolvedValue(null);
(runRipgrep as Mock).mockResolvedValue({
stdout: '',
truncated: false,
error: new Error('ripgrep binary not found.'),
});
const params: RipGrepToolParams = { pattern: 'world' };
const invocation = grepTool.build(params);
@@ -612,54 +447,6 @@ describe('RipGrepTool', () => {
const result = await invocation.execute(controller.signal);
expect(result).toBeDefined();
});
it('should abort streaming search when signal is triggered', async () => {
// Setup specific mock for this test - simulate process being killed due to abort
mockSpawn.mockImplementationOnce(() => {
const mockProcess = {
stdout: {
on: vi.fn(),
removeListener: vi.fn(),
},
stderr: {
on: vi.fn(),
removeListener: vi.fn(),
},
on: vi.fn(),
removeListener: vi.fn(),
kill: vi.fn(),
};
// Simulate process being aborted - use setTimeout to ensure handlers are registered first
setTimeout(() => {
const closeHandler = mockProcess.on.mock.calls.find(
(call) => call[0] === 'close',
)?.[1];
if (closeHandler) {
// Simulate process killed by signal (code is null, signal is SIGTERM)
closeHandler(null, 'SIGTERM');
}
}, 0);
return mockProcess as unknown as ChildProcess;
});
const controller = new AbortController();
const params: RipGrepToolParams = { pattern: 'test' };
const invocation = grepTool.build(params);
// Abort immediately before starting the search
controller.abort();
const result = await invocation.execute(controller.signal);
expect(result.llmContent).toContain(
'Error during grep search operation: ripgrep exited with code null',
);
expect(result.returnDisplay).toContain(
'Error: ripgrep exited with code null',
);
});
});
describe('error handling and edge cases', () => {
@@ -675,32 +462,10 @@ describe('RipGrepTool', () => {
await fs.mkdir(emptyDir);
// Setup specific mock for this test - searching in empty directory should return no matches
mockSpawn.mockImplementationOnce(() => {
const mockProcess = {
stdout: {
on: vi.fn(),
removeListener: vi.fn(),
},
stderr: {
on: vi.fn(),
removeListener: vi.fn(),
},
on: vi.fn(),
removeListener: vi.fn(),
kill: vi.fn(),
};
setTimeout(() => {
const onClose = mockProcess.on.mock.calls.find(
(call) => call[0] === 'close',
)?.[1];
if (onClose) {
onClose(1);
}
}, 0);
return mockProcess as unknown as ChildProcess;
(runRipgrep as Mock).mockResolvedValue({
stdout: '',
truncated: false,
error: undefined,
});
const params: RipGrepToolParams = { pattern: 'test', path: 'empty' };
@@ -715,32 +480,10 @@ describe('RipGrepTool', () => {
await fs.writeFile(path.join(tempRootDir, 'empty.txt'), '');
// Setup specific mock for this test - searching for anything in empty files should return no matches
mockSpawn.mockImplementationOnce(() => {
const mockProcess = {
stdout: {
on: vi.fn(),
removeListener: vi.fn(),
},
stderr: {
on: vi.fn(),
removeListener: vi.fn(),
},
on: vi.fn(),
removeListener: vi.fn(),
kill: vi.fn(),
};
setTimeout(() => {
const onClose = mockProcess.on.mock.calls.find(
(call) => call[0] === 'close',
)?.[1];
if (onClose) {
onClose(1);
}
}, 0);
return mockProcess as unknown as ChildProcess;
(runRipgrep as Mock).mockResolvedValue({
stdout: '',
truncated: false,
error: undefined,
});
const params: RipGrepToolParams = { pattern: 'anything' };
@@ -758,42 +501,10 @@ describe('RipGrepTool', () => {
);
// Setup specific mock for this test - searching for 'world' should find the file with special characters
mockSpawn.mockImplementationOnce(() => {
const mockProcess = {
stdout: {
on: vi.fn(),
removeListener: vi.fn(),
},
stderr: {
on: vi.fn(),
removeListener: vi.fn(),
},
on: vi.fn(),
removeListener: vi.fn(),
kill: vi.fn(),
};
setTimeout(() => {
const onData = mockProcess.stdout.on.mock.calls.find(
(call) => call[0] === 'data',
)?.[1];
const onClose = mockProcess.on.mock.calls.find(
(call) => call[0] === 'close',
)?.[1];
if (onData) {
onData(
Buffer.from(
`${specialFileName}:1:hello world with special chars${EOL}`,
),
);
}
if (onClose) {
onClose(0);
}
}, 0);
return mockProcess as unknown as ChildProcess;
(runRipgrep as Mock).mockResolvedValue({
stdout: `file with spaces & symbols!.txt:1:hello world with special chars${EOL}`,
truncated: false,
error: undefined,
});
const params: RipGrepToolParams = { pattern: 'world' };
@@ -813,42 +524,10 @@ describe('RipGrepTool', () => {
);
// Setup specific mock for this test - searching for 'deep' should find the deeply nested file
mockSpawn.mockImplementationOnce(() => {
const mockProcess = {
stdout: {
on: vi.fn(),
removeListener: vi.fn(),
},
stderr: {
on: vi.fn(),
removeListener: vi.fn(),
},
on: vi.fn(),
removeListener: vi.fn(),
kill: vi.fn(),
};
setTimeout(() => {
const onData = mockProcess.stdout.on.mock.calls.find(
(call) => call[0] === 'data',
)?.[1];
const onClose = mockProcess.on.mock.calls.find(
(call) => call[0] === 'close',
)?.[1];
if (onData) {
onData(
Buffer.from(
`a/b/c/d/e/deep.txt:1:content in deep directory${EOL}`,
),
);
}
if (onClose) {
onClose(0);
}
}, 0);
return mockProcess as unknown as ChildProcess;
(runRipgrep as Mock).mockResolvedValue({
stdout: `a/b/c/d/e/deep.txt:1:content in deep directory${EOL}`,
truncated: false,
error: undefined,
});
const params: RipGrepToolParams = { pattern: 'deep' };
@@ -868,42 +547,10 @@ describe('RipGrepTool', () => {
);
// Setup specific mock for this test - regex pattern should match function declarations
mockSpawn.mockImplementationOnce(() => {
const mockProcess = {
stdout: {
on: vi.fn(),
removeListener: vi.fn(),
},
stderr: {
on: vi.fn(),
removeListener: vi.fn(),
},
on: vi.fn(),
removeListener: vi.fn(),
kill: vi.fn(),
};
setTimeout(() => {
const onData = mockProcess.stdout.on.mock.calls.find(
(call) => call[0] === 'data',
)?.[1];
const onClose = mockProcess.on.mock.calls.find(
(call) => call[0] === 'close',
)?.[1];
if (onData) {
onData(
Buffer.from(
`code.js:1:function getName() { return "test"; }${EOL}`,
),
);
}
if (onClose) {
onClose(0);
}
}, 0);
return mockProcess as unknown as ChildProcess;
(runRipgrep as Mock).mockResolvedValue({
stdout: `code.js:1:function getName() { return "test"; }${EOL}`,
truncated: false,
error: undefined,
});
const params: RipGrepToolParams = { pattern: 'function\\s+\\w+\\s*\\(' };
@@ -921,42 +568,10 @@ describe('RipGrepTool', () => {
);
// Setup specific mock for this test - case insensitive search should match all variants
mockSpawn.mockImplementationOnce(() => {
const mockProcess = {
stdout: {
on: vi.fn(),
removeListener: vi.fn(),
},
stderr: {
on: vi.fn(),
removeListener: vi.fn(),
},
on: vi.fn(),
removeListener: vi.fn(),
kill: vi.fn(),
};
setTimeout(() => {
const onData = mockProcess.stdout.on.mock.calls.find(
(call) => call[0] === 'data',
)?.[1];
const onClose = mockProcess.on.mock.calls.find(
(call) => call[0] === 'close',
)?.[1];
if (onData) {
onData(
Buffer.from(
`case.txt:1:Hello World${EOL}case.txt:2:hello world${EOL}case.txt:3:HELLO WORLD${EOL}`,
),
);
}
if (onClose) {
onClose(0);
}
}, 0);
return mockProcess as unknown as ChildProcess;
(runRipgrep as Mock).mockResolvedValue({
stdout: `case.txt:1:Hello World${EOL}case.txt:2:hello world${EOL}case.txt:3:HELLO WORLD${EOL}`,
truncated: false,
error: undefined,
});
const params: RipGrepToolParams = { pattern: 'hello' };
@@ -975,38 +590,10 @@ describe('RipGrepTool', () => {
);
// Setup specific mock for this test - escaped regex pattern should match price format
mockSpawn.mockImplementationOnce(() => {
const mockProcess = {
stdout: {
on: vi.fn(),
removeListener: vi.fn(),
},
stderr: {
on: vi.fn(),
removeListener: vi.fn(),
},
on: vi.fn(),
removeListener: vi.fn(),
kill: vi.fn(),
};
setTimeout(() => {
const onData = mockProcess.stdout.on.mock.calls.find(
(call) => call[0] === 'data',
)?.[1];
const onClose = mockProcess.on.mock.calls.find(
(call) => call[0] === 'close',
)?.[1];
if (onData) {
onData(Buffer.from(`special.txt:1:Price: $19.99${EOL}`));
}
if (onClose) {
onClose(0);
}
}, 0);
return mockProcess as unknown as ChildProcess;
(runRipgrep as Mock).mockResolvedValue({
stdout: `special.txt:1:Price: $19.99${EOL}`,
truncated: false,
error: undefined,
});
const params: RipGrepToolParams = { pattern: '\\$\\d+\\.\\d+' };
@@ -1032,42 +619,10 @@ describe('RipGrepTool', () => {
await fs.writeFile(path.join(tempRootDir, 'test.txt'), 'text content');
// Setup specific mock for this test - glob pattern should filter to only ts/tsx files
mockSpawn.mockImplementationOnce(() => {
const mockProcess = {
stdout: {
on: vi.fn(),
removeListener: vi.fn(),
},
stderr: {
on: vi.fn(),
removeListener: vi.fn(),
},
on: vi.fn(),
removeListener: vi.fn(),
kill: vi.fn(),
};
setTimeout(() => {
const onData = mockProcess.stdout.on.mock.calls.find(
(call) => call[0] === 'data',
)?.[1];
const onClose = mockProcess.on.mock.calls.find(
(call) => call[0] === 'close',
)?.[1];
if (onData) {
onData(
Buffer.from(
`test.ts:1:typescript content${EOL}test.tsx:1:tsx content${EOL}`,
),
);
}
if (onClose) {
onClose(0);
}
}, 0);
return mockProcess as unknown as ChildProcess;
(runRipgrep as Mock).mockResolvedValue({
stdout: `test.ts:1:typescript content${EOL}test.tsx:1:tsx content${EOL}`,
truncated: false,
error: undefined,
});
const params: RipGrepToolParams = {
@@ -1092,38 +647,10 @@ describe('RipGrepTool', () => {
await fs.writeFile(path.join(tempRootDir, 'other.ts'), 'other code');
// Setup specific mock for this test - glob pattern should filter to only src/** files
mockSpawn.mockImplementationOnce(() => {
const mockProcess = {
stdout: {
on: vi.fn(),
removeListener: vi.fn(),
},
stderr: {
on: vi.fn(),
removeListener: vi.fn(),
},
on: vi.fn(),
removeListener: vi.fn(),
kill: vi.fn(),
};
setTimeout(() => {
const onData = mockProcess.stdout.on.mock.calls.find(
(call) => call[0] === 'data',
)?.[1];
const onClose = mockProcess.on.mock.calls.find(
(call) => call[0] === 'close',
)?.[1];
if (onData) {
onData(Buffer.from(`src/main.ts:1:source code${EOL}`));
}
if (onClose) {
onClose(0);
}
}, 0);
return mockProcess as unknown as ChildProcess;
(runRipgrep as Mock).mockResolvedValue({
stdout: `src/main.ts:1:source code${EOL}`,
truncated: false,
error: undefined,
});
const params: RipGrepToolParams = {

61
packages/core/src/tools/ripGrep.ts
View File

@@ -6,14 +6,13 @@
import fs from 'node:fs';
import path from 'node:path';
import { spawn } from 'node:child_process';
import type { ToolInvocation, ToolResult } from './tools.js';
import { BaseDeclarativeTool, BaseToolInvocation, Kind } from './tools.js';
import { ToolNames } from './tool-names.js';
import { resolveAndValidatePath } from '../utils/paths.js';
import { getErrorMessage } from '../utils/errors.js';
import type { Config } from '../config/config.js';
import { getRipgrepCommand } from '../utils/ripgrepUtils.js';
import { runRipgrep } from '../utils/ripgrepUtils.js';
import { SchemaValidator } from '../utils/schemaValidator.js';
import type { FileFilteringOptions } from '../config/constants.js';
import { DEFAULT_FILE_FILTERING_OPTIONS } from '../config/constants.js';
@@ -208,60 +207,12 @@ class GrepToolInvocation extends BaseToolInvocation<
rgArgs.push('--threads', '4');
rgArgs.push(absolutePath);
try {
const rgCommand = await getRipgrepCommand(
this.config.getUseBuiltinRipgrep(),
);
if (!rgCommand) {
throw new Error('ripgrep binary not found.');
}
const output = await new Promise<string>((resolve, reject) => {
const child = spawn(rgCommand, rgArgs, {
windowsHide: true,
});
const stdoutChunks: Buffer[] = [];
const stderrChunks: Buffer[] = [];
const cleanup = () => {
if (options.signal.aborted) {
child.kill();
}
};
options.signal.addEventListener('abort', cleanup, { once: true });
child.stdout.on('data', (chunk) => stdoutChunks.push(chunk));
child.stderr.on('data', (chunk) => stderrChunks.push(chunk));
child.on('error', (err) => {
options.signal.removeEventListener('abort', cleanup);
reject(new Error(`failed to start ripgrep: ${err.message}.`));
});
child.on('close', (code) => {
options.signal.removeEventListener('abort', cleanup);
const stdoutData = Buffer.concat(stdoutChunks).toString('utf8');
const stderrData = Buffer.concat(stderrChunks).toString('utf8');
if (code === 0) {
resolve(stdoutData);
} else if (code === 1) {
resolve(''); // No matches found
} else {
reject(
new Error(`ripgrep exited with code ${code}: ${stderrData}`),
);
}
});
});
return output;
} catch (error: unknown) {
console.error(`Ripgrep failed: ${getErrorMessage(error)}`);
throw error;
const result = await runRipgrep(rgArgs, options.signal);
if (result.error && !result.stdout) {
throw result.error;
}
return result.stdout;
}
private getFileFilteringOptions(): FileFilteringOptions {

119
packages/core/src/utils/ripgrepUtils.test.ts
View File

@@ -4,30 +4,12 @@
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect, beforeEach, vi, type Mock } from 'vitest';
import {
canUseRipgrep,
getRipgrepCommand,
getBuiltinRipgrep,
} from './ripgrepUtils.js';
import { fileExists } from './fileUtils.js';
import { describe, it, expect } from 'vitest';
import { getBuiltinRipgrep } from './ripgrepUtils.js';
import path from 'node:path';
// Mock fileUtils
vi.mock('./fileUtils.js', async (importOriginal) => {
const actual = await importOriginal<typeof import('./fileUtils.js')>();
return {
...actual,
fileExists: vi.fn(),
};
});
describe('ripgrepUtils', () => {
beforeEach(() => {
vi.clearAllMocks();
});
describe('getBulltinRipgrepPath', () => {
describe('getBuiltinRipgrep', () => {
it('should return path with .exe extension on Windows', () => {
const originalPlatform = process.platform;
const originalArch = process.arch;
@@ -150,99 +132,4 @@ describe('ripgrepUtils', () => {
Object.defineProperty(process, 'arch', { value: originalArch });
});
});
describe('canUseRipgrep', () => {
it('should return true if ripgrep binary exists (builtin)', async () => {
(fileExists as Mock).mockResolvedValue(true);
const result = await canUseRipgrep(true);
expect(result).toBe(true);
expect(fileExists).toHaveBeenCalledOnce();
});
it('should return true if ripgrep binary exists (default)', async () => {
(fileExists as Mock).mockResolvedValue(true);
const result = await canUseRipgrep();
expect(result).toBe(true);
expect(fileExists).toHaveBeenCalledOnce();
});
});
describe('ensureRipgrepPath', () => {
it('should return bundled ripgrep path if binary exists (useBuiltin=true)', async () => {
(fileExists as Mock).mockResolvedValue(true);
const rgPath = await getRipgrepCommand(true);
expect(rgPath).toBeDefined();
expect(rgPath).toContain('rg');
expect(rgPath).not.toBe('rg'); // Should be full path, not just 'rg'
expect(fileExists).toHaveBeenCalledOnce();
expect(fileExists).toHaveBeenCalledWith(rgPath);
});
it('should return bundled ripgrep path if binary exists (default)', async () => {
(fileExists as Mock).mockResolvedValue(true);
const rgPath = await getRipgrepCommand();
expect(rgPath).toBeDefined();
expect(rgPath).toContain('rg');
expect(fileExists).toHaveBeenCalledOnce();
});
it('should fall back to system rg if bundled binary does not exist', async () => {
(fileExists as Mock).mockResolvedValue(false);
// When useBuiltin is true but bundled binary doesn't exist,
// it should fall back to checking system rg
// The test result depends on whether system rg is actually available
const rgPath = await getRipgrepCommand(true);
expect(fileExists).toHaveBeenCalledOnce();
// If system rg is available, it should return 'rg' (or 'rg.exe' on Windows)
// This test will pass if system ripgrep is installed
expect(rgPath).toBeDefined();
});
it('should use system rg when useBuiltin=false', async () => {
// When useBuiltin is false, should skip bundled check and go straight to system rg
const rgPath = await getRipgrepCommand(false);
// Should not check for bundled binary
expect(fileExists).not.toHaveBeenCalled();
// If system rg is available, it should return 'rg' (or 'rg.exe' on Windows)
expect(rgPath).toBeDefined();
});
it('should throw error if neither bundled nor system ripgrep is available', async () => {
// This test only makes sense in an environment where system rg is not installed
// We'll skip this test in CI/local environments where rg might be available
// Instead, we test the error message format
const originalPlatform = process.platform;
// Use an unsupported platform to trigger the error path
Object.defineProperty(process, 'platform', { value: 'freebsd' });
try {
await getRipgrepCommand();
// If we get here without error, system rg was available, which is fine
} catch (error) {
expect(error).toBeInstanceOf(Error);
const errorMessage = (error as Error).message;
// Should contain helpful error information
expect(
errorMessage.includes('Ripgrep binary not found') ||
errorMessage.includes('Failed to locate ripgrep') ||
errorMessage.includes('Unsupported platform'),
).toBe(true);
}
// Restore original value
Object.defineProperty(process, 'platform', { value: originalPlatform });
});
});
});

260
packages/core/src/utils/ripgrepUtils.ts
View File

@@ -6,7 +6,53 @@
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { execFile } from 'node:child_process';
import { fileExists } from './fileUtils.js';
import { execCommand, isCommandAvailable } from './shell-utils.js';
const RIPGREP_COMMAND = 'rg';
const RIPGREP_BUFFER_LIMIT = 20_000_000; // Keep buffers aligned with the original bundle.
const RIPGREP_TEST_TIMEOUT_MS = 5_000;
const RIPGREP_RUN_TIMEOUT_MS = 10_000;
const RIPGREP_WSL_TIMEOUT_MS = 60_000;
type RipgrepMode = 'builtin' | 'system';
interface RipgrepSelection {
mode: RipgrepMode;
command: string;
}
interface RipgrepHealth {
working: boolean;
lastTested: number;
selection: RipgrepSelection;
}
export interface RipgrepRunResult {
/**
* The stdout output from ripgrep
*/
stdout: string;
/**
* Whether the results were truncated due to buffer overflow or signal termination
*/
truncated: boolean;
/**
* Any error that occurred during execution (non-fatal errors like no matches won't populate this)
*/
error?: Error;
}
let cachedSelection: RipgrepSelection | null = null;
let cachedHealth: RipgrepHealth | null = null;
let macSigningAttempted = false;
function wslTimeout(): number {
return process.platform === 'linux' && process.env['WSL_INTEROP']
? RIPGREP_WSL_TIMEOUT_MS
: RIPGREP_RUN_TIMEOUT_MS;
}
// Get the directory of the current module
const __filename = fileURLToPath(import.meta.url);
@@ -88,59 +134,201 @@ export function getBuiltinRipgrep(): string | null {
return vendorPath;
}
/**
* Checks if system ripgrep is available and returns the command to use
* @returns The ripgrep command ('rg' or 'rg.exe') if available, or null if not found
*/
export async function getSystemRipgrep(): Promise<string | null> {
try {
const { spawn } = await import('node:child_process');
const rgCommand = process.platform === 'win32' ? 'rg.exe' : 'rg';
const isAvailable = await new Promise<boolean>((resolve) => {
const proc = spawn(rgCommand, ['--version']);
proc.on('error', () => resolve(false));
proc.on('exit', (code) => resolve(code === 0));
});
return isAvailable ? rgCommand : null;
} catch (_error) {
return null;
}
}
/**
* Checks if ripgrep binary exists and returns its path
* @param useBuiltin If true, tries bundled ripgrep first, then falls back to system ripgrep.
* If false, only checks for system ripgrep.
* @returns The path to ripgrep binary ('rg' or 'rg.exe' for system ripgrep, or full path for bundled), or null if not available
* @throws {Error} If an error occurs while resolving the ripgrep binary.
*/
export async function getRipgrepCommand(
export async function resolveRipgrep(
useBuiltin: boolean = true,
): Promise<string | null> {
try {
if (useBuiltin) {
// Try bundled ripgrep first
const rgPath = getBuiltinRipgrep();
if (rgPath && (await fileExists(rgPath))) {
return rgPath;
}
// Fallback to system rg if bundled binary is not available
}
): Promise<RipgrepSelection | null> {
if (cachedSelection) return cachedSelection;
// Check for system ripgrep
return await getSystemRipgrep();
} catch (_error) {
return null;
if (useBuiltin) {
// Try bundled ripgrep first
const rgPath = getBuiltinRipgrep();
if (rgPath && (await fileExists(rgPath))) {
cachedSelection = { mode: 'builtin', command: rgPath };
return cachedSelection;
}
// Fallback to system rg if bundled binary is not available
}
const { available, error } = isCommandAvailable(RIPGREP_COMMAND);
if (available) {
cachedSelection = { mode: 'system', command: RIPGREP_COMMAND };
return cachedSelection;
}
if (error) {
throw error;
}
return null;
}
/**
* Ensures that ripgrep is healthy by checking its version.
* @param selection The ripgrep selection to check.
* @throws {Error} If ripgrep is not found or is not healthy.
*/
export async function ensureRipgrepHealthy(
selection: RipgrepSelection,
): Promise<void> {
if (
cachedHealth &&
cachedHealth.selection.command === selection.command &&
cachedHealth.working
)
return;
try {
const { stdout, code } = await execCommand(
selection.command,
['--version'],
{
timeout: RIPGREP_TEST_TIMEOUT_MS,
},
);
const working = code === 0 && stdout.startsWith('ripgrep');
cachedHealth = { working, lastTested: Date.now(), selection };
} catch (error) {
cachedHealth = { working: false, lastTested: Date.now(), selection };
throw error;
}
}
export async function ensureMacBinarySigned(
selection: RipgrepSelection,
): Promise<void> {
if (process.platform !== 'darwin') return;
if (macSigningAttempted) return;
macSigningAttempted = true;
if (selection.mode !== 'builtin') return;
const binaryPath = selection.command;
const inspect = await execCommand('codesign', ['-vv', '-d', binaryPath], {
preserveOutputOnError: false,
});
const alreadySigned =
inspect.stdout
?.split('\n')
.some((line) => line.includes('linker-signed')) ?? false;
if (!alreadySigned) return;
await execCommand('codesign', [
'--sign',
'-',
'--force',
'--preserve-metadata=entitlements,requirements,flags,runtime',
binaryPath,
]);
await execCommand('xattr', ['-d', 'com.apple.quarantine', binaryPath]);
}
/**
* Checks if ripgrep binary is available
* @param useBuiltin If true, tries bundled ripgrep first, then falls back to system ripgrep.
* If false, only checks for system ripgrep.
* @returns True if ripgrep is available, false otherwise.
* @throws {Error} If an error occurs while resolving the ripgrep binary.
*/
export async function canUseRipgrep(
useBuiltin: boolean = true,
): Promise<boolean> {
const rgPath = await getRipgrepCommand(useBuiltin);
return rgPath !== null;
const selection = await resolveRipgrep(useBuiltin);
if (!selection) {
return false;
}
await ensureRipgrepHealthy(selection);
return true;
}
/**
* Runs ripgrep with the provided arguments
* @param args The arguments to pass to ripgrep
* @param signal The signal to abort the ripgrep process
* @returns The result of running ripgrep
* @throws {Error} If an error occurs while running ripgrep.
*/
export async function runRipgrep(
args: string[],
signal?: AbortSignal,
): Promise<RipgrepRunResult> {
const selection = await resolveRipgrep();
if (!selection) {
throw new Error('ripgrep not found.');
}
await ensureRipgrepHealthy(selection);
return new Promise<RipgrepRunResult>((resolve) => {
const child = execFile(
selection.command,
args,
{
maxBuffer: RIPGREP_BUFFER_LIMIT,
timeout: wslTimeout(),
signal,
},
(error, stdout = '', stderr = '') => {
if (!error) {
// Success case
resolve({
stdout,
truncated: false,
});
return;
}
// Exit code 1 = no matches found (not an error)
// The error.code from execFile can be string | number | undefined | null
const errorCode = (
error as Error & { code?: string | number | undefined | null }
).code;
if (errorCode === 1) {
resolve({ stdout: '', truncated: false });
return;
}
// Detect various error conditions
const wasKilled =
error.signal === 'SIGTERM' || error.name === 'AbortError';
const overflow = errorCode === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER';
const syntaxError = errorCode === 2;
const truncated = wasKilled || overflow;
let partialOutput = stdout;
// If killed or overflow with partial output, remove the last potentially incomplete line
if (truncated && partialOutput.length > 0) {
const lines = partialOutput.split('\n');
if (lines.length > 0) {
lines.pop();
partialOutput = lines.join('\n');
}
}
// Log warnings for abnormal exits (except syntax errors)
if (!syntaxError && truncated) {
console.warn(
`ripgrep exited abnormally (signal=${error.signal} code=${error.code}) with stderr:\n${stderr.trim() || '(empty)'}`,
);
}
resolve({
stdout: partialOutput,
truncated,
error: error instanceof Error ? error : undefined,
});
},
);
// Handle spawn errors
child.on('error', (err) =>
resolve({ stdout: '', truncated: false, error: err }),
);
});
}

130
packages/core/src/utils/shell-utils.ts
View File

@@ -10,7 +10,12 @@ import os from 'node:os';
import { quote } from 'shell-quote';
import { doesToolInvocationMatch } from './tool-utils.js';
import { isShellCommandReadOnly } from './shellReadOnlyChecker.js';
import { spawn, type SpawnOptionsWithoutStdio } from 'node:child_process';
import {
execFile,
execFileSync,
type ExecFileOptions,
} from 'node:child_process';
import { accessSync, constants as fsConstants } from 'node:fs';
const SHELL_TOOL_NAMES = ['run_shell_command', 'ShellTool'];
@@ -455,46 +460,101 @@ export function checkCommandPermissions(
}
/**
* Determines whether a given shell command is allowed to execute based on
* the tool's configuration including allowlists and blocklists.
* Executes a command with the given arguments without using a shell.
*
* This function operates in "default allow" mode. It is a wrapper around
* `checkCommandPermissions`.
* This is a wrapper around Node.js's `execFile`, which spawns a process
* directly without invoking a shell, making it safer than `exec`.
* It's suitable for short-running commands with limited output.
*
* @param command The shell command string to validate.
* @param config The application configuration.
* @returns An object with 'allowed' boolean and optional 'reason' string if not allowed.
* @param command The command to execute (e.g., 'git', 'osascript').
* @param args Array of arguments to pass to the command.
* @param options Optional spawn options including:
* - preserveOutputOnError: If false (default), rejects on error.
* If true, resolves with output and error code.
* - Other standard spawn options (e.g., cwd, env).
* @returns A promise that resolves with stdout, stderr strings, and exit code.
* @throws Rejects with an error if the command fails (unless preserveOutputOnError is true).
*/
export const spawnAsync = (
export function execCommand(
command: string,
args: string[],
options?: SpawnOptionsWithoutStdio,
): Promise<{ stdout: string; stderr: string }> =>
new Promise((resolve, reject) => {
const child = spawn(command, args, options);
let stdout = '';
let stderr = '';
child.stdout.on('data', (data) => {
stdout += data.toString();
});
child.stderr.on('data', (data) => {
stderr += data.toString();
});
child.on('close', (code) => {
if (code === 0) {
resolve({ stdout, stderr });
} else {
reject(new Error(`Command failed with exit code ${code}:\n${stderr}`));
}
});
child.on('error', (err) => {
reject(err);
});
options: { preserveOutputOnError?: boolean } & ExecFileOptions = {},
): Promise<{ stdout: string; stderr: string; code: number }> {
return new Promise((resolve, reject) => {
const child = execFile(
command,
args,
{ encoding: 'utf8', ...options },
(error, stdout, stderr) => {
if (error) {
if (!options.preserveOutputOnError) {
reject(error);
} else {
resolve({
stdout: stdout ?? '',
stderr: stderr ?? '',
code: typeof error.code === 'number' ? error.code : 1,
});
}
return;
}
resolve({ stdout: stdout ?? '', stderr: stderr ?? '', code: 0 });
},
);
child.on('error', reject);
});
}
/**
* Resolves the path of a command in the system's PATH.
* @param {string} command The command name (e.g., 'git', 'grep').
* @returns {path: string | null; error?: Error} The path of the command, or null if it is not found and any error that occurred.
*/
export function resolveCommandPath(command: string): {
path: string | null;
error?: Error;
} {
try {
const isWin = process.platform === 'win32';
const checkCommand = isWin ? 'where' : 'command';
const checkArgs = isWin ? [command] : ['-v', command];
let result: string | null = null;
try {
result = execFileSync(checkCommand, checkArgs, {
encoding: 'utf8',
shell: isWin,
}).trim();
} catch {
console.warn(`Command ${checkCommand} not found`);
}
if (!result) return { path: null, error: undefined };
if (!isWin) {
accessSync(result, fsConstants.X_OK);
}
return { path: result, error: undefined };
} catch (error) {
return {
path: null,
error: error instanceof Error ? error : new Error(String(error)),
};
}
}
/**
* Checks if a command is available in the system's PATH.
* @param {string} command The command name (e.g., 'git', 'grep').
* @returns {available: boolean; error?: Error} The availability of the command and any error that occurred.
*/
export function isCommandAvailable(command: string): {
available: boolean;
error?: Error;
} {
const { path, error } = resolveCommandPath(command);
return { available: path !== null, error };
}
export function isCommandAllowed(
command: string,