feat: add multi websearch provider

This commit is contained in:
pomelo-nwu
2025-10-24 17:16:14 +08:00
parent 5cf609c367
commit f9f6eb52dd
11 changed files with 955 additions and 4 deletions

View File

@@ -771,6 +771,7 @@ export async function loadCliConfig(
output: {
format: (argv.outputFormat ?? settings.output?.format) as OutputFormat,
},
webSearch: settings.webSearch,
});
}

View File

@@ -1055,6 +1055,25 @@ const SETTINGS_SCHEMA = {
},
},
webSearch: {
type: 'object',
label: 'Web Search',
category: 'Advanced',
requiresRestart: true,
default: undefined as
| {
provider: Array<{
type: 'tavily' | 'google' | 'dashscope';
apiKey?: string;
searchEngineId?: string;
}>;
default: string;
}
| undefined,
description: 'Configuration for web search providers.',
showInDialog: false,
},
experimental: {
type: 'object',
label: 'Experimental',

View File

@@ -56,7 +56,7 @@ import { TaskTool } from '../tools/task.js';
import { TodoWriteTool } from '../tools/todoWrite.js';
import { ToolRegistry } from '../tools/tool-registry.js';
import { WebFetchTool } from '../tools/web-fetch.js';
import { WebSearchTool } from '../tools/web-search.js';
import { WebSearchTool } from '../tools/web-search/index.js';
import { WriteFileTool } from '../tools/write-file.js';
// Other modules
@@ -261,6 +261,14 @@ export interface ConfigParameters {
loadMemoryFromIncludeDirectories?: boolean;
// Web search providers
tavilyApiKey?: string;
webSearch?: {
provider: Array<{
type: 'tavily' | 'google' | 'dashscope';
apiKey?: string;
searchEngineId?: string;
}>;
default: string;
};
chatCompression?: ChatCompressionSettings;
interactive?: boolean;
trustedFolder?: boolean;
@@ -349,6 +357,14 @@ export class Config {
private readonly experimentalZedIntegration: boolean = false;
private readonly loadMemoryFromIncludeDirectories: boolean = false;
private readonly tavilyApiKey?: string;
private readonly webSearch?: {
provider: Array<{
type: 'tavily' | 'google' | 'dashscope';
apiKey?: string;
searchEngineId?: string;
}>;
default: string;
};
private readonly chatCompression: ChatCompressionSettings | undefined;
private readonly interactive: boolean;
private readonly trustedFolder: boolean | undefined;
@@ -454,6 +470,7 @@ export class Config {
// Web search
this.tavilyApiKey = params.tavilyApiKey;
this.webSearch = params.webSearch;
this.useRipgrep = params.useRipgrep ?? true;
this.shouldUseNodePtyShell = params.shouldUseNodePtyShell ?? false;
this.skipNextSpeakerCheck = params.skipNextSpeakerCheck ?? true;
@@ -891,6 +908,31 @@ export class Config {
return this.tavilyApiKey;
}
getWebSearchConfig():
| {
provider: Array<{
type: 'tavily' | 'google' | 'dashscope';
config: Record<string, unknown>;
}>;
default: string;
}
| undefined {
if (!this.webSearch) {
return undefined;
}
return {
provider: this.webSearch.provider.map((p) => ({
type: p.type,
config: {
apiKey: p.apiKey,
searchEngineId: p.searchEngineId,
},
})),
default: this.webSearch.default,
};
}
getIdeMode(): boolean {
return this.ideMode;
}
@@ -1118,8 +1160,12 @@ export class Config {
registerCoreTool(TodoWriteTool, this);
registerCoreTool(ExitPlanModeTool, this);
registerCoreTool(WebFetchTool, this);
// Conditionally register web search tool only if Tavily API key is set
if (this.getTavilyApiKey()) {
// Conditionally register web search tool if any web search provider is configured
// or if using qwen-oauth authentication
if (
this.getWebSearchConfig() ||
this.getAuthType() === AuthType.QWEN_OAUTH
) {
registerCoreTool(WebSearchTool, this);
}

View File

@@ -97,7 +97,7 @@ export * from './tools/write-file.js';
export * from './tools/web-fetch.js';
export * from './tools/memoryTool.js';
export * from './tools/shell.js';
export * from './tools/web-search.js';
export * from './tools/web-search/index.js';
export * from './tools/read-many-files.js';
export * from './tools/mcp-client.js';
export * from './tools/mcp-tool.js';

View File

@@ -0,0 +1,101 @@
/**
* @license
* Copyright 2025 Qwen
* SPDX-License-Identifier: Apache-2.0
*/
import type {
WebSearchProvider,
WebSearchResult,
WebSearchResultItem,
} from './types.js';
import { getErrorMessage } from '../../utils/errors.js';
/**
* Base implementation for web search providers.
*/
export abstract class BaseWebSearchProvider implements WebSearchProvider {
abstract readonly name: string;
/**
* Check if the provider is available (has required configuration).
*/
abstract isAvailable(): boolean;
/**
* Perform the actual search implementation.
* @param query The search query
* @param signal Abort signal for cancellation
* @returns Promise resolving to search results
*/
protected abstract performSearch(
query: string,
signal: AbortSignal,
): Promise<WebSearchResult>;
/**
* Execute a web search with error handling.
* @param query The search query
* @param signal Abort signal for cancellation
* @returns Promise resolving to search results
*/
async search(query: string, signal: AbortSignal): Promise<WebSearchResult> {
if (!this.isAvailable()) {
throw new Error(
`${this.name} provider is not available. Please check your configuration.`,
);
}
try {
return await this.performSearch(query, signal);
} catch (error: unknown) {
const errorMessage = getErrorMessage(error);
throw new Error(`Error during ${this.name} search: ${errorMessage}`);
}
}
/**
* Format search results into a consistent format.
* @param results Raw results from the provider
* @param query The original search query
* @returns Formatted search results
*/
protected formatResults(
results: WebSearchResultItem[],
query: string,
answer?: string,
): WebSearchResult {
return {
query,
answer,
results,
};
}
/**
* Create a formatted source list for display.
* @param results Search result items
* @returns Formatted source list
*/
protected createSourceList(results: WebSearchResultItem[]): string {
return results
.map((r, i) => `[${i + 1}] ${r.title || 'Untitled'} (${r.url})`)
.join('\n');
}
/**
* Build a concise summary from search results.
* @param results Search result items
* @param maxResults Maximum number of results to include
* @returns Concise summary string
*/
protected buildSummary(
results: WebSearchResultItem[],
maxResults: number = 3,
): string {
return results
.slice(0, maxResults)
.map((r, i) => `${i + 1}. ${r.title} - ${r.url}`)
.join('\n');
}
}

View File

@@ -0,0 +1,235 @@
/**
* @license
* Copyright 2025 Qwen
* SPDX-License-Identifier: Apache-2.0
*/
import {
BaseDeclarativeTool,
BaseToolInvocation,
Kind,
type ToolInvocation,
type ToolCallConfirmationDetails,
type ToolInfoConfirmationDetails,
ToolConfirmationOutcome,
} from '../tools.js';
import type { Config } from '../../config/config.js';
import { ApprovalMode } from '../../config/config.js';
import { getErrorMessage } from '../../utils/errors.js';
import { WebSearchProviderFactory } from './provider-factory.js';
import type {
WebSearchToolParams,
WebSearchToolResult,
WebSearchProvider,
WebSearchResultItem,
} from './types.js';
class WebSearchToolInvocation extends BaseToolInvocation<
WebSearchToolParams,
WebSearchToolResult
> {
constructor(
private readonly config: Config,
params: WebSearchToolParams,
) {
super(params);
}
override getDescription(): string {
// Try to determine which provider will be used
const webSearchConfig = this.config.getWebSearchConfig();
const provider =
this.params.provider || webSearchConfig?.default || 'tavily';
return ` (Searching the web via ${provider})`;
}
override async shouldConfirmExecute(
_abortSignal: AbortSignal,
): Promise<ToolCallConfirmationDetails | false> {
if (this.config.getApprovalMode() === ApprovalMode.AUTO_EDIT) {
return false;
}
const confirmationDetails: ToolInfoConfirmationDetails = {
type: 'info',
title: 'Confirm Web Search',
prompt: `Search the web for: "${this.params.query}"`,
onConfirm: async (outcome: ToolConfirmationOutcome) => {
if (outcome === ToolConfirmationOutcome.ProceedAlways) {
this.config.setApprovalMode(ApprovalMode.AUTO_EDIT);
}
},
};
return confirmationDetails;
}
async execute(signal: AbortSignal): Promise<WebSearchToolResult> {
const webSearchConfig = this.config.getWebSearchConfig();
if (!webSearchConfig) {
return {
llmContent:
'Web search is disabled because no web search configuration is available. Please configure web search providers in your settings.json.',
returnDisplay:
'Web search disabled. Configure web search providers to enable search.',
};
}
const providers = WebSearchProviderFactory.createProviders(
webSearchConfig.provider,
);
// Determine which provider to use
let selectedProvider: WebSearchProvider | null = null;
if (this.params.provider) {
// Use the specified provider if available
const provider = providers.get(this.params.provider);
if (provider && provider.isAvailable()) {
selectedProvider = provider;
} else {
return {
llmContent: `The specified provider "${this.params.provider}" is not available or not configured. Available providers: ${Array.from(providers.keys()).join(', ')}`,
returnDisplay: `The WebSearch Provider "${this.params.provider}" not available.`,
};
}
} else {
// Use default provider
selectedProvider = WebSearchProviderFactory.getDefaultProvider(
providers,
webSearchConfig.default,
);
}
if (!selectedProvider) {
return {
llmContent:
'Web search is disabled because no web search providers are available. Please check your configuration.',
returnDisplay: 'Web search disabled. No available providers.',
};
}
try {
const searchResult = await selectedProvider.search(
this.params.query,
signal,
);
const sources = searchResult.results.map((r: WebSearchResultItem) => ({
title: r.title,
url: r.url,
}));
const sourceListFormatted = sources.map(
(s: { title: string; url: string }, i: number) =>
`[${i + 1}] ${s.title || 'Untitled'} (${s.url})`,
);
let content = searchResult.answer?.trim() || '';
if (!content) {
// Fallback: build a concise summary from top results
content = sources
.slice(0, 3)
.map(
(s: { title: string; url: string }, i: number) =>
`${i + 1}. ${s.title} - ${s.url}`,
)
.join('\n');
}
if (sourceListFormatted.length > 0) {
content += `\n\nSources:\n${sourceListFormatted.join('\n')}`;
}
if (!content.trim()) {
return {
llmContent: `No search results or information found for query: "${this.params.query}" (searched via ${selectedProvider.name})`,
returnDisplay: `No information found for "${this.params.query}".`,
};
}
return {
llmContent: `Web search results for "${this.params.query}" (via ${selectedProvider.name}):\n\n${content}`,
returnDisplay: `Search results for "${this.params.query}".`,
sources,
};
} catch (error: unknown) {
const errorMessage = `Error during web search for query "${this.params.query}": ${getErrorMessage(
error,
)}`;
console.error(errorMessage, error);
return {
llmContent: `Error: ${errorMessage}`,
returnDisplay: `Error performing web search.`,
};
}
}
}
/**
* A tool to perform web searches using configurable providers.
*/
export class WebSearchTool extends BaseDeclarativeTool<
WebSearchToolParams,
WebSearchToolResult
> {
static readonly Name: string = 'web_search';
constructor(private readonly config: Config) {
super(
WebSearchTool.Name,
'WebSearch',
'Performs a web search using configurable providers and returns a concise answer with sources.',
Kind.Search,
{
type: 'object',
properties: {
query: {
type: 'string',
description: 'The search query to find information on the web.',
},
provider: {
type: 'string',
description:
'Optional provider to use for the search (e.g., "tavily", "google", "dashscope"). If not specified, the default provider will be used.',
},
},
required: ['query'],
},
);
}
/**
* Validates the parameters for the WebSearchTool.
* @param params The parameters to validate
* @returns An error message string if validation fails, null if valid
*/
protected override validateToolParamValues(
params: WebSearchToolParams,
): string | null {
if (!params.query || params.query.trim() === '') {
return "The 'query' parameter cannot be empty.";
}
// Validate provider parameter if provided
if (params.provider !== undefined && params.provider.trim() === '') {
return "The 'provider' parameter cannot be empty if specified.";
}
return null;
}
protected createInvocation(
params: WebSearchToolParams,
): ToolInvocation<WebSearchToolParams, WebSearchToolResult> {
return new WebSearchToolInvocation(this.config, params);
}
}
// Re-export types for external use
export type {
WebSearchToolParams,
WebSearchToolResult,
WebSearchConfig,
WebSearchProviderConfig,
} from './types.js';

View File

@@ -0,0 +1,110 @@
/**
* @license
* Copyright 2025 Qwen
* SPDX-License-Identifier: Apache-2.0
*/
import type { WebSearchProvider, WebSearchProviderConfig } from './types.js';
import {
TavilyProvider,
type TavilyConfig,
} from './providers/tavily-provider.js';
import {
GoogleProvider,
type GoogleConfig,
} from './providers/google-provider.js';
import {
DashScopeProvider,
type DashScopeConfig,
} from './providers/dashscope-provider.js';
/**
* Factory for creating web search providers based on configuration.
*/
export class WebSearchProviderFactory {
/**
* Create a web search provider from configuration.
* @param config Provider configuration
* @returns Web search provider instance
*/
static createProvider(config: WebSearchProviderConfig): WebSearchProvider {
switch (config.type) {
case 'tavily': {
const tavilyConfig = config.config as unknown as TavilyConfig;
if (!tavilyConfig?.apiKey) {
throw new Error('Tavily provider requires apiKey in configuration');
}
return new TavilyProvider(tavilyConfig);
}
case 'google': {
const googleConfig = config.config as unknown as GoogleConfig;
if (!googleConfig?.apiKey || !googleConfig?.searchEngineId) {
throw new Error(
'Google provider requires apiKey and searchEngineId in configuration',
);
}
return new GoogleProvider(googleConfig);
}
case 'dashscope': {
const dashscopeConfig = config.config as unknown as DashScopeConfig;
return new DashScopeProvider(dashscopeConfig);
}
default:
throw new Error(
`Unsupported web search provider type: ${(config as WebSearchProviderConfig).type}`,
);
}
}
/**
* Create multiple providers from configuration list.
* @param configs List of provider configurations
* @returns Map of provider name to provider instance
*/
static createProviders(
configs: WebSearchProviderConfig[],
): Map<string, WebSearchProvider> {
const providers = new Map<string, WebSearchProvider>();
for (const config of configs) {
try {
const provider = this.createProvider(config);
providers.set(config.type, provider);
} catch (error) {
console.warn(`Failed to create ${config.type} provider:`, error);
}
}
return providers;
}
/**
* Get the default provider from a list of providers.
* @param providers Map of available providers
* @param defaultProviderName Name of the default provider
* @returns Default provider or the first available provider
*/
static getDefaultProvider(
providers: Map<string, WebSearchProvider>,
defaultProviderName?: string,
): WebSearchProvider | null {
if (defaultProviderName && providers.has(defaultProviderName)) {
const provider = providers.get(defaultProviderName)!;
if (provider.isAvailable()) {
return provider;
}
}
// Fallback to first available provider
for (const provider of providers.values()) {
if (provider.isAvailable()) {
return provider;
}
}
return null;
}
}

View File

@@ -0,0 +1,138 @@
/**
* @license
* Copyright 2025 Qwen
* SPDX-License-Identifier: Apache-2.0
*/
import { BaseWebSearchProvider } from '../base-provider.js';
import type { WebSearchResult, WebSearchResultItem } from '../types.js';
interface DashScopeSearchItem {
_id: string;
snippet: string;
title: string;
url: string;
timestamp: number;
timestamp_format: string;
hostname: string;
hostlogo?: string;
web_main_body?: string;
_score?: number;
}
interface DashScopeSearchResponse {
headers: Record<string, unknown>;
rid: string;
status: number;
message: string | null;
data: {
total: number;
totalDistinct: number;
docs: DashScopeSearchItem[];
keywords?: string[];
qpInfos?: Array<{
query: string;
cleanQuery: string;
sensitive: boolean;
spellchecked: string;
spellcheck: boolean;
tokenized: string[];
stopWords: string[];
synonymWords: string[];
recognitions: unknown[];
rewrite: string;
operator: string;
}>;
aggs?: unknown;
extras?: Record<string, unknown>;
};
debug?: unknown;
success: boolean;
}
/**
* Configuration for DashScope provider.
*/
export interface DashScopeConfig {
apiKey: string;
uid: string;
appId: string;
maxResults?: number;
scene?: string;
timeout?: number;
}
/**
* Web search provider using Alibaba Cloud DashScope API.
*/
export class DashScopeProvider extends BaseWebSearchProvider {
readonly name = 'DashScope';
constructor(private readonly config: DashScopeConfig) {
super();
}
isAvailable(): boolean {
return !!(this.config.apiKey && this.config.uid && this.config.appId);
}
protected async performSearch(
query: string,
signal: AbortSignal,
): Promise<WebSearchResult> {
const requestBody = {
rid: '',
uid: this.config.uid,
scene: this.config.scene || 'dolphin_search_inner_turbo',
uq: query,
fields: [],
page: 1,
rows: this.config.maxResults || 10,
customConfigInfo: {},
headers: {
__d_head_qto: this.config.timeout || 8000,
__d_head_app: this.config.appId,
},
};
const response = await fetch(
'https://dashscope.aliyuncs.com/api/v1/indices/plugin/web_search',
{
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: this.config.apiKey,
},
body: JSON.stringify(requestBody),
signal,
},
);
if (!response.ok) {
const text = await response.text().catch(() => '');
throw new Error(
`DashScope API error: ${response.status} ${response.statusText}${text ? ` - ${text}` : ''}`,
);
}
const data = (await response.json()) as DashScopeSearchResponse;
if (data.status !== 0) {
throw new Error(
`DashScope API error: ${data.message || 'Unknown error'}`,
);
}
const results: WebSearchResultItem[] = (data.data?.docs || []).map(
(item) => ({
title: item.title,
url: item.url,
content: item.snippet,
score: item._score,
publishedDate: item.timestamp_format,
}),
);
return this.formatResults(results, query, undefined);
}
}

View File

@@ -0,0 +1,96 @@
/**
* @license
* Copyright 2025 Qwen
* SPDX-License-Identifier: Apache-2.0
*/
import { BaseWebSearchProvider } from '../base-provider.js';
import type { WebSearchResult, WebSearchResultItem } from '../types.js';
interface GoogleSearchItem {
title: string;
link: string;
snippet?: string;
displayLink?: string;
formattedUrl?: string;
}
interface GoogleSearchResponse {
items?: GoogleSearchItem[];
searchInformation?: {
totalResults?: string;
searchTime?: number;
};
}
/**
* Configuration for Google provider.
*/
export interface GoogleConfig {
apiKey: string;
searchEngineId: string;
maxResults?: number;
safeSearch?: 'off' | 'medium' | 'high';
language?: string;
country?: string;
}
/**
* Web search provider using Google Custom Search API.
*/
export class GoogleProvider extends BaseWebSearchProvider {
readonly name = 'Google';
constructor(private readonly config: GoogleConfig) {
super();
}
isAvailable(): boolean {
return !!(this.config.apiKey && this.config.searchEngineId);
}
protected async performSearch(
query: string,
signal: AbortSignal,
): Promise<WebSearchResult> {
const params = new URLSearchParams({
key: this.config.apiKey,
cx: this.config.searchEngineId,
q: query,
num: String(this.config.maxResults || 10),
safe: this.config.safeSearch || 'medium',
});
if (this.config.language) {
params.append('lr', `lang_${this.config.language}`);
}
if (this.config.country) {
params.append('cr', `country${this.config.country}`);
}
const url = `https://www.googleapis.com/customsearch/v1?${params.toString()}`;
const response = await fetch(url, {
method: 'GET',
signal,
});
if (!response.ok) {
const text = await response.text().catch(() => '');
throw new Error(
`Google Search API error: ${response.status} ${response.statusText}${text ? ` - ${text}` : ''}`,
);
}
const data = (await response.json()) as GoogleSearchResponse;
const results: WebSearchResultItem[] = (data.items || []).map((item) => ({
title: item.title,
url: item.link,
content: item.snippet,
}));
return this.formatResults(results, query, undefined);
}
}

View File

@@ -0,0 +1,86 @@
/**
* @license
* Copyright 2025 Qwen
* SPDX-License-Identifier: Apache-2.0
*/
import { BaseWebSearchProvider } from '../base-provider.js';
import type { WebSearchResult, WebSearchResultItem } from '../types.js';
interface TavilyResultItem {
title: string;
url: string;
content?: string;
score?: number;
published_date?: string;
}
interface TavilySearchResponse {
query: string;
answer?: string;
results: TavilyResultItem[];
}
/**
* Configuration for Tavily provider.
*/
export interface TavilyConfig {
apiKey: string;
searchDepth?: 'basic' | 'advanced';
maxResults?: number;
includeAnswer?: boolean;
}
/**
* Web search provider using Tavily API.
*/
export class TavilyProvider extends BaseWebSearchProvider {
readonly name = 'Tavily';
constructor(private readonly config: TavilyConfig) {
super();
}
isAvailable(): boolean {
return !!this.config.apiKey;
}
protected async performSearch(
query: string,
signal: AbortSignal,
): Promise<WebSearchResult> {
const response = await fetch('https://api.tavily.com/search', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
api_key: this.config.apiKey,
query,
search_depth: this.config.searchDepth || 'advanced',
max_results: this.config.maxResults || 5,
include_answer: this.config.includeAnswer !== false,
}),
signal,
});
if (!response.ok) {
const text = await response.text().catch(() => '');
throw new Error(
`Tavily API error: ${response.status} ${response.statusText}${text ? ` - ${text}` : ''}`,
);
}
const data = (await response.json()) as TavilySearchResponse;
const results: WebSearchResultItem[] = (data.results || []).map((r) => ({
title: r.title,
url: r.url,
content: r.content,
score: r.score,
publishedDate: r.published_date,
}));
return this.formatResults(results, query, data.answer?.trim());
}
}

View File

@@ -0,0 +1,119 @@
/**
* @license
* Copyright 2025 Qwen
* SPDX-License-Identifier: Apache-2.0
*/
import type { ToolResult } from '../tools.js';
/**
* Common interface for all web search providers.
*/
export interface WebSearchProvider {
/**
* The name of the provider.
*/
readonly name: string;
/**
* Whether the provider is available (has required configuration).
*/
isAvailable(): boolean;
/**
* Perform a web search with the given query.
* @param query The search query
* @param signal Abort signal for cancellation
* @returns Promise resolving to search results
*/
search(query: string, signal: AbortSignal): Promise<WebSearchResult>;
}
/**
* Result item from a web search.
*/
export interface WebSearchResultItem {
title: string;
url: string;
content?: string;
score?: number;
publishedDate?: string;
}
/**
* Result from a web search operation.
*/
export interface WebSearchResult {
/**
* The search query that was executed.
*/
query: string;
/**
* A concise answer if available from the provider.
*/
answer?: string;
/**
* List of search result items.
*/
results: WebSearchResultItem[];
/**
* Provider-specific metadata.
*/
metadata?: Record<string, unknown>;
}
/**
* Extended tool result that includes sources for web search.
*/
export interface WebSearchToolResult extends ToolResult {
sources?: Array<{ title: string; url: string }>;
}
/**
* Parameters for the WebSearchTool.
*/
export interface WebSearchToolParams {
/**
* The search query.
*/
query: string;
/**
* Optional provider to use for the search.
* If not specified, the default provider will be used.
*/
provider?: string;
}
/**
* Configuration for web search providers.
*/
export interface WebSearchConfig {
/**
* List of available providers with their configurations.
*/
providers: WebSearchProviderConfig[];
/**
* The default provider to use.
*/
default: string;
}
/**
* Base configuration for a web search provider.
*/
export interface WebSearchProviderConfig {
/**
* The type of provider.
*/
type: 'tavily' | 'google' | 'dashscope';
/**
* Provider-specific configuration.
*/
config?: Record<string, unknown>;
}