diff --git a/packages/cli/src/zed-integration/schema.ts b/packages/cli/src/zed-integration/schema.ts index 4c962131..b35cc47d 100644 --- a/packages/cli/src/zed-integration/schema.ts +++ b/packages/cli/src/zed-integration/schema.ts @@ -84,6 +84,8 @@ export type AgentCapabilities = z.infer; export type AuthMethod = z.infer; +export type PromptCapabilities = z.infer; + export type ClientResponse = z.infer; export type ClientNotification = z.infer; @@ -270,8 +272,15 @@ export const mcpServerSchema = z.object({ name: z.string(), }); +export const promptCapabilitiesSchema = z.object({ + audio: z.boolean().optional(), + embeddedContext: z.boolean().optional(), + image: z.boolean().optional(), +}); + export const agentCapabilitiesSchema = z.object({ - loadSession: z.boolean(), + loadSession: z.boolean().optional(), + promptCapabilities: promptCapabilitiesSchema.optional(), }); export const authMethodSchema = z.object({ diff --git a/packages/cli/src/zed-integration/zedIntegration.ts b/packages/cli/src/zed-integration/zedIntegration.ts index 79f90637..40948b39 100644 --- a/packages/cli/src/zed-integration/zedIntegration.ts +++ b/packages/cli/src/zed-integration/zedIntegration.ts @@ -99,6 +99,11 @@ class GeminiAgent { authMethods, agentCapabilities: { loadSession: false, + promptCapabilities: { + image: true, + audio: true, + embeddedContext: true, + }, }, }; } @@ -488,45 +493,59 @@ class Session { message: acp.ContentBlock[], abortSignal: AbortSignal, ): Promise { + const FILE_URI_SCHEME = 'file://'; + + const embeddedContext: acp.EmbeddedResourceResource[] = []; + const parts = message.map((part) => { switch (part.type) { case 'text': return { text: part.text }; - case 'resource_link': + case 'image': + case 'audio': return { - fileData: { - mimeData: part.mimeType, - name: part.name, - fileUri: part.uri, + inlineData: { + mimeType: part.mimeType, + data: part.data, }, }; + case 'resource_link': { + if (part.uri.startsWith(FILE_URI_SCHEME)) { + return { + fileData: { + mimeData: part.mimeType, + name: part.name, + fileUri: part.uri.slice(FILE_URI_SCHEME.length), + }, + }; + } else { + return { text: `@${part.uri}` }; + } + } case 'resource': { - return { - fileData: { - mimeData: part.resource.mimeType, - name: part.resource.uri, - fileUri: part.resource.uri, - }, - }; + embeddedContext.push(part.resource); + return { text: `@${part.resource.uri}` }; } default: { - throw new Error(`Unexpected chunk type: '${part.type}'`); + const unreachable: never = part; + throw new Error(`Unexpected chunk type: '${unreachable}'`); } } }); const atPathCommandParts = parts.filter((part) => 'fileData' in part); - if (atPathCommandParts.length === 0) { + if (atPathCommandParts.length === 0 && embeddedContext.length === 0) { return parts; } + const atPathToResolvedSpecMap = new Map(); + // Get centralized file discovery service const fileDiscovery = this.config.getFileService(); const respectGitIgnore = this.config.getFileFilteringRespectGitIgnore(); const pathSpecsToRead: string[] = []; - const atPathToResolvedSpecMap = new Map(); const contentLabelsForDisplay: string[] = []; const ignoredPaths: string[] = []; @@ -634,6 +653,7 @@ class Session { contentLabelsForDisplay.push(pathName); } } + // Construct the initial part of the query for the LLM let initialQueryText = ''; for (let i = 0; i < parts.length; i++) { @@ -687,93 +707,123 @@ class Session { `Ignored ${ignoredPaths.length} ${ignoreType} files: ${ignoredPaths.join(', ')}`, ); } - // Fallback for lone "@" or completely invalid @-commands resulting in empty initialQueryText - if (pathSpecsToRead.length === 0) { + + const processedQueryParts: Part[] = [{ text: initialQueryText }]; + + if (pathSpecsToRead.length === 0 && embeddedContext.length === 0) { + // Fallback for lone "@" or completely invalid @-commands resulting in empty initialQueryText console.warn('No valid file paths found in @ commands to read.'); return [{ text: initialQueryText }]; } - const processedQueryParts: Part[] = [{ text: initialQueryText }]; - const toolArgs = { - paths: pathSpecsToRead, - respectGitIgnore, // Use configuration setting - }; - const callId = `${readManyFilesTool.name}-${Date.now()}`; - - try { - const invocation = readManyFilesTool.build(toolArgs); - - await this.sendUpdate({ - sessionUpdate: 'tool_call', - toolCallId: callId, - status: 'in_progress', - title: invocation.getDescription(), - content: [], - locations: invocation.toolLocations(), - kind: readManyFilesTool.kind, - }); - - const result = await invocation.execute(abortSignal); - const content = toToolCallContent(result) || { - type: 'content', - content: { - type: 'text', - text: `Successfully read: ${contentLabelsForDisplay.join(', ')}`, - }, + if (pathSpecsToRead.length > 0) { + const toolArgs = { + paths: pathSpecsToRead, + respectGitIgnore, // Use configuration setting }; - await this.sendUpdate({ - sessionUpdate: 'tool_call_update', - toolCallId: callId, - status: 'completed', - content: content ? [content] : [], - }); - if (Array.isArray(result.llmContent)) { - const fileContentRegex = /^--- (.*?) ---\n\n([\s\S]*?)\n\n$/; - processedQueryParts.push({ - text: '\n--- Content from referenced files ---', + + const callId = `${readManyFilesTool.name}-${Date.now()}`; + + try { + const invocation = readManyFilesTool.build(toolArgs); + + await this.sendUpdate({ + sessionUpdate: 'tool_call', + toolCallId: callId, + status: 'in_progress', + title: invocation.getDescription(), + content: [], + locations: invocation.toolLocations(), + kind: readManyFilesTool.kind, }); - for (const part of result.llmContent) { - if (typeof part === 'string') { - const match = fileContentRegex.exec(part); - if (match) { - const filePathSpecInContent = match[1]; // This is a resolved pathSpec - const fileActualContent = match[2].trim(); - processedQueryParts.push({ - text: `\nContent from @${filePathSpecInContent}:\n`, - }); - processedQueryParts.push({ text: fileActualContent }); - } else { - processedQueryParts.push({ text: part }); - } - } else { - // part is a Part object. - processedQueryParts.push(part); - } - } - } else { - console.warn( - 'read_many_files tool returned no content or empty content.', - ); - } - return processedQueryParts; - } catch (error: unknown) { - await this.sendUpdate({ - sessionUpdate: 'tool_call_update', - toolCallId: callId, - status: 'failed', - content: [ - { - type: 'content', - content: { - type: 'text', - text: `Error reading files (${contentLabelsForDisplay.join(', ')}): ${getErrorMessage(error)}`, - }, + + const result = await invocation.execute(abortSignal); + const content = toToolCallContent(result) || { + type: 'content', + content: { + type: 'text', + text: `Successfully read: ${contentLabelsForDisplay.join(', ')}`, }, - ], + }; + await this.sendUpdate({ + sessionUpdate: 'tool_call_update', + toolCallId: callId, + status: 'completed', + content: content ? [content] : [], + }); + if (Array.isArray(result.llmContent)) { + const fileContentRegex = /^--- (.*?) ---\n\n([\s\S]*?)\n\n$/; + processedQueryParts.push({ + text: '\n--- Content from referenced files ---', + }); + for (const part of result.llmContent) { + if (typeof part === 'string') { + const match = fileContentRegex.exec(part); + if (match) { + const filePathSpecInContent = match[1]; // This is a resolved pathSpec + const fileActualContent = match[2].trim(); + processedQueryParts.push({ + text: `\nContent from @${filePathSpecInContent}:\n`, + }); + processedQueryParts.push({ text: fileActualContent }); + } else { + processedQueryParts.push({ text: part }); + } + } else { + // part is a Part object. + processedQueryParts.push(part); + } + } + } else { + console.warn( + 'read_many_files tool returned no content or empty content.', + ); + } + } catch (error: unknown) { + await this.sendUpdate({ + sessionUpdate: 'tool_call_update', + toolCallId: callId, + status: 'failed', + content: [ + { + type: 'content', + content: { + type: 'text', + text: `Error reading files (${contentLabelsForDisplay.join(', ')}): ${getErrorMessage(error)}`, + }, + }, + ], + }); + + throw error; + } + } + + if (embeddedContext.length > 0) { + processedQueryParts.push({ + text: '\n--- Content from referenced context ---', }); - throw error; + for (const contextPart of embeddedContext) { + processedQueryParts.push({ + text: `\nContent from @${contextPart.uri}:\n`, + }); + if ('text' in contextPart) { + processedQueryParts.push({ + text: contextPart.text, + }); + } else { + processedQueryParts.push({ + inlineData: { + mimeType: contextPart.mimeType ?? 'application/octet-stream', + data: contextPart.blob, + }, + }); + } + } } + + return processedQueryParts; } debug(msg: string) { @@ -784,6 +834,10 @@ class Session { } function toToolCallContent(toolResult: ToolResult): acp.ToolCallContent | null { + if (toolResult.error?.message) { + throw new Error(toolResult.error.message); + } + if (toolResult.returnDisplay) { if (typeof toolResult.returnDisplay === 'string') { return {