Support Images and PDFs (#447)

This commit is contained in:
Jacob Richman
2025-05-20 13:02:41 -07:00
committed by GitHub
parent 4002e980d9
commit 716f7875a2
14 changed files with 965 additions and 56 deletions

View File

@@ -0,0 +1,48 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { vi } from 'vitest';
import * as actualFsPromises from 'node:fs/promises';
const readFileMock = vi.fn();
// Export a control object so tests can access and manipulate the mock
export const mockControl = {
mockReadFile: readFileMock,
};
// Export all other functions from the actual fs/promises module
export const {
access,
appendFile,
chmod,
chown,
copyFile,
cp,
lchmod,
lchown,
link,
lstat,
mkdir,
open,
opendir,
readdir,
readlink,
realpath,
rename,
rmdir,
rm,
stat,
symlink,
truncate,
unlink,
utimes,
watch,
writeFile,
} = actualFsPromises;
// Override readFile with our mock
export const readFile = readFileMock;

View File

@@ -0,0 +1,71 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { type PartListUnion, type Part } from '@google/genai';
/**
* Represents a request to be sent to the Gemini API.
* For now, it's an alias to PartListUnion as the primary content.
* This can be expanded later to include other request parameters.
*/
export type GeminiCodeRequest = PartListUnion;
export function partListUnionToString(value: PartListUnion): string {
if (typeof value === 'string') {
return value;
}
if (Array.isArray(value)) {
return value.map(partListUnionToString).join('');
}
// Cast to Part, assuming it might contain project-specific fields
const part = value as Part & {
videoMetadata?: unknown;
thought?: string;
codeExecutionResult?: unknown;
executableCode?: unknown;
};
if (part.videoMetadata !== undefined) {
return `[Video Metadata]`;
}
if (part.thought !== undefined) {
return `[Thought: ${part.thought}]`;
}
if (part.codeExecutionResult !== undefined) {
return `[Code Execution Result]`;
}
if (part.executableCode !== undefined) {
return `[Executable Code]`;
}
// Standard Part fields
if (part.fileData !== undefined) {
return `[File Data]`;
}
if (part.functionCall !== undefined) {
return `[Function Call: ${part.functionCall.name}]`;
}
if (part.functionResponse !== undefined) {
return `[Function Response: ${part.functionResponse.name}]`;
}
if (part.inlineData !== undefined) {
return `<${part.inlineData.mimeType}>`;
}
if (part.text !== undefined) {
return part.text;
}
return '';
}

View File

@@ -11,6 +11,7 @@ export * from './config/config.js';
export * from './core/client.js';
export * from './core/prompts.js';
export * from './core/turn.js';
export * from './core/geminiRequest.js';
// Potentially export types from turn.ts if needed externally
// export { GeminiEventType } from './core/turn.js'; // Example

View File

@@ -5,6 +5,7 @@
*/
import { GlobTool, GlobToolParams } from './glob.js';
import { partListUnionToString } from '../core/geminiRequest.js';
// import { ToolResult } from './tools.js'; // ToolResult is implicitly used by execute
import path from 'path';
import fs from 'fs/promises';
@@ -134,9 +135,14 @@ describe('GlobTool', () => {
it('should correctly sort files by modification time (newest first)', async () => {
const params: GlobToolParams = { pattern: '*.sortme' };
const result = await globTool.execute(params, abortSignal);
expect(result.llmContent).toContain('Found 2 file(s)');
const filesListed = result.llmContent
.substring(result.llmContent.indexOf(':') + 1)
const llmContent = partListUnionToString(result.llmContent);
expect(llmContent).toContain('Found 2 file(s)');
// Ensure llmContent is a string for TypeScript type checking
expect(typeof llmContent).toBe('string');
const filesListed = llmContent
.substring(llmContent.indexOf(':') + 1)
.trim()
.split('\n');
expect(filesListed[0]).toContain(path.join(tempRootDir, 'newer.sortme'));

View File

@@ -0,0 +1,332 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { vi } from 'vitest';
import type { Mock } from 'vitest';
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import { mockControl } from '../__mocks__/fs/promises.js';
import { ReadManyFilesTool } from './read-many-files.js';
import path from 'path';
import fs from 'fs'; // Actual fs for setup
import os from 'os';
describe('ReadManyFilesTool', () => {
let tool: ReadManyFilesTool;
let tempRootDir: string;
let tempDirOutsideRoot: string;
let mockReadFileFn: Mock;
beforeEach(async () => {
tempRootDir = fs.mkdtempSync(
path.join(os.tmpdir(), 'read-many-files-root-'),
);
tempDirOutsideRoot = fs.mkdtempSync(
path.join(os.tmpdir(), 'read-many-files-external-'),
);
tool = new ReadManyFilesTool(tempRootDir);
mockReadFileFn = mockControl.mockReadFile;
mockReadFileFn.mockReset();
mockReadFileFn.mockImplementation(
async (filePath: fs.PathLike, options?: Record<string, unknown>) => {
const fp =
typeof filePath === 'string'
? filePath
: (filePath as Buffer).toString();
if (fs.existsSync(fp)) {
const originalFs = await vi.importActual<typeof fs>('fs');
return originalFs.promises.readFile(fp, options);
}
if (fp.endsWith('nonexistent-file.txt')) {
const err = new Error(
`ENOENT: no such file or directory, open '${fp}'`,
);
(err as NodeJS.ErrnoException).code = 'ENOENT';
throw err;
}
if (fp.endsWith('unreadable.txt')) {
const err = new Error(`EACCES: permission denied, open '${fp}'`);
(err as NodeJS.ErrnoException).code = 'EACCES';
throw err;
}
if (fp.endsWith('.png'))
return Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); // PNG header
if (fp.endsWith('.pdf')) return Buffer.from('%PDF-1.4...'); // PDF start
if (fp.endsWith('binary.bin'))
return Buffer.from([0x00, 0x01, 0x02, 0x00, 0x03]);
const err = new Error(
`ENOENT: no such file or directory, open '${fp}' (unmocked path)`,
);
(err as NodeJS.ErrnoException).code = 'ENOENT';
throw err;
},
);
});
afterEach(() => {
if (fs.existsSync(tempRootDir)) {
fs.rmSync(tempRootDir, { recursive: true, force: true });
}
if (fs.existsSync(tempDirOutsideRoot)) {
fs.rmSync(tempDirOutsideRoot, { recursive: true, force: true });
}
});
describe('validateParams', () => {
it('should return null for valid relative paths within root', () => {
const params = { paths: ['file1.txt', 'subdir/file2.txt'] };
expect(tool.validateParams(params)).toBeNull();
});
it('should return null for valid glob patterns within root', () => {
const params = { paths: ['*.txt', 'subdir/**/*.js'] };
expect(tool.validateParams(params)).toBeNull();
});
it('should return null for paths trying to escape the root (e.g., ../) as execute handles this', () => {
const params = { paths: ['../outside.txt'] };
expect(tool.validateParams(params)).toBeNull();
});
it('should return null for absolute paths as execute handles this', () => {
const params = { paths: [path.join(tempDirOutsideRoot, 'absolute.txt')] };
expect(tool.validateParams(params)).toBeNull();
});
it('should return error if paths array is empty', () => {
const params = { paths: [] };
expect(tool.validateParams(params)).toBe(
'The "paths" parameter is required and must be a non-empty array of strings/glob patterns.',
);
});
it('should return null for valid exclude and include patterns', () => {
const params = {
paths: ['src/**/*.ts'],
exclude: ['**/*.test.ts'],
include: ['src/utils/*.ts'],
};
expect(tool.validateParams(params)).toBeNull();
});
});
describe('execute', () => {
const createFile = (filePath: string, content = '') => {
const fullPath = path.join(tempRootDir, filePath);
fs.mkdirSync(path.dirname(fullPath), { recursive: true });
fs.writeFileSync(fullPath, content);
};
const createBinaryFile = (filePath: string, data: Uint8Array) => {
const fullPath = path.join(tempRootDir, filePath);
fs.mkdirSync(path.dirname(fullPath), { recursive: true });
fs.writeFileSync(fullPath, data);
};
it('should read a single specified file', async () => {
createFile('file1.txt', 'Content of file1');
const params = { paths: ['file1.txt'] };
const result = await tool.execute(params, new AbortController().signal);
expect(result.llmContent).toEqual([
'--- file1.txt ---\n\nContent of file1\n\n',
]);
expect(result.returnDisplay).toContain(
'Successfully read and concatenated content from **1 file(s)**',
);
});
it('should read multiple specified files', async () => {
createFile('file1.txt', 'Content1');
createFile('subdir/file2.js', 'Content2');
const params = { paths: ['file1.txt', 'subdir/file2.js'] };
const result = await tool.execute(params, new AbortController().signal);
const content = result.llmContent as string[];
expect(
content.some((c) => c.includes('--- file1.txt ---\n\nContent1\n\n')),
).toBe(true);
expect(
content.some((c) =>
c.includes('--- subdir/file2.js ---\n\nContent2\n\n'),
),
).toBe(true);
expect(result.returnDisplay).toContain(
'Successfully read and concatenated content from **2 file(s)**',
);
});
it('should handle glob patterns', async () => {
createFile('file.txt', 'Text file');
createFile('another.txt', 'Another text');
createFile('sub/data.json', '{}');
const params = { paths: ['*.txt'] };
const result = await tool.execute(params, new AbortController().signal);
const content = result.llmContent as string[];
expect(
content.some((c) => c.includes('--- file.txt ---\n\nText file\n\n')),
).toBe(true);
expect(
content.some((c) =>
c.includes('--- another.txt ---\n\nAnother text\n\n'),
),
).toBe(true);
expect(content.find((c) => c.includes('sub/data.json'))).toBeUndefined();
expect(result.returnDisplay).toContain(
'Successfully read and concatenated content from **2 file(s)**',
);
});
it('should respect exclude patterns', async () => {
createFile('src/main.ts', 'Main content');
createFile('src/main.test.ts', 'Test content');
const params = { paths: ['src/**/*.ts'], exclude: ['**/*.test.ts'] };
const result = await tool.execute(params, new AbortController().signal);
const content = result.llmContent as string[];
expect(content).toEqual(['--- src/main.ts ---\n\nMain content\n\n']);
expect(
content.find((c) => c.includes('src/main.test.ts')),
).toBeUndefined();
expect(result.returnDisplay).toContain(
'Successfully read and concatenated content from **1 file(s)**',
);
});
it('should handle non-existent specific files gracefully', async () => {
const params = { paths: ['nonexistent-file.txt'] };
const result = await tool.execute(params, new AbortController().signal);
expect(result.llmContent).toEqual([
'No files matching the criteria were found or all were skipped.',
]);
expect(result.returnDisplay).toContain(
'No files were read and concatenated based on the criteria.',
);
});
it('should use default excludes', async () => {
createFile('node_modules/some-lib/index.js', 'lib code');
createFile('src/app.js', 'app code');
const params = { paths: ['**/*.js'] };
const result = await tool.execute(params, new AbortController().signal);
const content = result.llmContent as string[];
expect(content).toEqual(['--- src/app.js ---\n\napp code\n\n']);
expect(
content.find((c) => c.includes('node_modules/some-lib/index.js')),
).toBeUndefined();
expect(result.returnDisplay).toContain(
'Successfully read and concatenated content from **1 file(s)**',
);
});
it('should NOT use default excludes if useDefaultExcludes is false', async () => {
createFile('node_modules/some-lib/index.js', 'lib code');
createFile('src/app.js', 'app code');
const params = { paths: ['**/*.js'], useDefaultExcludes: false };
const result = await tool.execute(params, new AbortController().signal);
const content = result.llmContent as string[];
expect(
content.some((c) =>
c.includes('--- node_modules/some-lib/index.js ---\n\nlib code\n\n'),
),
).toBe(true);
expect(
content.some((c) => c.includes('--- src/app.js ---\n\napp code\n\n')),
).toBe(true);
expect(result.returnDisplay).toContain(
'Successfully read and concatenated content from **2 file(s)**',
);
});
it('should include images as inlineData parts if explicitly requested by extension', async () => {
createBinaryFile(
'image.png',
Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
);
const params = { paths: ['*.png'] }; // Explicitly requesting .png
const result = await tool.execute(params, new AbortController().signal);
expect(result.llmContent).toEqual([
{
inlineData: {
data: Buffer.from([
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
]).toString('base64'),
mimeType: 'image/png',
},
},
]);
expect(result.returnDisplay).toContain(
'Successfully read and concatenated content from **1 file(s)**',
);
});
it('should include images as inlineData parts if explicitly requested by name', async () => {
createBinaryFile(
'myExactImage.png',
Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
);
const params = { paths: ['myExactImage.png'] }; // Explicitly requesting by full name
const result = await tool.execute(params, new AbortController().signal);
expect(result.llmContent).toEqual([
{
inlineData: {
data: Buffer.from([
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
]).toString('base64'),
mimeType: 'image/png',
},
},
]);
});
it('should skip PDF files if not explicitly requested by extension or name', async () => {
createBinaryFile('document.pdf', Buffer.from('%PDF-1.4...'));
createFile('notes.txt', 'text notes');
const params = { paths: ['*'] }; // Generic glob, not specific to .pdf
const result = await tool.execute(params, new AbortController().signal);
const content = result.llmContent as string[];
expect(
content.some(
(c) => typeof c === 'string' && c.includes('--- notes.txt ---'),
),
).toBe(true);
expect(result.returnDisplay).toContain(
'**Skipped 1 item(s) (up to 5 shown):**',
);
expect(result.returnDisplay).toContain(
'- `document.pdf` (Reason: asset file (image/pdf) was not explicitly requested by name or extension)',
);
});
it('should include PDF files as inlineData parts if explicitly requested by extension', async () => {
createBinaryFile('important.pdf', Buffer.from('%PDF-1.4...'));
const params = { paths: ['*.pdf'] }; // Explicitly requesting .pdf files
const result = await tool.execute(params, new AbortController().signal);
expect(result.llmContent).toEqual([
{
inlineData: {
data: Buffer.from('%PDF-1.4...').toString('base64'),
mimeType: 'application/pdf',
},
},
]);
});
it('should include PDF files as inlineData parts if explicitly requested by name', async () => {
createBinaryFile('report-final.pdf', Buffer.from('%PDF-1.4...'));
const params = { paths: ['report-final.pdf'] };
const result = await tool.execute(params, new AbortController().signal);
expect(result.llmContent).toEqual([
{
inlineData: {
data: Buffer.from('%PDF-1.4...').toString('base64'),
mimeType: 'application/pdf',
},
},
]);
});
});
});

View File

@@ -12,6 +12,8 @@ import * as path from 'path';
import fg from 'fast-glob';
import { GEMINI_MD_FILENAME } from './memoryTool.js';
import { PartListUnion } from '@google/genai';
import mime from 'mime-types';
/**
* Parameters for the ReadManyFilesTool.
*/
@@ -82,14 +84,6 @@ const DEFAULT_EXCLUDES: string[] = [
'**/*.bz2',
'**/*.rar',
'**/*.7z',
'**/*.png',
'**/*.jpg',
'**/*.jpeg',
'**/*.gif',
'**/*.bmp',
'**/*.tiff',
'**/*.ico',
'**/*.pdf',
'**/*.doc',
'**/*.docx',
'**/*.xls',
@@ -187,6 +181,13 @@ Default excludes apply to common non-text files and large dependency directories
}
validateParams(params: ReadManyFilesParams): string | null {
if (
!params.paths ||
!Array.isArray(params.paths) ||
params.paths.length === 0
) {
return 'The "paths" parameter is required and must be a non-empty array of strings/glob patterns.';
}
if (
this.schema.parameters &&
!SchemaValidator.validate(
@@ -263,7 +264,7 @@ Default excludes apply to common non-text files and large dependency directories
const filesToConsider = new Set<string>();
const skippedFiles: Array<{ path: string; reason: string }> = [];
const processedFilesRelativePaths: string[] = [];
let concatenatedContent = '';
const content: PartListUnion = [];
const effectiveExcludes = useDefaultExcludes
? [...DEFAULT_EXCLUDES, ...exclude]
@@ -319,28 +320,63 @@ Default excludes apply to common non-text files and large dependency directories
.relative(toolBaseDir, filePath)
.replace(/\\/g, '/');
try {
const contentBuffer = await fs.readFile(filePath);
// Basic binary detection: check for null bytes in the first 1KB
const sample = contentBuffer.subarray(
0,
Math.min(contentBuffer.length, 1024),
);
if (sample.includes(0)) {
skippedFiles.push({
path: relativePathForDisplay,
reason: 'Skipped (appears to be binary)',
const mimeType = mime.lookup(filePath);
if (
mimeType &&
(mimeType.startsWith('image/') || mimeType === 'application/pdf')
) {
const fileExtension = path.extname(filePath);
const fileNameWithoutExtension = path.basename(
filePath,
fileExtension,
);
const requestedExplicitly = inputPatterns.some(
(pattern: string) =>
pattern.toLowerCase().includes(fileExtension) ||
pattern.includes(fileNameWithoutExtension),
);
if (!requestedExplicitly) {
skippedFiles.push({
path: relativePathForDisplay,
reason:
'asset file (image/pdf) was not explicitly requested by name or extension',
});
continue;
}
const contentBuffer = await fs.readFile(filePath);
const base64Data = contentBuffer.toString('base64');
content.push({
inlineData: {
data: base64Data,
mimeType,
},
});
continue;
processedFilesRelativePaths.push(relativePathForDisplay);
} else {
const contentBuffer = await fs.readFile(filePath);
// Basic binary detection: check for null bytes in the first 1KB
const sample = contentBuffer.subarray(
0,
Math.min(contentBuffer.length, 1024),
);
if (sample.includes(0)) {
skippedFiles.push({
path: relativePathForDisplay,
reason: 'appears to be binary',
});
continue;
}
// Using default encoding
const fileContent = contentBuffer.toString(DEFAULT_ENCODING);
// Using default separator format
const separator = DEFAULT_OUTPUT_SEPARATOR_FORMAT.replace(
'{filePath}',
relativePathForDisplay,
);
content.push(`${separator}\n\n${fileContent}\n\n`);
processedFilesRelativePaths.push(relativePathForDisplay);
}
// Using default encoding
const fileContent = contentBuffer.toString(DEFAULT_ENCODING);
// Using default separator format
const separator = DEFAULT_OUTPUT_SEPARATOR_FORMAT.replace(
'{filePath}',
relativePathForDisplay,
);
concatenatedContent += `${separator}\n\n${fileContent}\n\n`;
processedFilesRelativePaths.push(relativePathForDisplay);
} catch (error) {
skippedFiles.push({
path: relativePathForDisplay,
@@ -352,18 +388,24 @@ Default excludes apply to common non-text files and large dependency directories
let displayMessage = `### ReadManyFiles Result (Target Dir: \`${this.targetDir}\`)\n\n`;
if (processedFilesRelativePaths.length > 0) {
displayMessage += `Successfully read and concatenated content from **${processedFilesRelativePaths.length} file(s)**.\n`;
displayMessage += `\n**Processed Files:**\n`;
processedFilesRelativePaths
.slice(0, 10)
.forEach((p) => (displayMessage += `- \`${p}\`\n`));
if (processedFilesRelativePaths.length > 10) {
if (processedFilesRelativePaths.length <= 10) {
displayMessage += `\n**Processed Files:**\n`;
processedFilesRelativePaths.forEach(
(p) => (displayMessage += `- \`${p}\`\n`),
);
} else {
displayMessage += `\n**Processed Files (first 10 shown):**\n`;
processedFilesRelativePaths
.slice(0, 10)
.forEach((p) => (displayMessage += `- \`${p}\`\n`));
displayMessage += `- ...and ${processedFilesRelativePaths.length - 10} more.\n`;
}
} else {
displayMessage += `No files were read and concatenated based on the criteria.\n`;
}
if (skippedFiles.length > 0) {
if (processedFilesRelativePaths.length === 0) {
displayMessage += `No files were read and concatenated based on the criteria.\n`;
}
displayMessage += `\n**Skipped ${skippedFiles.length} item(s) (up to 5 shown):**\n`;
skippedFiles
.slice(0, 5)
@@ -373,18 +415,21 @@ Default excludes apply to common non-text files and large dependency directories
if (skippedFiles.length > 5) {
displayMessage += `- ...and ${skippedFiles.length - 5} more.\n`;
}
}
if (
concatenatedContent.length === 0 &&
processedFilesRelativePaths.length === 0
} else if (
processedFilesRelativePaths.length === 0 &&
skippedFiles.length === 0
) {
concatenatedContent =
'No files matching the criteria were found or all were skipped.';
displayMessage += `No files were read and concatenated based on the criteria.\n`;
}
if (content.length === 0) {
content.push(
'No files matching the criteria were found or all were skipped.',
);
}
return {
llmContent: concatenatedContent,
returnDisplay: displayMessage,
llmContent: content,
returnDisplay: displayMessage.trim(),
};
}
}

View File

@@ -4,7 +4,7 @@
* SPDX-License-Identifier: Apache-2.0
*/
import { FunctionDeclaration, Schema } from '@google/genai';
import { FunctionDeclaration, PartListUnion, Schema } from '@google/genai';
/**
* Interface representing the base Tool functionality
@@ -152,7 +152,7 @@ export interface ToolResult {
* Content meant to be included in LLM history.
* This should represent the factual outcome of the tool execution.
*/
llmContent: string;
llmContent: PartListUnion;
/**
* Markdown string for user display.