Preflight and integration npx (#1096)

integration-tests/file-system.test.js

@@ -0,0 +1,30 @@
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { strict as assert } from 'assert';
+import { test } from 'node:test';
+import { TestRig } from './test-helper.js';
+
+test('reads a file', (t) => {
+  const rig = new TestRig();
+  rig.setup(t.name);
+  rig.createFile('test.txt', 'hello world');
+
+  const output = rig.run(`read the file name test.txt`);
+
+  assert.ok(output.toLowerCase().includes('hello'));
+});
+
+test('writes a file', (t) => {
+  const rig = new TestRig();
+  rig.setup(t.name);
+  rig.createFile('test.txt', '');
+
+  rig.run(`edit test.txt to have a hello world message`);
+
+  const fileContent = rig.readFile('test.txt');
+  assert.ok(fileContent.toLowerCase().includes('hello'));
+});

integration-tests/google_web_search.test.js

@@ -0,0 +1,19 @@
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { test } from 'node:test';
+import { strict as assert } from 'assert';
+import { TestRig } from './test-helper.js';
+
+test('should be able to search the web', async (t) => {
+  const rig = new TestRig();
+  rig.setup(t.name);
+
+  const prompt = `what planet do we live on`;
+  const result = await rig.run(prompt);
+
+  assert.ok(result.toLowerCase().includes('earth'));
+});

integration-tests/list_directory.test.js

@@ -0,0 +1,22 @@
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { test } from 'node:test';
+import { strict as assert } from 'assert';
+import { TestRig } from './test-helper.js';
+
+test('should be able to list a directory', async (t) => {
+  const rig = new TestRig();
+  rig.setup(t.name);
+  rig.createFile('file1.txt', 'file 1 content');
+  rig.mkdir('subdir');
+
+  const prompt = `Can you list the files in the current directory`;
+  const result = await rig.run(prompt);
+
+  assert.ok(result.includes('file1.txt'));
+  assert.ok(result.includes('subdir'));
+});

integration-tests/read_many_files.test.js

@@ -0,0 +1,22 @@
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { test } from 'node:test';
+import { strict as assert } from 'assert';
+import { TestRig } from './test-helper.js';
+
+test.skip('should be able to read multiple files', async (t) => {
+  const rig = new TestRig();
+  rig.setup(t.name);
+  rig.createFile('file1.txt', 'file 1 content');
+  rig.createFile('file2.txt', 'file 2 content');
+
+  const prompt = `Read the files in this directory, list them and print them to the screen`;
+  const result = await rig.run(prompt);
+
+  assert.ok(result.includes('file 1 content'));
+  assert.ok(result.includes('file 2 content'));
+});

integration-tests/replace.test.js

@@ -0,0 +1,22 @@
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { test } from 'node:test';
+import { strict as assert } from 'assert';
+import { TestRig } from './test-helper.js';
+
+test('should be able to replace content in a file', async (t) => {
+  const rig = new TestRig();
+  rig.setup(t.name);
+
+  const fileName = 'file_to_replace.txt';
+  rig.createFile(fileName, 'original content');
+  const prompt = `Can you replace 'original' with 'replaced' in the file 'file_to_replace.txt'`;
+
+  await rig.run(prompt);
+  const newFileContent = rig.readFile(fileName);
+  assert.strictEqual(newFileContent, 'replaced content');
+});

integration-tests/run-tests.js

@@ -0,0 +1,122 @@
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { spawnSync } from 'child_process';
+import { spawn } from 'child_process';
+import { mkdirSync, rmSync, createWriteStream } from 'fs';
+import { join, dirname, basename } from 'path';
+import { fileURLToPath } from 'url';
+import { glob } from 'glob';
+
+async function main() {
+  const __dirname = dirname(fileURLToPath(import.meta.url));
+  const rootDir = join(__dirname, '..');
+  const integrationTestsDir = join(rootDir, '.integration-tests');
+
+  if (process.env.GEMINI_SANDBOX === 'docker' && !process.env.IS_DOCKER) {
+    console.log('Building sandbox for Docker...');
+    const buildResult = spawnSync('npm', ['run', 'build:all'], {
+      stdio: 'inherit',
+    });
+    if (buildResult.status !== 0) {
+      console.error('Sandbox build failed.');
+      process.exit(1);
+    }
+  }
+
+  const runId = `${Date.now()}`;
+  const runDir = join(integrationTestsDir, runId);
+
+  mkdirSync(runDir, { recursive: true });
+
+  const args = process.argv.slice(2);
+  const keepOutput =
+    process.env.KEEP_OUTPUT === 'true' || args.includes('--keep-output');
+  if (keepOutput) {
+    const keepOutputIndex = args.indexOf('--keep-output');
+    if (keepOutputIndex > -1) {
+      args.splice(keepOutputIndex, 1);
+    }
+    console.log(`Keeping output for test run in: ${runDir}`);
+  }
+
+  const verbose = args.includes('--verbose');
+  if (verbose) {
+    const verboseIndex = args.indexOf('--verbose');
+    if (verboseIndex > -1) {
+      args.splice(verboseIndex, 1);
+    }
+  }
+
+  const testPatterns =
+    args.length > 0
+      ? args.map((arg) => `integration-tests/${arg}.test.js`)
+      : ['integration-tests/*.test.js'];
+  const testFiles = glob.sync(testPatterns, { cwd: rootDir, absolute: true });
+
+  for (const testFile of testFiles) {
+    const testFileName = basename(testFile);
+    console.log(`\tFound test file: ${testFileName}`);
+  }
+
+  let allTestsPassed = true;
+
+  for (const testFile of testFiles) {
+    const testFileName = basename(testFile);
+    const testFileDir = join(runDir, testFileName);
+    mkdirSync(testFileDir, { recursive: true });
+
+    console.log(
+      `------------- Running test file: ${testFileName} ------------------------------`,
+    );
+
+    const child = spawn('node', ['--test', testFile], {
+      stdio: 'pipe',
+      env: {
+        ...process.env,
+        INTEGRATION_TEST_FILE_DIR: testFileDir,
+        KEEP_OUTPUT: keepOutput.toString(),
+        TEST_FILE_NAME: testFileName,
+      },
+    });
+
+    if (verbose) {
+      child.stdout.pipe(process.stdout);
+      child.stderr.pipe(process.stderr);
+    }
+
+    if (keepOutput) {
+      const outputFile = join(testFileDir, 'output.log');
+      const outputStream = createWriteStream(outputFile);
+      child.stdout.pipe(outputStream);
+      child.stderr.pipe(outputStream);
+      console.log(`Output for ${testFileName} written to: ${outputFile}`);
+    } else if (!verbose) {
+      child.stdout.pipe(process.stdout);
+      child.stderr.pipe(process.stderr);
+    }
+
+    const exitCode = await new Promise((resolve) => {
+      child.on('close', resolve);
+    });
+
+    if (exitCode !== 0) {
+      console.error(`Test file failed: ${testFileName}`);
+      allTestsPassed = false;
+    }
+  }
+
+  if (!keepOutput) {
+    rmSync(runDir, { recursive: true, force: true });
+  }
+
+  if (!allTestsPassed) {
+    console.error('One or more test files failed.');
+    process.exit(1);
+  }
+}
+
+main();

integration-tests/run_shell_command.test.js

@@ -0,0 +1,20 @@
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { test } from 'node:test';
+import { strict as assert } from 'assert';
+import { TestRig } from './test-helper.js';
+
+test('should be able to run a shell command', async (t) => {
+  const rig = new TestRig();
+  rig.setup(t.name);
+  rig.createFile('blah.txt', 'some content');
+
+  const prompt = `Can you use ls to list the contexts of the current folder`;
+  const result = await rig.run(prompt);
+
+  assert.ok(result.includes('blah.txt'));
+});

integration-tests/save_memory.test.js

@@ -0,0 +1,22 @@
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { test } from 'node:test';
+import { strict as assert } from 'assert';
+import { TestRig } from './test-helper.js';
+
+test('should be able to save to memory', async (t) => {
+  const rig = new TestRig();
+  rig.setup(t.name);
+
+  const prompt = `remember that my favorite color is  blue`;
+  await rig.run(prompt);
+  const result = await rig.run(
+    'what is my favorite color? tell me that and surround it with $ symbol',
+  );
+
+  assert.ok(result.toLowerCase().includes('$blue$'));
+});

integration-tests/test-helper.js

@@ -0,0 +1,80 @@
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { execSync } from 'child_process';
+import { mkdirSync, writeFileSync, readFileSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import { env } from 'process';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+function sanitizeTestName(name) {
+  return name
+    .toLowerCase()
+    .replace(/[^a-z0-9]/g, '-')
+    .replace(/-+/g, '-');
+}
+
+export class TestRig {
+  constructor() {
+    this.bundlePath = join(__dirname, '..', 'bundle/gemini.js');
+    this.testDir = null;
+  }
+
+  setup(testName) {
+    this.testName = testName;
+    const sanitizedName = sanitizeTestName(testName);
+    this.testDir = join(env.INTEGRATION_TEST_FILE_DIR, sanitizedName);
+    mkdirSync(this.testDir, { recursive: true });
+  }
+
+  createFile(fileName, content) {
+    const filePath = join(this.testDir, fileName);
+    writeFileSync(filePath, content);
+    return filePath;
+  }
+
+  mkdir(dir) {
+    mkdirSync(join(this.testDir, dir));
+  }
+
+  run(prompt, ...args) {
+    const output = execSync(
+      `node ${this.bundlePath} --yolo --prompt "${prompt}" ${args.join(' ')}`,
+      {
+        cwd: this.testDir,
+        encoding: 'utf-8',
+      },
+    );
+
+    if (env.KEEP_OUTPUT === 'true') {
+      const testId = `${env.TEST_FILE_NAME.replace(
+        '.test.js',
+        '',
+      )}:${this.testName.replace(/ /g, '-')}`;
+      console.log(`--- TEST: ${testId} ---`);
+      console.log(output);
+      console.log(`--- END TEST: ${testId} ---`);
+    }
+
+    return output;
+  }
+
+  readFile(fileName) {
+    const content = readFileSync(join(this.testDir, fileName), 'utf-8');
+    if (env.KEEP_OUTPUT === 'true') {
+      const testId = `${env.TEST_FILE_NAME.replace(
+        '.test.js',
+        '',
+      )}:${this.testName.replace(/ /g, '-')}`;
+      console.log(`--- FILE: ${testId}/${fileName} ---`);
+      console.log(content);
+      console.log(`--- END FILE: ${testId}/${fileName} ---`);
+    }
+    return content;
+  }
+}

integration-tests/write_file.test.js

@@ -0,0 +1,21 @@
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { test } from 'node:test';
+import { strict as assert } from 'assert';
+import { TestRig } from './test-helper.js';
+
+test('should be able to write a file', async (t) => {
+  const rig = new TestRig();
+  rig.setup(t.name);
+  const prompt = `show me an example of using the write tool. put a dad joke in dad.txt`;
+
+  await rig.run(prompt);
+  const newFilePath = 'dad.txt';
+
+  const newFileContent = rig.readFile(newFilePath);
+  assert.notEqual(newFileContent, '');
+});