fix: enhance error handling and output for non-interactive authentication in JSON and STREAM_JSON modes

integration-tests/json-output.test.ts

@@ -74,35 +74,236 @@ describe('JSON output', () => {
     expect(thrown).toBeDefined();
     const message = (thrown as Error).message;
 
-    // The error JSON is written to stderr, so it should be in the error message
-    // Use a regex to find the first complete JSON object in the string
-    const jsonMatch = message.match(/{[\s\S]*}/);
-
-    // Fail if no JSON-like text was found
+    // The error JSON is written to stdout as a CLIResultMessageError
+    // Extract stdout from the error message
+    const stdoutMatch = message.match(/Stdout:\n([\s\S]*?)(?:\n\nStderr:|$)/);
     expect(
-      jsonMatch,
-      'Expected to find a JSON object in the error output',
+      stdoutMatch,
+      'Expected to find stdout in the error message',
     ).toBeTruthy();
 
-    let payload;
+    const stdout = stdoutMatch![1];
+    let parsed: unknown[];
     try {
-      // Parse the matched JSON string
-      payload = JSON.parse(jsonMatch![0]);
+      // Parse the JSON array from stdout
+      parsed = JSON.parse(stdout);
     } catch (parseError) {
-      console.error('Failed to parse the following JSON:', jsonMatch![0]);
+      console.error('Failed to parse the following JSON:', stdout);
       throw new Error(
-        `Test failed: Could not parse JSON from error message. Details: ${parseError}`,
+        `Test failed: Could not parse JSON from stdout. Details: ${parseError}`,
       );
     }
 
-    // The JsonFormatter.formatError() outputs: { error: { type, message, code } }
-    expect(payload).toHaveProperty('error');
-    expect(payload.error).toBeDefined();
-    expect(payload.error.type).toBe('Error');
-    expect(payload.error.code).toBe(1);
-    expect(payload.error.message).toContain(
+    // The output should be an array of messages
+    expect(Array.isArray(parsed)).toBe(true);
+    expect(parsed.length).toBeGreaterThan(0);
+
+    // Find the result message with error
+    const resultMessage = parsed.find(
+      (msg: unknown) =>
+        typeof msg === 'object' &&
+        msg !== null &&
+        'type' in msg &&
+        msg.type === 'result' &&
+        'is_error' in msg &&
+        msg.is_error === true,
+    ) as {
+      type: string;
+      is_error: boolean;
+      subtype: string;
+      error?: { message: string; type?: string };
+    };
+
+    expect(resultMessage).toBeDefined();
+    expect(resultMessage.is_error).toBe(true);
+    expect(resultMessage).toHaveProperty('subtype');
+    expect(resultMessage.subtype).toBe('error_during_execution');
+    expect(resultMessage).toHaveProperty('error');
+    expect(resultMessage.error).toBeDefined();
+    expect(resultMessage.error?.message).toContain(
       'configured auth type is qwen-oauth',
     );
-    expect(payload.error.message).toContain('current auth type is openai');
+    expect(resultMessage.error?.message).toContain(
+      'current auth type is openai',
+    );
+  });
+
+  it('should return line-delimited JSON messages for stream-json output format', async () => {
+    const result = await rig.run(
+      'What is the capital of France?',
+      '--output-format',
+      'stream-json',
+    );
+
+    // Stream-json output is line-delimited JSON (one JSON object per line)
+    const lines = result
+      .trim()
+      .split('\n')
+      .filter((line) => line.trim());
+    expect(lines.length).toBeGreaterThan(0);
+
+    // Parse each line as a JSON object
+    const messages: unknown[] = [];
+    for (const line of lines) {
+      try {
+        const parsed = JSON.parse(line);
+        messages.push(parsed);
+      } catch (parseError) {
+        throw new Error(
+          `Failed to parse JSON line: ${line}. Error: ${parseError}`,
+        );
+      }
+    }
+
+    // Should have at least system, assistant, and result messages
+    expect(messages.length).toBeGreaterThanOrEqual(3);
+
+    // Find system message
+    const systemMessage = messages.find(
+      (msg: unknown) =>
+        typeof msg === 'object' &&
+        msg !== null &&
+        'type' in msg &&
+        msg.type === 'system',
+    );
+    expect(systemMessage).toBeDefined();
+    expect(systemMessage).toHaveProperty('subtype');
+    expect(systemMessage).toHaveProperty('session_id');
+
+    // Find assistant message
+    const assistantMessage = messages.find(
+      (msg: unknown) =>
+        typeof msg === 'object' &&
+        msg !== null &&
+        'type' in msg &&
+        msg.type === 'assistant',
+    );
+    expect(assistantMessage).toBeDefined();
+    expect(assistantMessage).toHaveProperty('message');
+    expect(assistantMessage).toHaveProperty('session_id');
+
+    // Find result message (should be the last message)
+    const resultMessage = messages[messages.length - 1] as {
+      type: string;
+      is_error: boolean;
+      result: string;
+    };
+    expect(resultMessage).toBeDefined();
+    expect(
+      typeof resultMessage === 'object' &&
+        resultMessage !== null &&
+        'type' in resultMessage &&
+        resultMessage.type === 'result',
+    ).toBe(true);
+    expect(resultMessage).toHaveProperty('is_error');
+    expect(resultMessage.is_error).toBe(false);
+    expect(resultMessage).toHaveProperty('result');
+    expect(typeof resultMessage.result).toBe('string');
+    expect(resultMessage.result.toLowerCase()).toContain('paris');
+  });
+
+  it('should include stream events when using stream-json with include-partial-messages', async () => {
+    const result = await rig.run(
+      'What is the capital of France?',
+      '--output-format',
+      'stream-json',
+      '--include-partial-messages',
+    );
+
+    // Stream-json output is line-delimited JSON (one JSON object per line)
+    const lines = result
+      .trim()
+      .split('\n')
+      .filter((line) => line.trim());
+    expect(lines.length).toBeGreaterThan(0);
+
+    // Parse each line as a JSON object
+    const messages: unknown[] = [];
+    for (const line of lines) {
+      try {
+        const parsed = JSON.parse(line);
+        messages.push(parsed);
+      } catch (parseError) {
+        throw new Error(
+          `Failed to parse JSON line: ${line}. Error: ${parseError}`,
+        );
+      }
+    }
+
+    // Should have more messages than without include-partial-messages
+    // because we're including stream events
+    expect(messages.length).toBeGreaterThan(3);
+
+    // Find stream_event messages
+    const streamEvents = messages.filter(
+      (msg: unknown) =>
+        typeof msg === 'object' &&
+        msg !== null &&
+        'type' in msg &&
+        msg.type === 'stream_event',
+    );
+    expect(streamEvents.length).toBeGreaterThan(0);
+
+    // Verify stream event structure
+    const firstStreamEvent = streamEvents[0];
+    expect(firstStreamEvent).toHaveProperty('event');
+    expect(firstStreamEvent).toHaveProperty('session_id');
+    expect(firstStreamEvent).toHaveProperty('uuid');
+
+    // Check for expected stream event types
+    const eventTypes = streamEvents.map((event: unknown) =>
+      typeof event === 'object' &&
+      event !== null &&
+      'event' in event &&
+      typeof event.event === 'object' &&
+      event.event !== null &&
+      'type' in event.event
+        ? event.event.type
+        : null,
+    );
+
+    // Should have message_start event
+    expect(eventTypes).toContain('message_start');
+
+    // Should have content_block_start event
+    expect(eventTypes).toContain('content_block_start');
+
+    // Should have content_block_delta events
+    expect(eventTypes).toContain('content_block_delta');
+
+    // Should have content_block_stop event
+    expect(eventTypes).toContain('content_block_stop');
+
+    // Should have message_stop event
+    expect(eventTypes).toContain('message_stop');
+
+    // Verify that we still have the complete assistant message
+    const assistantMessage = messages.find(
+      (msg: unknown) =>
+        typeof msg === 'object' &&
+        msg !== null &&
+        'type' in msg &&
+        msg.type === 'assistant',
+    );
+    expect(assistantMessage).toBeDefined();
+    expect(assistantMessage).toHaveProperty('message');
+
+    // Verify that we still have the result message
+    const resultMessage = messages[messages.length - 1] as {
+      type: string;
+      is_error: boolean;
+      result: string;
+    };
+    expect(resultMessage).toBeDefined();
+    expect(
+      typeof resultMessage === 'object' &&
+        resultMessage !== null &&
+        'type' in resultMessage &&
+        resultMessage.type === 'result',
+    ).toBe(true);
+    expect(resultMessage).toHaveProperty('is_error');
+    expect(resultMessage.is_error).toBe(false);
+    expect(resultMessage).toHaveProperty('result');
+    expect(resultMessage.result.toLowerCase()).toContain('paris');
   });
 });

integration-tests/test-helper.ts

@@ -340,7 +340,8 @@ export class TestRig {
           // as it would corrupt the JSON
           const isJsonOutput =
             commandArgs.includes('--output-format') &&
-            commandArgs.includes('json');
+            (commandArgs.includes('json') ||
+              commandArgs.includes('stream-json'));
 
           // If we have stderr output and it's not a JSON test, include that also
           if (stderr && !isJsonOutput) {
@@ -349,7 +350,23 @@ export class TestRig {
 
           resolve(result);
         } else {
-          reject(new Error(`Process exited with code ${code}:\n${stderr}`));
+          // Check if this is a JSON output test - for JSON errors, the error is in stdout
+          const isJsonOutputOnError =
+            commandArgs.includes('--output-format') &&
+            (commandArgs.includes('json') ||
+              commandArgs.includes('stream-json'));
+
+          // For JSON output tests, include stdout in the error message
+          // as the error JSON is written to stdout
+          if (isJsonOutputOnError && stdout) {
+            reject(
+              new Error(
+                `Process exited with code ${code}:\nStdout:\n${stdout}\n\nStderr:\n${stderr}`,
+              ),
+            );
+          } else {
+            reject(new Error(`Process exited with code ${code}:\n${stderr}`));
+          }
         }
       });
     });