/** * @license * Copyright 2025 Qwen * SPDX-License-Identifier: Apache-2.0 */ import { type PartListUnion, type Part } from '@google/genai'; import { AuthType, type Config } from '@qwen-code/qwen-code-core'; import { useCallback, useRef } from 'react'; import { VisionSwitchOutcome } from '../components/ModelSwitchDialog.js'; import { getDefaultVisionModel, isVisionModel, } from '../models/availableModels.js'; import { MessageType } from '../types.js'; import type { UseHistoryManagerReturn } from './useHistoryManager.js'; /** * Checks if a PartListUnion contains image parts */ function hasImageParts(parts: PartListUnion): boolean { if (typeof parts === 'string') { return false; } if (Array.isArray(parts)) { return parts.some((part) => { // Skip string parts if (typeof part === 'string') return false; return isImagePart(part); }); } // If it's a single Part (not a string), check if it's an image if (typeof parts === 'object') { return isImagePart(parts); } return false; } /** * Checks if a single Part is an image part */ function isImagePart(part: Part): boolean { // Check for inlineData with image mime type if ('inlineData' in part && part.inlineData?.mimeType?.startsWith('image/')) { return true; } // Check for fileData with image mime type if ('fileData' in part && part.fileData?.mimeType?.startsWith('image/')) { return true; } return false; } /** * Determines if we should offer vision switch for the given parts, auth type, and current model */ export function shouldOfferVisionSwitch( parts: PartListUnion, authType: AuthType, currentModel: string, ): boolean { // Only trigger for qwen-oauth if (authType !== AuthType.QWEN_OAUTH) { return false; } // If current model is already a vision model, no need to switch if (isVisionModel(currentModel)) { return false; } // Check if the current message contains image parts return hasImageParts(parts); } /** * Interface for vision switch result */ export interface VisionSwitchResult { modelOverride?: string; persistSessionModel?: string; showGuidance?: boolean; } /** * Processes the vision switch outcome and returns the appropriate result */ export function processVisionSwitchOutcome( outcome: VisionSwitchOutcome, ): VisionSwitchResult { const vlModelId = getDefaultVisionModel(); switch (outcome) { case VisionSwitchOutcome.SwitchOnce: return { modelOverride: vlModelId }; case VisionSwitchOutcome.SwitchSessionToVL: return { persistSessionModel: vlModelId }; case VisionSwitchOutcome.DisallowWithGuidance: return { showGuidance: true }; default: return { showGuidance: true }; } } /** * Gets the guidance message for when vision switch is disallowed */ export function getVisionSwitchGuidanceMessage(): string { const vlModelId = getDefaultVisionModel(); return `To use images with your query, you can: • Use /model set ${vlModelId} to switch to a vision-capable model • Or remove the image and provide a text description instead`; } /** * Interface for vision switch handling result */ export interface VisionSwitchHandlingResult { shouldProceed: boolean; originalModel?: string; } /** * Custom hook for handling vision model auto-switching */ export function useVisionAutoSwitch( config: Config, addItem: UseHistoryManagerReturn['addItem'], onVisionSwitchRequired?: (query: PartListUnion) => Promise<{ modelOverride?: string; persistSessionModel?: string; showGuidance?: boolean; }>, ) { const originalModelRef = useRef(null); const handleVisionSwitch = useCallback( async ( query: PartListUnion, userMessageTimestamp: number, isContinuation: boolean, ): Promise => { // Skip vision switch handling for continuations or if no handler provided if (isContinuation || !onVisionSwitchRequired) { return { shouldProceed: true }; } const contentGeneratorConfig = config.getContentGeneratorConfig(); // Only handle qwen-oauth auth type if (contentGeneratorConfig?.authType !== AuthType.QWEN_OAUTH) { return { shouldProceed: true }; } // Check if vision switch is needed if ( !shouldOfferVisionSwitch( query, contentGeneratorConfig.authType, config.getModel(), ) ) { return { shouldProceed: true }; } try { const visionSwitchResult = await onVisionSwitchRequired(query); if (visionSwitchResult.showGuidance) { // Show guidance and don't proceed with the request addItem( { type: MessageType.INFO, text: getVisionSwitchGuidanceMessage(), }, userMessageTimestamp, ); return { shouldProceed: false }; } if (visionSwitchResult.modelOverride) { // One-time model override originalModelRef.current = config.getModel(); config.setModel(visionSwitchResult.modelOverride); return { shouldProceed: true, originalModel: originalModelRef.current, }; } else if (visionSwitchResult.persistSessionModel) { // Persistent session model change config.setModel(visionSwitchResult.persistSessionModel); return { shouldProceed: true }; } return { shouldProceed: true }; } catch (_error) { // If vision switch dialog was cancelled or errored, don't proceed return { shouldProceed: false }; } }, [config, addItem, onVisionSwitchRequired], ); const restoreOriginalModel = useCallback(() => { if (originalModelRef.current) { config.setModel(originalModelRef.current); originalModelRef.current = null; } }, [config]); return { handleVisionSwitch, restoreOriginalModel, }; }