From a8ca4ebf89ad479344e1c7b6dce2b7ff9305879d Mon Sep 17 00:00:00 2001 From: "mingholy.lmh" Date: Wed, 17 Sep 2025 20:58:54 +0800 Subject: [PATCH] feat: add `visionModelPreview` to control default visibility of vision models --- packages/cli/src/config/settingsSchema.ts | 10 ++++ packages/cli/src/ui/App.tsx | 10 +++- packages/cli/src/ui/hooks/useGeminiStream.ts | 2 + .../src/ui/hooks/useVisionAutoSwitch.test.ts | 58 ++++++++++++++++--- .../cli/src/ui/hooks/useVisionAutoSwitch.ts | 10 +++- packages/cli/src/ui/models/availableModels.ts | 12 ++++ .../core/openaiContentGenerator/pipeline.ts | 2 +- 7 files changed, 91 insertions(+), 13 deletions(-) diff --git a/packages/cli/src/config/settingsSchema.ts b/packages/cli/src/config/settingsSchema.ts index 1cbd63e0..c7f1e94e 100644 --- a/packages/cli/src/config/settingsSchema.ts +++ b/packages/cli/src/config/settingsSchema.ts @@ -741,6 +741,16 @@ export const SETTINGS_SCHEMA = { description: 'Enable extension management features.', showInDialog: false, }, + visionModelPreview: { + type: 'boolean', + label: 'Vision Model Preview', + category: 'Experimental', + requiresRestart: false, + default: false, + description: + 'Enable vision model support and auto-switching functionality. When disabled, vision models like qwen-vl-max-latest will be hidden and auto-switching will not occur.', + showInDialog: true, + }, }, }, diff --git a/packages/cli/src/ui/App.tsx b/packages/cli/src/ui/App.tsx index 09040cfa..0f2c9e6c 100644 --- a/packages/cli/src/ui/App.tsx +++ b/packages/cli/src/ui/App.tsx @@ -59,8 +59,8 @@ import { type VisionSwitchOutcome, } from './components/ModelSwitchDialog.js'; import { - AVAILABLE_MODELS_QWEN, getOpenAIAvailableModelFromEnv, + getFilteredQwenModels, type AvailableModel, } from './models/availableModels.js'; import { processVisionSwitchOutcome } from './hooks/useVisionAutoSwitch.js'; @@ -669,9 +669,12 @@ const App = ({ config, settings, startupWarnings = [], version }: AppProps) => { const contentGeneratorConfig = config.getContentGeneratorConfig(); if (!contentGeneratorConfig) return []; + const visionModelPreviewEnabled = + settings.merged.experimental?.visionModelPreview ?? false; + switch (contentGeneratorConfig.authType) { case AuthType.QWEN_OAUTH: - return AVAILABLE_MODELS_QWEN; + return getFilteredQwenModels(visionModelPreviewEnabled); case AuthType.USE_OPENAI: { const openAIModel = getOpenAIAvailableModelFromEnv(); return openAIModel ? [openAIModel] : []; @@ -679,7 +682,7 @@ const App = ({ config, settings, startupWarnings = [], version }: AppProps) => { default: return []; } - }, [config]); + }, [config, settings.merged.experimental?.visionModelPreview]); // Core hooks and processors const { @@ -756,6 +759,7 @@ const App = ({ config, settings, startupWarnings = [], version }: AppProps) => { setModelSwitchedFromQuotaError, refreshStatic, () => cancelHandlerRef.current(), + settings.merged.experimental?.visionModelPreview ?? false, handleVisionSwitchRequired, ); diff --git a/packages/cli/src/ui/hooks/useGeminiStream.ts b/packages/cli/src/ui/hooks/useGeminiStream.ts index 2fd4fbcb..352ec6da 100644 --- a/packages/cli/src/ui/hooks/useGeminiStream.ts +++ b/packages/cli/src/ui/hooks/useGeminiStream.ts @@ -89,6 +89,7 @@ export const useGeminiStream = ( setModelSwitchedFromQuotaError: React.Dispatch>, onEditorClose: () => void, onCancelSubmit: () => void, + visionModelPreviewEnabled: boolean = false, onVisionSwitchRequired?: (query: PartListUnion) => Promise<{ modelOverride?: string; persistSessionModel?: string; @@ -164,6 +165,7 @@ export const useGeminiStream = ( const { handleVisionSwitch, restoreOriginalModel } = useVisionAutoSwitch( config, addItem, + visionModelPreviewEnabled, onVisionSwitchRequired, ); diff --git a/packages/cli/src/ui/hooks/useVisionAutoSwitch.test.ts b/packages/cli/src/ui/hooks/useVisionAutoSwitch.test.ts index ee7c1059..dd8c6a06 100644 --- a/packages/cli/src/ui/hooks/useVisionAutoSwitch.test.ts +++ b/packages/cli/src/ui/hooks/useVisionAutoSwitch.test.ts @@ -29,6 +29,7 @@ describe('useVisionAutoSwitch helpers', () => { parts, AuthType.USE_GEMINI, 'qwen3-coder-plus', + true, ); expect(result).toBe(false); }); @@ -41,6 +42,7 @@ describe('useVisionAutoSwitch helpers', () => { parts, AuthType.QWEN_OAUTH, 'qwen-vl-max-latest', + true, ); expect(result).toBe(false); }); @@ -54,6 +56,7 @@ describe('useVisionAutoSwitch helpers', () => { parts, AuthType.QWEN_OAUTH, 'qwen3-coder-plus', + true, ); expect(result).toBe(true); }); @@ -66,6 +69,7 @@ describe('useVisionAutoSwitch helpers', () => { singleImagePart, AuthType.QWEN_OAUTH, 'qwen3-coder-plus', + true, ); expect(result).toBe(true); }); @@ -76,6 +80,7 @@ describe('useVisionAutoSwitch helpers', () => { parts, AuthType.QWEN_OAUTH, 'qwen3-coder-plus', + true, ); expect(result).toBe(false); }); @@ -86,6 +91,20 @@ describe('useVisionAutoSwitch helpers', () => { parts, AuthType.QWEN_OAUTH, 'qwen3-coder-plus', + true, + ); + expect(result).toBe(false); + }); + + it('returns false when visionModelPreviewEnabled is false', () => { + const parts: PartListUnion = [ + { inlineData: { mimeType: 'image/png', data: '...' } }, + ]; + const result = shouldOfferVisionSwitch( + parts, + AuthType.QWEN_OAUTH, + 'qwen3-coder-plus', + false, ); expect(result).toBe(false); }); @@ -159,7 +178,7 @@ describe('useVisionAutoSwitch hook', () => { it('returns shouldProceed=true immediately for continuations', async () => { const config = createMockConfig(AuthType.QWEN_OAUTH, 'qwen3-coder-plus'); const { result } = renderHook(() => - useVisionAutoSwitch(config, addItem as any, vi.fn()), + useVisionAutoSwitch(config, addItem as any, true, vi.fn()), ); const parts: PartListUnion = [ @@ -177,7 +196,7 @@ describe('useVisionAutoSwitch hook', () => { const config = createMockConfig(AuthType.USE_GEMINI, 'qwen3-coder-plus'); const onVisionSwitchRequired = vi.fn(); const { result } = renderHook(() => - useVisionAutoSwitch(config, addItem as any, onVisionSwitchRequired), + useVisionAutoSwitch(config, addItem as any, true, onVisionSwitchRequired), ); const parts: PartListUnion = [ @@ -195,7 +214,7 @@ describe('useVisionAutoSwitch hook', () => { const config = createMockConfig(AuthType.QWEN_OAUTH, 'qwen3-coder-plus'); const onVisionSwitchRequired = vi.fn(); const { result } = renderHook(() => - useVisionAutoSwitch(config, addItem as any, onVisionSwitchRequired), + useVisionAutoSwitch(config, addItem as any, true, onVisionSwitchRequired), ); const parts: PartListUnion = [{ text: 'no images here' }]; @@ -213,7 +232,7 @@ describe('useVisionAutoSwitch hook', () => { .fn() .mockResolvedValue({ showGuidance: true }); const { result } = renderHook(() => - useVisionAutoSwitch(config, addItem as any, onVisionSwitchRequired), + useVisionAutoSwitch(config, addItem as any, true, onVisionSwitchRequired), ); const parts: PartListUnion = [ @@ -241,7 +260,7 @@ describe('useVisionAutoSwitch hook', () => { .fn() .mockResolvedValue({ modelOverride: 'qwen-vl-max-latest' }); const { result } = renderHook(() => - useVisionAutoSwitch(config, addItem as any, onVisionSwitchRequired), + useVisionAutoSwitch(config, addItem as any, true, onVisionSwitchRequired), ); const parts: PartListUnion = [ @@ -269,7 +288,7 @@ describe('useVisionAutoSwitch hook', () => { .fn() .mockResolvedValue({ persistSessionModel: 'qwen-vl-max-latest' }); const { result } = renderHook(() => - useVisionAutoSwitch(config, addItem as any, onVisionSwitchRequired), + useVisionAutoSwitch(config, addItem as any, true, onVisionSwitchRequired), ); const parts: PartListUnion = [ @@ -298,7 +317,7 @@ describe('useVisionAutoSwitch hook', () => { const config = createMockConfig(AuthType.QWEN_OAUTH, 'qwen3-coder-plus'); const onVisionSwitchRequired = vi.fn().mockResolvedValue({}); const { result } = renderHook(() => - useVisionAutoSwitch(config, addItem as any, onVisionSwitchRequired), + useVisionAutoSwitch(config, addItem as any, true, onVisionSwitchRequired), ); const parts: PartListUnion = [ @@ -316,7 +335,7 @@ describe('useVisionAutoSwitch hook', () => { const config = createMockConfig(AuthType.QWEN_OAUTH, 'qwen3-coder-plus'); const onVisionSwitchRequired = vi.fn().mockRejectedValue(new Error('x')); const { result } = renderHook(() => - useVisionAutoSwitch(config, addItem as any, onVisionSwitchRequired), + useVisionAutoSwitch(config, addItem as any, true, onVisionSwitchRequired), ); const parts: PartListUnion = [ @@ -329,4 +348,27 @@ describe('useVisionAutoSwitch hook', () => { expect(res).toEqual({ shouldProceed: false }); expect(config.setModel).not.toHaveBeenCalled(); }); + + it('does nothing when visionModelPreviewEnabled is false', async () => { + const config = createMockConfig(AuthType.QWEN_OAUTH, 'qwen3-coder-plus'); + const onVisionSwitchRequired = vi.fn(); + const { result } = renderHook(() => + useVisionAutoSwitch( + config, + addItem as any, + false, + onVisionSwitchRequired, + ), + ); + + const parts: PartListUnion = [ + { inlineData: { mimeType: 'image/png', data: '...' } }, + ]; + let res: any; + await act(async () => { + res = await result.current.handleVisionSwitch(parts, 6060, false); + }); + expect(res).toEqual({ shouldProceed: true }); + expect(onVisionSwitchRequired).not.toHaveBeenCalled(); + }); }); diff --git a/packages/cli/src/ui/hooks/useVisionAutoSwitch.ts b/packages/cli/src/ui/hooks/useVisionAutoSwitch.ts index 36764ee5..7b839c08 100644 --- a/packages/cli/src/ui/hooks/useVisionAutoSwitch.ts +++ b/packages/cli/src/ui/hooks/useVisionAutoSwitch.ts @@ -63,12 +63,18 @@ export function shouldOfferVisionSwitch( parts: PartListUnion, authType: AuthType, currentModel: string, + visionModelPreviewEnabled: boolean = false, ): boolean { // Only trigger for qwen-oauth if (authType !== AuthType.QWEN_OAUTH) { return false; } + // If vision model preview is disabled, never offer vision switch + if (!visionModelPreviewEnabled) { + return false; + } + // If current model is already a vision model, no need to switch if (isVisionModel(currentModel)) { return false; @@ -134,6 +140,7 @@ export interface VisionSwitchHandlingResult { export function useVisionAutoSwitch( config: Config, addItem: UseHistoryManagerReturn['addItem'], + visionModelPreviewEnabled: boolean = false, onVisionSwitchRequired?: (query: PartListUnion) => Promise<{ modelOverride?: string; persistSessionModel?: string; @@ -166,6 +173,7 @@ export function useVisionAutoSwitch( query, contentGeneratorConfig.authType, config.getModel(), + visionModelPreviewEnabled, ) ) { return { shouldProceed: true }; @@ -206,7 +214,7 @@ export function useVisionAutoSwitch( return { shouldProceed: false }; } }, - [config, addItem, onVisionSwitchRequired], + [config, addItem, visionModelPreviewEnabled, onVisionSwitchRequired], ); const restoreOriginalModel = useCallback(() => { diff --git a/packages/cli/src/ui/models/availableModels.ts b/packages/cli/src/ui/models/availableModels.ts index 0bfeba6d..7c3a1cf5 100644 --- a/packages/cli/src/ui/models/availableModels.ts +++ b/packages/cli/src/ui/models/availableModels.ts @@ -15,6 +15,18 @@ export const AVAILABLE_MODELS_QWEN: AvailableModel[] = [ { id: 'qwen-vl-max-latest', label: 'qwen-vl-max', isVision: true }, ]; +/** + * Get available Qwen models filtered by vision model preview setting + */ +export function getFilteredQwenModels( + visionModelPreviewEnabled: boolean, +): AvailableModel[] { + if (visionModelPreviewEnabled) { + return AVAILABLE_MODELS_QWEN; + } + return AVAILABLE_MODELS_QWEN.filter((model) => !model.isVision); +} + /** * Currently we use the single model of `OPENAI_MODEL` in the env. * In the future, after settings.json is updated, we will allow users to configure this themselves. diff --git a/packages/core/src/core/openaiContentGenerator/pipeline.ts b/packages/core/src/core/openaiContentGenerator/pipeline.ts index a6bd5533..85d279e6 100644 --- a/packages/core/src/core/openaiContentGenerator/pipeline.ts +++ b/packages/core/src/core/openaiContentGenerator/pipeline.ts @@ -12,7 +12,7 @@ import { import type { Config } from '../../config/config.js'; import type { ContentGeneratorConfig } from '../contentGenerator.js'; import type { OpenAICompatibleProvider } from './provider/index.js'; -import type { OpenAIContentConverter } from './converter.js'; +import { OpenAIContentConverter } from './converter.js'; import type { TelemetryService, RequestContext } from './telemetryService.js'; import type { ErrorHandler } from './errorHandler.js';