feat: add visionModelPreview to control default visibility of vision models

This commit is contained in:
mingholy.lmh
2025-09-17 20:58:54 +08:00
parent caedd8338f
commit a8ca4ebf89
7 changed files with 91 additions and 13 deletions

View File

@@ -741,6 +741,16 @@ export const SETTINGS_SCHEMA = {
description: 'Enable extension management features.', description: 'Enable extension management features.',
showInDialog: false, showInDialog: false,
}, },
visionModelPreview: {
type: 'boolean',
label: 'Vision Model Preview',
category: 'Experimental',
requiresRestart: false,
default: false,
description:
'Enable vision model support and auto-switching functionality. When disabled, vision models like qwen-vl-max-latest will be hidden and auto-switching will not occur.',
showInDialog: true,
},
}, },
}, },

View File

@@ -59,8 +59,8 @@ import {
type VisionSwitchOutcome, type VisionSwitchOutcome,
} from './components/ModelSwitchDialog.js'; } from './components/ModelSwitchDialog.js';
import { import {
AVAILABLE_MODELS_QWEN,
getOpenAIAvailableModelFromEnv, getOpenAIAvailableModelFromEnv,
getFilteredQwenModels,
type AvailableModel, type AvailableModel,
} from './models/availableModels.js'; } from './models/availableModels.js';
import { processVisionSwitchOutcome } from './hooks/useVisionAutoSwitch.js'; import { processVisionSwitchOutcome } from './hooks/useVisionAutoSwitch.js';
@@ -669,9 +669,12 @@ const App = ({ config, settings, startupWarnings = [], version }: AppProps) => {
const contentGeneratorConfig = config.getContentGeneratorConfig(); const contentGeneratorConfig = config.getContentGeneratorConfig();
if (!contentGeneratorConfig) return []; if (!contentGeneratorConfig) return [];
const visionModelPreviewEnabled =
settings.merged.experimental?.visionModelPreview ?? false;
switch (contentGeneratorConfig.authType) { switch (contentGeneratorConfig.authType) {
case AuthType.QWEN_OAUTH: case AuthType.QWEN_OAUTH:
return AVAILABLE_MODELS_QWEN; return getFilteredQwenModels(visionModelPreviewEnabled);
case AuthType.USE_OPENAI: { case AuthType.USE_OPENAI: {
const openAIModel = getOpenAIAvailableModelFromEnv(); const openAIModel = getOpenAIAvailableModelFromEnv();
return openAIModel ? [openAIModel] : []; return openAIModel ? [openAIModel] : [];
@@ -679,7 +682,7 @@ const App = ({ config, settings, startupWarnings = [], version }: AppProps) => {
default: default:
return []; return [];
} }
}, [config]); }, [config, settings.merged.experimental?.visionModelPreview]);
// Core hooks and processors // Core hooks and processors
const { const {
@@ -756,6 +759,7 @@ const App = ({ config, settings, startupWarnings = [], version }: AppProps) => {
setModelSwitchedFromQuotaError, setModelSwitchedFromQuotaError,
refreshStatic, refreshStatic,
() => cancelHandlerRef.current(), () => cancelHandlerRef.current(),
settings.merged.experimental?.visionModelPreview ?? false,
handleVisionSwitchRequired, handleVisionSwitchRequired,
); );

View File

@@ -89,6 +89,7 @@ export const useGeminiStream = (
setModelSwitchedFromQuotaError: React.Dispatch<React.SetStateAction<boolean>>, setModelSwitchedFromQuotaError: React.Dispatch<React.SetStateAction<boolean>>,
onEditorClose: () => void, onEditorClose: () => void,
onCancelSubmit: () => void, onCancelSubmit: () => void,
visionModelPreviewEnabled: boolean = false,
onVisionSwitchRequired?: (query: PartListUnion) => Promise<{ onVisionSwitchRequired?: (query: PartListUnion) => Promise<{
modelOverride?: string; modelOverride?: string;
persistSessionModel?: string; persistSessionModel?: string;
@@ -164,6 +165,7 @@ export const useGeminiStream = (
const { handleVisionSwitch, restoreOriginalModel } = useVisionAutoSwitch( const { handleVisionSwitch, restoreOriginalModel } = useVisionAutoSwitch(
config, config,
addItem, addItem,
visionModelPreviewEnabled,
onVisionSwitchRequired, onVisionSwitchRequired,
); );

View File

@@ -29,6 +29,7 @@ describe('useVisionAutoSwitch helpers', () => {
parts, parts,
AuthType.USE_GEMINI, AuthType.USE_GEMINI,
'qwen3-coder-plus', 'qwen3-coder-plus',
true,
); );
expect(result).toBe(false); expect(result).toBe(false);
}); });
@@ -41,6 +42,7 @@ describe('useVisionAutoSwitch helpers', () => {
parts, parts,
AuthType.QWEN_OAUTH, AuthType.QWEN_OAUTH,
'qwen-vl-max-latest', 'qwen-vl-max-latest',
true,
); );
expect(result).toBe(false); expect(result).toBe(false);
}); });
@@ -54,6 +56,7 @@ describe('useVisionAutoSwitch helpers', () => {
parts, parts,
AuthType.QWEN_OAUTH, AuthType.QWEN_OAUTH,
'qwen3-coder-plus', 'qwen3-coder-plus',
true,
); );
expect(result).toBe(true); expect(result).toBe(true);
}); });
@@ -66,6 +69,7 @@ describe('useVisionAutoSwitch helpers', () => {
singleImagePart, singleImagePart,
AuthType.QWEN_OAUTH, AuthType.QWEN_OAUTH,
'qwen3-coder-plus', 'qwen3-coder-plus',
true,
); );
expect(result).toBe(true); expect(result).toBe(true);
}); });
@@ -76,6 +80,7 @@ describe('useVisionAutoSwitch helpers', () => {
parts, parts,
AuthType.QWEN_OAUTH, AuthType.QWEN_OAUTH,
'qwen3-coder-plus', 'qwen3-coder-plus',
true,
); );
expect(result).toBe(false); expect(result).toBe(false);
}); });
@@ -86,6 +91,20 @@ describe('useVisionAutoSwitch helpers', () => {
parts, parts,
AuthType.QWEN_OAUTH, AuthType.QWEN_OAUTH,
'qwen3-coder-plus', 'qwen3-coder-plus',
true,
);
expect(result).toBe(false);
});
it('returns false when visionModelPreviewEnabled is false', () => {
const parts: PartListUnion = [
{ inlineData: { mimeType: 'image/png', data: '...' } },
];
const result = shouldOfferVisionSwitch(
parts,
AuthType.QWEN_OAUTH,
'qwen3-coder-plus',
false,
); );
expect(result).toBe(false); expect(result).toBe(false);
}); });
@@ -159,7 +178,7 @@ describe('useVisionAutoSwitch hook', () => {
it('returns shouldProceed=true immediately for continuations', async () => { it('returns shouldProceed=true immediately for continuations', async () => {
const config = createMockConfig(AuthType.QWEN_OAUTH, 'qwen3-coder-plus'); const config = createMockConfig(AuthType.QWEN_OAUTH, 'qwen3-coder-plus');
const { result } = renderHook(() => const { result } = renderHook(() =>
useVisionAutoSwitch(config, addItem as any, vi.fn()), useVisionAutoSwitch(config, addItem as any, true, vi.fn()),
); );
const parts: PartListUnion = [ const parts: PartListUnion = [
@@ -177,7 +196,7 @@ describe('useVisionAutoSwitch hook', () => {
const config = createMockConfig(AuthType.USE_GEMINI, 'qwen3-coder-plus'); const config = createMockConfig(AuthType.USE_GEMINI, 'qwen3-coder-plus');
const onVisionSwitchRequired = vi.fn(); const onVisionSwitchRequired = vi.fn();
const { result } = renderHook(() => const { result } = renderHook(() =>
useVisionAutoSwitch(config, addItem as any, onVisionSwitchRequired), useVisionAutoSwitch(config, addItem as any, true, onVisionSwitchRequired),
); );
const parts: PartListUnion = [ const parts: PartListUnion = [
@@ -195,7 +214,7 @@ describe('useVisionAutoSwitch hook', () => {
const config = createMockConfig(AuthType.QWEN_OAUTH, 'qwen3-coder-plus'); const config = createMockConfig(AuthType.QWEN_OAUTH, 'qwen3-coder-plus');
const onVisionSwitchRequired = vi.fn(); const onVisionSwitchRequired = vi.fn();
const { result } = renderHook(() => const { result } = renderHook(() =>
useVisionAutoSwitch(config, addItem as any, onVisionSwitchRequired), useVisionAutoSwitch(config, addItem as any, true, onVisionSwitchRequired),
); );
const parts: PartListUnion = [{ text: 'no images here' }]; const parts: PartListUnion = [{ text: 'no images here' }];
@@ -213,7 +232,7 @@ describe('useVisionAutoSwitch hook', () => {
.fn() .fn()
.mockResolvedValue({ showGuidance: true }); .mockResolvedValue({ showGuidance: true });
const { result } = renderHook(() => const { result } = renderHook(() =>
useVisionAutoSwitch(config, addItem as any, onVisionSwitchRequired), useVisionAutoSwitch(config, addItem as any, true, onVisionSwitchRequired),
); );
const parts: PartListUnion = [ const parts: PartListUnion = [
@@ -241,7 +260,7 @@ describe('useVisionAutoSwitch hook', () => {
.fn() .fn()
.mockResolvedValue({ modelOverride: 'qwen-vl-max-latest' }); .mockResolvedValue({ modelOverride: 'qwen-vl-max-latest' });
const { result } = renderHook(() => const { result } = renderHook(() =>
useVisionAutoSwitch(config, addItem as any, onVisionSwitchRequired), useVisionAutoSwitch(config, addItem as any, true, onVisionSwitchRequired),
); );
const parts: PartListUnion = [ const parts: PartListUnion = [
@@ -269,7 +288,7 @@ describe('useVisionAutoSwitch hook', () => {
.fn() .fn()
.mockResolvedValue({ persistSessionModel: 'qwen-vl-max-latest' }); .mockResolvedValue({ persistSessionModel: 'qwen-vl-max-latest' });
const { result } = renderHook(() => const { result } = renderHook(() =>
useVisionAutoSwitch(config, addItem as any, onVisionSwitchRequired), useVisionAutoSwitch(config, addItem as any, true, onVisionSwitchRequired),
); );
const parts: PartListUnion = [ const parts: PartListUnion = [
@@ -298,7 +317,7 @@ describe('useVisionAutoSwitch hook', () => {
const config = createMockConfig(AuthType.QWEN_OAUTH, 'qwen3-coder-plus'); const config = createMockConfig(AuthType.QWEN_OAUTH, 'qwen3-coder-plus');
const onVisionSwitchRequired = vi.fn().mockResolvedValue({}); const onVisionSwitchRequired = vi.fn().mockResolvedValue({});
const { result } = renderHook(() => const { result } = renderHook(() =>
useVisionAutoSwitch(config, addItem as any, onVisionSwitchRequired), useVisionAutoSwitch(config, addItem as any, true, onVisionSwitchRequired),
); );
const parts: PartListUnion = [ const parts: PartListUnion = [
@@ -316,7 +335,7 @@ describe('useVisionAutoSwitch hook', () => {
const config = createMockConfig(AuthType.QWEN_OAUTH, 'qwen3-coder-plus'); const config = createMockConfig(AuthType.QWEN_OAUTH, 'qwen3-coder-plus');
const onVisionSwitchRequired = vi.fn().mockRejectedValue(new Error('x')); const onVisionSwitchRequired = vi.fn().mockRejectedValue(new Error('x'));
const { result } = renderHook(() => const { result } = renderHook(() =>
useVisionAutoSwitch(config, addItem as any, onVisionSwitchRequired), useVisionAutoSwitch(config, addItem as any, true, onVisionSwitchRequired),
); );
const parts: PartListUnion = [ const parts: PartListUnion = [
@@ -329,4 +348,27 @@ describe('useVisionAutoSwitch hook', () => {
expect(res).toEqual({ shouldProceed: false }); expect(res).toEqual({ shouldProceed: false });
expect(config.setModel).not.toHaveBeenCalled(); expect(config.setModel).not.toHaveBeenCalled();
}); });
it('does nothing when visionModelPreviewEnabled is false', async () => {
const config = createMockConfig(AuthType.QWEN_OAUTH, 'qwen3-coder-plus');
const onVisionSwitchRequired = vi.fn();
const { result } = renderHook(() =>
useVisionAutoSwitch(
config,
addItem as any,
false,
onVisionSwitchRequired,
),
);
const parts: PartListUnion = [
{ inlineData: { mimeType: 'image/png', data: '...' } },
];
let res: any;
await act(async () => {
res = await result.current.handleVisionSwitch(parts, 6060, false);
});
expect(res).toEqual({ shouldProceed: true });
expect(onVisionSwitchRequired).not.toHaveBeenCalled();
});
}); });

View File

@@ -63,12 +63,18 @@ export function shouldOfferVisionSwitch(
parts: PartListUnion, parts: PartListUnion,
authType: AuthType, authType: AuthType,
currentModel: string, currentModel: string,
visionModelPreviewEnabled: boolean = false,
): boolean { ): boolean {
// Only trigger for qwen-oauth // Only trigger for qwen-oauth
if (authType !== AuthType.QWEN_OAUTH) { if (authType !== AuthType.QWEN_OAUTH) {
return false; return false;
} }
// If vision model preview is disabled, never offer vision switch
if (!visionModelPreviewEnabled) {
return false;
}
// If current model is already a vision model, no need to switch // If current model is already a vision model, no need to switch
if (isVisionModel(currentModel)) { if (isVisionModel(currentModel)) {
return false; return false;
@@ -134,6 +140,7 @@ export interface VisionSwitchHandlingResult {
export function useVisionAutoSwitch( export function useVisionAutoSwitch(
config: Config, config: Config,
addItem: UseHistoryManagerReturn['addItem'], addItem: UseHistoryManagerReturn['addItem'],
visionModelPreviewEnabled: boolean = false,
onVisionSwitchRequired?: (query: PartListUnion) => Promise<{ onVisionSwitchRequired?: (query: PartListUnion) => Promise<{
modelOverride?: string; modelOverride?: string;
persistSessionModel?: string; persistSessionModel?: string;
@@ -166,6 +173,7 @@ export function useVisionAutoSwitch(
query, query,
contentGeneratorConfig.authType, contentGeneratorConfig.authType,
config.getModel(), config.getModel(),
visionModelPreviewEnabled,
) )
) { ) {
return { shouldProceed: true }; return { shouldProceed: true };
@@ -206,7 +214,7 @@ export function useVisionAutoSwitch(
return { shouldProceed: false }; return { shouldProceed: false };
} }
}, },
[config, addItem, onVisionSwitchRequired], [config, addItem, visionModelPreviewEnabled, onVisionSwitchRequired],
); );
const restoreOriginalModel = useCallback(() => { const restoreOriginalModel = useCallback(() => {

View File

@@ -15,6 +15,18 @@ export const AVAILABLE_MODELS_QWEN: AvailableModel[] = [
{ id: 'qwen-vl-max-latest', label: 'qwen-vl-max', isVision: true }, { id: 'qwen-vl-max-latest', label: 'qwen-vl-max', isVision: true },
]; ];
/**
* Get available Qwen models filtered by vision model preview setting
*/
export function getFilteredQwenModels(
visionModelPreviewEnabled: boolean,
): AvailableModel[] {
if (visionModelPreviewEnabled) {
return AVAILABLE_MODELS_QWEN;
}
return AVAILABLE_MODELS_QWEN.filter((model) => !model.isVision);
}
/** /**
* Currently we use the single model of `OPENAI_MODEL` in the env. * Currently we use the single model of `OPENAI_MODEL` in the env.
* In the future, after settings.json is updated, we will allow users to configure this themselves. * In the future, after settings.json is updated, we will allow users to configure this themselves.

View File

@@ -12,7 +12,7 @@ import {
import type { Config } from '../../config/config.js'; import type { Config } from '../../config/config.js';
import type { ContentGeneratorConfig } from '../contentGenerator.js'; import type { ContentGeneratorConfig } from '../contentGenerator.js';
import type { OpenAICompatibleProvider } from './provider/index.js'; import type { OpenAICompatibleProvider } from './provider/index.js';
import type { OpenAIContentConverter } from './converter.js'; import { OpenAIContentConverter } from './converter.js';
import type { TelemetryService, RequestContext } from './telemetryService.js'; import type { TelemetryService, RequestContext } from './telemetryService.js';
import type { ErrorHandler } from './errorHandler.js'; import type { ErrorHandler } from './errorHandler.js';