feat(ai): complete models runtime migration
This commit is contained in:
Mario Zechner
@@ -21,6 +21,7 @@ import type {
|
||||
Model,
|
||||
OpenAICompletionsCompat,
|
||||
ProviderEnv,
|
||||
ProviderHeaders,
|
||||
SimpleStreamOptions,
|
||||
StopReason,
|
||||
StreamFunction,
|
||||
@@ -36,7 +37,6 @@ import { headersToRecord } from "../utils/headers.ts";
|
||||
import { parseStreamingJson } from "../utils/json-parse.ts";
|
||||
import { getProviderEnvValue } from "../utils/provider-env.ts";
|
||||
import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts";
|
||||
import { isCloudflareProvider, resolveCloudflareBaseUrl } from "./cloudflare.ts";
|
||||
import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.ts";
|
||||
import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts";
|
||||
import { buildBaseOptions } from "./simple-options.ts";
|
||||
@@ -47,6 +47,21 @@ import { transformMessages } from "./transform-messages.ts";
|
||||
* This is needed because Anthropic (via proxy) requires the tools param
|
||||
* to be present when messages include tool_calls or tool role messages.
|
||||
*/
|
||||
function hasHeader(headers: ProviderHeaders | undefined, name: string): boolean {
|
||||
if (!headers) return false;
|
||||
const expected = name.toLowerCase();
|
||||
for (const [key, value] of Object.entries(headers)) {
|
||||
if (key.toLowerCase() === expected && value !== null && value.trim().length > 0) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function getClientApiKey(provider: string, apiKey: string | undefined, headers: ProviderHeaders | undefined): string {
|
||||
if (apiKey) return apiKey;
|
||||
if (hasHeader(headers, "authorization") || hasHeader(headers, "cf-aig-authorization")) return "unused";
|
||||
throw new Error(`No API key for provider: ${provider}`);
|
||||
}
|
||||
|
||||
function hasToolHistory(messages: Message[]): boolean {
|
||||
for (const msg of messages) {
|
||||
if (msg.role === "toolResult") {
|
||||
@@ -160,14 +175,11 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio
|
||||
};
|
||||
|
||||
try {
|
||||
const apiKey = options?.apiKey;
|
||||
if (!apiKey) {
|
||||
throw new Error(`No API key for provider: ${model.provider}`);
|
||||
}
|
||||
const apiKey = getClientApiKey(model.provider, options?.apiKey, options?.headers);
|
||||
const compat = getCompat(model);
|
||||
const cacheRetention = resolveCacheRetention(options?.cacheRetention, options?.env);
|
||||
const cacheSessionId = cacheRetention === "none" ? undefined : options?.sessionId;
|
||||
const client = createClient(model, context, apiKey, options?.headers, cacheSessionId, compat, options?.env);
|
||||
const client = createClient(model, context, apiKey, options?.headers, cacheSessionId, compat);
|
||||
let params = buildParams(model, context, options, compat, cacheRetention);
|
||||
const nextParams = await options?.onPayload?.(params, model);
|
||||
if (nextParams !== undefined) {
|
||||
@@ -468,12 +480,9 @@ export const streamSimple: StreamFunction<"openai-completions", SimpleStreamOpti
|
||||
context: Context,
|
||||
options?: SimpleStreamOptions,
|
||||
): AssistantMessageEventStream => {
|
||||
const apiKey = options?.apiKey;
|
||||
if (!apiKey) {
|
||||
throw new Error(`No API key for provider: ${model.provider}`);
|
||||
}
|
||||
getClientApiKey(model.provider, options?.apiKey, options?.headers);
|
||||
|
||||
const base = buildBaseOptions(model, options, apiKey);
|
||||
const base = buildBaseOptions(model, options, options?.apiKey);
|
||||
const clampedReasoning = options?.reasoning ? clampThinkingLevel(model, options.reasoning) : undefined;
|
||||
const reasoningEffort = clampedReasoning === "off" ? undefined : clampedReasoning;
|
||||
const toolChoice = (options as OpenAICompletionsOptions | undefined)?.toolChoice;
|
||||
@@ -489,12 +498,11 @@ function createClient(
|
||||
model: Model<"openai-completions">,
|
||||
context: Context,
|
||||
apiKey: string,
|
||||
optionsHeaders?: Record<string, string>,
|
||||
optionsHeaders?: ProviderHeaders,
|
||||
sessionId?: string,
|
||||
compat: ResolvedOpenAICompletionsCompat = getCompat(model),
|
||||
env?: ProviderEnv,
|
||||
) {
|
||||
const headers = { ...model.headers };
|
||||
const headers: ProviderHeaders = { ...model.headers };
|
||||
if (model.provider === "github-copilot") {
|
||||
const hasImages = hasCopilotVisionInput(context.messages);
|
||||
const copilotHeaders = buildCopilotDynamicHeaders({
|
||||
@@ -515,20 +523,11 @@ function createClient(
|
||||
Object.assign(headers, optionsHeaders);
|
||||
}
|
||||
|
||||
const defaultHeaders =
|
||||
model.provider === "cloudflare-ai-gateway"
|
||||
? {
|
||||
...headers,
|
||||
Authorization: headers.Authorization ?? null,
|
||||
"cf-aig-authorization": `Bearer ${apiKey}`,
|
||||
}
|
||||
: headers;
|
||||
|
||||
return new OpenAI({
|
||||
apiKey,
|
||||
baseURL: isCloudflareProvider(model.provider) ? resolveCloudflareBaseUrl(model, env) : model.baseUrl,
|
||||
baseURL: model.baseUrl,
|
||||
dangerouslyAllowBrowser: true,
|
||||
defaultHeaders,
|
||||
defaultHeaders: headers,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -674,7 +673,7 @@ function buildParams(
|
||||
}
|
||||
|
||||
// Vercel AI Gateway provider routing preferences
|
||||
if (model.baseUrl.includes("ai-gateway.vercel.sh") && model.compat?.vercelGatewayRouting) {
|
||||
if (model.compat?.vercelGatewayRouting) {
|
||||
const routing = model.compat.vercelGatewayRouting;
|
||||
if (routing.only || routing.order) {
|
||||
const gatewayOptions: Record<string, string[]> = {};
|
||||
@@ -1164,123 +1163,55 @@ function mapStopReason(reason: ChatCompletionChunk.Choice["finish_reason"] | str
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect compatibility settings from provider and baseUrl for known providers.
|
||||
* Provider takes precedence over URL-based detection since it's explicitly configured.
|
||||
* Returns a fully resolved OpenAICompletionsCompat object with all fields set.
|
||||
*/
|
||||
function detectCompat(model: Model<"openai-completions">): ResolvedOpenAICompletionsCompat {
|
||||
const provider = model.provider;
|
||||
const baseUrl = model.baseUrl;
|
||||
|
||||
const isZai =
|
||||
provider === "zai" ||
|
||||
provider === "zai-coding-cn" ||
|
||||
baseUrl.includes("api.z.ai") ||
|
||||
baseUrl.includes("open.bigmodel.cn");
|
||||
const isTogether =
|
||||
provider === "together" || baseUrl.includes("api.together.ai") || baseUrl.includes("api.together.xyz");
|
||||
const isMoonshot = provider === "moonshotai" || provider === "moonshotai-cn" || baseUrl.includes("api.moonshot.");
|
||||
const isOpenRouter = provider === "openrouter" || baseUrl.includes("openrouter.ai");
|
||||
const isCloudflareWorkersAI = provider === "cloudflare-workers-ai" || baseUrl.includes("api.cloudflare.com");
|
||||
const isCloudflareAiGateway = provider === "cloudflare-ai-gateway" || baseUrl.includes("gateway.ai.cloudflare.com");
|
||||
const isNvidia = provider === "nvidia" || baseUrl.includes("integrate.api.nvidia.com");
|
||||
const isAntLing = provider === "ant-ling" || baseUrl.includes("api.ant-ling.com");
|
||||
|
||||
const isNonStandard =
|
||||
isNvidia ||
|
||||
provider === "cerebras" ||
|
||||
baseUrl.includes("cerebras.ai") ||
|
||||
provider === "xai" ||
|
||||
baseUrl.includes("api.x.ai") ||
|
||||
isTogether ||
|
||||
baseUrl.includes("chutes.ai") ||
|
||||
baseUrl.includes("deepseek.com") ||
|
||||
isZai ||
|
||||
isMoonshot ||
|
||||
provider === "opencode" ||
|
||||
baseUrl.includes("opencode.ai") ||
|
||||
isCloudflareWorkersAI ||
|
||||
isCloudflareAiGateway ||
|
||||
isAntLing;
|
||||
|
||||
const useMaxTokens =
|
||||
baseUrl.includes("chutes.ai") || isMoonshot || isCloudflareAiGateway || isTogether || isNvidia || isAntLing;
|
||||
|
||||
const isGrok = provider === "xai" || baseUrl.includes("api.x.ai");
|
||||
const isDeepSeek = provider === "deepseek" || baseUrl.includes("deepseek.com");
|
||||
const isOpenRouterDeveloperRoleModel =
|
||||
isOpenRouter && (model.id.startsWith("anthropic/") || model.id.startsWith("openai/"));
|
||||
const cacheControlFormat = provider === "openrouter" && model.id.startsWith("anthropic/") ? "anthropic" : undefined;
|
||||
|
||||
return {
|
||||
supportsStore: !isNonStandard,
|
||||
supportsDeveloperRole: isOpenRouterDeveloperRoleModel || (!isNonStandard && !isOpenRouter),
|
||||
supportsReasoningEffort:
|
||||
!isGrok && !isZai && !isMoonshot && !isTogether && !isCloudflareAiGateway && !isNvidia && !isAntLing,
|
||||
supportsUsageInStreaming: true,
|
||||
maxTokensField: useMaxTokens ? "max_tokens" : "max_completion_tokens",
|
||||
requiresToolResultName: false,
|
||||
requiresAssistantAfterToolResult: false,
|
||||
requiresThinkingAsText: false,
|
||||
requiresReasoningContentOnAssistantMessages: isDeepSeek,
|
||||
thinkingFormat: isDeepSeek
|
||||
? "deepseek"
|
||||
: isZai
|
||||
? "zai"
|
||||
: isTogether
|
||||
? "together"
|
||||
: isAntLing
|
||||
? "ant-ling"
|
||||
: isOpenRouter
|
||||
? "openrouter"
|
||||
: "openai",
|
||||
openRouterRouting: {},
|
||||
vercelGatewayRouting: {},
|
||||
chatTemplateKwargs: {},
|
||||
zaiToolStream: false,
|
||||
supportsStrictMode: !isMoonshot && !isTogether && !isCloudflareAiGateway && !isNvidia,
|
||||
cacheControlFormat,
|
||||
sendSessionAffinityHeaders: false,
|
||||
supportsLongCacheRetention: !(
|
||||
isTogether ||
|
||||
isCloudflareWorkersAI ||
|
||||
isCloudflareAiGateway ||
|
||||
isNvidia ||
|
||||
isAntLing
|
||||
),
|
||||
};
|
||||
}
|
||||
const DEFAULT_COMPAT: ResolvedOpenAICompletionsCompat = {
|
||||
supportsStore: true,
|
||||
supportsDeveloperRole: true,
|
||||
supportsReasoningEffort: true,
|
||||
supportsUsageInStreaming: true,
|
||||
maxTokensField: "max_completion_tokens",
|
||||
requiresToolResultName: false,
|
||||
requiresAssistantAfterToolResult: false,
|
||||
requiresThinkingAsText: false,
|
||||
requiresReasoningContentOnAssistantMessages: false,
|
||||
thinkingFormat: "openai",
|
||||
openRouterRouting: {},
|
||||
vercelGatewayRouting: {},
|
||||
chatTemplateKwargs: {},
|
||||
zaiToolStream: false,
|
||||
supportsStrictMode: true,
|
||||
cacheControlFormat: undefined,
|
||||
sendSessionAffinityHeaders: false,
|
||||
supportsLongCacheRetention: true,
|
||||
};
|
||||
|
||||
/**
|
||||
* Get resolved compatibility settings for a model.
|
||||
* Uses explicit model.compat if provided, otherwise auto-detects from provider/URL.
|
||||
* Uses explicit generated/custom model.compat over OpenAI-standard defaults.
|
||||
*/
|
||||
function getCompat(model: Model<"openai-completions">): ResolvedOpenAICompletionsCompat {
|
||||
const detected = detectCompat(model);
|
||||
if (!model.compat) return detected;
|
||||
if (!model.compat) return DEFAULT_COMPAT;
|
||||
|
||||
return {
|
||||
supportsStore: model.compat.supportsStore ?? detected.supportsStore,
|
||||
supportsDeveloperRole: model.compat.supportsDeveloperRole ?? detected.supportsDeveloperRole,
|
||||
supportsReasoningEffort: model.compat.supportsReasoningEffort ?? detected.supportsReasoningEffort,
|
||||
supportsUsageInStreaming: model.compat.supportsUsageInStreaming ?? detected.supportsUsageInStreaming,
|
||||
maxTokensField: model.compat.maxTokensField ?? detected.maxTokensField,
|
||||
requiresToolResultName: model.compat.requiresToolResultName ?? detected.requiresToolResultName,
|
||||
supportsStore: model.compat.supportsStore ?? DEFAULT_COMPAT.supportsStore,
|
||||
supportsDeveloperRole: model.compat.supportsDeveloperRole ?? DEFAULT_COMPAT.supportsDeveloperRole,
|
||||
supportsReasoningEffort: model.compat.supportsReasoningEffort ?? DEFAULT_COMPAT.supportsReasoningEffort,
|
||||
supportsUsageInStreaming: model.compat.supportsUsageInStreaming ?? DEFAULT_COMPAT.supportsUsageInStreaming,
|
||||
maxTokensField: model.compat.maxTokensField ?? DEFAULT_COMPAT.maxTokensField,
|
||||
requiresToolResultName: model.compat.requiresToolResultName ?? DEFAULT_COMPAT.requiresToolResultName,
|
||||
requiresAssistantAfterToolResult:
|
||||
model.compat.requiresAssistantAfterToolResult ?? detected.requiresAssistantAfterToolResult,
|
||||
requiresThinkingAsText: model.compat.requiresThinkingAsText ?? detected.requiresThinkingAsText,
|
||||
model.compat.requiresAssistantAfterToolResult ?? DEFAULT_COMPAT.requiresAssistantAfterToolResult,
|
||||
requiresThinkingAsText: model.compat.requiresThinkingAsText ?? DEFAULT_COMPAT.requiresThinkingAsText,
|
||||
requiresReasoningContentOnAssistantMessages:
|
||||
model.compat.requiresReasoningContentOnAssistantMessages ??
|
||||
detected.requiresReasoningContentOnAssistantMessages,
|
||||
thinkingFormat: model.compat.thinkingFormat ?? detected.thinkingFormat,
|
||||
openRouterRouting: model.compat.openRouterRouting ?? {},
|
||||
vercelGatewayRouting: model.compat.vercelGatewayRouting ?? detected.vercelGatewayRouting,
|
||||
chatTemplateKwargs: model.compat.chatTemplateKwargs ?? detected.chatTemplateKwargs,
|
||||
zaiToolStream: model.compat.zaiToolStream ?? detected.zaiToolStream,
|
||||
supportsStrictMode: model.compat.supportsStrictMode ?? detected.supportsStrictMode,
|
||||
cacheControlFormat: model.compat.cacheControlFormat ?? detected.cacheControlFormat,
|
||||
sendSessionAffinityHeaders: model.compat.sendSessionAffinityHeaders ?? detected.sendSessionAffinityHeaders,
|
||||
supportsLongCacheRetention: model.compat.supportsLongCacheRetention ?? detected.supportsLongCacheRetention,
|
||||
DEFAULT_COMPAT.requiresReasoningContentOnAssistantMessages,
|
||||
thinkingFormat: model.compat.thinkingFormat ?? DEFAULT_COMPAT.thinkingFormat,
|
||||
openRouterRouting: model.compat.openRouterRouting ?? DEFAULT_COMPAT.openRouterRouting,
|
||||
vercelGatewayRouting: model.compat.vercelGatewayRouting ?? DEFAULT_COMPAT.vercelGatewayRouting,
|
||||
chatTemplateKwargs: model.compat.chatTemplateKwargs ?? DEFAULT_COMPAT.chatTemplateKwargs,
|
||||
zaiToolStream: model.compat.zaiToolStream ?? DEFAULT_COMPAT.zaiToolStream,
|
||||
supportsStrictMode: model.compat.supportsStrictMode ?? DEFAULT_COMPAT.supportsStrictMode,
|
||||
cacheControlFormat: model.compat.cacheControlFormat ?? DEFAULT_COMPAT.cacheControlFormat,
|
||||
sendSessionAffinityHeaders: model.compat.sendSessionAffinityHeaders ?? DEFAULT_COMPAT.sendSessionAffinityHeaders,
|
||||
supportsLongCacheRetention: model.compat.supportsLongCacheRetention ?? DEFAULT_COMPAT.supportsLongCacheRetention,
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user