Armin Ronacher
706 lines
25 KiB
TypeScript
706 lines
25 KiB
TypeScript
import { lazyStream } from "./api/lazy.ts";
|
|
import { defaultProviderAuthContext as defaultAuthContext } from "./auth/context.ts";
|
|
import { InMemoryCredentialStore } from "./auth/credential-store.ts";
|
|
import { type AuthResolutionOverrides, ModelsError, resolveProviderAuth } from "./auth/resolve.ts";
|
|
import type {
|
|
AuthCheck,
|
|
AuthContext,
|
|
AuthInteraction,
|
|
AuthResult,
|
|
AuthType,
|
|
Credential,
|
|
CredentialStore,
|
|
ProviderAuth,
|
|
} from "./auth/types.ts";
|
|
import { InMemoryModelsStore, type ModelsStore, type ProviderModelsStore } from "./models-store.ts";
|
|
import type {
|
|
Api,
|
|
ApiStreamOptions,
|
|
AssistantMessage,
|
|
AssistantMessageEventStream,
|
|
Context,
|
|
Model,
|
|
ModelCostRates,
|
|
ModelThinkingLevel,
|
|
ProviderHeaders,
|
|
ProviderStreams,
|
|
SimpleStreamOptions,
|
|
StreamOptions,
|
|
Usage,
|
|
} from "./types.ts";
|
|
|
|
export { ModelsError, type ModelsErrorCode } from "./auth/resolve.ts";
|
|
|
|
export interface RefreshModelsContext {
|
|
/** Effective configured credential. OAuth credentials are refreshed before network access. */
|
|
credential?: Credential;
|
|
/** Persistent model storage scoped to this provider ID. */
|
|
store: ProviderModelsStore;
|
|
/** False during offline/cache-only initialization. */
|
|
allowNetwork: boolean;
|
|
/** Bypass provider freshness checks and fetch immediately when network access is allowed. */
|
|
force?: boolean;
|
|
signal?: AbortSignal;
|
|
}
|
|
|
|
export interface ModelsRefreshOptions {
|
|
allowNetwork?: boolean;
|
|
/** Bypass provider freshness checks and fetch immediately when network access is allowed. */
|
|
force?: boolean;
|
|
signal?: AbortSignal;
|
|
}
|
|
|
|
export interface ModelsRefreshResult {
|
|
aborted: boolean;
|
|
errors: ReadonlyMap<string, Error>;
|
|
}
|
|
|
|
export interface ModelsStreamTransforms {
|
|
/** Transform fully assembled model/auth/request headers before provider dispatch. */
|
|
transformHeaders?: (headers: ProviderHeaders) => ProviderHeaders | Promise<ProviderHeaders>;
|
|
}
|
|
|
|
export type ModelsApiStreamOptions<TApi extends Api> = ApiStreamOptions<TApi> & ModelsStreamTransforms;
|
|
export type ModelsSimpleStreamOptions = SimpleStreamOptions & ModelsStreamTransforms;
|
|
|
|
/**
|
|
* A provider is the concrete runtime unit. It owns id/name/base metadata,
|
|
* auth methods, model listing, and stream behavior.
|
|
*
|
|
* `TApi` lets concrete provider factories declare which APIs their models
|
|
* use (e.g. `openaiProvider(): Provider<"openai-responses" | "openai-completions">`),
|
|
* giving typed model lists to direct factory users. Inside a `Models`
|
|
* collection providers are held as `Provider<Api>`.
|
|
*/
|
|
export interface Provider<TApi extends Api = Api> {
|
|
readonly id: string;
|
|
readonly name: string;
|
|
|
|
readonly baseUrl?: string;
|
|
readonly headers?: ProviderHeaders;
|
|
|
|
/**
|
|
* Required: at least one of `apiKey`/`oauth`. Every provider has auth
|
|
* semantics — even providers with only ambient credentials (env vars, AWS
|
|
* profiles, ADC files) and keyless local servers provide `apiKey` auth
|
|
* whose `resolve()` reports whether the provider is configured.
|
|
* `Models.getAuth()` returns undefined when the provider is unconfigured.
|
|
*/
|
|
readonly auth: ProviderAuth;
|
|
|
|
/**
|
|
* Current known models, sync. Static providers return their catalog;
|
|
* dynamic providers return the list as of the last `refreshModels()`
|
|
* (empty before the first). Must not throw; `Models` treats a throwing
|
|
* implementation as having no models.
|
|
*/
|
|
getModels(): readonly Model<TApi>[];
|
|
|
|
/**
|
|
* Dynamic providers only: restore the provider-scoped stored catalog and optionally fetch
|
|
* a newer list using the effective credential. Implementations must retain their previous
|
|
* list on failure and honor the shared abort signal for network requests.
|
|
*/
|
|
refreshModels?(context: RefreshModelsContext): Promise<void>;
|
|
|
|
/**
|
|
* Optional provider policy for credential-specific model availability.
|
|
* `getModels()` remains the complete synchronous catalog; `Models.getAvailable()`
|
|
* applies this filter after confirming that provider auth is configured.
|
|
*/
|
|
filterModels?(models: readonly Model<TApi>[], credential: Credential | undefined): readonly Model<TApi>[];
|
|
|
|
stream<T extends TApi>(
|
|
model: Model<T>,
|
|
context: Context,
|
|
options?: ApiStreamOptions<T>,
|
|
): AssistantMessageEventStream;
|
|
|
|
streamSimple(model: Model<TApi>, context: Context, options?: SimpleStreamOptions): AssistantMessageEventStream;
|
|
}
|
|
|
|
/**
|
|
* Runtime collection of providers plus auth application and stream
|
|
* convenience. Providers own stream behavior; `Models` resolves auth and
|
|
* delegates each request to the provider that owns the model.
|
|
*/
|
|
export interface Models {
|
|
getProviders(): readonly Provider[];
|
|
getProvider(id: string): Provider | undefined;
|
|
|
|
/**
|
|
* Sync read of last-known models from one provider or all providers.
|
|
* Best-effort: a provider whose `getModels()` throws yields no models.
|
|
*/
|
|
getModels(provider?: string): readonly Model<Api>[];
|
|
|
|
/**
|
|
* Sync runtime model lookup against last-known lists. Dynamic model lists
|
|
* are typed as `Model<Api>`; narrow with the `hasApi()` type guard.
|
|
*/
|
|
getModel(provider: string, id: string): Model<Api> | undefined;
|
|
|
|
/**
|
|
* Refresh every configured dynamic provider concurrently. Provider errors and cancellation
|
|
* are returned without rejecting; static and unconfigured providers are skipped.
|
|
*/
|
|
refresh(options?: ModelsRefreshOptions): Promise<ModelsRefreshResult>;
|
|
|
|
/** Check whether a provider has complete auth configuration without refreshing OAuth. */
|
|
checkAuth(providerId: string): Promise<AuthCheck | undefined>;
|
|
|
|
/** Return models whose providers have complete auth configuration. */
|
|
getAvailable(providerId?: string): Promise<readonly Model<Api>[]>;
|
|
|
|
/**
|
|
* Resolve provider-scoped auth by provider id, or provider auth plus static
|
|
* model headers when passed a model. Includes a source label for status UI.
|
|
* Resolves `undefined` when the provider is unknown or unconfigured.
|
|
* Rejects with `ModelsError`: code "oauth" when a token refresh fails (the
|
|
* stored credential is preserved for retry; re-login fixes it), code "auth"
|
|
* when api-key resolution or the credential store fails. Request paths
|
|
* surface rejections as stream errors.
|
|
*/
|
|
getAuth(providerId: string, overrides?: AuthResolutionOverrides): Promise<AuthResult | undefined>;
|
|
getAuth(model: Model<Api>, overrides?: AuthResolutionOverrides): Promise<AuthResult | undefined>;
|
|
|
|
/** Run a provider-owned login flow and persist its returned credential. */
|
|
login(providerId: string, type: AuthType, interaction: AuthInteraction): Promise<Credential>;
|
|
|
|
/** Remove the stored credential for a provider. */
|
|
logout(providerId: string): Promise<void>;
|
|
|
|
stream<TApi extends Api>(
|
|
model: Model<TApi>,
|
|
context: Context,
|
|
options?: ModelsApiStreamOptions<TApi>,
|
|
): AssistantMessageEventStream;
|
|
|
|
complete<TApi extends Api>(
|
|
model: Model<TApi>,
|
|
context: Context,
|
|
options?: ModelsApiStreamOptions<TApi>,
|
|
): Promise<AssistantMessage>;
|
|
|
|
streamSimple(model: Model<Api>, context: Context, options?: ModelsSimpleStreamOptions): AssistantMessageEventStream;
|
|
completeSimple(model: Model<Api>, context: Context, options?: ModelsSimpleStreamOptions): Promise<AssistantMessage>;
|
|
}
|
|
|
|
export interface MutableModels extends Models {
|
|
/** Upsert/replace by provider.id. Provider ids are unique. */
|
|
setProvider(provider: Provider): void;
|
|
deleteProvider(id: string): void;
|
|
clearProviders(): void;
|
|
}
|
|
|
|
export interface CreateModelsOptions {
|
|
credentials?: CredentialStore;
|
|
modelsStore?: ModelsStore;
|
|
authContext?: AuthContext;
|
|
}
|
|
|
|
function mergeHeaders(
|
|
base: ProviderHeaders | undefined,
|
|
override: ProviderHeaders | undefined,
|
|
): ProviderHeaders | undefined {
|
|
if (!base && !override) return undefined;
|
|
const merged = { ...base };
|
|
for (const [name, value] of Object.entries(override ?? {})) {
|
|
const lowerName = name.toLowerCase();
|
|
for (const existingName of Object.keys(merged)) {
|
|
if (existingName.toLowerCase() === lowerName) delete merged[existingName];
|
|
}
|
|
merged[name] = value;
|
|
}
|
|
return merged;
|
|
}
|
|
|
|
class ModelsImpl implements MutableModels {
|
|
private providers = new Map<string, Provider>();
|
|
private credentials: CredentialStore;
|
|
private modelsStore: ModelsStore;
|
|
private authContext: AuthContext;
|
|
|
|
constructor(options?: CreateModelsOptions) {
|
|
this.credentials = options?.credentials ?? new InMemoryCredentialStore();
|
|
this.modelsStore = options?.modelsStore ?? new InMemoryModelsStore();
|
|
this.authContext = options?.authContext ?? defaultAuthContext();
|
|
}
|
|
|
|
setProvider(provider: Provider): void {
|
|
this.providers.set(provider.id, provider);
|
|
}
|
|
|
|
deleteProvider(id: string): void {
|
|
this.providers.delete(id);
|
|
}
|
|
|
|
clearProviders(): void {
|
|
this.providers.clear();
|
|
}
|
|
|
|
getProviders(): readonly Provider[] {
|
|
return Array.from(this.providers.values());
|
|
}
|
|
|
|
getProvider(id: string): Provider | undefined {
|
|
return this.providers.get(id);
|
|
}
|
|
|
|
getModels(provider?: string): readonly Model<Api>[] {
|
|
if (provider !== undefined) {
|
|
const entry = this.providers.get(provider);
|
|
if (!entry) return [];
|
|
try {
|
|
return entry.getModels();
|
|
} catch {
|
|
return [];
|
|
}
|
|
}
|
|
|
|
const models: Model<Api>[] = [];
|
|
for (const entry of this.providers.values()) {
|
|
try {
|
|
models.push(...entry.getModels());
|
|
} catch {
|
|
// Best-effort: ill-behaved providers yield no models.
|
|
}
|
|
}
|
|
return models;
|
|
}
|
|
|
|
getModel(provider: string, id: string): Model<Api> | undefined {
|
|
return this.getModels(provider).find((model) => model.id === id);
|
|
}
|
|
|
|
async refresh(options: ModelsRefreshOptions = {}): Promise<ModelsRefreshResult> {
|
|
const allowNetwork = options.allowNetwork ?? true;
|
|
const errors = new Map<string, Error>();
|
|
const refreshable = Array.from(this.providers.values()).filter(
|
|
(provider): provider is Provider & Required<Pick<Provider, "refreshModels">> =>
|
|
provider.refreshModels !== undefined,
|
|
);
|
|
|
|
await Promise.all(
|
|
refreshable.map(async (provider) => {
|
|
if (options.signal?.aborted) return;
|
|
const store: ProviderModelsStore = {
|
|
read: () => this.modelsStore.read(provider.id),
|
|
write: (entry) => this.modelsStore.write(provider.id, entry),
|
|
delete: () => this.modelsStore.delete(provider.id),
|
|
};
|
|
let stored: Credential | undefined;
|
|
try {
|
|
stored = await this.readCredential(provider.id);
|
|
const credential = await this.resolveRefreshCredential(provider, stored, allowNetwork, options.signal);
|
|
if (!credential) return;
|
|
await provider.refreshModels({
|
|
credential,
|
|
store,
|
|
allowNetwork,
|
|
force: options.force,
|
|
signal: options.signal,
|
|
});
|
|
} catch (error) {
|
|
if (!options.signal?.aborted) {
|
|
errors.set(
|
|
provider.id,
|
|
error instanceof Error
|
|
? error
|
|
: new ModelsError("model_source", `Model refresh failed for ${provider.id}`, { cause: error }),
|
|
);
|
|
}
|
|
try {
|
|
await provider.refreshModels({
|
|
credential: stored,
|
|
store,
|
|
allowNetwork: false,
|
|
signal: options.signal,
|
|
});
|
|
} catch {
|
|
// Preserve the original auth/network error; cache restoration is best-effort here.
|
|
}
|
|
}
|
|
}),
|
|
);
|
|
|
|
return { aborted: options.signal?.aborted ?? false, errors };
|
|
}
|
|
|
|
private async resolveRefreshCredential(
|
|
provider: Provider,
|
|
stored: Credential | undefined,
|
|
allowNetwork: boolean,
|
|
signal?: AbortSignal,
|
|
): Promise<Credential | undefined> {
|
|
if (stored?.type === "oauth") {
|
|
const oauth = provider.auth.oauth;
|
|
if (!oauth) return undefined;
|
|
if (!allowNetwork || Date.now() < stored.expires) return stored;
|
|
if (signal?.aborted) return undefined;
|
|
const post = await this.credentials.modify(provider.id, async (current) => {
|
|
if (current?.type !== "oauth" || Date.now() < current.expires) return undefined;
|
|
return oauth.refresh(current, signal);
|
|
});
|
|
return post?.type === "oauth" ? post : undefined;
|
|
}
|
|
|
|
const apiKey = provider.auth.apiKey;
|
|
if (!apiKey) return undefined;
|
|
const credential = stored?.type === "api_key" ? stored : undefined;
|
|
const result = await apiKey.resolve({ ctx: this.authContext, credential });
|
|
if (!result) return undefined;
|
|
return { type: "api_key", key: result.auth.apiKey, env: result.env };
|
|
}
|
|
|
|
private async readCredential(providerId: string): Promise<Credential | undefined> {
|
|
try {
|
|
return await this.credentials.read(providerId);
|
|
} catch (error) {
|
|
throw new ModelsError("auth", `Credential store read failed for ${providerId}`, { cause: error });
|
|
}
|
|
}
|
|
|
|
private async checkProviderAuth(
|
|
provider: Provider,
|
|
credential: Credential | undefined,
|
|
): Promise<AuthCheck | undefined> {
|
|
if (credential?.type === "oauth") {
|
|
return provider.auth.oauth ? { source: "OAuth", type: "oauth" } : undefined;
|
|
}
|
|
const apiKey = provider.auth.apiKey;
|
|
if (!apiKey) return undefined;
|
|
if (apiKey.check) {
|
|
try {
|
|
return await apiKey.check({
|
|
ctx: this.authContext,
|
|
credential: credential?.type === "api_key" ? credential : undefined,
|
|
});
|
|
} catch (error) {
|
|
throw new ModelsError("auth", `API key auth check failed for provider ${provider.id}`, { cause: error });
|
|
}
|
|
}
|
|
|
|
const resolution = await resolveProviderAuth(provider, this.credentials, this.authContext);
|
|
return resolution ? { source: resolution.source, type: "api_key" } : undefined;
|
|
}
|
|
|
|
async checkAuth(providerId: string): Promise<AuthCheck | undefined> {
|
|
const provider = this.providers.get(providerId);
|
|
if (!provider) return undefined;
|
|
return this.checkProviderAuth(provider, await this.readCredential(providerId));
|
|
}
|
|
|
|
async getAvailable(providerId?: string): Promise<readonly Model<Api>[]> {
|
|
const providers = providerId
|
|
? [this.providers.get(providerId)].filter((entry) => entry !== undefined)
|
|
: this.getProviders();
|
|
const checks = await Promise.all(
|
|
providers.map(async (provider) => {
|
|
const credential = await this.readCredential(provider.id);
|
|
return { provider, credential, auth: await this.checkProviderAuth(provider, credential) };
|
|
}),
|
|
);
|
|
return checks.flatMap(({ provider, credential, auth }) => {
|
|
if (!auth) return [];
|
|
const models = provider.getModels();
|
|
return provider.filterModels?.(models, credential) ?? models;
|
|
});
|
|
}
|
|
|
|
getAuth(providerId: string, overrides?: AuthResolutionOverrides): Promise<AuthResult | undefined>;
|
|
getAuth(model: Model<Api>, overrides?: AuthResolutionOverrides): Promise<AuthResult | undefined>;
|
|
async getAuth(
|
|
providerOrModel: string | Model<Api>,
|
|
overrides?: AuthResolutionOverrides,
|
|
): Promise<AuthResult | undefined> {
|
|
const providerId = typeof providerOrModel === "string" ? providerOrModel : providerOrModel.provider;
|
|
const provider = this.providers.get(providerId);
|
|
if (!provider) return undefined;
|
|
const result = await resolveProviderAuth(provider, this.credentials, this.authContext, overrides);
|
|
if (!result || typeof providerOrModel === "string" || !providerOrModel.headers) return result;
|
|
return {
|
|
...result,
|
|
auth: {
|
|
...result.auth,
|
|
headers: mergeHeaders(result.auth.headers, providerOrModel.headers),
|
|
},
|
|
};
|
|
}
|
|
|
|
async login(providerId: string, type: AuthType, interaction: AuthInteraction): Promise<Credential> {
|
|
const provider = this.providers.get(providerId);
|
|
if (!provider) throw new ModelsError("provider", `Unknown provider: ${providerId}`);
|
|
const method = type === "oauth" ? provider.auth.oauth : provider.auth.apiKey;
|
|
if (!method?.login) {
|
|
throw new ModelsError("auth", `${provider.name} does not support ${type} login`);
|
|
}
|
|
const credential = await method.login(interaction);
|
|
try {
|
|
await this.credentials.modify(providerId, async () => credential);
|
|
} catch (error) {
|
|
throw new ModelsError("auth", `Credential store modify failed for ${providerId}`, { cause: error });
|
|
}
|
|
return credential;
|
|
}
|
|
|
|
async logout(providerId: string): Promise<void> {
|
|
try {
|
|
await this.credentials.delete(providerId);
|
|
} catch (error) {
|
|
throw new ModelsError("auth", `Credential store delete failed for ${providerId}`, { cause: error });
|
|
}
|
|
}
|
|
|
|
private requireProvider(model: Model<Api>): Provider {
|
|
const provider = this.providers.get(model.provider);
|
|
if (!provider) {
|
|
throw new ModelsError("provider", `Unknown provider: ${model.provider}`);
|
|
}
|
|
return provider;
|
|
}
|
|
|
|
private async applyAuth<TOptions extends StreamOptions & ModelsStreamTransforms>(
|
|
model: Model<Api>,
|
|
options: TOptions | undefined,
|
|
): Promise<{ requestModel: Model<Api>; requestOptions: StreamOptions | undefined }> {
|
|
this.requireProvider(model);
|
|
const resolution = await this.getAuth(model, {
|
|
apiKey: options?.apiKey,
|
|
env: options?.env,
|
|
});
|
|
if (!resolution) {
|
|
throw new ModelsError("auth", `Provider is not configured: ${model.provider}`);
|
|
}
|
|
const auth = resolution.auth;
|
|
|
|
// Explicit request options win per-field; the Models-only transform runs last.
|
|
const apiKey = options?.apiKey ?? auth.apiKey;
|
|
let headers = mergeHeaders(auth.headers, options?.headers);
|
|
if (options?.transformHeaders) headers = await options.transformHeaders(headers ?? {});
|
|
const env = resolution.env || options?.env ? { ...(resolution.env ?? {}), ...(options?.env ?? {}) } : undefined;
|
|
const requestModel = auth.baseUrl ? { ...model, baseUrl: auth.baseUrl } : model;
|
|
const { transformHeaders: _transformHeaders, ...providerOptions } = options ?? {};
|
|
const requestOptions = { ...providerOptions, apiKey, headers, env } as StreamOptions;
|
|
|
|
return { requestModel, requestOptions };
|
|
}
|
|
|
|
stream<TApi extends Api>(
|
|
model: Model<TApi>,
|
|
context: Context,
|
|
options?: ModelsApiStreamOptions<TApi>,
|
|
): AssistantMessageEventStream {
|
|
return lazyStream(model, async () => {
|
|
const provider = this.requireProvider(model);
|
|
const { requestModel, requestOptions } = await this.applyAuth(
|
|
model,
|
|
options as ModelsApiStreamOptions<Api> | undefined,
|
|
);
|
|
return provider.stream(requestModel as Model<TApi>, context, requestOptions as ApiStreamOptions<TApi>);
|
|
});
|
|
}
|
|
|
|
async complete<TApi extends Api>(
|
|
model: Model<TApi>,
|
|
context: Context,
|
|
options?: ModelsApiStreamOptions<TApi>,
|
|
): Promise<AssistantMessage> {
|
|
return this.stream(model, context, options).result();
|
|
}
|
|
|
|
streamSimple(model: Model<Api>, context: Context, options?: ModelsSimpleStreamOptions): AssistantMessageEventStream {
|
|
return lazyStream(model, async () => {
|
|
const provider = this.requireProvider(model);
|
|
const { requestModel, requestOptions } = await this.applyAuth(model, options);
|
|
return provider.streamSimple(requestModel, context, requestOptions as SimpleStreamOptions);
|
|
});
|
|
}
|
|
|
|
async completeSimple(
|
|
model: Model<Api>,
|
|
context: Context,
|
|
options?: ModelsSimpleStreamOptions,
|
|
): Promise<AssistantMessage> {
|
|
return this.streamSimple(model, context, options).result();
|
|
}
|
|
}
|
|
|
|
export function createModels(options?: CreateModelsOptions): MutableModels {
|
|
return new ModelsImpl(options);
|
|
}
|
|
|
|
export interface CreateProviderOptions<TApi extends Api = Api> {
|
|
id: string;
|
|
/** Display name. Default: `id`. */
|
|
name?: string;
|
|
baseUrl?: string;
|
|
headers?: ProviderHeaders;
|
|
/** Required — every provider has auth semantics, even ambient/keyless ones. */
|
|
auth: ProviderAuth;
|
|
/** Static baseline model list (empty for purely dynamic providers). */
|
|
models: readonly Model<TApi>[];
|
|
/** Fetch a dynamic model overlay. createProvider restores/persists it through ModelsStore. */
|
|
fetchModels?: (context: RefreshModelsContext) => Promise<readonly Model<TApi>[]>;
|
|
filterModels?: (models: readonly Model<TApi>[], credential: Credential | undefined) => readonly Model<TApi>[];
|
|
/** Single implementation, or map keyed by `model.api` for mixed-API providers. */
|
|
api: ProviderStreams | Partial<Record<TApi, ProviderStreams>>;
|
|
}
|
|
|
|
/**
|
|
* Builds a provider from parts. Built-in provider factories and models.json
|
|
* custom providers both go through this. A single `api` streams all models;
|
|
* an `api` map dispatches on `model.api`, and a model whose api has no entry
|
|
* produces a stream error.
|
|
*/
|
|
export function createProvider<TApi extends Api = Api>(input: CreateProviderOptions<TApi>): Provider<TApi> {
|
|
const baselineModels = input.models;
|
|
let dynamicModels: readonly Model<TApi>[] = [];
|
|
let inflightRefresh: Promise<void> | undefined;
|
|
const fetchModels = input.fetchModels;
|
|
const currentModels = (): readonly Model<TApi>[] => {
|
|
const merged = [...baselineModels];
|
|
for (const model of dynamicModels) {
|
|
const index = merged.findIndex((entry) => entry.id === model.id);
|
|
if (index >= 0) merged[index] = model;
|
|
else merged.push(model);
|
|
}
|
|
return merged;
|
|
};
|
|
const single =
|
|
typeof (input.api as ProviderStreams).stream === "function" ? (input.api as ProviderStreams) : undefined;
|
|
const byApi = single ? undefined : (input.api as Partial<Record<string, ProviderStreams>>);
|
|
|
|
const apiFor = (model: Model<Api>): ProviderStreams | undefined => single ?? byApi?.[model.api];
|
|
|
|
const dispatch = (
|
|
model: Model<Api>,
|
|
run: (streams: ProviderStreams) => AssistantMessageEventStream,
|
|
): AssistantMessageEventStream => {
|
|
const streams = apiFor(model);
|
|
if (!streams) {
|
|
return lazyStream(model, async () => {
|
|
throw new ModelsError("stream", `Provider ${input.id} has no API implementation for "${model.api}"`);
|
|
});
|
|
}
|
|
return run(streams);
|
|
};
|
|
|
|
return {
|
|
id: input.id,
|
|
name: input.name ?? input.id,
|
|
baseUrl: input.baseUrl,
|
|
headers: input.headers,
|
|
auth: input.auth,
|
|
getModels: currentModels,
|
|
refreshModels: fetchModels
|
|
? (context) => {
|
|
inflightRefresh ??= (async () => {
|
|
try {
|
|
const stored = await context.store.read();
|
|
if (stored) {
|
|
dynamicModels = stored.models
|
|
.filter((model) => model.provider === input.id)
|
|
.map((model) => model as Model<TApi>);
|
|
}
|
|
if (!context.allowNetwork || context.signal?.aborted) return;
|
|
const refreshed = await fetchModels(context);
|
|
if (context.signal?.aborted) return;
|
|
dynamicModels = refreshed;
|
|
await context.store.write({ models: refreshed, checkedAt: Date.now() });
|
|
} finally {
|
|
inflightRefresh = undefined;
|
|
}
|
|
})();
|
|
return inflightRefresh;
|
|
}
|
|
: undefined,
|
|
filterModels: input.filterModels,
|
|
stream: (model, context, options) => dispatch(model, (streams) => streams.stream(model, context, options)),
|
|
streamSimple: (model, context, options) =>
|
|
dispatch(model, (streams) => streams.streamSimple(model, context, options)),
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Runtime-checked narrowing for dynamically looked-up models:
|
|
*
|
|
* ```ts
|
|
* const model = models.getModel("anthropic", "claude-opus-4-7");
|
|
* if (model && hasApi(model, "anthropic-messages")) {
|
|
* // model: Model<"anthropic-messages">, stream options fully typed
|
|
* }
|
|
* ```
|
|
*/
|
|
export function hasApi<TApi extends Api>(model: Model<Api>, api: TApi): model is Model<TApi> {
|
|
return model.api === api;
|
|
}
|
|
|
|
export function calculateCost<TApi extends Api>(model: Model<TApi>, usage: Usage): Usage["cost"] {
|
|
const inputTokens = usage.input + usage.cacheRead + usage.cacheWrite;
|
|
let rates: ModelCostRates = model.cost;
|
|
let matchedThreshold = -1;
|
|
for (const tier of model.cost.tiers ?? []) {
|
|
if (inputTokens > tier.inputTokensAbove && tier.inputTokensAbove > matchedThreshold) {
|
|
rates = tier;
|
|
matchedThreshold = tier.inputTokensAbove;
|
|
}
|
|
}
|
|
|
|
// Anthropic charges 2x base input for 1h cache writes.
|
|
const longWrite = usage.cacheWrite1h ?? 0;
|
|
const shortWrite = usage.cacheWrite - longWrite;
|
|
usage.cost.input = (rates.input / 1000000) * usage.input;
|
|
usage.cost.output = (rates.output / 1000000) * usage.output;
|
|
usage.cost.cacheRead = (rates.cacheRead / 1000000) * usage.cacheRead;
|
|
usage.cost.cacheWrite = (rates.cacheWrite * shortWrite + rates.input * 2 * longWrite) / 1000000;
|
|
usage.cost.total = usage.cost.input + usage.cost.output + usage.cost.cacheRead + usage.cost.cacheWrite;
|
|
return usage.cost;
|
|
}
|
|
|
|
const EXTENDED_THINKING_LEVELS: ModelThinkingLevel[] = ["off", "minimal", "low", "medium", "high", "xhigh", "max"];
|
|
|
|
export function getSupportedThinkingLevels<TApi extends Api>(model: Model<TApi>): ModelThinkingLevel[] {
|
|
if (!model.reasoning) return ["off"];
|
|
|
|
return EXTENDED_THINKING_LEVELS.filter((level) => {
|
|
const mapped = model.thinkingLevelMap?.[level];
|
|
if (mapped === null) return false;
|
|
if (level === "xhigh" || level === "max") return mapped !== undefined;
|
|
return true;
|
|
});
|
|
}
|
|
|
|
export function clampThinkingLevel<TApi extends Api>(
|
|
model: Model<TApi>,
|
|
level: ModelThinkingLevel,
|
|
): ModelThinkingLevel {
|
|
const availableLevels = getSupportedThinkingLevels(model);
|
|
if (availableLevels.includes(level)) return level;
|
|
|
|
const requestedIndex = EXTENDED_THINKING_LEVELS.indexOf(level);
|
|
if (requestedIndex === -1) return availableLevels[0] ?? "off";
|
|
|
|
for (let i = requestedIndex; i < EXTENDED_THINKING_LEVELS.length; i++) {
|
|
const candidate = EXTENDED_THINKING_LEVELS[i];
|
|
if (availableLevels.includes(candidate)) return candidate;
|
|
}
|
|
for (let i = requestedIndex - 1; i >= 0; i--) {
|
|
const candidate = EXTENDED_THINKING_LEVELS[i];
|
|
if (availableLevels.includes(candidate)) return candidate;
|
|
}
|
|
return availableLevels[0] ?? "off";
|
|
}
|
|
|
|
/**
|
|
* Check if two models are equal by comparing both their id and provider.
|
|
* Returns false if either model is null or undefined.
|
|
*/
|
|
export function modelsAreEqual<TApi extends Api>(
|
|
a: Model<TApi> | null | undefined,
|
|
b: Model<TApi> | null | undefined,
|
|
): boolean {
|
|
if (!a || !b) return false;
|
|
return a.id === b.id && a.provider === b.provider;
|
|
}
|