feat(ai): ImagesModels collections mirroring the chat-side design
createImagesModels()/ImagesProvider/createImagesProvider() give image generation the same shape as chat: sync model reads, explicit async refresh(provider?) with in-flight dedupe, provider-resolved auth, and never-rejecting generateImages() (failures return AssistantImages with stopReason error). Auth resolution is shared with the chat side via the free-standing resolveProviderAuth() in auth/resolve.ts, which also owns ModelsError; both collections pass their store/context as arguments. The OpenRouter implementation moves to api/openrouter-images.ts with a lazy wrapper; openrouterImagesProvider() factory plus builtinImagesProviders()/builtinImagesModels() land in providers/all. The ImagesProvider id type alias is renamed to ImagesProviderId (mirror of Provider -> ProviderId). The old global image API (getImageModel*, generateImages, registerImagesApiProvider) stays on /compat, its registration shim repointed at the moved implementation. README: Quick Start uses builtinModels(), the full streaming event switch, image generation and the development checklist are restored in full, image generation documents the new collections with compat noted for the old API, plus the review fixes (builtinModels options, credential-store mention for browsers, ImagesModels notes).
This commit is contained in:
Mario Zechner
@@ -1,3 +1,4 @@
|
||||
import { createImagesModels, type ImagesProvider, type MutableImagesModels } from "../images-models.ts";
|
||||
import { MODELS } from "../models.generated.ts";
|
||||
import { type CreateModelsOptions, createModels, type MutableModels, type Provider } from "../models.ts";
|
||||
import type { Api, KnownProvider, Model } from "../types.ts";
|
||||
@@ -27,6 +28,7 @@ import { openaiCodexProvider } from "./openai-codex.ts";
|
||||
import { opencodeProvider } from "./opencode.ts";
|
||||
import { opencodeGoProvider } from "./opencode-go.ts";
|
||||
import { openrouterProvider } from "./openrouter.ts";
|
||||
import { openrouterImagesProvider } from "./openrouter-images.ts";
|
||||
import { togetherProvider } from "./together.ts";
|
||||
import { vercelAIGatewayProvider } from "./vercel-ai-gateway.ts";
|
||||
import { xaiProvider } from "./xai.ts";
|
||||
@@ -113,3 +115,17 @@ export function builtinModels(options?: CreateModelsOptions): MutableModels {
|
||||
}
|
||||
return models;
|
||||
}
|
||||
|
||||
/** All built-in image-generation providers, freshly constructed. */
|
||||
export function builtinImagesProviders(): ImagesProvider[] {
|
||||
return [openrouterImagesProvider()];
|
||||
}
|
||||
|
||||
/** An `ImagesModels` collection with every built-in image-generation provider registered. */
|
||||
export function builtinImagesModels(options?: CreateModelsOptions): MutableImagesModels {
|
||||
const models = createImagesModels(options);
|
||||
for (const provider of builtinImagesProviders()) {
|
||||
models.setProvider(provider);
|
||||
}
|
||||
return models;
|
||||
}
|
||||
|
||||
@@ -1,186 +0,0 @@
|
||||
import OpenAI from "openai";
|
||||
import type {
|
||||
ChatCompletion,
|
||||
ChatCompletionContentPart,
|
||||
ChatCompletionContentPartImage,
|
||||
ChatCompletionContentPartText,
|
||||
ChatCompletionCreateParamsNonStreaming,
|
||||
} from "openai/resources/chat/completions.js";
|
||||
import type {
|
||||
AssistantImages,
|
||||
ImageContent,
|
||||
ImagesContext,
|
||||
ImagesFunction,
|
||||
ImagesModel,
|
||||
ImagesOptions,
|
||||
TextContent,
|
||||
} from "../../types.ts";
|
||||
import { headersToRecord } from "../../utils/headers.ts";
|
||||
import { sanitizeSurrogates } from "../../utils/sanitize-unicode.ts";
|
||||
|
||||
interface OpenRouterGeneratedImage {
|
||||
image_url?: string | { url?: string };
|
||||
}
|
||||
|
||||
type OpenRouterImageGenerationMessage = ChatCompletion["choices"][number]["message"] & {
|
||||
images?: OpenRouterGeneratedImage[];
|
||||
};
|
||||
|
||||
type OpenRouterImageGenerationChoice = ChatCompletion["choices"][number] & {
|
||||
message: OpenRouterImageGenerationMessage;
|
||||
};
|
||||
|
||||
type OpenRouterImageGenerationResponse = ChatCompletion & {
|
||||
choices: OpenRouterImageGenerationChoice[];
|
||||
};
|
||||
|
||||
export const generateImagesOpenRouter: ImagesFunction<"openrouter-images", ImagesOptions> = async (
|
||||
model: ImagesModel<"openrouter-images">,
|
||||
context: ImagesContext,
|
||||
options?: ImagesOptions,
|
||||
) => {
|
||||
const output: AssistantImages = {
|
||||
api: model.api,
|
||||
provider: model.provider,
|
||||
model: model.id,
|
||||
output: [],
|
||||
stopReason: "stop",
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
|
||||
try {
|
||||
const apiKey = options?.apiKey;
|
||||
if (!apiKey) {
|
||||
throw new Error(`No API key for provider: ${model.provider}`);
|
||||
}
|
||||
const client = createClient(model, apiKey, options?.headers);
|
||||
let params = buildParams(model, context);
|
||||
const nextParams = await options?.onPayload?.(params, model);
|
||||
if (nextParams !== undefined) {
|
||||
params = nextParams as typeof params;
|
||||
}
|
||||
const requestOptions = {
|
||||
...(options?.signal ? { signal: options.signal } : {}),
|
||||
...(options?.timeoutMs !== undefined ? { timeout: options.timeoutMs } : {}),
|
||||
maxRetries: options?.maxRetries ?? 0,
|
||||
};
|
||||
const { data: response, response: rawResponse } = await client.chat.completions
|
||||
.create(params as unknown as ChatCompletionCreateParamsNonStreaming, requestOptions)
|
||||
.withResponse();
|
||||
await options?.onResponse?.({ status: rawResponse.status, headers: headersToRecord(rawResponse.headers) }, model);
|
||||
|
||||
const imageResponse = response as OpenRouterImageGenerationResponse;
|
||||
output.responseId = imageResponse.id;
|
||||
if (imageResponse.usage) {
|
||||
output.usage = parseUsage(imageResponse.usage, model);
|
||||
}
|
||||
|
||||
const choice = imageResponse.choices[0];
|
||||
if (choice) {
|
||||
const content = choice.message.content;
|
||||
if (typeof content === "string" && content.length > 0) {
|
||||
output.output.push({ type: "text", text: content } satisfies TextContent);
|
||||
}
|
||||
|
||||
for (const image of choice.message.images ?? []) {
|
||||
const imageUrl = typeof image.image_url === "string" ? image.image_url : image.image_url?.url;
|
||||
if (!imageUrl?.startsWith("data:")) continue;
|
||||
const matches = imageUrl.match(/^data:([^;]+);base64,(.+)$/);
|
||||
if (!matches) continue;
|
||||
output.output.push({
|
||||
type: "image",
|
||||
mimeType: matches[1],
|
||||
data: matches[2],
|
||||
} satisfies ImageContent);
|
||||
}
|
||||
}
|
||||
|
||||
return output;
|
||||
} catch (error) {
|
||||
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
|
||||
output.errorMessage = error instanceof Error ? error.message : JSON.stringify(error);
|
||||
return output;
|
||||
}
|
||||
};
|
||||
|
||||
function createClient(
|
||||
model: ImagesModel<"openrouter-images">,
|
||||
apiKey: string,
|
||||
optionsHeaders?: Record<string, string>,
|
||||
): OpenAI {
|
||||
return new OpenAI({
|
||||
apiKey,
|
||||
baseURL: model.baseUrl,
|
||||
dangerouslyAllowBrowser: true,
|
||||
defaultHeaders: {
|
||||
...model.headers,
|
||||
...optionsHeaders,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
type OpenRouterImagesCreateParams = Omit<ChatCompletionCreateParamsNonStreaming, "modalities"> & {
|
||||
modalities: Array<"image" | "text">;
|
||||
};
|
||||
|
||||
function buildParams(model: ImagesModel<"openrouter-images">, context: ImagesContext): OpenRouterImagesCreateParams {
|
||||
const content: ChatCompletionContentPart[] = context.input.map((item): ChatCompletionContentPart => {
|
||||
if (item.type === "text") {
|
||||
return {
|
||||
type: "text",
|
||||
text: sanitizeSurrogates(item.text),
|
||||
} satisfies ChatCompletionContentPartText;
|
||||
}
|
||||
return {
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: `data:${item.mimeType};base64,${item.data}`,
|
||||
},
|
||||
} satisfies ChatCompletionContentPartImage;
|
||||
});
|
||||
|
||||
return {
|
||||
model: model.id,
|
||||
messages: [
|
||||
{
|
||||
role: "user" as const,
|
||||
content,
|
||||
},
|
||||
],
|
||||
stream: false,
|
||||
modalities: model.output.includes("text") ? ["image", "text"] : ["image"],
|
||||
};
|
||||
}
|
||||
|
||||
function parseUsage(
|
||||
rawUsage: {
|
||||
prompt_tokens?: number;
|
||||
completion_tokens?: number;
|
||||
prompt_tokens_details?: { cached_tokens?: number; cache_write_tokens?: number };
|
||||
},
|
||||
model: ImagesModel<"openrouter-images">,
|
||||
) {
|
||||
const promptTokens = rawUsage.prompt_tokens || 0;
|
||||
const reportedCachedTokens = rawUsage.prompt_tokens_details?.cached_tokens || 0;
|
||||
const cacheWriteTokens = rawUsage.prompt_tokens_details?.cache_write_tokens || 0;
|
||||
const cacheReadTokens =
|
||||
cacheWriteTokens > 0 ? Math.max(0, reportedCachedTokens - cacheWriteTokens) : reportedCachedTokens;
|
||||
const input = Math.max(0, promptTokens - cacheReadTokens - cacheWriteTokens);
|
||||
const output = rawUsage.completion_tokens || 0;
|
||||
const usage = {
|
||||
input,
|
||||
output,
|
||||
cacheRead: cacheReadTokens,
|
||||
cacheWrite: cacheWriteTokens,
|
||||
totalTokens: input + output + cacheReadTokens + cacheWriteTokens,
|
||||
cost: {
|
||||
input: (model.cost.input / 1000000) * input,
|
||||
output: (model.cost.output / 1000000) * output,
|
||||
cacheRead: (model.cost.cacheRead / 1000000) * cacheReadTokens,
|
||||
cacheWrite: (model.cost.cacheWrite / 1000000) * cacheWriteTokens,
|
||||
total: 0,
|
||||
},
|
||||
};
|
||||
usage.cost.total = usage.cost.input + usage.cost.output + usage.cost.cacheRead + usage.cost.cacheWrite;
|
||||
return usage;
|
||||
}
|
||||
@@ -1,9 +1,9 @@
|
||||
import type { generateImages as generateImagesOpenRouterFunction } from "../../api/openrouter-images.ts";
|
||||
import { registerImagesApiProvider } from "../../images-api-registry.ts";
|
||||
import type { AssistantImages, ImagesContext, ImagesFunction, ImagesModel, ImagesOptions } from "../../types.ts";
|
||||
import type { generateImagesOpenRouter as generateImagesOpenRouterFunction } from "./openrouter.ts";
|
||||
|
||||
interface OpenRouterImagesProviderModule {
|
||||
generateImagesOpenRouter: typeof generateImagesOpenRouterFunction;
|
||||
generateImages: typeof generateImagesOpenRouterFunction;
|
||||
}
|
||||
|
||||
let openRouterImagesProviderModulePromise: Promise<OpenRouterImagesProviderModule> | undefined;
|
||||
@@ -21,7 +21,7 @@ function createLazyLoadErrorImages(model: ImagesModel<"openrouter-images">, erro
|
||||
}
|
||||
|
||||
function loadOpenRouterImagesProviderModule(): Promise<OpenRouterImagesProviderModule> {
|
||||
openRouterImagesProviderModulePromise ||= import("./openrouter.ts").then(
|
||||
openRouterImagesProviderModulePromise ||= import("../../api/openrouter-images.ts").then(
|
||||
(module) => module as OpenRouterImagesProviderModule,
|
||||
);
|
||||
return openRouterImagesProviderModulePromise;
|
||||
@@ -34,7 +34,7 @@ export const generateImagesOpenRouter: ImagesFunction<"openrouter-images", Image
|
||||
) => {
|
||||
try {
|
||||
const module = await loadOpenRouterImagesProviderModule();
|
||||
return await module.generateImagesOpenRouter(model, context, options);
|
||||
return await module.generateImages(model, context, options);
|
||||
} catch (error) {
|
||||
return createLazyLoadErrorImages(model, error);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
import { openrouterImagesApi } from "../api/openrouter-images.lazy.ts";
|
||||
import { envApiKeyAuth } from "../auth/helpers.ts";
|
||||
import { IMAGE_MODELS } from "../image-models.generated.ts";
|
||||
import { createImagesProvider, type ImagesProvider } from "../images-models.ts";
|
||||
|
||||
export function openrouterImagesProvider(): ImagesProvider {
|
||||
return createImagesProvider({
|
||||
id: "openrouter",
|
||||
name: "OpenRouter",
|
||||
auth: { apiKey: envApiKeyAuth("OpenRouter API key", ["OPENROUTER_API_KEY"]) },
|
||||
models: Object.values(IMAGE_MODELS.openrouter),
|
||||
api: openrouterImagesApi(),
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user