feat(ai): ImagesModels collections mirroring the chat-side design
createImagesModels()/ImagesProvider/createImagesProvider() give image generation the same shape as chat: sync model reads, explicit async refresh(provider?) with in-flight dedupe, provider-resolved auth, and never-rejecting generateImages() (failures return AssistantImages with stopReason error). Auth resolution is shared with the chat side via the free-standing resolveProviderAuth() in auth/resolve.ts, which also owns ModelsError; both collections pass their store/context as arguments. The OpenRouter implementation moves to api/openrouter-images.ts with a lazy wrapper; openrouterImagesProvider() factory plus builtinImagesProviders()/builtinImagesModels() land in providers/all. The ImagesProvider id type alias is renamed to ImagesProviderId (mirror of Provider -> ProviderId). The old global image API (getImageModel*, generateImages, registerImagesApiProvider) stays on /compat, its registration shim repointed at the moved implementation. README: Quick Start uses builtinModels(), the full streaming event switch, image generation and the development checklist are restored in full, image generation documents the new collections with compat noted for the old API, plus the review fixes (builtinModels options, credential-store mention for browsers, ImagesModels notes).
This commit is contained in:
Mario Zechner
@@ -0,0 +1,10 @@
|
||||
import type { ImagesModel, ProviderImages } from "../types.ts";
|
||||
|
||||
export const openrouterImagesApi = (): ProviderImages => ({
|
||||
generateImages: async (model, context, options) =>
|
||||
(await import("./openrouter-images.ts")).generateImages(
|
||||
model as ImagesModel<"openrouter-images">,
|
||||
context,
|
||||
options,
|
||||
),
|
||||
});
|
||||
@@ -0,0 +1,186 @@
|
||||
import OpenAI from "openai";
|
||||
import type {
|
||||
ChatCompletion,
|
||||
ChatCompletionContentPart,
|
||||
ChatCompletionContentPartImage,
|
||||
ChatCompletionContentPartText,
|
||||
ChatCompletionCreateParamsNonStreaming,
|
||||
} from "openai/resources/chat/completions.js";
|
||||
import type {
|
||||
AssistantImages,
|
||||
ImageContent,
|
||||
ImagesContext,
|
||||
ImagesFunction,
|
||||
ImagesModel,
|
||||
ImagesOptions,
|
||||
TextContent,
|
||||
} from "../types.ts";
|
||||
import { headersToRecord } from "../utils/headers.ts";
|
||||
import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts";
|
||||
|
||||
interface OpenRouterGeneratedImage {
|
||||
image_url?: string | { url?: string };
|
||||
}
|
||||
|
||||
type OpenRouterImageGenerationMessage = ChatCompletion["choices"][number]["message"] & {
|
||||
images?: OpenRouterGeneratedImage[];
|
||||
};
|
||||
|
||||
type OpenRouterImageGenerationChoice = ChatCompletion["choices"][number] & {
|
||||
message: OpenRouterImageGenerationMessage;
|
||||
};
|
||||
|
||||
type OpenRouterImageGenerationResponse = ChatCompletion & {
|
||||
choices: OpenRouterImageGenerationChoice[];
|
||||
};
|
||||
|
||||
export const generateImages: ImagesFunction<"openrouter-images", ImagesOptions> = async (
|
||||
model: ImagesModel<"openrouter-images">,
|
||||
context: ImagesContext,
|
||||
options?: ImagesOptions,
|
||||
) => {
|
||||
const output: AssistantImages = {
|
||||
api: model.api,
|
||||
provider: model.provider,
|
||||
model: model.id,
|
||||
output: [],
|
||||
stopReason: "stop",
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
|
||||
try {
|
||||
const apiKey = options?.apiKey;
|
||||
if (!apiKey) {
|
||||
throw new Error(`No API key for provider: ${model.provider}`);
|
||||
}
|
||||
const client = createClient(model, apiKey, options?.headers);
|
||||
let params = buildParams(model, context);
|
||||
const nextParams = await options?.onPayload?.(params, model);
|
||||
if (nextParams !== undefined) {
|
||||
params = nextParams as typeof params;
|
||||
}
|
||||
const requestOptions = {
|
||||
...(options?.signal ? { signal: options.signal } : {}),
|
||||
...(options?.timeoutMs !== undefined ? { timeout: options.timeoutMs } : {}),
|
||||
maxRetries: options?.maxRetries ?? 0,
|
||||
};
|
||||
const { data: response, response: rawResponse } = await client.chat.completions
|
||||
.create(params as unknown as ChatCompletionCreateParamsNonStreaming, requestOptions)
|
||||
.withResponse();
|
||||
await options?.onResponse?.({ status: rawResponse.status, headers: headersToRecord(rawResponse.headers) }, model);
|
||||
|
||||
const imageResponse = response as OpenRouterImageGenerationResponse;
|
||||
output.responseId = imageResponse.id;
|
||||
if (imageResponse.usage) {
|
||||
output.usage = parseUsage(imageResponse.usage, model);
|
||||
}
|
||||
|
||||
const choice = imageResponse.choices[0];
|
||||
if (choice) {
|
||||
const content = choice.message.content;
|
||||
if (typeof content === "string" && content.length > 0) {
|
||||
output.output.push({ type: "text", text: content } satisfies TextContent);
|
||||
}
|
||||
|
||||
for (const image of choice.message.images ?? []) {
|
||||
const imageUrl = typeof image.image_url === "string" ? image.image_url : image.image_url?.url;
|
||||
if (!imageUrl?.startsWith("data:")) continue;
|
||||
const matches = imageUrl.match(/^data:([^;]+);base64,(.+)$/);
|
||||
if (!matches) continue;
|
||||
output.output.push({
|
||||
type: "image",
|
||||
mimeType: matches[1],
|
||||
data: matches[2],
|
||||
} satisfies ImageContent);
|
||||
}
|
||||
}
|
||||
|
||||
return output;
|
||||
} catch (error) {
|
||||
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
|
||||
output.errorMessage = error instanceof Error ? error.message : JSON.stringify(error);
|
||||
return output;
|
||||
}
|
||||
};
|
||||
|
||||
function createClient(
|
||||
model: ImagesModel<"openrouter-images">,
|
||||
apiKey: string,
|
||||
optionsHeaders?: Record<string, string>,
|
||||
): OpenAI {
|
||||
return new OpenAI({
|
||||
apiKey,
|
||||
baseURL: model.baseUrl,
|
||||
dangerouslyAllowBrowser: true,
|
||||
defaultHeaders: {
|
||||
...model.headers,
|
||||
...optionsHeaders,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
type OpenRouterImagesCreateParams = Omit<ChatCompletionCreateParamsNonStreaming, "modalities"> & {
|
||||
modalities: Array<"image" | "text">;
|
||||
};
|
||||
|
||||
function buildParams(model: ImagesModel<"openrouter-images">, context: ImagesContext): OpenRouterImagesCreateParams {
|
||||
const content: ChatCompletionContentPart[] = context.input.map((item): ChatCompletionContentPart => {
|
||||
if (item.type === "text") {
|
||||
return {
|
||||
type: "text",
|
||||
text: sanitizeSurrogates(item.text),
|
||||
} satisfies ChatCompletionContentPartText;
|
||||
}
|
||||
return {
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: `data:${item.mimeType};base64,${item.data}`,
|
||||
},
|
||||
} satisfies ChatCompletionContentPartImage;
|
||||
});
|
||||
|
||||
return {
|
||||
model: model.id,
|
||||
messages: [
|
||||
{
|
||||
role: "user" as const,
|
||||
content,
|
||||
},
|
||||
],
|
||||
stream: false,
|
||||
modalities: model.output.includes("text") ? ["image", "text"] : ["image"],
|
||||
};
|
||||
}
|
||||
|
||||
function parseUsage(
|
||||
rawUsage: {
|
||||
prompt_tokens?: number;
|
||||
completion_tokens?: number;
|
||||
prompt_tokens_details?: { cached_tokens?: number; cache_write_tokens?: number };
|
||||
},
|
||||
model: ImagesModel<"openrouter-images">,
|
||||
) {
|
||||
const promptTokens = rawUsage.prompt_tokens || 0;
|
||||
const reportedCachedTokens = rawUsage.prompt_tokens_details?.cached_tokens || 0;
|
||||
const cacheWriteTokens = rawUsage.prompt_tokens_details?.cache_write_tokens || 0;
|
||||
const cacheReadTokens =
|
||||
cacheWriteTokens > 0 ? Math.max(0, reportedCachedTokens - cacheWriteTokens) : reportedCachedTokens;
|
||||
const input = Math.max(0, promptTokens - cacheReadTokens - cacheWriteTokens);
|
||||
const output = rawUsage.completion_tokens || 0;
|
||||
const usage = {
|
||||
input,
|
||||
output,
|
||||
cacheRead: cacheReadTokens,
|
||||
cacheWrite: cacheWriteTokens,
|
||||
totalTokens: input + output + cacheReadTokens + cacheWriteTokens,
|
||||
cost: {
|
||||
input: (model.cost.input / 1000000) * input,
|
||||
output: (model.cost.output / 1000000) * output,
|
||||
cacheRead: (model.cost.cacheRead / 1000000) * cacheReadTokens,
|
||||
cacheWrite: (model.cost.cacheWrite / 1000000) * cacheWriteTokens,
|
||||
total: 0,
|
||||
},
|
||||
};
|
||||
usage.cost.total = usage.cost.input + usage.cost.output + usage.cost.cacheRead + usage.cost.cacheWrite;
|
||||
return usage;
|
||||
}
|
||||
Reference in New Issue
Block a user