feat(ai): add Models runtime with provider-owned auth (phase 1)
New Models/MutableModels/createModels collection: provider map, async
model listing (best-effort aggregation), getAuth decision tree with
double-checked locked OAuth refresh, stream/complete with per-field
auth merge over lazyStream.
Auth substrate: ProviderAuth { apiKey?, oauth? }, one type-tagged
credential per provider, CredentialStore (read/modify/delete; modify
is the only write path, serialized RMW), OAuthAuth login/refresh/toAuth
split, prompt()/notify() login callbacks, browser-safe default
AuthContext.
types.ts: Provider alias renamed to ProviderId; ApiOptionsMap and
ApiStreamOptions<TApi> for typed per-API stream options; hasApi()
runtime narrowing guard.
This commit is contained in:
Mario Zechner
@@ -0,0 +1,399 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { InMemoryCredentialStore } from "../src/auth/credential-store.ts";
|
||||
import type { ApiKeyAuth, CredentialStore, OAuthAuth, ProviderAuth } from "../src/auth/types.ts";
|
||||
import { createModels, hasApi, type Provider } from "../src/models.ts";
|
||||
import type { Api, AssistantMessage, Context, Model, SimpleStreamOptions, StreamOptions } from "../src/types.ts";
|
||||
import { AssistantMessageEventStream } from "../src/utils/event-stream.ts";
|
||||
|
||||
function testModel(provider: string, id: string): Model<Api> {
|
||||
return {
|
||||
id,
|
||||
name: id,
|
||||
api: "test-api",
|
||||
provider,
|
||||
baseUrl: "https://example.test/v1",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 10000,
|
||||
maxTokens: 1000,
|
||||
};
|
||||
}
|
||||
|
||||
function doneMessage(model: Model<Api>, text: string): AssistantMessage {
|
||||
return {
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text }],
|
||||
api: model.api,
|
||||
provider: model.provider,
|
||||
model: model.id,
|
||||
usage: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 0,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
},
|
||||
stopReason: "stop",
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
interface ProviderCall {
|
||||
model: Model<Api>;
|
||||
options: StreamOptions | undefined;
|
||||
}
|
||||
|
||||
/** Ambient auth for keyless test providers; reports "configured" with no auth values. */
|
||||
const ambientAuth: ApiKeyAuth = {
|
||||
name: "Ambient",
|
||||
resolve: async () => ({ auth: {} }),
|
||||
};
|
||||
|
||||
function testProvider(input: {
|
||||
id: string;
|
||||
models?: Model<Api>[];
|
||||
auth?: ProviderAuth;
|
||||
getModels?: () => Promise<readonly Model<Api>[]>;
|
||||
calls?: ProviderCall[];
|
||||
}): Provider {
|
||||
const models = input.models ?? [testModel(input.id, "model-a")];
|
||||
const respond = (model: Model<Api>, options: StreamOptions | undefined) => {
|
||||
input.calls?.push({ model, options });
|
||||
const stream = new AssistantMessageEventStream();
|
||||
const message = doneMessage(model, "ok");
|
||||
stream.push({ type: "start", partial: message });
|
||||
stream.push({ type: "done", reason: "stop", message });
|
||||
stream.end(message);
|
||||
return stream;
|
||||
};
|
||||
return {
|
||||
id: input.id,
|
||||
name: input.id,
|
||||
auth: input.auth ?? { apiKey: ambientAuth },
|
||||
getModels: input.getModels ?? (async () => models),
|
||||
stream: (model, _context, options) => respond(model, options as StreamOptions | undefined),
|
||||
streamSimple: (model, _context, options) => respond(model, options as SimpleStreamOptions | undefined),
|
||||
};
|
||||
}
|
||||
|
||||
const context: Context = { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] };
|
||||
|
||||
function envKeyAuth(key: string | undefined): ApiKeyAuth {
|
||||
return {
|
||||
name: "Test API key",
|
||||
resolve: async ({ credential }) => {
|
||||
const resolved = credential?.key ?? key;
|
||||
if (!resolved) return undefined;
|
||||
return { auth: { apiKey: resolved }, source: credential ? "stored" : "env" };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function testOAuth(overrides?: Partial<OAuthAuth>): OAuthAuth {
|
||||
return {
|
||||
name: "Test OAuth",
|
||||
login: async () => {
|
||||
throw new Error("not used");
|
||||
},
|
||||
refresh: async (credential) => credential,
|
||||
toAuth: async (credential) => ({ apiKey: credential.access }),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("Models runtime", () => {
|
||||
it("registers, replaces, and deletes providers", () => {
|
||||
const models = createModels();
|
||||
models.setProvider(testProvider({ id: "p1" }));
|
||||
models.setProvider(testProvider({ id: "p2" }));
|
||||
expect(models.getProviders().map((p) => p.id)).toEqual(["p1", "p2"]);
|
||||
|
||||
const replacement = testProvider({ id: "p1" });
|
||||
models.setProvider(replacement);
|
||||
expect(models.getProvider("p1")).toBe(replacement);
|
||||
expect(models.getProviders()).toHaveLength(2);
|
||||
|
||||
models.deleteProvider("p1");
|
||||
expect(models.getProvider("p1")).toBeUndefined();
|
||||
|
||||
models.clearProviders();
|
||||
expect(models.getProviders()).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("lists and finds models per provider", async () => {
|
||||
const models = createModels();
|
||||
models.setProvider(testProvider({ id: "p1", models: [testModel("p1", "m1"), testModel("p1", "m2")] }));
|
||||
models.setProvider(testProvider({ id: "p2", models: [testModel("p2", "m3")] }));
|
||||
|
||||
expect((await models.getModels()).map((m) => m.id)).toEqual(["m1", "m2", "m3"]);
|
||||
expect((await models.getModels("p1")).map((m) => m.id)).toEqual(["m1", "m2"]);
|
||||
expect((await models.getModels("nope")).length).toBe(0);
|
||||
expect((await models.getModel("p2", "m3"))?.id).toBe("m3");
|
||||
expect(await models.getModel("p2", "missing")).toBeUndefined();
|
||||
|
||||
// hasApi() narrows dynamically looked-up models with a runtime check
|
||||
const found = await models.getModel("p2", "m3");
|
||||
expect(found && hasApi(found, "openai-completions")).toBe(false);
|
||||
expect(found && hasApi(found, "test-api")).toBe(true);
|
||||
if (found && hasApi(found, "test-api")) {
|
||||
const _typed: Model<"test-api"> = found;
|
||||
expect(_typed.id).toBe("m3");
|
||||
}
|
||||
});
|
||||
|
||||
it("swallows provider source failures for both all-provider and single-provider listing", async () => {
|
||||
const models = createModels();
|
||||
models.setProvider(
|
||||
testProvider({
|
||||
id: "broken",
|
||||
getModels: async () => {
|
||||
throw new Error("boom");
|
||||
},
|
||||
}),
|
||||
);
|
||||
models.setProvider(testProvider({ id: "ok", models: [testModel("ok", "m1")] }));
|
||||
|
||||
expect((await models.getModels()).map((m) => m.id)).toEqual(["m1"]);
|
||||
expect(await models.getModels("broken")).toEqual([]);
|
||||
// precise failures come from the provider directly
|
||||
await expect(models.getProvider("broken")?.getModels()).rejects.toThrow("boom");
|
||||
|
||||
// even sync-throwing (non-async) provider implementations are isolated
|
||||
models.setProvider({
|
||||
...testProvider({ id: "sync-broken" }),
|
||||
getModels: () => {
|
||||
throw new Error("sync boom");
|
||||
},
|
||||
});
|
||||
expect((await models.getModels()).map((m) => m.id)).toEqual(["m1"]);
|
||||
});
|
||||
|
||||
it("supports getModels(options) without a provider id", async () => {
|
||||
const seen: ({ forceRefresh?: boolean } | undefined)[] = [];
|
||||
const models = createModels();
|
||||
models.setProvider(testProvider({ id: "p1", models: [testModel("p1", "m1")] }));
|
||||
models.setProvider({
|
||||
...testProvider({ id: "p2" }),
|
||||
getModels: async (options) => {
|
||||
seen.push(options);
|
||||
return [testModel("p2", "m2")];
|
||||
},
|
||||
});
|
||||
|
||||
const all = await models.getModels({ forceRefresh: true });
|
||||
expect(all.map((m) => m.id)).toEqual(["m1", "m2"]);
|
||||
expect(seen).toEqual([{ forceRefresh: true }]);
|
||||
});
|
||||
|
||||
it("resolves auth: stored credential owns the provider, ambient only when nothing stored", async () => {
|
||||
const credentials = new InMemoryCredentialStore();
|
||||
const models = createModels({ credentials });
|
||||
models.setProvider(testProvider({ id: "p1", auth: { apiKey: envKeyAuth("env-key"), oauth: testOAuth() } }));
|
||||
const model = testModel("p1", "model-a");
|
||||
|
||||
// nothing stored: ambient env resolves
|
||||
expect((await models.getAuth(model))?.auth.apiKey).toBe("env-key");
|
||||
|
||||
// stored oauth credential (persisted via the single write path): beats ambient env
|
||||
await credentials.modify("p1", async () => ({
|
||||
type: "oauth",
|
||||
access: "oauth-token",
|
||||
refresh: "r",
|
||||
expires: Date.now() + 100000,
|
||||
}));
|
||||
const resolution = await models.getAuth(model);
|
||||
expect(resolution?.auth.apiKey).toBe("oauth-token");
|
||||
expect(resolution?.source).toBe("OAuth");
|
||||
|
||||
// stored api-key credential resolves through apiKey auth, beats env
|
||||
await credentials.modify("p1", async () => ({ type: "api-key", key: "stored-key" }));
|
||||
const apiKeyResolution = await models.getAuth(model);
|
||||
expect(apiKeyResolution?.auth.apiKey).toBe("stored-key");
|
||||
expect(apiKeyResolution?.source).toBe("stored");
|
||||
});
|
||||
|
||||
it("a stored credential without a matching handler blocks ambient fallback", async () => {
|
||||
const credentials = new InMemoryCredentialStore();
|
||||
const models = createModels({ credentials });
|
||||
// provider has only apiKey auth, but an oauth credential is stored (stale config)
|
||||
models.setProvider(testProvider({ id: "p1", auth: { apiKey: envKeyAuth("env-key") } }));
|
||||
await credentials.modify("p1", async () => ({ type: "oauth", access: "a", refresh: "r", expires: 0 }));
|
||||
|
||||
expect(await models.getAuth(testModel("p1", "model-a"))).toBeUndefined();
|
||||
});
|
||||
|
||||
it("refreshes expired oauth credentials and persists the rotated credential", async () => {
|
||||
const credentials = new InMemoryCredentialStore();
|
||||
const oauth = testOAuth({
|
||||
refresh: async (credential) => ({ ...credential, access: "new-token", expires: Date.now() + 60_000 }),
|
||||
});
|
||||
const models = createModels({ credentials });
|
||||
models.setProvider(testProvider({ id: "p1", auth: { oauth } }));
|
||||
await credentials.modify("p1", async () => ({
|
||||
type: "oauth",
|
||||
access: "old-token",
|
||||
refresh: "r",
|
||||
expires: 0,
|
||||
}));
|
||||
|
||||
const resolution = await models.getAuth(testModel("p1", "model-a"));
|
||||
expect(resolution?.auth.apiKey).toBe("new-token");
|
||||
expect(((await credentials.read("p1")) as { access: string }).access).toBe("new-token");
|
||||
});
|
||||
|
||||
it("rejects with code oauth when refresh fails, preserving the stored credential", async () => {
|
||||
const credentials = new InMemoryCredentialStore();
|
||||
const oauth = testOAuth({
|
||||
refresh: async () => {
|
||||
throw new Error("invalid_grant");
|
||||
},
|
||||
});
|
||||
const models = createModels({ credentials });
|
||||
models.setProvider(testProvider({ id: "p1", auth: { oauth } }));
|
||||
await credentials.modify("p1", async () => ({ type: "oauth", access: "old", refresh: "r", expires: 0 }));
|
||||
|
||||
await expect(models.getAuth(testModel("p1", "model-a"))).rejects.toMatchObject({ code: "oauth" });
|
||||
// credential preserved for retry / re-login
|
||||
expect(((await credentials.read("p1")) as { access: string }).access).toBe("old");
|
||||
});
|
||||
|
||||
it("serializes concurrent OAuth refreshes through store.modify (no double refresh)", async () => {
|
||||
const credentials = new InMemoryCredentialStore();
|
||||
await credentials.modify("p1", async () => ({ type: "oauth", access: "old", refresh: "r1", expires: 0 }));
|
||||
|
||||
let refreshes = 0;
|
||||
const oauth = testOAuth({
|
||||
refresh: async () => {
|
||||
refreshes++;
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
return { type: "oauth", access: `new-${refreshes}`, refresh: "r2", expires: Date.now() + 60_000 };
|
||||
},
|
||||
});
|
||||
const models = createModels({ credentials });
|
||||
models.setProvider(testProvider({ id: "p1", auth: { oauth } }));
|
||||
const model = testModel("p1", "model-a");
|
||||
|
||||
const [a, b] = await Promise.all([models.getAuth(model), models.getAuth(model)]);
|
||||
expect(refreshes).toBe(1);
|
||||
expect(a?.auth.apiKey).toBe("new-1");
|
||||
expect(b?.auth.apiKey).toBe("new-1");
|
||||
});
|
||||
|
||||
it("valid oauth tokens resolve without touching modify", async () => {
|
||||
let modifies = 0;
|
||||
const base = new InMemoryCredentialStore();
|
||||
const credentials: CredentialStore = {
|
||||
read: (pid) => base.read(pid),
|
||||
modify: (pid, fn) => {
|
||||
modifies++;
|
||||
return base.modify(pid, fn);
|
||||
},
|
||||
delete: (pid) => base.delete(pid),
|
||||
};
|
||||
await base.modify("p1", async () => ({
|
||||
type: "oauth",
|
||||
access: "valid",
|
||||
refresh: "r",
|
||||
expires: Date.now() + 60_000,
|
||||
}));
|
||||
const models = createModels({ credentials });
|
||||
models.setProvider(testProvider({ id: "p1", auth: { oauth: testOAuth() } }));
|
||||
|
||||
expect((await models.getAuth(testModel("p1", "model-a")))?.auth.apiKey).toBe("valid");
|
||||
expect(modifies).toBe(0);
|
||||
});
|
||||
|
||||
it("wraps credential store failures in ModelsError", async () => {
|
||||
// read failure
|
||||
const readFailing: CredentialStore = {
|
||||
read: async () => {
|
||||
throw new Error("disk on fire");
|
||||
},
|
||||
modify: async () => undefined,
|
||||
delete: async () => {},
|
||||
};
|
||||
const models = createModels({ credentials: readFailing });
|
||||
models.setProvider(testProvider({ id: "p1", auth: { apiKey: envKeyAuth("env-key") } }));
|
||||
await expect(models.getAuth(testModel("p1", "model-a"))).rejects.toMatchObject({ code: "auth" });
|
||||
|
||||
// modify failure during refresh
|
||||
const modifyFailing: CredentialStore = {
|
||||
read: async () => ({ type: "oauth", access: "old", refresh: "r", expires: 0 }),
|
||||
modify: async () => {
|
||||
throw new Error("disk on fire");
|
||||
},
|
||||
delete: async () => {},
|
||||
};
|
||||
const oauthModels = createModels({ credentials: modifyFailing });
|
||||
oauthModels.setProvider(testProvider({ id: "p1", auth: { oauth: testOAuth() } }));
|
||||
await expect(oauthModels.getAuth(testModel("p1", "model-a"))).rejects.toMatchObject({ code: "auth" });
|
||||
});
|
||||
|
||||
it("wraps api-key auth failures in ModelsError", async () => {
|
||||
const failing: ApiKeyAuth = {
|
||||
name: "Failing",
|
||||
resolve: async () => {
|
||||
throw new Error("nope");
|
||||
},
|
||||
};
|
||||
const models = createModels();
|
||||
models.setProvider(testProvider({ id: "p1", auth: { apiKey: failing } }));
|
||||
await expect(models.getAuth(testModel("p1", "model-a"))).rejects.toMatchObject({ code: "auth" });
|
||||
});
|
||||
|
||||
it("merges resolved auth into stream options; explicit options win per field", async () => {
|
||||
const calls: ProviderCall[] = [];
|
||||
const apiKey: ApiKeyAuth = {
|
||||
name: "Test",
|
||||
resolve: async () => ({
|
||||
auth: {
|
||||
apiKey: "resolved-key",
|
||||
headers: { "x-a": "auth", "x-b": "auth" },
|
||||
baseUrl: "https://auth.test/v1",
|
||||
},
|
||||
}),
|
||||
};
|
||||
const models = createModels();
|
||||
models.setProvider(testProvider({ id: "p1", auth: { apiKey }, calls }));
|
||||
const model = testModel("p1", "model-a");
|
||||
|
||||
const result = await models.completeSimple(model, context, {
|
||||
apiKey: "explicit-key",
|
||||
headers: { "x-b": "explicit" },
|
||||
});
|
||||
expect(result.stopReason).toBe("stop");
|
||||
expect(calls).toHaveLength(1);
|
||||
expect(calls[0].options?.apiKey).toBe("explicit-key");
|
||||
expect(calls[0].options?.headers).toEqual({ "x-a": "auth", "x-b": "explicit" });
|
||||
expect(calls[0].model.baseUrl).toBe("https://auth.test/v1");
|
||||
|
||||
// without explicit options, resolved auth applies
|
||||
const result2 = await models.completeSimple(model, context);
|
||||
expect(result2.stopReason).toBe("stop");
|
||||
expect(calls[1].options?.apiKey).toBe("resolved-key");
|
||||
});
|
||||
|
||||
it("produces an error stream for unknown providers instead of throwing", async () => {
|
||||
const models = createModels();
|
||||
const result = await models.completeSimple(testModel("ghost", "model-a"), context);
|
||||
expect(result.stopReason).toBe("error");
|
||||
expect(result.errorMessage).toContain("Unknown provider: ghost");
|
||||
});
|
||||
|
||||
it("streams through the provider", async () => {
|
||||
const models = createModels();
|
||||
models.setProvider(testProvider({ id: "p1" }));
|
||||
const model = testModel("p1", "model-a");
|
||||
|
||||
const events: string[] = [];
|
||||
const stream = models.streamSimple(model, context);
|
||||
for await (const event of stream) {
|
||||
events.push(event.type);
|
||||
}
|
||||
expect(events).toEqual(["start", "done"]);
|
||||
const message = await stream.result();
|
||||
expect(message.stopReason).toBe("stop");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user