ton/pi_harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into feat/pi-orchestrator

Browse Source
This commit is contained in:
Cristina Poncela Cubeiro
2026-06-25 09:53:29 +02:00
committed by GitHub
parent 0563baa51d 371adcf371
commit be64062f97
375 changed files with 35524 additions and 25982 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/APPROVED_CONTRIBUTORS
+2
View File
@@ -241,3 +241,5 @@ dangooddd pr
Mearman pr
dodiego pr
any-victor pr
.github/ISSUE_TEMPLATE/package-report.yml
+49
View File
@@ -0,0 +1,49 @@
name: Package Report
description: Report a problematic Pi package listed on pi.dev
labels: ["package-report"]
body:
- type: markdown
attributes:
value: |
Use this form to report a package listed on pi.dev. For Pi core bugs, use the bug report template instead.
New issues from new contributors are auto-closed by default. Maintainers review auto-closed issues daily. Issues that do not meet the quality bar in [CONTRIBUTING.md](https://github.com/earendil-works/pi-mono/blob/main/CONTRIBUTING.md) will not be reopened or receive a reply.
Keep this short. If it doesn't fit on one screen, it's too long. Write in your own voice.
- type: input
id: package-name
attributes:
label: Package name
description: The npm package name from pi.dev.
placeholder: "@scope/package"
validations:
required: true
- type: input
id: package-version
attributes:
label: Version
description: The package version shown on pi.dev.
placeholder: "0.1.0"
validations:
required: false
- type: dropdown
id: report-type
attributes:
label: What are you reporting?
options:
- Malicious or unsafe behavior
- Impersonation
- Trademark / TOS Violations
validations:
required: true
- type: textarea
id: details
attributes:
label: Details
description: Describe the concern and include links, logs, or screenshots if helpful.
validations:
required: true
.github/workflows/build-binaries.yml
+160 -21
View File
@@ -17,11 +17,17 @@ on:
permissions: {}
concurrency:
group: build-binaries-${{ github.event.inputs.tag || github.ref_name }}
cancel-in-progress: false
jobs:
# Keep the public GitHub Release publication last. Binary assets are staged in
# a draft release first; cleanup removes the draft if later publishing fails.
build:
runs-on: ubuntu-latest
permissions:
contents: write
contents: read
env:
RELEASE_TAG: ${{ github.event.inputs.tag || github.ref_name }}
SOURCE_REF: ${{ github.event.inputs.source_ref || github.event.inputs.tag || github.ref_name }}
@@ -46,17 +52,16 @@ jobs:
- name: Build binaries
run: ./scripts/build-binaries.sh
- name: Extract changelog for this version
id: changelog
- name: Prepare GitHub release payload
run: |
set -euo pipefail
mkdir -p release-assets
VERSION="${RELEASE_TAG}"
VERSION="${VERSION#v}" # Remove 'v' prefix
node scripts/release-notes.mjs extract --version "${VERSION}" --tag "${RELEASE_TAG}" --out /tmp/release-notes.md
node scripts/release-notes.mjs extract --version "${VERSION}" --tag "${RELEASE_TAG}" --out release-assets/RELEASE_NOTES.md
- name: Create GitHub Release and upload binaries
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
cd packages/coding-agent/binaries
release_assets=(
@@ -67,24 +72,107 @@ jobs:
pi-windows-x64.zip
pi-windows-arm64.zip
)
sha256sum "${release_assets[@]}" > SHA256SUMS
release_assets+=(SHA256SUMS)
if gh release view "${RELEASE_TAG}" >/dev/null 2>&1; then
gh release edit "${RELEASE_TAG}" \
--title "${RELEASE_TAG}" \
--notes-file /tmp/release-notes.md
gh release upload "${RELEASE_TAG}" "${release_assets[@]}" --clobber
else
for asset in "${release_assets[@]}"; do
test -f "${asset}"
done
sha256sum "${release_assets[@]}" > "${GITHUB_WORKSPACE}/release-assets/SHA256SUMS"
cp "${release_assets[@]}" "${GITHUB_WORKSPACE}/release-assets/"
- name: Upload GitHub release payload
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: release-assets-${{ env.RELEASE_TAG }}
path: release-assets/*
if-no-files-found: error
retention-days: 14
stage-github-release:
runs-on: ubuntu-latest
needs: build
permissions:
actions: read
contents: write
env:
GH_REPO: ${{ github.repository }}
RELEASE_TAG: ${{ github.event.inputs.tag || github.ref_name }}
steps:
- name: Download GitHub release payload
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: release-assets-${{ env.RELEASE_TAG }}
path: release-assets
- name: Validate GitHub release payload
run: |
set -euo pipefail
cd release-assets
expected_assets=(
pi-darwin-arm64.tar.gz
pi-darwin-x64.tar.gz
pi-linux-x64.tar.gz
pi-linux-arm64.tar.gz
pi-windows-x64.zip
pi-windows-arm64.zip
SHA256SUMS
RELEASE_NOTES.md
)
for asset in "${expected_assets[@]}"; do
test -f "${asset}"
done
sha256sum -c SHA256SUMS
- name: Create draft GitHub Release and upload binaries
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
cd release-assets
release_assets=(
pi-darwin-arm64.tar.gz
pi-darwin-x64.tar.gz
pi-linux-x64.tar.gz
pi-linux-arm64.tar.gz
pi-windows-x64.zip
pi-windows-arm64.zip
SHA256SUMS
)
existing_release="$(gh release view "${RELEASE_TAG}" --json isDraft --jq .isDraft 2>/dev/null || true)"
if [[ "${existing_release}" == "false" ]]; then
echo "::error::GitHub Release ${RELEASE_TAG} is already published. Refusing to mutate a public release."
exit 1
fi
if [[ "${existing_release}" == "true" ]]; then
gh release delete "${RELEASE_TAG}" --yes
fi
gh release create "${RELEASE_TAG}" \
--verify-tag \
--draft \
--title "${RELEASE_TAG}" \
--notes-file /tmp/release-notes.md \
--notes-file RELEASE_NOTES.md \
"${release_assets[@]}"
expected_asset_names="$(printf '%s\n' "${release_assets[@]}" | sort)"
actual_asset_names="$(gh release view "${RELEASE_TAG}" --json assets --jq '.assets[].name' | sort)"
if [[ "${actual_asset_names}" != "${expected_asset_names}" ]]; then
echo "::error::Draft GitHub Release asset set does not match expected files."
diff -u <(printf '%s\n' "${expected_asset_names}") <(printf '%s\n' "${actual_asset_names}") || true
exit 1
fi
publish-npm:
runs-on: ubuntu-latest
needs: build
needs: stage-github-release
environment: npm-publish
permissions:
contents: read
@@ -124,9 +212,6 @@ jobs:
- name: Test
run: npm test
- name: Verify release artifacts are committed
run: git diff --exit-code
- name: Upgrade npm for trusted publishing
run: |
npm install -g npm@11.16.0 --ignore-scripts
@@ -134,3 +219,57 @@ jobs:
- name: Publish npm packages
run: node scripts/publish.mjs
publish-github-release:
runs-on: ubuntu-latest
needs:
- stage-github-release
- publish-npm
permissions:
contents: write
env:
GH_REPO: ${{ github.repository }}
RELEASE_TAG: ${{ github.event.inputs.tag || github.ref_name }}
steps:
- name: Publish staged GitHub Release
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
existing_release="$(gh release view "${RELEASE_TAG}" --json isDraft --jq .isDraft 2>/dev/null || true)"
if [[ "${existing_release}" == "" ]]; then
echo "::error::Draft GitHub Release ${RELEASE_TAG} does not exist."
exit 1
fi
if [[ "${existing_release}" == "false" ]]; then
echo "::error::GitHub Release ${RELEASE_TAG} is already published."
exit 1
fi
gh release edit "${RELEASE_TAG}" --draft=false
cleanup-draft-github-release:
runs-on: ubuntu-latest
needs:
- build
- stage-github-release
- publish-npm
- publish-github-release
if: ${{ always() && needs.stage-github-release.result != 'skipped' && (needs.stage-github-release.result != 'success' || needs.publish-npm.result != 'success' || needs.publish-github-release.result != 'success') }}
permissions:
contents: write
env:
GH_REPO: ${{ github.repository }}
RELEASE_TAG: ${{ github.event.inputs.tag || github.ref_name }}
steps:
- name: Delete draft GitHub Release after failure
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
existing_release="$(gh release view "${RELEASE_TAG}" --json isDraft --jq .isDraft 2>/dev/null || true)"
if [[ "${existing_release}" == "true" ]]; then
gh release delete "${RELEASE_TAG}" --yes
fi
.github/workflows/issue-gate.yml
+8
View File
@@ -111,9 +111,17 @@ jobs:
body: message,
});
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: ['untriaged'],
});
await github.rest.issues.update({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
state: 'closed',
state_reason: 'not_planned',
});
.github/workflows/issue-triage-labels.yml
+142
View File
@@ -0,0 +1,142 @@
name: Issue Triage Labels
on:
issues:
types: [reopened, labeled]
jobs:
update-labels:
runs-on: ubuntu-latest
permissions:
issues: write
steps:
- name: Update triage labels
uses: actions/github-script@v7
with:
script: |
const UNTRIAGED_LABEL = 'untriaged';
const NO_ACTION_LABEL = 'no-action';
const LAST_READ_LABEL = 'last-read';
const TO_DISCUSS_LABEL = 'to-discuss';
const INPROGRESS_LABEL = 'inprogress';
function issueHasLabel(issue, labelName) {
return (issue.labels ?? []).some((label) => label.name === labelName);
}
async function removeLabelIfPresent(issueNumber, issue, labelName) {
if (!issueHasLabel(issue, labelName)) {
console.log(`Issue #${issueNumber} does not have ${labelName}`);
return;
}
try {
await github.rest.issues.removeLabel({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: issueNumber,
name: labelName,
});
console.log(`Removed ${labelName} from #${issueNumber}`);
} catch (error) {
if (error.status === 404) {
console.log(`Label ${labelName} was already absent from #${issueNumber}`);
return;
}
throw error;
}
}
if (context.payload.action === 'reopened') {
await removeLabelIfPresent(context.issue.number, context.payload.issue, UNTRIAGED_LABEL);
await removeLabelIfPresent(context.issue.number, context.payload.issue, NO_ACTION_LABEL);
return;
}
if (context.payload.action === 'labeled' && context.payload.label?.name === NO_ACTION_LABEL) {
await removeLabelIfPresent(context.issue.number, context.payload.issue, UNTRIAGED_LABEL);
return;
}
if (context.payload.action !== 'labeled' || context.payload.label?.name !== LAST_READ_LABEL) {
console.log('Not a last-read label event');
return;
}
const currentIssueNumber = context.issue.number;
const lastReadIssues = await github.paginate(github.rest.issues.listForRepo, {
owner: context.repo.owner,
repo: context.repo.repo,
state: 'all',
labels: LAST_READ_LABEL,
per_page: 100,
});
const previousIssueNumbers = lastReadIssues
.filter((issue) => !issue.pull_request)
.map((issue) => issue.number)
.filter((issueNumber) => issueNumber !== currentIssueNumber);
if (previousIssueNumbers.length === 0) {
console.log('No previous last-read issue found');
return;
}
const previousIssueNumber = Math.max(...previousIssueNumbers);
if (currentIssueNumber <= previousIssueNumber) {
console.log(
`Last-read was added to old issue #${currentIssueNumber}; latest last-read is #${previousIssueNumber}`,
);
return;
}
const untriagedIssues = await github.paginate(github.rest.issues.listForRepo, {
owner: context.repo.owner,
repo: context.repo.repo,
state: 'all',
labels: UNTRIAGED_LABEL,
per_page: 100,
});
const issuesToMark = untriagedIssues
.filter((issue) => !issue.pull_request)
.filter((issue) => issue.number >= previousIssueNumber && issue.number <= currentIssueNumber)
.sort((a, b) => a.number - b.number);
if (issuesToMark.length === 0) {
console.log(`No untriaged issues found from #${previousIssueNumber} to #${currentIssueNumber}`);
return;
}
for (const issue of issuesToMark) {
if (issueHasLabel(issue, TO_DISCUSS_LABEL)) {
console.log(`Skipped ${NO_ACTION_LABEL} for #${issue.number} because it has ${TO_DISCUSS_LABEL}`);
} else {
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: issue.number,
labels: [NO_ACTION_LABEL],
});
console.log(`Added ${NO_ACTION_LABEL} to #${issue.number}`);
}
await github.rest.issues.update({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: issue.number,
state: 'closed',
state_reason: 'not_planned',
});
console.log(`Closed #${issue.number} as not planned`);
await removeLabelIfPresent(issue.number, issue, INPROGRESS_LABEL);
await github.rest.issues.removeLabel({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: issue.number,
name: UNTRIAGED_LABEL,
});
console.log(`Removed ${UNTRIAGED_LABEL} from #${issue.number}`);
}
.github/workflows/openclaw-gate.yml
-122
View File
@@ -1,122 +0,0 @@
name: OpenClaw Gate
on:
issues:
types: [opened]
pull_request_target:
types: [opened]
jobs:
check-contributor:
runs-on: ubuntu-latest
permissions:
contents: read
issues: write
pull-requests: write
steps:
- name: Check contributor
uses: actions/github-script@v7
with:
script: |
const isPR = !!context.payload.pull_request;
const author = isPR
? context.payload.pull_request.user.login
: context.payload.issue.user.login;
const number = isPR
? context.payload.pull_request.number
: context.payload.issue.number;
const defaultBranch = context.payload.repository.default_branch;
if (author.endsWith('[bot]') || author === 'dependabot[bot]') {
console.log(`Skipping bot: ${author}`);
return;
}
const APPROVED_FILE = '.github/APPROVED_CONTRIBUTORS';
const VALID_CAPABILITIES = new Set(['issue', 'pr']);
// --- Check APPROVED_CONTRIBUTORS ---
async function getTextFile(path) {
const { data } = await github.rest.repos.getContent({
owner: context.repo.owner,
repo: context.repo.repo,
path,
ref: defaultBranch,
});
if (!('content' in data) || typeof data.content !== 'string') {
throw new Error(`Expected file content for ${path}`);
}
return Buffer.from(data.content, 'base64').toString('utf8');
}
try {
const content = await getTextFile(APPROVED_FILE);
const approved = new Map();
for (const rawLine of content.split('\n')) {
const line = rawLine.trim();
if (!line || line.startsWith('#')) continue;
const parts = line.split(/\s+/);
if (parts.length !== 2) continue;
const [username, capability] = parts;
const normalizedCapability = capability.toLowerCase();
if (!VALID_CAPABILITIES.has(normalizedCapability)) continue;
approved.set(username.toLowerCase(), normalizedCapability);
}
if (approved.has(author.toLowerCase())) {
console.log(`${author} is in APPROVED_CONTRIBUTORS, passing`);
return;
}
} catch (err) {
console.log(`Could not read APPROVED_CONTRIBUTORS: ${err.message}`);
}
// --- Also pass collaborators with write+ access ---
try {
const { data: perm } = await github.rest.repos.getCollaboratorPermissionLevel({
owner: context.repo.owner,
repo: context.repo.repo,
username: author,
});
if (['admin', 'maintain', 'write'].includes(perm.permission)) {
console.log(`${author} is a collaborator (${perm.permission}), passing`);
return;
}
} catch {
// not a collaborator
}
// --- Check if user opened issues/PRs on openclaw/openclaw ---
async function hasOpenClawActivity(username) {
try {
const { data } = await github.rest.search.issuesAndPullRequests({
q: `repo:openclaw/openclaw author:${username}`,
per_page: 1,
});
if (data.total_count > 0) {
console.log(`${username} has opened ${data.total_count} issues/PRs on openclaw/openclaw`);
return true;
}
} catch (err) {
console.log(`Search failed: ${err.message}`);
}
return false;
}
const hasActivity = await hasOpenClawActivity(author);
if (!hasActivity) {
console.log(`${author} has no openclaw/openclaw activity, passing`);
return;
}
// --- Add openclaw label ---
console.log(`${author} has openclaw/openclaw activity, adding label`);
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: number,
labels: ['possibly-openclaw-clanker'],
});
.github/workflows/remove-inprogress-on-close.yml
+31
View File
@@ -0,0 +1,31 @@
name: Remove In Progress Label On Close
on:
issues:
types: [closed]
jobs:
remove-label:
runs-on: ubuntu-latest
permissions:
issues: write
steps:
- name: Remove inprogress label
uses: actions/github-script@v7
with:
script: |
const labelName = 'inprogress';
const labels = context.payload.issue.labels ?? [];
const hasLabel = labels.some((label) => label.name === labelName);
if (!hasLabel) {
console.log(`Issue does not have ${labelName} label`);
return;
}
await github.rest.issues.removeLabel({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
name: labelName,
});
README.md
+26 -26
View File
@@ -5,46 +5,24 @@
</p>
<p align="center">
<a href="https://discord.com/invite/3cU7Bz4UPx"><img alt="Discord" src="https://img.shields.io/badge/discord-community-5865F2?style=flat-square&logo=discord&logoColor=white" /></a>
</p>
<p align="center">
<a href="https://pi.dev">pi.dev</a> domain graciously donated by
<br /><br />
<a href="https://exe.dev"><img src="packages/coding-agent/docs/images/exy.png" alt="Exy mascot" width="48" /><br />exe.dev</a>
<a href="https://www.npmjs.com/package/@earendil-works/pi-coding-agent"><img alt="npm" src="https://img.shields.io/npm/v/@earendil-works/pi-coding-agent?style=flat-square" /></a>
</p>
> New issues and PRs from new contributors are auto-closed by default. Maintainers review auto-closed issues daily. See [CONTRIBUTING.md](CONTRIBUTING.md).
---
# Pi Agent Harness
# Pi Agent Harness Mono Repo
This is the home of the pi agent harness project including our self extensible coding agent.
This is the home of the Pi agent harness project including our self extensible coding agent.
* **[@earendil-works/pi-coding-agent](packages/coding-agent)**: Interactive coding agent CLI
* **[@earendil-works/pi-agent-core](packages/agent)**: Agent runtime with tool calling and state management
* **[@earendil-works/pi-ai](packages/ai)**: Unified multi-provider LLM API (OpenAI, Anthropic, Google, …)
To learn more about pi:
To learn more about Pi:
* [Visit pi.dev](https://pi.dev), the project website with demos
* [Read the documentation](https://pi.dev/docs/latest), but you can also ask the agent to explain itself
## Share your OSS coding agent sessions
If you use pi or other coding agents for open source work, please share your sessions.
Public OSS session data helps improve coding agents with real-world tasks, tool use, failures, and fixes instead of toy benchmarks.
For the full explanation, see [this post on X](https://x.com/badlogicgames/status/2037811643774652911).
To publish sessions, use [`badlogic/pi-share-hf`](https://github.com/badlogic/pi-share-hf). Read its README.md for setup instructions. All you need is a Hugging Face account, the Hugging Face CLI, and `pi-share-hf`.
You can also watch [this video](https://x.com/badlogicgames/status/2041151967695634619), where I show how I publish my `pi-mono` sessions.
I regularly publish my own `pi-mono` work sessions here:
- [badlogicgames/pi-mono on Hugging Face](https://huggingface.co/datasets/badlogicgames/pi-mono)
## All Packages
| Package | Description |
@@ -94,6 +72,28 @@ We treat npm dependency changes as reviewed code changes.
- CI installs with `npm ci --ignore-scripts`, and a scheduled GitHub workflow runs `npm audit --omit=dev` plus `npm audit signatures --omit=dev`.
- Shrinkwrap generation has an explicit allowlist for dependency lifecycle scripts; new lifecycle-script deps fail checks until reviewed.
## Share your OSS coding agent sessions
If you use Pi or other coding agents for open source work, please share your sessions.
Public OSS session data helps improve coding agents with real-world tasks, tool use, failures, and fixes instead of toy benchmarks.
For the full explanation, see [this post on X](https://x.com/badlogicgames/status/2037811643774652911).
To publish sessions, use [`badlogic/pi-share-hf`](https://github.com/badlogic/pi-share-hf). Read its README.md for setup instructions. All you need is a Hugging Face account, the Hugging Face CLI, and `pi-share-hf`.
You can also watch [this video](https://x.com/badlogicgames/status/2041151967695634619), where I show how I publish my `pi-mono` sessions.
I regularly publish my own `pi-mono` work sessions here:
- [badlogicgames/pi-mono on Hugging Face](https://huggingface.co/datasets/badlogicgames/pi-mono)
## License
MIT
<p align="center">
<a href="https://pi.dev">pi.dev</a> domain graciously donated by
<br /><br />
<a href="https://exe.dev"><img src="packages/coding-agent/docs/images/exy.png" alt="Exy mascot" width="48" /><br />exe.dev</a>
</p>
biome.json
+1
View File
@@ -31,6 +31,7 @@
"!**/node_modules/**/*",
"!**/test-sessions.ts",
"!**/models.generated.ts",
"!**/*.models.ts",
"!packages/mom/data/**/*",
"!!**/node_modules"
]
package-lock.json
Generated
+1574 -1804
View File
File diff suppressed because it is too large Load Diff
package.json
+1 -1
View File
@@ -41,7 +41,7 @@
"@biomejs/biome": "2.3.5",
"@types/node": "22.19.19",
"@typescript/native-preview": "7.0.0-dev.20260120.1",
"esbuild": "0.28.0",
"esbuild": "0.28.1",
"husky": "9.1.7",
"jiti": "2.7.0",
"shx": "0.4.0",
packages/agent/CHANGELOG.md
+38
View File
@@ -2,6 +2,44 @@
## [Unreleased]
## [0.80.2] - 2026-06-23
### Changed
- Renamed the public harness shell execution options type from `ExecutionEnvExecOptions` to `ShellExecOptions`.
## [0.80.1] - 2026-06-23
## [0.80.0] - 2026-06-23
### Breaking Changes
- `AgentHarnessOptions.models` is required and is the only auth path: the harness streams turns, compaction, and branch summarization through the provided `Models` instance (`models.streamSimple()`/`completeSimple()`), resolving auth through the providers. `AgentHarnessOptions.getApiKeyAndHeaders` is removed — apps that resolved keys per request now express that as provider auth (`ApiKeyAuth`/`OAuthAuth`) on the providers in the `Models` collection. Build one with `createModels()` + provider factories (or `builtinModels()` from `@earendil-works/pi-ai/providers/all`); tests use `fauxProvider()`.
- `compact()`, `generateSummary()`, and `generateBranchSummary()` take a `Models` parameter and no longer accept explicit `apiKey`/`headers`.
- `StreamFn` is defined structurally (`(model, context, options?) => AssistantMessageEventStream | Promise<...>`); `Models.streamSimple` satisfies it.
- Removed the `@earendil-works/pi-agent-core/base` selective-provider entrypoint; use the root package with an explicit `Models` instance instead.
### Fixed
- Fixed harness session names to normalize newline characters before storing labels ([#5999](https://github.com/earendil-works/pi/pull/5999) by [@haoqixu](https://github.com/haoqixu)).
- Fixed harness compaction estimates to ignore malformed all-zero assistant usage after truncated responses ([#5526](https://github.com/earendil-works/pi/pull/5526) by [@dmmulroy](https://github.com/dmmulroy)).
## [0.79.10] - 2026-06-22
## [0.79.9] - 2026-06-20
### Fixed
- Fixed Node execution environment commands through legacy WSL `bash.exe` to pass scripts over stdin so shell variables expand in the target bash ([#5893](https://github.com/earendil-works/pi/issues/5893)).
## [0.79.8] - 2026-06-19
### Added
- Added `@earendil-works/pi-agent-core/base` for bundlers that want to pair the agent core with selective `@earendil-works/pi-ai/base` provider registration ([#5348](https://github.com/earendil-works/pi/pull/5348) by [@FredKSchott](https://github.com/FredKSchott)).
## [0.79.7] - 2026-06-18
## [0.79.6] - 2026-06-16
## [0.79.5] - 2026-06-16
packages/agent/docs/models.md
+966
View File
@@ -0,0 +1,966 @@
# Models architecture
This document describes the target design for the next `pi-ai` model/provider refactor. It describes the desired shape, not the current implementation. It is intended to be complete enough to start implementing from a fresh session.
Goals:
- `Models` is a dumb runtime collection of providers.
- Concrete providers own metadata, auth, model listing, and stream behavior.
- API implementations live under `src/api/` and are reusable/lazy.
- Concrete provider factories live under `src/providers/`.
- Users can import only the providers they need.
- Importing a provider must not eagerly import heavy SDKs.
- Dynamic model lists are first-class: reads are sync (last-known list), fetching happens in an explicit async `refresh`.
- `models.json` and extensions layer by wrapping providers, not by mutating provider internals ad hoc.
- Old global APIs survive only in an explicit, temporary `/compat` entrypoint.
Non-goals for the immediate `pi-ai` pass:
- Do not migrate coding-agent `ModelRegistry` yet.
- Do not keep the stream/API registry inside `Models`.
- Do not implement web OAuth flows yet.
- Image generation mirrors the chat-side design (`ImagesModels`/`ImagesProvider` in `images-models.ts`); the old global image API (`images.ts`, `images-api-registry.ts`) lives on compat.
## Package layout
Target source layout:
```txt
packages/ai/src/
index.ts # core exports only; no built-in provider imports
models.ts # Models runtime, Provider
images-models.ts # ImagesModels runtime, ImagesProvider (mirrors models.ts)
compat.ts # temporary old-API compatibility entrypoint
auth/ # auth method types, helpers, shared resolveProviderAuth(), login callbacks
api/ # API implementations and lazy wrappers
openai-completions.ts # real implementation, imports SDKs, exports stream/streamSimple
openai-completions.lazy.ts
openai-responses.ts
openai-responses.lazy.ts
openai-codex-responses.ts
openai-codex-responses.lazy.ts
azure-openai-responses.ts
azure-openai-responses.lazy.ts
anthropic-messages.ts
anthropic-messages.lazy.ts
google-generative-ai.ts
google-generative-ai.lazy.ts
google-vertex.ts
google-vertex.lazy.ts
mistral-conversations.ts
mistral-conversations.lazy.ts
bedrock-converse-stream.ts
bedrock-converse-stream.lazy.ts
openrouter-images.ts # image-generation API implementation
openrouter-images.lazy.ts
lazy.ts # lazyStream()/lazyApi() helpers
(shared helpers: openai-responses-shared, google-shared, transform-messages, ...)
providers/ # concrete provider factories and per-provider catalogs
openai.ts
openai.models.ts # generated OpenAI catalog
openai-codex.ts
openai-codex.models.ts
anthropic.ts
anthropic.models.ts
google.ts
google.models.ts
...one pair per built-in provider...
openrouter-images.ts # image-generation provider factory
faux.ts # test provider factory
all.ts # explicit aggregate: builtinModels(), builtinImagesModels(), getBuiltin*()
utils/oauth/ # OAuth flow implementations (node), lazy-loaded
```
`src/index.ts` must stay core-only. It must not import:
- generated model catalogs
- built-in provider factories
- provider SDK implementations
- Node-only OAuth modules
- `providers/all`
- `compat`
Provider, API, and compat entrypoints are explicit subpath exports.
## Public usage
Minimal provider usage:
```ts
import { createModels } from "@earendil-works/pi-ai";
import { openaiProvider } from "@earendil-works/pi-ai/providers/openai";
const models = createModels();
models.setProvider(openaiProvider());
const model = models.getModel("openai", "gpt-4o-mini");
if (!model) throw new Error("model not found");
const response = await models.complete(model, context);
```
Multiple providers:
```ts
const models = createModels();
models.setProvider(openaiProvider());
models.setProvider(openrouterProvider());
```
All built-ins, explicitly heavy metadata entrypoint:
```ts
import { builtinModels } from "@earendil-works/pi-ai/providers/all";
const models = builtinModels();
```
`providers/all` may import all provider metadata/catalogs. It still must not eagerly import SDK implementations; provider streams use lazy wrappers.
## Core runtime: Models
`Models` is a provider collection plus auth application and stream convenience. No stream registry, no auth resolver strategy object.
```ts
export function createModels(options?: {
/** App-owned credential storage. Default: in-memory store. */
credentials?: CredentialStore;
/** Environment access for auth resolution (env vars, file existence). Default: process.env/node:fs backed; injectable for tests and non-Node hosts. */
authContext?: AuthContext;
}): MutableModels;
export interface Models {
getProviders(): readonly Provider[];
getProvider(id: string): Provider | undefined;
/** Sync read of last-known models. Best-effort: a provider whose getModels() throws yields no models. */
getModels(provider?: string): readonly Model<Api>[];
/** Dynamic lists are honestly Model<Api>; narrow with the hasApi() guard. */
getModel(provider: string, id: string): Model<Api> | undefined;
/**
* Ask dynamic providers to re-fetch their model lists. With a provider id,
* rejects on that provider's failure; without, refreshes all concurrently
* best-effort. Static providers are no-ops.
*/
refresh(provider?: string): Promise<void>;
/**
* Resolve request auth for a model. Includes source label for status UI.
* Resolves undefined when the provider is unknown or unconfigured. Rejects
* with ModelsError ("oauth" on refresh failure, "auth" on api-key/store
* failure); status/availability UIs catch rejections and render
* "needs re-login" instead of treating them as unconfigured.
*/
getAuth(model: Model<Api>): Promise<AuthResult | undefined>;
stream<TApi extends Api>(
model: Model<TApi>,
context: Context,
options?: ApiStreamOptions<TApi>,
): AssistantMessageEventStream;
complete<TApi extends Api>(
model: Model<TApi>,
context: Context,
options?: ApiStreamOptions<TApi>,
): Promise<AssistantMessage>;
streamSimple(model: Model<Api>, context: Context, options?: SimpleStreamOptions): AssistantMessageEventStream;
completeSimple(model: Model<Api>, context: Context, options?: SimpleStreamOptions): Promise<AssistantMessage>;
}
export interface MutableModels extends Models {
/** Upsert/replace by provider.id. Provider ids are unique. */
setProvider(provider: Provider): void;
deleteProvider(id: string): void;
clearProviders(): void;
}
```
Removed concepts:
```txt
no Models.setStreamFunctions() / getStreamFunctions()
no api-registry as a real dispatch mechanism
no Models.provider(id) builder, no setModel/upsertModel/patchModel lifecycle
no ModelAuthResolver / setAuthResolver — resolution policy is fixed, store is injected
```
If an app needs different auth policy, it wraps providers (wrap auth methods or `getModels`) or passes explicit request auth in stream options.
## Provider
A provider is the concrete runtime unit. It owns id/name/base metadata, auth methods, model listing, and stream behavior.
`Provider` is generic over the APIs its models use. Concrete factories declare what they emit (`openaiProvider(): Provider<"openai-responses" | "openai-completions">`), giving typed model lists to direct factory users. A `Models` collection holds providers as `Provider<Api>`.
```ts
export interface Provider<TApi extends Api = Api> {
readonly id: string;
readonly name: string;
readonly baseUrl?: string;
readonly headers?: Record<string, string>;
/**
* Required: at least one of apiKey/oauth. Even ambient-credential providers
* (env vars, AWS profiles, ADC) and keyless local servers provide apiKey
* auth whose resolve() reports whether the provider is configured.
* getAuth() returning undefined = not configured.
*/
readonly auth: ProviderAuth;
/** Current known models, sync. Static providers: the catalog. Dynamic providers: as of the last refresh (empty before the first). */
getModels(): readonly Model<TApi>[];
/** Dynamic providers only: fetch and update the model list. Concurrent calls share one in-flight fetch. */
refreshModels?(): Promise<void>;
stream<T extends TApi>(model: Model<T>, context: Context, options?: ApiStreamOptions<T>): AssistantMessageEventStream;
streamSimple(model: Model<TApi>, context: Context, options?: SimpleStreamOptions): AssistantMessageEventStream;
}
```
There is no `Provider.api` field. `model.api` carries API identity; the provider dispatches internally (see `createProvider()`).
`Model.api` remains: existing metadata and tests use it, it is useful for diagnostics, and provider construction uses it for API implementation selection. But `Models` never dispatches on it; the provider does.
### Typed stream options
Full stream options are API-specific. `Model<TApi>` pays off by deriving the option type from the API:
```ts
// types.ts — type-only imports from API impl modules are erased, so this is tree-shake safe
export interface ApiOptionsMap {
"anthropic-messages": AnthropicOptions;
"openai-completions": OpenAICompletionsOptions;
"openai-responses": OpenAIResponsesOptions;
"openai-codex-responses": OpenAICodexResponsesOptions;
"azure-openai-responses": AzureOpenAIResponsesOptions;
"google-generative-ai": GoogleOptions;
"google-vertex": GoogleVertexOptions;
"mistral-conversations": MistralOptions;
"bedrock-converse-stream": BedrockOptions;
}
export type ApiStreamOptions<TApi extends Api> = TApi extends keyof ApiOptionsMap
? ApiOptionsMap[TApi]
: StreamOptions & Record<string, unknown>;
```
Custom api strings fall back to the generic shape.
### Typed model narrowing
Runtime model lists are dynamic, so `models.getModel()`/`getModels()` honestly return `Model<Api>`. Typing improves at three points:
1. **`hasApi()` type guard** — runtime-checked narrowing for dynamic lookups (no blind casts):
```ts
export function hasApi<TApi extends Api>(model: Model<Api>, api: TApi): model is Model<TApi>;
const model = models.getModel("anthropic", "claude-opus-4-7");
if (model && hasApi(model, "anthropic-messages")) {
// model: Model<"anthropic-messages">, stream options fully typed
}
```
2. **`getBuiltinModel()`** — sync, generated-catalog lookup with typed overloads: `(provider, id) -> Model<exact-api-literal>`. The path for hardcoded known models.
3. **`Provider<TApi>` factories** — typed model lists when using a provider directly, without a `Models` collection.
Deliberately not done: tying `models.getModel(provider, ...)` to typed provider/model ids would require statically knowing which providers are installed in a mutable runtime collection. The harness path (`streamSimple` + `SimpleStreamOptions`) is API-agnostic and unaffected.
For comparison: Vercel AI SDK attaches the implementation to the model object, which dissolves dispatch typing but makes models non-serializable (no sessions/RPC/catalogs as plain data), and its `providerOptions` bag is `Record<string, JSON>` checked only by `satisfies` convention. Plain-data models + provider-owned behavior keeps stronger typing where it matters.
### Name collision
`types.ts` currently exports `type Provider = KnownProvider | string` (a provider id). Rename that alias to `ProviderId` and fix call sites. The `Provider` interface above takes the name.
## Provider model listing
Reads are sync; fetching is an explicit async verb. `Provider.getModels()` returns the current known list — the full catalog for static providers, the last-refreshed list for dynamic ones (llama.cpp, OpenRouter live listing). `refreshModels()` is where dynamic providers fetch.
This split exists because a sync-or-async union (`Promise<T> | T`) invites latent sync assumptions that detonate on the first async provider, while async-only reads force every consumer (UI lists, extension `find`/`getAll` surfaces) through Promises for data that is almost always static. Sync reads + explicit refresh keeps the staleness visible and the contract single: `getModels()` = last known, `refresh()` = make it current. A fetched list is stale the moment it returns anyway; naming the refresh point is honest about it.
Apps own the refresh lifecycle: startup, registry reload, opening a model selector. Freshness-critical lookups are two-step: `await models.refresh("llamacpp"); models.getModel("llamacpp", id)`.
Dynamic refresh must be side-effect-free discovery:
```txt
OK: fetch /v1/models, enumerate local catalog, refresh cached remote model list
Not OK: load model, download model, mutate server state, run request probe
```
Provider-specific model lifecycle (load/unload) belongs in app/provider-management commands, not in `refreshModels()`.
## Streaming path
`Models.stream()` finds the provider by `model.provider`, resolves auth, merges it into request options, and delegates:
```ts
function stream(model, context, options) {
const provider = this.getProvider(model.provider);
if (!provider) {
// produce an error stream, not a throw — see Error behavior
}
// async setup happens inside the returned stream (lazyStream pattern)
const resolution = await this.getAuth(model);
const requestModel = resolution?.auth.baseUrl ? { ...model, baseUrl: resolution.auth.baseUrl } : model;
const requestOptions = mergeAuth(options, resolution?.auth); // explicit options win per-field
return provider.stream(requestModel, context, requestOptions);
}
```
`stream()` returns `AssistantMessageEventStream` synchronously; async setup (auth resolution, lazy module load) happens inside the returned stream. The forwarding pattern already exists in today's `register-builtins.ts` (`createLazyStream`); extract it as `lazyStream()` in `src/api/lazy.ts`.
No request hot-path model canonicalization: `stream()` uses the supplied model object as-is. If an app wants fresh model metadata, it refreshes the provider and re-reads (`await models.refresh(p); models.getModel(p, id)`) before starting the turn.
## API implementations under `src/api`
An API implementation is reusable stream behavior. It is not a provider.
Uniform export contract — every real implementation module exports exactly:
```ts
// src/api/anthropic-messages.ts — imports SDKs
export function stream(model, context, options) { ... }
export function streamSimple(model, context, options) { ... }
```
This makes the module itself satisfy `ProviderStreams`, so the lazy wrapper is one generic helper instead of bespoke per-API plumbing. `ProviderStreams` is the untyped dispatch shape (implementation modules export concretely typed functions, which would not be assignable to a generic method); per-API option typing lives on the modules themselves and on `Provider.stream()` via `ApiStreamOptions`:
```ts
export interface ProviderStreams {
stream(model: Model<Api>, context: Context, options?: StreamOptions): AssistantMessageEventStream;
streamSimple(model: Model<Api>, context: Context, options?: SimpleStreamOptions): AssistantMessageEventStream;
}
// src/api/lazy.ts
export function lazyApi(load: () => Promise<ProviderStreams>): ProviderStreams;
// src/api/anthropic-messages.lazy.ts
export const anthropicMessagesApi = (): ProviderStreams => lazyApi(() => import("./anthropic-messages.ts"));
```
Import chain:
```txt
provider module -> lazy API wrapper -> dynamic import(real API impl) -> SDK deps
```
Notes:
- Bedrock keeps the node-only dynamic import trick (`importNodeOnlyProvider`, `.ts`/`.js` specifier rewrite) inside its lazy wrapper. `setBedrockProviderModule()` (used by the Bun build) moves into the bedrock lazy wrapper module.
- Shared helper modules (`openai-responses-shared.ts`, `google-shared.ts`, `transform-messages.ts`, prompt-cache, copilot headers) move to `src/api/` alongside the implementations.
## Shared API implementations across concrete providers
Many concrete providers share an API implementation (OpenAI-completions: OpenRouter, Groq, Cerebras, xAI, ZAI, ...). They share lazy API objects by reference:
```ts
import { openAICompletionsApi } from "../api/openai-completions.lazy.ts";
export function openrouterProvider(): Provider {
return createProvider({
id: "openrouter",
name: "OpenRouter",
baseUrl: "https://openrouter.ai/api/v1",
auth: { apiKey: envApiKeyAuth("OpenRouter API key", ["OPENROUTER_API_KEY"]) },
models: OPENROUTER_MODELS,
api: openAICompletionsApi(),
});
}
```
This copies Vercel AI SDK's useful property: users import concrete providers; shared protocol implementation is internal.
## Auth
Request auth output stays small:
```ts
export interface ModelAuth {
apiKey?: string;
headers?: Record<string, string>;
baseUrl?: string;
}
```
If a value cannot be expressed as `apiKey`, `headers`, or `baseUrl`, it is provider config, not auth (Vertex project/location, Bedrock region/profile, Azure apiVersion are provider factory options).
### Provider auth
`Provider.auth` has exactly two slots; real providers have at most one api-key path and at most one OAuth path, and the slot names carry the UI's oauth-vs-api-key split without a `kind` discriminant or method ids:
```ts
export interface ProviderAuth {
apiKey?: ApiKeyAuth; // stored key/provider env + ambient env/files/ADC/IAM
oauth?: OAuthAuth; // login flow + refresh
}
export interface ApiKeyAuth {
name: string; // "Anthropic API key"
/** Interactive setup (prompt for key/provider env). Absent = ambient-only (env, ADC, IAM). */
login?(callbacks: AuthLoginCallbacks): Promise<ApiKeyCredential>;
/**
* Resolve auth from the stored credential and/or ambient sources, merging
* per field (credential.key ?? env("..."), credential.env?.NAME ?? env("...")).
* undefined = not configured.
*/
resolve(input: {
model: Model<Api>;
ctx: AuthContext;
credential?: ApiKeyCredential;
}): Promise<AuthResult | undefined>;
}
export interface OAuthAuth {
name: string; // "Anthropic (Claude Pro/Max)"
login(callbacks: AuthLoginCallbacks): Promise<OAuthCredential>;
/** Exchange the refresh token. Network call; throws on failure (invalid_grant etc.). Runs under the store lock. */
refresh(credential: OAuthCredential): Promise<OAuthCredential>;
/** Side-effect-free derivation of request auth from a valid credential. Covers Copilot-style per-credential baseUrl. Async so lazy wrappers can load the implementation. */
toAuth(credential: OAuthCredential): Promise<ModelAuth>;
}
export interface AuthResult {
auth: ModelAuth;
/** Human-readable label for status UI: "ANTHROPIC_API_KEY", "OAuth", "~/.aws/credentials". */
source?: string;
}
export interface AuthContext {
env(name: string): Promise<string | undefined>;
fileExists(path: string): Promise<boolean>; // supports leading ~
}
```
The OAuth split (`refresh` + `toAuth` instead of one `resolve`) matches the old `OAuthProviderInterface` (`refreshToken` + `getApiKey`) and lets `Models` own the locking pattern without closure gymnastics: refresh produces a credential, `toAuth` derives request auth from whatever credential ends up stored.
There is no `usesCallbackServer` flag. With `prompt()/notify()` callbacks the flow self-describes at runtime: a flow that runs a callback server issues a `manual_code` prompt racing the server and aborts the prompt when the callback wins. The UI needs no static foreknowledge.
### Credentials
One credential per provider, type-tagged — exactly the shape of today's auth.json (`type: "api_key" | "oauth"` per provider id):
```ts
export interface ApiKeyCredential {
type: "api_key";
key?: string;
env?: ProviderEnv; // e.g. Cloudflare account/gateway ids, Azure/Vertex/Bedrock scoped config
}
export interface OAuthCredential extends OAuthCredentials {
type: "oauth"; // access, refresh, expires from OAuthCredentials
}
export type Credential = ApiKeyCredential | OAuthCredential;
```
`ApiKeyCredential.env` stores provider-scoped environment/config values alongside or instead of a key. `ApiKeyAuth.resolve()` merges per field: `credential.key ?? env("CLOUDFLARE_API_KEY")`, `credential.env?.CLOUDFLARE_ACCOUNT_ID ?? env("CLOUDFLARE_ACCOUNT_ID")`, etc. The credential discriminator intentionally matches today's `auth.json` (`api_key`) so the file-backed store does not need lossy type translation.
### Credential store
The app injects storage; `pi-ai` ships an in-memory default. Keyed by provider id, one credential per provider:
```ts
export interface CredentialStore {
/** Read the stored credential, possibly expired. Display/status use; request auth comes from Models.getAuth(). */
read(providerId: string): Promise<Credential | undefined>;
/**
* Serialized write — the only write path. fn sees the current credential
* because correct writes (refresh, login-during-refresh) depend on it;
* return the new credential, or undefined to leave the entry unchanged.
* Mutual exclusion per provider id, cross-process too where the backing
* store supports it (file lock). Resolves with the post-write credential.
*/
modify(
providerId: string,
fn: (current: Credential | undefined) => Promise<Credential | undefined>,
): Promise<Credential | undefined>;
/** Remove (logout). Serialized against modify. */
delete(providerId: string): Promise<void>;
}
```
There is deliberately no `set`: an unserialized write path invites read-modify-write races (login-during-refresh clobbering a fresh credential, double token refresh). Call sites:
```ts
await store.modify(pid, async () => credential); // login: store this
await store.read(pid); // status UI ("logged in via OAuth")
await store.delete(pid); // logout
// refresh RMW happens inside Models.getAuth
```
Error semantics: `read` resolves `undefined` for missing entries; methods reject only on storage failure, and `Models` wraps such rejections in `ModelsError` code `"auth"`. Best-effort stores that serve an in-memory view and record persistence errors internally (today's AuthStorage behavior) are valid implementations.
### Resolution policy (fixed)
`Models.getAuth(model)` is a decision tree, not a loop. A stored credential owns the provider — ambient/env is consulted only when nothing is stored (AuthStorage parity: no silent env fallback after a failed refresh or for an unmatched credential type):
```ts
const stored = await store.read(provider.id);
if (stored) {
if (stored.type === "oauth" && provider.auth.oauth) {
const oauth = provider.auth.oauth;
let credential = stored;
if (Date.now() >= credential.expires) { // optimistic check, lock-free
const post = await store.modify(provider.id, async (current) => {
if (current?.type !== "oauth") return undefined; // logged out meanwhile
return Date.now() >= current.expires // authoritative check, under lock
? oauth.refresh(current) // throws -> ModelsError("oauth")
: undefined; // another process/request refreshed
});
if (post?.type !== "oauth") return undefined;
credential = post;
}
return { auth: await oauth.toAuth(credential), source: "OAuth" };
}
if (stored.type === "api_key" && provider.auth.apiKey) {
return provider.auth.apiKey.resolve({ model, ctx, credential: stored });
}
return undefined; // stored credential without matching handler blocks ambient
}
return provider.auth.apiKey?.resolve({ model, ctx, credential: undefined }); // ambient
```
Properties:
- Double-checked locking, same as today's `refreshOAuthTokenWithLock`: valid tokens cost one `read` and zero locks; expired tokens lock, re-check under the lock, refresh once globally, persist before release.
- Explicit request auth (stream options `apiKey`/`headers`) is merged per-field on top in `stream()`, winning over everything.
- Refresh failure rejects with `ModelsError("oauth")`; the stored credential is untouched (preserved for retry). Request paths surface this as a stream error with the real cause ("run /login"); status/availability UIs catch the rejection and render "needs re-login" — documented contract on `getAuth`.
### Replacing AuthStorage
The end state for coding-agent: AuthStorage is deleted; its capabilities map onto a `CredentialStore` implementation plus composition.
Today's `getApiKey` priority and its new home:
| AuthStorage today | New design |
|---|---|
| runtime override (CLI `--api-key`) | `withRuntimeOverrides(store, overrides)` decorator: `read` returns the override as an `ApiKeyCredential`; never persisted |
| stored `api_key` (with `$ENV`/`!command` via `resolveConfigValue`) | stored `ApiKeyCredential`; config-value resolution happens at `read` in coding-agent's adapter/decorator (command execution stays app policy) |
| stored `oauth` + locked refresh, undefined on failure | `getAuth` decision tree above; failure rejects with cause instead of silently unconfiguring |
| env var (only when nothing stored) | ambient branch of `apiKey.resolve` |
| `fallbackResolver` (models.json custom providers) | gone — custom providers carry their own `auth.apiKey` |
```txt
FileCredentialStore ports AuthStorage's lock backend: read = memory snapshot,
modify = withLockAsync(re-read, fn, merge-write), delete,
internal error recording (drainErrors equivalent)
└─ withConfigValues $ENV / !command at read
└─ withRuntimeOverrides --api-key
└─ createModels({ credentials: store })
login/logout UI provider.auth.{oauth,apiKey}.login(callbacks) + store.modify/delete
status UI store.read(pid) + getAuth try/catch ("needs /login" on rejection)
getOAuthProviders presence of provider.auth.oauth across registered providers
```
### Login callbacks
One interface serves api-key and OAuth login:
```ts
export interface AuthLoginCallbacks {
/** Aborts the whole login flow. Per-prompt cancellation uses AuthPrompt.signal. */
signal?: AbortSignal;
prompt(prompt: AuthPrompt): Promise<string>;
notify(event: AuthEvent): void;
}
/** `signal` lets the flow cancel a pending prompt when an out-of-band event resolves the step. */
export type AuthPrompt = { signal?: AbortSignal } & (
| { type: "text"; message: string; placeholder?: string }
| { type: "secret"; message: string; placeholder?: string }
| { type: "select"; message: string; options: readonly { id: string; label: string; description?: string }[] }
| { type: "manual_code"; message: string; placeholder?: string }
);
export type AuthEvent =
| { type: "auth_url"; url: string; instructions?: string }
| { type: "device_code"; userCode: string; verificationUri: string; intervalSeconds?: number; expiresInSeconds?: number }
| { type: "progress"; message: string };
```
`prompt()` returns the entered/selected string (`select` returns the option id). Flows race a `manual_code` prompt against a callback server by setting `AuthPrompt.signal` and aborting the prompt when the callback wins.
### OAuth attachment
Providers that support OAuth always attach it. There is no factory toggle: the flow is lazy-loaded, so advertising OAuth costs nothing until `login()`/`refresh()` actually runs, and a host that never logs in never loads it.
```ts
export function anthropicProvider(): Provider {
return createProvider({
id: "anthropic",
name: "Anthropic",
baseUrl: "https://api.anthropic.com/v1",
auth: {
apiKey: envApiKeyAuth("Anthropic API key", ["ANTHROPIC_API_KEY"]),
oauth: lazyOAuth({
name: "Anthropic (Claude Pro/Max)",
load: () => import("../utils/oauth/anthropic.ts").then((m) => m.anthropicOAuth),
}),
},
models: ANTHROPIC_MODELS,
api: anthropicMessagesApi(),
});
}
```
`lazyOAuth()` wraps a dynamically imported `OAuthAuth` so provider definitions can advertise OAuth without importing the implementation (`toAuth` is async for exactly this reason):
```ts
export function lazyOAuth(input: {
name: string;
load: () => Promise<OAuthAuth>;
}): OAuthAuth;
```
OAuth must not force Node-only code (`node:http`, `node:crypto`) into browser bundles: the dynamic import inside `lazyOAuth()` uses the same bundler-opaque variable-specifier trick as the bedrock lazy wrapper. Browser hosts never trigger the load (no stored node OAuth credentials, no login flow). If web OAuth lands later (sitegeist proved feasibility: Web Crypto PKCE, auth tab, fetch token exchange, device-code polling), it is just a different `OAuthAuth` implementation — no reserved option values.
The existing flows in `src/utils/oauth/` (anthropic, openai-codex, github-copilot) are adapted to `OAuthAuth` (`login`/`refresh`/`toAuth`, replacing `login`/`refreshToken`/`getApiKey`/`modifyModels`) with the new callbacks, staying Node-targeted and lazy-loaded. Copilot's `modifyModels` baseUrl rewriting becomes `toAuth` returning `ModelAuth.baseUrl`.
## Provider wrappers and models.json
`models.json` is a provider wrapper layer. It does not mutate providers in place:
```ts
function withProviderOverrides(base: Provider, overrides: ProviderOverrides): Provider {
return {
...base,
name: overrides.name ?? base.name,
baseUrl: overrides.baseUrl ?? base.baseUrl,
headers: mergeHeaders(base.headers, overrides.headers),
getModels: () => applyModelOverrides(base.getModels(), overrides.models),
refreshModels: base.refreshModels?.bind(base),
stream: base.stream,
streamSimple: base.streamSimple,
};
}
```
This composes with dynamic providers because `getModels()` delegates to the base source and `refreshModels()` passes through.
Request-auth config from models.json (`$ENV`, `!command`, inline keys) remains app-owned sidecar state, surfaced either as explicit request auth or as a custom `ApiKeyAuth` the app sets on the wrapped provider's `auth.apiKey`.
## Custom providers: createProvider()
One helper builds providers from parts; it handles both single-API and mixed-API providers:
```ts
export function createProvider(input: {
id: string;
name?: string; // default: id
baseUrl?: string;
headers?: Record<string, string>;
auth: ProviderAuth; // required, at least one of apiKey/oauth (no "no-auth" providers)
/** Initial model list (empty for purely dynamic providers). */
models: readonly Model<Api>[];
/** Dynamic providers: fetch the current list; createProvider stores it and dedupes in-flight calls. */
refreshModels?: () => Promise<readonly Model<Api>[]>;
/** Single implementation, or map keyed by model.api for mixed-API providers. */
api: ProviderStreams | Record<string, ProviderStreams>;
}): Provider;
```
- Single `api`: all models stream through it.
- Map `api`: `stream()`/`streamSimple()` dispatch on `model.api`; unknown api produces a stream error.
Mixed-API custom providers must be supported (opencode Go/Zen-style providers expose models backed by different APIs under one provider id).
Built-in provider factories use `createProvider()` internally. models.json custom providers map onto it directly:
```json
{
"providers": {
"my-openai-proxy": {
"api": "openai-completions",
"baseUrl": "https://proxy.example/v1",
"models": [ ... ]
}
}
}
```
## Compat entrypoint
`@earendil-works/pi-ai/compat` preserves the old global API surface until the coding-agent migration deletes it. New code never imports it.
Old semantics being preserved: global `stream()` can still dispatch by `model.api` through the legacy api-registry for custom providers, mutated models, and tests/extensions that override a built-in API implementation.
- `stream/complete/streamSimple/completeSimple(model, ctx, opts)`: real built-in provider/model/api matches route through a singleton `builtinModels()` collection, so provider auth/env/baseUrl behavior is shared with the new runtime. Unknown providers, mutated models, or overridden API registrations fall back to api-registry dispatch plus `getEnvApiKey` injection.
- The builtin api registration side effect moves from the root barrel into compat. It skips api ids that already have a registration, since compat may load after a test or extension has already registered an override. `registerApiProvider()/unregisterApiProviders()` keep feeding the compat-local registry; `resetApiProviders()` clears and re-registers builtins.
- Sync `getModel/getModels/getProviders` are deprecated aliases of `getBuiltinModel/getBuiltinModels/getBuiltinProviders` from `providers/all` (they were always pure generated-catalog reads — verified: nothing ever mutated the old `modelRegistry`).
- Re-exports the per-API lazy stream wrappers (incl. `setBedrockProviderModule`), `env-api-keys.ts`, and the image-generation registry/catalogs; none of these stay on the root barrel.
- `export * from "./index.ts"`: compat is a strict superset of the core entrypoint, so consumers switch a file's import path wholesale without symbol surgery.
coding-agent (and the interim agent package) switch imports of these symbols from `@earendil-works/pi-ai` to `@earendil-works/pi-ai/compat` (import-path-only change) and are otherwise untouched until the ModelManager migration.
Extension grace period: the coding-agent extension loader (jiti aliases + Bun `virtualModules`) resolves the `@earendil-works/pi-ai` ROOT specifier to the compat entrypoint. Existing user extensions using the old global API (`complete`, `getModel`, `registerApiProvider`, ...) keep working at runtime without changes; they break only when compat is removed at the ModelManager migration, with a migration guide in the changelog. Typechecking is the nudge: editors resolve the root to the slim core types, so extension sources that typecheck must import old globals from `/compat` — which is what the repo example extensions demonstrate.
## Builtin static helpers
Typed, sync, generated-catalog-only helpers live with the catalogs (exported from `providers/all`):
```ts
getBuiltinModel(provider, id) // sync, typed overloads from generated catalog
getBuiltinModels(provider) // sync
getBuiltinProviders() // sync
```
Runtime lookup through a `Models` instance is sync over the last-known provider lists: `models.getModel(...)`. Freshness-critical callers run `await models.refresh(provider)` first.
Generated catalogs are split per provider (`providers/<id>.models.ts`) by updating `packages/ai/scripts/generate-models.ts`. If the generator change turns out too large for this pass, splitting may be deferred; `providers/all` and provider factories may temporarily import the monolithic `models.generated.ts`, relying on `sideEffects: false` for pruning.
## Tree-shaking and lazy imports
Rules:
1. Main `@earendil-works/pi-ai` import is core-only.
2. Provider modules import their catalog, auth helpers, and lazy API wrappers only.
3. Lazy API wrappers dynamically import real API implementations.
4. Real API implementations import SDK dependencies.
5. OAuth implementations are always attached via `lazyOAuth()` and lazy-loaded behind a bundler-opaque dynamic import; provider metadata never eagerly imports Node-only OAuth code.
6. `providers/all` imports every built-in provider factory and all catalogs. It is the explicit heavy entrypoint.
7. Provider modules are side-effect-free; importing a provider does not register anything globally.
8. `package.json` lists only effectful compat/image registration files in `sideEffects`; root and provider modules stay tree-shakeable.
9. With code splitting, provider SDKs stay in lazy chunks. Without code splitting, bundlers fold statically reachable lazy API implementations into the single bundle; `providers/all` then pulls all statically visible SDKs. Bedrock is the exception because its AWS SDK implementation is behind a bundler-opaque Node-only import and needs `setBedrockProviderModule()` for standalone single-file bundles.
Exports map sketch:
```json
{
"exports": {
".": "./dist/index.js",
"./compat": "./dist/compat.js",
"./providers/all": "./dist/providers/all.js",
"./providers/openai": "./dist/providers/openai.js",
"./providers/anthropic": "./dist/providers/anthropic.js",
"./providers/*": "./dist/providers/*.js",
"./api/*": "./dist/api/*.js"
}
}
```
Browser smoke check (`scripts/check-browser-smoke.mjs`) must keep passing: bundling the core entrypoint (and any non-node provider entrypoint) must not pull `node:http`/`node:crypto`.
## AgentHarness integration
`AgentHarness` receives a `Models` instance.
- `AgentHarnessOptions.models` is required.
- The harness does not snapshot `Models` into turn state.
- Request path calls `this.models.streamSimple(model, context, options)`; same for compaction/branch-summarization paths.
- Request path never calls async `models.getModel()` to canonicalize; if model metadata needs refresh, the app updates the selected model before starting a turn.
- Harness tests build `createModels()` and install the faux provider (`fauxProvider()` factory from `providers/faux`).
## coding-agent next phase (not this pass)
coding-agent builds providers in layers and binds them per session:
```txt
built-in providers (builtinModels)
-> models.json provider wrappers / custom providers (createProvider)
-> extension provider wrappers/additions
```
```ts
sessionModels.clearProviders();
for (const provider of layeredProviders) sessionModels.setProvider(provider);
```
coding-agent owns: `FileCredentialStore` + decorators replacing AuthStorage (see "Replacing AuthStorage"), models.json auth sidecar (`$ENV`, `!command`), command execution policy, provider status labels (from `AuthResult.source`), login/logout UI (driving `auth.{apiKey,oauth}.login()` with `prompt()/notify()`), extension lifecycle, provider-management slash commands.
Current interim state:
- `AgentHarness` already accepts a `Models` instance and uses it for turn streaming, compaction, and branch summaries.
- coding-agent does not use `AgentHarness` yet; `AgentSession` still drives the low-level `Agent` with a `streamFn`.
- coding-agent still uses legacy `AuthStorage` + `ModelRegistry` and imports old global pi-ai APIs through `@earendil-works/pi-ai/compat`.
- The extension loader still aliases the pi-ai root to `/compat` as the runtime grace period for old extensions.
## Implementation TODOs
Check items off as they land. Keep this list current; it is the working state for resumed sessions.
### Phase 1 — core types/runtime
- [x] Rename `types.ts` `Provider` alias to `ProviderId`; fix call sites.
- [x] Add `ApiOptionsMap` and `ApiStreamOptions<TApi>` to `types.ts` (type-only imports).
- [x] New `models.ts`: `Provider<TApi>` interface, `hasApi()` guard, `ModelsError` + codes. Auth types live in `src/auth/types.ts` (`ProviderAuth` = `{ apiKey?, oauth? }`, credentials, `CredentialStore` (`read`/`modify`/`delete`, one credential per provider), `AuthResult`, `AuthContext`, `ModelAuth`, login callbacks), in-memory store in `src/auth/credential-store.ts`, default context in `src/auth/context.ts` (browser-safe node:fs trick), `lazyStream()` in `src/api/lazy.ts`.
- [x] `Models`/`MutableModels`/`createModels({ credentials?, authContext? })` with provider map, sync `getModel(s)` (per-provider failure isolation), explicit async `refresh(provider?)`, `getAuth` (decision tree, double-checked locked refresh), `stream/complete/streamSimple/completeSimple` with per-field auth merge. Tests: `packages/ai/test/models-runtime.test.ts`.
- [x] Keep metadata helpers: `calculateCost`, `getSupportedThinkingLevels`, `clampThinkingLevel`, `modelsAreEqual`.
### Phase 2 — `src/api/`
- [x] Move stream implementations from `src/providers/` to `src/api/`, renamed by API id (`anthropic.ts` -> `api/anthropic-messages.ts`, etc.).
- [x] Normalize each implementation module to export exactly `stream` and `streamSimple`.
- [x] Move shared helpers (`openai-responses-shared`, `google-shared`, `transform-messages`, `openai-prompt-cache`, `github-copilot-headers`, `cloudflare`, `simple-options`) to `src/api/`.
- [x] Extract `lazyStream()`/`lazyApi()` into `src/api/lazy.ts`.
- [x] Add `*.lazy.ts` wrappers per API; bedrock keeps node-only import trick and `setBedrockProviderModule()`.
- [x] Delete `providers/register-builtins.ts`. Interim until Phase 5 compat: builtin api-registry registration lives in `stream.ts`; lazy API wrappers are exported from the root barrel.
### Phase 3 — provider factories + catalogs
- [x] Auth helpers in `src/auth/helpers.ts`: `envApiKeyAuth()` (with secret-prompt `login`), `lazyOAuth()`. OAuth flow loads go through `utils/oauth/load.ts` (bundler-opaque dynamic import); the `OAuthAuth` exports it references land in Phase 4.
- [x] `createProvider()` in `models.ts` (single + mixed `api` map, dispatch on `model.api`, unknown api -> stream error).
- [x] Per-provider factories under `src/providers/` for all built-in catalog providers; OAuth attached via `lazyOAuth()` (anthropic, openai-codex, github-copilot); ambient `ApiKeyAuth` for amazon-bedrock (AWS env/profile) and google-vertex (key or ADC+project+location).
- [x] `providers/all.ts`: `builtinProviders()`, `builtinModels()`, `getBuiltinModel/getBuiltinModels/getBuiltinProviders` re-exports.
- [x] Faux provider factory (`fauxProvider()` in `providers/faux.ts`) for tests; legacy `registerFauxProvider()` kept until compat dies.
- [x] Split generated catalogs per provider via `scripts/generate-models.ts` (`providers/<id>.models.ts`); `models.generated.ts` becomes a generated aggregator.
### Phase 4 — OAuth adaptation
- [x] Adapt `utils/oauth/anthropic.ts`, `openai-codex.ts`, `github-copilot.ts` to `OAuthAuth` (`login`/`refresh`/`toAuth`) + `prompt()/notify()`; `modifyModels` baseUrl rewriting becomes `toAuth().baseUrl`. New exports (`anthropicOAuth`, `openaiCodexOAuth`, `githubCopilotOAuth`) sit next to the old `OAuthProviderInterface` objects, which survive until Phase 7.
- [x] No `usesCallbackServer` on `OAuthAuth`: callback-server flows race a `manual_code` prompt (aborted via `AuthPrompt.signal` once the flow settles). The old interface keeps its flag until it dies with compat.
### Phase 5 — packaging
- [x] `index.ts` core-only and side-effect free (no catalogs, no provider factories, no api-registry, no env-api-keys, no images, no OAuth, no compat). Typed catalog reads (`getBuiltin*`) implemented in `providers/all.ts`; `models.ts` no longer imports `models.generated.ts`.
- [x] `compat.ts`: superset of index + old api-dispatch globals, deprecated `getModel/getModels/getProviders` aliases, lazy api wrappers + `setBedrockProviderModule`, `getEnvApiKey`, images. Registration side effect lives here (skip-if-present).
- [x] Subpath exports map (`./compat`, `./providers/*`, `./api/*`); `sideEffects` array listing the effectful modules (`compat`, images registration) instead of `false`.
- [x] Browser smoke (entry now imports old globals from `/compat`) + shrinkwrap checks green. Internal old-global imports switched to `/compat` already (42 files in agent/coding-agent/examples; vitest configs alias `/compat` to src; spawn-CLI tests resolve workspace dist, so `packages/ai` + `packages/agent` dists were rebuilt).
### Phase 6 — AgentHarness
- [x] `AgentHarnessOptions.models` required (`readonly models` on the harness); the harness stream path uses `models.streamSimple()`. `StreamFn` redefined structurally (no compat type dependency); `Models.streamSimple` satisfies it.
- [x] Compaction/branch-summarization take the harness `Models` instance. `getApiKeyAndHeaders` is removed entirely — `Models` is the only auth path; per-request key resolution becomes provider auth on the collection. `compact()`/`generateSummary()`/`generateBranchSummary()` lose their explicit `apiKey`/`headers` parameters.
- [x] Harness tests use `createModels()` + `fauxProvider()` with unique per-fake provider ids; no global api-registry state, no unregister bookkeeping.
### Phase 7 — coding-agent bridge (minimal)
- [x] Switch old-global imports to `@earendil-works/pi-ai/compat` (landed with Phase 5; compat is a superset so the switch was path-only). Extension loader resolves the pi-ai root to compat as the runtime grace period.
- [x] Everything else originally sketched here is gated on coding-agent actually streaming through a `Models` instance — coding-agent's `AgentSession` drives the low-level `Agent` via `streamFn`, not the harness — and moved to Phase 9.
### Phase 8 — wrap-up
- [x] Update/add tests; run affected suites (tests landed with each phase; `./test.sh` green throughout).
- [x] `packages/ai/CHANGELOG.md`: `### Breaking Changes` with migration guide (compat entrypoint, `Provider` -> `ProviderId`, api module moves) + `### Added` for the new Models/provider/auth API.
- [x] `packages/coding-agent/CHANGELOG.md`: `### Changed` entry for extension authors — runtime unaffected (loader resolves the pi-ai root to compat), typecheck nudges to `/compat` or the new API; removal happens later with a migration guide.
- [x] `packages/agent/CHANGELOG.md`: `### Breaking Changes` for required `AgentHarnessOptions.models`, compaction signature changes, structural `StreamFn`.
- [x] `npm run check` clean.
### Phase 9 — coding-agent on Models + CredentialStore (in scope)
coding-agent replaces AuthStorage and ModelRegistry's internals with `FileCredentialStore` + a `MutableModels` collection. AgentSession itself stays (AgentHarness adoption is pi 2.0); only its model/auth substrate swaps. Layering is strictly one-directional:
```txt
FileCredentialStore (auth.json, locked, $ENV/!command resolution) + explicit --api-key overlay
↑
MutableModels: builtin factories (wrapped per models.json config) + custom providers (models.json ∪ extensions)
↑
ModelRegistry: compatibility facade — sync last-known reads delegate to the collection; registerProvider/login/logout/status for extensions + UI
↑
AgentSession / sdk / interactive-mode (stream via models; await only auth/refresh paths)
```
Decisions:
- `AuthStorage` is deleted as a type — it would otherwise depend on provider auth while provider auth depends on its store (circular). Its surface splits: `get`/`set`/`remove` -> `CredentialStore`; `getApiKey` -> `Models.getAuth`; `login`/`logout`/`getAuthStatus` -> ModelRegistry facade methods over `provider.auth.oauth` + the store.
- `FileCredentialStore` is self-contained (path, locking, parse/write, chmod, error buffering) and owns `auth.json` semantics, including `$ENV`/`!command` resolution for stored API-key credentials. Persisted values stay raw; resolution returns copies for auth use.
- Runtime `--api-key` overrides are an explicit store overlay (an override reads as an ephemeral stored api-key credential, masking stored OAuth — matches today's priority). Every registered provider is guaranteed an `apiKey` auth slot so overrides apply to OAuth-only providers too.
- `ModelRegistry.getAll`/`find`/`getAvailable` stay sync for SDK and extension compatibility, delegating to the collection's last-known sync model lists and fast configured-looking status checks. Dynamic providers update through explicit async `refresh()`, and request auth remains async through `getApiKeyAndHeaders()`/`Models.getAuth()`. Extensions also get the collection itself as the forward API.
- models.json keeps FULL feature parity, implemented as provider decoration: builtin factories wrapped so `getModels()` applies provider `baseUrl`/`compat` overlays, `modelOverrides`, and custom-model merges (async-safe); provider `apiKey`/`headers`/`authHeader` configs become that provider's `ApiKeyAuth` (config first, factory auth fallback); parse errors keep `getError()` semantics.
- Extension `ProviderConfig` parity: provider-keyed `streamSimple`, old-style `oauth` adapted to `OAuthAuth` (`modifyModels` -> `getModels` wrap + `toAuth`), full model replacement per provider. Legacy `registerApiProvider` writes stay compat-local for consumers that call global `complete()`; they die with compat.
- Copilot: stored-credential baseUrl applied in the wrapped `getModels()` (extension-visible models stay correct) plus per-request `toAuth().baseUrl`.
- Cloudflare: provider-auth substitution (key + `CLOUDFLARE_ACCOUNT_ID`/`CLOUDFLARE_GATEWAY_ID` from credential `env` or ambient `AuthContext.env()` -> `ModelAuth.baseUrl`). Built-in compat calls route through `Models`, so they use the same provider auth path.
Ordering for new sessions:
1. [x] pi-ai rework first: `Provider.getModels()` sync + optional `refreshModels()`; `Models.getModels`/`getModel` sync, `Models.refresh(provider?)` async; `createProvider` takes `models` array + optional `refreshModels` fetcher (in-flight dedupe). Reverses Phase 1's async-listing decision — see "Provider model listing" for rationale (sync-or-async unions breed latent sync assumptions; async-only breaks sync consumer surfaces like extension `find`/`getAll`).
2. [x] Cloudflare provider auth in pi-ai factories: Workers AI and AI Gateway validate their required account/gateway env/config and return resolved `baseUrl`, provider-scoped env, and header suppression/override metadata from provider auth.
3. [ ] Add `FileCredentialStore` in coding-agent.
- Implement the pi-ai `CredentialStore` interface as a self-contained `auth.json` store; do not depend on the old `AuthStorageBackend` abstraction, though its lock/retry semantics may be ported.
- Preserve the existing file format. `ApiKeyCredential` uses `{ type: "api_key", key?, env? }`, matching today's `auth.json`; do not translate `env` into metadata or rewrite discriminators.
- Resolve `$ENV`/`!command` in stored API-key `key` and `env` values out of the box using an injected execution/config environment. `$ENV` lookup should come from that environment, and `!command` should run through the shared shell execution path rather than direct `execSync`.
- Persist raw config values; resolved credentials returned for auth use must be copies and must not rewrite `$ENV`/`!command` strings unless a caller explicitly stores new values.
- `read(provider)` returns the current credential snapshot and records parse/storage errors for status UI parity.
- `modify(provider, fn)` must lock, re-read, run `fn`, merge-write the provider entry, chmod `0600`, and return the post-write credential.
- `delete(provider)` must lock and remove only that provider's entry.
- Add file-backed and in-memory tests covering lock/RMW behavior, `api_key` reads with config-value resolution, OAuth reads, provider `env` preservation, delete, parse errors, and concurrent refresh-style modifications.
4. [ ] Add runtime override overlay for coding-agent policy.
- `withRuntimeOverrides(store, overrides)` implements CLI `--api-key`: read returns an ephemeral `{ type: "api_key", key }` for each overridden provider, masking stored OAuth/API credentials without persisting.
- Runtime overrides must apply even to OAuth-capable providers; every provider registered in coding-agent must retain or gain an `apiKey` auth slot so the overlay is meaningful.
- Tests cover precedence: runtime override > stored credential > models.json config auth > ambient provider env, with stored credential blocking ambient fallback.
5. [ ] Build provider decoration helpers for `models.json`.
- Start from built-in provider factories, not generated model arrays.
- Wrap provider `getModels()` so provider-level `baseUrl`/`headers`/`compat`, per-model `modelOverrides`, and custom model merges apply on every sync read.
- Preserve `refreshModels()` passthrough so dynamic providers compose with decorations.
- Convert provider `apiKey`/`headers`/`authHeader` models.json config into a wrapped `ApiKeyAuth` that resolves config values first and falls back to the base provider auth.
- Custom providers with `models` use `createProvider()` with the appropriate lazy API wrapper or extension-provided stream implementation.
- Parse errors must keep current `ModelRegistry.getError()` behavior: built-ins remain available, and the error is visible.
6. [ ] Copilot `getModels()` baseUrl wrap.
- GitHub Copilot OAuth `toAuth()` already returns per-credential request `baseUrl` for streaming.
- Wrap Copilot's provider `getModels()` when an OAuth credential is present so extension/UI-visible model metadata also carries the authenticated account base URL.
- Keep API-key/env-token Copilot behavior unchanged.
- Add tests for model metadata before login, after OAuth credential, after refresh/baseUrl change, and logout.
7. [ ] Extension OAuth adapter.
- Adapt old extension `OAuthProviderInterface` configs to pi-ai `OAuthAuth`.
- `login` maps old callbacks/events to `prompt()/notify()`.
- `refreshToken` maps to `refresh`.
- `getApiKey` maps to `toAuth`.
- `modifyModels` becomes a provider `getModels()` wrapper plus `toAuth().baseUrl` where applicable.
- Preserve existing extension runtime compatibility through the `/compat` alias until Phase 10.
8. [ ] Rebuild coding-agent `ModelRegistry` over `MutableModels`.
- It owns a `MutableModels` instance built from decorated built-ins + models.json custom providers + extension providers.
- `getAll()`, `find()`, and `getAvailable()` remain sync compatibility methods over last-known model lists and fast configured-looking auth status. Do not break the extension-facing `modelRegistry` surface for these reads.
- `refresh()` is the explicit async freshness boundary: rebuild provider layers and call `models.refresh()` where needed; no global api-registry reset should be part of the new path except compat-only grace behavior.
- `registerProvider()`/`unregisterProvider()` mutate provider layers and rebuild the collection.
- Facade auth ops (`login`, `logout`, provider status, available OAuth providers) drive `provider.auth.{apiKey,oauth}` and the `CredentialStore`; no `AuthStorage` type remains.
- Legacy `registerApiProvider` writes stay only for `/compat` callers and are removed in Phase 10.
9. [ ] Rewire consumers.
- `AgentSession` stream function resolves through `ModelRegistry`/`Models`, not `getApiKeyAndHeaders()` + compat globals.
- SDK options replace `authStorage` with `credentials?: CredentialStore` or an agent-dir-backed default; update `sdk.md` and examples.
- `model-resolver`, `--list-models`, model selector, login/logout/status UI, and provider attribution use sync last-known model reads and await only explicit refresh/auth operations.
- CLI `--api-key` populates the runtime override decorator instead of mutating `AuthStorage`.
- Keep extension loader root-to-compat alias until Phase 10, but expose the new collection/facade as the forward API.
10. [ ] Test migration and real-provider validation.
- Unit tests for `FileCredentialStore`, runtime override overlay, provider decoration, extension OAuth adapter, Models-backed ModelRegistry facade, and consumer rewiring.
- Regression tests for Cloudflare account/gateway env, Copilot OAuth baseUrl wrapping, runtime `--api-key` precedence, `$ENV`/`!command` resolution, and stored credential blocking ambient fallback.
- Update existing tests for sync last-known `ModelRegistry.getAll/find/getAvailable` plus explicit async refresh behavior.
- Run targeted non-e2e suites plus tmux validation of login flows against real providers (Anthropic OAuth/API key, OpenAI Codex OAuth, GitHub Copilot OAuth, Cloudflare AI Gateway, Bedrock if credentials are available).
### Phase 10 — compat deletion (pi 2.0 era, separate)
- [ ] AgentSession -> AgentHarness; the registry facade dies in favor of harness `Models`.
- [ ] Move ALL internal `/compat` imports to the new API: every package's src, all tests, and the example extensions (examples then demonstrate the new API). Nothing inside the repo may import `/compat` at that point.
- [ ] Delete `/compat`, `env-api-keys.ts`, the extension-loader root-to-compat alias, the old `pi-ai/oauth` registry and `OAuthProviderInterface` (incl. `usesCallbackServer`), and the compat-local legacy API registry. This is the extension-author breaking release; changelog carries the migration guide.
### Deferred / follow-ups
- [ ] Web OAuth implementations (sitegeist-style) as an alternative `OAuthAuth`.
- [x] Images API redesign: `ImagesModels`/`ImagesProvider`/`createImagesProvider` mirror the chat-side design (sync reads, explicit refresh, never-reject generation); auth resolution shared with the chat side via the free-standing `resolveProviderAuth()` in `auth/resolve.ts` (which also owns `ModelsError`; both collections pass their store/context as arguments — no resolver object). `openrouterImagesProvider()` factory + `builtinImagesProviders()`/`builtinImagesModels()` in `providers/all`; impl moved to `api/openrouter-images.ts` with a lazy wrapper. The old global image API (registry + `getImageModel*` + `generateImages`) stays on compat; `ImagesProvider` id alias in types.ts renamed to `ImagesProviderId` (mirror of `Provider` -> `ProviderId`).
## Error behavior
`undefined` means not found or not configured. Real failures reject or become stream errors.
```ts
export type ModelsErrorCode =
| "model_source" // provider model refresh failed
| "model_validation" // model object invalid
| "provider" // unknown provider, dispatch failure
| "stream" // stream setup failure
| "auth" // auth resolution failure
| "oauth"; // oauth login/refresh failure
```
- `Models.stream()` produces stream errors (error event + error result) for async setup failures; it does not throw after returning the stream.
- `Models.getModels()` is a sync best-effort read: a provider whose `getModels()` throws yields no models. `Models.refresh(provider)` rejects on that provider's fetch failure; `Models.refresh()` (all providers) is concurrent best-effort. Apps that need a concrete listing failure refresh the single provider.
- Auth resolution and credential store failures reject loudly (`ModelsError` codes `auth`/`oauth`); silent fallback to a different auth path after a failure risks billing surprises. A stored credential always blocks ambient/env fallback, including after a failed refresh.
- Status/availability UIs catch `getAuth` rejections and render "needs re-login"; they do not treat rejection as "unconfigured".
packages/agent/package.json
+4 -4
View File
@@ -1,6 +1,6 @@
{
"name": "@earendil-works/pi-agent-core",
"version": "0.79.6",
"version": "0.80.2",
"description": "General-purpose agent with transport abstraction, state management, and attachment support",
"type": "module",
"main": "./dist/index.js",
@@ -29,7 +29,7 @@
"prepublishOnly": "npm run clean && npm run build"
},
"dependencies": {
"@earendil-works/pi-ai": "^0.79.6",
"@earendil-works/pi-ai": "^0.80.2",
"ignore": "7.0.5",
"typebox": "1.1.38",
"yaml": "2.9.0"
@@ -53,8 +53,8 @@
},
"devDependencies": {
"@types/node": "24.12.4",
"@vitest/coverage-v8": "3.2.4",
"@vitest/coverage-v8": "4.1.9",
"typescript": "5.9.3",
"vitest": "3.2.4"
"vitest": "4.1.9"
}
}
packages/agent/src/agent-loop.ts
+1 -1
View File
@@ -10,7 +10,7 @@ import {
streamSimple,
type ToolResultMessage,
validateToolArguments,
} from "@earendil-works/pi-ai";
} from "@earendil-works/pi-ai/compat";
import type {
AgentContext,
AgentEvent,
packages/agent/src/agent.ts
+1 -1
View File
@@ -7,7 +7,7 @@ import {
type TextContent,
type ThinkingBudgets,
type Transport,
} from "@earendil-works/pi-ai";
} from "@earendil-works/pi-ai/compat";
import { runAgentLoop, runAgentLoopContinue } from "./agent-loop.ts";
import type {
AfterToolCallContext,
packages/agent/src/harness/agent-harness.ts
+7 -42
View File
@@ -1,10 +1,4 @@
import {
type AssistantMessage,
type ImageContent,
type Model,
streamSimple,
type UserMessage,
} from "@earendil-works/pi-ai";
import type { AssistantMessage, ImageContent, Model, Models, UserMessage } from "@earendil-works/pi-ai";
import { runAgentLoop } from "../agent-loop.ts";
import type {
AgentContext,
@@ -75,17 +69,6 @@ function cloneStreamOptions(streamOptions?: AgentHarnessStreamOptions): AgentHar
};
}
function mergeHeaders(...headers: Array<Record<string, string> | undefined>): Record<string, string> | undefined {
const merged: Record<string, string> = {};
let hasHeaders = false;
for (const entry of headers) {
if (!entry) continue;
Object.assign(merged, entry);
hasHeaders = true;
}
return hasHeaders ? merged : undefined;
}
function findDuplicateNames(names: string[]): string[] {
const seen = new Set<string>();
const duplicates = new Set<string>();
@@ -178,6 +161,7 @@ export class AgentHarness<
> {
readonly env: ExecutionEnv;
private session: Session;
readonly models: Models;
private phase: AgentHarnessPhase = "idle";
private runAbortController?: AbortController;
private runPromise?: Promise<void>;
@@ -186,7 +170,6 @@ export class AgentHarness<
private thinkingLevel: ThinkingLevel;
private systemPrompt: AgentHarnessOptions<TSkill, TPromptTemplate, TTool>["systemPrompt"];
private streamOptions: AgentHarnessStreamOptions;
private getApiKeyAndHeaders?: AgentHarnessOptions["getApiKeyAndHeaders"];
private resources: AgentHarnessResources<TSkill, TPromptTemplate>;
private tools = new Map<string, TTool>();
private activeToolNames: string[];
@@ -200,10 +183,10 @@ export class AgentHarness<
constructor(options: AgentHarnessOptions<TSkill, TPromptTemplate, TTool>) {
this.env = options.env;
this.session = options.session;
this.models = options.models;
this.resources = options.resources ?? {};
this.streamOptions = cloneStreamOptions(options.streamOptions);
this.systemPrompt = options.systemPrompt;
this.getApiKeyAndHeaders = options.getApiKeyAndHeaders;
this.validateUniqueNames(
(options.tools ?? []).map((tool) => tool.name),
"Duplicate tool name(s)",
@@ -376,13 +359,9 @@ export class AgentHarness<
private createStreamFn(getTurnState: () => AgentHarnessTurnState<TSkill, TPromptTemplate, TTool>): StreamFn {
return async (model, context, streamOptions) => {
const turnState = getTurnState();
const auth = await this.getApiKeyAndHeaders?.(model);
const snapshotOptions: AgentHarnessStreamOptions = {
...turnState.streamOptions,
headers: mergeHeaders(turnState.streamOptions.headers, auth?.headers),
};
const snapshotOptions: AgentHarnessStreamOptions = { ...turnState.streamOptions };
const requestOptions = await this.emitBeforeProviderRequest(model, turnState.sessionId, snapshotOptions);
return streamSimple(model, context, {
return this.models.streamSimple(model, context, {
cacheRetention: requestOptions.cacheRetention,
headers: requestOptions.headers,
maxRetries: requestOptions.maxRetries,
@@ -401,7 +380,6 @@ export class AgentHarness<
sessionId: turnState.sessionId,
timeoutMs: requestOptions.timeoutMs,
transport: requestOptions.transport,
apiKey: auth?.apiKey,
});
};
}
@@ -713,8 +691,6 @@ export class AgentHarness<
try {
const model = this.model;
if (!model) throw new AgentHarnessError("invalid_state", "No model set for compaction");
const auth = await this.getApiKeyAndHeaders?.(model);
if (!auth) throw new AgentHarnessError("auth", "No auth available for compaction");
const branchEntries = await this.session.getBranch();
const preparationResult = prepareCompaction(branchEntries, DEFAULT_COMPACTION_SETTINGS);
if (!preparationResult.ok) throw preparationResult.error;
@@ -731,15 +707,7 @@ export class AgentHarness<
const provided = hookResult?.compaction;
const compactResult = provided
? { ok: true as const, value: provided }
: await compact(
preparation,
model,
auth.apiKey,
auth.headers,
customInstructions,
undefined,
this.thinkingLevel,
);
: await compact(preparation, this.models, model, customInstructions, undefined, this.thinkingLevel);
if (!compactResult.ok) throw compactResult.error;
const result = compactResult.value;
const entryId = await this.session.appendCompaction(
@@ -792,12 +760,9 @@ export class AgentHarness<
if (!summaryText && options?.summarize && entries.length > 0) {
const model = this.model;
if (!model) throw new AgentHarnessError("invalid_state", "No model set for branch summary");
const auth = await this.getApiKeyAndHeaders?.(model);
if (!auth) throw new AgentHarnessError("auth", "No auth available for branch summary");
const branchSummary = await generateBranchSummary(entries, {
models: this.models,
model,
apiKey: auth.apiKey,
headers: auth.headers,
signal: new AbortController().signal,
customInstructions: hookResult?.customInstructions ?? options?.customInstructions,
replaceInstructions: hookResult?.replaceInstructions ?? options?.replaceInstructions,
packages/agent/src/harness/compaction/branch-summarization.ts
+7 -9
View File
@@ -1,5 +1,5 @@
import type { Model } from "@earendil-works/pi-ai";
import { completeSimple } from "@earendil-works/pi-ai";
import type { Model, Models } from "@earendil-works/pi-ai";
import type { AgentMessage } from "../../types.ts";
import {
convertToLlm,
@@ -49,12 +49,10 @@ export interface CollectEntriesResult {
/** Options for generating a branch summary. */
export interface GenerateBranchSummaryOptions {
/** Provider collection the summarization request goes through; owns auth resolution. */
models: Models;
/** Model used for summarization. */
model: Model<any>;
/** API key forwarded to the provider. */
apiKey: string;
/** Optional request headers forwarded to the provider. */
headers?: Record<string, string>;
/** Abort signal for the summarization request. */
signal: AbortSignal;
/** Optional instructions appended to or replacing the default prompt. */
@@ -202,7 +200,7 @@ export async function generateBranchSummary(
entries: SessionTreeEntry[],
options: GenerateBranchSummaryOptions,
): Promise<Result<BranchSummaryResult, BranchSummaryError>> {
const { model, apiKey, headers, signal, customInstructions, replaceInstructions, reserveTokens = 16384 } = options;
const { models, model, signal, customInstructions, replaceInstructions, reserveTokens = 16384 } = options;
const contextWindow = model.contextWindow || 128000;
const tokenBudget = contextWindow - reserveTokens;
@@ -230,10 +228,10 @@ export async function generateBranchSummary(
timestamp: Date.now(),
},
];
const response = await completeSimple(
const response = await models.completeSimple(
model,
{ systemPrompt: SUMMARIZATION_SYSTEM_PROMPT, messages: summarizationMessages },
{ apiKey, headers, signal, maxTokens: 2048 },
{ signal, maxTokens: 2048 },
);
if (response.stopReason === "aborted") {
return err(new BranchSummaryError("aborted", response.errorMessage || "Branch summary aborted"));
packages/agent/src/harness/compaction/compaction.ts
+20 -29
View File
@@ -1,5 +1,4 @@
import type { AssistantMessage, ImageContent, Model, TextContent, Usage } from "@earendil-works/pi-ai";
import { completeSimple } from "@earendil-works/pi-ai";
import type { AssistantMessage, ImageContent, Model, Models, TextContent, Usage } from "@earendil-works/pi-ai";
import type { AgentMessage, ThinkingLevel } from "../../types.ts";
import {
convertToLlm,
@@ -122,14 +121,19 @@ export function calculateContextTokens(usage: Usage): number {
function getAssistantUsage(msg: AgentMessage): Usage | undefined {
if (msg.role === "assistant" && "usage" in msg) {
const assistantMsg = msg as AssistantMessage;
if (assistantMsg.stopReason !== "aborted" && assistantMsg.stopReason !== "error" && assistantMsg.usage) {
if (
assistantMsg.stopReason !== "aborted" &&
assistantMsg.stopReason !== "error" &&
assistantMsg.usage &&
calculateContextTokens(assistantMsg.usage) > 0
) {
return assistantMsg.usage;
}
}
return undefined;
}
/** Return usage from the last successful assistant message in session entries. */
/** Return usage from the last valid assistant message in session entries. */
export function getLastAssistantUsage(entries: SessionTreeEntry[]): Usage | undefined {
for (let i = entries.length - 1; i >= 0; i--) {
const entry = entries[i];
@@ -455,10 +459,9 @@ Keep each section concise. Preserve exact file paths, function names, and error
/** Generate or update a conversation summary for compaction. */
export async function generateSummary(
currentMessages: AgentMessage[],
models: Models,
model: Model<any>,
reserveTokens: number,
apiKey: string,
headers?: Record<string, string>,
signal?: AbortSignal,
customInstructions?: string,
previousSummary?: string,
@@ -490,10 +493,10 @@ export async function generateSummary(
const completionOptions =
model.reasoning && thinkingLevel && thinkingLevel !== "off"
? { maxTokens, signal, apiKey, headers, reasoning: thinkingLevel }
: { maxTokens, signal, apiKey, headers };
? { maxTokens, signal, reasoning: thinkingLevel }
: { maxTokens, signal };
const response = await completeSimple(
const response = await models.completeSimple(
model,
{ systemPrompt: SUMMARIZATION_SYSTEM_PROMPT, messages: summarizationMessages },
completionOptions,
@@ -626,9 +629,8 @@ export { serializeConversation } from "./utils.ts";
/** Generate compaction summary data from prepared session history. */
export async function compact(
preparation: CompactionPreparation,
models: Models,
model: Model<any>,
apiKey: string,
headers?: Record<string, string>,
customInstructions?: string,
signal?: AbortSignal,
thinkingLevel?: ThinkingLevel,
@@ -655,25 +657,16 @@ export async function compact(
messagesToSummarize.length > 0
? generateSummary(
messagesToSummarize,
models,
model,
settings.reserveTokens,
apiKey,
headers,
signal,
customInstructions,
previousSummary,
thinkingLevel,
)
: Promise.resolve(ok<string, CompactionError>("No prior history.")),
generateTurnPrefixSummary(
turnPrefixMessages,
model,
settings.reserveTokens,
apiKey,
headers,
signal,
thinkingLevel,
),
generateTurnPrefixSummary(turnPrefixMessages, models, model, settings.reserveTokens, signal, thinkingLevel),
]);
if (!historyResult.ok) return err(historyResult.error);
if (!turnPrefixResult.ok) return err(turnPrefixResult.error);
@@ -681,10 +674,9 @@ export async function compact(
} else {
const summaryResult = await generateSummary(
messagesToSummarize,
models,
model,
settings.reserveTokens,
apiKey,
headers,
signal,
customInstructions,
previousSummary,
@@ -706,10 +698,9 @@ export async function compact(
}
async function generateTurnPrefixSummary(
messages: AgentMessage[],
models: Models,
model: Model<any>,
reserveTokens: number,
apiKey: string,
headers?: Record<string, string>,
signal?: AbortSignal,
thinkingLevel?: ThinkingLevel,
): Promise<Result<string, CompactionError>> {
@@ -728,12 +719,12 @@ async function generateTurnPrefixSummary(
},
];
const response = await completeSimple(
const response = await models.completeSimple(
model,
{ systemPrompt: SUMMARIZATION_SYSTEM_PROMPT, messages: summarizationMessages },
model.reasoning && thinkingLevel && thinkingLevel !== "off"
? { maxTokens, signal, apiKey, headers, reasoning: thinkingLevel }
: { maxTokens, signal, apiKey, headers },
? { maxTokens, signal, reasoning: thinkingLevel }
: { maxTokens, signal },
);
if (response.stopReason === "aborted") {
return err(new CompactionError("aborted", response.errorMessage || "Turn prefix summarization aborted"));
packages/agent/src/harness/env/nodejs.ts
Vendored
+33 -11
View File
@@ -144,12 +144,25 @@ async function findBashOnPath(): Promise<string | null> {
return firstMatch && (await pathExists(firstMatch)) ? firstMatch : null;
}
async function getShellConfig(
customShellPath?: string,
): Promise<Result<{ shell: string; args: string[] }, ExecutionError>> {
interface ShellConfig {
shell: string;
args: string[];
commandTransport?: "argv" | "stdin";
}
function isLegacyWslBashPath(path: string): boolean {
const normalized = path.replace(/\//g, "\\").toLowerCase();
return /^[a-z]:\\windows\\(?:system32|sysnative)\\bash\.exe$/.test(normalized);
}
function getBashShellConfig(shell: string): ShellConfig {
return isLegacyWslBashPath(shell) ? { shell, args: ["-s"], commandTransport: "stdin" } : { shell, args: ["-c"] };
}
async function getShellConfig(customShellPath?: string): Promise<Result<ShellConfig, ExecutionError>> {
if (customShellPath) {
if (await pathExists(customShellPath)) {
return ok({ shell: customShellPath, args: ["-c"] });
return ok(getBashShellConfig(customShellPath));
}
return err(new ExecutionError("shell_unavailable", `Custom shell path not found: ${customShellPath}`));
}
@@ -161,22 +174,22 @@ async function getShellConfig(
if (programFilesX86) candidates.push(`${programFilesX86}\\Git\\bin\\bash.exe`);
for (const candidate of candidates) {
if (await pathExists(candidate)) {
return ok({ shell: candidate, args: ["-c"] });
return ok(getBashShellConfig(candidate));
}
}
const bashOnPath = await findBashOnPath();
if (bashOnPath) {
return ok({ shell: bashOnPath, args: ["-c"] });
return ok(getBashShellConfig(bashOnPath));
}
return err(new ExecutionError("shell_unavailable", "No bash shell found"));
}
if (await pathExists("/bin/bash")) {
return ok({ shell: "/bin/bash", args: ["-c"] });
return ok(getBashShellConfig("/bin/bash"));
}
const bashOnPath = await findBashOnPath();
if (bashOnPath) {
return ok({ shell: bashOnPath, args: ["-c"] });
return ok(getBashShellConfig(bashOnPath));
}
return ok({ shell: "sh", args: ["-c"] });
}
@@ -274,13 +287,22 @@ export class NodeExecutionEnv implements ExecutionEnv {
};
try {
child = spawn(shellConfig.value.shell, [...shellConfig.value.args, command], {
const commandFromStdin = shellConfig.value.commandTransport === "stdin";
child = spawn(
shellConfig.value.shell,
commandFromStdin ? shellConfig.value.args : [...shellConfig.value.args, command],
{
cwd,
detached: process.platform !== "win32",
env: getShellEnv(this.shellEnv, options?.env),
stdio: ["ignore", "pipe", "pipe"],
stdio: [commandFromStdin ? "pipe" : "ignore", "pipe", "pipe"],
windowsHide: true,
});
},
);
if (commandFromStdin) {
child.stdin?.on("error", () => {});
child.stdin?.end(command);
}
} catch (error) {
const cause = toError(error);
settle(err(new ExecutionError("spawn_error", cause.message, cause)));
packages/agent/src/harness/session/session.ts
+2 -1
View File
@@ -234,12 +234,13 @@ export class Session<TMetadata extends SessionMetadata = SessionMetadata> {
}
async appendSessionName(name: string): Promise<string> {
const sanitizedName = name.replace(/[\r\n]+/g, " ").trim();
return this.appendTypedEntry({
type: "session_info",
id: await this.storage.createEntryId(),
parentId: await this.storage.getLeafId(),
timestamp: new Date().toISOString(),
name: name.trim(),
name: sanitizedName,
} satisfies SessionInfoEntry);
}
packages/agent/src/harness/types.ts
+24 -21
View File
@@ -1,4 +1,4 @@
import type { ImageContent, Model, SimpleStreamOptions, TextContent, Transport } from "@earendil-works/pi-ai";
import type { ImageContent, Model, Models, SimpleStreamOptions, TextContent, Transport } from "@earendil-works/pi-ai";
import type { AgentEvent, AgentMessage, AgentTool, QueueMode, ThinkingLevel } from "../index.ts";
import type { Session } from "./session/session.ts";
@@ -240,22 +240,6 @@ export interface FileInfo {
mtimeMs: number;
}
/** Options for {@link Shell.exec}. */
export interface ExecutionEnvExecOptions {
/** Working directory for the command. Relative paths are resolved against {@link ExecutionEnv.cwd}. Defaults to {@link ExecutionEnv.cwd}. */
cwd?: string;
/** Additional environment variables for the command. Values override the environment defaults. Defaults to no overrides. */
env?: Record<string, string>;
/** Timeout in seconds. Implementations should return a timeout error when the command exceeds this duration. Defaults to no timeout. */
timeout?: number;
/** Abort signal used to terminate the command. Defaults to no abort signal. */
abortSignal?: AbortSignal;
/** Called with stdout chunks as they are produced. */
onStdout?: (chunk: string) => void;
/** Called with stderr chunks as they are produced. */
onStderr?: (chunk: string) => void;
}
/**
* Filesystem capability used by the harness.
*
@@ -317,12 +301,28 @@ export interface FileSystem {
cleanup(): Promise<void>;
}
/** Options for {@link Shell.exec}. */
export interface ShellExecOptions {
/** Working directory for the command. Relative paths are resolved against {@link ExecutionEnv.cwd}. Defaults to {@link ExecutionEnv.cwd}. */
cwd?: string;
/** Additional environment variables for the command. Values override the environment defaults. Defaults to no overrides. */
env?: Record<string, string>;
/** Timeout in seconds. Implementations should return a timeout error when the command exceeds this duration. Defaults to no timeout. */
timeout?: number;
/** Abort signal used to terminate the command. Defaults to no abort signal. */
abortSignal?: AbortSignal;
/** Called with stdout chunks as they are produced. */
onStdout?: (chunk: string) => void;
/** Called with stderr chunks as they are produced. */
onStderr?: (chunk: string) => void;
}
/** Shell execution capability used by the harness. */
export interface Shell {
/** Execute a shell command in {@link FileSystem.cwd} unless `options.cwd` is provided. */
exec(
command: string,
options?: ExecutionEnvExecOptions,
options?: ShellExecOptions,
): Promise<Result<{ stdout: string; stderr: string; exitCode: number }, ExecutionError>>;
/** Release shell resources. Must be best-effort and must not throw or reject. */
cleanup(): Promise<void>;
@@ -802,6 +802,12 @@ export interface AgentHarnessOptions<
> {
env: ExecutionEnv;
session: Session;
/**
* Provider collection used for all model requests (turn streaming,
* compaction, branch summarization). Auth resolves through the providers'
* auth.
*/
models: Models;
tools?: TTool[];
/**
* Concrete resources available to explicit invocation methods and system-prompt callbacks.
@@ -818,9 +824,6 @@ export interface AgentHarnessOptions<
activeTools: TTool[];
resources: AgentHarnessResources<TSkill, TPromptTemplate>;
}) => string | Promise<string>);
getApiKeyAndHeaders?: (
model: Model<any>,
) => Promise<{ apiKey: string; headers?: Record<string, string> } | undefined>;
/** Curated stream/provider request options. Snapshotted at turn start. */
streamOptions?: AgentHarnessStreamOptions;
model: Model<any>;
packages/agent/src/harness/utils/shell-output.ts
+2 -10
View File
@@ -1,15 +1,7 @@
import {
type ExecutionEnv,
type ExecutionEnvExecOptions,
ExecutionError,
err,
ok,
type Result,
toError,
} from "../types.ts";
import { type ExecutionEnv, ExecutionError, err, ok, type Result, type ShellExecOptions, toError } from "../types.ts";
import { DEFAULT_MAX_BYTES, truncateTail } from "./truncate.ts";
export interface ShellCaptureOptions extends Omit<ExecutionEnvExecOptions, "onStdout" | "onStderr"> {
export interface ShellCaptureOptions extends Omit<ShellExecOptions, "onStdout" | "onStderr"> {
onChunk?: (chunk: string) => void;
}
packages/agent/src/types.ts
+9 -4
View File
@@ -1,11 +1,13 @@
import type {
Api,
AssistantMessage,
AssistantMessageEvent,
AssistantMessageEventStream,
Context,
ImageContent,
Message,
Model,
SimpleStreamOptions,
streamSimple,
TextContent,
Tool,
ToolResultMessage,
@@ -13,7 +15,8 @@ import type {
import type { Static, TSchema } from "typebox";
/**
* Stream function used by the agent loop.
* Stream function used by the agent loop. `Models.streamSimple` satisfies
* this shape.
*
* Contract:
* - Must not throw or return a rejected promise for request/model/runtime failures.
@@ -22,8 +25,10 @@ import type { Static, TSchema } from "typebox";
* final AssistantMessage with stopReason "error" or "aborted" and errorMessage.
*/
export type StreamFn = (
...args: Parameters<typeof streamSimple>
) => ReturnType<typeof streamSimple> | Promise<ReturnType<typeof streamSimple>>;
model: Model<Api>,
context: Context,
options?: SimpleStreamOptions,
) => AssistantMessageEventStream | Promise<AssistantMessageEventStream>;
/**
* Configuration for how tool calls from a single assistant message are executed.
packages/agent/test/agent.test.ts
+1 -1
View File
@@ -1,4 +1,4 @@
import { type AssistantMessage, type AssistantMessageEvent, EventStream, getModel } from "@earendil-works/pi-ai";
import { type AssistantMessage, type AssistantMessageEvent, EventStream, getModel } from "@earendil-works/pi-ai/compat";
import { Type } from "typebox";
import { describe, expect, it } from "vitest";
import { Agent, type AgentEvent, type AgentTool, type AgentToolUpdateCallback } from "../src/index.ts";
packages/agent/test/e2e.test.ts
+1 -1
View File
@@ -9,7 +9,7 @@ import {
registerFauxProvider,
type ToolResultMessage,
type UserMessage,
} from "@earendil-works/pi-ai";
} from "@earendil-works/pi-ai/compat";
import { afterEach, describe, expect, it } from "vitest";
import { Agent, type AgentEvent } from "../src/index.ts";
import { calculateTool } from "./utils/calculate.ts";
packages/agent/test/harness/agent-harness-stream.test.ts
+27 -20
View File
@@ -1,18 +1,27 @@
import { fauxAssistantMessage, fauxToolCall, registerFauxProvider, type StreamOptions } from "@earendil-works/pi-ai";
import { afterEach, describe, expect, it } from "vitest";
import {
createModels,
type FauxProviderHandle,
fauxAssistantMessage,
fauxProvider,
fauxToolCall,
type StreamOptions,
} from "@earendil-works/pi-ai";
import { describe, expect, it } from "vitest";
import { AgentHarness } from "../../src/harness/agent-harness.ts";
import { NodeExecutionEnv } from "../../src/harness/env/nodejs.ts";
import { InMemorySessionStorage } from "../../src/harness/session/memory-storage.ts";
import { Session } from "../../src/harness/session/session.ts";
import { calculateTool } from "../utils/calculate.ts";
const registrations: Array<{ unregister(): void }> = [];
/** Shared collection; each faux provider gets a unique id so coexisting fakes route correctly. */
const models = createModels();
let fauxCount = 0;
afterEach(() => {
for (const registration of registrations.splice(0)) {
registration.unregister();
function newFaux(): FauxProviderHandle {
const faux = fauxProvider({ provider: `faux-${++fauxCount}` });
models.setProvider(faux.provider);
return faux;
}
});
function createHarness(options: ConstructorParameters<typeof AgentHarness>[0]): AgentHarness {
return new AgentHarness(options);
@@ -27,10 +36,9 @@ function captureOptions(options: StreamOptions | undefined): StreamOptions {
}
describe("AgentHarness stream configuration", () => {
it("snapshots stream options and merges auth headers before provider request hooks", async () => {
it("snapshots stream options before provider request hooks", async () => {
let capturedOptions: StreamOptions | undefined;
const registration = registerFauxProvider();
registrations.push(registration);
const registration = newFaux();
registration.setResponses([
(_context, options) => {
capturedOptions = options;
@@ -40,6 +48,7 @@ describe("AgentHarness stream configuration", () => {
const session = new Session(new InMemorySessionStorage({ metadata: { id: "session-1", createdAt: "now" } }));
const harness = createHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session,
model: registration.getModel(),
@@ -51,12 +60,11 @@ describe("AgentHarness stream configuration", () => {
metadata: { base: true },
cacheRetention: "none",
},
getApiKeyAndHeaders: async () => ({ apiKey: "secret", headers: { "x-auth": "auth" } }),
});
harness.on("before_provider_request", (event) => {
expect(event.sessionId).toBe("session-1");
expect(event.streamOptions.headers).toEqual({ "x-base": "base", "x-auth": "auth" });
expect(event.streamOptions.headers).toEqual({ "x-base": "base" });
return {
streamOptions: {
headers: { "x-hook": "hook" },
@@ -68,21 +76,19 @@ describe("AgentHarness stream configuration", () => {
await harness.prompt("hello");
expect(capturedOptions).toMatchObject({
apiKey: "secret",
timeoutMs: 1000,
maxRetries: 2,
maxRetryDelayMs: 3000,
sessionId: "session-1",
cacheRetention: "none",
});
expect(capturedOptions?.headers).toEqual({ "x-base": "base", "x-auth": "auth", "x-hook": "hook" });
expect(capturedOptions?.headers).toEqual({ "x-base": "base", "x-hook": "hook" });
expect(capturedOptions?.metadata).toEqual({ base: true, hook: true });
});
it("chains provider request patches and supports deletion semantics", async () => {
let capturedOptions: StreamOptions | undefined;
const registration = registerFauxProvider();
registrations.push(registration);
const registration = newFaux();
registration.setResponses([
(_context, options) => {
capturedOptions = options;
@@ -91,6 +97,7 @@ describe("AgentHarness stream configuration", () => {
]);
const harness = createHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session: new Session(new InMemorySessionStorage()),
model: registration.getModel(),
@@ -133,8 +140,7 @@ describe("AgentHarness stream configuration", () => {
it("uses updated stream options for save-point snapshots without mutating the active request", async () => {
const capturedOptions: StreamOptions[] = [];
const registration = registerFauxProvider();
registrations.push(registration);
const registration = newFaux();
registration.setResponses([
(_context, options) => {
capturedOptions.push(captureOptions(options));
@@ -149,6 +155,7 @@ describe("AgentHarness stream configuration", () => {
]);
const harness = createHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session: new Session(new InMemorySessionStorage()),
model: registration.getModel(),
@@ -174,8 +181,7 @@ describe("AgentHarness stream configuration", () => {
it("chains provider payload hooks", async () => {
const seenPayloads: unknown[] = [];
let finalPayload: unknown;
const registration = registerFauxProvider();
registrations.push(registration);
const registration = newFaux();
registration.setResponses([
async (_context, options, _state, model) => {
finalPayload = await options?.onPayload?.({ steps: ["provider"] }, model);
@@ -184,6 +190,7 @@ describe("AgentHarness stream configuration", () => {
]);
const harness = createHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session: new Session(new InMemorySessionStorage()),
model: registration.getModel(),
packages/agent/test/harness/agent-harness.test.ts
+43 -29
View File
@@ -1,5 +1,13 @@
import { fauxAssistantMessage, fauxToolCall, getModel, registerFauxProvider } from "@earendil-works/pi-ai";
import { afterEach, describe, expect, it } from "vitest";
import {
createModels,
type FauxProviderHandle,
fauxAssistantMessage,
fauxProvider,
fauxToolCall,
type RegisterFauxProviderOptions,
} from "@earendil-works/pi-ai";
import { getModel } from "@earendil-works/pi-ai/compat";
import { describe, expect, it } from "vitest";
import { AgentHarness } from "../../src/harness/agent-harness.ts";
import { NodeExecutionEnv } from "../../src/harness/env/nodejs.ts";
import { InMemorySessionStorage } from "../../src/harness/session/memory-storage.ts";
@@ -17,7 +25,15 @@ interface AppPromptTemplate extends PromptTemplate {
source: "project" | "user";
}
const registrations: Array<{ unregister(): void }> = [];
/** Shared collection; each faux provider gets a unique id so coexisting fakes route correctly. */
const models = createModels();
let fauxCount = 0;
function newFaux(options: RegisterFauxProviderOptions = {}): FauxProviderHandle {
const faux = fauxProvider({ provider: `faux-${++fauxCount}`, ...options });
models.setProvider(faux.provider);
return faux;
}
function textFromUserMessages(messages: Array<{ role: string; content: unknown }>): string[] {
return messages.flatMap((message) => {
@@ -44,18 +60,13 @@ function getReasoning(options: unknown): unknown {
return options.reasoning;
}
afterEach(() => {
for (const registration of registrations.splice(0)) {
registration.unregister();
}
});
describe("AgentHarness", () => {
it("constructs directly and exposes queue modes", () => {
const session = new Session(new InMemorySessionStorage());
const env = new NodeExecutionEnv({ cwd: process.cwd() });
const initialModel = getModel("anthropic", "claude-sonnet-4-5");
const harness = new AgentHarness({
models,
env,
session,
model: initialModel,
@@ -76,8 +87,7 @@ describe("AgentHarness", () => {
});
it("drains one queued steering message at a time and emits queue updates", async () => {
const registration = registerFauxProvider();
registrations.push(registration);
const registration = newFaux();
const userCounts: number[] = [];
registration.setResponses([
(context) => {
@@ -94,6 +104,7 @@ describe("AgentHarness", () => {
},
]);
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session: new Session(new InMemorySessionStorage()),
model: registration.getModel(),
@@ -119,8 +130,7 @@ describe("AgentHarness", () => {
});
it("appends before_agent_start messages and persists them", async () => {
const registration = registerFauxProvider();
registrations.push(registration);
const registration = newFaux();
let requestText: string[] = [];
registration.setResponses([
(context) => {
@@ -130,6 +140,7 @@ describe("AgentHarness", () => {
]);
const session = new Session(new InMemorySessionStorage());
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session,
model: registration.getModel(),
@@ -151,8 +162,7 @@ describe("AgentHarness", () => {
});
it("abort clears steer and follow-up queues but preserves next-turn messages", async () => {
const registration = registerFauxProvider();
registrations.push(registration);
const registration = newFaux();
let releaseFirstResponse: (() => void) | undefined;
let abortedSignal: AbortSignal | undefined;
const firstResponseReleased = new Promise<void>((resolve) => {
@@ -171,6 +181,7 @@ describe("AgentHarness", () => {
},
]);
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session: new Session(new InMemorySessionStorage()),
model: registration.getModel(),
@@ -206,8 +217,7 @@ describe("AgentHarness", () => {
});
it("drains follow-up messages one at a time after the agent would otherwise stop", async () => {
const registration = registerFauxProvider();
registrations.push(registration);
const registration = newFaux();
const userCounts: number[] = [];
registration.setResponses([
(context) => {
@@ -224,6 +234,7 @@ describe("AgentHarness", () => {
},
]);
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session: new Session(new InMemorySessionStorage()),
model: registration.getModel(),
@@ -249,11 +260,11 @@ describe("AgentHarness", () => {
});
it("settles thrown hook failures with persisted assistant error messages", async () => {
const registration = registerFauxProvider();
registrations.push(registration);
const registration = newFaux();
registration.setResponses([() => fauxAssistantMessage("should not be used")]);
const session = new Session(new InMemorySessionStorage());
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session,
model: registration.getModel(),
@@ -280,13 +291,12 @@ describe("AgentHarness", () => {
});
it("refreshes model, thinking level, resources, system prompt, and active tools at save points", async () => {
const registration = registerFauxProvider({
const registration = newFaux({
models: [
{ id: "first", reasoning: true },
{ id: "second", reasoning: true },
],
});
registrations.push(registration);
const secondModel = registration.getModel("second");
if (!secondModel) throw new Error("missing second faux model");
const captured: Array<{ modelId: string; reasoning: unknown; systemPrompt: string; tools: string[] }> = [];
@@ -313,6 +323,7 @@ describe("AgentHarness", () => {
},
]);
const harness = new AgentHarness<Skill, PromptTemplate, AgentTool>({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session: new Session(new InMemorySessionStorage()),
model: registration.getModel(),
@@ -345,11 +356,11 @@ describe("AgentHarness", () => {
});
it("orders pending listener session writes after agent-emitted messages", async () => {
const registration = registerFauxProvider();
registrations.push(registration);
const registration = newFaux();
registration.setResponses([() => fauxAssistantMessage("ok")]);
const session = new Session(new InMemorySessionStorage());
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session,
model: registration.getModel(),
@@ -376,11 +387,11 @@ describe("AgentHarness", () => {
});
it("waitForIdle waits for external run settlement and awaited listeners", async () => {
const registration = registerFauxProvider();
registrations.push(registration);
const registration = newFaux();
registration.setResponses([() => fauxAssistantMessage("ok")]);
const barrier = deferred();
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session: new Session(new InMemorySessionStorage()),
model: registration.getModel(),
@@ -408,8 +419,7 @@ describe("AgentHarness", () => {
});
it("runs tool_call and tool_result hooks through the direct loop", async () => {
const registration = registerFauxProvider();
registrations.push(registration);
const registration = newFaux();
registration.setResponses([
() =>
fauxAssistantMessage(fauxToolCall("calculate", { expression: "2 + 2" }, { id: "call-1" }), {
@@ -418,6 +428,7 @@ describe("AgentHarness", () => {
]);
const session = new Session(new InMemorySessionStorage());
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session,
model: registration.getModel(),
@@ -462,6 +473,7 @@ describe("AgentHarness", () => {
const inspectTool: AppTool = { ...calculateTool, name: "inspect", source: "builtin" };
const searchTool: AppTool = { ...calculateTool, name: "search", source: "extension" };
const harness = new AgentHarness<AppSkill, AppPromptTemplate, AppTool>({
models,
env,
session,
model,
@@ -530,11 +542,12 @@ describe("AgentHarness", () => {
const env = new NodeExecutionEnv({ cwd: process.cwd() });
const model = getModel("anthropic", "claude-sonnet-4-5");
expect(
() => new AgentHarness({ env, session, model, tools: [calculateTool], activeToolNames: ["missing"] }),
() => new AgentHarness({ env, session, models, model, tools: [calculateTool], activeToolNames: ["missing"] }),
).toThrow(/Unknown tool/);
expect(
() =>
new AgentHarness({
models,
env,
session,
model,
@@ -545,6 +558,7 @@ describe("AgentHarness", () => {
expect(
() =>
new AgentHarness({
models,
env,
session,
model,
@@ -558,7 +572,7 @@ describe("AgentHarness", () => {
const session = new Session(new InMemorySessionStorage());
const env = new NodeExecutionEnv({ cwd: process.cwd() });
const model = getModel("anthropic", "claude-sonnet-4-5");
const harness = new AgentHarness<AppSkill, AppPromptTemplate, AgentTool>({ env, session, model });
const harness = new AgentHarness<AppSkill, AppPromptTemplate, AgentTool>({ env, session, models, model });
const skill: AppSkill = {
name: "inspect",
description: "Inspect things",
packages/agent/test/harness/compaction.test.ts
+43 -62
View File
@@ -1,13 +1,14 @@
import {
type AssistantMessage,
type FauxProviderRegistration,
createModels,
type FauxProviderHandle,
fauxAssistantMessage,
fauxProvider,
type Message,
type Model,
registerFauxProvider,
type Usage,
} from "@earendil-works/pi-ai";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { beforeEach, describe, expect, it } from "vitest";
import {
type CompactionPreparation,
calculateContextTokens,
@@ -121,11 +122,13 @@ function createModelChangeEntry(provider: string, modelId: string, parentId: str
};
}
function createFauxModel(
reasoning: boolean,
maxTokens = 8192,
): { faux: FauxProviderRegistration; model: Model<string> } {
const faux = registerFauxProvider({
/** Shared collection; each faux provider gets a unique id so coexisting fakes route correctly. */
const models = createModels();
let fauxCount = 0;
function createFauxModel(reasoning: boolean, maxTokens = 8192): { faux: FauxProviderHandle; model: Model<string> } {
const faux = fauxProvider({
provider: `faux-${++fauxCount}`,
models: [
{
id: reasoning ? "reasoning-model" : "non-reasoning-model",
@@ -135,18 +138,10 @@ function createFauxModel(
},
],
});
fauxRegistrations.push(faux);
models.setProvider(faux.provider);
return { faux, model: faux.getModel() };
}
const fauxRegistrations: FauxProviderRegistration[] = [];
afterEach(() => {
while (fauxRegistrations.length > 0) {
fauxRegistrations.pop()?.unregister();
}
});
describe("harness compaction", () => {
beforeEach(() => {
nextId = 0;
@@ -306,11 +301,28 @@ describe("harness compaction", () => {
createMessageEntry({ ...assistant, stopReason: "error" }),
]),
).toBeUndefined();
expect(
getLastAssistantUsage([
createMessageEntry(createUserMessage("user")),
createMessageEntry(assistant),
createMessageEntry(createAssistantMessage("partial", createMockUsage(0, 0))),
]),
).toBe(usage);
expect(estimateContextTokens([createUserMessage("no usage")]).lastUsageIndex).toBeNull();
expect(estimateContextTokens([assistant, createUserMessage("tail")])).toMatchObject({
usageTokens: 20,
lastUsageIndex: 0,
});
const estimate = estimateContextTokens([
createUserMessage("Hello"),
assistant,
createUserMessage("continue"),
createAssistantMessage("Partial thinking", createMockUsage(0, 0)),
]);
expect(estimate.usageTokens).toBe(20);
expect(estimate.lastUsageIndex).toBe(1);
expect(estimate.trailingTokens).toBeGreaterThan(0);
expect(estimate.tokens).toBe(20 + estimate.trailingTokens);
});
it("builds session context with a compaction entry", () => {
@@ -445,19 +457,9 @@ describe("harness compaction", () => {
},
]);
getOrThrow(
await generateSummary(
messages,
reasoningModel,
2000,
"test-key",
undefined,
undefined,
undefined,
undefined,
"medium",
),
await generateSummary(messages, models, reasoningModel, 2000, undefined, undefined, undefined, "medium"),
);
expect(seenOptions[0]).toMatchObject({ reasoning: "medium", apiKey: "test-key" });
expect(seenOptions[0]).toMatchObject({ reasoning: "medium" });
const { faux: fauxOff, model: offModel } = createFauxModel(true);
fauxOff.setResponses([
@@ -466,9 +468,7 @@ describe("harness compaction", () => {
return fauxAssistantMessage("## Goal\nTest summary");
},
]);
getOrThrow(
await generateSummary(messages, offModel, 2000, "test-key", undefined, undefined, undefined, undefined, "off"),
);
getOrThrow(await generateSummary(messages, models, offModel, 2000, undefined, undefined, undefined, "off"));
expect(seenOptions[1]).not.toHaveProperty("reasoning");
const { faux: fauxNonReasoning, model: nonReasoningModel } = createFauxModel(false);
@@ -479,17 +479,7 @@ describe("harness compaction", () => {
},
]);
getOrThrow(
await generateSummary(
messages,
nonReasoningModel,
2000,
"test-key",
undefined,
undefined,
undefined,
undefined,
"medium",
),
await generateSummary(messages, models, nonReasoningModel, 2000, undefined, undefined, undefined, "medium"),
);
expect(seenOptions[2]).not.toHaveProperty("reasoning");
});
@@ -508,16 +498,7 @@ describe("harness compaction", () => {
]);
const summary = getOrThrow(
await generateSummary(
messages,
model,
2000,
"test-key",
{ "x-test": "yes" },
undefined,
"focus",
"old summary",
),
await generateSummary(messages, models, model, 2000, undefined, "focus", "old summary"),
);
expect(summary).toContain("Test summary");
@@ -529,7 +510,7 @@ describe("harness compaction", () => {
const messages: AgentMessage[] = [createUserMessage("Summarize this.")];
const { faux: errorFaux, model: errorModel } = createFauxModel(false);
errorFaux.setResponses([fauxAssistantMessage("", { stopReason: "error", errorMessage: "boom" })]);
const errorResult = await generateSummary(messages, errorModel, 2000, "test-key");
const errorResult = await generateSummary(messages, models, errorModel, 2000);
expect(errorResult).toMatchObject({
ok: false,
error: { code: "summarization_failed", message: "Summarization failed: boom" },
@@ -537,7 +518,7 @@ describe("harness compaction", () => {
const { faux: abortedFaux, model: abortedModel } = createFauxModel(false);
abortedFaux.setResponses([fauxAssistantMessage("", { stopReason: "aborted", errorMessage: "stopped" })]);
const abortedResult = await generateSummary(messages, abortedModel, 2000, "test-key");
const abortedResult = await generateSummary(messages, models, abortedModel, 2000);
expect(abortedResult).toMatchObject({ ok: false, error: { code: "aborted", message: "stopped" } });
});
@@ -565,7 +546,7 @@ describe("harness compaction", () => {
settings: { enabled: true, reserveTokens: 500000, keepRecentTokens: 20000 },
};
getOrThrow(await compact(preparation, model, "test-key"));
getOrThrow(await compact(preparation, models, model));
expect(seenOptions.map((options) => options?.maxTokens)).toEqual([128000, 128000]);
});
@@ -583,7 +564,7 @@ describe("harness compaction", () => {
};
const { faux: historyFaux, model: historyModel } = createFauxModel(false);
historyFaux.setResponses([fauxAssistantMessage("", { stopReason: "error", errorMessage: "history failed" })]);
expect(await compact(preparation, historyModel, "test-key")).toMatchObject({
expect(await compact(preparation, models, historyModel)).toMatchObject({
ok: false,
error: { code: "summarization_failed", message: "Summarization failed: history failed" },
});
@@ -591,8 +572,8 @@ describe("harness compaction", () => {
const { model: invalidModel } = createFauxModel(false);
const invalidResult = await compact(
{ ...preparation, messagesToSummarize: [], firstKeptEntryId: "" },
models,
invalidModel,
"test-key",
);
expect(invalidResult).toMatchObject({ ok: false, error: { code: "invalid_session" } });
});
@@ -617,7 +598,7 @@ describe("harness compaction", () => {
settings: { enabled: true, reserveTokens: 2000, keepRecentTokens: 20 },
};
getOrThrow(await compact(preparation, model, "test-key", undefined, undefined, undefined, "high"));
getOrThrow(await compact(preparation, models, model, undefined, undefined, "high"));
expect(seenOptions[0]).toMatchObject({ reasoning: "high" });
});
@@ -636,14 +617,14 @@ describe("harness compaction", () => {
const { faux, model } = createFauxModel(false);
faux.setResponses([fauxAssistantMessage("", { stopReason: "error", errorMessage: "prefix failed" })]);
expect(await compact(preparation, model, "test-key")).toMatchObject({
expect(await compact(preparation, models, model)).toMatchObject({
ok: false,
error: { code: "summarization_failed", message: "Turn prefix summarization failed: prefix failed" },
});
const { faux: abortedFaux, model: abortedModel } = createFauxModel(false);
abortedFaux.setResponses([fauxAssistantMessage("", { stopReason: "aborted", errorMessage: "prefix stopped" })]);
expect(await compact(preparation, abortedModel, "test-key")).toMatchObject({
expect(await compact(preparation, models, abortedModel)).toMatchObject({
ok: false,
error: { code: "aborted", message: "prefix stopped" },
});
@@ -662,7 +643,7 @@ describe("harness compaction", () => {
expect(preparation).toBeDefined();
const { faux, model } = createFauxModel(false);
faux.setResponses([fauxAssistantMessage("## Goal\nTest summary")]);
const result = getOrThrow(await compact(preparation!, model, "test-key"));
const result = getOrThrow(await compact(preparation!, models, model));
expect(result.summary.length).toBeGreaterThan(0);
expect(result.firstKeptEntryId).toBeTruthy();
expect(result.details).toBeDefined();
packages/agent/test/harness/nodejs-env.test.ts
+34 -1
View File
@@ -1,5 +1,5 @@
import { access, chmod, realpath, symlink } from "node:fs/promises";
import { join } from "node:path";
import { delimiter, join } from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { NodeExecutionEnv } from "../../src/harness/env/nodejs.ts";
import { FileError, getOrThrow } from "../../src/harness/types.ts";
@@ -201,6 +201,39 @@ describe("NodeExecutionEnv", () => {
expect(result).toEqual({ stdout: `${await realpath(root)}:ok`, stderr: "", exitCode: 0 });
});
it("uses stdin command transport for legacy WSL bash paths", async () => {
if (process.platform === "win32") return;
const root = createTempDir();
const shellPath = "C:\\Windows\\System32\\bash.exe";
const env = new NodeExecutionEnv({ cwd: root });
getOrThrow(await env.writeFile(shellPath, '#!/bin/sh\nprintf \'args:%s\\n\' "$*" >&2\nexec /bin/bash "$@"\n'));
await chmod(join(root, shellPath), 0o755);
const originalCwd = process.cwd();
const originalPath = process.env.PATH;
const platformDescriptor = Object.getOwnPropertyDescriptor(process, "platform");
try {
process.chdir(root);
process.env.PATH = `${root}${delimiter}${originalPath ?? ""}`;
Object.defineProperty(process, "platform", {
configurable: true,
value: "win32",
});
const wslEnv = new NodeExecutionEnv({ cwd: root, shellPath });
const nameExpansion = "$" + "{name}";
const result = getOrThrow(await wslEnv.exec(`name='World'; echo "Hello, ${nameExpansion}!"`));
expect(result).toEqual({ stdout: "Hello, World!\n", stderr: "args:-s\n", exitCode: 0 });
} finally {
process.chdir(originalCwd);
process.env.PATH = originalPath;
if (platformDescriptor) {
Object.defineProperty(process, "platform", platformDescriptor);
}
}
});
it("streams stdout and stderr chunks", async () => {
const root = createTempDir();
const env = new NodeExecutionEnv({ cwd: root });
packages/agent/test/harness/session.test.ts
+6
View File
@@ -86,6 +86,12 @@ async function runSessionSuite(
expect(context.messages[1]?.role).toBe("custom");
});
it("normalizes session names", async () => {
const session = new Session(await createStorage());
await session.appendSessionName(" hello\nworld\r\nagain ");
expect(await session.getSessionName()).toBe("hello world again");
});
it("supports labels and session info entries without affecting context", async () => {
const session = new Session(await createStorage());
const user1 = await session.appendMessage(createUserMessage("one"));
packages/agent/test/scratch/simple.ts
+15 -2
View File
@@ -1,6 +1,8 @@
import { homedir } from "node:os";
import { join } from "node:path";
import { getModel } from "@earendil-works/pi-ai";
import { createModels } from "@earendil-works/pi-ai";
import { cloudflareAIGatewayProvider } from "@earendil-works/pi-ai/providers/cloudflare-ai-gateway";
import { openaiProvider } from "@earendil-works/pi-ai/providers/openai";
import { NodeExecutionEnv } from "../../src/harness/env/nodejs.ts";
import { InMemorySessionStorage } from "../../src/harness/session/memory-storage.ts";
import {
@@ -35,11 +37,22 @@ const { promptTemplates: sourcedPromptTemplates } = await loadSourcedPromptTempl
(promptTemplate, source) => ({ ...promptTemplate, source }),
);
const models = createModels();
models.setProvider(openaiProvider());
models.setProvider(cloudflareAIGatewayProvider());
const model = models.getModel("openai", "gpt-5.5");
// const model = models.getModel("cloudflare-ai-gateway", "claude-haiku-4-5");
if (!model) {
console.log("Model not found");
process.exit(-1);
}
const session = new Session(new InMemorySessionStorage());
const agent = new AgentHarness({
env,
session,
model: getModel("openai", "gpt-5.5"),
models,
model,
thinkingLevel: "low",
systemPrompt: ({ env, resources }) =>
[
packages/agent/vitest.config.ts
+10
View File
@@ -1,9 +1,19 @@
import { fileURLToPath } from "node:url";
import { defineConfig } from "vitest/config";
const aiSrcIndex = fileURLToPath(new URL("../ai/src/index.ts", import.meta.url));
const aiSrcCompat = fileURLToPath(new URL("../ai/src/compat.ts", import.meta.url));
export default defineConfig({
test: {
globals: true,
environment: "node",
testTimeout: 30000, // 30 seconds for API calls
},
resolve: {
alias: [
{ find: /^@earendil-works\/pi-ai$/, replacement: aiSrcIndex },
{ find: /^@earendil-works\/pi-ai\/compat$/, replacement: aiSrcCompat },
],
},
});
packages/agent/vitest.harness.config.ts
+10
View File
@@ -1,5 +1,9 @@
import { fileURLToPath } from "node:url";
import { defineConfig } from "vitest/config";
const aiSrcIndex = fileURLToPath(new URL("../ai/src/index.ts", import.meta.url));
const aiSrcCompat = fileURLToPath(new URL("../ai/src/compat.ts", import.meta.url));
export default defineConfig({
test: {
globals: true,
@@ -15,4 +19,10 @@ export default defineConfig({
reportsDirectory: "coverage/harness",
},
},
resolve: {
alias: [
{ find: /^@earendil-works\/pi-ai$/, replacement: aiSrcIndex },
{ find: /^@earendil-works\/pi-ai\/compat$/, replacement: aiSrcCompat },
],
},
});
packages/ai/CHANGELOG.md
+127
View File
@@ -2,6 +2,133 @@
## [Unreleased]
### Fixed
- Fixed retry classification for provider errors that explicitly tell callers to retry the request ([#6019](https://github.com/earendil-works/pi/issues/6019)).
## [0.80.2] - 2026-06-23
### Changed
- Changed `ApiKeyCredential` to use the `auth.json`-compatible discriminator `type: "api_key"` and provider-scoped `env` values instead of `type: "api-key"` and metadata.
### Fixed
- Fixed Anthropic-compatible custom models to use explicit compatibility metadata instead of provider-name heuristics for session-affinity headers and unsupported tool-field omissions.
- Fixed request-scoped `apiKey` and `env` values to participate in provider auth resolution, so providers such as Cloudflare can derive request-specific base URLs from explicit call options ([#6021](https://github.com/earendil-works/pi/issues/6021)).
- Restored temporary legacy per-API stream aliases such as `streamSimpleOpenAICompletions` on the compat entrypoint ([#6016](https://github.com/earendil-works/pi/issues/6016), [#6017](https://github.com/earendil-works/pi/issues/6017)).
- Restored runtime `detectCompat` fallback in `openai-completions` for models without explicit compat metadata ([#6020](https://github.com/earendil-works/pi/issues/6020)).
## [0.80.1] - 2026-06-23
### Fixed
- Fixed a regression in Amazon Bedrock scoped `AWS_PROFILE` endpoint resolution for built-in inference profile endpoints.
- Fixed Fireworks Anthropic-compatible requests to apply session-affinity and unsupported tool-field defaults for custom Fireworks models.
- Fixed Together MiniMax M2.7 metadata to avoid unsupported Together reasoning toggles.
## [0.80.0] - 2026-06-23
### Breaking Changes
- The root entrypoint (`@earendil-works/pi-ai`) is now core-only and side-effect free. The old global API moved to the temporary `@earendil-works/pi-ai/compat` entrypoint, a strict superset of the root: switching a file's import path is the only migration step. Moved symbols include `stream`/`complete`/`streamSimple`/`completeSimple`, `getModel`/`getModels`/`getProviders` (now deprecated aliases of `getBuiltinModel`/`getBuiltinModels`/`getBuiltinProviders` from `@earendil-works/pi-ai/providers/all`), `registerApiProvider`/`unregisterApiProviders`/`resetApiProviders`/`getApiProvider`, `getEnvApiKey`/`findEnvKeys`, `setBedrockProviderModule`, the per-API lazy stream wrappers (`anthropicMessagesApi`, ...), and the image-generation API.
- Renamed the `Provider` type to `ProviderId`. `Provider` now names the runtime provider interface (id, name, auth, model listing, stream behavior).
- API implementation modules moved from `src/providers/` to `@earendil-works/pi-ai/api/*`, renamed by API id (`anthropic` -> `api/anthropic-messages`, `google` -> `api/google-generative-ai`, `mistral` -> `api/mistral-conversations`, `amazon-bedrock` -> `api/bedrock-converse-stream`), each exporting exactly `stream` and `streamSimple`. The old per-impl export names (`streamAnthropic`, `streamSimpleAnthropic`, ...) and legacy raw API subpaths (`./anthropic`, `./google`, `./openai-completions`, ...) are gone; import raw API implementations through `@earendil-works/pi-ai/api/*`.
- Removed the `@earendil-works/pi-ai/base` selective-provider entrypoint; use the root/core APIs with explicit `createModels()` collections and provider factories for isolated bundles.
Migration guide:
- Read `packages/ai/README.md` in full for the new `Models` API, provider factories, auth configuration, image generation, and custom provider examples.
- To keep the old global API temporarily, change imports from `@earendil-works/pi-ai` to `@earendil-works/pi-ai/compat`. The compat entrypoint preserves `stream`/`complete`, generated catalog reads, API registry helpers, env API-key helpers, lazy API wrappers, and image globals, but it will be removed in a future release.
- To migrate to the new runtime, create a `Models` collection and call methods on it:
```ts
import { builtinModels } from "@earendil-works/pi-ai/providers/all";
const models = builtinModels();
const model = models.getModel("anthropic", "claude-haiku-4-5");
if (!model) throw new Error("model not found");
const message = await models.complete(model, {
messages: [{ role: "user", content: "Hello", timestamp: Date.now() }],
});
```
- For an isolated provider set, register provider factories explicitly:
```ts
import { createModels } from "@earendil-works/pi-ai";
import { anthropicProvider } from "@earendil-works/pi-ai/providers/anthropic";
const models = createModels();
models.setProvider(anthropicProvider());
```
- To call a raw API implementation directly, import from `@earendil-works/pi-ai/api/*` and pass a compatible model plus auth/options yourself. Raw API modules export `stream` and `streamSimple`; use `.result()` on the returned stream for `complete`/`completeSimple` behavior:
```ts
import { streamSimple } from "@earendil-works/pi-ai/api/anthropic-messages";
import { getBuiltinModel } from "@earendil-works/pi-ai/providers/all";
const model = getBuiltinModel("anthropic", "claude-haiku-4-5");
const stream = streamSimple(
model,
{ messages: [{ role: "user", content: "Hello", timestamp: Date.now() }] },
{ apiKey: process.env.ANTHROPIC_API_KEY },
);
const message = await stream.result();
```
Custom raw models must set the matching `api` value (for example `"anthropic-messages"` for `api/anthropic-messages`) and any required provider compatibility metadata in `model.compat`.
### Added
- New `Models` runtime: `createModels()` builds an isolated provider collection with sync model reads (`getModels`/`getModel` return the last-known lists), an explicit async `refresh(provider?)` for dynamic providers, auth resolution (`getAuth`), and `stream`/`complete`/`streamSimple`/`completeSimple` that resolve auth through the owning provider. `createProvider()` builds providers from parts (single API implementation or a map dispatched on `model.api`; static `models` array plus an optional `refreshModels` fetcher with in-flight dedupe); `hasApi()` narrows dynamically listed models.
- Provider auth substrate: `ProviderAuth` (`{ apiKey?, oauth? }`), one type-tagged credential per provider, `CredentialStore` (`read`/`modify`/`delete` with serialized writes; in-memory default), `envApiKeyAuth()`, `lazyOAuth()`, and injectable `AuthContext`. OAuth refresh runs under the store lock with double-checked expiry; a stored credential owns its provider (no silent env fallback).
- One provider factory per built-in provider under `@earendil-works/pi-ai/providers/*` (e.g. `anthropicProvider()`, `openrouterProvider()`), plus `@earendil-works/pi-ai/providers/all` with `builtinProviders()`/`builtinModels()` and typed `getBuiltin*` catalog reads. Generated catalogs are split per provider, so importing one provider pulls one catalog; `sideEffects` metadata makes the package tree-shakeable.
- OAuth flows (Anthropic, OpenAI Codex, GitHub Copilot) gained `OAuthAuth` adapters (`login`/`refresh`/`toAuth`) on unified `prompt()`/`notify()` login callbacks; Copilot's per-credential base URL is derived in `toAuth()`.
- `fauxProvider()` returns a faux `Provider` for tests built on explicit `Models` collections.
- Image generation mirrors the chat-side design: `createImagesModels()`/`ImagesProvider`/`createImagesProvider()` with sync model reads, explicit `refresh()`, provider-resolved auth, and never-rejecting `generateImages()`; `openrouterImagesProvider()` factory plus `builtinImagesProviders()`/`builtinImagesModels()` in `providers/all`. The `ImagesProvider` id type alias is renamed to `ImagesProviderId`; the old global image API stays on `/compat`.
- Provider auth results can carry provider-scoped environment values that `Models` and `ImagesModels` merge into API implementation options.
### Fixed
- Fixed OpenAI Responses streams to fail when they end before a terminal response event and to treat `response.incomplete` as a length stop ([#5526](https://github.com/earendil-works/pi/pull/5526) by [@dmmulroy](https://github.com/dmmulroy)).
- Fixed Amazon Bedrock endpoint resolution to honor scoped `AWS_PROFILE` values.
- Fixed Cloudflare providers to require account/gateway configuration and route built-in `/compat` requests through provider auth.
- Fixed `/compat` API-key injection to honor request-scoped `env` values.
- Fixed OpenAI Codex Responses WebSocket sessions to reconnect once when OpenAI's connection limit is reached before output starts ([#5973](https://github.com/earendil-works/pi/issues/5973)).
- Fixed OpenCode Go GLM-5.2 metadata to expose `xhigh` reasoning and send `reasoning_effort: "max"` ([#5967](https://github.com/earendil-works/pi/issues/5967)).
## [0.79.10] - 2026-06-22
### Fixed
- Fixed OpenAI-compatible streaming to preserve encrypted `reasoning_details` that arrive before matching tool call deltas ([#5114](https://github.com/earendil-works/pi/issues/5114)).
## [0.79.9] - 2026-06-20
### Added
- Added configurable `chat-template` thinking support for OpenAI-compatible providers that use `chat_template_kwargs`, such as DeepSeek models behind vLLM ([#5673](https://github.com/earendil-works/pi/issues/5673)).
### Fixed
- Fixed Fireworks GLM-5.2 metadata to use the OpenAI-compatible Chat Completions endpoint with `reasoning_effort` support ([#5923](https://github.com/earendil-works/pi/issues/5923)).
- Fixed OpenRouter GLM-5.2 metadata to expose `xhigh` reasoning and send OpenRouter's native `xhigh` effort ([#5770](https://github.com/earendil-works/pi/issues/5770)).
- Fixed GitHub Copilot OAuth model availability to use the authenticated account's model picker catalog ([#5897](https://github.com/earendil-works/pi/issues/5897)).
## [0.79.8] - 2026-06-19
### Added
- Added `@earendil-works/pi-ai/base` and direct provider registration exports for bundlers that want selective provider transports without root built-in registration ([#5348](https://github.com/earendil-works/pi/pull/5348) by [@FredKSchott](https://github.com/FredKSchott)).
- Added prompt caching for Mistral requests using the pi session ID as `prompt_cache_key`, including cached-token usage and cost accounting ([#5854](https://github.com/earendil-works/pi/issues/5854)).
- Added the OpenRouter Fusion alias as `openrouter/fusion` ([#5866](https://github.com/earendil-works/pi/pull/5866) by [@dannote](https://github.com/dannote)).
## [0.79.7] - 2026-06-18
### Added
- Added GLM-5.2 model to the OpenCode Go subscription model catalog ([#5860](https://github.com/earendil-works/pi/issues/5860)).
packages/ai/README.md
+648 -494
View File
File diff suppressed because it is too large Load Diff
packages/ai/package.json
+19 -33
View File
@@ -1,46 +1,31 @@
{
"name": "@earendil-works/pi-ai",
"version": "0.79.6",
"version": "0.80.2",
"description": "Unified LLM API with automatic model discovery and provider configuration",
"type": "module",
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
"sideEffects": [
"./dist/compat.js",
"./dist/images.js",
"./dist/providers/images/register-builtins.js"
],
"exports": {
".": {
"types": "./dist/index.d.ts",
"import": "./dist/index.js"
},
"./anthropic": {
"types": "./dist/providers/anthropic.d.ts",
"import": "./dist/providers/anthropic.js"
"./compat": {
"types": "./dist/compat.d.ts",
"import": "./dist/compat.js"
},
"./azure-openai-responses": {
"types": "./dist/providers/azure-openai-responses.d.ts",
"import": "./dist/providers/azure-openai-responses.js"
"./providers/*": {
"types": "./dist/providers/*.d.ts",
"import": "./dist/providers/*.js"
},
"./google": {
"types": "./dist/providers/google.d.ts",
"import": "./dist/providers/google.js"
},
"./google-vertex": {
"types": "./dist/providers/google-vertex.d.ts",
"import": "./dist/providers/google-vertex.js"
},
"./mistral": {
"types": "./dist/providers/mistral.d.ts",
"import": "./dist/providers/mistral.js"
},
"./openai-codex-responses": {
"types": "./dist/providers/openai-codex-responses.d.ts",
"import": "./dist/providers/openai-codex-responses.js"
},
"./openai-completions": {
"types": "./dist/providers/openai-completions.d.ts",
"import": "./dist/providers/openai-completions.js"
},
"./openai-responses": {
"types": "./dist/providers/openai-responses.d.ts",
"import": "./dist/providers/openai-responses.js"
"./api/*": {
"types": "./dist/api/*.d.ts",
"import": "./dist/api/*.js"
},
"./oauth": {
"types": "./dist/oauth.d.ts",
@@ -69,9 +54,10 @@
"dependencies": {
"@anthropic-ai/sdk": "0.91.1",
"@aws-sdk/client-bedrock-runtime": "3.1048.0",
"@smithy/node-http-handler": "4.7.3",
"@google/genai": "1.52.0",
"@mistralai/mistralai": "2.2.1",
"@mistralai/mistralai": "2.2.6",
"@opentelemetry/api": "1.9.0",
"@smithy/node-http-handler": "4.7.3",
"http-proxy-agent": "7.0.2",
"https-proxy-agent": "7.0.6",
"openai": "6.26.0",
@@ -101,6 +87,6 @@
"devDependencies": {
"@types/node": "24.12.4",
"canvas": "3.2.3",
"vitest": "3.2.4"
"vitest": "4.1.9"
}
}
packages/ai/scripts/generate-models.ts
+252 -46
View File
@@ -1,6 +1,6 @@
#!/usr/bin/env node
import { writeFileSync } from "fs";
import { readdirSync, rmSync, writeFileSync } from "fs";
import { join, dirname } from "path";
import { fileURLToPath } from "url";
import {
@@ -8,7 +8,7 @@ import {
CLOUDFLARE_AI_GATEWAY_COMPAT_BASE_URL,
CLOUDFLARE_AI_GATEWAY_OPENAI_BASE_URL,
CLOUDFLARE_WORKERS_AI_BASE_URL,
} from "../src/providers/cloudflare.ts";
} from "../src/api/cloudflare.ts";
import type { AnthropicMessagesCompat, Api, KnownProvider, Model, OpenAICompletionsCompat } from "../src/types.ts";
const __filename = fileURLToPath(import.meta.url);
@@ -164,6 +164,14 @@ const ZAI_GLM52_THINKING_LEVEL_MAP = {
high: "high",
xhigh: "max",
} as const;
const OPENCODE_GO_GLM52_THINKING_LEVEL_MAP = {
off: null,
minimal: null,
low: null,
medium: null,
high: "high",
xhigh: "max",
} as const;
const EAGER_TOOL_INPUT_STREAMING_UNSUPPORTED_ANTHROPIC_MODELS = new Set([
"github-copilot:claude-haiku-4.5",
"github-copilot:claude-sonnet-4",
@@ -269,10 +277,149 @@ function isAnthropicTemperatureUnsupportedModel(modelId: string): boolean {
return id.includes("opus-4-7") || id.includes("opus-4.7") || id.includes("opus-4-8") || id.includes("opus-4.8");
}
const OPENAI_COMPLETIONS_DEFAULT_COMPAT = {
supportsStore: true,
supportsDeveloperRole: true,
supportsReasoningEffort: true,
supportsUsageInStreaming: true,
maxTokensField: "max_completion_tokens",
requiresToolResultName: false,
requiresAssistantAfterToolResult: false,
requiresThinkingAsText: false,
requiresReasoningContentOnAssistantMessages: false,
thinkingFormat: "openai",
openRouterRouting: {},
vercelGatewayRouting: {},
chatTemplateKwargs: {},
zaiToolStream: false,
supportsStrictMode: true,
sendSessionAffinityHeaders: false,
supportsLongCacheRetention: true,
} satisfies Required<Omit<OpenAICompletionsCompat, "cacheControlFormat">> & {
cacheControlFormat?: OpenAICompletionsCompat["cacheControlFormat"];
};
type OpenAICompletionsResolvedCompat = typeof OPENAI_COMPLETIONS_DEFAULT_COMPAT & {
cacheControlFormat?: OpenAICompletionsCompat["cacheControlFormat"];
};
function mergeAnthropicMessagesCompat(model: Model<Api>, compat: AnthropicMessagesCompat): void {
model.compat = { ...(model.compat as AnthropicMessagesCompat | undefined), ...compat };
}
function detectOpenAICompletionsCompat(model: Model<"openai-completions">): OpenAICompletionsResolvedCompat {
const provider = model.provider;
const baseUrl = model.baseUrl;
const isZai =
provider === "zai" ||
provider === "zai-coding-cn" ||
baseUrl.includes("api.z.ai") ||
baseUrl.includes("open.bigmodel.cn");
const isTogether =
provider === "together" || baseUrl.includes("api.together.ai") || baseUrl.includes("api.together.xyz");
const isMoonshot = provider === "moonshotai" || provider === "moonshotai-cn" || baseUrl.includes("api.moonshot.");
const isOpenRouter = provider === "openrouter" || baseUrl.includes("openrouter.ai");
const isCloudflareWorkersAI = provider === "cloudflare-workers-ai" || baseUrl.includes("api.cloudflare.com");
const isCloudflareAiGateway = provider === "cloudflare-ai-gateway" || baseUrl.includes("gateway.ai.cloudflare.com");
const isNvidia = provider === "nvidia" || baseUrl.includes("integrate.api.nvidia.com");
const isAntLing = provider === "ant-ling" || baseUrl.includes("api.ant-ling.com");
const isTogetherReasoningOnly = isTogether && TOGETHER_REASONING_ONLY_MODELS.has(model.id);
const isNonStandard =
isNvidia ||
provider === "cerebras" ||
baseUrl.includes("cerebras.ai") ||
provider === "xai" ||
baseUrl.includes("api.x.ai") ||
isTogether ||
baseUrl.includes("chutes.ai") ||
baseUrl.includes("deepseek.com") ||
isZai ||
isMoonshot ||
provider === "opencode" ||
baseUrl.includes("opencode.ai") ||
isCloudflareWorkersAI ||
isCloudflareAiGateway ||
isAntLing;
const useMaxTokens =
baseUrl.includes("chutes.ai") || isMoonshot || isCloudflareAiGateway || isTogether || isNvidia || isAntLing;
const isGrok = provider === "xai" || baseUrl.includes("api.x.ai");
const isDeepSeek = provider === "deepseek" || baseUrl.includes("deepseek.com");
const isOpenRouterDeveloperRoleModel =
isOpenRouter && (model.id.startsWith("anthropic/") || model.id.startsWith("openai/"));
const cacheControlFormat = provider === "openrouter" && model.id.startsWith("anthropic/") ? "anthropic" : undefined;
return {
supportsStore: !isNonStandard,
supportsDeveloperRole: isOpenRouterDeveloperRoleModel || (!isNonStandard && !isOpenRouter),
supportsReasoningEffort:
!isGrok && !isZai && !isMoonshot && !isTogether && !isCloudflareAiGateway && !isNvidia && !isAntLing,
supportsUsageInStreaming: true,
maxTokensField: useMaxTokens ? "max_tokens" : "max_completion_tokens",
requiresToolResultName: false,
requiresAssistantAfterToolResult: false,
requiresThinkingAsText: false,
requiresReasoningContentOnAssistantMessages: isDeepSeek,
thinkingFormat: isDeepSeek
? "deepseek"
: isZai
? "zai"
: isTogether && !isTogetherReasoningOnly
? "together"
: isAntLing
? "ant-ling"
: isOpenRouter
? "openrouter"
: "openai",
openRouterRouting: {},
vercelGatewayRouting: {},
chatTemplateKwargs: {},
zaiToolStream: false,
supportsStrictMode: !isMoonshot && !isTogether && !isCloudflareAiGateway && !isNvidia,
...(cacheControlFormat ? { cacheControlFormat } : {}),
sendSessionAffinityHeaders: false,
supportsLongCacheRetention: !(
isTogether ||
isCloudflareWorkersAI ||
isCloudflareAiGateway ||
isNvidia ||
isAntLing
),
};
}
function isPlainEmptyObject(value: unknown): boolean {
return typeof value === "object" && value !== null && !Array.isArray(value) && Object.keys(value).length === 0;
}
function openAICompletionsCompatDelta(compat: OpenAICompletionsResolvedCompat): OpenAICompletionsCompat {
const delta: OpenAICompletionsCompat = {};
for (const [key, value] of Object.entries(compat)) {
const defaultValue = OPENAI_COMPLETIONS_DEFAULT_COMPAT[key as keyof typeof OPENAI_COMPLETIONS_DEFAULT_COMPAT];
if (isPlainEmptyObject(value) && isPlainEmptyObject(defaultValue)) continue;
if (value !== defaultValue) {
(delta as Record<string, unknown>)[key] = value;
}
}
return delta;
}
function mergeOpenAICompletionsCompat(model: Model<Api>, compat: OpenAICompletionsCompat): void {
model.compat = { ...(model.compat as OpenAICompletionsCompat | undefined), ...compat };
}
function applyOpenAICompletionsCompatMetadata(model: Model<Api>): void {
if (model.api !== "openai-completions") return;
const detected = openAICompletionsCompatDelta(detectOpenAICompletionsCompat(model as Model<"openai-completions">));
model.compat = { ...detected, ...(model.compat as OpenAICompletionsCompat | undefined) };
if (Object.keys(model.compat).length === 0) {
delete model.compat;
}
}
function isGemini3ProModel(modelId: string): boolean {
return /gemini-3(?:\.\d+)?-pro/.test(modelId.toLowerCase());
}
@@ -374,6 +521,15 @@ function applyThinkingLevelMetadata(model: Model<any>): void {
// Pi's low/medium/high pass through verbatim; OpenRouter normalizes to Mercury's vocabulary.
mergeThinkingLevelMap(model, { off: null });
}
if (model.provider === "openrouter" && model.id === "z-ai/glm-5.2") {
mergeThinkingLevelMap(model, { xhigh: "xhigh" });
}
if (model.provider === "fireworks" && model.id === "accounts/fireworks/models/glm-5p2") {
mergeThinkingLevelMap(model, { off: "none", minimal: null, low: "high", medium: "high", xhigh: "max" });
}
if (model.provider === "opencode-go" && model.id === "glm-5.2") {
mergeThinkingLevelMap(model, OPENCODE_GO_GLM52_THINKING_LEVEL_MAP);
}
if (model.provider === "opencode-go" && model.id === "kimi-k2.6") {
// OpenCode Go exposes Kimi K2.6 thinking as on/off, not distinct effort tiers.
mergeThinkingLevelMap(model, { minimal: null, low: null, medium: null });
@@ -837,9 +993,10 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
continue;
}
// workers-ai/* through the gateway forwards x-session-affinity to
// the underlying Workers AI runtime for prefix-cache routing.
const compat = upstream === "workers-ai" ? { sendSessionAffinityHeaders: true } : undefined;
// Gateway passthroughs forward session affinity headers to upstreams that
// use them for cache/routing affinity.
const compat =
upstream === "anthropic" || upstream === "workers-ai" ? { sendSessionAffinityHeaders: true } : undefined;
models.push({
id,
@@ -948,7 +1105,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
cost: {
input: m.cost?.input || 0,
output: m.cost?.output || 0,
cacheRead: m.cost?.cache_read || 0,
cacheRead: m.cost?.cache_read ?? (m.cost?.input ? roundCost(m.cost.input * 0.1) : 0),
cacheWrite: m.cost?.cache_write || 0,
},
contextWindow: m.limit?.context || 4096,
@@ -1511,7 +1668,11 @@ async function generateModels() {
candidate.cost.output = 1.9;
candidate.cost.cacheRead = 0.119;
}
if (candidate.provider === "fireworks" && candidate.id === "accounts/fireworks/models/glm-5p2") {
candidate.api = "openai-completions";
candidate.baseUrl = "https://api.fireworks.ai/inference/v1";
candidate.compat = { supportsStore: false, supportsDeveloperRole: false };
}
}
@@ -2030,6 +2191,32 @@ async function generateModels() {
});
}
// Add "fusion" alias for openrouter/fusion. OpenRouter exposes Fusion as a
// router alias/plugin entry point; its model metadata does not advertise
// tools, but the alias resolves to a concrete model that can invoke caller
// tools and has the openrouter:fusion server tool auto-injected.
if (!allModels.some(m => m.provider === "openrouter" && m.id === "openrouter/fusion")) {
allModels.push({
id: "openrouter/fusion",
name: "OpenRouter: Fusion",
api: "openai-completions",
provider: "openrouter",
baseUrl: "https://openrouter.ai/api/v1",
reasoning: true,
input: ["text"],
cost: {
// we dont know about the costs because Fusion routes to multiple models
// and then charges you for the underlying used models
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 1000000,
maxTokens: 30000,
});
}
// Azure Foundry deploys these with larger context windows than OpenAI's own API,
// which caps gpt-5.4/gpt-5.5 at 272k. See models-sold-directly-by-azure docs.
const AZURE_CONTEXT_WINDOW_OVERRIDES: Record<string, number> = {
@@ -2049,6 +2236,7 @@ async function generateModels() {
for (const model of allModels) {
applyThinkingLevelMetadata(model);
applyOpenAICompletionsCompatMetadata(model);
}
// Group by provider and deduplicate by model ID
@@ -2064,62 +2252,80 @@ async function generateModels() {
}
}
// Generate TypeScript file
let output = `// This file is auto-generated by scripts/generate-models.ts
// Generate TypeScript files: one catalog per provider plus an aggregator
const generatedHeader = `// This file is auto-generated by scripts/generate-models.ts
// Do not edit manually - run 'npm run generate-models' to update
import type { Model } from "./types.ts";
export const MODELS = {
`;
const catalogConstName = (providerId: string) => `${providerId.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_MODELS`;
// Generate provider sections (sorted for deterministic output)
const sortedProviderIds = Object.keys(providers).sort();
for (const providerId of sortedProviderIds) {
const models = providers[providerId];
output += `\t${JSON.stringify(providerId)}: {\n`;
const sortedModelIds = Object.keys(models).sort();
for (const modelId of sortedModelIds) {
const model = models[modelId];
output += `\t\t"${model.id}": {\n`;
output += `\t\t\tid: "${model.id}",\n`;
output += `\t\t\tname: "${model.name}",\n`;
output += `\t\t\tapi: "${model.api}",\n`;
output += `\t\t\tprovider: "${model.provider}",\n`;
function emitModel(model: Model<any>, indent: string): string {
let output = `${indent}"${model.id}": {\n`;
output += `${indent}\tid: "${model.id}",\n`;
output += `${indent}\tname: "${model.name}",\n`;
output += `${indent}\tapi: "${model.api}",\n`;
output += `${indent}\tprovider: "${model.provider}",\n`;
if (model.baseUrl !== undefined) {
output += `\t\t\tbaseUrl: "${model.baseUrl}",\n`;
output += `${indent}\tbaseUrl: "${model.baseUrl}",\n`;
}
if (model.headers) {
output += `\t\t\theaders: ${JSON.stringify(model.headers)},\n`;
output += `${indent}\theaders: ${JSON.stringify(model.headers)},\n`;
}
if (model.compat) {
output += ` compat: ${JSON.stringify(model.compat)},
`;
output += `${indent}\tcompat: ${JSON.stringify(model.compat)},\n`;
}
output += `\t\t\treasoning: ${model.reasoning},\n`;
output += `${indent}\treasoning: ${model.reasoning},\n`;
if (model.thinkingLevelMap) {
output += `\t\t\tthinkingLevelMap: ${JSON.stringify(model.thinkingLevelMap)},\n`;
output += `${indent}\tthinkingLevelMap: ${JSON.stringify(model.thinkingLevelMap)},\n`;
}
output += `\t\t\tinput: [${model.input.map(i => `"${i}"`).join(", ")}],\n`;
output += `\t\t\tcost: {\n`;
output += `\t\t\t\tinput: ${model.cost.input},\n`;
output += `\t\t\t\toutput: ${model.cost.output},\n`;
output += `\t\t\t\tcacheRead: ${model.cost.cacheRead},\n`;
output += `\t\t\t\tcacheWrite: ${model.cost.cacheWrite},\n`;
output += `\t\t\t},\n`;
output += `\t\t\tcontextWindow: ${model.contextWindow},\n`;
output += `\t\t\tmaxTokens: ${model.maxTokens},\n`;
output += `\t\t} satisfies Model<"${model.api}">,\n`;
output += `${indent}\tinput: [${model.input.map(i => `"${i}"`).join(", ")}],\n`;
output += `${indent}\tcost: {\n`;
output += `${indent}\t\tinput: ${model.cost.input},\n`;
output += `${indent}\t\toutput: ${model.cost.output},\n`;
output += `${indent}\t\tcacheRead: ${model.cost.cacheRead},\n`;
output += `${indent}\t\tcacheWrite: ${model.cost.cacheWrite},\n`;
output += `${indent}\t},\n`;
output += `${indent}\tcontextWindow: ${model.contextWindow},\n`;
output += `${indent}\tmaxTokens: ${model.maxTokens},\n`;
output += `${indent}} satisfies Model<"${model.api}">,\n`;
return output;
}
output += `\t},\n`;
const sortedProviderIds = Object.keys(providers).sort();
const providersDir = join(packageRoot, "src/providers");
// Remove stale per-provider catalogs
for (const entry of readdirSync(providersDir)) {
if (entry.endsWith(".models.ts")) {
rmSync(join(providersDir, entry));
}
}
output += `} as const;
`;
// Per-provider catalogs (sorted for deterministic output)
for (const providerId of sortedProviderIds) {
const models = providers[providerId];
let output = generatedHeader;
output += `import type { Model } from "../types.ts";\n\n`;
output += `export const ${catalogConstName(providerId)} = {\n`;
const sortedModelIds = Object.keys(models).sort();
for (const modelId of sortedModelIds) {
output += emitModel(models[modelId], "\t");
}
output += `} as const;\n`;
writeFileSync(join(providersDir, `${providerId}.models.ts`), output);
}
console.log(`Generated ${sortedProviderIds.length} catalogs under src/providers/`);
// Write file
// Aggregator
let output = generatedHeader;
for (const providerId of sortedProviderIds) {
output += `import { ${catalogConstName(providerId)} } from "./providers/${providerId}.models.ts";\n`;
}
output += `\nexport const MODELS = {\n`;
for (const providerId of sortedProviderIds) {
output += `\t${JSON.stringify(providerId)}: ${catalogConstName(providerId)},\n`;
}
output += `} as const;\n`;
writeFileSync(join(packageRoot, "src/models.generated.ts"), output);
console.log("Generated src/models.generated.ts");
packages/ai/src/api-registry.ts
-98
View File
@@ -1,98 +0,0 @@
import type {
Api,
AssistantMessageEventStream,
Context,
Model,
SimpleStreamOptions,
StreamFunction,
StreamOptions,
} from "./types.ts";
export type ApiStreamFunction = (
model: Model<Api>,
context: Context,
options?: StreamOptions,
) => AssistantMessageEventStream;
export type ApiStreamSimpleFunction = (
model: Model<Api>,
context: Context,
options?: SimpleStreamOptions,
) => AssistantMessageEventStream;
export interface ApiProvider<TApi extends Api = Api, TOptions extends StreamOptions = StreamOptions> {
api: TApi;
stream: StreamFunction<TApi, TOptions>;
streamSimple: StreamFunction<TApi, SimpleStreamOptions>;
}
interface ApiProviderInternal {
api: Api;
stream: ApiStreamFunction;
streamSimple: ApiStreamSimpleFunction;
}
type RegisteredApiProvider = {
provider: ApiProviderInternal;
sourceId?: string;
};
const apiProviderRegistry = new Map<string, RegisteredApiProvider>();
function wrapStream<TApi extends Api, TOptions extends StreamOptions>(
api: TApi,
stream: StreamFunction<TApi, TOptions>,
): ApiStreamFunction {
return (model, context, options) => {
if (model.api !== api) {
throw new Error(`Mismatched api: ${model.api} expected ${api}`);
}
return stream(model as Model<TApi>, context, options as TOptions);
};
}
function wrapStreamSimple<TApi extends Api>(
api: TApi,
streamSimple: StreamFunction<TApi, SimpleStreamOptions>,
): ApiStreamSimpleFunction {
return (model, context, options) => {
if (model.api !== api) {
throw new Error(`Mismatched api: ${model.api} expected ${api}`);
}
return streamSimple(model as Model<TApi>, context, options);
};
}
export function registerApiProvider<TApi extends Api, TOptions extends StreamOptions>(
provider: ApiProvider<TApi, TOptions>,
sourceId?: string,
): void {
apiProviderRegistry.set(provider.api, {
provider: {
api: provider.api,
stream: wrapStream(provider.api, provider.stream),
streamSimple: wrapStreamSimple(provider.api, provider.streamSimple),
},
sourceId,
});
}
export function getApiProvider(api: Api): ApiProviderInternal | undefined {
return apiProviderRegistry.get(api)?.provider;
}
export function getApiProviders(): ApiProviderInternal[] {
return Array.from(apiProviderRegistry.values(), (entry) => entry.provider);
}
export function unregisterApiProviders(sourceId: string): void {
for (const [api, entry] of apiProviderRegistry.entries()) {
if (entry.sourceId === sourceId) {
apiProviderRegistry.delete(api);
}
}
}
export function clearApiProviders(): void {
apiProviderRegistry.clear();
}
packages/ai/src/api/anthropic-messages.lazy.ts
+4
View File
@@ -0,0 +1,4 @@
import type { ProviderStreams } from "../types.ts";
import { lazyApi } from "./lazy.ts";
export const anthropicMessagesApi = (): ProviderStreams => lazyApi(() => import("./anthropic-messages.ts"));
packages/ai/src/api/anthropic-messages.ts
+1229
View File
File diff suppressed because it is too large Load Diff
packages/ai/src/api/azure-openai-responses.lazy.ts
+4
View File
@@ -0,0 +1,4 @@
import type { ProviderStreams } from "../types.ts";
import { lazyApi } from "./lazy.ts";
export const azureOpenAIResponsesApi = (): ProviderStreams => lazyApi(() => import("./azure-openai-responses.ts"));
packages/ai/src/api/azure-openai-responses.ts
+307
View File
@@ -0,0 +1,307 @@
import { AzureOpenAI } from "openai";
import type { ResponseCreateParamsStreaming } from "openai/resources/responses/responses.js";
import { clampThinkingLevel } from "../models.ts";
import type {
Api,
AssistantMessage,
Context,
Model,
SimpleStreamOptions,
StreamFunction,
StreamOptions,
} from "../types.ts";
import { AssistantMessageEventStream } from "../utils/event-stream.ts";
import { headersToRecord } from "../utils/headers.ts";
import { getProviderEnvValue } from "../utils/provider-env.ts";
import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts";
import { convertResponsesMessages, convertResponsesTools, processResponsesStream } from "./openai-responses-shared.ts";
import { buildBaseOptions } from "./simple-options.ts";
const DEFAULT_AZURE_API_VERSION = "v1";
const AZURE_TOOL_CALL_PROVIDERS = new Set(["openai", "openai-codex", "opencode", "azure-openai-responses"]);
function parseDeploymentNameMap(value: string | undefined): Map<string, string> {
const map = new Map<string, string>();
if (!value) return map;
for (const entry of value.split(",")) {
const trimmed = entry.trim();
if (!trimmed) continue;
const [modelId, deploymentName] = trimmed.split("=", 2);
if (!modelId || !deploymentName) continue;
map.set(modelId.trim(), deploymentName.trim());
}
return map;
}
function resolveDeploymentName(model: Model<"azure-openai-responses">, options?: AzureOpenAIResponsesOptions): string {
if (options?.azureDeploymentName) {
return options.azureDeploymentName;
}
const mappedDeployment = parseDeploymentNameMap(
getProviderEnvValue("AZURE_OPENAI_DEPLOYMENT_NAME_MAP", options?.env),
).get(model.id);
return mappedDeployment || model.id;
}
function formatAzureOpenAIError(error: unknown): string {
if (error instanceof Error) {
const status = (error as Error & { status?: unknown }).status;
const statusCode = typeof status === "number" ? status : undefined;
if (statusCode !== undefined) {
return `Azure OpenAI API error (${statusCode}): ${error.message}`;
}
return error.message;
}
try {
return JSON.stringify(error);
} catch {
return String(error);
}
}
// Azure OpenAI Responses-specific options
export interface AzureOpenAIResponsesOptions extends StreamOptions {
reasoningEffort?: "minimal" | "low" | "medium" | "high" | "xhigh";
reasoningSummary?: "auto" | "detailed" | "concise" | null;
azureApiVersion?: string;
azureResourceName?: string;
azureBaseUrl?: string;
azureDeploymentName?: string;
}
/**
* Generate function for Azure OpenAI Responses API
*/
export const stream: StreamFunction<"azure-openai-responses", AzureOpenAIResponsesOptions> = (
model: Model<"azure-openai-responses">,
context: Context,
options?: AzureOpenAIResponsesOptions,
): AssistantMessageEventStream => {
const stream = new AssistantMessageEventStream();
// Start async processing
(async () => {
const deploymentName = resolveDeploymentName(model, options);
const output: AssistantMessage = {
role: "assistant",
content: [],
api: "azure-openai-responses" as Api,
provider: model.provider,
model: model.id,
usage: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "stop",
timestamp: Date.now(),
};
try {
// Create Azure OpenAI client
const apiKey = options?.apiKey;
if (!apiKey) {
throw new Error(`No API key for provider: ${model.provider}`);
}
const client = createClient(model, apiKey, options);
let params = buildParams(model, context, options, deploymentName);
const nextParams = await options?.onPayload?.(params, model);
if (nextParams !== undefined) {
params = nextParams as ResponseCreateParamsStreaming;
}
const requestOptions = {
...(options?.signal ? { signal: options.signal } : {}),
...(options?.timeoutMs !== undefined ? { timeout: options.timeoutMs } : {}),
maxRetries: options?.maxRetries ?? 0,
};
const { data: openaiStream, response } = await client.responses.create(params, requestOptions).withResponse();
await options?.onResponse?.({ status: response.status, headers: headersToRecord(response.headers) }, model);
stream.push({ type: "start", partial: output });
await processResponsesStream(openaiStream, output, stream, model);
if (options?.signal?.aborted) {
throw new Error("Request was aborted");
}
if (output.stopReason === "aborted" || output.stopReason === "error") {
throw new Error("An unknown error occurred");
}
stream.push({ type: "done", reason: output.stopReason, message: output });
stream.end();
} catch (error) {
for (const block of output.content) {
delete (block as { index?: number }).index;
// partialJson is only a streaming scratch buffer; never persist it.
delete (block as { partialJson?: string }).partialJson;
}
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
output.errorMessage = formatAzureOpenAIError(error);
stream.push({ type: "error", reason: output.stopReason, error: output });
stream.end();
}
})();
return stream;
};
export const streamSimple: StreamFunction<"azure-openai-responses", SimpleStreamOptions> = (
model: Model<"azure-openai-responses">,
context: Context,
options?: SimpleStreamOptions,
): AssistantMessageEventStream => {
const apiKey = options?.apiKey;
if (!apiKey) {
throw new Error(`No API key for provider: ${model.provider}`);
}
const base = buildBaseOptions(model, options, apiKey);
const clampedReasoning = options?.reasoning ? clampThinkingLevel(model, options.reasoning) : undefined;
const reasoningEffort = clampedReasoning === "off" ? undefined : clampedReasoning;
return stream(model, context, {
...base,
reasoningEffort,
} satisfies AzureOpenAIResponsesOptions);
};
function normalizeAzureBaseUrl(baseUrl: string): string {
const trimmed = baseUrl.trim().replace(/\/+$/, "");
let url: URL;
try {
url = new URL(trimmed);
} catch {
throw new Error(`Invalid Azure OpenAI base URL: ${baseUrl}`);
}
const isAzureHost =
url.hostname.endsWith(".openai.azure.com") ||
url.hostname.endsWith(".cognitiveservices.azure.com") ||
url.hostname.endsWith(".ai.azure.com");
const normalizedPath = url.pathname.replace(/\/+$/, "");
// Ensure Azure hosts have /openai/v1 as base path so the AzureOpenAI SDK
// can append /deployments/<model>/... and ?api-version=v1 correctly.
if (
isAzureHost &&
(normalizedPath === "" ||
normalizedPath === "/" ||
normalizedPath === "/openai" ||
normalizedPath === "/openai/v1/responses")
) {
url.pathname = "/openai/v1";
url.search = "";
}
return url.toString().replace(/\/+$/, "");
}
function buildDefaultBaseUrl(resourceName: string): string {
return `https://${resourceName}.openai.azure.com/openai/v1`;
}
function resolveAzureConfig(
model: Model<"azure-openai-responses">,
options?: AzureOpenAIResponsesOptions,
): { baseUrl: string; apiVersion: string } {
const apiVersion =
options?.azureApiVersion ||
getProviderEnvValue("AZURE_OPENAI_API_VERSION", options?.env) ||
DEFAULT_AZURE_API_VERSION;
const baseUrl =
options?.azureBaseUrl?.trim() || getProviderEnvValue("AZURE_OPENAI_BASE_URL", options?.env)?.trim() || undefined;
const resourceName = options?.azureResourceName || getProviderEnvValue("AZURE_OPENAI_RESOURCE_NAME", options?.env);
let resolvedBaseUrl = baseUrl;
if (!resolvedBaseUrl && resourceName) {
resolvedBaseUrl = buildDefaultBaseUrl(resourceName);
}
if (!resolvedBaseUrl && model.baseUrl) {
resolvedBaseUrl = model.baseUrl;
}
if (!resolvedBaseUrl) {
throw new Error(
"Azure OpenAI base URL is required. Set AZURE_OPENAI_BASE_URL or AZURE_OPENAI_RESOURCE_NAME, or pass azureBaseUrl, azureResourceName, or model.baseUrl.",
);
}
return {
baseUrl: normalizeAzureBaseUrl(resolvedBaseUrl),
apiVersion,
};
}
function createClient(model: Model<"azure-openai-responses">, apiKey: string, options?: AzureOpenAIResponsesOptions) {
const headers = { ...model.headers };
if (options?.headers) {
Object.assign(headers, options.headers);
}
const { baseUrl, apiVersion } = resolveAzureConfig(model, options);
return new AzureOpenAI({
apiKey,
apiVersion,
dangerouslyAllowBrowser: true,
defaultHeaders: headers,
baseURL: baseUrl,
});
}
function buildParams(
model: Model<"azure-openai-responses">,
context: Context,
options: AzureOpenAIResponsesOptions | undefined,
deploymentName: string,
) {
const messages = convertResponsesMessages(model, context, AZURE_TOOL_CALL_PROVIDERS);
const params: ResponseCreateParamsStreaming = {
model: deploymentName,
input: messages,
stream: true,
prompt_cache_key: clampOpenAIPromptCacheKey(options?.sessionId),
store: false,
};
if (options?.maxTokens) {
params.max_output_tokens = options?.maxTokens;
}
if (options?.temperature !== undefined) {
params.temperature = options?.temperature;
}
if (context.tools && context.tools.length > 0) {
params.tools = convertResponsesTools(context.tools);
}
if (model.reasoning) {
if (options?.reasoningEffort || options?.reasoningSummary) {
const effort = options?.reasoningEffort
? (model.thinkingLevelMap?.[options.reasoningEffort] ?? options.reasoningEffort)
: "medium";
params.reasoning = {
effort: effort as NonNullable<typeof params.reasoning>["effort"],
summary: options?.reasoningSummary || "auto",
};
params.include = ["reasoning.encrypted_content"];
} else if (model.thinkingLevelMap?.off !== null) {
params.reasoning = {
effort: (model.thinkingLevelMap?.off ?? "none") as NonNullable<typeof params.reasoning>["effort"],
};
}
}
return params;
}
packages/ai/src/api/bedrock-converse-stream.lazy.ts
+30
View File
@@ -0,0 +1,30 @@
import type { ProviderStreams } from "../types.ts";
import { lazyApi } from "./lazy.ts";
/**
* Loads the bedrock implementation through a variable specifier so bundlers
* (browser smoke, Bun compile) cannot follow the import into the Node-only
* AWS SDK. The `.ts`/`.js` rewrite keeps the trick working from both source
* and built output.
*/
const importNodeOnlyApi = (specifier: string): Promise<unknown> => {
const runtimeSpecifier = import.meta.url.endsWith(".js") ? specifier.replace(/\.ts$/, ".js") : specifier;
return import(runtimeSpecifier);
};
let bedrockModuleOverride: ProviderStreams | undefined;
/**
* Overrides the dynamically imported bedrock implementation. Used by the Bun
* binary build, where the variable-specifier import cannot be bundled; the
* build registers a statically imported module instead.
*/
export function setBedrockProviderModule(module: ProviderStreams): void {
bedrockModuleOverride = module;
}
export const bedrockConverseStreamApi = (): ProviderStreams =>
lazyApi(
async () =>
bedrockModuleOverride ?? ((await importNodeOnlyApi("./bedrock-converse-stream.ts")) as ProviderStreams),
);
packages/ai/src/api/bedrock-converse-stream.ts
+1063
View File
File diff suppressed because it is too large Load Diff
packages/ai/src/providers/cloudflare.ts → packages/ai/src/api/cloudflare.ts
-21
View File
@@ -1,6 +1,3 @@
import type { Api, Model, ProviderEnv } from "../types.ts";
import { getProviderEnvValue } from "../utils/provider-env.ts";
/** Workers AI direct endpoint. */
export const CLOUDFLARE_WORKERS_AI_BASE_URL =
"https://api.cloudflare.com/client/v4/accounts/{CLOUDFLARE_ACCOUNT_ID}/ai/v1";
@@ -16,21 +13,3 @@ export const CLOUDFLARE_AI_GATEWAY_OPENAI_BASE_URL =
/** AI Gateway → Anthropic passthrough. */
export const CLOUDFLARE_AI_GATEWAY_ANTHROPIC_BASE_URL =
"https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/anthropic";
export function isCloudflareProvider(provider: string): boolean {
return provider === "cloudflare-workers-ai" || provider === "cloudflare-ai-gateway";
}
/** Substitute `{VAR}` placeholders in a Cloudflare baseUrl from provider env or process.env. */
export function resolveCloudflareBaseUrl(model: Model<Api>, env?: ProviderEnv): string {
const url = model.baseUrl;
if (!url.includes("{")) return url;
const baseUrl = url.replace(/\{([A-Z_][A-Z0-9_]*)\}/g, (_match, name: string) => {
const value = getProviderEnvValue(name, env);
if (!value) {
throw new Error(`${name} is required for provider ${model.provider} but is not set.`);
}
return value;
});
return baseUrl;
}
packages/ai/src/providers/github-copilot-headers.ts → packages/ai/src/api/github-copilot-headers.ts
View File
packages/ai/src/api/google-generative-ai.lazy.ts
+4
View File
@@ -0,0 +1,4 @@
import type { ProviderStreams } from "../types.ts";
import { lazyApi } from "./lazy.ts";
export const googleGenerativeAIApi = (): ProviderStreams => lazyApi(() => import("./google-generative-ai.ts"));
packages/ai/src/api/google-generative-ai.ts
+507
View File
@@ -0,0 +1,507 @@
import {
type GenerateContentConfig,
type GenerateContentParameters,
GoogleGenAI,
type ThinkingConfig,
} from "@google/genai";
import { calculateCost, clampThinkingLevel } from "../models.ts";
import type {
Api,
AssistantMessage,
Context,
Model,
ProviderHeaders,
SimpleStreamOptions,
StreamFunction,
StreamOptions,
TextContent,
ThinkingBudgets,
ThinkingContent,
ThinkingLevel,
ToolCall,
} from "../types.ts";
import { AssistantMessageEventStream } from "../utils/event-stream.ts";
import { providerHeadersToRecord } from "../utils/headers.ts";
import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts";
import type { GoogleThinkingLevel } from "./google-shared.ts";
import {
convertMessages,
convertTools,
isThinkingPart,
mapStopReason,
mapToolChoice,
retainThoughtSignature,
} from "./google-shared.ts";
import { buildBaseOptions } from "./simple-options.ts";
export interface GoogleOptions extends StreamOptions {
toolChoice?: "auto" | "none" | "any";
thinking?: {
enabled: boolean;
budgetTokens?: number; // -1 for dynamic, 0 to disable
level?: GoogleThinkingLevel;
};
}
// Counter for generating unique tool call IDs
let toolCallCounter = 0;
export const stream: StreamFunction<"google-generative-ai", GoogleOptions> = (
model: Model<"google-generative-ai">,
context: Context,
options?: GoogleOptions,
): AssistantMessageEventStream => {
const stream = new AssistantMessageEventStream();
(async () => {
const output: AssistantMessage = {
role: "assistant",
content: [],
api: "google-generative-ai" as Api,
provider: model.provider,
model: model.id,
usage: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "stop",
timestamp: Date.now(),
};
try {
const apiKey = options?.apiKey;
if (!apiKey) {
throw new Error(`No API key for provider: ${model.provider}`);
}
const client = createClient(model, apiKey, options?.headers);
let params = buildParams(model, context, options);
const nextParams = await options?.onPayload?.(params, model);
if (nextParams !== undefined) {
params = nextParams as GenerateContentParameters;
}
const googleStream = await client.models.generateContentStream(params);
stream.push({ type: "start", partial: output });
let currentBlock: TextContent | ThinkingContent | null = null;
const blocks = output.content;
const blockIndex = () => blocks.length - 1;
for await (const chunk of googleStream) {
// @google/genai documents GenerateContentResponse.responseId as an output-only field
// used to identify each response. Keep the first non-empty one from the stream.
output.responseId ||= chunk.responseId;
const candidate = chunk.candidates?.[0];
if (candidate?.content?.parts) {
for (const part of candidate.content.parts) {
if (part.text !== undefined) {
const isThinking = isThinkingPart(part);
if (
!currentBlock ||
(isThinking && currentBlock.type !== "thinking") ||
(!isThinking && currentBlock.type !== "text")
) {
if (currentBlock) {
if (currentBlock.type === "text") {
stream.push({
type: "text_end",
contentIndex: blocks.length - 1,
content: currentBlock.text,
partial: output,
});
} else {
stream.push({
type: "thinking_end",
contentIndex: blockIndex(),
content: currentBlock.thinking,
partial: output,
});
}
}
if (isThinking) {
currentBlock = { type: "thinking", thinking: "", thinkingSignature: undefined };
output.content.push(currentBlock);
stream.push({ type: "thinking_start", contentIndex: blockIndex(), partial: output });
} else {
currentBlock = { type: "text", text: "" };
output.content.push(currentBlock);
stream.push({ type: "text_start", contentIndex: blockIndex(), partial: output });
}
}
if (currentBlock.type === "thinking") {
currentBlock.thinking += part.text;
currentBlock.thinkingSignature = retainThoughtSignature(
currentBlock.thinkingSignature,
part.thoughtSignature,
);
stream.push({
type: "thinking_delta",
contentIndex: blockIndex(),
delta: part.text,
partial: output,
});
} else {
currentBlock.text += part.text;
currentBlock.textSignature = retainThoughtSignature(
currentBlock.textSignature,
part.thoughtSignature,
);
stream.push({
type: "text_delta",
contentIndex: blockIndex(),
delta: part.text,
partial: output,
});
}
}
if (part.functionCall) {
if (currentBlock) {
if (currentBlock.type === "text") {
stream.push({
type: "text_end",
contentIndex: blockIndex(),
content: currentBlock.text,
partial: output,
});
} else {
stream.push({
type: "thinking_end",
contentIndex: blockIndex(),
content: currentBlock.thinking,
partial: output,
});
}
currentBlock = null;
}
// Generate unique ID if not provided or if it's a duplicate
const providedId = part.functionCall.id;
const needsNewId =
!providedId || output.content.some((b) => b.type === "toolCall" && b.id === providedId);
const toolCallId = needsNewId
? `${part.functionCall.name}_${Date.now()}_${++toolCallCounter}`
: providedId;
const toolCall: ToolCall = {
type: "toolCall",
id: toolCallId,
name: part.functionCall.name || "",
arguments: (part.functionCall.args as Record<string, any>) ?? {},
...(part.thoughtSignature && { thoughtSignature: part.thoughtSignature }),
};
output.content.push(toolCall);
stream.push({ type: "toolcall_start", contentIndex: blockIndex(), partial: output });
stream.push({
type: "toolcall_delta",
contentIndex: blockIndex(),
delta: JSON.stringify(toolCall.arguments),
partial: output,
});
stream.push({ type: "toolcall_end", contentIndex: blockIndex(), toolCall, partial: output });
}
}
}
if (candidate?.finishReason) {
output.stopReason = mapStopReason(candidate.finishReason);
if (output.content.some((b) => b.type === "toolCall")) {
output.stopReason = "toolUse";
}
}
if (chunk.usageMetadata) {
output.usage = {
input:
(chunk.usageMetadata.promptTokenCount || 0) - (chunk.usageMetadata.cachedContentTokenCount || 0),
output:
(chunk.usageMetadata.candidatesTokenCount || 0) + (chunk.usageMetadata.thoughtsTokenCount || 0),
cacheRead: chunk.usageMetadata.cachedContentTokenCount || 0,
cacheWrite: 0,
totalTokens: chunk.usageMetadata.totalTokenCount || 0,
cost: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
total: 0,
},
};
calculateCost(model, output.usage);
}
}
if (currentBlock) {
if (currentBlock.type === "text") {
stream.push({
type: "text_end",
contentIndex: blockIndex(),
content: currentBlock.text,
partial: output,
});
} else {
stream.push({
type: "thinking_end",
contentIndex: blockIndex(),
content: currentBlock.thinking,
partial: output,
});
}
}
if (options?.signal?.aborted) {
throw new Error("Request was aborted");
}
if (output.stopReason === "aborted" || output.stopReason === "error") {
throw new Error("An unknown error occurred");
}
stream.push({ type: "done", reason: output.stopReason, message: output });
stream.end();
} catch (error) {
// Remove internal index property used during streaming
for (const block of output.content) {
if ("index" in block) {
delete (block as { index?: number }).index;
}
}
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
output.errorMessage = error instanceof Error ? error.message : JSON.stringify(error);
stream.push({ type: "error", reason: output.stopReason, error: output });
stream.end();
}
})();
return stream;
};
export const streamSimple: StreamFunction<"google-generative-ai", SimpleStreamOptions> = (
model: Model<"google-generative-ai">,
context: Context,
options?: SimpleStreamOptions,
): AssistantMessageEventStream => {
const apiKey = options?.apiKey;
if (!apiKey) {
throw new Error(`No API key for provider: ${model.provider}`);
}
const base = buildBaseOptions(model, options, apiKey);
if (!options?.reasoning) {
return stream(model, context, { ...base, thinking: { enabled: false } } satisfies GoogleOptions);
}
const clampedReasoning = clampThinkingLevel(model, options.reasoning);
const effort = (clampedReasoning === "off" ? "high" : clampedReasoning) as ClampedThinkingLevel;
const googleModel = model as Model<"google-generative-ai">;
if (isGemini3ProModel(googleModel) || isGemini3FlashModel(googleModel) || isGemma4Model(googleModel)) {
return stream(model, context, {
...base,
thinking: {
enabled: true,
level: getThinkingLevel(effort, googleModel),
},
} satisfies GoogleOptions);
}
return stream(model, context, {
...base,
thinking: {
enabled: true,
budgetTokens: getGoogleBudget(googleModel, effort, options.thinkingBudgets),
},
} satisfies GoogleOptions);
};
function createClient(
model: Model<"google-generative-ai">,
apiKey?: string,
optionsHeaders?: ProviderHeaders,
): GoogleGenAI {
const httpOptions: { baseUrl?: string; apiVersion?: string; headers?: Record<string, string> } = {};
if (model.baseUrl) {
httpOptions.baseUrl = model.baseUrl;
httpOptions.apiVersion = ""; // baseUrl already includes version path, don't append
}
const headers = providerHeadersToRecord({ ...model.headers, ...optionsHeaders });
if (headers) {
httpOptions.headers = headers;
}
return new GoogleGenAI({
apiKey,
httpOptions: Object.keys(httpOptions).length > 0 ? httpOptions : undefined,
});
}
function buildParams(
model: Model<"google-generative-ai">,
context: Context,
options: GoogleOptions = {},
): GenerateContentParameters {
const contents = convertMessages(model, context);
const generationConfig: GenerateContentConfig = {};
if (options.temperature !== undefined) {
generationConfig.temperature = options.temperature;
}
if (options.maxTokens !== undefined) {
generationConfig.maxOutputTokens = options.maxTokens;
}
const config: GenerateContentConfig = {
...(Object.keys(generationConfig).length > 0 && generationConfig),
...(context.systemPrompt && { systemInstruction: sanitizeSurrogates(context.systemPrompt) }),
...(context.tools && context.tools.length > 0 && { tools: convertTools(context.tools) }),
};
if (context.tools && context.tools.length > 0 && options.toolChoice) {
config.toolConfig = {
functionCallingConfig: {
mode: mapToolChoice(options.toolChoice),
},
};
} else {
config.toolConfig = undefined;
}
if (options.thinking?.enabled && model.reasoning) {
const thinkingConfig: ThinkingConfig = { includeThoughts: true };
if (options.thinking.level !== undefined) {
// Cast to any since our GoogleThinkingLevel mirrors Google's ThinkingLevel enum values
thinkingConfig.thinkingLevel = options.thinking.level as any;
} else if (options.thinking.budgetTokens !== undefined) {
thinkingConfig.thinkingBudget = options.thinking.budgetTokens;
}
config.thinkingConfig = thinkingConfig;
} else if (model.reasoning && options.thinking && !options.thinking.enabled) {
config.thinkingConfig = getDisabledThinkingConfig(model);
}
if (options.signal) {
if (options.signal.aborted) {
throw new Error("Request aborted");
}
config.abortSignal = options.signal;
}
const params: GenerateContentParameters = {
model: model.id,
contents,
config,
};
return params;
}
type ClampedThinkingLevel = Exclude<ThinkingLevel, "xhigh">;
function isGemma4Model(model: Model<"google-generative-ai">): boolean {
return /gemma-?4/.test(model.id.toLowerCase());
}
function isGemini3ProModel(model: Model<"google-generative-ai">): boolean {
return /gemini-3(?:\.\d+)?-pro/.test(model.id.toLowerCase());
}
function isGemini3FlashModel(model: Model<"google-generative-ai">): boolean {
const id = model.id.toLowerCase();
return /gemini-3(?:\.\d+)?-flash/.test(id) || id === "gemini-flash-latest" || id === "gemini-flash-lite-latest";
}
function getDisabledThinkingConfig(model: Model<"google-generative-ai">): ThinkingConfig {
// Google docs: Gemini 3.1 Pro cannot disable thinking, and Gemini 3 Flash / Flash-Lite
// do not support full thinking-off either. For Gemini 3 models, use the lowest supported
// thinkingLevel without includeThoughts so hidden thinking remains invisible to pi.
if (isGemini3ProModel(model)) {
return { thinkingLevel: "LOW" as any };
}
if (isGemini3FlashModel(model)) {
return { thinkingLevel: "MINIMAL" as any };
}
if (isGemma4Model(model)) {
return { thinkingLevel: "MINIMAL" as any };
}
// Gemini 2.x supports disabling via thinkingBudget = 0.
return { thinkingBudget: 0 };
}
function getThinkingLevel(effort: ClampedThinkingLevel, model: Model<"google-generative-ai">): GoogleThinkingLevel {
if (isGemini3ProModel(model)) {
switch (effort) {
case "minimal":
case "low":
return "LOW";
case "medium":
case "high":
return "HIGH";
}
}
if (isGemma4Model(model)) {
switch (effort) {
case "minimal":
case "low":
return "MINIMAL";
case "medium":
case "high":
return "HIGH";
}
}
switch (effort) {
case "minimal":
return "MINIMAL";
case "low":
return "LOW";
case "medium":
return "MEDIUM";
case "high":
return "HIGH";
}
}
function getGoogleBudget(
model: Model<"google-generative-ai">,
effort: ClampedThinkingLevel,
customBudgets?: ThinkingBudgets,
): number {
if (customBudgets?.[effort] !== undefined) {
return customBudgets[effort]!;
}
if (model.id.includes("2.5-pro")) {
const budgets: Record<ClampedThinkingLevel, number> = {
minimal: 128,
low: 2048,
medium: 8192,
high: 32768,
};
return budgets[effort];
}
if (model.id.includes("2.5-flash-lite")) {
const budgets: Record<ClampedThinkingLevel, number> = {
minimal: 512,
low: 2048,
medium: 8192,
high: 24576,
};
return budgets[effort];
}
if (model.id.includes("2.5-flash")) {
const budgets: Record<ClampedThinkingLevel, number> = {
minimal: 128,
low: 2048,
medium: 8192,
high: 24576,
};
return budgets[effort];
}
return -1;
}
packages/ai/src/providers/google-shared.ts → packages/ai/src/api/google-shared.ts
View File
packages/ai/src/api/google-vertex.lazy.ts
+4
View File
@@ -0,0 +1,4 @@
import type { ProviderStreams } from "../types.ts";
import { lazyApi } from "./lazy.ts";
export const googleVertexApi = (): ProviderStreams => lazyApi(() => import("./google-vertex.ts"));
packages/ai/src/api/google-vertex.ts
+582
View File
@@ -0,0 +1,582 @@
import {
type GenerateContentConfig,
type GenerateContentParameters,
GoogleGenAI,
type HttpOptions,
ResourceScope,
type ThinkingConfig,
ThinkingLevel,
} from "@google/genai";
import { calculateCost, clampThinkingLevel } from "../models.ts";
import type {
Api,
AssistantMessage,
Context,
Model,
ThinkingLevel as PiThinkingLevel,
ProviderEnv,
ProviderHeaders,
SimpleStreamOptions,
StreamFunction,
StreamOptions,
TextContent,
ThinkingBudgets,
ThinkingContent,
ToolCall,
} from "../types.ts";
import { AssistantMessageEventStream } from "../utils/event-stream.ts";
import { providerHeadersToRecord } from "../utils/headers.ts";
import { getProviderEnvValue } from "../utils/provider-env.ts";
import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts";
import type { GoogleThinkingLevel } from "./google-shared.ts";
import {
convertMessages,
convertTools,
isThinkingPart,
mapStopReason,
mapToolChoice,
retainThoughtSignature,
} from "./google-shared.ts";
import { buildBaseOptions } from "./simple-options.ts";
export interface GoogleVertexOptions extends StreamOptions {
toolChoice?: "auto" | "none" | "any";
thinking?: {
enabled: boolean;
budgetTokens?: number; // -1 for dynamic, 0 to disable
level?: GoogleThinkingLevel;
};
project?: string;
location?: string;
}
const API_VERSION = "v1";
const GCP_VERTEX_CREDENTIALS_MARKER = "gcp-vertex-credentials";
const THINKING_LEVEL_MAP: Record<GoogleThinkingLevel, ThinkingLevel> = {
THINKING_LEVEL_UNSPECIFIED: ThinkingLevel.THINKING_LEVEL_UNSPECIFIED,
MINIMAL: ThinkingLevel.MINIMAL,
LOW: ThinkingLevel.LOW,
MEDIUM: ThinkingLevel.MEDIUM,
HIGH: ThinkingLevel.HIGH,
};
// Counter for generating unique tool call IDs
let toolCallCounter = 0;
export const stream: StreamFunction<"google-vertex", GoogleVertexOptions> = (
model: Model<"google-vertex">,
context: Context,
options?: GoogleVertexOptions,
): AssistantMessageEventStream => {
const stream = new AssistantMessageEventStream();
(async () => {
const output: AssistantMessage = {
role: "assistant",
content: [],
api: "google-vertex" as Api,
provider: model.provider,
model: model.id,
usage: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "stop",
timestamp: Date.now(),
};
try {
const apiKey = resolveApiKey(options);
// Create the client using either a Vertex API key, if provided, or ADC with project and location
const client = apiKey
? createClientWithApiKey(model, apiKey, options?.headers)
: createClient(model, resolveProject(options), resolveLocation(options), options?.headers, options?.env);
let params = buildParams(model, context, options);
const nextParams = await options?.onPayload?.(params, model);
if (nextParams !== undefined) {
params = nextParams as GenerateContentParameters;
}
const googleStream = await client.models.generateContentStream(params);
stream.push({ type: "start", partial: output });
let currentBlock: TextContent | ThinkingContent | null = null;
const blocks = output.content;
const blockIndex = () => blocks.length - 1;
for await (const chunk of googleStream) {
// Vertex uses the same @google/genai GenerateContentResponse type as Gemini.
// responseId is documented there as an output-only identifier for each response.
output.responseId ||= chunk.responseId;
const candidate = chunk.candidates?.[0];
if (candidate?.content?.parts) {
for (const part of candidate.content.parts) {
if (part.text !== undefined) {
const isThinking = isThinkingPart(part);
if (
!currentBlock ||
(isThinking && currentBlock.type !== "thinking") ||
(!isThinking && currentBlock.type !== "text")
) {
if (currentBlock) {
if (currentBlock.type === "text") {
stream.push({
type: "text_end",
contentIndex: blocks.length - 1,
content: currentBlock.text,
partial: output,
});
} else {
stream.push({
type: "thinking_end",
contentIndex: blockIndex(),
content: currentBlock.thinking,
partial: output,
});
}
}
if (isThinking) {
currentBlock = { type: "thinking", thinking: "", thinkingSignature: undefined };
output.content.push(currentBlock);
stream.push({ type: "thinking_start", contentIndex: blockIndex(), partial: output });
} else {
currentBlock = { type: "text", text: "" };
output.content.push(currentBlock);
stream.push({ type: "text_start", contentIndex: blockIndex(), partial: output });
}
}
if (currentBlock.type === "thinking") {
currentBlock.thinking += part.text;
currentBlock.thinkingSignature = retainThoughtSignature(
currentBlock.thinkingSignature,
part.thoughtSignature,
);
stream.push({
type: "thinking_delta",
contentIndex: blockIndex(),
delta: part.text,
partial: output,
});
} else {
currentBlock.text += part.text;
currentBlock.textSignature = retainThoughtSignature(
currentBlock.textSignature,
part.thoughtSignature,
);
stream.push({
type: "text_delta",
contentIndex: blockIndex(),
delta: part.text,
partial: output,
});
}
}
if (part.functionCall) {
if (currentBlock) {
if (currentBlock.type === "text") {
stream.push({
type: "text_end",
contentIndex: blockIndex(),
content: currentBlock.text,
partial: output,
});
} else {
stream.push({
type: "thinking_end",
contentIndex: blockIndex(),
content: currentBlock.thinking,
partial: output,
});
}
currentBlock = null;
}
const providedId = part.functionCall.id;
const needsNewId =
!providedId || output.content.some((b) => b.type === "toolCall" && b.id === providedId);
const toolCallId = needsNewId
? `${part.functionCall.name}_${Date.now()}_${++toolCallCounter}`
: providedId;
const toolCall: ToolCall = {
type: "toolCall",
id: toolCallId,
name: part.functionCall.name || "",
arguments: (part.functionCall.args as Record<string, any>) ?? {},
...(part.thoughtSignature && { thoughtSignature: part.thoughtSignature }),
};
output.content.push(toolCall);
stream.push({ type: "toolcall_start", contentIndex: blockIndex(), partial: output });
stream.push({
type: "toolcall_delta",
contentIndex: blockIndex(),
delta: JSON.stringify(toolCall.arguments),
partial: output,
});
stream.push({ type: "toolcall_end", contentIndex: blockIndex(), toolCall, partial: output });
}
}
}
if (candidate?.finishReason) {
output.stopReason = mapStopReason(candidate.finishReason);
if (output.content.some((b) => b.type === "toolCall")) {
output.stopReason = "toolUse";
}
}
if (chunk.usageMetadata) {
output.usage = {
input:
(chunk.usageMetadata.promptTokenCount || 0) - (chunk.usageMetadata.cachedContentTokenCount || 0),
output:
(chunk.usageMetadata.candidatesTokenCount || 0) + (chunk.usageMetadata.thoughtsTokenCount || 0),
cacheRead: chunk.usageMetadata.cachedContentTokenCount || 0,
cacheWrite: 0,
totalTokens: chunk.usageMetadata.totalTokenCount || 0,
cost: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
total: 0,
},
};
calculateCost(model, output.usage);
}
}
if (currentBlock) {
if (currentBlock.type === "text") {
stream.push({
type: "text_end",
contentIndex: blockIndex(),
content: currentBlock.text,
partial: output,
});
} else {
stream.push({
type: "thinking_end",
contentIndex: blockIndex(),
content: currentBlock.thinking,
partial: output,
});
}
}
if (options?.signal?.aborted) {
throw new Error("Request was aborted");
}
if (output.stopReason === "aborted" || output.stopReason === "error") {
throw new Error("An unknown error occurred");
}
stream.push({ type: "done", reason: output.stopReason, message: output });
stream.end();
} catch (error) {
// Remove internal index property used during streaming
for (const block of output.content) {
if ("index" in block) {
delete (block as { index?: number }).index;
}
}
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
output.errorMessage = error instanceof Error ? error.message : JSON.stringify(error);
stream.push({ type: "error", reason: output.stopReason, error: output });
stream.end();
}
})();
return stream;
};
export const streamSimple: StreamFunction<"google-vertex", SimpleStreamOptions> = (
model: Model<"google-vertex">,
context: Context,
options?: SimpleStreamOptions,
): AssistantMessageEventStream => {
const base = buildBaseOptions(model, options, undefined);
if (!options?.reasoning) {
return stream(model, context, {
...base,
thinking: { enabled: false },
} satisfies GoogleVertexOptions);
}
const clampedReasoning = clampThinkingLevel(model, options.reasoning);
const effort = (clampedReasoning === "off" ? "high" : clampedReasoning) as ClampedThinkingLevel;
const geminiModel = model as unknown as Model<"google-generative-ai">;
if (isGemini3ProModel(geminiModel) || isGemini3FlashModel(geminiModel)) {
return stream(model, context, {
...base,
thinking: {
enabled: true,
level: getGemini3ThinkingLevel(effort, geminiModel),
},
} satisfies GoogleVertexOptions);
}
return stream(model, context, {
...base,
thinking: {
enabled: true,
budgetTokens: getGoogleBudget(geminiModel, effort, options.thinkingBudgets),
},
} satisfies GoogleVertexOptions);
};
function createClient(
model: Model<"google-vertex">,
project: string,
location: string,
optionsHeaders?: ProviderHeaders,
env?: ProviderEnv,
): GoogleGenAI {
const googleAuthOptions = buildGoogleAuthOptions(env);
return new GoogleGenAI({
vertexai: true,
project,
location,
apiVersion: API_VERSION,
...(googleAuthOptions ? { googleAuthOptions } : {}),
httpOptions: buildHttpOptions(model, optionsHeaders),
});
}
function createClientWithApiKey(
model: Model<"google-vertex">,
apiKey: string,
optionsHeaders?: ProviderHeaders,
): GoogleGenAI {
return new GoogleGenAI({
vertexai: true,
apiKey,
apiVersion: API_VERSION,
httpOptions: buildHttpOptions(model, optionsHeaders),
});
}
function buildHttpOptions(model: Model<"google-vertex">, optionsHeaders?: ProviderHeaders): HttpOptions | undefined {
const httpOptions: HttpOptions = {};
const baseUrl = resolveCustomBaseUrl(model.baseUrl);
if (baseUrl) {
httpOptions.baseUrl = baseUrl;
httpOptions.baseUrlResourceScope = ResourceScope.COLLECTION;
if (baseUrlIncludesApiVersion(baseUrl)) {
httpOptions.apiVersion = "";
}
}
const headers = providerHeadersToRecord({ ...model.headers, ...optionsHeaders });
if (headers) {
httpOptions.headers = headers;
}
return Object.keys(httpOptions).length > 0 ? httpOptions : undefined;
}
function resolveCustomBaseUrl(baseUrl: string): string | undefined {
const trimmed = baseUrl.trim();
if (!trimmed || trimmed.includes("{location}")) {
return undefined;
}
return trimmed;
}
function baseUrlIncludesApiVersion(baseUrl: string): boolean {
try {
const url = new URL(baseUrl);
return url.pathname.split("/").some((part) => /^v\d+(?:beta\d*)?$/.test(part));
} catch {
return /(?:^|\/)v\d+(?:beta\d*)?(?:\/|$)/.test(baseUrl);
}
}
function buildGoogleAuthOptions(env?: ProviderEnv): { keyFilename: string } | undefined {
const keyFilename = getProviderEnvValue("GOOGLE_APPLICATION_CREDENTIALS", env);
return keyFilename ? { keyFilename } : undefined;
}
function resolveApiKey(options?: GoogleVertexOptions): string | undefined {
const apiKey = options?.apiKey?.trim();
if (!apiKey || apiKey === GCP_VERTEX_CREDENTIALS_MARKER || isPlaceholderApiKey(apiKey)) {
return undefined;
}
return apiKey;
}
function isPlaceholderApiKey(apiKey: string): boolean {
return /^<[^>]+>$/.test(apiKey);
}
function resolveProject(options?: GoogleVertexOptions): string {
const project =
options?.project ||
getProviderEnvValue("GOOGLE_CLOUD_PROJECT", options?.env) ||
getProviderEnvValue("GCLOUD_PROJECT", options?.env);
if (!project) {
throw new Error(
"Vertex AI requires a project ID. Set GOOGLE_CLOUD_PROJECT/GCLOUD_PROJECT or pass project in options.",
);
}
return project;
}
function resolveLocation(options?: GoogleVertexOptions): string {
const location = options?.location || getProviderEnvValue("GOOGLE_CLOUD_LOCATION", options?.env);
if (!location) {
throw new Error("Vertex AI requires a location. Set GOOGLE_CLOUD_LOCATION or pass location in options.");
}
return location;
}
function buildParams(
model: Model<"google-vertex">,
context: Context,
options: GoogleVertexOptions = {},
): GenerateContentParameters {
const contents = convertMessages(model, context);
const generationConfig: GenerateContentConfig = {};
if (options.temperature !== undefined) {
generationConfig.temperature = options.temperature;
}
if (options.maxTokens !== undefined) {
generationConfig.maxOutputTokens = options.maxTokens;
}
const config: GenerateContentConfig = {
...(Object.keys(generationConfig).length > 0 && generationConfig),
...(context.systemPrompt && { systemInstruction: sanitizeSurrogates(context.systemPrompt) }),
...(context.tools && context.tools.length > 0 && { tools: convertTools(context.tools) }),
};
if (context.tools && context.tools.length > 0 && options.toolChoice) {
config.toolConfig = {
functionCallingConfig: {
mode: mapToolChoice(options.toolChoice),
},
};
} else {
config.toolConfig = undefined;
}
if (options.thinking?.enabled && model.reasoning) {
const thinkingConfig: ThinkingConfig = { includeThoughts: true };
if (options.thinking.level !== undefined) {
thinkingConfig.thinkingLevel = THINKING_LEVEL_MAP[options.thinking.level];
} else if (options.thinking.budgetTokens !== undefined) {
thinkingConfig.thinkingBudget = options.thinking.budgetTokens;
}
config.thinkingConfig = thinkingConfig;
} else if (model.reasoning && options.thinking && !options.thinking.enabled) {
config.thinkingConfig = getDisabledThinkingConfig(model);
}
if (options.signal) {
if (options.signal.aborted) {
throw new Error("Request aborted");
}
config.abortSignal = options.signal;
}
const params: GenerateContentParameters = {
model: model.id,
contents,
config,
};
return params;
}
type ClampedThinkingLevel = Exclude<PiThinkingLevel, "xhigh">;
function isGemini3ProModel(model: Model<"google-generative-ai">): boolean {
return /gemini-3(?:\.\d+)?-pro/.test(model.id.toLowerCase());
}
function isGemini3FlashModel(model: Model<"google-generative-ai">): boolean {
const id = model.id.toLowerCase();
return /gemini-3(?:\.\d+)?-flash/.test(id) || id === "gemini-flash-latest" || id === "gemini-flash-lite-latest";
}
function getDisabledThinkingConfig(model: Model<"google-vertex">): ThinkingConfig {
// Google docs: Gemini 3.1 Pro cannot disable thinking, and Gemini 3 Flash / Flash-Lite
// do not support full thinking-off either. For Gemini 3 models, use the lowest supported
// thinkingLevel without includeThoughts so hidden thinking remains invisible to pi.
const geminiModel = model as unknown as Model<"google-generative-ai">;
if (isGemini3ProModel(geminiModel)) {
return { thinkingLevel: ThinkingLevel.LOW };
}
if (isGemini3FlashModel(geminiModel)) {
return { thinkingLevel: ThinkingLevel.MINIMAL };
}
// Gemini 2.x supports disabling via thinkingBudget = 0.
return { thinkingBudget: 0 };
}
function getGemini3ThinkingLevel(
effort: ClampedThinkingLevel,
model: Model<"google-generative-ai">,
): GoogleThinkingLevel {
if (isGemini3ProModel(model)) {
switch (effort) {
case "minimal":
case "low":
return "LOW";
case "medium":
case "high":
return "HIGH";
}
}
switch (effort) {
case "minimal":
return "MINIMAL";
case "low":
return "LOW";
case "medium":
return "MEDIUM";
case "high":
return "HIGH";
}
}
function getGoogleBudget(
model: Model<"google-generative-ai">,
effort: ClampedThinkingLevel,
customBudgets?: ThinkingBudgets,
): number {
if (customBudgets?.[effort] !== undefined) {
return customBudgets[effort]!;
}
if (model.id.includes("2.5-pro")) {
const budgets: Record<ClampedThinkingLevel, number> = {
minimal: 128,
low: 2048,
medium: 8192,
high: 32768,
};
return budgets[effort];
}
if (model.id.includes("2.5-flash")) {
const budgets: Record<ClampedThinkingLevel, number> = {
minimal: 128,
low: 2048,
medium: 8192,
high: 24576,
};
return budgets[effort];
}
return -1;
}
packages/ai/src/api/lazy.ts
+70
View File
@@ -0,0 +1,70 @@
import type { Api, AssistantMessage, AssistantMessageEvent, Model, ProviderStreams } from "../types.ts";
import { AssistantMessageEventStream } from "../utils/event-stream.ts";
function createSetupErrorMessage(model: Model<Api>, error: unknown): AssistantMessage {
return {
role: "assistant",
content: [],
api: model.api,
provider: model.provider,
model: model.id,
usage: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "error",
errorMessage: error instanceof Error ? error.message : String(error),
timestamp: Date.now(),
};
}
function forwardStream(target: AssistantMessageEventStream, source: AsyncIterable<AssistantMessageEvent>): void {
(async () => {
for await (const event of source) {
target.push(event);
}
target.end();
})();
}
/**
* Returns a stream synchronously while running async setup (auth resolution,
* lazy module loading) behind it. Setup failures terminate the stream with an
* error event.
*/
export function lazyStream(
model: Model<Api>,
setup: () => Promise<AsyncIterable<AssistantMessageEvent>>,
): AssistantMessageEventStream {
const outer = new AssistantMessageEventStream();
setup()
.then((inner) => {
forwardStream(outer, inner);
})
.catch((error) => {
const message = createSetupErrorMessage(model, error);
outer.push({ type: "error", reason: "error", error: message });
outer.end(message);
});
return outer;
}
/**
* Wraps a dynamically imported API implementation module as `ProviderStreams`.
* The module loads on first stream call; the host's import cache deduplicates
* loads. Load failures terminate the returned stream with an error event.
*/
export function lazyApi(load: () => Promise<ProviderStreams>): ProviderStreams {
return {
stream: (model, context, options) =>
lazyStream(model, async () => (await load()).stream(model, context, options)),
streamSimple: (model, context, options) =>
lazyStream(model, async () => (await load()).streamSimple(model, context, options)),
};
}
packages/ai/src/api/mistral-conversations.lazy.ts
+4
View File
@@ -0,0 +1,4 @@
import type { ProviderStreams } from "../types.ts";
import { lazyApi } from "./lazy.ts";
export const mistralConversationsApi = (): ProviderStreams => lazyApi(() => import("./mistral-conversations.ts"));
packages/ai/src/api/mistral-conversations.ts
+664
View File
@@ -0,0 +1,664 @@
import { Mistral } from "@mistralai/mistralai";
import type {
ChatCompletionStreamRequest,
ChatCompletionStreamRequestMessage,
CompletionEvent,
ContentChunk,
FunctionTool,
} from "@mistralai/mistralai/models/components";
import { calculateCost, clampThinkingLevel } from "../models.ts";
import type {
AssistantMessage,
Context,
Message,
Model,
SimpleStreamOptions,
StopReason,
StreamFunction,
StreamOptions,
TextContent,
ThinkingContent,
Tool,
ToolCall,
} from "../types.ts";
import { AssistantMessageEventStream } from "../utils/event-stream.ts";
import { shortHash } from "../utils/hash.ts";
import { parseStreamingJson } from "../utils/json-parse.ts";
import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts";
import { buildBaseOptions } from "./simple-options.ts";
import { transformMessages } from "./transform-messages.ts";
const MISTRAL_TOOL_CALL_ID_LENGTH = 9;
const MAX_MISTRAL_ERROR_BODY_CHARS = 4000;
/**
* Provider-specific options for the Mistral API.
*/
type MistralReasoningEffort = "none" | "high";
export interface MistralOptions extends StreamOptions {
toolChoice?: "auto" | "none" | "any" | "required" | { type: "function"; function: { name: string } };
promptMode?: "reasoning";
reasoningEffort?: MistralReasoningEffort;
}
/**
* Stream responses from Mistral using `chat.stream`.
*/
export const stream: StreamFunction<"mistral-conversations", MistralOptions> = (
model: Model<"mistral-conversations">,
context: Context,
options?: MistralOptions,
): AssistantMessageEventStream => {
const stream = new AssistantMessageEventStream();
(async () => {
const output = createOutput(model);
try {
const apiKey = options?.apiKey;
if (!apiKey) {
throw new Error(`No API key for provider: ${model.provider}`);
}
// Intentionally per-request: avoids shared SDK mutable state across concurrent consumers.
const mistral = new Mistral({
apiKey,
serverURL: model.baseUrl,
});
const normalizeMistralToolCallId = createMistralToolCallIdNormalizer();
const transformedMessages = transformMessages(context.messages, model, (id) => normalizeMistralToolCallId(id));
let payload = buildChatPayload(model, context, transformedMessages, options);
const nextPayload = await options?.onPayload?.(payload, model);
if (nextPayload !== undefined) {
payload = nextPayload as ChatCompletionStreamRequest;
}
const mistralStream = await mistral.chat.stream(payload, buildRequestOptions(model, options));
stream.push({ type: "start", partial: output });
await consumeChatStream(model, output, stream, mistralStream);
if (options?.signal?.aborted) {
throw new Error("Request was aborted");
}
if (output.stopReason === "aborted" || output.stopReason === "error") {
throw new Error("An unknown error occurred");
}
stream.push({ type: "done", reason: output.stopReason, message: output });
stream.end();
} catch (error) {
for (const block of output.content) {
// partialArgs is only a streaming scratch buffer; never persist it.
delete (block as { partialArgs?: string }).partialArgs;
}
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
output.errorMessage = formatMistralError(error);
stream.push({ type: "error", reason: output.stopReason, error: output });
stream.end();
}
})();
return stream;
};
/**
* Maps provider-agnostic `SimpleStreamOptions` to Mistral options.
*/
export const streamSimple: StreamFunction<"mistral-conversations", SimpleStreamOptions> = (
model: Model<"mistral-conversations">,
context: Context,
options?: SimpleStreamOptions,
): AssistantMessageEventStream => {
const apiKey = options?.apiKey;
if (!apiKey) {
throw new Error(`No API key for provider: ${model.provider}`);
}
const base = buildBaseOptions(model, options, apiKey);
const clampedReasoning = options?.reasoning ? clampThinkingLevel(model, options.reasoning) : undefined;
const reasoning = clampedReasoning === "off" ? undefined : clampedReasoning;
const shouldUseReasoning = model.reasoning && reasoning !== undefined;
return stream(model, context, {
...base,
promptMode: shouldUseReasoning && usesPromptModeReasoning(model) ? "reasoning" : undefined,
reasoningEffort:
shouldUseReasoning && usesReasoningEffort(model) ? mapReasoningEffort(model, reasoning) : undefined,
} satisfies MistralOptions);
};
function createOutput(model: Model<"mistral-conversations">): AssistantMessage {
return {
role: "assistant",
content: [],
api: model.api,
provider: model.provider,
model: model.id,
usage: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "stop",
timestamp: Date.now(),
};
}
function createMistralToolCallIdNormalizer(): (id: string) => string {
const idMap = new Map<string, string>();
const reverseMap = new Map<string, string>();
return (id: string): string => {
const existing = idMap.get(id);
if (existing) return existing;
let attempt = 0;
while (true) {
const candidate = deriveMistralToolCallId(id, attempt);
const owner = reverseMap.get(candidate);
if (!owner || owner === id) {
idMap.set(id, candidate);
reverseMap.set(candidate, id);
return candidate;
}
attempt++;
}
};
}
function deriveMistralToolCallId(id: string, attempt: number): string {
const normalized = id.replace(/[^a-zA-Z0-9]/g, "");
if (attempt === 0 && normalized.length === MISTRAL_TOOL_CALL_ID_LENGTH) return normalized;
const seedBase = normalized || id;
const seed = attempt === 0 ? seedBase : `${seedBase}:${attempt}`;
return shortHash(seed)
.replace(/[^a-zA-Z0-9]/g, "")
.slice(0, MISTRAL_TOOL_CALL_ID_LENGTH);
}
function formatMistralError(error: unknown): string {
if (error instanceof Error) {
const sdkError = error as Error & { statusCode?: unknown; body?: unknown };
const statusCode = typeof sdkError.statusCode === "number" ? sdkError.statusCode : undefined;
const bodyText = typeof sdkError.body === "string" ? sdkError.body.trim() : undefined;
if (statusCode !== undefined && bodyText) {
return `Mistral API error (${statusCode}): ${truncateErrorText(bodyText, MAX_MISTRAL_ERROR_BODY_CHARS)}`;
}
if (statusCode !== undefined) return `Mistral API error (${statusCode}): ${error.message}`;
return error.message;
}
return safeJsonStringify(error);
}
function truncateErrorText(text: string, maxChars: number): string {
if (text.length <= maxChars) return text;
return `${text.slice(0, maxChars)}... [truncated ${text.length - maxChars} chars]`;
}
function safeJsonStringify(value: unknown): string {
try {
const serialized = JSON.stringify(value);
return serialized === undefined ? String(value) : serialized;
} catch {
return String(value);
}
}
function buildRequestOptions(model: Model<"mistral-conversations">, options?: MistralOptions) {
const requestOptions: {
signal?: AbortSignal;
retries: { strategy: "none" };
headers?: Record<string, string>;
} = {
retries: { strategy: "none" },
};
if (options?.signal) requestOptions.signal = options.signal;
const headers: Record<string, string> = {};
if (model.headers) Object.assign(headers, model.headers);
if (options?.headers) Object.assign(headers, options.headers);
// Mistral infrastructure uses `x-affinity` for KV-cache reuse (prefix caching).
// Respect explicit caller-provided header values.
if (shouldUsePromptCaching(options) && !headers["x-affinity"]) {
headers["x-affinity"] = options.sessionId;
}
if (Object.keys(headers).length > 0) {
requestOptions.headers = headers;
}
return requestOptions;
}
function buildChatPayload(
model: Model<"mistral-conversations">,
context: Context,
messages: Message[],
options?: MistralOptions,
): ChatCompletionStreamRequest {
const payload: ChatCompletionStreamRequest = {
model: model.id,
stream: true,
messages: toChatMessages(messages, model.input.includes("image")),
};
if (context.tools?.length) payload.tools = toFunctionTools(context.tools);
if (options?.temperature !== undefined) payload.temperature = options.temperature;
if (options?.maxTokens !== undefined) payload.maxTokens = options.maxTokens;
if (options?.toolChoice) payload.toolChoice = mapToolChoice(options.toolChoice);
if (options?.promptMode) payload.promptMode = options.promptMode;
if (options?.reasoningEffort) payload.reasoningEffort = options.reasoningEffort;
if (shouldUsePromptCaching(options)) payload.promptCacheKey = options.sessionId;
if (context.systemPrompt) {
payload.messages.unshift({
role: "system",
content: sanitizeSurrogates(context.systemPrompt),
});
}
return payload;
}
function shouldUsePromptCaching(options?: MistralOptions): options is MistralOptions & { sessionId: string } {
return options?.cacheRetention !== "none" && !!options?.sessionId;
}
function getMistralCachedPromptTokens(usage: unknown, promptTokens: number): number {
const rawUsage = usage as {
promptTokensDetails?: { cachedTokens?: unknown } | null;
prompt_tokens_details?: { cached_tokens?: unknown } | null;
promptTokenDetails?: { cachedTokens?: unknown } | null;
prompt_token_details?: { cached_tokens?: unknown } | null;
numCachedTokens?: unknown;
num_cached_tokens?: unknown;
};
const rawCachedTokens =
rawUsage.promptTokensDetails?.cachedTokens ??
rawUsage.prompt_tokens_details?.cached_tokens ??
rawUsage.promptTokenDetails?.cachedTokens ??
rawUsage.prompt_token_details?.cached_tokens ??
rawUsage.numCachedTokens ??
rawUsage.num_cached_tokens ??
0;
const cachedTokens = typeof rawCachedTokens === "number" && Number.isFinite(rawCachedTokens) ? rawCachedTokens : 0;
return Math.min(promptTokens, Math.max(0, cachedTokens));
}
async function consumeChatStream(
model: Model<"mistral-conversations">,
output: AssistantMessage,
stream: AssistantMessageEventStream,
mistralStream: AsyncIterable<CompletionEvent>,
): Promise<void> {
let currentBlock: TextContent | ThinkingContent | null = null;
const blocks = output.content;
const blockIndex = () => blocks.length - 1;
const toolBlocksByKey = new Map<string, number>();
const finishCurrentBlock = (block?: typeof currentBlock) => {
if (!block) return;
if (block.type === "text") {
stream.push({
type: "text_end",
contentIndex: blockIndex(),
content: block.text,
partial: output,
});
return;
}
if (block.type === "thinking") {
stream.push({
type: "thinking_end",
contentIndex: blockIndex(),
content: block.thinking,
partial: output,
});
}
};
for await (const event of mistralStream) {
const chunk = event.data;
// Mistral's streamed CompletionChunk carries an id field. Keep the first non-empty one,
// mirroring how OpenAI-style streaming exposes a stable response identifier per stream.
output.responseId ||= chunk.id;
if (chunk.usage) {
const promptTokens = chunk.usage.promptTokens || 0;
const cachedPromptTokens = getMistralCachedPromptTokens(chunk.usage, promptTokens);
output.usage.input = Math.max(0, promptTokens - cachedPromptTokens);
output.usage.output = chunk.usage.completionTokens || 0;
output.usage.cacheRead = cachedPromptTokens;
output.usage.cacheWrite = 0;
output.usage.totalTokens =
chunk.usage.totalTokens ||
output.usage.input + output.usage.output + output.usage.cacheRead + output.usage.cacheWrite;
calculateCost(model, output.usage);
}
const choice = chunk.choices[0];
if (!choice) continue;
if (choice.finishReason) {
output.stopReason = mapChatStopReason(choice.finishReason);
}
const delta = choice.delta;
if (delta.content !== null && delta.content !== undefined) {
const contentItems = typeof delta.content === "string" ? [delta.content] : delta.content;
for (const item of contentItems) {
if (typeof item === "string") {
const textDelta = sanitizeSurrogates(item);
if (!currentBlock || currentBlock.type !== "text") {
finishCurrentBlock(currentBlock);
currentBlock = { type: "text", text: "" };
output.content.push(currentBlock);
stream.push({ type: "text_start", contentIndex: blockIndex(), partial: output });
}
currentBlock.text += textDelta;
stream.push({
type: "text_delta",
contentIndex: blockIndex(),
delta: textDelta,
partial: output,
});
continue;
}
if (item.type === "thinking") {
const deltaText = item.thinking
.map((part) => ("text" in part ? part.text : ""))
.filter((text) => text.length > 0)
.join("");
const thinkingDelta = sanitizeSurrogates(deltaText);
if (!thinkingDelta) continue;
if (!currentBlock || currentBlock.type !== "thinking") {
finishCurrentBlock(currentBlock);
currentBlock = { type: "thinking", thinking: "" };
output.content.push(currentBlock);
stream.push({ type: "thinking_start", contentIndex: blockIndex(), partial: output });
}
currentBlock.thinking += thinkingDelta;
stream.push({
type: "thinking_delta",
contentIndex: blockIndex(),
delta: thinkingDelta,
partial: output,
});
continue;
}
if (item.type === "text") {
const textDelta = sanitizeSurrogates(item.text);
if (!currentBlock || currentBlock.type !== "text") {
finishCurrentBlock(currentBlock);
currentBlock = { type: "text", text: "" };
output.content.push(currentBlock);
stream.push({ type: "text_start", contentIndex: blockIndex(), partial: output });
}
currentBlock.text += textDelta;
stream.push({
type: "text_delta",
contentIndex: blockIndex(),
delta: textDelta,
partial: output,
});
}
}
}
const toolCalls = delta.toolCalls || [];
for (const toolCall of toolCalls) {
if (currentBlock) {
finishCurrentBlock(currentBlock);
currentBlock = null;
}
const callId =
toolCall.id && toolCall.id !== "null"
? toolCall.id
: deriveMistralToolCallId(`toolcall:${toolCall.index ?? 0}`, 0);
const key = `${callId}:${toolCall.index || 0}`;
const existingIndex = toolBlocksByKey.get(key);
let block: (ToolCall & { partialArgs?: string }) | undefined;
if (existingIndex !== undefined) {
const existing = output.content[existingIndex];
if (existing?.type === "toolCall") {
block = existing as ToolCall & { partialArgs?: string };
}
}
if (!block) {
block = {
type: "toolCall",
id: callId,
name: toolCall.function.name,
arguments: {},
partialArgs: "",
};
output.content.push(block);
toolBlocksByKey.set(key, output.content.length - 1);
stream.push({ type: "toolcall_start", contentIndex: output.content.length - 1, partial: output });
}
const argsDelta =
typeof toolCall.function.arguments === "string"
? toolCall.function.arguments
: JSON.stringify(toolCall.function.arguments || {});
block.partialArgs = (block.partialArgs || "") + argsDelta;
block.arguments = parseStreamingJson<Record<string, unknown>>(block.partialArgs);
stream.push({
type: "toolcall_delta",
contentIndex: toolBlocksByKey.get(key)!,
delta: argsDelta,
partial: output,
});
}
}
finishCurrentBlock(currentBlock);
for (const index of toolBlocksByKey.values()) {
const block = output.content[index];
if (block.type !== "toolCall") continue;
const toolBlock = block as ToolCall & { partialArgs?: string };
toolBlock.arguments = parseStreamingJson<Record<string, unknown>>(toolBlock.partialArgs);
// Finalize in-place and strip the scratch buffer so replay only
// carries parsed arguments.
delete toolBlock.partialArgs;
stream.push({
type: "toolcall_end",
contentIndex: index,
toolCall: toolBlock,
partial: output,
});
}
}
function toFunctionTools(tools: Tool[]): Array<FunctionTool & { type: "function" }> {
return tools.map((tool) => ({
type: "function",
function: {
name: tool.name,
description: tool.description,
parameters: stripSymbolKeys(tool.parameters) as Record<string, unknown>,
strict: false,
},
}));
}
function stripSymbolKeys(value: unknown): unknown {
if (Array.isArray(value)) {
return value.map((item) => stripSymbolKeys(item));
}
if (value && typeof value === "object") {
const result: Record<string, unknown> = {};
for (const [key, entry] of Object.entries(value)) {
result[key] = stripSymbolKeys(entry);
}
return result;
}
return value;
}
function toChatMessages(messages: Message[], supportsImages: boolean): ChatCompletionStreamRequestMessage[] {
const result: ChatCompletionStreamRequestMessage[] = [];
for (const msg of messages) {
if (msg.role === "user") {
if (typeof msg.content === "string") {
result.push({ role: "user", content: sanitizeSurrogates(msg.content) });
continue;
}
const hadImages = msg.content.some((item) => item.type === "image");
const content: ContentChunk[] = msg.content
.filter((item) => item.type === "text" || supportsImages)
.map((item) => {
if (item.type === "text") return { type: "text", text: sanitizeSurrogates(item.text) };
return { type: "image_url", imageUrl: `data:${item.mimeType};base64,${item.data}` };
});
if (content.length > 0) {
result.push({ role: "user", content });
continue;
}
if (hadImages && !supportsImages) {
result.push({ role: "user", content: "(image omitted: model does not support images)" });
}
continue;
}
if (msg.role === "assistant") {
const contentParts: ContentChunk[] = [];
const toolCalls: Array<{ id: string; type: "function"; function: { name: string; arguments: string } }> = [];
for (const block of msg.content) {
if (block.type === "text") {
if (block.text.trim().length > 0) {
contentParts.push({ type: "text", text: sanitizeSurrogates(block.text) });
}
continue;
}
if (block.type === "thinking") {
if (block.thinking.trim().length > 0) {
contentParts.push({
type: "thinking",
thinking: [{ type: "text", text: sanitizeSurrogates(block.thinking) }],
});
}
continue;
}
toolCalls.push({
id: block.id,
type: "function",
function: { name: block.name, arguments: JSON.stringify(block.arguments || {}) },
});
}
const assistantMessage: ChatCompletionStreamRequestMessage = { role: "assistant" };
if (contentParts.length > 0) assistantMessage.content = contentParts;
if (toolCalls.length > 0) assistantMessage.toolCalls = toolCalls;
if (contentParts.length > 0 || toolCalls.length > 0) result.push(assistantMessage);
continue;
}
const toolContent: ContentChunk[] = [];
const textResult = msg.content
.filter((part) => part.type === "text")
.map((part) => (part.type === "text" ? sanitizeSurrogates(part.text) : ""))
.join("\n");
const hasImages = msg.content.some((part) => part.type === "image");
const toolText = buildToolResultText(textResult, hasImages, supportsImages, msg.isError);
toolContent.push({ type: "text", text: toolText });
for (const part of msg.content) {
if (!supportsImages) continue;
if (part.type !== "image") continue;
toolContent.push({
type: "image_url",
imageUrl: `data:${part.mimeType};base64,${part.data}`,
});
}
result.push({
role: "tool",
toolCallId: msg.toolCallId,
name: msg.toolName,
content: toolContent,
});
}
return result;
}
function buildToolResultText(text: string, hasImages: boolean, supportsImages: boolean, isError: boolean): string {
const trimmed = text.trim();
const errorPrefix = isError ? "[tool error] " : "";
if (trimmed.length > 0) {
const imageSuffix = hasImages && !supportsImages ? "\n[tool image omitted: model does not support images]" : "";
return `${errorPrefix}${trimmed}${imageSuffix}`;
}
if (hasImages) {
if (supportsImages) {
return isError ? "[tool error] (see attached image)" : "(see attached image)";
}
return isError
? "[tool error] (image omitted: model does not support images)"
: "(image omitted: model does not support images)";
}
return isError ? "[tool error] (no tool output)" : "(no tool output)";
}
function usesReasoningEffort(model: Model<"mistral-conversations">): boolean {
return model.id === "mistral-small-2603" || model.id === "mistral-small-latest" || model.id === "mistral-medium-3.5";
}
function usesPromptModeReasoning(model: Model<"mistral-conversations">): boolean {
return model.reasoning && !usesReasoningEffort(model);
}
function mapReasoningEffort(
model: Model<"mistral-conversations">,
level: Exclude<SimpleStreamOptions["reasoning"], undefined>,
): MistralReasoningEffort {
return (model.thinkingLevelMap?.[level] ?? "high") as MistralReasoningEffort;
}
function mapToolChoice(
choice: MistralOptions["toolChoice"],
): "auto" | "none" | "any" | "required" | { type: "function"; function: { name: string } } | undefined {
if (!choice) return undefined;
if (choice === "auto" || choice === "none" || choice === "any" || choice === "required") {
return choice as any;
}
return {
type: "function",
function: { name: choice.function.name },
};
}
function mapChatStopReason(reason: string | null): StopReason {
if (reason === null) return "stop";
switch (reason) {
case "stop":
return "stop";
case "length":
case "model_length":
return "length";
case "tool_calls":
return "toolUse";
case "error":
return "error";
default:
return "stop";
}
}
packages/ai/src/api/openai-codex-responses.lazy.ts
+4
View File
@@ -0,0 +1,4 @@
import type { ProviderStreams } from "../types.ts";
import { lazyApi } from "./lazy.ts";
export const openAICodexResponsesApi = (): ProviderStreams => lazyApi(() => import("./openai-codex-responses.ts"));
packages/ai/src/providers/openai-codex-responses.ts → packages/ai/src/api/openai-codex-responses.ts
+42 -10
View File
@@ -28,6 +28,7 @@ import type {
Context,
Model,
ProviderEnv,
ProviderHeaders,
SimpleStreamOptions,
StreamFunction,
StreamOptions,
@@ -61,6 +62,7 @@ const DEFAULT_SSE_HEADER_TIMEOUT_MS = 20_000;
const DEFAULT_WEBSOCKET_CONNECT_TIMEOUT_MS = 15_000;
const CODEX_TOOL_CALL_PROVIDERS = new Set(["openai", "openai-codex", "opencode"]);
const WEBSOCKET_MESSAGE_TOO_BIG_CLOSE_CODE = 1009;
const WEBSOCKET_CONNECTION_LIMIT_REACHED_CODE = "websocket_connection_limit_reached";
const CODEX_RESPONSE_STATUSES = new Set<CodexResponseStatus>([
"completed",
@@ -195,7 +197,7 @@ function createSSEHeaderTimeout(): { signal: AbortSignal; clear: () => void; err
// Main Stream Function
// ============================================================================
export const streamOpenAICodexResponses: StreamFunction<"openai-codex-responses", OpenAICodexResponsesOptions> = (
export const stream: StreamFunction<"openai-codex-responses", OpenAICodexResponsesOptions> = (
model: Model<"openai-codex-responses">,
context: Context,
options?: OpenAICodexResponsesOptions,
@@ -253,6 +255,9 @@ export const streamOpenAICodexResponses: StreamFunction<"openai-codex-responses"
if (transport !== "sse" && !websocketDisabledForSession) {
let websocketStarted = false;
let retriedWebSocketConnectionLimit = false;
while (true) {
websocketStarted = false;
try {
await processWebSocketStream(
resolveCodexWebSocketUrl(model.baseUrl),
@@ -281,7 +286,12 @@ export const streamOpenAICodexResponses: StreamFunction<"openai-codex-responses"
return;
} catch (error) {
const aborted = options?.signal?.aborted;
if (aborted || isCodexNonTransportError(error)) {
const connectionLimitBeforeStart = !websocketStarted && isWebSocketConnectionLimitReachedError(error);
if (!aborted && connectionLimitBeforeStart && !retriedWebSocketConnectionLimit) {
retriedWebSocketConnectionLimit = true;
continue;
}
if (aborted || (isCodexNonTransportError(error) && !connectionLimitBeforeStart)) {
throw error;
}
appendAssistantMessageDiagnostic(
@@ -299,6 +309,8 @@ export const streamOpenAICodexResponses: StreamFunction<"openai-codex-responses"
throw error;
}
recordWebSocketSseFallback(options?.sessionId);
break;
}
}
}
@@ -408,7 +420,7 @@ export const streamOpenAICodexResponses: StreamFunction<"openai-codex-responses"
return stream;
};
export const streamSimpleOpenAICodexResponses: StreamFunction<"openai-codex-responses", SimpleStreamOptions> = (
export const streamSimple: StreamFunction<"openai-codex-responses", SimpleStreamOptions> = (
model: Model<"openai-codex-responses">,
context: Context,
options?: SimpleStreamOptions,
@@ -422,7 +434,7 @@ export const streamSimpleOpenAICodexResponses: StreamFunction<"openai-codex-resp
const clampedReasoning = options?.reasoning ? clampThinkingLevel(model, options.reasoning) : undefined;
const reasoningEffort = clampedReasoning === "off" ? undefined : clampedReasoning;
return streamOpenAICodexResponses(model, context, {
return stream(model, context, {
...base,
reasoningEffort,
} satisfies OpenAICodexResponsesOptions);
@@ -582,16 +594,32 @@ function isCodexNonTransportError(error: unknown): boolean {
return error instanceof CodexApiError || error instanceof CodexProtocolError;
}
function isWebSocketConnectionLimitReachedError(error: unknown): boolean {
return error instanceof CodexApiError && error.code === WEBSOCKET_CONNECTION_LIMIT_REACHED_CODE;
}
function extractCodexEventError(event: Record<string, unknown>): { code?: string; message?: string } {
const nested = event.error && typeof event.error === "object" ? (event.error as Record<string, unknown>) : undefined;
return {
code: typeof event.code === "string" ? event.code : typeof nested?.code === "string" ? nested.code : undefined,
message:
typeof event.message === "string"
? event.message
: typeof nested?.message === "string"
? nested.message
: undefined,
};
}
async function* mapCodexEvents(events: AsyncIterable<Record<string, unknown>>): AsyncGenerator<ResponseStreamEvent> {
for await (const event of events) {
const type = typeof event.type === "string" ? event.type : undefined;
if (!type) continue;
if (type === "error") {
const code = (event as { code?: string }).code || "";
const message = (event as { message?: string }).message || "";
const { code, message } = extractCodexEventError(event);
throw new CodexApiError(`Codex error: ${message || code || JSON.stringify(event)}`, {
code: code || undefined,
code,
payload: event,
});
}
@@ -1440,14 +1468,18 @@ function createCodexRequestId(): string {
function buildBaseCodexHeaders(
initHeaders: Record<string, string> | undefined,
additionalHeaders: Record<string, string> | undefined,
additionalHeaders: ProviderHeaders | undefined,
accountId: string,
token: string,
): Headers {
const headers = new Headers(initHeaders);
for (const [key, value] of Object.entries(additionalHeaders || {})) {
if (value === null) {
headers.delete(key);
} else {
headers.set(key, value);
}
}
headers.set("Authorization", `Bearer ${token}`);
headers.set("chatgpt-account-id", accountId);
headers.set("originator", "pi");
@@ -1458,7 +1490,7 @@ function buildBaseCodexHeaders(
function buildSSEHeaders(
initHeaders: Record<string, string> | undefined,
additionalHeaders: Record<string, string> | undefined,
additionalHeaders: ProviderHeaders | undefined,
accountId: string,
token: string,
sessionId?: string,
@@ -1478,7 +1510,7 @@ function buildSSEHeaders(
function buildWebSocketHeaders(
initHeaders: Record<string, string> | undefined,
additionalHeaders: Record<string, string> | undefined,
additionalHeaders: ProviderHeaders | undefined,
accountId: string,
token: string,
requestId: string,
packages/ai/src/api/openai-completions.lazy.ts
+4
View File
@@ -0,0 +1,4 @@
import type { ProviderStreams } from "../types.ts";
import { lazyApi } from "./lazy.ts";
export const openAICompletionsApi = (): ProviderStreams => lazyApi(() => import("./openai-completions.ts"));
packages/ai/src/providers/openai-completions.ts → packages/ai/src/api/openai-completions.ts
+120 -40
View File
@@ -14,12 +14,14 @@ import { calculateCost, clampThinkingLevel } from "../models.ts";
import type {
AssistantMessage,
CacheRetention,
ChatTemplateKwargValue,
Context,
ImageContent,
Message,
Model,
OpenAICompletionsCompat,
ProviderEnv,
ProviderHeaders,
SimpleStreamOptions,
StopReason,
StreamFunction,
@@ -35,7 +37,6 @@ import { headersToRecord } from "../utils/headers.ts";
import { parseStreamingJson } from "../utils/json-parse.ts";
import { getProviderEnvValue } from "../utils/provider-env.ts";
import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts";
import { isCloudflareProvider, resolveCloudflareBaseUrl } from "./cloudflare.ts";
import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.ts";
import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts";
import { buildBaseOptions } from "./simple-options.ts";
@@ -46,6 +47,21 @@ import { transformMessages } from "./transform-messages.ts";
* This is needed because Anthropic (via proxy) requires the tools param
* to be present when messages include tool_calls or tool role messages.
*/
function hasHeader(headers: ProviderHeaders | undefined, name: string): boolean {
if (!headers) return false;
const expected = name.toLowerCase();
for (const [key, value] of Object.entries(headers)) {
if (key.toLowerCase() === expected && value !== null && value.trim().length > 0) return true;
}
return false;
}
function getClientApiKey(provider: string, apiKey: string | undefined, headers: ProviderHeaders | undefined): string {
if (apiKey) return apiKey;
if (hasHeader(headers, "authorization") || hasHeader(headers, "cf-aig-authorization")) return "unused";
throw new Error(`No API key for provider: ${provider}`);
}
function hasToolHistory(messages: Message[]): boolean {
for (const msg of messages) {
if (msg.role === "toolResult") {
@@ -76,6 +92,20 @@ function isImageContentBlock(block: { type: string }): block is ImageContent {
return block.type === "image";
}
function isEncryptedReasoningDetail(detail: unknown): detail is OpenAIEncryptedReasoningDetail {
if (typeof detail !== "object" || detail === null) {
return false;
}
const candidate = detail as Record<string, unknown>;
return (
candidate.type === "reasoning.encrypted" &&
typeof candidate.id === "string" &&
candidate.id.length > 0 &&
typeof candidate.data === "string" &&
candidate.data.length > 0
);
}
export interface OpenAICompletionsOptions extends StreamOptions {
toolChoice?: "auto" | "none" | "required" | { type: "function"; function: { name: string } };
reasoningEffort?: "minimal" | "low" | "medium" | "high" | "xhigh";
@@ -90,8 +120,16 @@ type ResolvedOpenAICompletionsCompat = Omit<Required<OpenAICompletionsCompat>, "
cacheControlFormat?: OpenAICompletionsCompat["cacheControlFormat"];
};
type ResolvedChatTemplateKwargValue = string | number | boolean | null;
type ChatCompletionInstructionMessageParam = ChatCompletionDeveloperMessageParam | ChatCompletionSystemMessageParam;
type OpenAIEncryptedReasoningDetail = {
type: "reasoning.encrypted";
id: string;
data: string;
};
type ChatCompletionTextPartWithCacheControl = ChatCompletionContentPartText & {
cache_control?: OpenAICompatCacheControl;
};
@@ -110,7 +148,7 @@ function resolveCacheRetention(cacheRetention?: CacheRetention, env?: ProviderEn
return "short";
}
export const streamOpenAICompletions: StreamFunction<"openai-completions", OpenAICompletionsOptions> = (
export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptions> = (
model: Model<"openai-completions">,
context: Context,
options?: OpenAICompletionsOptions,
@@ -137,14 +175,11 @@ export const streamOpenAICompletions: StreamFunction<"openai-completions", OpenA
};
try {
const apiKey = options?.apiKey;
if (!apiKey) {
throw new Error(`No API key for provider: ${model.provider}`);
}
const apiKey = getClientApiKey(model.provider, options?.apiKey, options?.headers);
const compat = getCompat(model);
const cacheRetention = resolveCacheRetention(options?.cacheRetention, options?.env);
const cacheSessionId = cacheRetention === "none" ? undefined : options?.sessionId;
const client = createClient(model, context, apiKey, options?.headers, cacheSessionId, compat, options?.env);
const client = createClient(model, context, apiKey, options?.headers, cacheSessionId, compat);
let params = buildParams(model, context, options, compat, cacheRetention);
const nextParams = await options?.onPayload?.(params, model);
if (nextParams !== undefined) {
@@ -173,6 +208,7 @@ export const streamOpenAICompletions: StreamFunction<"openai-completions", OpenA
let hasFinishReason = false;
const toolCallBlocksByIndex = new Map<number, StreamingToolCallBlock>();
const toolCallBlocksById = new Map<string, StreamingToolCallBlock>();
const pendingReasoningDetailsByToolCallId = new Map<string, string>();
const blocks = output.content as StreamingBlock[];
const getContentIndex = (block: StreamingBlock) => blocks.indexOf(block);
const finishBlock = (block: StreamingBlock) => {
@@ -228,6 +264,16 @@ export const streamOpenAICompletions: StreamFunction<"openai-completions", OpenA
}
return thinkingBlock;
};
const applyPendingReasoningDetail = (block: StreamingToolCallBlock) => {
if (!block.id) {
return;
}
const pendingReasoningDetail = pendingReasoningDetailsByToolCallId.get(block.id);
if (pendingReasoningDetail) {
block.thoughtSignature = pendingReasoningDetail;
pendingReasoningDetailsByToolCallId.delete(block.id);
}
};
const ensureToolCallBlock = (toolCall: StreamingToolCallDelta) => {
const streamIndex = typeof toolCall.index === "number" ? toolCall.index : undefined;
let block = streamIndex !== undefined ? toolCallBlocksByIndex.get(streamIndex) : undefined;
@@ -263,6 +309,7 @@ export const streamOpenAICompletions: StreamFunction<"openai-completions", OpenA
if (toolCall.id) {
toolCallBlocksById.set(toolCall.id, block);
}
applyPendingReasoningDetail(block);
return block;
};
@@ -372,15 +419,16 @@ export const streamOpenAICompletions: StreamFunction<"openai-completions", OpenA
}
}
const reasoningDetails = (choice.delta as any).reasoning_details;
if (reasoningDetails && Array.isArray(reasoningDetails)) {
const reasoningDetails = (choice.delta as { reasoning_details?: unknown }).reasoning_details;
if (Array.isArray(reasoningDetails)) {
for (const detail of reasoningDetails) {
if (detail.type === "reasoning.encrypted" && detail.id && detail.data) {
const matchingToolCall = output.content.find(
(b) => b.type === "toolCall" && b.id === detail.id,
) as ToolCall | undefined;
if (isEncryptedReasoningDetail(detail)) {
const serializedDetail = JSON.stringify(detail);
const matchingToolCall = toolCallBlocksById.get(detail.id);
if (matchingToolCall) {
matchingToolCall.thoughtSignature = JSON.stringify(detail);
matchingToolCall.thoughtSignature = serializedDetail;
} else {
pendingReasoningDetailsByToolCallId.set(detail.id, serializedDetail);
}
}
}
@@ -427,22 +475,19 @@ export const streamOpenAICompletions: StreamFunction<"openai-completions", OpenA
return stream;
};
export const streamSimpleOpenAICompletions: StreamFunction<"openai-completions", SimpleStreamOptions> = (
export const streamSimple: StreamFunction<"openai-completions", SimpleStreamOptions> = (
model: Model<"openai-completions">,
context: Context,
options?: SimpleStreamOptions,
): AssistantMessageEventStream => {
const apiKey = options?.apiKey;
if (!apiKey) {
throw new Error(`No API key for provider: ${model.provider}`);
}
getClientApiKey(model.provider, options?.apiKey, options?.headers);
const base = buildBaseOptions(model, options, apiKey);
const base = buildBaseOptions(model, options, options?.apiKey);
const clampedReasoning = options?.reasoning ? clampThinkingLevel(model, options.reasoning) : undefined;
const reasoningEffort = clampedReasoning === "off" ? undefined : clampedReasoning;
const toolChoice = (options as OpenAICompletionsOptions | undefined)?.toolChoice;
return streamOpenAICompletions(model, context, {
return stream(model, context, {
...base,
reasoningEffort,
toolChoice,
@@ -453,12 +498,11 @@ function createClient(
model: Model<"openai-completions">,
context: Context,
apiKey: string,
optionsHeaders?: Record<string, string>,
optionsHeaders?: ProviderHeaders,
sessionId?: string,
compat: ResolvedOpenAICompletionsCompat = getCompat(model),
env?: ProviderEnv,
) {
const headers = { ...model.headers };
const headers: ProviderHeaders = { ...model.headers };
if (model.provider === "github-copilot") {
const hasImages = hasCopilotVisionInput(context.messages);
const copilotHeaders = buildCopilotDynamicHeaders({
@@ -479,20 +523,11 @@ function createClient(
Object.assign(headers, optionsHeaders);
}
const defaultHeaders =
model.provider === "cloudflare-ai-gateway"
? {
...headers,
Authorization: headers.Authorization ?? null,
"cf-aig-authorization": `Bearer ${apiKey}`,
}
: headers;
return new OpenAI({
apiKey,
baseURL: isCloudflareProvider(model.provider) ? resolveCloudflareBaseUrl(model, env) : model.baseUrl,
baseURL: model.baseUrl,
dangerouslyAllowBrowser: true,
defaultHeaders,
defaultHeaders: headers,
});
}
@@ -576,6 +611,11 @@ function buildParams(
enable_thinking: !!options?.reasoningEffort,
preserve_thinking: true,
};
} else if (compat.thinkingFormat === "chat-template" && model.reasoning) {
const chatTemplateKwargs = buildChatTemplateKwargs(model, options, compat);
if (chatTemplateKwargs) {
(params as any).chat_template_kwargs = chatTemplateKwargs;
}
} else if (compat.thinkingFormat === "deepseek" && model.reasoning) {
if (options?.reasoningEffort) {
(params as any).thinking = { type: "enabled" };
@@ -633,7 +673,7 @@ function buildParams(
}
// Vercel AI Gateway provider routing preferences
if (model.baseUrl.includes("ai-gateway.vercel.sh") && model.compat?.vercelGatewayRouting) {
if (model.compat?.vercelGatewayRouting) {
const routing = model.compat.vercelGatewayRouting;
if (routing.only || routing.order) {
const gatewayOptions: Record<string, string[]> = {};
@@ -646,6 +686,44 @@ function buildParams(
return params;
}
function buildChatTemplateKwargs(
model: Model<"openai-completions">,
options: OpenAICompletionsOptions | undefined,
compat: ResolvedOpenAICompletionsCompat,
): Record<string, ResolvedChatTemplateKwargValue> | undefined {
const kwargs: Record<string, ResolvedChatTemplateKwargValue> = {};
for (const [key, value] of Object.entries(compat.chatTemplateKwargs)) {
const resolved = resolveChatTemplateKwargValue(model, options, value);
if (resolved !== undefined) {
kwargs[key] = resolved;
}
}
return Object.keys(kwargs).length > 0 ? kwargs : undefined;
}
function resolveChatTemplateKwargValue(
model: Model<"openai-completions">,
options: OpenAICompletionsOptions | undefined,
value: ChatTemplateKwargValue,
): ResolvedChatTemplateKwargValue | undefined {
if (typeof value !== "object" || value === null) {
return value;
}
const reasoningEffort = options?.reasoningEffort;
if (!reasoningEffort && value.omitWhenOff) {
return undefined;
}
if (value.$var === "thinking.enabled") {
return !!reasoningEffort;
}
const mappedValue = reasoningEffort ? model.thinkingLevelMap?.[reasoningEffort] : model.thinkingLevelMap?.off;
return mappedValue === undefined ? reasoningEffort : typeof mappedValue === "string" ? mappedValue : undefined;
}
function getCompatCacheControl(
compat: ResolvedOpenAICompletionsCompat,
cacheRetention: CacheRetention,
@@ -1086,9 +1164,9 @@ function mapStopReason(reason: ChatCompletionChunk.Choice["finish_reason"] | str
}
/**
* Detect compatibility settings from provider and baseUrl for known providers.
* Provider takes precedence over URL-based detection since it's explicitly configured.
* Returns a fully resolved OpenAICompletionsCompat object with all fields set.
* Auto-detect compatibility settings from provider name and baseUrl.
* Used as the base when model.compat is not set; explicit model.compat
* entries override these detected values.
*/
function detectCompat(model: Model<"openai-completions">): ResolvedOpenAICompletionsCompat {
const provider = model.provider;
@@ -1158,6 +1236,7 @@ function detectCompat(model: Model<"openai-completions">): ResolvedOpenAIComplet
: "openai",
openRouterRouting: {},
vercelGatewayRouting: {},
chatTemplateKwargs: {},
zaiToolStream: false,
supportsStrictMode: !isMoonshot && !isTogether && !isCloudflareAiGateway && !isNvidia,
cacheControlFormat,
@@ -1174,7 +1253,7 @@ function detectCompat(model: Model<"openai-completions">): ResolvedOpenAIComplet
/**
* Get resolved compatibility settings for a model.
* Uses explicit model.compat if provided, otherwise auto-detects from provider/URL.
* Auto-detects from provider/URL then overrides with explicit model.compat.
*/
function getCompat(model: Model<"openai-completions">): ResolvedOpenAICompletionsCompat {
const detected = detectCompat(model);
@@ -1196,6 +1275,7 @@ function getCompat(model: Model<"openai-completions">): ResolvedOpenAICompletion
thinkingFormat: model.compat.thinkingFormat ?? detected.thinkingFormat,
openRouterRouting: model.compat.openRouterRouting ?? {},
vercelGatewayRouting: model.compat.vercelGatewayRouting ?? detected.vercelGatewayRouting,
chatTemplateKwargs: model.compat.chatTemplateKwargs ?? detected.chatTemplateKwargs,
zaiToolStream: model.compat.zaiToolStream ?? detected.zaiToolStream,
supportsStrictMode: model.compat.supportsStrictMode ?? detected.supportsStrictMode,
cacheControlFormat: model.compat.cacheControlFormat ?? detected.cacheControlFormat,
packages/ai/src/providers/openai-prompt-cache.ts → packages/ai/src/api/openai-prompt-cache.ts
View File
packages/ai/src/providers/openai-responses-shared.ts → packages/ai/src/api/openai-responses-shared.ts
+39 -29
View File
@@ -294,8 +294,41 @@ export async function processResponsesStream<TApi extends Api>(
): Promise<void> {
let currentItem: ResponseReasoningItem | ResponseOutputMessage | ResponseFunctionToolCall | null = null;
let currentBlock: ThinkingContent | TextContent | (ToolCall & { partialJson: string }) | null = null;
let sawTerminalResponseEvent = false;
const blocks = output.content;
const blockIndex = () => blocks.length - 1;
const finalizeResponse = (
response: Extract<ResponseStreamEvent, { type: "response.completed" | "response.incomplete" }>["response"],
): void => {
sawTerminalResponseEvent = true;
if (response?.id) {
output.responseId = response.id;
}
if (response?.usage) {
const cachedTokens = response.usage.input_tokens_details?.cached_tokens || 0;
output.usage = {
// OpenAI includes cached tokens in input_tokens, so subtract to get non-cached input
input: (response.usage.input_tokens || 0) - cachedTokens,
output: response.usage.output_tokens || 0,
cacheRead: cachedTokens,
cacheWrite: 0,
totalTokens: response.usage.total_tokens || 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
};
}
calculateCost(model, output.usage);
if (options?.applyServiceTierPricing) {
const serviceTier = options.resolveServiceTier
? options.resolveServiceTier(response?.service_tier, options.serviceTier)
: (response?.service_tier ?? options.serviceTier);
options.applyServiceTierPricing(output.usage, serviceTier);
}
// Map status to stop reason
output.stopReason = mapStopReason(response?.status);
if (output.content.some((b) => b.type === "toolCall") && output.stopReason === "stop") {
output.stopReason = "toolUse";
}
};
for await (const event of openaiStream) {
if (event.type === "response.created") {
@@ -491,38 +524,12 @@ export async function processResponsesStream<TApi extends Api>(
currentBlock = null;
stream.push({ type: "toolcall_end", contentIndex: blockIndex(), toolCall, partial: output });
}
} else if (event.type === "response.completed") {
const response = event.response;
if (response?.id) {
output.responseId = response.id;
}
if (response?.usage) {
const cachedTokens = response.usage.input_tokens_details?.cached_tokens || 0;
output.usage = {
// OpenAI includes cached tokens in input_tokens, so subtract to get non-cached input
input: (response.usage.input_tokens || 0) - cachedTokens,
output: response.usage.output_tokens || 0,
cacheRead: cachedTokens,
cacheWrite: 0,
totalTokens: response.usage.total_tokens || 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
};
}
calculateCost(model, output.usage);
if (options?.applyServiceTierPricing) {
const serviceTier = options.resolveServiceTier
? options.resolveServiceTier(response?.service_tier, options.serviceTier)
: (response?.service_tier ?? options.serviceTier);
options.applyServiceTierPricing(output.usage, serviceTier);
}
// Map status to stop reason
output.stopReason = mapStopReason(response?.status);
if (output.content.some((b) => b.type === "toolCall") && output.stopReason === "stop") {
output.stopReason = "toolUse";
}
} else if (event.type === "response.completed" || event.type === "response.incomplete") {
finalizeResponse(event.response);
} else if (event.type === "error") {
throw new Error(`Error Code ${event.code}: ${event.message}` || "Unknown error");
} else if (event.type === "response.failed") {
sawTerminalResponseEvent = true;
const error = event.response?.error;
const details = event.response?.incomplete_details;
const msg = error
@@ -533,6 +540,9 @@ export async function processResponsesStream<TApi extends Api>(
throw new Error(msg);
}
}
if (!sawTerminalResponseEvent) {
throw new Error("OpenAI Responses stream ended before a terminal response event");
}
}
function mapStopReason(status: OpenAI.Responses.ResponseStatus | undefined): StopReason {
packages/ai/src/api/openai-responses.lazy.ts
+4
View File
@@ -0,0 +1,4 @@
import type { ProviderStreams } from "../types.ts";
import { lazyApi } from "./lazy.ts";
export const openAIResponsesApi = (): ProviderStreams => lazyApi(() => import("./openai-responses.ts"));
packages/ai/src/providers/openai-responses.ts → packages/ai/src/api/openai-responses.ts
+27 -28
View File
@@ -9,6 +9,7 @@ import type {
Model,
OpenAIResponsesCompat,
ProviderEnv,
ProviderHeaders,
SimpleStreamOptions,
StreamFunction,
StreamOptions,
@@ -17,7 +18,6 @@ import type {
import { AssistantMessageEventStream } from "../utils/event-stream.ts";
import { headersToRecord } from "../utils/headers.ts";
import { getProviderEnvValue } from "../utils/provider-env.ts";
import { isCloudflareProvider, resolveCloudflareBaseUrl } from "./cloudflare.ts";
import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.ts";
import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts";
import { convertResponsesMessages, convertResponsesTools, processResponsesStream } from "./openai-responses-shared.ts";
@@ -25,6 +25,21 @@ import { buildBaseOptions } from "./simple-options.ts";
const OPENAI_TOOL_CALL_PROVIDERS = new Set(["openai", "openai-codex", "opencode"]);
function hasHeader(headers: ProviderHeaders | undefined, name: string): boolean {
if (!headers) return false;
const expected = name.toLowerCase();
for (const [key, value] of Object.entries(headers)) {
if (key.toLowerCase() === expected && value !== null && value.trim().length > 0) return true;
}
return false;
}
function getClientApiKey(provider: string, apiKey: string | undefined, headers: ProviderHeaders | undefined): string {
if (apiKey) return apiKey;
if (hasHeader(headers, "authorization") || hasHeader(headers, "cf-aig-authorization")) return "unused";
throw new Error(`No API key for provider: ${provider}`);
}
/**
* Resolve cache retention preference.
* Defaults to "short" and uses PI_CACHE_RETENTION for backward compatibility.
@@ -80,7 +95,7 @@ export interface OpenAIResponsesOptions extends StreamOptions {
/**
* Generate function for OpenAI Responses API
*/
export const streamOpenAIResponses: StreamFunction<"openai-responses", OpenAIResponsesOptions> = (
export const stream: StreamFunction<"openai-responses", OpenAIResponsesOptions> = (
model: Model<"openai-responses">,
context: Context,
options?: OpenAIResponsesOptions,
@@ -109,13 +124,10 @@ export const streamOpenAIResponses: StreamFunction<"openai-responses", OpenAIRes
try {
// Create OpenAI client
const apiKey = options?.apiKey;
if (!apiKey) {
throw new Error(`No API key for provider: ${model.provider}`);
}
const apiKey = getClientApiKey(model.provider, options?.apiKey, options?.headers);
const cacheRetention = resolveCacheRetention(options?.cacheRetention, options?.env);
const cacheSessionId = cacheRetention === "none" ? undefined : options?.sessionId;
const client = createClient(model, context, apiKey, options?.headers, cacheSessionId, options?.env);
const client = createClient(model, context, apiKey, options?.headers, cacheSessionId);
let params = buildParams(model, context, options);
const nextParams = await options?.onPayload?.(params, model);
if (nextParams !== undefined) {
@@ -161,21 +173,18 @@ export const streamOpenAIResponses: StreamFunction<"openai-responses", OpenAIRes
return stream;
};
export const streamSimpleOpenAIResponses: StreamFunction<"openai-responses", SimpleStreamOptions> = (
export const streamSimple: StreamFunction<"openai-responses", SimpleStreamOptions> = (
model: Model<"openai-responses">,
context: Context,
options?: SimpleStreamOptions,
): AssistantMessageEventStream => {
const apiKey = options?.apiKey;
if (!apiKey) {
throw new Error(`No API key for provider: ${model.provider}`);
}
getClientApiKey(model.provider, options?.apiKey, options?.headers);
const base = buildBaseOptions(model, options, apiKey);
const base = buildBaseOptions(model, options, options?.apiKey);
const clampedReasoning = options?.reasoning ? clampThinkingLevel(model, options.reasoning) : undefined;
const reasoningEffort = clampedReasoning === "off" ? undefined : clampedReasoning;
return streamOpenAIResponses(model, context, {
return stream(model, context, {
...base,
reasoningEffort,
} satisfies OpenAIResponsesOptions);
@@ -185,12 +194,11 @@ function createClient(
model: Model<"openai-responses">,
context: Context,
apiKey: string,
optionsHeaders?: Record<string, string>,
optionsHeaders?: ProviderHeaders,
sessionId?: string,
env?: ProviderEnv,
) {
const compat = getCompat(model);
const headers = { ...model.headers };
const headers: ProviderHeaders = { ...model.headers };
if (model.provider === "github-copilot") {
const hasImages = hasCopilotVisionInput(context.messages);
const copilotHeaders = buildCopilotDynamicHeaders({
@@ -212,20 +220,11 @@ function createClient(
Object.assign(headers, optionsHeaders);
}
const defaultHeaders =
model.provider === "cloudflare-ai-gateway"
? {
...headers,
Authorization: headers.Authorization ?? null,
"cf-aig-authorization": `Bearer ${apiKey}`,
}
: headers;
return new OpenAI({
apiKey,
baseURL: isCloudflareProvider(model.provider) ? resolveCloudflareBaseUrl(model, env) : model.baseUrl,
baseURL: model.baseUrl,
dangerouslyAllowBrowser: true,
defaultHeaders,
defaultHeaders: headers,
});
}
packages/ai/src/api/openrouter-images.lazy.ts
+10
View File
@@ -0,0 +1,10 @@
import type { ImagesModel, ProviderImages } from "../types.ts";
export const openrouterImagesApi = (): ProviderImages => ({
generateImages: async (model, context, options) =>
(await import("./openrouter-images.ts")).generateImages(
model as ImagesModel<"openrouter-images">,
context,
options,
),
});
packages/ai/src/providers/images/openrouter.ts → packages/ai/src/api/openrouter-images.ts
+7 -9
View File
@@ -13,10 +13,11 @@ import type {
ImagesFunction,
ImagesModel,
ImagesOptions,
ProviderHeaders,
TextContent,
} from "../../types.ts";
import { headersToRecord } from "../../utils/headers.ts";
import { sanitizeSurrogates } from "../../utils/sanitize-unicode.ts";
} from "../types.ts";
import { headersToRecord, providerHeadersToRecord } from "../utils/headers.ts";
import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts";
interface OpenRouterGeneratedImage {
image_url?: string | { url?: string };
@@ -34,7 +35,7 @@ type OpenRouterImageGenerationResponse = ChatCompletion & {
choices: OpenRouterImageGenerationChoice[];
};
export const generateImagesOpenRouter: ImagesFunction<"openrouter-images", ImagesOptions> = async (
export const generateImages: ImagesFunction<"openrouter-images", ImagesOptions> = async (
model: ImagesModel<"openrouter-images">,
context: ImagesContext,
options?: ImagesOptions,
@@ -106,16 +107,13 @@ export const generateImagesOpenRouter: ImagesFunction<"openrouter-images", Image
function createClient(
model: ImagesModel<"openrouter-images">,
apiKey: string,
optionsHeaders?: Record<string, string>,
optionsHeaders?: ProviderHeaders,
): OpenAI {
return new OpenAI({
apiKey,
baseURL: model.baseUrl,
dangerouslyAllowBrowser: true,
defaultHeaders: {
...model.headers,
...optionsHeaders,
},
defaultHeaders: providerHeadersToRecord({ ...model.headers, ...optionsHeaders }),
});
}
packages/ai/src/providers/simple-options.ts → packages/ai/src/api/simple-options.ts
View File
packages/ai/src/providers/transform-messages.ts → packages/ai/src/api/transform-messages.ts
View File
packages/ai/src/auth/context.ts
+45
View File
@@ -0,0 +1,45 @@
import type { AuthContext } from "./types.ts";
interface NodeFsModule {
access(path: string): Promise<void>;
}
interface NodeOsModule {
homedir(): string;
}
// Variable specifier so browser bundlers do not try to resolve node builtins.
const importNodeModule = (specifier: string): Promise<unknown> => import(specifier);
function getProcessEnv(): Record<string, string | undefined> | undefined {
const proc = (globalThis as { process?: { env?: Record<string, string | undefined> } }).process;
return proc?.env;
}
/**
* Default auth context: env vars from `process.env` (undefined in browsers),
* file existence via node:fs (always false in browsers).
*/
export function defaultProviderAuthContext(): AuthContext {
return {
async env(name: string): Promise<string | undefined> {
const value = getProcessEnv()?.[name];
return typeof value === "string" && value.trim().length > 0 ? value : undefined;
},
async fileExists(path: string): Promise<boolean> {
try {
const fs = (await importNodeModule("node:fs/promises")) as NodeFsModule;
let resolved = path;
if (resolved.startsWith("~")) {
const os = (await importNodeModule("node:os")) as NodeOsModule;
resolved = os.homedir() + resolved.slice(1);
}
await fs.access(resolved);
return true;
} catch {
return false;
}
},
};
}
packages/ai/src/auth/credential-store.ts
+47
View File
@@ -0,0 +1,47 @@
import type { Credential, CredentialStore } from "./types.ts";
/**
* Default in-memory credential store. Apps inject persistent stores.
* Keyed by `Provider.id`, one credential per provider; see `CredentialStore`.
* Writes are serialized per provider through a promise chain.
*/
export class InMemoryCredentialStore implements CredentialStore {
private credentials = new Map<string, Credential>();
private chains = new Map<string, Promise<unknown>>();
/** Serialize tasks per provider id. */
private enqueue<T>(providerId: string, task: () => Promise<T>): Promise<T> {
const previous = this.chains.get(providerId) ?? Promise.resolve();
const next = (async () => {
await previous.catch(() => {});
return task();
})();
this.chains.set(
providerId,
next.catch(() => {}),
);
return next;
}
async read(providerId: string): Promise<Credential | undefined> {
return this.credentials.get(providerId);
}
modify(
providerId: string,
fn: (current: Credential | undefined) => Promise<Credential | undefined>,
): Promise<Credential | undefined> {
return this.enqueue(providerId, async () => {
const current = this.credentials.get(providerId);
const next = await fn(current);
if (next !== undefined) this.credentials.set(providerId, next);
return next ?? current;
});
}
delete(providerId: string): Promise<void> {
return this.enqueue(providerId, async () => {
this.credentials.delete(providerId);
});
}
}
packages/ai/src/auth/helpers.ts
+46
View File
@@ -0,0 +1,46 @@
import type { ApiKeyAuth, OAuthAuth } from "./types.ts";
/**
* Standard api-key auth: a stored credential key wins, otherwise the first
* set env var resolves. Includes a `login` that prompts for the key.
* Providers with non-standard resolution (provider env, ambient files, IAM)
* write their own `ApiKeyAuth`.
*/
export function envApiKeyAuth(name: string, envVars: readonly string[]): ApiKeyAuth {
return {
name,
login: async (callbacks) => {
const key = await callbacks.prompt({ type: "secret", message: `Enter ${name}` });
return { type: "api_key", key };
},
resolve: async ({ ctx, credential }) => {
if (credential?.key) return { auth: { apiKey: credential.key }, source: "stored credential" };
for (const envVar of envVars) {
const value = await ctx.env(envVar);
if (value) return { auth: { apiKey: value }, source: envVar };
}
return undefined;
},
};
}
/**
* Wraps a dynamically imported `OAuthAuth` so provider definitions can
* advertise OAuth without importing the implementation. The flow loads on
* first `login`/`refresh`/`toAuth` call; callers keep Node-only flow code out
* of bundles by loading through a bundler-opaque dynamic import (variable
* specifier, see the bedrock lazy wrapper).
*/
export function lazyOAuth(input: { name: string; load: () => Promise<OAuthAuth> }): OAuthAuth {
let promise: Promise<OAuthAuth> | undefined;
const loaded = () => {
promise ??= input.load();
return promise;
};
return {
name: input.name,
login: async (callbacks) => (await loaded()).login(callbacks),
refresh: async (credential) => (await loaded()).refresh(credential),
toAuth: async (credential) => (await loaded()).toAuth(credential),
};
}
packages/ai/src/auth/resolve.ts
+141
View File
@@ -0,0 +1,141 @@
import type { Api, ImagesApi, ImagesModel, Model, ProviderEnv } from "../types.ts";
import type {
ApiKeyAuth,
ApiKeyCredential,
AuthContext,
AuthResult,
Credential,
CredentialStore,
OAuthAuth,
OAuthCredential,
ProviderAuth,
} from "./types.ts";
export type ModelsErrorCode = "model_source" | "model_validation" | "provider" | "stream" | "auth" | "oauth";
export interface AuthResolutionOverrides {
apiKey?: string;
env?: ProviderEnv;
}
export class ModelsError extends Error {
readonly code: ModelsErrorCode;
constructor(code: ModelsErrorCode, message: string, options?: { cause?: unknown }) {
super(message, options);
this.name = "ModelsError";
this.code = code;
}
}
/** Model shape auth resolution receives: chat or image-generation models. */
export type AuthModel = Model<Api> | ImagesModel<ImagesApi>;
/**
* Auth resolution shared by the `Models` and `ImagesModels` collections.
* A stored credential owns the provider: ambient/env is consulted only when
* nothing is stored. No silent env fallback after a failed refresh or for a
* credential type without a matching handler.
*/
export async function resolveProviderAuth(
provider: { id: string; auth: ProviderAuth },
model: AuthModel,
credentials: CredentialStore,
authContext: AuthContext,
overrides?: AuthResolutionOverrides,
): Promise<AuthResult | undefined> {
const requestAuthContext = overrides?.env ? overlayEnvAuthContext(authContext, overrides.env) : authContext;
if (overrides?.apiKey !== undefined && provider.auth.apiKey) {
return resolveApiKey(requestAuthContext, provider.auth.apiKey, model, {
type: "api_key",
key: overrides.apiKey,
env: overrides.env,
});
}
const stored = await readCredential(credentials, provider.id);
if (stored) {
if (stored.type === "oauth" && provider.auth.oauth) {
return resolveStoredOAuth(credentials, provider.id, provider.auth.oauth, stored);
}
if (stored.type === "api_key" && provider.auth.apiKey) {
const credential = overrides?.env ? { ...stored, env: { ...stored.env, ...overrides.env } } : stored;
return resolveApiKey(requestAuthContext, provider.auth.apiKey, model, credential);
}
return undefined;
}
// Ambient (env vars, AWS profiles, ADC files).
return provider.auth.apiKey ? resolveApiKey(requestAuthContext, provider.auth.apiKey, model, undefined) : undefined;
}
function overlayEnvAuthContext(base: AuthContext, env: ProviderEnv): AuthContext {
return {
env: async (name) => env[name] || (await base.env(name)),
fileExists: (path) => base.fileExists(path),
};
}
/**
* OAuth resolution with double-checked locking (same pattern as today's
* AuthStorage): valid tokens cost zero locks; expired tokens lock, re-check
* expiry under the lock, refresh once globally, and persist the rotated
* credential before release.
*/
async function resolveStoredOAuth(
credentials: CredentialStore,
providerId: string,
oauth: OAuthAuth,
stored: OAuthCredential,
): Promise<AuthResult | undefined> {
let credential = stored;
if (Date.now() >= credential.expires) {
// Optimistic check said expired; the authoritative check runs under the lock.
let post: Credential | undefined;
try {
post = await credentials.modify(providerId, async (current) => {
if (current?.type !== "oauth") return undefined; // logged out meanwhile
if (Date.now() < current.expires) return undefined; // another process/request refreshed
try {
return await oauth.refresh(current);
} catch (error) {
throw new ModelsError("oauth", `OAuth refresh failed for ${providerId}`, { cause: error });
}
});
} catch (error) {
if (error instanceof ModelsError) throw error;
throw new ModelsError("auth", `Credential store modify failed for ${providerId}`, { cause: error });
}
if (post?.type !== "oauth") return undefined; // logged out meanwhile
credential = post;
}
try {
return { auth: await oauth.toAuth(credential), source: "OAuth" };
} catch (error) {
throw new ModelsError("oauth", `OAuth auth derivation failed for ${providerId}`, { cause: error });
}
}
async function resolveApiKey(
authContext: AuthContext,
apiKey: ApiKeyAuth,
model: AuthModel,
credential: ApiKeyCredential | undefined,
): Promise<AuthResult | undefined> {
try {
return await apiKey.resolve({ model, ctx: authContext, credential });
} catch (error) {
throw new ModelsError("auth", `API key auth failed for provider ${model.provider}`, { cause: error });
}
}
async function readCredential(credentials: CredentialStore, providerId: string): Promise<Credential | undefined> {
try {
return await credentials.read(providerId);
} catch (error) {
throw new ModelsError("auth", `Credential store read failed for ${providerId}`, { cause: error });
}
}
packages/ai/src/auth/types.ts
+182
View File
@@ -0,0 +1,182 @@
import type { Api, ImagesApi, ImagesModel, Model, ProviderEnv, ProviderHeaders } from "../types.ts";
import type { OAuthCredentials } from "../utils/oauth/types.ts";
/**
* Request auth for a single model request. If a value cannot be expressed as
* `apiKey`, `headers`, or `baseUrl`, it is provider config, not auth.
*/
export interface ModelAuth {
apiKey?: string;
headers?: ProviderHeaders;
baseUrl?: string;
}
/**
* Stored api-key credential. `env` holds provider-scoped environment/config
* values such as Cloudflare account/gateway ids.
*/
export interface ApiKeyCredential {
type: "api_key";
key?: string;
env?: ProviderEnv;
}
/** Stored OAuth credential (`access`, `refresh`, `expires` from OAuthCredentials). */
export interface OAuthCredential extends OAuthCredentials {
type: "oauth";
}
/** One type-tagged credential per provider — the shape of today's auth.json. */
export type Credential = ApiKeyCredential | OAuthCredential;
/**
* App-owned credential storage, keyed by `Provider.id`, one credential per
* provider. `modify` is the only write path, so every mutation is a
* serialized read-modify-write; `Models.getAuth()` runs OAuth refresh inside
* `modify` so concurrent requests cannot double-refresh a rotated token. The
* app persists a credential after login via
* `modify(provider.id, async () => credential)`. Login/logout orchestration
* is app-owned.
*
* Error semantics: `read` resolves `undefined` for missing entries. Methods
* reject only on storage failure; `Models` wraps such rejections in
* `ModelsError` with code "auth". Best-effort stores that serve an in-memory
* view and record persistence errors internally (like coding-agent's
* AuthStorage) are valid implementations.
*/
export interface CredentialStore {
/**
* Read the stored credential, possibly expired. Display/status use;
* resolved request auth comes from `Models.getAuth()`.
*/
read(providerId: string): Promise<Credential | undefined>;
/**
* Serialized write — the only write path. `fn` sees the current credential
* because correct writes (refresh, login-during-refresh) depend on it;
* return the new credential, or undefined to leave the entry unchanged.
* Mutual exclusion per provider id, cross-process too where the backing
* store supports it (e.g. a file lock). Resolves with the post-write
* credential. Rejections from `fn` propagate.
*/
modify(
providerId: string,
fn: (current: Credential | undefined) => Promise<Credential | undefined>,
): Promise<Credential | undefined>;
/** Remove a credential (logout). Implementations serialize this against `modify`. */
delete(providerId: string): Promise<void>;
}
/** Environment access for auth resolution. Injectable for tests and browsers. */
export interface AuthContext {
env(name: string): Promise<string | undefined>;
/** Check whether a file exists. Supports a leading `~`. Always false in browsers. */
fileExists(path: string): Promise<boolean>;
}
/** Result of resolving auth for a model. */
export interface AuthResult {
auth: ModelAuth;
/** Provider-scoped environment/config values resolved from credentials and ambient context. */
env?: ProviderEnv;
/** Human-readable label for status UI: "ANTHROPIC_API_KEY", "OAuth", "~/.aws/credentials". */
source?: string;
}
/**
* Prompt shown to the user during login. `signal` lets the flow cancel a
* pending prompt when an out-of-band event resolves the step, e.g. a
* `manual_code` prompt raced against a callback server, aborted when the
* callback wins.
*/
export type AuthPrompt = { signal?: AbortSignal } & (
| { type: "text"; message: string; placeholder?: string }
| { type: "secret"; message: string; placeholder?: string }
| { type: "select"; message: string; options: readonly { id: string; label: string; description?: string }[] }
| { type: "manual_code"; message: string; placeholder?: string }
);
export type AuthEvent =
| { type: "auth_url"; url: string; instructions?: string }
| {
type: "device_code";
userCode: string;
verificationUri: string;
intervalSeconds?: number;
expiresInSeconds?: number;
}
| { type: "progress"; message: string };
/**
* Login interaction callbacks serving both api-key and OAuth flows.
*
* `prompt()` returns the entered/selected string (`select` returns the option
* id). Rejects on cancel/abort. `signal` aborts the whole login flow;
* per-prompt cancellation uses `AuthPrompt.signal`.
*/
export interface AuthLoginCallbacks {
signal?: AbortSignal;
prompt(prompt: AuthPrompt): Promise<string>;
notify(event: AuthEvent): void;
}
/**
* Api-key auth: stored key/provider env plus ambient sources (env vars, AWS
* profiles, ADC files). Ambient-only providers omit `login`.
*/
export interface ApiKeyAuth {
/** Display name, e.g. "Anthropic API key". */
name: string;
/** Interactive setup (prompt for key/provider env). Absent = ambient-only. */
login?(callbacks: AuthLoginCallbacks): Promise<ApiKeyCredential>;
/**
* Resolve auth from the stored credential and/or ambient sources, merging
* per field (`credential.key ?? env("...")`, `credential.env?.NAME ?? env("...")`).
* undefined = not configured. Receives the chat or image-generation model
* the request is for (both carry `provider` and `baseUrl`).
*/
resolve(input: {
model: Model<Api> | ImagesModel<ImagesApi>;
ctx: AuthContext;
credential?: ApiKeyCredential;
}): Promise<AuthResult | undefined>;
}
/**
* OAuth auth. The `refresh`/`toAuth` split lets `Models` own the locked
* refresh pattern: `refresh` produces a credential, `toAuth` derives request
* auth from whatever credential ends up stored.
*/
export interface OAuthAuth {
/** Display name, e.g. "Anthropic (Claude Pro/Max)". */
name: string;
login(callbacks: AuthLoginCallbacks): Promise<OAuthCredential>;
/**
* Exchange the refresh token. Network call; throws on failure
* (invalid_grant etc.). `Models` runs this under the store lock.
*/
refresh(credential: OAuthCredential): Promise<OAuthCredential>;
/**
* Side-effect-free derivation of request auth from a valid credential.
* Covers per-credential baseUrl (GitHub Copilot). Async so lazy wrappers
* can load the implementation on first use.
*/
toAuth(credential: OAuthCredential): Promise<ModelAuth>;
}
/**
* Provider auth. At least one of `apiKey`/`oauth` must be present: even
* ambient-credential providers and keyless local servers provide `apiKey`
* auth whose `resolve()` reports whether the provider is configured.
*/
export interface ProviderAuth {
apiKey?: ApiKeyAuth;
oauth?: OAuthAuth;
}
packages/ai/src/bedrock-provider.ts
+3 -3
View File
@@ -1,6 +1,6 @@
import { streamBedrock, streamSimpleBedrock } from "./providers/amazon-bedrock.ts";
import { stream, streamSimple } from "./api/bedrock-converse-stream.ts";
export const bedrockProviderModule = {
streamBedrock,
streamSimpleBedrock,
stream,
streamSimple,
};
packages/ai/src/compat.ts
+277
View File
@@ -0,0 +1,277 @@
/**
* Temporary compatibility entrypoint preserving the old global pi-ai API
* surface: api-dispatch `stream()`/`complete()` with env API key injection,
* the api-registry, generated catalog reads (`getModel`/`getModels`/
* `getProviders`), per-API lazy stream wrappers, and image generation.
*
* Existing apps switch imports from "@earendil-works/pi-ai" to
* "@earendil-works/pi-ai/compat" unchanged; new code uses `createModels()`
* and the provider factories. This module is deleted with the coding-agent
* ModelManager migration.
*/
export * from "./api/anthropic-messages.lazy.ts";
export * from "./api/azure-openai-responses.lazy.ts";
export * from "./api/bedrock-converse-stream.lazy.ts";
export * from "./api/google-generative-ai.lazy.ts";
export * from "./api/google-vertex.lazy.ts";
export * from "./api/mistral-conversations.lazy.ts";
export * from "./api/openai-codex-responses.lazy.ts";
export * from "./api/openai-completions.lazy.ts";
export * from "./api/openai-responses.lazy.ts";
export * from "./env-api-keys.ts";
export * from "./image-models.ts";
export * from "./images.ts";
export * from "./images-api-registry.ts";
export * from "./index.ts";
export * from "./legacy-api-aliases.ts";
export * from "./providers/images/register-builtins.ts";
import { anthropicMessagesApi } from "./api/anthropic-messages.lazy.ts";
import { azureOpenAIResponsesApi } from "./api/azure-openai-responses.lazy.ts";
import { bedrockConverseStreamApi } from "./api/bedrock-converse-stream.lazy.ts";
import { googleGenerativeAIApi } from "./api/google-generative-ai.lazy.ts";
import { googleVertexApi } from "./api/google-vertex.lazy.ts";
import { mistralConversationsApi } from "./api/mistral-conversations.lazy.ts";
import { openAICodexResponsesApi } from "./api/openai-codex-responses.lazy.ts";
import { openAICompletionsApi } from "./api/openai-completions.lazy.ts";
import { openAIResponsesApi } from "./api/openai-responses.lazy.ts";
import { getEnvApiKey } from "./env-api-keys.ts";
import { builtinModels, getBuiltinModel, getBuiltinModels, getBuiltinProviders } from "./providers/all.ts";
import { createFauxCore, type FauxProviderRegistration, type RegisterFauxProviderOptions } from "./providers/faux.ts";
import type {
Api,
ApiStreamOptions,
AssistantMessage,
AssistantMessageEventStream,
Context,
Model,
ProviderStreamOptions,
ProviderStreams,
SimpleStreamOptions,
StreamFunction,
StreamOptions,
} from "./types.ts";
/** @deprecated Static catalog read. Use `getBuiltinModel` from "@earendil-works/pi-ai/providers/all" or `Models.getModel()`. */
export const getModel = getBuiltinModel;
/** @deprecated Static catalog read. Use `getBuiltinModels` from "@earendil-works/pi-ai/providers/all" or `Models.getModels()`. */
export const getModels = getBuiltinModels;
/** @deprecated Static catalog read. Use `getBuiltinProviders` from "@earendil-works/pi-ai/providers/all" or `Models.getProviders()`. */
export const getProviders = getBuiltinProviders;
export type ApiStreamFunction = (
model: Model<Api>,
context: Context,
options?: StreamOptions,
) => AssistantMessageEventStream;
export type ApiStreamSimpleFunction = (
model: Model<Api>,
context: Context,
options?: SimpleStreamOptions,
) => AssistantMessageEventStream;
export interface ApiProvider<TApi extends Api = Api, TOptions extends StreamOptions = StreamOptions> {
api: TApi;
stream: StreamFunction<TApi, TOptions>;
streamSimple: StreamFunction<TApi, SimpleStreamOptions>;
}
interface ApiProviderInternal {
api: Api;
stream: ApiStreamFunction;
streamSimple: ApiStreamSimpleFunction;
}
type RegisteredApiProvider = {
provider: ApiProviderInternal;
sourceId?: string;
};
const apiProviderRegistry = new Map<string, RegisteredApiProvider>();
function wrapStream<TApi extends Api, TOptions extends StreamOptions>(
api: TApi,
stream: StreamFunction<TApi, TOptions>,
): ApiStreamFunction {
return (model, context, options) => {
if (model.api !== api) {
throw new Error(`Mismatched api: ${model.api} expected ${api}`);
}
return stream(model as Model<TApi>, context, options as TOptions);
};
}
function wrapStreamSimple<TApi extends Api>(
api: TApi,
streamSimple: StreamFunction<TApi, SimpleStreamOptions>,
): ApiStreamSimpleFunction {
return (model, context, options) => {
if (model.api !== api) {
throw new Error(`Mismatched api: ${model.api} expected ${api}`);
}
return streamSimple(model as Model<TApi>, context, options);
};
}
export function registerApiProvider<TApi extends Api, TOptions extends StreamOptions>(
provider: ApiProvider<TApi, TOptions>,
sourceId?: string,
): void {
apiProviderRegistry.set(provider.api, {
provider: {
api: provider.api,
stream: wrapStream(provider.api, provider.stream),
streamSimple: wrapStreamSimple(provider.api, provider.streamSimple),
},
sourceId,
});
}
export function getApiProvider(api: Api): ApiProviderInternal | undefined {
return apiProviderRegistry.get(api)?.provider;
}
export function getApiProviders(): ApiProviderInternal[] {
return Array.from(apiProviderRegistry.values(), (entry) => entry.provider);
}
export function unregisterApiProviders(sourceId: string): void {
for (const [api, entry] of apiProviderRegistry.entries()) {
if (entry.sourceId === sourceId) {
apiProviderRegistry.delete(api);
}
}
}
function clearApiProviders(): void {
apiProviderRegistry.clear();
}
export function registerFauxProvider(options: RegisterFauxProviderOptions = {}): FauxProviderRegistration {
const core = createFauxCore(options);
const sourceId = `faux-provider-${Math.random().toString(36).slice(2, 10)}`;
registerApiProvider({ api: core.api, stream: core.stream, streamSimple: core.streamSimple }, sourceId);
return {
api: core.api,
models: core.models,
getModel: core.getModel,
state: core.state,
setResponses: core.setResponses,
appendResponses: core.appendResponses,
getPendingResponseCount: core.getPendingResponseCount,
unregister() {
unregisterApiProviders(sourceId);
},
};
}
const BUILTIN_APIS: [Api, ProviderStreams][] = [
["anthropic-messages", anthropicMessagesApi()],
["openai-completions", openAICompletionsApi()],
["openai-responses", openAIResponsesApi()],
["openai-codex-responses", openAICodexResponsesApi()],
["azure-openai-responses", azureOpenAIResponsesApi()],
["google-generative-ai", googleGenerativeAIApi()],
["google-vertex", googleVertexApi()],
["mistral-conversations", mistralConversationsApi()],
["bedrock-converse-stream", bedrockConverseStreamApi()],
];
const builtinApiProviderInstances = new Map<Api, ReturnType<typeof getApiProvider>>();
/**
* Registers the builtin API implementations into the api-registry without
* clobbering existing entries: compat may load after a test or extension has
* already registered an override for a builtin api id.
*/
export function registerBuiltInApiProviders(): void {
for (const [api, streams] of BUILTIN_APIS) {
if (!getApiProvider(api)) {
registerApiProvider({ api, stream: streams.stream, streamSimple: streams.streamSimple });
}
builtinApiProviderInstances.set(api, getApiProvider(api));
}
}
export function resetApiProviders(): void {
clearApiProviders();
builtinApiProviderInstances.clear();
registerBuiltInApiProviders();
}
registerBuiltInApiProviders();
const compatModels = builtinModels();
function hasExplicitApiKey(apiKey: string | undefined): apiKey is string {
return typeof apiKey === "string" && apiKey.trim().length > 0;
}
function withEnvApiKey<TOptions extends StreamOptions>(
model: Model<Api>,
options: TOptions | undefined,
): TOptions | undefined {
if (hasExplicitApiKey(options?.apiKey)) return options;
const apiKey = getEnvApiKey(model.provider, options?.env);
if (!apiKey) return options;
return { ...options, apiKey } as TOptions;
}
function shouldUseBuiltinModels(model: Model<Api>): boolean {
const builtin = compatModels.getModel(model.provider, model.id);
return builtin?.api === model.api && getApiProvider(model.api) === builtinApiProviderInstances.get(model.api);
}
function resolveApiProvider(api: Api) {
const provider = getApiProvider(api);
if (!provider) {
throw new Error(`No API provider registered for api: ${api}`);
}
return provider;
}
export function stream<TApi extends Api>(
model: Model<TApi>,
context: Context,
options?: ProviderStreamOptions,
): AssistantMessageEventStream {
if (shouldUseBuiltinModels(model)) {
return compatModels.stream(model, context, options as ApiStreamOptions<TApi> | undefined);
}
const provider = resolveApiProvider(model.api);
return provider.stream(model, context, withEnvApiKey(model, options) as StreamOptions);
}
export async function complete<TApi extends Api>(
model: Model<TApi>,
context: Context,
options?: ProviderStreamOptions,
): Promise<AssistantMessage> {
const s = stream(model, context, options);
return s.result();
}
export function streamSimple<TApi extends Api>(
model: Model<TApi>,
context: Context,
options?: SimpleStreamOptions,
): AssistantMessageEventStream {
if (shouldUseBuiltinModels(model)) {
return compatModels.streamSimple(model, context, options);
}
const provider = resolveApiProvider(model.api);
return provider.streamSimple(model, context, withEnvApiKey(model, options));
}
export async function completeSimple<TApi extends Api>(
model: Model<TApi>,
context: Context,
options?: SimpleStreamOptions,
): Promise<AssistantMessage> {
const s = streamSimple(model, context, options);
return s.result();
}
packages/ai/src/image-models.generated.ts
+30
View File
@@ -95,6 +95,21 @@ export const IMAGE_MODELS = {
cacheWrite: 0.08333333333333334,
},
} satisfies ImagesModel<"openrouter-images">,
"google/gemini-3-pro-image": {
id: "google/gemini-3-pro-image",
name: "Google: Nano Banana Pro (Gemini 3 Pro Image)",
api: "openrouter-images",
provider: "openrouter",
baseUrl: "https://openrouter.ai/api/v1",
input: ["image", "text"],
output: ["image", "text"],
cost: {
input: 2,
output: 12,
cacheRead: 0.19999999999999998,
cacheWrite: 0.375,
},
} satisfies ImagesModel<"openrouter-images">,
"google/gemini-3-pro-image-preview": {
id: "google/gemini-3-pro-image-preview",
name: "Google: Nano Banana Pro (Gemini 3 Pro Image Preview)",
@@ -110,6 +125,21 @@ export const IMAGE_MODELS = {
cacheWrite: 0.375,
},
} satisfies ImagesModel<"openrouter-images">,
"google/gemini-3.1-flash-image": {
id: "google/gemini-3.1-flash-image",
name: "Google: Nano Banana 2 (Gemini 3.1 Flash Image)",
api: "openrouter-images",
provider: "openrouter",
baseUrl: "https://openrouter.ai/api/v1",
input: ["image", "text"],
output: ["image", "text"],
cost: {
input: 0.5,
output: 3,
cacheRead: 0,
cacheWrite: 0,
},
} satisfies ImagesModel<"openrouter-images">,
"google/gemini-3.1-flash-image-preview": {
id: "google/gemini-3.1-flash-image-preview",
name: "Google: Nano Banana 2 (Gemini 3.1 Flash Image Preview)",
packages/ai/src/images-models.ts
+267
View File
@@ -0,0 +1,267 @@
import { defaultProviderAuthContext as defaultAuthContext } from "./auth/context.ts";
import { InMemoryCredentialStore } from "./auth/credential-store.ts";
import { ModelsError, resolveProviderAuth } from "./auth/resolve.ts";
import type { AuthContext, AuthResult, CredentialStore, ProviderAuth } from "./auth/types.ts";
import type { CreateModelsOptions } from "./models.ts";
import type { AssistantImages, ImagesApi, ImagesContext, ImagesModel, ImagesOptions, ProviderImages } from "./types.ts";
/**
* An image-generation provider: the image-side counterpart of `Provider`.
* Owns id/name metadata, auth, model listing, and generation behavior.
*/
export interface ImagesProvider {
readonly id: string;
readonly name: string;
/**
* Required: at least one of `apiKey`/`oauth`. Same semantics as chat
* providers; `ImagesModels.getAuth()` returns undefined when the provider
* is unconfigured.
*/
readonly auth: ProviderAuth;
/**
* Current known models, sync. Static providers return their catalog;
* dynamic providers return the list as of the last `refreshModels()`
* (empty before the first). Must not throw; `ImagesModels` treats a
* throwing implementation as having no models.
*/
getModels(): readonly ImagesModel<ImagesApi>[];
/**
* Dynamic providers only: fetch and update the model list. May reject
* (network); on rejection the model list stays at its last-known state
* and a later call retries.
*/
refreshModels?(): Promise<void>;
generateImages(
model: ImagesModel<ImagesApi>,
context: ImagesContext,
options?: ImagesOptions,
): Promise<AssistantImages>;
}
/**
* Runtime collection of image-generation providers plus auth application and
* generation convenience: the image-side counterpart of `Models`.
*/
export interface ImagesModels {
getProviders(): readonly ImagesProvider[];
getProvider(id: string): ImagesProvider | undefined;
/**
* Sync read of last-known models from one provider or all providers.
* Best-effort: a provider whose `getModels()` throws yields no models.
*/
getModels(provider?: string): readonly ImagesModel<ImagesApi>[];
/** Sync runtime model lookup against last-known lists. */
getModel(provider: string, id: string): ImagesModel<ImagesApi> | undefined;
/**
* Ask dynamic providers to re-fetch their model lists. With a provider id,
* rejects with `ModelsError` ("model_source") on that provider's fetch
* failure; without one, refreshes all providers concurrently best-effort.
* Static providers (no `refreshModels`) are no-ops.
*/
refresh(provider?: string): Promise<void>;
/**
* Resolve request auth for an image model. Same contract as
* `Models.getAuth()`: undefined when unknown/unconfigured, rejects with
* `ModelsError` ("oauth"/"auth") on real failures.
*/
getAuth(model: ImagesModel<ImagesApi>): Promise<AuthResult | undefined>;
/**
* Generate images through the owning provider with auth resolved and
* merged (explicit options win per field). Never rejects; failures are
* returned as an `AssistantImages` with `stopReason: "error"`.
*/
generateImages(
model: ImagesModel<ImagesApi>,
context: ImagesContext,
options?: ImagesOptions,
): Promise<AssistantImages>;
}
export interface MutableImagesModels extends ImagesModels {
/** Upsert/replace by provider.id. Provider ids are unique. */
setProvider(provider: ImagesProvider): void;
deleteProvider(id: string): void;
clearProviders(): void;
}
class ImagesModelsImpl implements MutableImagesModels {
private providers = new Map<string, ImagesProvider>();
private credentials: CredentialStore;
private authContext: AuthContext;
constructor(options?: CreateModelsOptions) {
this.credentials = options?.credentials ?? new InMemoryCredentialStore();
this.authContext = options?.authContext ?? defaultAuthContext();
}
setProvider(provider: ImagesProvider): void {
this.providers.set(provider.id, provider);
}
deleteProvider(id: string): void {
this.providers.delete(id);
}
clearProviders(): void {
this.providers.clear();
}
getProviders(): readonly ImagesProvider[] {
return Array.from(this.providers.values());
}
getProvider(id: string): ImagesProvider | undefined {
return this.providers.get(id);
}
getModels(provider?: string): readonly ImagesModel<ImagesApi>[] {
if (provider !== undefined) {
const entry = this.providers.get(provider);
if (!entry) return [];
try {
return entry.getModels();
} catch {
return [];
}
}
const models: ImagesModel<ImagesApi>[] = [];
for (const entry of this.providers.values()) {
try {
models.push(...entry.getModels());
} catch {
// Best-effort: ill-behaved providers yield no models.
}
}
return models;
}
getModel(provider: string, id: string): ImagesModel<ImagesApi> | undefined {
return this.getModels(provider).find((model) => model.id === id);
}
async refresh(provider?: string): Promise<void> {
if (provider !== undefined) {
const entry = this.providers.get(provider);
if (!entry?.refreshModels) return;
try {
await entry.refreshModels();
} catch (error) {
if (error instanceof ModelsError) throw error;
throw new ModelsError("model_source", `Model refresh failed for ${provider}`, { cause: error });
}
return;
}
// Cannot reject: the async mapper turns even sync throws from ill-behaved
// providers into rejections, and allSettled captures all of them.
await Promise.allSettled(Array.from(this.providers.values(), async (entry) => entry.refreshModels?.()));
}
async getAuth(model: ImagesModel<ImagesApi>): Promise<AuthResult | undefined> {
const provider = this.providers.get(model.provider);
if (!provider) return undefined;
return resolveProviderAuth(provider, model, this.credentials, this.authContext);
}
async generateImages(
model: ImagesModel<ImagesApi>,
context: ImagesContext,
options?: ImagesOptions,
): Promise<AssistantImages> {
try {
const provider = this.providers.get(model.provider);
if (!provider) {
throw new ModelsError("provider", `Unknown provider: ${model.provider}`);
}
const resolution = await resolveProviderAuth(provider, model, this.credentials, this.authContext, {
apiKey: options?.apiKey,
env: options?.env,
});
const auth = resolution?.auth;
if (!auth) {
return provider.generateImages(model, context, options);
}
const requestModel = auth.baseUrl ? { ...model, baseUrl: auth.baseUrl } : model;
// Explicit request options win per-field; headers/env merge per key.
const apiKey = options?.apiKey ?? auth.apiKey;
const headers = auth.headers || options?.headers ? { ...auth.headers, ...options?.headers } : undefined;
const env =
resolution.env || options?.env ? { ...(resolution.env ?? {}), ...(options?.env ?? {}) } : undefined;
return await provider.generateImages(requestModel, context, { ...options, apiKey, headers, env });
} catch (error) {
return {
api: model.api,
provider: model.provider,
model: model.id,
output: [],
stopReason: "error",
errorMessage: error instanceof Error ? error.message : String(error),
timestamp: Date.now(),
};
}
}
}
export function createImagesModels(options?: CreateModelsOptions): MutableImagesModels {
return new ImagesModelsImpl(options);
}
export interface CreateImagesProviderOptions {
id: string;
/** Display name. Default: `id`. */
name?: string;
/** Required — every provider has auth semantics, even ambient/keyless ones. */
auth: ProviderAuth;
/** Initial model list (empty for purely dynamic providers). */
models: readonly ImagesModel<ImagesApi>[];
/**
* Dynamic providers: fetch the current list. Stored on success; concurrent
* calls share one in-flight fetch. May reject: the stored list then stays
* at its last-known state, the rejection propagates to the caller of
* `refreshModels()` (wrapped as ModelsError "model_source" by
* `ImagesModels.refresh(provider)`), and a later call retries.
*/
refreshModels?: () => Promise<readonly ImagesModel<ImagesApi>[]>;
api: ProviderImages;
}
/** Builds an image-generation provider from parts. */
export function createImagesProvider(input: CreateImagesProviderOptions): ImagesProvider {
let models = input.models;
let inflightRefresh: Promise<void> | undefined;
const refreshModels = input.refreshModels;
return {
id: input.id,
name: input.name ?? input.id,
auth: input.auth,
getModels: () => models,
refreshModels: refreshModels
? () => {
inflightRefresh ??= (async () => {
try {
models = await refreshModels();
} finally {
inflightRefresh = undefined;
}
})();
return inflightRefresh;
}
: undefined,
generateImages: (model, context, options) => input.api.generateImages(model, context, options),
};
}
packages/ai/src/index.ts
+22 -21
View File
@@ -1,30 +1,30 @@
export type { Static, TSchema } from "typebox";
export { Type } from "typebox";
export * from "./api-registry.ts";
export * from "./env-api-keys.ts";
export * from "./image-models.ts";
export * from "./images.ts";
export * from "./images-api-registry.ts";
// Core only, side-effect free: no generated catalogs, no provider factories,
// no api-registry, no OAuth implementations, no compat. Provider factories
// live under "@earendil-works/pi-ai/providers/*", API implementations under
// "@earendil-works/pi-ai/api/*", the old global API under
// "@earendil-works/pi-ai/compat".
export type { AnthropicEffort, AnthropicOptions, AnthropicThinkingDisplay } from "./api/anthropic-messages.ts";
export type { AzureOpenAIResponsesOptions } from "./api/azure-openai-responses.ts";
export type { BedrockOptions, BedrockThinkingDisplay } from "./api/bedrock-converse-stream.ts";
export type { GoogleOptions } from "./api/google-generative-ai.ts";
export type { GoogleThinkingLevel } from "./api/google-shared.ts";
export type { GoogleVertexOptions } from "./api/google-vertex.ts";
export * from "./api/lazy.ts";
export type { MistralOptions } from "./api/mistral-conversations.ts";
export type { OpenAICodexResponsesOptions, OpenAICodexWebSocketDebugStats } from "./api/openai-codex-responses.ts";
export type { OpenAICompletionsOptions } from "./api/openai-completions.ts";
export type { OpenAIResponsesOptions } from "./api/openai-responses.ts";
export * from "./auth/context.ts";
export * from "./auth/credential-store.ts";
export * from "./auth/helpers.ts";
export * from "./auth/types.ts";
export * from "./images-models.ts";
export * from "./models.ts";
export type { BedrockOptions, BedrockThinkingDisplay } from "./providers/amazon-bedrock.ts";
export type { AnthropicEffort, AnthropicOptions, AnthropicThinkingDisplay } from "./providers/anthropic.ts";
export type { AzureOpenAIResponsesOptions } from "./providers/azure-openai-responses.ts";
export * from "./providers/faux.ts";
export type { GoogleOptions } from "./providers/google.ts";
export type { GoogleThinkingLevel } from "./providers/google-shared.ts";
export type { GoogleVertexOptions } from "./providers/google-vertex.ts";
export * from "./providers/images/register-builtins.ts";
export type { MistralOptions } from "./providers/mistral.ts";
export type {
OpenAICodexResponsesOptions,
OpenAICodexWebSocketDebugStats,
} from "./providers/openai-codex-responses.ts";
export type { OpenAICompletionsOptions } from "./providers/openai-completions.ts";
export type { OpenAIResponsesOptions } from "./providers/openai-responses.ts";
export * from "./providers/register-builtins.ts";
export * from "./session-resources.ts";
export * from "./stream.ts";
export * from "./types.ts";
export * from "./utils/diagnostics.ts";
export * from "./utils/event-stream.ts";
@@ -43,5 +43,6 @@ export type {
OAuthSelectPrompt,
} from "./utils/oauth/types.ts";
export * from "./utils/overflow.ts";
export * from "./utils/retry.ts";
export * from "./utils/typebox-helpers.ts";
export * from "./utils/validation.ts";
packages/ai/src/legacy-api-aliases.ts
+108
View File
@@ -0,0 +1,108 @@
import { anthropicMessagesApi } from "./api/anthropic-messages.lazy.ts";
import type { AnthropicOptions } from "./api/anthropic-messages.ts";
import { azureOpenAIResponsesApi } from "./api/azure-openai-responses.lazy.ts";
import type { AzureOpenAIResponsesOptions } from "./api/azure-openai-responses.ts";
import { googleGenerativeAIApi } from "./api/google-generative-ai.lazy.ts";
import type { GoogleOptions } from "./api/google-generative-ai.ts";
import { googleVertexApi } from "./api/google-vertex.lazy.ts";
import type { GoogleVertexOptions } from "./api/google-vertex.ts";
import { mistralConversationsApi } from "./api/mistral-conversations.lazy.ts";
import type { MistralOptions } from "./api/mistral-conversations.ts";
import { openAICodexResponsesApi } from "./api/openai-codex-responses.lazy.ts";
import type { OpenAICodexResponsesOptions } from "./api/openai-codex-responses.ts";
import { openAICompletionsApi } from "./api/openai-completions.lazy.ts";
import type { OpenAICompletionsOptions } from "./api/openai-completions.ts";
import { openAIResponsesApi } from "./api/openai-responses.lazy.ts";
import type { OpenAIResponsesOptions } from "./api/openai-responses.ts";
import type { SimpleStreamOptions, StreamFunction } from "./types.ts";
const anthropicMessagesStreams = anthropicMessagesApi();
const azureOpenAIResponsesStreams = azureOpenAIResponsesApi();
const googleGenerativeAIStreams = googleGenerativeAIApi();
const googleVertexStreams = googleVertexApi();
const mistralConversationsStreams = mistralConversationsApi();
const openAICodexResponsesStreams = openAICodexResponsesApi();
const openAICompletionsStreams = openAICompletionsApi();
const openAIResponsesStreams = openAIResponsesApi();
/** @deprecated Use `stream` from `@earendil-works/pi-ai/api/anthropic-messages` or `anthropicMessagesApi().stream`. */
export const streamAnthropic = anthropicMessagesStreams.stream as StreamFunction<
"anthropic-messages",
AnthropicOptions
>;
/** @deprecated Use `streamSimple` from `@earendil-works/pi-ai/api/anthropic-messages` or `anthropicMessagesApi().streamSimple`. */
export const streamSimpleAnthropic = anthropicMessagesStreams.streamSimple as StreamFunction<
"anthropic-messages",
SimpleStreamOptions
>;
/** @deprecated Use `stream` from `@earendil-works/pi-ai/api/azure-openai-responses` or `azureOpenAIResponsesApi().stream`. */
export const streamAzureOpenAIResponses = azureOpenAIResponsesStreams.stream as StreamFunction<
"azure-openai-responses",
AzureOpenAIResponsesOptions
>;
/** @deprecated Use `streamSimple` from `@earendil-works/pi-ai/api/azure-openai-responses` or `azureOpenAIResponsesApi().streamSimple`. */
export const streamSimpleAzureOpenAIResponses = azureOpenAIResponsesStreams.streamSimple as StreamFunction<
"azure-openai-responses",
SimpleStreamOptions
>;
/** @deprecated Use `stream` from `@earendil-works/pi-ai/api/google-generative-ai` or `googleGenerativeAIApi().stream`. */
export const streamGoogle = googleGenerativeAIStreams.stream as StreamFunction<"google-generative-ai", GoogleOptions>;
/** @deprecated Use `streamSimple` from `@earendil-works/pi-ai/api/google-generative-ai` or `googleGenerativeAIApi().streamSimple`. */
export const streamSimpleGoogle = googleGenerativeAIStreams.streamSimple as StreamFunction<
"google-generative-ai",
SimpleStreamOptions
>;
/** @deprecated Use `stream` from `@earendil-works/pi-ai/api/google-vertex` or `googleVertexApi().stream`. */
export const streamGoogleVertex = googleVertexStreams.stream as StreamFunction<"google-vertex", GoogleVertexOptions>;
/** @deprecated Use `streamSimple` from `@earendil-works/pi-ai/api/google-vertex` or `googleVertexApi().streamSimple`. */
export const streamSimpleGoogleVertex = googleVertexStreams.streamSimple as StreamFunction<
"google-vertex",
SimpleStreamOptions
>;
/** @deprecated Use `stream` from `@earendil-works/pi-ai/api/mistral-conversations` or `mistralConversationsApi().stream`. */
export const streamMistral = mistralConversationsStreams.stream as StreamFunction<
"mistral-conversations",
MistralOptions
>;
/** @deprecated Use `streamSimple` from `@earendil-works/pi-ai/api/mistral-conversations` or `mistralConversationsApi().streamSimple`. */
export const streamSimpleMistral = mistralConversationsStreams.streamSimple as StreamFunction<
"mistral-conversations",
SimpleStreamOptions
>;
/** @deprecated Use `stream` from `@earendil-works/pi-ai/api/openai-codex-responses` or `openAICodexResponsesApi().stream`. */
export const streamOpenAICodexResponses = openAICodexResponsesStreams.stream as StreamFunction<
"openai-codex-responses",
OpenAICodexResponsesOptions
>;
/** @deprecated Use `streamSimple` from `@earendil-works/pi-ai/api/openai-codex-responses` or `openAICodexResponsesApi().streamSimple`. */
export const streamSimpleOpenAICodexResponses = openAICodexResponsesStreams.streamSimple as StreamFunction<
"openai-codex-responses",
SimpleStreamOptions
>;
/** @deprecated Use `stream` from `@earendil-works/pi-ai/api/openai-completions` or `openAICompletionsApi().stream`. */
export const streamOpenAICompletions = openAICompletionsStreams.stream as StreamFunction<
"openai-completions",
OpenAICompletionsOptions
>;
/** @deprecated Use `streamSimple` from `@earendil-works/pi-ai/api/openai-completions` or `openAICompletionsApi().streamSimple`. */
export const streamSimpleOpenAICompletions = openAICompletionsStreams.streamSimple as StreamFunction<
"openai-completions",
SimpleStreamOptions
>;
/** @deprecated Use `stream` from `@earendil-works/pi-ai/api/openai-responses` or `openAIResponsesApi().stream`. */
export const streamOpenAIResponses = openAIResponsesStreams.stream as StreamFunction<
"openai-responses",
OpenAIResponsesOptions
>;
/** @deprecated Use `streamSimple` from `@earendil-works/pi-ai/api/openai-responses` or `openAIResponsesApi().streamSimple`. */
export const streamSimpleOpenAIResponses = openAIResponsesStreams.streamSimple as StreamFunction<
"openai-responses",
SimpleStreamOptions
>;
packages/ai/src/models.generated.ts
+70 -17175
View File
File diff suppressed because it is too large Load Diff
packages/ai/src/models.ts
+373 -27
View File
@@ -1,39 +1,385 @@
import { MODELS } from "./models.generated.ts";
import type { Api, KnownProvider, Model, ModelThinkingLevel, Usage } from "./types.ts";
import { lazyStream } from "./api/lazy.ts";
import { defaultProviderAuthContext as defaultAuthContext } from "./auth/context.ts";
import { InMemoryCredentialStore } from "./auth/credential-store.ts";
import { ModelsError, resolveProviderAuth } from "./auth/resolve.ts";
import type { AuthContext, AuthResult, CredentialStore, ProviderAuth } from "./auth/types.ts";
import type {
Api,
ApiStreamOptions,
AssistantMessage,
AssistantMessageEventStream,
Context,
Model,
ModelThinkingLevel,
ProviderHeaders,
ProviderStreams,
SimpleStreamOptions,
StreamOptions,
Usage,
} from "./types.ts";
const modelRegistry: Map<string, Map<string, Model<Api>>> = new Map();
export { type AuthModel, ModelsError, type ModelsErrorCode } from "./auth/resolve.ts";
// Initialize registry from MODELS on module load
for (const [provider, models] of Object.entries(MODELS)) {
const providerModels = new Map<string, Model<Api>>();
for (const [id, model] of Object.entries(models)) {
providerModels.set(id, model as Model<Api>);
}
modelRegistry.set(provider, providerModels);
/**
* A provider is the concrete runtime unit. It owns id/name/base metadata,
* auth methods, model listing, and stream behavior.
*
* `TApi` lets concrete provider factories declare which APIs their models
* use (e.g. `openaiProvider(): Provider<"openai-responses" | "openai-completions">`),
* giving typed model lists to direct factory users. Inside a `Models`
* collection providers are held as `Provider<Api>`.
*/
export interface Provider<TApi extends Api = Api> {
readonly id: string;
readonly name: string;
readonly baseUrl?: string;
readonly headers?: ProviderHeaders;
/**
* Required: at least one of `apiKey`/`oauth`. Every provider has auth
* semantics — even providers with only ambient credentials (env vars, AWS
* profiles, ADC files) and keyless local servers provide `apiKey` auth
* whose `resolve()` reports whether the provider is configured.
* `Models.getAuth()` returns undefined when the provider is unconfigured.
*/
readonly auth: ProviderAuth;
/**
* Current known models, sync. Static providers return their catalog;
* dynamic providers return the list as of the last `refreshModels()`
* (empty before the first). Must not throw; `Models` treats a throwing
* implementation as having no models.
*/
getModels(): readonly Model<TApi>[];
/**
* Dynamic providers only: fetch and update the model list. Side-effect-free
* discovery (no loading/downloading); provider-specific model lifecycle
* belongs in app commands. Concurrent calls share one in-flight fetch.
* May reject (network); on rejection the model list stays at its last-known
* state and a later call retries.
*/
refreshModels?(): Promise<void>;
stream<T extends TApi>(
model: Model<T>,
context: Context,
options?: ApiStreamOptions<T>,
): AssistantMessageEventStream;
streamSimple(model: Model<TApi>, context: Context, options?: SimpleStreamOptions): AssistantMessageEventStream;
}
type ModelApi<
TProvider extends KnownProvider,
TModelId extends keyof (typeof MODELS)[TProvider],
> = (typeof MODELS)[TProvider][TModelId] extends { api: infer TApi } ? (TApi extends Api ? TApi : never) : never;
/**
* Runtime collection of providers plus auth application and stream
* convenience. Providers own stream behavior; `Models` resolves auth and
* delegates each request to the provider that owns the model.
*/
export interface Models {
getProviders(): readonly Provider[];
getProvider(id: string): Provider | undefined;
export function getModel<TProvider extends KnownProvider, TModelId extends keyof (typeof MODELS)[TProvider]>(
provider: TProvider,
modelId: TModelId,
): Model<ModelApi<TProvider, TModelId>> {
const providerModels = modelRegistry.get(provider);
return providerModels?.get(modelId as string) as Model<ModelApi<TProvider, TModelId>>;
/**
* Sync read of last-known models from one provider or all providers.
* Best-effort: a provider whose `getModels()` throws yields no models.
*/
getModels(provider?: string): readonly Model<Api>[];
/**
* Sync runtime model lookup against last-known lists. Dynamic model lists
* are typed as `Model<Api>`; narrow with the `hasApi()` type guard.
*/
getModel(provider: string, id: string): Model<Api> | undefined;
/**
* Ask dynamic providers to re-fetch their model lists. With a provider id,
* rejects with `ModelsError` ("model_source") on that provider's fetch
* failure; without one, refreshes all providers concurrently best-effort.
* Static providers (no `refreshModels`) are no-ops.
*/
refresh(provider?: string): Promise<void>;
/**
* Resolve request auth for a model. Includes a source label for status UI.
* Resolves `undefined` when the provider is unknown or unconfigured.
* Rejects with `ModelsError`: code "oauth" when a token refresh fails (the
* stored credential is preserved for retry; re-login fixes it), code "auth"
* when api-key resolution or the credential store fails. Request paths
* surface rejections as stream errors; status/availability UIs catch them
* and render "needs re-login" instead of treating them as unconfigured.
*/
getAuth(model: Model<Api>): Promise<AuthResult | undefined>;
stream<TApi extends Api>(
model: Model<TApi>,
context: Context,
options?: ApiStreamOptions<TApi>,
): AssistantMessageEventStream;
complete<TApi extends Api>(
model: Model<TApi>,
context: Context,
options?: ApiStreamOptions<TApi>,
): Promise<AssistantMessage>;
streamSimple(model: Model<Api>, context: Context, options?: SimpleStreamOptions): AssistantMessageEventStream;
completeSimple(model: Model<Api>, context: Context, options?: SimpleStreamOptions): Promise<AssistantMessage>;
}
export function getProviders(): KnownProvider[] {
return Array.from(modelRegistry.keys()) as KnownProvider[];
export interface MutableModels extends Models {
/** Upsert/replace by provider.id. Provider ids are unique. */
setProvider(provider: Provider): void;
deleteProvider(id: string): void;
clearProviders(): void;
}
export function getModels<TProvider extends KnownProvider>(
provider: TProvider,
): Model<ModelApi<TProvider, keyof (typeof MODELS)[TProvider]>>[] {
const models = modelRegistry.get(provider);
return models ? (Array.from(models.values()) as Model<ModelApi<TProvider, keyof (typeof MODELS)[TProvider]>>[]) : [];
export interface CreateModelsOptions {
credentials?: CredentialStore;
authContext?: AuthContext;
}
class ModelsImpl implements MutableModels {
private providers = new Map<string, Provider>();
private credentials: CredentialStore;
private authContext: AuthContext;
constructor(options?: CreateModelsOptions) {
this.credentials = options?.credentials ?? new InMemoryCredentialStore();
this.authContext = options?.authContext ?? defaultAuthContext();
}
setProvider(provider: Provider): void {
this.providers.set(provider.id, provider);
}
deleteProvider(id: string): void {
this.providers.delete(id);
}
clearProviders(): void {
this.providers.clear();
}
getProviders(): readonly Provider[] {
return Array.from(this.providers.values());
}
getProvider(id: string): Provider | undefined {
return this.providers.get(id);
}
getModels(provider?: string): readonly Model<Api>[] {
if (provider !== undefined) {
const entry = this.providers.get(provider);
if (!entry) return [];
try {
return entry.getModels();
} catch {
return [];
}
}
const models: Model<Api>[] = [];
for (const entry of this.providers.values()) {
try {
models.push(...entry.getModels());
} catch {
// Best-effort: ill-behaved providers yield no models.
}
}
return models;
}
getModel(provider: string, id: string): Model<Api> | undefined {
return this.getModels(provider).find((model) => model.id === id);
}
async refresh(provider?: string): Promise<void> {
if (provider !== undefined) {
const entry = this.providers.get(provider);
if (!entry?.refreshModels) return;
try {
await entry.refreshModels();
} catch (error) {
if (error instanceof ModelsError) throw error;
throw new ModelsError("model_source", `Model refresh failed for ${provider}`, { cause: error });
}
return;
}
// Cannot reject: the async mapper turns even sync throws from ill-behaved
// providers into rejections, and allSettled captures all of them.
await Promise.allSettled(Array.from(this.providers.values(), async (entry) => entry.refreshModels?.()));
}
async getAuth(model: Model<Api>): Promise<AuthResult | undefined> {
const provider = this.providers.get(model.provider);
if (!provider) return undefined;
return resolveProviderAuth(provider, model, this.credentials, this.authContext);
}
private requireProvider(model: Model<Api>): Provider {
const provider = this.providers.get(model.provider);
if (!provider) {
throw new ModelsError("provider", `Unknown provider: ${model.provider}`);
}
return provider;
}
private async applyAuth<TOptions extends StreamOptions>(
model: Model<Api>,
options: TOptions | undefined,
): Promise<{ requestModel: Model<Api>; requestOptions: TOptions | undefined }> {
const resolution = await resolveProviderAuth(
this.requireProvider(model),
model,
this.credentials,
this.authContext,
{
apiKey: options?.apiKey,
env: options?.env,
},
);
const auth = resolution?.auth;
if (!auth) return { requestModel: model, requestOptions: options };
const requestModel = auth.baseUrl ? { ...model, baseUrl: auth.baseUrl } : model;
// Explicit request options win per-field; headers/env merge per key.
const apiKey = options?.apiKey ?? auth.apiKey;
const headers = auth.headers || options?.headers ? { ...auth.headers, ...options?.headers } : undefined;
const env = resolution.env || options?.env ? { ...(resolution.env ?? {}), ...(options?.env ?? {}) } : undefined;
const requestOptions = { ...options, apiKey, headers, env } as TOptions;
return { requestModel, requestOptions };
}
stream<TApi extends Api>(
model: Model<TApi>,
context: Context,
options?: ApiStreamOptions<TApi>,
): AssistantMessageEventStream {
return lazyStream(model, async () => {
const provider = this.requireProvider(model);
const { requestModel, requestOptions } = await this.applyAuth(model, options as StreamOptions | undefined);
return provider.stream(requestModel as Model<TApi>, context, requestOptions as ApiStreamOptions<TApi>);
});
}
async complete<TApi extends Api>(
model: Model<TApi>,
context: Context,
options?: ApiStreamOptions<TApi>,
): Promise<AssistantMessage> {
return this.stream(model, context, options).result();
}
streamSimple(model: Model<Api>, context: Context, options?: SimpleStreamOptions): AssistantMessageEventStream {
return lazyStream(model, async () => {
const provider = this.requireProvider(model);
const { requestModel, requestOptions } = await this.applyAuth(model, options);
return provider.streamSimple(requestModel, context, requestOptions);
});
}
async completeSimple(model: Model<Api>, context: Context, options?: SimpleStreamOptions): Promise<AssistantMessage> {
return this.streamSimple(model, context, options).result();
}
}
export function createModels(options?: CreateModelsOptions): MutableModels {
return new ModelsImpl(options);
}
export interface CreateProviderOptions<TApi extends Api = Api> {
id: string;
/** Display name. Default: `id`. */
name?: string;
baseUrl?: string;
headers?: ProviderHeaders;
/** Required — every provider has auth semantics, even ambient/keyless ones. */
auth: ProviderAuth;
/** Initial model list (empty for purely dynamic providers). */
models: readonly Model<TApi>[];
/**
* Dynamic providers: fetch the current list. Stored on success; concurrent
* calls share one in-flight fetch. May reject: the stored list then stays
* at its last-known state, the rejection propagates to the caller of
* `refreshModels()` (wrapped as ModelsError "model_source" by
* `Models.refresh(provider)`), and a later call retries.
*/
refreshModels?: () => Promise<readonly Model<TApi>[]>;
/** Single implementation, or map keyed by `model.api` for mixed-API providers. */
api: ProviderStreams | Partial<Record<TApi, ProviderStreams>>;
}
/**
* Builds a provider from parts. Built-in provider factories and models.json
* custom providers both go through this. A single `api` streams all models;
* an `api` map dispatches on `model.api`, and a model whose api has no entry
* produces a stream error.
*/
export function createProvider<TApi extends Api = Api>(input: CreateProviderOptions<TApi>): Provider<TApi> {
let models = input.models;
let inflightRefresh: Promise<void> | undefined;
const refreshModels = input.refreshModels;
const single =
typeof (input.api as ProviderStreams).stream === "function" ? (input.api as ProviderStreams) : undefined;
const byApi = single ? undefined : (input.api as Partial<Record<string, ProviderStreams>>);
const apiFor = (model: Model<Api>): ProviderStreams | undefined => single ?? byApi?.[model.api];
const dispatch = (
model: Model<Api>,
run: (streams: ProviderStreams) => AssistantMessageEventStream,
): AssistantMessageEventStream => {
const streams = apiFor(model);
if (!streams) {
return lazyStream(model, async () => {
throw new ModelsError("stream", `Provider ${input.id} has no API implementation for "${model.api}"`);
});
}
return run(streams);
};
return {
id: input.id,
name: input.name ?? input.id,
baseUrl: input.baseUrl,
headers: input.headers,
auth: input.auth,
getModels: () => models,
refreshModels: refreshModels
? () => {
inflightRefresh ??= (async () => {
try {
models = await refreshModels();
} finally {
inflightRefresh = undefined;
}
})();
return inflightRefresh;
}
: undefined,
stream: (model, context, options) => dispatch(model, (streams) => streams.stream(model, context, options)),
streamSimple: (model, context, options) =>
dispatch(model, (streams) => streams.streamSimple(model, context, options)),
};
}
/**
* Runtime-checked narrowing for dynamically looked-up models:
*
* ```ts
* const model = models.getModel("anthropic", "claude-opus-4-7");
* if (model && hasApi(model, "anthropic-messages")) {
* // model: Model<"anthropic-messages">, stream options fully typed
* }
* ```
*/
export function hasApi<TApi extends Api>(model: Model<Api>, api: TApi): model is Model<TApi> {
return model.api === api;
}
export function calculateCost<TApi extends Api>(model: Model<TApi>, usage: Usage): Usage["cost"] {
packages/ai/src/providers/all.ts
+131
View File
@@ -0,0 +1,131 @@
import { createImagesModels, type ImagesProvider, type MutableImagesModels } from "../images-models.ts";
import { MODELS } from "../models.generated.ts";
import { type CreateModelsOptions, createModels, type MutableModels, type Provider } from "../models.ts";
import type { Api, KnownProvider, Model } from "../types.ts";
import { amazonBedrockProvider } from "./amazon-bedrock.ts";
import { antLingProvider } from "./ant-ling.ts";
import { anthropicProvider } from "./anthropic.ts";
import { azureOpenAIResponsesProvider } from "./azure-openai-responses.ts";
import { cerebrasProvider } from "./cerebras.ts";
import { cloudflareAIGatewayProvider } from "./cloudflare-ai-gateway.ts";
import { cloudflareWorkersAIProvider } from "./cloudflare-workers-ai.ts";
import { deepseekProvider } from "./deepseek.ts";
import { fireworksProvider } from "./fireworks.ts";
import { githubCopilotProvider } from "./github-copilot.ts";
import { googleProvider } from "./google.ts";
import { googleVertexProvider } from "./google-vertex.ts";
import { groqProvider } from "./groq.ts";
import { huggingfaceProvider } from "./huggingface.ts";
import { kimiCodingProvider } from "./kimi-coding.ts";
import { minimaxProvider } from "./minimax.ts";
import { minimaxCnProvider } from "./minimax-cn.ts";
import { mistralProvider } from "./mistral.ts";
import { moonshotaiProvider } from "./moonshotai.ts";
import { moonshotaiCnProvider } from "./moonshotai-cn.ts";
import { nvidiaProvider } from "./nvidia.ts";
import { openaiProvider } from "./openai.ts";
import { openaiCodexProvider } from "./openai-codex.ts";
import { opencodeProvider } from "./opencode.ts";
import { opencodeGoProvider } from "./opencode-go.ts";
import { openrouterProvider } from "./openrouter.ts";
import { openrouterImagesProvider } from "./openrouter-images.ts";
import { togetherProvider } from "./together.ts";
import { vercelAIGatewayProvider } from "./vercel-ai-gateway.ts";
import { xaiProvider } from "./xai.ts";
import { xiaomiProvider } from "./xiaomi.ts";
import { xiaomiTokenPlanAmsProvider } from "./xiaomi-token-plan-ams.ts";
import { xiaomiTokenPlanCnProvider } from "./xiaomi-token-plan-cn.ts";
import { xiaomiTokenPlanSgpProvider } from "./xiaomi-token-plan-sgp.ts";
import { zaiProvider } from "./zai.ts";
import { zaiCodingCnProvider } from "./zai-coding-cn.ts";
type BuiltinModelApi<
TProvider extends KnownProvider,
TModelId extends keyof (typeof MODELS)[TProvider],
> = (typeof MODELS)[TProvider][TModelId] extends { api: infer TApi } ? (TApi extends Api ? TApi : never) : never;
/** Typed read of the generated built-in catalog. */
export function getBuiltinModel<TProvider extends KnownProvider, TModelId extends keyof (typeof MODELS)[TProvider]>(
provider: TProvider,
modelId: TModelId,
): Model<BuiltinModelApi<TProvider, TModelId>> {
const models = MODELS[provider] as Record<string, Model<Api>> | undefined;
return models?.[modelId as string] as Model<BuiltinModelApi<TProvider, TModelId>>;
}
export function getBuiltinProviders(): KnownProvider[] {
return Object.keys(MODELS) as KnownProvider[];
}
export function getBuiltinModels<TProvider extends KnownProvider>(
provider: TProvider,
): Model<BuiltinModelApi<TProvider, keyof (typeof MODELS)[TProvider]>>[] {
const models = MODELS[provider] as Record<string, Model<Api>> | undefined;
return models
? (Object.values(models) as Model<BuiltinModelApi<TProvider, keyof (typeof MODELS)[TProvider]>>[])
: [];
}
/** All built-in providers, freshly constructed. */
export function builtinProviders(): Provider[] {
return [
amazonBedrockProvider(),
antLingProvider(),
anthropicProvider(),
azureOpenAIResponsesProvider(),
cerebrasProvider(),
cloudflareAIGatewayProvider(),
cloudflareWorkersAIProvider(),
deepseekProvider(),
fireworksProvider(),
githubCopilotProvider(),
googleProvider(),
googleVertexProvider(),
groqProvider(),
huggingfaceProvider(),
kimiCodingProvider(),
minimaxProvider(),
minimaxCnProvider(),
mistralProvider(),
moonshotaiProvider(),
moonshotaiCnProvider(),
nvidiaProvider(),
openaiProvider(),
openaiCodexProvider(),
opencodeProvider(),
opencodeGoProvider(),
openrouterProvider(),
togetherProvider(),
vercelAIGatewayProvider(),
xaiProvider(),
xiaomiProvider(),
xiaomiTokenPlanAmsProvider(),
xiaomiTokenPlanCnProvider(),
xiaomiTokenPlanSgpProvider(),
zaiProvider(),
zaiCodingCnProvider(),
];
}
/** A `Models` collection with every built-in provider registered. */
export function builtinModels(options?: CreateModelsOptions): MutableModels {
const models = createModels(options);
for (const provider of builtinProviders()) {
models.setProvider(provider);
}
return models;
}
/** All built-in image-generation providers, freshly constructed. */
export function builtinImagesProviders(): ImagesProvider[] {
return [openrouterImagesProvider()];
}
/** An `ImagesModels` collection with every built-in image-generation provider registered. */
export function builtinImagesModels(options?: CreateModelsOptions): MutableImagesModels {
const models = createImagesModels(options);
for (const provider of builtinImagesProviders()) {
models.setProvider(provider);
}
return models;
}
packages/ai/src/providers/amazon-bedrock.models.ts
+1677
View File
File diff suppressed because it is too large Load Diff
packages/ai/src/providers/amazon-bedrock.ts
+27 -1053
View File
File diff suppressed because it is too large Load Diff
packages/ai/src/providers/ant-ling.models.ts
+62
View File
@@ -0,0 +1,62 @@
// This file is auto-generated by scripts/generate-models.ts
// Do not edit manually - run 'npm run generate-models' to update
import type { Model } from "../types.ts";
export const ANT_LING_MODELS = {
"Ling-2.6-1T": {
id: "Ling-2.6-1T",
name: "Ling 2.6 1T",
api: "openai-completions",
provider: "ant-ling",
baseUrl: "https://api.ant-ling.com/v1",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsReasoningEffort":false,"maxTokensField":"max_tokens","thinkingFormat":"ant-ling","supportsLongCacheRetention":false},
reasoning: false,
input: ["text"],
cost: {
input: 0.06,
output: 0.25,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 262144,
maxTokens: 65536,
} satisfies Model<"openai-completions">,
"Ling-2.6-flash": {
id: "Ling-2.6-flash",
name: "Ling 2.6 Flash",
api: "openai-completions",
provider: "ant-ling",
baseUrl: "https://api.ant-ling.com/v1",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsReasoningEffort":false,"maxTokensField":"max_tokens","thinkingFormat":"ant-ling","supportsLongCacheRetention":false},
reasoning: false,
input: ["text"],
cost: {
input: 0.01,
output: 0.02,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 262144,
maxTokens: 65536,
} satisfies Model<"openai-completions">,
"Ring-2.6-1T": {
id: "Ring-2.6-1T",
name: "Ring 2.6 1T",
api: "openai-completions",
provider: "ant-ling",
baseUrl: "https://api.ant-ling.com/v1",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsReasoningEffort":false,"maxTokensField":"max_tokens","thinkingFormat":"ant-ling","supportsLongCacheRetention":false},
reasoning: true,
thinkingLevelMap: {"off":null,"minimal":null,"low":null,"medium":null,"high":"high","xhigh":"xhigh"},
input: ["text"],
cost: {
input: 0.06,
output: 0.25,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 262144,
maxTokens: 65536,
} satisfies Model<"openai-completions">,
} as const;
packages/ai/src/providers/ant-ling.ts
+15
View File
@@ -0,0 +1,15 @@
import { openAICompletionsApi } from "../api/openai-completions.lazy.ts";
import { envApiKeyAuth } from "../auth/helpers.ts";
import { createProvider, type Provider } from "../models.ts";
import { ANT_LING_MODELS } from "./ant-ling.models.ts";
export function antLingProvider(): Provider<"openai-completions"> {
return createProvider({
id: "ant-ling",
name: "Ant Ling",
baseUrl: "https://api.ant-ling.com/v1",
auth: { apiKey: envApiKeyAuth("Ant Ling API key", ["ANT_LING_API_KEY"]) },
models: Object.values(ANT_LING_MODELS),
api: openAICompletionsApi(),
});
}
packages/ai/src/providers/anthropic.models.ts
+441
View File
@@ -0,0 +1,441 @@
// This file is auto-generated by scripts/generate-models.ts
// Do not edit manually - run 'npm run generate-models' to update
import type { Model } from "../types.ts";
export const ANTHROPIC_MODELS = {
"claude-3-5-haiku-20241022": {
id: "claude-3-5-haiku-20241022",
name: "Claude Haiku 3.5",
api: "anthropic-messages",
provider: "anthropic",
baseUrl: "https://api.anthropic.com",
reasoning: false,
input: ["text", "image"],
cost: {
input: 0.8,
output: 4,
cacheRead: 0.08,
cacheWrite: 1,
},
contextWindow: 200000,
maxTokens: 8192,
} satisfies Model<"anthropic-messages">,
"claude-3-5-haiku-latest": {
id: "claude-3-5-haiku-latest",
name: "Claude Haiku 3.5 (latest)",
api: "anthropic-messages",
provider: "anthropic",
baseUrl: "https://api.anthropic.com",
reasoning: false,
input: ["text", "image"],
cost: {
input: 0.8,
output: 4,
cacheRead: 0.08,
cacheWrite: 1,
},
contextWindow: 200000,
maxTokens: 8192,
} satisfies Model<"anthropic-messages">,
"claude-3-5-sonnet-20240620": {
id: "claude-3-5-sonnet-20240620",
name: "Claude Sonnet 3.5",
api: "anthropic-messages",
provider: "anthropic",
baseUrl: "https://api.anthropic.com",
reasoning: false,
input: ["text", "image"],
cost: {
input: 3,
output: 15,
cacheRead: 0.3,
cacheWrite: 3.75,
},
contextWindow: 200000,
maxTokens: 8192,
} satisfies Model<"anthropic-messages">,
"claude-3-5-sonnet-20241022": {
id: "claude-3-5-sonnet-20241022",
name: "Claude Sonnet 3.5 v2",
api: "anthropic-messages",
provider: "anthropic",
baseUrl: "https://api.anthropic.com",
reasoning: false,
input: ["text", "image"],
cost: {
input: 3,
output: 15,
cacheRead: 0.3,
cacheWrite: 3.75,
},
contextWindow: 200000,
maxTokens: 8192,
} satisfies Model<"anthropic-messages">,
"claude-3-7-sonnet-20250219": {
id: "claude-3-7-sonnet-20250219",
name: "Claude Sonnet 3.7",
api: "anthropic-messages",
provider: "anthropic",
baseUrl: "https://api.anthropic.com",
reasoning: true,
input: ["text", "image"],
cost: {
input: 3,
output: 15,
cacheRead: 0.3,
cacheWrite: 3.75,
},
contextWindow: 200000,
maxTokens: 64000,
} satisfies Model<"anthropic-messages">,
"claude-3-haiku-20240307": {
id: "claude-3-haiku-20240307",
name: "Claude Haiku 3",
api: "anthropic-messages",
provider: "anthropic",
baseUrl: "https://api.anthropic.com",
reasoning: false,
input: ["text", "image"],
cost: {
input: 0.25,
output: 1.25,
cacheRead: 0.03,
cacheWrite: 0.3,
},
contextWindow: 200000,
maxTokens: 4096,
} satisfies Model<"anthropic-messages">,
"claude-3-opus-20240229": {
id: "claude-3-opus-20240229",
name: "Claude Opus 3",
api: "anthropic-messages",
provider: "anthropic",
baseUrl: "https://api.anthropic.com",
reasoning: false,
input: ["text", "image"],
cost: {
input: 15,
output: 75,
cacheRead: 1.5,
cacheWrite: 18.75,
},
contextWindow: 200000,
maxTokens: 4096,
} satisfies Model<"anthropic-messages">,
"claude-3-sonnet-20240229": {
id: "claude-3-sonnet-20240229",
name: "Claude Sonnet 3",
api: "anthropic-messages",
provider: "anthropic",
baseUrl: "https://api.anthropic.com",
reasoning: false,
input: ["text", "image"],
cost: {
input: 3,
output: 15,
cacheRead: 0.3,
cacheWrite: 0.3,
},
contextWindow: 200000,
maxTokens: 4096,
} satisfies Model<"anthropic-messages">,
"claude-fable-5": {
id: "claude-fable-5",
name: "Claude Fable 5",
api: "anthropic-messages",
provider: "anthropic",
baseUrl: "https://api.anthropic.com",
compat: {"forceAdaptiveThinking":true},
reasoning: true,
thinkingLevelMap: {"off":null,"xhigh":"xhigh"},
input: ["text", "image"],
cost: {
input: 10,
output: 50,
cacheRead: 1,
cacheWrite: 12.5,
},
contextWindow: 1000000,
maxTokens: 128000,
} satisfies Model<"anthropic-messages">,
"claude-haiku-4-5": {
id: "claude-haiku-4-5",
name: "Claude Haiku 4.5 (latest)",
api: "anthropic-messages",
provider: "anthropic",
baseUrl: "https://api.anthropic.com",
reasoning: true,
input: ["text", "image"],
cost: {
input: 1,
output: 5,
cacheRead: 0.1,
cacheWrite: 1.25,
},
contextWindow: 200000,
maxTokens: 64000,
} satisfies Model<"anthropic-messages">,
"claude-haiku-4-5-20251001": {
id: "claude-haiku-4-5-20251001",
name: "Claude Haiku 4.5",
api: "anthropic-messages",
provider: "anthropic",
baseUrl: "https://api.anthropic.com",
reasoning: true,
input: ["text", "image"],
cost: {
input: 1,
output: 5,
cacheRead: 0.1,
cacheWrite: 1.25,
},
contextWindow: 200000,
maxTokens: 64000,
} satisfies Model<"anthropic-messages">,
"claude-opus-4-0": {
id: "claude-opus-4-0",
name: "Claude Opus 4 (latest)",
api: "anthropic-messages",
provider: "anthropic",
baseUrl: "https://api.anthropic.com",
reasoning: true,
input: ["text", "image"],
cost: {
input: 15,
output: 75,
cacheRead: 1.5,
cacheWrite: 18.75,
},
contextWindow: 200000,
maxTokens: 32000,
} satisfies Model<"anthropic-messages">,
"claude-opus-4-1": {
id: "claude-opus-4-1",
name: "Claude Opus 4.1 (latest)",
api: "anthropic-messages",
provider: "anthropic",
baseUrl: "https://api.anthropic.com",
reasoning: true,
input: ["text", "image"],
cost: {
input: 15,
output: 75,
cacheRead: 1.5,
cacheWrite: 18.75,
},
contextWindow: 200000,
maxTokens: 32000,
} satisfies Model<"anthropic-messages">,
"claude-opus-4-1-20250805": {
id: "claude-opus-4-1-20250805",
name: "Claude Opus 4.1",
api: "anthropic-messages",
provider: "anthropic",
baseUrl: "https://api.anthropic.com",
reasoning: true,
input: ["text", "image"],
cost: {
input: 15,
output: 75,
cacheRead: 1.5,
cacheWrite: 18.75,
},
contextWindow: 200000,
maxTokens: 32000,
} satisfies Model<"anthropic-messages">,
"claude-opus-4-20250514": {
id: "claude-opus-4-20250514",
name: "Claude Opus 4",
api: "anthropic-messages",
provider: "anthropic",
baseUrl: "https://api.anthropic.com",
reasoning: true,
input: ["text", "image"],
cost: {
input: 15,
output: 75,
cacheRead: 1.5,
cacheWrite: 18.75,
},
contextWindow: 200000,
maxTokens: 32000,
} satisfies Model<"anthropic-messages">,
"claude-opus-4-5": {
id: "claude-opus-4-5",
name: "Claude Opus 4.5 (latest)",
api: "anthropic-messages",
provider: "anthropic",
baseUrl: "https://api.anthropic.com",
reasoning: true,
input: ["text", "image"],
cost: {
input: 5,
output: 25,
cacheRead: 0.5,
cacheWrite: 6.25,
},
contextWindow: 200000,
maxTokens: 64000,
} satisfies Model<"anthropic-messages">,
"claude-opus-4-5-20251101": {
id: "claude-opus-4-5-20251101",
name: "Claude Opus 4.5",
api: "anthropic-messages",
provider: "anthropic",
baseUrl: "https://api.anthropic.com",
reasoning: true,
input: ["text", "image"],
cost: {
input: 5,
output: 25,
cacheRead: 0.5,
cacheWrite: 6.25,
},
contextWindow: 200000,
maxTokens: 64000,
} satisfies Model<"anthropic-messages">,
"claude-opus-4-6": {
id: "claude-opus-4-6",
name: "Claude Opus 4.6",
api: "anthropic-messages",
provider: "anthropic",
baseUrl: "https://api.anthropic.com",
compat: {"forceAdaptiveThinking":true},
reasoning: true,
thinkingLevelMap: {"xhigh":"max"},
input: ["text", "image"],
cost: {
input: 5,
output: 25,
cacheRead: 0.5,
cacheWrite: 6.25,
},
contextWindow: 1000000,
maxTokens: 128000,
} satisfies Model<"anthropic-messages">,
"claude-opus-4-7": {
id: "claude-opus-4-7",
name: "Claude Opus 4.7",
api: "anthropic-messages",
provider: "anthropic",
baseUrl: "https://api.anthropic.com",
compat: {"forceAdaptiveThinking":true,"supportsTemperature":false},
reasoning: true,
thinkingLevelMap: {"xhigh":"xhigh"},
input: ["text", "image"],
cost: {
input: 5,
output: 25,
cacheRead: 0.5,
cacheWrite: 6.25,
},
contextWindow: 1000000,
maxTokens: 128000,
} satisfies Model<"anthropic-messages">,
"claude-opus-4-8": {
id: "claude-opus-4-8",
name: "Claude Opus 4.8",
api: "anthropic-messages",
provider: "anthropic",
baseUrl: "https://api.anthropic.com",
compat: {"forceAdaptiveThinking":true,"supportsTemperature":false},
reasoning: true,
thinkingLevelMap: {"xhigh":"xhigh"},
input: ["text", "image"],
cost: {
input: 5,
output: 25,
cacheRead: 0.5,
cacheWrite: 6.25,
},
contextWindow: 1000000,
maxTokens: 128000,
} satisfies Model<"anthropic-messages">,
"claude-sonnet-4-0": {
id: "claude-sonnet-4-0",
name: "Claude Sonnet 4 (latest)",
api: "anthropic-messages",
provider: "anthropic",
baseUrl: "https://api.anthropic.com",
reasoning: true,
input: ["text", "image"],
cost: {
input: 3,
output: 15,
cacheRead: 0.3,
cacheWrite: 3.75,
},
contextWindow: 200000,
maxTokens: 64000,
} satisfies Model<"anthropic-messages">,
"claude-sonnet-4-20250514": {
id: "claude-sonnet-4-20250514",
name: "Claude Sonnet 4",
api: "anthropic-messages",
provider: "anthropic",
baseUrl: "https://api.anthropic.com",
reasoning: true,
input: ["text", "image"],
cost: {
input: 3,
output: 15,
cacheRead: 0.3,
cacheWrite: 3.75,
},
contextWindow: 200000,
maxTokens: 64000,
} satisfies Model<"anthropic-messages">,
"claude-sonnet-4-5": {
id: "claude-sonnet-4-5",
name: "Claude Sonnet 4.5 (latest)",
api: "anthropic-messages",
provider: "anthropic",
baseUrl: "https://api.anthropic.com",
reasoning: true,
input: ["text", "image"],
cost: {
input: 3,
output: 15,
cacheRead: 0.3,
cacheWrite: 3.75,
},
contextWindow: 200000,
maxTokens: 64000,
} satisfies Model<"anthropic-messages">,
"claude-sonnet-4-5-20250929": {
id: "claude-sonnet-4-5-20250929",
name: "Claude Sonnet 4.5",
api: "anthropic-messages",
provider: "anthropic",
baseUrl: "https://api.anthropic.com",
reasoning: true,
input: ["text", "image"],
cost: {
input: 3,
output: 15,
cacheRead: 0.3,
cacheWrite: 3.75,
},
contextWindow: 200000,
maxTokens: 64000,
} satisfies Model<"anthropic-messages">,
"claude-sonnet-4-6": {
id: "claude-sonnet-4-6",
name: "Claude Sonnet 4.6",
api: "anthropic-messages",
provider: "anthropic",
baseUrl: "https://api.anthropic.com",
compat: {"forceAdaptiveThinking":true},
reasoning: true,
input: ["text", "image"],
cost: {
input: 3,
output: 15,
cacheRead: 0.3,
cacheWrite: 3.75,
},
contextWindow: 1000000,
maxTokens: 64000,
} satisfies Model<"anthropic-messages">,
} as const;
packages/ai/src/providers/anthropic.ts
+16 -1238
View File
File diff suppressed because it is too large Load Diff
packages/ai/src/providers/azure-openai-responses.models.ts
+745
View File
@@ -0,0 +1,745 @@
// This file is auto-generated by scripts/generate-models.ts
// Do not edit manually - run 'npm run generate-models' to update
import type { Model } from "../types.ts";
export const AZURE_OPENAI_RESPONSES_MODELS = {
"gpt-4": {
id: "gpt-4",
name: "GPT-4",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: false,
input: ["text"],
cost: {
input: 30,
output: 60,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 8192,
maxTokens: 8192,
} satisfies Model<"azure-openai-responses">,
"gpt-4-turbo": {
id: "gpt-4-turbo",
name: "GPT-4 Turbo",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: false,
input: ["text", "image"],
cost: {
input: 10,
output: 30,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 128000,
maxTokens: 4096,
} satisfies Model<"azure-openai-responses">,
"gpt-4.1": {
id: "gpt-4.1",
name: "GPT-4.1",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: false,
input: ["text", "image"],
cost: {
input: 2,
output: 8,
cacheRead: 0.5,
cacheWrite: 0,
},
contextWindow: 1047576,
maxTokens: 32768,
} satisfies Model<"azure-openai-responses">,
"gpt-4.1-mini": {
id: "gpt-4.1-mini",
name: "GPT-4.1 mini",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: false,
input: ["text", "image"],
cost: {
input: 0.4,
output: 1.6,
cacheRead: 0.1,
cacheWrite: 0,
},
contextWindow: 1047576,
maxTokens: 32768,
} satisfies Model<"azure-openai-responses">,
"gpt-4.1-nano": {
id: "gpt-4.1-nano",
name: "GPT-4.1 nano",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: false,
input: ["text", "image"],
cost: {
input: 0.1,
output: 0.4,
cacheRead: 0.025,
cacheWrite: 0,
},
contextWindow: 1047576,
maxTokens: 32768,
} satisfies Model<"azure-openai-responses">,
"gpt-4o": {
id: "gpt-4o",
name: "GPT-4o",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: false,
input: ["text", "image"],
cost: {
input: 2.5,
output: 10,
cacheRead: 1.25,
cacheWrite: 0,
},
contextWindow: 128000,
maxTokens: 16384,
} satisfies Model<"azure-openai-responses">,
"gpt-4o-2024-05-13": {
id: "gpt-4o-2024-05-13",
name: "GPT-4o (2024-05-13)",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: false,
input: ["text", "image"],
cost: {
input: 5,
output: 15,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 128000,
maxTokens: 4096,
} satisfies Model<"azure-openai-responses">,
"gpt-4o-2024-08-06": {
id: "gpt-4o-2024-08-06",
name: "GPT-4o (2024-08-06)",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: false,
input: ["text", "image"],
cost: {
input: 2.5,
output: 10,
cacheRead: 1.25,
cacheWrite: 0,
},
contextWindow: 128000,
maxTokens: 16384,
} satisfies Model<"azure-openai-responses">,
"gpt-4o-2024-11-20": {
id: "gpt-4o-2024-11-20",
name: "GPT-4o (2024-11-20)",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: false,
input: ["text", "image"],
cost: {
input: 2.5,
output: 10,
cacheRead: 1.25,
cacheWrite: 0,
},
contextWindow: 128000,
maxTokens: 16384,
} satisfies Model<"azure-openai-responses">,
"gpt-4o-mini": {
id: "gpt-4o-mini",
name: "GPT-4o mini",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: false,
input: ["text", "image"],
cost: {
input: 0.15,
output: 0.6,
cacheRead: 0.075,
cacheWrite: 0,
},
contextWindow: 128000,
maxTokens: 16384,
} satisfies Model<"azure-openai-responses">,
"gpt-5": {
id: "gpt-5",
name: "GPT-5",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
thinkingLevelMap: {"off":null},
input: ["text", "image"],
cost: {
input: 1.25,
output: 10,
cacheRead: 0.125,
cacheWrite: 0,
},
contextWindow: 400000,
maxTokens: 128000,
} satisfies Model<"azure-openai-responses">,
"gpt-5-chat-latest": {
id: "gpt-5-chat-latest",
name: "GPT-5 Chat Latest",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: false,
thinkingLevelMap: {"off":null},
input: ["text", "image"],
cost: {
input: 1.25,
output: 10,
cacheRead: 0.125,
cacheWrite: 0,
},
contextWindow: 128000,
maxTokens: 16384,
} satisfies Model<"azure-openai-responses">,
"gpt-5-codex": {
id: "gpt-5-codex",
name: "GPT-5-Codex",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
thinkingLevelMap: {"off":null},
input: ["text", "image"],
cost: {
input: 1.25,
output: 10,
cacheRead: 0.125,
cacheWrite: 0,
},
contextWindow: 400000,
maxTokens: 128000,
} satisfies Model<"azure-openai-responses">,
"gpt-5-mini": {
id: "gpt-5-mini",
name: "GPT-5 Mini",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
thinkingLevelMap: {"off":null},
input: ["text", "image"],
cost: {
input: 0.25,
output: 2,
cacheRead: 0.025,
cacheWrite: 0,
},
contextWindow: 400000,
maxTokens: 128000,
} satisfies Model<"azure-openai-responses">,
"gpt-5-nano": {
id: "gpt-5-nano",
name: "GPT-5 Nano",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
thinkingLevelMap: {"off":null},
input: ["text", "image"],
cost: {
input: 0.05,
output: 0.4,
cacheRead: 0.005,
cacheWrite: 0,
},
contextWindow: 400000,
maxTokens: 128000,
} satisfies Model<"azure-openai-responses">,
"gpt-5-pro": {
id: "gpt-5-pro",
name: "GPT-5 Pro",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
thinkingLevelMap: {"off":null},
input: ["text", "image"],
cost: {
input: 15,
output: 120,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 400000,
maxTokens: 128000,
} satisfies Model<"azure-openai-responses">,
"gpt-5.1": {
id: "gpt-5.1",
name: "GPT-5.1",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
thinkingLevelMap: {"off":null},
input: ["text", "image"],
cost: {
input: 1.25,
output: 10,
cacheRead: 0.125,
cacheWrite: 0,
},
contextWindow: 400000,
maxTokens: 128000,
} satisfies Model<"azure-openai-responses">,
"gpt-5.1-chat-latest": {
id: "gpt-5.1-chat-latest",
name: "GPT-5.1 Chat",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
thinkingLevelMap: {"off":null},
input: ["text", "image"],
cost: {
input: 1.25,
output: 10,
cacheRead: 0.125,
cacheWrite: 0,
},
contextWindow: 128000,
maxTokens: 16384,
} satisfies Model<"azure-openai-responses">,
"gpt-5.1-codex": {
id: "gpt-5.1-codex",
name: "GPT-5.1 Codex",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
thinkingLevelMap: {"off":null},
input: ["text", "image"],
cost: {
input: 1.25,
output: 10,
cacheRead: 0.125,
cacheWrite: 0,
},
contextWindow: 400000,
maxTokens: 128000,
} satisfies Model<"azure-openai-responses">,
"gpt-5.1-codex-max": {
id: "gpt-5.1-codex-max",
name: "GPT-5.1 Codex Max",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
thinkingLevelMap: {"off":null},
input: ["text", "image"],
cost: {
input: 1.25,
output: 10,
cacheRead: 0.125,
cacheWrite: 0,
},
contextWindow: 400000,
maxTokens: 128000,
} satisfies Model<"azure-openai-responses">,
"gpt-5.1-codex-mini": {
id: "gpt-5.1-codex-mini",
name: "GPT-5.1 Codex mini",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
thinkingLevelMap: {"off":null},
input: ["text", "image"],
cost: {
input: 0.25,
output: 2,
cacheRead: 0.025,
cacheWrite: 0,
},
contextWindow: 400000,
maxTokens: 128000,
} satisfies Model<"azure-openai-responses">,
"gpt-5.2": {
id: "gpt-5.2",
name: "GPT-5.2",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
thinkingLevelMap: {"off":null,"xhigh":"xhigh"},
input: ["text", "image"],
cost: {
input: 1.75,
output: 14,
cacheRead: 0.175,
cacheWrite: 0,
},
contextWindow: 400000,
maxTokens: 128000,
} satisfies Model<"azure-openai-responses">,
"gpt-5.2-chat-latest": {
id: "gpt-5.2-chat-latest",
name: "GPT-5.2 Chat",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
thinkingLevelMap: {"off":null,"xhigh":"xhigh"},
input: ["text", "image"],
cost: {
input: 1.75,
output: 14,
cacheRead: 0.175,
cacheWrite: 0,
},
contextWindow: 128000,
maxTokens: 16384,
} satisfies Model<"azure-openai-responses">,
"gpt-5.2-codex": {
id: "gpt-5.2-codex",
name: "GPT-5.2 Codex",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
thinkingLevelMap: {"off":null,"xhigh":"xhigh"},
input: ["text", "image"],
cost: {
input: 1.75,
output: 14,
cacheRead: 0.175,
cacheWrite: 0,
},
contextWindow: 400000,
maxTokens: 128000,
} satisfies Model<"azure-openai-responses">,
"gpt-5.2-pro": {
id: "gpt-5.2-pro",
name: "GPT-5.2 Pro",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
thinkingLevelMap: {"off":null,"xhigh":"xhigh"},
input: ["text", "image"],
cost: {
input: 21,
output: 168,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 400000,
maxTokens: 128000,
} satisfies Model<"azure-openai-responses">,
"gpt-5.3-chat-latest": {
id: "gpt-5.3-chat-latest",
name: "GPT-5.3 Chat (latest)",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: false,
thinkingLevelMap: {"off":null,"xhigh":"xhigh"},
input: ["text", "image"],
cost: {
input: 1.75,
output: 14,
cacheRead: 0.175,
cacheWrite: 0,
},
contextWindow: 128000,
maxTokens: 16384,
} satisfies Model<"azure-openai-responses">,
"gpt-5.3-codex": {
id: "gpt-5.3-codex",
name: "GPT-5.3 Codex",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
thinkingLevelMap: {"off":null,"xhigh":"xhigh"},
input: ["text", "image"],
cost: {
input: 1.75,
output: 14,
cacheRead: 0.175,
cacheWrite: 0,
},
contextWindow: 400000,
maxTokens: 128000,
} satisfies Model<"azure-openai-responses">,
"gpt-5.3-codex-spark": {
id: "gpt-5.3-codex-spark",
name: "GPT-5.3 Codex Spark",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
thinkingLevelMap: {"off":null,"xhigh":"xhigh"},
input: ["text", "image"],
cost: {
input: 1.75,
output: 14,
cacheRead: 0.175,
cacheWrite: 0,
},
contextWindow: 128000,
maxTokens: 32000,
} satisfies Model<"azure-openai-responses">,
"gpt-5.4": {
id: "gpt-5.4",
name: "GPT-5.4",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
thinkingLevelMap: {"off":null,"xhigh":"xhigh"},
input: ["text", "image"],
cost: {
input: 2.5,
output: 15,
cacheRead: 0.25,
cacheWrite: 0,
},
contextWindow: 1050000,
maxTokens: 128000,
} satisfies Model<"azure-openai-responses">,
"gpt-5.4-mini": {
id: "gpt-5.4-mini",
name: "GPT-5.4 mini",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
thinkingLevelMap: {"off":null,"xhigh":"xhigh"},
input: ["text", "image"],
cost: {
input: 0.75,
output: 4.5,
cacheRead: 0.075,
cacheWrite: 0,
},
contextWindow: 400000,
maxTokens: 128000,
} satisfies Model<"azure-openai-responses">,
"gpt-5.4-nano": {
id: "gpt-5.4-nano",
name: "GPT-5.4 nano",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
thinkingLevelMap: {"off":null,"xhigh":"xhigh"},
input: ["text", "image"],
cost: {
input: 0.2,
output: 1.25,
cacheRead: 0.02,
cacheWrite: 0,
},
contextWindow: 400000,
maxTokens: 128000,
} satisfies Model<"azure-openai-responses">,
"gpt-5.4-pro": {
id: "gpt-5.4-pro",
name: "GPT-5.4 Pro",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
thinkingLevelMap: {"off":null,"xhigh":"xhigh"},
input: ["text", "image"],
cost: {
input: 30,
output: 180,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 1050000,
maxTokens: 128000,
} satisfies Model<"azure-openai-responses">,
"gpt-5.5": {
id: "gpt-5.5",
name: "GPT-5.5",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
thinkingLevelMap: {"off":null,"xhigh":"xhigh"},
input: ["text", "image"],
cost: {
input: 5,
output: 30,
cacheRead: 0.5,
cacheWrite: 0,
},
contextWindow: 1050000,
maxTokens: 128000,
} satisfies Model<"azure-openai-responses">,
"gpt-5.5-pro": {
id: "gpt-5.5-pro",
name: "GPT-5.5 Pro",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
thinkingLevelMap: {"off":null,"xhigh":"xhigh","minimal":null,"low":null},
input: ["text", "image"],
cost: {
input: 30,
output: 180,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 1050000,
maxTokens: 128000,
} satisfies Model<"azure-openai-responses">,
"o1": {
id: "o1",
name: "o1",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
input: ["text", "image"],
cost: {
input: 15,
output: 60,
cacheRead: 7.5,
cacheWrite: 0,
},
contextWindow: 200000,
maxTokens: 100000,
} satisfies Model<"azure-openai-responses">,
"o1-pro": {
id: "o1-pro",
name: "o1-pro",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
input: ["text", "image"],
cost: {
input: 150,
output: 600,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 200000,
maxTokens: 100000,
} satisfies Model<"azure-openai-responses">,
"o3": {
id: "o3",
name: "o3",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
input: ["text", "image"],
cost: {
input: 2,
output: 8,
cacheRead: 0.5,
cacheWrite: 0,
},
contextWindow: 200000,
maxTokens: 100000,
} satisfies Model<"azure-openai-responses">,
"o3-deep-research": {
id: "o3-deep-research",
name: "o3-deep-research",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
input: ["text", "image"],
cost: {
input: 10,
output: 40,
cacheRead: 2.5,
cacheWrite: 0,
},
contextWindow: 200000,
maxTokens: 100000,
} satisfies Model<"azure-openai-responses">,
"o3-mini": {
id: "o3-mini",
name: "o3-mini",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
input: ["text"],
cost: {
input: 1.1,
output: 4.4,
cacheRead: 0.55,
cacheWrite: 0,
},
contextWindow: 200000,
maxTokens: 100000,
} satisfies Model<"azure-openai-responses">,
"o3-pro": {
id: "o3-pro",
name: "o3-pro",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
input: ["text", "image"],
cost: {
input: 20,
output: 80,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 200000,
maxTokens: 100000,
} satisfies Model<"azure-openai-responses">,
"o4-mini": {
id: "o4-mini",
name: "o4-mini",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
input: ["text", "image"],
cost: {
input: 1.1,
output: 4.4,
cacheRead: 0.275,
cacheWrite: 0,
},
contextWindow: 200000,
maxTokens: 100000,
} satisfies Model<"azure-openai-responses">,
"o4-mini-deep-research": {
id: "o4-mini-deep-research",
name: "o4-mini-deep-research",
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
reasoning: true,
input: ["text", "image"],
cost: {
input: 2,
output: 8,
cacheRead: 0.5,
cacheWrite: 0,
},
contextWindow: 200000,
maxTokens: 100000,
} satisfies Model<"azure-openai-responses">,
} as const;
packages/ai/src/providers/azure-openai-responses.ts
+11 -296
View File
@@ -1,299 +1,14 @@
import { AzureOpenAI } from "openai";
import type { ResponseCreateParamsStreaming } from "openai/resources/responses/responses.js";
import { clampThinkingLevel } from "../models.ts";
import type {
Api,
AssistantMessage,
Context,
Model,
SimpleStreamOptions,
StreamFunction,
StreamOptions,
} from "../types.ts";
import { AssistantMessageEventStream } from "../utils/event-stream.ts";
import { headersToRecord } from "../utils/headers.ts";
import { getProviderEnvValue } from "../utils/provider-env.ts";
import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts";
import { convertResponsesMessages, convertResponsesTools, processResponsesStream } from "./openai-responses-shared.ts";
import { buildBaseOptions } from "./simple-options.ts";
import { azureOpenAIResponsesApi } from "../api/azure-openai-responses.lazy.ts";
import { envApiKeyAuth } from "../auth/helpers.ts";
import { createProvider, type Provider } from "../models.ts";
import { AZURE_OPENAI_RESPONSES_MODELS } from "./azure-openai-responses.models.ts";
const DEFAULT_AZURE_API_VERSION = "v1";
const AZURE_TOOL_CALL_PROVIDERS = new Set(["openai", "openai-codex", "opencode", "azure-openai-responses"]);
function parseDeploymentNameMap(value: string | undefined): Map<string, string> {
const map = new Map<string, string>();
if (!value) return map;
for (const entry of value.split(",")) {
const trimmed = entry.trim();
if (!trimmed) continue;
const [modelId, deploymentName] = trimmed.split("=", 2);
if (!modelId || !deploymentName) continue;
map.set(modelId.trim(), deploymentName.trim());
}
return map;
}
function resolveDeploymentName(model: Model<"azure-openai-responses">, options?: AzureOpenAIResponsesOptions): string {
if (options?.azureDeploymentName) {
return options.azureDeploymentName;
}
const mappedDeployment = parseDeploymentNameMap(
getProviderEnvValue("AZURE_OPENAI_DEPLOYMENT_NAME_MAP", options?.env),
).get(model.id);
return mappedDeployment || model.id;
}
function formatAzureOpenAIError(error: unknown): string {
if (error instanceof Error) {
const status = (error as Error & { status?: unknown }).status;
const statusCode = typeof status === "number" ? status : undefined;
if (statusCode !== undefined) {
return `Azure OpenAI API error (${statusCode}): ${error.message}`;
}
return error.message;
}
try {
return JSON.stringify(error);
} catch {
return String(error);
}
}
// Azure OpenAI Responses-specific options
export interface AzureOpenAIResponsesOptions extends StreamOptions {
reasoningEffort?: "minimal" | "low" | "medium" | "high" | "xhigh";
reasoningSummary?: "auto" | "detailed" | "concise" | null;
azureApiVersion?: string;
azureResourceName?: string;
azureBaseUrl?: string;
azureDeploymentName?: string;
}
/**
* Generate function for Azure OpenAI Responses API
*/
export const streamAzureOpenAIResponses: StreamFunction<"azure-openai-responses", AzureOpenAIResponsesOptions> = (
model: Model<"azure-openai-responses">,
context: Context,
options?: AzureOpenAIResponsesOptions,
): AssistantMessageEventStream => {
const stream = new AssistantMessageEventStream();
// Start async processing
(async () => {
const deploymentName = resolveDeploymentName(model, options);
const output: AssistantMessage = {
role: "assistant",
content: [],
api: "azure-openai-responses" as Api,
provider: model.provider,
model: model.id,
usage: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "stop",
timestamp: Date.now(),
};
try {
// Create Azure OpenAI client
const apiKey = options?.apiKey;
if (!apiKey) {
throw new Error(`No API key for provider: ${model.provider}`);
}
const client = createClient(model, apiKey, options);
let params = buildParams(model, context, options, deploymentName);
const nextParams = await options?.onPayload?.(params, model);
if (nextParams !== undefined) {
params = nextParams as ResponseCreateParamsStreaming;
}
const requestOptions = {
...(options?.signal ? { signal: options.signal } : {}),
...(options?.timeoutMs !== undefined ? { timeout: options.timeoutMs } : {}),
maxRetries: options?.maxRetries ?? 0,
};
const { data: openaiStream, response } = await client.responses.create(params, requestOptions).withResponse();
await options?.onResponse?.({ status: response.status, headers: headersToRecord(response.headers) }, model);
stream.push({ type: "start", partial: output });
await processResponsesStream(openaiStream, output, stream, model);
if (options?.signal?.aborted) {
throw new Error("Request was aborted");
}
if (output.stopReason === "aborted" || output.stopReason === "error") {
throw new Error("An unknown error occurred");
}
stream.push({ type: "done", reason: output.stopReason, message: output });
stream.end();
} catch (error) {
for (const block of output.content) {
delete (block as { index?: number }).index;
// partialJson is only a streaming scratch buffer; never persist it.
delete (block as { partialJson?: string }).partialJson;
}
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
output.errorMessage = formatAzureOpenAIError(error);
stream.push({ type: "error", reason: output.stopReason, error: output });
stream.end();
}
})();
return stream;
};
export const streamSimpleAzureOpenAIResponses: StreamFunction<"azure-openai-responses", SimpleStreamOptions> = (
model: Model<"azure-openai-responses">,
context: Context,
options?: SimpleStreamOptions,
): AssistantMessageEventStream => {
const apiKey = options?.apiKey;
if (!apiKey) {
throw new Error(`No API key for provider: ${model.provider}`);
}
const base = buildBaseOptions(model, options, apiKey);
const clampedReasoning = options?.reasoning ? clampThinkingLevel(model, options.reasoning) : undefined;
const reasoningEffort = clampedReasoning === "off" ? undefined : clampedReasoning;
return streamAzureOpenAIResponses(model, context, {
...base,
reasoningEffort,
} satisfies AzureOpenAIResponsesOptions);
};
function normalizeAzureBaseUrl(baseUrl: string): string {
const trimmed = baseUrl.trim().replace(/\/+$/, "");
let url: URL;
try {
url = new URL(trimmed);
} catch {
throw new Error(`Invalid Azure OpenAI base URL: ${baseUrl}`);
}
const isAzureHost =
url.hostname.endsWith(".openai.azure.com") || url.hostname.endsWith(".cognitiveservices.azure.com");
const normalizedPath = url.pathname.replace(/\/+$/, "");
// Ensure Azure hosts have /openai/v1 as base path so the AzureOpenAI SDK
// can append /deployments/<model>/... and ?api-version=v1 correctly.
if (isAzureHost && (normalizedPath === "" || normalizedPath === "/" || normalizedPath === "/openai")) {
url.pathname = "/openai/v1";
url.search = "";
}
return url.toString().replace(/\/+$/, "");
}
function buildDefaultBaseUrl(resourceName: string): string {
return `https://${resourceName}.openai.azure.com/openai/v1`;
}
function resolveAzureConfig(
model: Model<"azure-openai-responses">,
options?: AzureOpenAIResponsesOptions,
): { baseUrl: string; apiVersion: string } {
const apiVersion =
options?.azureApiVersion ||
getProviderEnvValue("AZURE_OPENAI_API_VERSION", options?.env) ||
DEFAULT_AZURE_API_VERSION;
const baseUrl =
options?.azureBaseUrl?.trim() || getProviderEnvValue("AZURE_OPENAI_BASE_URL", options?.env)?.trim() || undefined;
const resourceName = options?.azureResourceName || getProviderEnvValue("AZURE_OPENAI_RESOURCE_NAME", options?.env);
let resolvedBaseUrl = baseUrl;
if (!resolvedBaseUrl && resourceName) {
resolvedBaseUrl = buildDefaultBaseUrl(resourceName);
}
if (!resolvedBaseUrl && model.baseUrl) {
resolvedBaseUrl = model.baseUrl;
}
if (!resolvedBaseUrl) {
throw new Error(
"Azure OpenAI base URL is required. Set AZURE_OPENAI_BASE_URL or AZURE_OPENAI_RESOURCE_NAME, or pass azureBaseUrl, azureResourceName, or model.baseUrl.",
);
}
return {
baseUrl: normalizeAzureBaseUrl(resolvedBaseUrl),
apiVersion,
};
}
function createClient(model: Model<"azure-openai-responses">, apiKey: string, options?: AzureOpenAIResponsesOptions) {
const headers = { ...model.headers };
if (options?.headers) {
Object.assign(headers, options.headers);
}
const { baseUrl, apiVersion } = resolveAzureConfig(model, options);
return new AzureOpenAI({
apiKey,
apiVersion,
dangerouslyAllowBrowser: true,
defaultHeaders: headers,
baseURL: baseUrl,
export function azureOpenAIResponsesProvider(): Provider<"azure-openai-responses"> {
return createProvider({
id: "azure-openai-responses",
name: "Azure OpenAI",
auth: { apiKey: envApiKeyAuth("Azure OpenAI API key", ["AZURE_OPENAI_API_KEY"]) },
models: Object.values(AZURE_OPENAI_RESPONSES_MODELS),
api: azureOpenAIResponsesApi(),
});
}
function buildParams(
model: Model<"azure-openai-responses">,
context: Context,
options: AzureOpenAIResponsesOptions | undefined,
deploymentName: string,
) {
const messages = convertResponsesMessages(model, context, AZURE_TOOL_CALL_PROVIDERS);
const params: ResponseCreateParamsStreaming = {
model: deploymentName,
input: messages,
stream: true,
prompt_cache_key: clampOpenAIPromptCacheKey(options?.sessionId),
store: false,
};
if (options?.maxTokens) {
params.max_output_tokens = options?.maxTokens;
}
if (options?.temperature !== undefined) {
params.temperature = options?.temperature;
}
if (context.tools && context.tools.length > 0) {
params.tools = convertResponsesTools(context.tools);
}
if (model.reasoning) {
if (options?.reasoningEffort || options?.reasoningSummary) {
const effort = options?.reasoningEffort
? (model.thinkingLevelMap?.[options.reasoningEffort] ?? options.reasoningEffort)
: "medium";
params.reasoning = {
effort: effort as NonNullable<typeof params.reasoning>["effort"],
summary: options?.reasoningSummary || "auto",
};
params.include = ["reasoning.encrypted_content"];
} else if (model.thinkingLevelMap?.off !== null) {
params.reasoning = {
effort: (model.thinkingLevelMap?.off ?? "none") as NonNullable<typeof params.reasoning>["effort"],
};
}
}
return params;
}
packages/ai/src/providers/cerebras.models.ts
+43
View File
@@ -0,0 +1,43 @@
// This file is auto-generated by scripts/generate-models.ts
// Do not edit manually - run 'npm run generate-models' to update
import type { Model } from "../types.ts";
export const CEREBRAS_MODELS = {
"gpt-oss-120b": {
id: "gpt-oss-120b",
name: "GPT OSS 120B",
api: "openai-completions",
provider: "cerebras",
baseUrl: "https://api.cerebras.ai/v1",
compat: {"supportsStore":false,"supportsDeveloperRole":false},
reasoning: true,
input: ["text"],
cost: {
input: 0.35,
output: 0.75,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 131072,
maxTokens: 40960,
} satisfies Model<"openai-completions">,
"zai-glm-4.7": {
id: "zai-glm-4.7",
name: "Z.AI GLM-4.7",
api: "openai-completions",
provider: "cerebras",
baseUrl: "https://api.cerebras.ai/v1",
compat: {"supportsStore":false,"supportsDeveloperRole":false},
reasoning: true,
input: ["text"],
cost: {
input: 2.25,
output: 2.75,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 131072,
maxTokens: 40960,
} satisfies Model<"openai-completions">,
} as const;
packages/ai/src/providers/cerebras.ts
+15
View File
@@ -0,0 +1,15 @@
import { openAICompletionsApi } from "../api/openai-completions.lazy.ts";
import { envApiKeyAuth } from "../auth/helpers.ts";
import { createProvider, type Provider } from "../models.ts";
import { CEREBRAS_MODELS } from "./cerebras.models.ts";
export function cerebrasProvider(): Provider<"openai-completions"> {
return createProvider({
id: "cerebras",
name: "Cerebras",
baseUrl: "https://api.cerebras.ai/v1",
auth: { apiKey: envApiKeyAuth("Cerebras API key", ["CEREBRAS_API_KEY"]) },
models: Object.values(CEREBRAS_MODELS),
api: openAICompletionsApi(),
});
}
packages/ai/src/providers/cloudflare-ai-gateway.models.ts
+668
View File
@@ -0,0 +1,668 @@
// This file is auto-generated by scripts/generate-models.ts
// Do not edit manually - run 'npm run generate-models' to update
import type { Model } from "../types.ts";
export const CLOUDFLARE_AI_GATEWAY_MODELS = {
"claude-3-5-haiku": {
id: "claude-3-5-haiku",
name: "Claude Haiku 3.5 (latest)",
api: "anthropic-messages",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/anthropic",
compat: {"sendSessionAffinityHeaders":true},
reasoning: false,
input: ["text", "image"],
cost: {
input: 0.8,
output: 4,
cacheRead: 0.08,
cacheWrite: 1,
},
contextWindow: 200000,
maxTokens: 8192,
} satisfies Model<"anthropic-messages">,
"claude-3-haiku": {
id: "claude-3-haiku",
name: "Claude Haiku 3",
api: "anthropic-messages",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/anthropic",
compat: {"sendSessionAffinityHeaders":true},
reasoning: false,
input: ["text", "image"],
cost: {
input: 0.25,
output: 1.25,
cacheRead: 0.03,
cacheWrite: 0.3,
},
contextWindow: 200000,
maxTokens: 4096,
} satisfies Model<"anthropic-messages">,
"claude-3-opus": {
id: "claude-3-opus",
name: "Claude Opus 3",
api: "anthropic-messages",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/anthropic",
compat: {"sendSessionAffinityHeaders":true},
reasoning: false,
input: ["text", "image"],
cost: {
input: 15,
output: 75,
cacheRead: 1.5,
cacheWrite: 18.75,
},
contextWindow: 200000,
maxTokens: 4096,
} satisfies Model<"anthropic-messages">,
"claude-3-sonnet": {
id: "claude-3-sonnet",
name: "Claude Sonnet 3",
api: "anthropic-messages",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/anthropic",
compat: {"sendSessionAffinityHeaders":true},
reasoning: false,
input: ["text", "image"],
cost: {
input: 3,
output: 15,
cacheRead: 0.3,
cacheWrite: 0.3,
},
contextWindow: 200000,
maxTokens: 4096,
} satisfies Model<"anthropic-messages">,
"claude-3.5-haiku": {
id: "claude-3.5-haiku",
name: "Claude Haiku 3.5 (latest)",
api: "anthropic-messages",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/anthropic",
compat: {"sendSessionAffinityHeaders":true},
reasoning: false,
input: ["text", "image"],
cost: {
input: 0.8,
output: 4,
cacheRead: 0.08,
cacheWrite: 1,
},
contextWindow: 200000,
maxTokens: 8192,
} satisfies Model<"anthropic-messages">,
"claude-3.5-sonnet": {
id: "claude-3.5-sonnet",
name: "Claude Sonnet 3.5 v2",
api: "anthropic-messages",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/anthropic",
compat: {"sendSessionAffinityHeaders":true},
reasoning: false,
input: ["text", "image"],
cost: {
input: 3,
output: 15,
cacheRead: 0.3,
cacheWrite: 3.75,
},
contextWindow: 200000,
maxTokens: 8192,
} satisfies Model<"anthropic-messages">,
"claude-fable-5": {
id: "claude-fable-5",
name: "Claude Fable 5",
api: "anthropic-messages",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/anthropic",
compat: {"sendSessionAffinityHeaders":true,"forceAdaptiveThinking":true},
reasoning: true,
thinkingLevelMap: {"off":null,"xhigh":"xhigh"},
input: ["text", "image"],
cost: {
input: 10,
output: 50,
cacheRead: 1,
cacheWrite: 12.5,
},
contextWindow: 1000000,
maxTokens: 128000,
} satisfies Model<"anthropic-messages">,
"claude-haiku-4-5": {
id: "claude-haiku-4-5",
name: "Claude Haiku 4.5 (latest)",
api: "anthropic-messages",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/anthropic",
compat: {"sendSessionAffinityHeaders":true},
reasoning: true,
input: ["text", "image"],
cost: {
input: 1,
output: 5,
cacheRead: 0.1,
cacheWrite: 1.25,
},
contextWindow: 200000,
maxTokens: 64000,
} satisfies Model<"anthropic-messages">,
"claude-opus-4": {
id: "claude-opus-4",
name: "Claude Opus 4 (latest)",
api: "anthropic-messages",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/anthropic",
compat: {"sendSessionAffinityHeaders":true},
reasoning: true,
input: ["text", "image"],
cost: {
input: 15,
output: 75,
cacheRead: 1.5,
cacheWrite: 18.75,
},
contextWindow: 200000,
maxTokens: 32000,
} satisfies Model<"anthropic-messages">,
"claude-opus-4-1": {
id: "claude-opus-4-1",
name: "Claude Opus 4.1 (latest)",
api: "anthropic-messages",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/anthropic",
compat: {"sendSessionAffinityHeaders":true},
reasoning: true,
input: ["text", "image"],
cost: {
input: 15,
output: 75,
cacheRead: 1.5,
cacheWrite: 18.75,
},
contextWindow: 200000,
maxTokens: 32000,
} satisfies Model<"anthropic-messages">,
"claude-opus-4-5": {
id: "claude-opus-4-5",
name: "Claude Opus 4.5 (latest)",
api: "anthropic-messages",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/anthropic",
compat: {"sendSessionAffinityHeaders":true},
reasoning: true,
input: ["text", "image"],
cost: {
input: 5,
output: 25,
cacheRead: 0.5,
cacheWrite: 6.25,
},
contextWindow: 200000,
maxTokens: 64000,
} satisfies Model<"anthropic-messages">,
"claude-opus-4-6": {
id: "claude-opus-4-6",
name: "Claude Opus 4.6 (latest)",
api: "anthropic-messages",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/anthropic",
compat: {"sendSessionAffinityHeaders":true,"forceAdaptiveThinking":true},
reasoning: true,
thinkingLevelMap: {"xhigh":"max"},
input: ["text", "image"],
cost: {
input: 5,
output: 25,
cacheRead: 0.5,
cacheWrite: 6.25,
},
contextWindow: 1000000,
maxTokens: 128000,
} satisfies Model<"anthropic-messages">,
"claude-opus-4-7": {
id: "claude-opus-4-7",
name: "Claude Opus 4.7",
api: "anthropic-messages",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/anthropic",
compat: {"sendSessionAffinityHeaders":true,"forceAdaptiveThinking":true,"supportsTemperature":false},
reasoning: true,
thinkingLevelMap: {"xhigh":"xhigh"},
input: ["text", "image"],
cost: {
input: 5,
output: 25,
cacheRead: 0.5,
cacheWrite: 6.25,
},
contextWindow: 1000000,
maxTokens: 128000,
} satisfies Model<"anthropic-messages">,
"claude-opus-4-8": {
id: "claude-opus-4-8",
name: "Claude Opus 4.8",
api: "anthropic-messages",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/anthropic",
compat: {"sendSessionAffinityHeaders":true,"forceAdaptiveThinking":true,"supportsTemperature":false},
reasoning: true,
thinkingLevelMap: {"xhigh":"xhigh"},
input: ["text", "image"],
cost: {
input: 5,
output: 25,
cacheRead: 0.5,
cacheWrite: 6.25,
},
contextWindow: 1000000,
maxTokens: 128000,
} satisfies Model<"anthropic-messages">,
"claude-sonnet-4": {
id: "claude-sonnet-4",
name: "Claude Sonnet 4 (latest)",
api: "anthropic-messages",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/anthropic",
compat: {"sendSessionAffinityHeaders":true},
reasoning: true,
input: ["text", "image"],
cost: {
input: 3,
output: 15,
cacheRead: 0.3,
cacheWrite: 3.75,
},
contextWindow: 200000,
maxTokens: 64000,
} satisfies Model<"anthropic-messages">,
"claude-sonnet-4-5": {
id: "claude-sonnet-4-5",
name: "Claude Sonnet 4.5 (latest)",
api: "anthropic-messages",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/anthropic",
compat: {"sendSessionAffinityHeaders":true},
reasoning: true,
input: ["text", "image"],
cost: {
input: 3,
output: 15,
cacheRead: 0.3,
cacheWrite: 3.75,
},
contextWindow: 200000,
maxTokens: 64000,
} satisfies Model<"anthropic-messages">,
"claude-sonnet-4-6": {
id: "claude-sonnet-4-6",
name: "Claude Sonnet 4.6",
api: "anthropic-messages",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/anthropic",
compat: {"sendSessionAffinityHeaders":true,"forceAdaptiveThinking":true},
reasoning: true,
input: ["text", "image"],
cost: {
input: 3,
output: 15,
cacheRead: 0.3,
cacheWrite: 3.75,
},
contextWindow: 1000000,
maxTokens: 64000,
} satisfies Model<"anthropic-messages">,
"gpt-4": {
id: "gpt-4",
name: "GPT-4",
api: "openai-responses",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/openai",
reasoning: false,
input: ["text"],
cost: {
input: 30,
output: 60,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 8192,
maxTokens: 8192,
} satisfies Model<"openai-responses">,
"gpt-4-turbo": {
id: "gpt-4-turbo",
name: "GPT-4 Turbo",
api: "openai-responses",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/openai",
reasoning: false,
input: ["text", "image"],
cost: {
input: 10,
output: 30,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 128000,
maxTokens: 4096,
} satisfies Model<"openai-responses">,
"gpt-4o": {
id: "gpt-4o",
name: "GPT-4o",
api: "openai-responses",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/openai",
reasoning: false,
input: ["text", "image"],
cost: {
input: 2.5,
output: 10,
cacheRead: 1.25,
cacheWrite: 0,
},
contextWindow: 128000,
maxTokens: 16384,
} satisfies Model<"openai-responses">,
"gpt-4o-mini": {
id: "gpt-4o-mini",
name: "GPT-4o mini",
api: "openai-responses",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/openai",
reasoning: false,
input: ["text", "image"],
cost: {
input: 0.15,
output: 0.6,
cacheRead: 0.08,
cacheWrite: 0,
},
contextWindow: 128000,
maxTokens: 16384,
} satisfies Model<"openai-responses">,
"gpt-5.1": {
id: "gpt-5.1",
name: "GPT-5.1",
api: "openai-responses",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/openai",
reasoning: true,
thinkingLevelMap: {"off":null},
input: ["text", "image"],
cost: {
input: 1.25,
output: 10,
cacheRead: 0.13,
cacheWrite: 0,
},
contextWindow: 400000,
maxTokens: 128000,
} satisfies Model<"openai-responses">,
"gpt-5.1-codex": {
id: "gpt-5.1-codex",
name: "GPT-5.1 Codex",
api: "openai-responses",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/openai",
reasoning: true,
thinkingLevelMap: {"off":null},
input: ["text", "image"],
cost: {
input: 1.25,
output: 10,
cacheRead: 0.125,
cacheWrite: 0,
},
contextWindow: 400000,
maxTokens: 128000,
} satisfies Model<"openai-responses">,
"gpt-5.2": {
id: "gpt-5.2",
name: "GPT-5.2",
api: "openai-responses",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/openai",
reasoning: true,
thinkingLevelMap: {"off":null,"xhigh":"xhigh"},
input: ["text", "image"],
cost: {
input: 1.75,
output: 14,
cacheRead: 0.175,
cacheWrite: 0,
},
contextWindow: 400000,
maxTokens: 128000,
} satisfies Model<"openai-responses">,
"gpt-5.2-codex": {
id: "gpt-5.2-codex",
name: "GPT-5.2 Codex",
api: "openai-responses",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/openai",
reasoning: true,
thinkingLevelMap: {"off":null,"xhigh":"xhigh"},
input: ["text", "image"],
cost: {
input: 1.75,
output: 14,
cacheRead: 0.175,
cacheWrite: 0,
},
contextWindow: 400000,
maxTokens: 128000,
} satisfies Model<"openai-responses">,
"gpt-5.3-codex": {
id: "gpt-5.3-codex",
name: "GPT-5.3 Codex",
api: "openai-responses",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/openai",
reasoning: true,
thinkingLevelMap: {"off":null,"xhigh":"xhigh"},
input: ["text", "image"],
cost: {
input: 1.75,
output: 14,
cacheRead: 0.175,
cacheWrite: 0,
},
contextWindow: 400000,
maxTokens: 128000,
} satisfies Model<"openai-responses">,
"gpt-5.4": {
id: "gpt-5.4",
name: "GPT-5.4",
api: "openai-responses",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/openai",
reasoning: true,
thinkingLevelMap: {"off":null,"xhigh":"xhigh"},
input: ["text", "image"],
cost: {
input: 2.5,
output: 15,
cacheRead: 0.25,
cacheWrite: 0,
},
contextWindow: 1050000,
maxTokens: 128000,
} satisfies Model<"openai-responses">,
"gpt-5.5": {
id: "gpt-5.5",
name: "GPT-5.5",
api: "openai-responses",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/openai",
reasoning: true,
thinkingLevelMap: {"off":null,"xhigh":"xhigh"},
input: ["text", "image"],
cost: {
input: 5,
output: 30,
cacheRead: 0.5,
cacheWrite: 0,
},
contextWindow: 1050000,
maxTokens: 128000,
} satisfies Model<"openai-responses">,
"o1": {
id: "o1",
name: "o1",
api: "openai-responses",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/openai",
reasoning: true,
input: ["text", "image"],
cost: {
input: 15,
output: 60,
cacheRead: 7.5,
cacheWrite: 0,
},
contextWindow: 200000,
maxTokens: 100000,
} satisfies Model<"openai-responses">,
"o3": {
id: "o3",
name: "o3",
api: "openai-responses",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/openai",
reasoning: true,
input: ["text", "image"],
cost: {
input: 2,
output: 8,
cacheRead: 0.5,
cacheWrite: 0,
},
contextWindow: 200000,
maxTokens: 100000,
} satisfies Model<"openai-responses">,
"o3-mini": {
id: "o3-mini",
name: "o3-mini",
api: "openai-responses",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/openai",
reasoning: true,
input: ["text"],
cost: {
input: 1.1,
output: 4.4,
cacheRead: 0.55,
cacheWrite: 0,
},
contextWindow: 200000,
maxTokens: 100000,
} satisfies Model<"openai-responses">,
"o3-pro": {
id: "o3-pro",
name: "o3-pro",
api: "openai-responses",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/openai",
reasoning: true,
input: ["text", "image"],
cost: {
input: 20,
output: 80,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 200000,
maxTokens: 100000,
} satisfies Model<"openai-responses">,
"o4-mini": {
id: "o4-mini",
name: "o4-mini",
api: "openai-responses",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/openai",
reasoning: true,
input: ["text", "image"],
cost: {
input: 1.1,
output: 4.4,
cacheRead: 0.28,
cacheWrite: 0,
},
contextWindow: 200000,
maxTokens: 100000,
} satisfies Model<"openai-responses">,
"workers-ai/@cf/moonshotai/kimi-k2.5": {
id: "workers-ai/@cf/moonshotai/kimi-k2.5",
name: "Kimi K2.5",
api: "openai-completions",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/compat",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsReasoningEffort":false,"maxTokensField":"max_tokens","supportsStrictMode":false,"supportsLongCacheRetention":false,"sendSessionAffinityHeaders":true},
reasoning: true,
input: ["text", "image"],
cost: {
input: 0.6,
output: 3,
cacheRead: 0.1,
cacheWrite: 0,
},
contextWindow: 256000,
maxTokens: 256000,
} satisfies Model<"openai-completions">,
"workers-ai/@cf/moonshotai/kimi-k2.6": {
id: "workers-ai/@cf/moonshotai/kimi-k2.6",
name: "Kimi K2.6",
api: "openai-completions",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/compat",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsReasoningEffort":false,"maxTokensField":"max_tokens","supportsStrictMode":false,"supportsLongCacheRetention":false,"sendSessionAffinityHeaders":true},
reasoning: true,
input: ["text", "image"],
cost: {
input: 0.95,
output: 4,
cacheRead: 0.16,
cacheWrite: 0,
},
contextWindow: 256000,
maxTokens: 256000,
} satisfies Model<"openai-completions">,
"workers-ai/@cf/nvidia/nemotron-3-120b-a12b": {
id: "workers-ai/@cf/nvidia/nemotron-3-120b-a12b",
name: "Nemotron 3 Super 120B",
api: "openai-completions",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/compat",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsReasoningEffort":false,"maxTokensField":"max_tokens","supportsStrictMode":false,"supportsLongCacheRetention":false,"sendSessionAffinityHeaders":true},
reasoning: true,
input: ["text"],
cost: {
input: 0.5,
output: 1.5,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 256000,
maxTokens: 256000,
} satisfies Model<"openai-completions">,
"workers-ai/@cf/zai-org/glm-4.7-flash": {
id: "workers-ai/@cf/zai-org/glm-4.7-flash",
name: "GLM-4.7-Flash",
api: "openai-completions",
provider: "cloudflare-ai-gateway",
baseUrl: "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/compat",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsReasoningEffort":false,"maxTokensField":"max_tokens","supportsStrictMode":false,"supportsLongCacheRetention":false,"sendSessionAffinityHeaders":true},
reasoning: true,
input: ["text"],
cost: {
input: 0.06,
output: 0.4,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 131072,
maxTokens: 131072,
} satisfies Model<"openai-completions">,
} as const;
packages/ai/src/providers/cloudflare-ai-gateway.ts
+22
View File
@@ -0,0 +1,22 @@
import { anthropicMessagesApi } from "../api/anthropic-messages.lazy.ts";
import { openAICompletionsApi } from "../api/openai-completions.lazy.ts";
import { openAIResponsesApi } from "../api/openai-responses.lazy.ts";
import { createProvider, type Provider } from "../models.ts";
import { CLOUDFLARE_AI_GATEWAY_MODELS } from "./cloudflare-ai-gateway.models.ts";
import { cloudflareAIGatewayAuth } from "./cloudflare-auth.ts";
export function cloudflareAIGatewayProvider(): Provider<
"anthropic-messages" | "openai-completions" | "openai-responses"
> {
return createProvider({
id: "cloudflare-ai-gateway",
name: "Cloudflare AI Gateway",
auth: { apiKey: cloudflareAIGatewayAuth() },
models: Object.values(CLOUDFLARE_AI_GATEWAY_MODELS),
api: {
"anthropic-messages": anthropicMessagesApi(),
"openai-completions": openAICompletionsApi(),
"openai-responses": openAIResponsesApi(),
},
});
}
packages/ai/src/providers/cloudflare-auth.ts
+105
View File
@@ -0,0 +1,105 @@
import type { ApiKeyAuth, ApiKeyCredential, AuthContext } from "../auth/types.ts";
import type { Api, ImagesApi, ImagesModel, Model, ProviderEnv } from "../types.ts";
const CLOUDFLARE_API_KEY = "CLOUDFLARE_API_KEY";
const CLOUDFLARE_ACCOUNT_ID = "CLOUDFLARE_ACCOUNT_ID";
const CLOUDFLARE_GATEWAY_ID = "CLOUDFLARE_GATEWAY_ID";
type CloudflareAuthKind = "workers-ai" | "ai-gateway";
async function resolveValue(
name: string,
ctx: AuthContext,
credential: ApiKeyCredential | undefined,
): Promise<string | undefined> {
if (credential) {
if (name === CLOUDFLARE_API_KEY) return credential.key;
return credential.env?.[name];
}
return ctx.env(name);
}
function resolveCloudflareBaseUrl(
model: Model<Api> | ImagesModel<ImagesApi>,
accountId: string,
gatewayId: string | undefined,
): string {
return model.baseUrl
.replaceAll(`{${CLOUDFLARE_ACCOUNT_ID}}`, accountId)
.replaceAll(`{${CLOUDFLARE_GATEWAY_ID}}`, gatewayId ?? "");
}
async function resolveCloudflareEnv(
kind: CloudflareAuthKind,
model: Model<Api> | ImagesModel<ImagesApi>,
ctx: AuthContext,
credential: ApiKeyCredential | undefined,
): Promise<{ apiKey: string; env: ProviderEnv; baseUrl: string; source: string } | undefined> {
const apiKey = await resolveValue(CLOUDFLARE_API_KEY, ctx, credential);
const accountId = await resolveValue(CLOUDFLARE_ACCOUNT_ID, ctx, credential);
const gatewayId = kind === "ai-gateway" ? await resolveValue(CLOUDFLARE_GATEWAY_ID, ctx, credential) : undefined;
if (!apiKey || !accountId || (kind === "ai-gateway" && !gatewayId)) return undefined;
return {
apiKey,
env: {
CLOUDFLARE_ACCOUNT_ID: accountId,
...(gatewayId ? { CLOUDFLARE_GATEWAY_ID: gatewayId } : {}),
},
baseUrl: resolveCloudflareBaseUrl(model, accountId, gatewayId),
source: credential ? "stored credential" : CLOUDFLARE_API_KEY,
};
}
export function cloudflareWorkersAIAuth(): ApiKeyAuth {
return {
name: "Cloudflare API key",
login: async (callbacks) => {
const key = await callbacks.prompt({ type: "secret", message: "Enter Cloudflare API key" });
const accountId = await callbacks.prompt({ type: "text", message: "Enter Cloudflare account ID" });
return { type: "api_key", key, env: { CLOUDFLARE_ACCOUNT_ID: accountId } };
},
resolve: async ({ model, ctx, credential }) => {
const resolved = await resolveCloudflareEnv("workers-ai", model, ctx, credential);
if (!resolved) return undefined;
return {
auth: { apiKey: resolved.apiKey, baseUrl: resolved.baseUrl },
env: resolved.env,
source: resolved.source,
};
},
};
}
export function cloudflareAIGatewayAuth(): ApiKeyAuth {
return {
name: "Cloudflare API key",
login: async (callbacks) => {
const key = await callbacks.prompt({ type: "secret", message: "Enter Cloudflare API key" });
const accountId = await callbacks.prompt({ type: "text", message: "Enter Cloudflare account ID" });
const gatewayId = await callbacks.prompt({ type: "text", message: "Enter Cloudflare AI Gateway ID" });
return {
type: "api_key",
key,
env: { CLOUDFLARE_ACCOUNT_ID: accountId, CLOUDFLARE_GATEWAY_ID: gatewayId },
};
},
resolve: async ({ model, ctx, credential }) => {
const resolved = await resolveCloudflareEnv("ai-gateway", model, ctx, credential);
if (!resolved) return undefined;
return {
auth: {
headers: {
"cf-aig-authorization": `Bearer ${resolved.apiKey}`,
Authorization: null,
"x-api-key": null,
},
baseUrl: resolved.baseUrl,
},
env: resolved.env,
source: resolved.source,
};
},
};
}
packages/ai/src/providers/cloudflare-workers-ai.models.ts
+241
View File
@@ -0,0 +1,241 @@
// This file is auto-generated by scripts/generate-models.ts
// Do not edit manually - run 'npm run generate-models' to update
import type { Model } from "../types.ts";
export const CLOUDFLARE_WORKERS_AI_MODELS = {
"@cf/google/gemma-4-26b-a4b-it": {
id: "@cf/google/gemma-4-26b-a4b-it",
name: "Gemma 4 26B A4B IT",
api: "openai-completions",
provider: "cloudflare-workers-ai",
baseUrl: "https://api.cloudflare.com/client/v4/accounts/{CLOUDFLARE_ACCOUNT_ID}/ai/v1",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsLongCacheRetention":false,"sendSessionAffinityHeaders":true},
reasoning: true,
input: ["text", "image"],
cost: {
input: 0.1,
output: 0.3,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 256000,
maxTokens: 16384,
} satisfies Model<"openai-completions">,
"@cf/ibm-granite/granite-4.0-h-micro": {
id: "@cf/ibm-granite/granite-4.0-h-micro",
name: "Granite 4.0 H Micro",
api: "openai-completions",
provider: "cloudflare-workers-ai",
baseUrl: "https://api.cloudflare.com/client/v4/accounts/{CLOUDFLARE_ACCOUNT_ID}/ai/v1",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsLongCacheRetention":false,"sendSessionAffinityHeaders":true},
reasoning: false,
input: ["text"],
cost: {
input: 0.017,
output: 0.112,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 131000,
maxTokens: 131000,
} satisfies Model<"openai-completions">,
"@cf/meta/llama-3.3-70b-instruct-fp8-fast": {
id: "@cf/meta/llama-3.3-70b-instruct-fp8-fast",
name: "Llama 3.3 70B Instruct fp8 Fast",
api: "openai-completions",
provider: "cloudflare-workers-ai",
baseUrl: "https://api.cloudflare.com/client/v4/accounts/{CLOUDFLARE_ACCOUNT_ID}/ai/v1",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsLongCacheRetention":false,"sendSessionAffinityHeaders":true},
reasoning: false,
input: ["text"],
cost: {
input: 0.293,
output: 2.253,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 24000,
maxTokens: 24000,
} satisfies Model<"openai-completions">,
"@cf/meta/llama-4-scout-17b-16e-instruct": {
id: "@cf/meta/llama-4-scout-17b-16e-instruct",
name: "Llama 4 Scout 17B 16E Instruct",
api: "openai-completions",
provider: "cloudflare-workers-ai",
baseUrl: "https://api.cloudflare.com/client/v4/accounts/{CLOUDFLARE_ACCOUNT_ID}/ai/v1",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsLongCacheRetention":false,"sendSessionAffinityHeaders":true},
reasoning: false,
input: ["text", "image"],
cost: {
input: 0.27,
output: 0.85,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 131000,
maxTokens: 16384,
} satisfies Model<"openai-completions">,
"@cf/mistralai/mistral-small-3.1-24b-instruct": {
id: "@cf/mistralai/mistral-small-3.1-24b-instruct",
name: "Mistral Small 3.1 24B Instruct",
api: "openai-completions",
provider: "cloudflare-workers-ai",
baseUrl: "https://api.cloudflare.com/client/v4/accounts/{CLOUDFLARE_ACCOUNT_ID}/ai/v1",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsLongCacheRetention":false,"sendSessionAffinityHeaders":true},
reasoning: false,
input: ["text"],
cost: {
input: 0.351,
output: 0.555,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 128000,
maxTokens: 128000,
} satisfies Model<"openai-completions">,
"@cf/moonshotai/kimi-k2.6": {
id: "@cf/moonshotai/kimi-k2.6",
name: "Kimi K2.6",
api: "openai-completions",
provider: "cloudflare-workers-ai",
baseUrl: "https://api.cloudflare.com/client/v4/accounts/{CLOUDFLARE_ACCOUNT_ID}/ai/v1",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsLongCacheRetention":false,"sendSessionAffinityHeaders":true},
reasoning: true,
input: ["text", "image"],
cost: {
input: 0.95,
output: 4,
cacheRead: 0.16,
cacheWrite: 0,
},
contextWindow: 262144,
maxTokens: 256000,
} satisfies Model<"openai-completions">,
"@cf/moonshotai/kimi-k2.7-code": {
id: "@cf/moonshotai/kimi-k2.7-code",
name: "Kimi K2.7 Code",
api: "openai-completions",
provider: "cloudflare-workers-ai",
baseUrl: "https://api.cloudflare.com/client/v4/accounts/{CLOUDFLARE_ACCOUNT_ID}/ai/v1",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsLongCacheRetention":false,"sendSessionAffinityHeaders":true},
reasoning: true,
input: ["text", "image"],
cost: {
input: 0.95,
output: 4,
cacheRead: 0.19,
cacheWrite: 0,
},
contextWindow: 262144,
maxTokens: 262144,
} satisfies Model<"openai-completions">,
"@cf/nvidia/nemotron-3-120b-a12b": {
id: "@cf/nvidia/nemotron-3-120b-a12b",
name: "Nemotron 3 Super 120B",
api: "openai-completions",
provider: "cloudflare-workers-ai",
baseUrl: "https://api.cloudflare.com/client/v4/accounts/{CLOUDFLARE_ACCOUNT_ID}/ai/v1",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsLongCacheRetention":false,"sendSessionAffinityHeaders":true},
reasoning: true,
input: ["text"],
cost: {
input: 0.5,
output: 1.5,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 256000,
maxTokens: 256000,
} satisfies Model<"openai-completions">,
"@cf/openai/gpt-oss-120b": {
id: "@cf/openai/gpt-oss-120b",
name: "GPT OSS 120B",
api: "openai-completions",
provider: "cloudflare-workers-ai",
baseUrl: "https://api.cloudflare.com/client/v4/accounts/{CLOUDFLARE_ACCOUNT_ID}/ai/v1",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsLongCacheRetention":false,"sendSessionAffinityHeaders":true},
reasoning: true,
input: ["text"],
cost: {
input: 0.35,
output: 0.75,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 128000,
maxTokens: 16384,
} satisfies Model<"openai-completions">,
"@cf/openai/gpt-oss-20b": {
id: "@cf/openai/gpt-oss-20b",
name: "GPT OSS 20B",
api: "openai-completions",
provider: "cloudflare-workers-ai",
baseUrl: "https://api.cloudflare.com/client/v4/accounts/{CLOUDFLARE_ACCOUNT_ID}/ai/v1",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsLongCacheRetention":false,"sendSessionAffinityHeaders":true},
reasoning: true,
input: ["text"],
cost: {
input: 0.2,
output: 0.3,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 128000,
maxTokens: 16384,
} satisfies Model<"openai-completions">,
"@cf/qwen/qwen3-30b-a3b-fp8": {
id: "@cf/qwen/qwen3-30b-a3b-fp8",
name: "Qwen3 30B A3b fp8",
api: "openai-completions",
provider: "cloudflare-workers-ai",
baseUrl: "https://api.cloudflare.com/client/v4/accounts/{CLOUDFLARE_ACCOUNT_ID}/ai/v1",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsLongCacheRetention":false,"sendSessionAffinityHeaders":true},
reasoning: true,
input: ["text"],
cost: {
input: 0.0509,
output: 0.335,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 32768,
maxTokens: 32768,
} satisfies Model<"openai-completions">,
"@cf/zai-org/glm-4.7-flash": {
id: "@cf/zai-org/glm-4.7-flash",
name: "GLM-4.7-Flash",
api: "openai-completions",
provider: "cloudflare-workers-ai",
baseUrl: "https://api.cloudflare.com/client/v4/accounts/{CLOUDFLARE_ACCOUNT_ID}/ai/v1",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsLongCacheRetention":false,"sendSessionAffinityHeaders":true},
reasoning: true,
input: ["text"],
cost: {
input: 0.0605,
output: 0.4,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 131072,
maxTokens: 131072,
} satisfies Model<"openai-completions">,
"@cf/zai-org/glm-5.2": {
id: "@cf/zai-org/glm-5.2",
name: "Glm 5.2",
api: "openai-completions",
provider: "cloudflare-workers-ai",
baseUrl: "https://api.cloudflare.com/client/v4/accounts/{CLOUDFLARE_ACCOUNT_ID}/ai/v1",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsLongCacheRetention":false,"sendSessionAffinityHeaders":true},
reasoning: true,
input: ["text"],
cost: {
input: 1.4,
output: 4.4,
cacheRead: 0.26,
cacheWrite: 0,
},
contextWindow: 262144,
maxTokens: 262144,
} satisfies Model<"openai-completions">,
} as const;
packages/ai/src/providers/cloudflare-workers-ai.ts
+14
View File
@@ -0,0 +1,14 @@
import { openAICompletionsApi } from "../api/openai-completions.lazy.ts";
import { createProvider, type Provider } from "../models.ts";
import { cloudflareWorkersAIAuth } from "./cloudflare-auth.ts";
import { CLOUDFLARE_WORKERS_AI_MODELS } from "./cloudflare-workers-ai.models.ts";
export function cloudflareWorkersAIProvider(): Provider<"openai-completions"> {
return createProvider({
id: "cloudflare-workers-ai",
name: "Cloudflare Workers AI",
auth: { apiKey: cloudflareWorkersAIAuth() },
models: Object.values(CLOUDFLARE_WORKERS_AI_MODELS),
api: openAICompletionsApi(),
});
}
packages/ai/src/providers/deepseek.models.ts
+45
View File
@@ -0,0 +1,45 @@
// This file is auto-generated by scripts/generate-models.ts
// Do not edit manually - run 'npm run generate-models' to update
import type { Model } from "../types.ts";
export const DEEPSEEK_MODELS = {
"deepseek-v4-flash": {
id: "deepseek-v4-flash",
name: "DeepSeek V4 Flash",
api: "openai-completions",
provider: "deepseek",
baseUrl: "https://api.deepseek.com",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"requiresReasoningContentOnAssistantMessages":true,"thinkingFormat":"deepseek"},
reasoning: true,
thinkingLevelMap: {"minimal":null,"low":null,"medium":null,"high":"high","xhigh":"max"},
input: ["text"],
cost: {
input: 0.14,
output: 0.28,
cacheRead: 0.0028,
cacheWrite: 0,
},
contextWindow: 1000000,
maxTokens: 384000,
} satisfies Model<"openai-completions">,
"deepseek-v4-pro": {
id: "deepseek-v4-pro",
name: "DeepSeek V4 Pro",
api: "openai-completions",
provider: "deepseek",
baseUrl: "https://api.deepseek.com",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"requiresReasoningContentOnAssistantMessages":true,"thinkingFormat":"deepseek"},
reasoning: true,
thinkingLevelMap: {"minimal":null,"low":null,"medium":null,"high":"high","xhigh":"max"},
input: ["text"],
cost: {
input: 0.435,
output: 0.87,
cacheRead: 0.003625,
cacheWrite: 0,
},
contextWindow: 1000000,
maxTokens: 384000,
} satisfies Model<"openai-completions">,
} as const;
packages/ai/src/providers/deepseek.ts
+15
View File
@@ -0,0 +1,15 @@
import { openAICompletionsApi } from "../api/openai-completions.lazy.ts";
import { envApiKeyAuth } from "../auth/helpers.ts";
import { createProvider, type Provider } from "../models.ts";
import { DEEPSEEK_MODELS } from "./deepseek.models.ts";
export function deepseekProvider(): Provider<"openai-completions"> {
return createProvider({
id: "deepseek",
name: "DeepSeek",
baseUrl: "https://api.deepseek.com",
auth: { apiKey: envApiKeyAuth("DeepSeek API key", ["DEEPSEEK_API_KEY"]) },
models: Object.values(DEEPSEEK_MODELS),
api: openAICompletionsApi(),
});
}
packages/ai/src/providers/faux.ts
+49 -10
View File
@@ -1,4 +1,4 @@
import { registerApiProvider, unregisterApiProviders } from "../api-registry.ts";
import { createProvider, type Provider } from "../models.ts";
import type {
AssistantMessage,
AssistantMessageEventStream,
@@ -125,6 +125,18 @@ export interface FauxProviderRegistration {
unregister: () => void;
}
export interface FauxProviderHandle {
provider: Provider;
api: string;
models: [Model<string>, ...Model<string>[]];
getModel(): Model<string>;
getModel(modelId: string): Model<string> | undefined;
state: { callCount: number };
setResponses: (responses: FauxResponseStep[]) => void;
appendResponses: (responses: FauxResponseStep[]) => void;
getPendingResponseCount: () => number;
}
function estimateTokens(text: string): number {
return Math.ceil(text.length / 4);
}
@@ -388,10 +400,9 @@ async function streamWithDeltas(
stream.end(message);
}
export function registerFauxProvider(options: RegisterFauxProviderOptions = {}): FauxProviderRegistration {
export function createFauxCore(options: RegisterFauxProviderOptions) {
const api = options.api ?? randomId(DEFAULT_API);
const provider = options.provider ?? DEFAULT_PROVIDER;
const sourceId = randomId("faux-provider");
const minTokenSize = Math.max(
1,
Math.min(options.tokenSize?.min ?? DEFAULT_MIN_TOKEN_SIZE, options.tokenSize?.max ?? DEFAULT_MAX_TOKEN_SIZE),
@@ -467,8 +478,6 @@ export function registerFauxProvider(options: RegisterFauxProviderOptions = {}):
const streamSimple: StreamFunction<string, SimpleStreamOptions> = (streamModel, context, streamOptions) =>
stream(streamModel, context, streamOptions);
registerApiProvider({ api, stream, streamSimple }, sourceId);
function getModel(): Model<string>;
function getModel(requestedModelId: string): Model<string> | undefined;
function getModel(requestedModelId?: string): Model<string> | undefined {
@@ -480,20 +489,50 @@ export function registerFauxProvider(options: RegisterFauxProviderOptions = {}):
return {
api,
provider,
models,
stream,
streamSimple,
getModel,
state,
setResponses(responses) {
setResponses(responses: FauxResponseStep[]) {
pendingResponses = [...responses];
},
appendResponses(responses) {
appendResponses(responses: FauxResponseStep[]) {
pendingResponses.push(...responses);
},
getPendingResponseCount() {
return pendingResponses.length;
},
unregister() {
unregisterApiProviders(sourceId);
},
};
}
/**
* Faux provider for tests built on explicit `Models` collections:
*
* ```ts
* const faux = fauxProvider();
* const models = createModels();
* models.setProvider(faux.provider);
* faux.setResponses([fauxAssistantMessage("hi")]);
* ```
*/
export function fauxProvider(options: RegisterFauxProviderOptions = {}): FauxProviderHandle {
const core = createFauxCore(options);
const provider = createProvider({
id: core.provider,
auth: { apiKey: { name: "Faux", resolve: async () => ({ auth: {} }) } },
models: core.models,
api: { stream: core.stream, streamSimple: core.streamSimple },
});
return {
provider,
api: core.api,
models: core.models,
getModel: core.getModel,
state: core.state,
setResponses: core.setResponses,
appendResponses: core.appendResponses,
getPendingResponseCount: core.getPendingResponseCount,
};
}
packages/ai/src/providers/fireworks.models.ts
+278
View File
@@ -0,0 +1,278 @@
// This file is auto-generated by scripts/generate-models.ts
// Do not edit manually - run 'npm run generate-models' to update
import type { Model } from "../types.ts";
export const FIREWORKS_MODELS = {
"accounts/fireworks/models/deepseek-v4-flash": {
id: "accounts/fireworks/models/deepseek-v4-flash",
name: "DeepSeek V4 Flash",
api: "anthropic-messages",
provider: "fireworks",
baseUrl: "https://api.fireworks.ai/inference",
compat: {"sendSessionAffinityHeaders":true,"supportsEagerToolInputStreaming":false,"supportsCacheControlOnTools":false,"supportsLongCacheRetention":false},
reasoning: true,
input: ["text"],
cost: {
input: 0.14,
output: 0.28,
cacheRead: 0.028,
cacheWrite: 0,
},
contextWindow: 1000000,
maxTokens: 384000,
} satisfies Model<"anthropic-messages">,
"accounts/fireworks/models/deepseek-v4-pro": {
id: "accounts/fireworks/models/deepseek-v4-pro",
name: "DeepSeek V4 Pro",
api: "anthropic-messages",
provider: "fireworks",
baseUrl: "https://api.fireworks.ai/inference",
compat: {"sendSessionAffinityHeaders":true,"supportsEagerToolInputStreaming":false,"supportsCacheControlOnTools":false,"supportsLongCacheRetention":false},
reasoning: true,
input: ["text"],
cost: {
input: 1.74,
output: 3.48,
cacheRead: 0.145,
cacheWrite: 0,
},
contextWindow: 1000000,
maxTokens: 384000,
} satisfies Model<"anthropic-messages">,
"accounts/fireworks/models/glm-5p1": {
id: "accounts/fireworks/models/glm-5p1",
name: "GLM 5.1",
api: "anthropic-messages",
provider: "fireworks",
baseUrl: "https://api.fireworks.ai/inference",
compat: {"sendSessionAffinityHeaders":true,"supportsEagerToolInputStreaming":false,"supportsCacheControlOnTools":false,"supportsLongCacheRetention":false},
reasoning: true,
input: ["text"],
cost: {
input: 1.4,
output: 4.4,
cacheRead: 0.26,
cacheWrite: 0,
},
contextWindow: 202800,
maxTokens: 131072,
} satisfies Model<"anthropic-messages">,
"accounts/fireworks/models/glm-5p2": {
id: "accounts/fireworks/models/glm-5p2",
name: "GLM 5.2",
api: "openai-completions",
provider: "fireworks",
baseUrl: "https://api.fireworks.ai/inference/v1",
compat: {"supportsStore":false,"supportsDeveloperRole":false},
reasoning: true,
thinkingLevelMap: {"off":"none","minimal":null,"low":"high","medium":"high","xhigh":"max"},
input: ["text"],
cost: {
input: 1.4,
output: 4.4,
cacheRead: 0.26,
cacheWrite: 0,
},
contextWindow: 1048576,
maxTokens: 131072,
} satisfies Model<"openai-completions">,
"accounts/fireworks/models/gpt-oss-120b": {
id: "accounts/fireworks/models/gpt-oss-120b",
name: "GPT OSS 120B",
api: "anthropic-messages",
provider: "fireworks",
baseUrl: "https://api.fireworks.ai/inference",
compat: {"sendSessionAffinityHeaders":true,"supportsEagerToolInputStreaming":false,"supportsCacheControlOnTools":false,"supportsLongCacheRetention":false},
reasoning: true,
input: ["text"],
cost: {
input: 0.15,
output: 0.6,
cacheRead: 0.015,
cacheWrite: 0,
},
contextWindow: 131072,
maxTokens: 32768,
} satisfies Model<"anthropic-messages">,
"accounts/fireworks/models/gpt-oss-20b": {
id: "accounts/fireworks/models/gpt-oss-20b",
name: "GPT OSS 20B",
api: "anthropic-messages",
provider: "fireworks",
baseUrl: "https://api.fireworks.ai/inference",
compat: {"sendSessionAffinityHeaders":true,"supportsEagerToolInputStreaming":false,"supportsCacheControlOnTools":false,"supportsLongCacheRetention":false},
reasoning: true,
input: ["text"],
cost: {
input: 0.07,
output: 0.3,
cacheRead: 0.035,
cacheWrite: 0,
},
contextWindow: 131072,
maxTokens: 32768,
} satisfies Model<"anthropic-messages">,
"accounts/fireworks/models/kimi-k2p6": {
id: "accounts/fireworks/models/kimi-k2p6",
name: "Kimi K2.6",
api: "anthropic-messages",
provider: "fireworks",
baseUrl: "https://api.fireworks.ai/inference",
compat: {"sendSessionAffinityHeaders":true,"supportsEagerToolInputStreaming":false,"supportsCacheControlOnTools":false,"supportsLongCacheRetention":false},
reasoning: true,
input: ["text", "image"],
cost: {
input: 0.95,
output: 4,
cacheRead: 0.16,
cacheWrite: 0,
},
contextWindow: 262000,
maxTokens: 262000,
} satisfies Model<"anthropic-messages">,
"accounts/fireworks/models/kimi-k2p7-code": {
id: "accounts/fireworks/models/kimi-k2p7-code",
name: "Kimi K2.7 Code",
api: "anthropic-messages",
provider: "fireworks",
baseUrl: "https://api.fireworks.ai/inference",
compat: {"sendSessionAffinityHeaders":true,"supportsEagerToolInputStreaming":false,"supportsCacheControlOnTools":false,"supportsLongCacheRetention":false},
reasoning: true,
input: ["text", "image"],
cost: {
input: 0.95,
output: 4,
cacheRead: 0.19,
cacheWrite: 0,
},
contextWindow: 262000,
maxTokens: 262000,
} satisfies Model<"anthropic-messages">,
"accounts/fireworks/models/minimax-m2p7": {
id: "accounts/fireworks/models/minimax-m2p7",
name: "MiniMax-M2.7",
api: "anthropic-messages",
provider: "fireworks",
baseUrl: "https://api.fireworks.ai/inference",
compat: {"sendSessionAffinityHeaders":true,"supportsEagerToolInputStreaming":false,"supportsCacheControlOnTools":false,"supportsLongCacheRetention":false},
reasoning: true,
input: ["text"],
cost: {
input: 0.3,
output: 1.2,
cacheRead: 0.06,
cacheWrite: 0,
},
contextWindow: 196608,
maxTokens: 196608,
} satisfies Model<"anthropic-messages">,
"accounts/fireworks/models/minimax-m3": {
id: "accounts/fireworks/models/minimax-m3",
name: "MiniMax-M3",
api: "anthropic-messages",
provider: "fireworks",
baseUrl: "https://api.fireworks.ai/inference",
compat: {"sendSessionAffinityHeaders":true,"supportsEagerToolInputStreaming":false,"supportsCacheControlOnTools":false,"supportsLongCacheRetention":false},
reasoning: true,
input: ["text"],
cost: {
input: 0.3,
output: 1.2,
cacheRead: 0.06,
cacheWrite: 0,
},
contextWindow: 512000,
maxTokens: 512000,
} satisfies Model<"anthropic-messages">,
"accounts/fireworks/models/qwen3p7-plus": {
id: "accounts/fireworks/models/qwen3p7-plus",
name: "Qwen 3.7 Plus",
api: "anthropic-messages",
provider: "fireworks",
baseUrl: "https://api.fireworks.ai/inference",
compat: {"sendSessionAffinityHeaders":true,"supportsEagerToolInputStreaming":false,"supportsCacheControlOnTools":false,"supportsLongCacheRetention":false},
reasoning: true,
input: ["text", "image"],
cost: {
input: 0.4,
output: 1.6,
cacheRead: 0.08,
cacheWrite: 0,
},
contextWindow: 262144,
maxTokens: 65536,
} satisfies Model<"anthropic-messages">,
"accounts/fireworks/routers/glm-5p1-fast": {
id: "accounts/fireworks/routers/glm-5p1-fast",
name: "GLM 5.1 Fast",
api: "anthropic-messages",
provider: "fireworks",
baseUrl: "https://api.fireworks.ai/inference",
compat: {"sendSessionAffinityHeaders":true,"supportsEagerToolInputStreaming":false,"supportsCacheControlOnTools":false,"supportsLongCacheRetention":false},
reasoning: true,
input: ["text"],
cost: {
input: 2.8,
output: 8.8,
cacheRead: 0.52,
cacheWrite: 0,
},
contextWindow: 202800,
maxTokens: 131072,
} satisfies Model<"anthropic-messages">,
"accounts/fireworks/routers/kimi-k2p6-fast": {
id: "accounts/fireworks/routers/kimi-k2p6-fast",
name: "Kimi K2.6 Fast",
api: "anthropic-messages",
provider: "fireworks",
baseUrl: "https://api.fireworks.ai/inference",
compat: {"sendSessionAffinityHeaders":true,"supportsEagerToolInputStreaming":false,"supportsCacheControlOnTools":false,"supportsLongCacheRetention":false},
reasoning: true,
input: ["text", "image"],
cost: {
input: 2,
output: 8,
cacheRead: 0.3,
cacheWrite: 0,
},
contextWindow: 262000,
maxTokens: 262000,
} satisfies Model<"anthropic-messages">,
"accounts/fireworks/routers/kimi-k2p6-turbo": {
id: "accounts/fireworks/routers/kimi-k2p6-turbo",
name: "Kimi K2.6 Turbo",
api: "anthropic-messages",
provider: "fireworks",
baseUrl: "https://api.fireworks.ai/inference",
compat: {"sendSessionAffinityHeaders":true,"supportsEagerToolInputStreaming":false,"supportsCacheControlOnTools":false,"supportsLongCacheRetention":false},
reasoning: true,
input: ["text", "image"],
cost: {
input: 2,
output: 8,
cacheRead: 0.3,
cacheWrite: 0,
},
contextWindow: 262000,
maxTokens: 262000,
} satisfies Model<"anthropic-messages">,
"accounts/fireworks/routers/kimi-k2p7-code-fast": {
id: "accounts/fireworks/routers/kimi-k2p7-code-fast",
name: "Kimi K2.7 Code Fast",
api: "anthropic-messages",
provider: "fireworks",
baseUrl: "https://api.fireworks.ai/inference",
compat: {"sendSessionAffinityHeaders":true,"supportsEagerToolInputStreaming":false,"supportsCacheControlOnTools":false,"supportsLongCacheRetention":false},
reasoning: true,
input: ["text", "image"],
cost: {
input: 1.9,
output: 8,
cacheRead: 0.38,
cacheWrite: 0,
},
contextWindow: 262000,
maxTokens: 262000,
} satisfies Model<"anthropic-messages">,
} as const;

Some files were not shown because too many files have changed in this diff Show More