new_pull
This commit is contained in:
@@ -2,6 +2,10 @@
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.82.1] - 2026-07-25
|
||||
|
||||
## [0.82.0] - 2026-07-24
|
||||
|
||||
### Breaking Changes
|
||||
|
||||
- Replaced `AgentHarness`'s `ExecutionEnv` dependency and context-free `AgentTool` inputs with application-defined `toolContext` values and context-aware `AgentHarnessTool` definitions.
|
||||
@@ -14,6 +18,10 @@
|
||||
|
||||
- Aligned harness tool path handling, edit serialization, shell output capture, explicit non-inherited environments, and cross-platform process cleanup with coding-agent behavior.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed compaction and branch-summary requests to use fresh routing session IDs with prompt caching disabled where supported ([#6618](https://github.com/earendil-works/pi/pull/6618) by [@tmustier](https://github.com/tmustier)).
|
||||
|
||||
## [0.81.1] - 2026-07-21
|
||||
|
||||
### Added
|
||||
|
||||
@@ -1,473 +0,0 @@
|
||||
# Agent Workflow: Message Handling and Response Generation
|
||||
|
||||
## Overview
|
||||
|
||||
This document explains the complete step-by-step flow of how the agent processes a user message and generates a response, from the moment a user asks "what product do you have in stock" to when the agent responds with an answer.
|
||||
|
||||
---
|
||||
|
||||
## Architecture Diagram
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ USER INPUT │
|
||||
│ "what product do you have in stock?" │
|
||||
└─────────────────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 1. AGENT.PROMPT() ENTRY │
|
||||
│ File: packages/agent/src/agent.ts:339 │
|
||||
│ ┌──────────────────────────────────────────────────────────────────────────────┐ │
|
||||
│ │ • Validate no active run (throws if busy) │ │
|
||||
│ │ • normalizePromptInput() converts string to AgentMessage[] │ │
|
||||
│ │ • runPromptMessages() launches execution with lifecycle events │ │
|
||||
│ └──────────────────────────────────────────────────────────────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 2. AGENT LIFECYCLE INITIALIZATION │
|
||||
│ File: packages/agent/src/agent.ts:398-412 │
|
||||
│ ┌──────────────────────────────────────────────────────────────────────────────┐ │
|
||||
│ │ Events Emitted: │ │
|
||||
│ │ • agent_start │ │
|
||||
│ │ • turn_start │ │
|
||||
│ │ • message_start / message_end (for each prompt message) │ │
|
||||
│ │ │ │
|
||||
│ │ Context Snapshot Created: │ │
|
||||
│ │ • systemPrompt │ │
|
||||
│ │ • messages (copy) │ │
|
||||
│ │ • tools (copy) │ │
|
||||
│ └──────────────────────────────────────────────────────────────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 3. AGENT LOOP STARTS │
|
||||
│ File: packages/agent/src/agent-loop.ts:95 │
|
||||
│ ┌──────────────────────────────────────────────────────────────────────────────┐ │
|
||||
│ │ runPromptMessages() → runAgentLoop() │ │
|
||||
│ │ • Prompts added to context.messages │ │
|
||||
│ │ • Lifecycle events emitted for prompts │ │
|
||||
│ │ • Calls runLoop() (main processing loop) │ │
|
||||
│ └──────────────────────────────────────────────────────────────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 4. LLM CALL BOUNDARY - MESSAGE TRANSFORMATION │
|
||||
│ File: packages/agent/src/agent-loop.ts:281-372 (streamAssistantResponse) │
|
||||
│ ┌──────────────────────────────────────────────────────────────────────────────┐ │
|
||||
│ │ Step 4.1: Context Transform (optional) │ │
|
||||
│ │ transformContext(messages) → transformed messages │ │
|
||||
│ │ (Used for context pruning/injection) │ │
|
||||
│ │ │ │
|
||||
│ │ Step 4.2: LLM Conversion │ │
|
||||
│ │ convertToLlm(messages) → Message[] │ │
|
||||
│ │ - Filters non-LLM messages (bashExecution, branchSummary, etc.) │ │
|
||||
│ │ - Converts: user → user, assistant → assistant, toolResult → toolResult │ │
|
||||
│ │ │ │
|
||||
│ │ Step 4.3: Build LLM Context │ │
|
||||
│ │ { │ │
|
||||
│ │ systemPrompt: context.systemPrompt, │ │
|
||||
│ │ messages: llmMessages, │ │
|
||||
│ │ tools: context.tools │ │
|
||||
│ │ } │ │
|
||||
│ │ │ │
|
||||
│ │ Step 4.4: Resolve API Key │ │
|
||||
│ │ getApiKey(model.provider) → apiKey │ │
|
||||
│ │ │ │
|
||||
│ │ Step 4.5: Stream Function Call │ │
|
||||
│ │ streamFunction(model, llmContext, options) │ │
|
||||
│ │ - Default: Models.streamSimple() from @earendil-works/pi-ai │ │
|
||||
│ │ - Makes actual LLM API call │ │
|
||||
│ └──────────────────────────────────────────────────────────────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 5. LLM RESPONSE STREAMING │
|
||||
│ File: packages/agent/src/agent-loop.ts:317-371 │
|
||||
│ ┌──────────────────────────────────────────────────────────────────────────────┐ │
|
||||
│ │ Stream yields events: │ │
|
||||
│ │ • start → creates partial AssistantMessage │ │
|
||||
│ │ • text_start → streaming text begins │ │
|
||||
│ │ • text_delta → text chunks arrive │ │
|
||||
│ │ • toolcall_start → tool call block begins │ │
|
||||
│ │ • toolcall_delta → tool call arguments arrive │ │
|
||||
│ │ • toolcall_end → tool call block complete │ │
|
||||
│ │ • text_end → text block complete │ │
|
||||
│ │ • done → final message complete │ │
|
||||
│ │ │ │
|
||||
│ │ State Updates: │ │
|
||||
│ │ • Partial message pushed to context.messages │ │
|
||||
│ │ • message_start event emitted │ │
|
||||
│ │ • message_update events emitted as text/tools stream in │ │
|
||||
│ │ • Final message committed to context.messages │ │
|
||||
│ │ • message_end event emitted │ │
|
||||
│ │ │ │
|
||||
│ │ Stop Reasons: │ │
|
||||
│ │ • stop - normal completion │ │
|
||||
│ │ • toolUse - model requested tool calls │ │
|
||||
│ │ • length - token limit reached │ │
|
||||
│ │ • error - failure │ │
|
||||
│ │ • aborted - operation aborted │ │
|
||||
│ └──────────────────────────────────────────────────────────────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 6. TOOL CALL PARSING AND EXECUTION │
|
||||
│ File: packages/agent/src/agent-loop.ts:408-554 │
|
||||
│ ┌──────────────────────────────────────────────────────────────────────────────┐ │
|
||||
│ │ Step 6.1: Extract Tool Calls │ │
|
||||
│ │ toolCalls = message.content.filter(c => c.type === "toolCall") │ │
|
||||
│ │ │ │
|
||||
│ │ Step 6.2: Determine Execution Mode │ │
|
||||
│ │ - Check config.toolExecution: "parallel" or "sequential" │ │
|
||||
│ │ - Check individual tool executionMode setting │ │
|
||||
│ │ - Decides how to execute tool batch │ │
|
||||
│ │ │ │
|
||||
│ │ Step 6.3: For Each Tool Call │ │
|
||||
│ │ ┌──────────────────────────────────────────────────────────────────────┐ │ │
|
||||
│ │ │ 1. Tool Lookup │ │ │
|
||||
│ │ │ tool = context.tools.find(t => t.name === toolCall.name) │ │ │
|
||||
│ │ │ │ │ │
|
||||
│ │ │ 2. Argument Preparation │ │ │
|
||||
│ │ │ prepared = tool.prepareArguments?(toolCall.arguments) │ │ │
|
||||
│ │ │ │ │ │
|
||||
│ │ │ 3. Argument Validation │ │ │
|
||||
│ │ │ validateToolArguments(tool, preparedToolCall) │ │ │
|
||||
│ │ │ │ │ │
|
||||
│ │ │ 4. Before Tool Hook │ │ │
|
||||
│ │ │ beforeToolCall({ assistantMessage, toolCall, args, context }) │ │ │
|
||||
│ │ │ - Can block execution by returning { block: true, reason } │ │ │
|
||||
│ │ │ │ │ │
|
||||
│ │ │ 5. Execution │ │ │
|
||||
│ │ │ execute(toolCallId, params, signal, onUpdate, context) │ │ │
|
||||
│ │ │ │ │ │
|
||||
│ │ │ - Parallel Mode: Tools execute concurrently │ │ │
|
||||
│ │ │ - Sequential Mode: Tools execute one-by-one │ │ │
|
||||
│ │ └──────────────────────────────────────────────────────────────────────┘ │ │
|
||||
│ └──────────────────────────────────────────────────────────────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 7. TOOL EXECUTION EXAMPLE - READ TOOL │
|
||||
│ File: packages/agent/src/harness/tools/read.ts │
|
||||
│ ┌──────────────────────────────────────────────────────────────────────────────┐ │
|
||||
│ │ User asks: "what product do you have in stock?" │ │
|
||||
│ │ │ │
|
||||
│ │ Agent decides to read catalog file "products.json" │ │
|
||||
│ │ │ │
|
||||
│ │ Input Arguments: │ │
|
||||
│ │ { │ │
|
||||
│ │ "path": "products.json", │ │
|
||||
│ │ "offset": 1, │ │
|
||||
│ │ "limit": 100 │ │
|
||||
│ │ } │ │
|
||||
│ │ │ │
|
||||
│ │ Execution Steps: │ │
|
||||
│ │ 1. resolveReadToolPath(env, path, signal) → absolutePath │ │
|
||||
│ │ 2. env.readBinaryFile(absolutePath, signal) → bytes │ │
|
||||
│ │ 3. Detect mimeType (check if image) │ │
|
||||
│ │ 4. For text files: │ │
|
||||
│ │ - Decode UTF-8 → textContent │ │
|
||||
│ │ - Split by lines → allLines │ │
|
||||
│ │ - Apply offset/limit slicing │ │
|
||||
│ │ - Truncate if exceeds DEFAULT_MAX_BYTES or DEFAULT_MAX_LINES │ │
|
||||
│ │ - Add truncation notice to output │ │
|
||||
│ │ 5. Return result: │ │
|
||||
│ │ { │ │
|
||||
│ │ content: [{ type: "text", text: output }], │ │
|
||||
│ │ details: { truncation: ... } │ │
|
||||
│ │ } │ │
|
||||
│ │ │ │
|
||||
│ │ Output Example: │ │
|
||||
│ │ "Showing lines 1-50 of 150. [Showing 50 lines of 150. Use offset=51 to │ │
|
||||
│ │ continue.]" │ │
|
||||
│ │ │ │
|
||||
│ │ Tool Result: │ │
|
||||
│ │ { │ │
|
||||
│ │ "role": "toolResult", │ │
|
||||
│ │ "toolCallId": "tool_abc123", │ │
|
||||
│ │ "toolName": "read", │ │
|
||||
│ │ "content": [{ "type": "text", "text": "..." }], │ │
|
||||
│ │ "isError": false, │ │
|
||||
│ │ "timestamp": 1721721600000 │ │
|
||||
│ │ } │ │
|
||||
│ └──────────────────────────────────────────────────────────────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 8. TOOL RESULT HANDLING │
|
||||
│ File: packages/agent/src/agent-loop.ts:556-792 │
|
||||
│ ┌──────────────────────────────────────────────────────────────────────────────┐ │
|
||||
│ │ Step 8.1: After Tool Hook │ │
|
||||
│ │ afterToolCall({ assistantMessage, toolCall, args, result, isError, ctx }) │ │
|
||||
│ │ - Can override result content, details, usage, terminate hint │ │
|
||||
│ │ │ │
|
||||
│ │ Step 8.2: Create Tool Result Message │ │
|
||||
│ │ { │ │
|
||||
│ │ role: "toolResult", │ │
|
||||
│ │ toolCallId: toolCall.id, │ │
|
||||
│ │ toolName: toolCall.name, │ │
|
||||
│ │ content: result.content ?? [], │ │
|
||||
│ │ details: result.details, │ │
|
||||
│ │ usage: result.usage, │ │
|
||||
│ │ isError: false, │ │
|
||||
│ │ timestamp: Date.now() │ │
|
||||
│ │ } │ │
|
||||
│ │ │ │
|
||||
│ │ Step 8.3: Emit Events │ │
|
||||
│ │ • tool_execution_start │ │
|
||||
│ │ • tool_execution_end │ │
|
||||
│ │ • message_start (toolResult message) │ │
|
||||
│ │ • message_end (toolResult message) │ │
|
||||
│ │ │ │
|
||||
│ │ Step 8.4: Update Context │ │
|
||||
│ │ • Push tool result message to currentContext.messages │ │
|
||||
│ │ • Push to newMessages array │ │
|
||||
│ │ │ │
|
||||
│ │ Step 8.5: Batch Termination Check │ │
|
||||
│ │ shouldTerminateToolBatch(finalizedCalls) │ │
|
||||
│ │ - Returns true if ALL tools set terminate: true │ │
|
||||
│ │ - If true, agent may stop after this batch │ │
|
||||
│ └──────────────────────────────────────────────────────────────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 9. NEXT TURN PREPARATION │
|
||||
│ File: packages/agent/src/agent-loop.ts:224-257 │
|
||||
│ ┌──────────────────────────────────────────────────────────────────────────────┐ │
|
||||
│ │ Step 9.1: Turn End Event │ │
|
||||
│ │ turn_end emitted with message and toolResults │ │
|
||||
│ │ │ │
|
||||
│ │ Step 9.2: prepareNextTurn Hook │ │
|
||||
│ │ prepareNextTurn({ message, toolResults, context, newMessages }) │ │
|
||||
│ │ - Can return updated context, model, or thinking level │ │
|
||||
│ │ - Used for dynamic context management │ │
|
||||
│ │ │ │
|
||||
│ │ Step 9.3: Queue Polling │ │
|
||||
│ │ getSteeringMessages() → inject messages for immediate processing │ │
|
||||
│ │ getFollowUpMessages() → check for queued follow-up messages │ │
|
||||
│ │ │ │
|
||||
│ │ Step 9.4: Loop Decision │ │
|
||||
│ │ • If steering messages exist → process them, continue loop │ │
|
||||
│ │ • If follow-up messages exist → process them, continue loop │ │
|
||||
│ │ • If tool calls remain in message → continue inner loop │ │
|
||||
│ │ • If no messages → emit agent_end, exit loop │ │
|
||||
│ └──────────────────────────────────────────────────────────────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 10. AGENT RESPONSE GENERATION │
|
||||
│ File: packages/agent/src/agent-loop.ts:58-163 │
|
||||
│ ┌──────────────────────────────────────────────────────────────────────────────┐ │
|
||||
│ │ The agent loop continues until: │ │
|
||||
│ │ • No tool calls remain in assistant messages │ │
|
||||
│ │ • No steering/follow-up messages queued │ │
|
||||
│ │ • shouldStopAfterTurn() returns true (if configured) │ │
|
||||
│ │ │ │
|
||||
│ │ Final Response Generation: │ │
|
||||
│ │ 1. LLM streams text content blocks │ │
|
||||
│ │ 2. Message committed to context │ │
|
||||
│ │ 3. turn_end emitted │ │
|
||||
│ │ 4. agent_end emitted with all new messages │ │
|
||||
│ │ 5. Agent returns to idle state │ │
|
||||
│ │ │ │
|
||||
│ │ Final Agent Response: │ │
|
||||
│ │ "We have 15 products in stock:" │ │
|
||||
│ │ • Product A - $29.99 │ │
|
||||
│ │ • Product B - $49.99 │ │
|
||||
│ │ • Product C - $19.99 │ │
|
||||
│ │ (and 2 more products) │ │
|
||||
│ └──────────────────────────────────────────────────────────────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────────────────────────┘
|
||||
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Complete Event Flow Diagram
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ COMPLETE EVENT SEQUENCE FOR A TURN │
|
||||
│ WITH TOOL USE (Product Catalog Query) │
|
||||
└─────────────────────────────────────────────────────────────────────────────────────┘
|
||||
|
||||
agent_start
|
||||
↓
|
||||
turn_start
|
||||
↓
|
||||
message_start (user message: "what product do you have in stock?")
|
||||
message_end (user message)
|
||||
↓
|
||||
message_start (assistant message - streaming from LLM)
|
||||
message_update (text delta: "We have")
|
||||
message_update (toolcall delta: {"name":"read","arguments":{...}})
|
||||
message_update (toolcall end)
|
||||
message_end (assistant message with tool calls)
|
||||
↓
|
||||
tool_execution_start (tool call: read products.json)
|
||||
tool_execution_end (tool call: read complete)
|
||||
message_start (toolResult message)
|
||||
message_end (toolResult message)
|
||||
↓
|
||||
turn_end
|
||||
↓
|
||||
[Inner loop continues: send tool result to LLM]
|
||||
↓
|
||||
turn_start
|
||||
↓
|
||||
message_start (assistant message - streaming from LLM)
|
||||
message_update (text delta: "We have 15 products in stock:")
|
||||
message_update (text delta: "• Product A - $29.99")
|
||||
message_update (text delta: "• Product B - $49.99")
|
||||
message_end (final assistant message)
|
||||
↓
|
||||
turn_end
|
||||
↓
|
||||
agent_end
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Key Files Summary
|
||||
|
||||
| Component | File | Purpose |
|
||||
|-----------|------|---------|
|
||||
| Agent class | `packages/agent/src/agent.ts` | Stateful wrapper, event emission, queue management |
|
||||
| Agent loop | `packages/agent/src/agent-loop.ts` | Core loop, LLM calls, tool execution |
|
||||
| Message types | `packages/agent/src/types.ts` | AgentMessage, AgentTool, AgentEvent definitions |
|
||||
| Harness | `packages/agent/src/harness/agent-harness.ts` | Session integration, hooks, persistence |
|
||||
| Read tool | `packages/agent/src/harness/tools/read.ts` | File reading implementation |
|
||||
| Stream function | `packages/agent/src/stream-fn.ts` | Default stream function management |
|
||||
| Types | `packages/agent/src/types.ts` | All type definitions |
|
||||
|
||||
---
|
||||
|
||||
## Hook Points for Customization
|
||||
|
||||
The agent supports multiple extension points:
|
||||
|
||||
| Hook | Location | Purpose |
|
||||
|------|----------|---------|
|
||||
| `convertToLlm` | `agent.ts:99` | Transform messages before LLM call |
|
||||
| `transformContext` | `agent.ts:100` | Modify context (pruning, injection) |
|
||||
| `beforeToolCall` | `agent.ts:105` | Block or modify tool execution |
|
||||
| `afterToolCall` | `agent.ts:106` | Override tool results |
|
||||
| `prepareNextTurn` | `agent.ts:107` | Dynamic context/model updates |
|
||||
| `shouldStopAfterTurn` | `agent-loop.ts` | Graceful termination |
|
||||
| `getSteeringMessages` | `agent.ts:114` | Inject messages mid-turn |
|
||||
| `getFollowUpMessages` | `agent.ts:115` | Queue follow-up messages |
|
||||
|
||||
---
|
||||
|
||||
## Tool Execution Flow
|
||||
|
||||
```
|
||||
Tool Call Received from LLM
|
||||
↓
|
||||
1. Tool Lookup (find by name)
|
||||
↓
|
||||
2. prepareArguments? (transform if defined)
|
||||
↓
|
||||
3. validateToolArguments (JSON Schema)
|
||||
↓
|
||||
4. beforeToolCall hook (can block)
|
||||
↓
|
||||
5. execute (parallel or sequential)
|
||||
↓
|
||||
6. onUpdate (stream partial results)
|
||||
↓
|
||||
7. afterToolCall hook (can override)
|
||||
↓
|
||||
8. Create toolResult message
|
||||
↓
|
||||
9. Emit events (start, end)
|
||||
↓
|
||||
10. Add to context.messages
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Example: "What product do you have in stock?"
|
||||
|
||||
### Step-by-Step Execution:
|
||||
|
||||
1. **User sends message**
|
||||
```
|
||||
"what product do you have in stock?"
|
||||
```
|
||||
|
||||
2. **Agent normalizes input**
|
||||
```typescript
|
||||
[{
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "what product do you have in stock?" }],
|
||||
timestamp: Date.now()
|
||||
}]
|
||||
```
|
||||
|
||||
3. **LLM processes and decides to use `read` tool**
|
||||
```json
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [{
|
||||
"type": "toolCall",
|
||||
"name": "read",
|
||||
"arguments": {
|
||||
"path": "products.json",
|
||||
"offset": 1,
|
||||
"limit": 50
|
||||
},
|
||||
"id": "tool_abc123"
|
||||
}]
|
||||
}
|
||||
```
|
||||
|
||||
4. **Tool execution**
|
||||
- Read `products.json` (150 lines total)
|
||||
- Return lines 1-50 with truncation notice
|
||||
- Add to context as toolResult
|
||||
|
||||
5. **LLM generates final response**
|
||||
```
|
||||
We have 15 products in stock:
|
||||
|
||||
• Product A - $29.99
|
||||
• Product B - $49.99
|
||||
• Product C - $19.99
|
||||
• Product D - $99.99
|
||||
• Product E - $149.99
|
||||
(and 10 more products)
|
||||
|
||||
Use offset=51 to continue viewing.
|
||||
```
|
||||
|
||||
6. **Agent emits final response to user**
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
The agent workflow follows a clear pattern:
|
||||
|
||||
1. **Message Input** → Normalize and validate
|
||||
2. **Context Setup** → Create snapshot with system prompt, messages, tools
|
||||
3. **LLM Call** → Transform messages, resolve API key, stream response
|
||||
4. **Tool Detection** → Check for tool calls in assistant message
|
||||
5. **Tool Execution** → Validate, hook, execute, stream updates
|
||||
6. **Result Handling** → Create toolResult message, emit events
|
||||
7. **Next Turn** → Check for steering/follow-up messages, prepare context
|
||||
8. **Response Generation** → Continue until no more tool calls needed
|
||||
9. **Completion** → Emit final response to user
|
||||
|
||||
The entire flow is event-driven, allowing for real-time updates and hook-based customization at every step.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,624 +0,0 @@
|
||||
# Memory & Context Management in Pi Agent
|
||||
|
||||
## 1. Architecture Overview
|
||||
|
||||
The agent manages memory at **two layers**:
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Agent Harness │
|
||||
│ ┌──────────────┐ ┌──────────────┐ ┌─────────────────────┐ │
|
||||
│ │ Session │ │ Compaction │ │ Branch Navigation │ │
|
||||
│ │ (JSONL tree)│ │ (summarize) │ │ (reset + summarize)│ │
|
||||
│ └──────┬───────┘ └──────────────┘ └─────────────────────┘ │
|
||||
└─────────┼───────────────────────────────────────────────────────┘
|
||||
│ builds context
|
||||
┌─────────▼───────────────────────────────────────────────────────┐
|
||||
│ Agent Class │
|
||||
│ ┌───────────────────────────────────────────────────────────┐ │
|
||||
│ │ _state.messages: AgentMessage[] (linear transcript) │ │
|
||||
│ │ _state.tools, systemPrompt, model │ │
|
||||
│ └───────────────────────────────────────────────────────────┘ │
|
||||
│ subscribe() → events → UI updates │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**Key separation:**
|
||||
- **In-memory** (`Agent`): linear transcript for the current run. Cleared on `reset()`.
|
||||
- **On-disk** (`Session`): persistent tree of entries in JSONL files. Survives restarts.
|
||||
- **Compaction**: replaces old on-disk history with an LLM-generated summary, controlling context window usage.
|
||||
|
||||
---
|
||||
|
||||
## 2. Data Flow: From Prompt to LLM Call
|
||||
|
||||
```
|
||||
agent.prompt("Read README.md")
|
||||
│
|
||||
▼
|
||||
┌──────────────────────────┐
|
||||
│ normalizePromptInput() │ → { role: "user", content: "..." }
|
||||
└──────────┬───────────────┘
|
||||
│
|
||||
▼
|
||||
┌──────────────────────────┐
|
||||
│ runWithLifecycle() │ → sets isStreaming=true
|
||||
│ runAgentLoop() │
|
||||
└──────────┬───────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ runLoop() — the main while(true) loop │
|
||||
│ │
|
||||
│ Inner loop: │
|
||||
│ 1. Inject steering/follow-up messages │
|
||||
│ 2. streamAssistantResponse() │
|
||||
│ ┌─────────────────────────────────────────────┐ │
|
||||
│ │ transformContext() │ │
|
||||
│ │ AgentMessage[] → AgentMessage[] │ │
|
||||
│ │ (prune, inject external context) │ │
|
||||
│ └──────────────┬──────────────────────────────┘ │
|
||||
│ ▼ │
|
||||
│ ┌─────────────────────────────────────────────┐ │
|
||||
│ │ convertToLlm() │ │
|
||||
│ │ AgentMessage[] → Message[] │ │
|
||||
│ │ (filter to user/assistant/toolResult only) │ │
|
||||
│ └──────────────┬──────────────────────────────┘ │
|
||||
│ ▼ │
|
||||
│ ┌─────────────────────────────────────────────┐ │
|
||||
│ │ streamFunction() → LLM provider │ │
|
||||
│ │ { systemPrompt, messages, tools } │ │
|
||||
│ └──────────────┬──────────────────────────────┘ │
|
||||
│ ▼ │
|
||||
│ Stream events: start → delta* → done │
|
||||
│ ▼ │
|
||||
│ Return AssistantMessage │
|
||||
│ 3. Extract toolCall blocks │
|
||||
│ 4. If toolCalls: executeToolCalls() │
|
||||
│ → create toolResult messages │
|
||||
│ → append to context │
|
||||
│ 5. Check shouldStopAfterTurn / prepareNextTurn │
|
||||
│ 6. Check steering/follow-up queues │
|
||||
│ → Loop if more tool calls or queued messages │
|
||||
│ │
|
||||
│ Outer loop: │
|
||||
│ → Check follow-up queue for messages after agent would │
|
||||
│ stop │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. Context Building (On-Disk → In-Memory)
|
||||
|
||||
The `AgentHarness` bridges on-disk session data to the in-memory agent loop.
|
||||
|
||||
```
|
||||
session.buildContext()
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ getBranch() — walk from leaf → root via parentId │
|
||||
│ │
|
||||
│ Tree structure: │
|
||||
│ │
|
||||
│ ┌──────┐ ┌──────┐ ┌──────────┐ ┌──────────┐ │
|
||||
│ │msg 1 │───▶│msg 2 │───▶│ msg 3 │───▶│ msg 4 │ │
|
||||
│ │user │ │assist│ │ toolCall │ │ user │ │
|
||||
│ └──────┘ └──────┘ └──────────┘ └──────────┘ │
|
||||
│ │
|
||||
│ Path to root: [msg1, msg2, msg3, msg4] │
|
||||
└──────────┬────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ defaultContextEntryTransform() — THE KEY STEP │
|
||||
│ │
|
||||
│ Finds latest "compaction" entry in path: │
|
||||
│ │
|
||||
│ ┌──────┐ ┌──────────┐ ┌──────┐ ┌──────┐ │
|
||||
│ │msg 1 │ │compaction│ │msg 3 │ │msg 4 │ │
|
||||
│ │user │ │summary X │ │msg 2 │ │assist│ │
|
||||
│ └──────┘ └──────────┘ └──────┘ └──────┘ │
|
||||
│ │ │ │ │ │
|
||||
│ ├──────────────┤ │ │ │
|
||||
│ │ SKIPPED │ │ │ │
|
||||
│ │ (summarized)│ │ │ │
|
||||
│ └──────────────┼───────────┘ │ │
|
||||
│ ▼ ▼ │
|
||||
│ Include compaction entry Include entries after │
|
||||
│ + firstKeptEntryId the compaction point │
|
||||
│ │
|
||||
│ Result: [compaction, msg3, msg4] │
|
||||
│ → compaction entry becomes a "compactionSummary" message │
|
||||
└──────────┬────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ sessionEntryToContextMessages() │
|
||||
│ │
|
||||
│ For each entry: │
|
||||
│ message → [message] │
|
||||
│ compaction → [compactionSummary, ...retainedTail] │
|
||||
│ branch_summary → [branchSummaryMessage] │
|
||||
│ custom_message → [customMessage] │
|
||||
│ other → [] (omitted from LLM context) │
|
||||
│ │
|
||||
│ Flat result: [compactionSummary, msg3, msg4] │
|
||||
└──────────┬────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ deriveSessionContextState() │
|
||||
│ │
|
||||
│ Extracts from entries: │
|
||||
│ thinkingLevel ← latest thinking_level_change or assistant │
|
||||
│ model ← latest model_change or assistant │
|
||||
│ activeTools ← latest active_tools_change │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. Token Estimation
|
||||
|
||||
Before compaction can decide whether to trigger, it needs to know how many tokens the context uses.
|
||||
|
||||
```
|
||||
estimateContextTokens(messages)
|
||||
│
|
||||
├── Has provider-reported usage on last assistant msg?
|
||||
│ ├── YES → use actual usage + estimate tail
|
||||
│ │ (accurate — avoids compounding error)
|
||||
│ │
|
||||
│ └── NO → estimate all messages from scratch
|
||||
│
|
||||
▼
|
||||
estimateTokens(message) [character heuristic: chars / 4]
|
||||
│
|
||||
├── role === "user"
|
||||
│ content.length / 4
|
||||
│ (images ≈ 4800 chars each)
|
||||
│
|
||||
├── role === "assistant"
|
||||
│ sum of all content blocks:
|
||||
│ text blocks → text.length
|
||||
│ thinking blocks → thinking.length
|
||||
│ toolCall blocks → name.length + JSON.stringify(args).length
|
||||
│
|
||||
├── role === "toolResult" / "custom"
|
||||
│ content.length / 4
|
||||
│
|
||||
├── role === "bashExecution"
|
||||
│ (command.length + output.length) / 4
|
||||
│
|
||||
└── role === "compactionSummary" / "branchSummary"
|
||||
summary.length / 4
|
||||
```
|
||||
|
||||
**Why `chars / 4`?** Rough heuristic: ~4 ASCII characters ≈ 1 token. Conservative estimate to avoid under-counting.
|
||||
|
||||
---
|
||||
|
||||
## 5. Compaction — The Core Memory Management
|
||||
|
||||
### 5.1 Trigger Condition
|
||||
|
||||
```
|
||||
shouldCompact(contextTokens, contextWindow, settings)
|
||||
→ contextTokens > contextWindow - reserveTokens
|
||||
|
||||
Defaults:
|
||||
reserveTokens: 16384 (~16K tokens for summary prompt + output)
|
||||
keepRecentTokens: 20000 (~20K tokens of recent history to keep)
|
||||
|
||||
Example (Claude with 200K context window):
|
||||
Triggers when: contextTokens > 200000 - 16384 = 183616
|
||||
```
|
||||
|
||||
### 5.2 Finding the Cut Point
|
||||
|
||||
```
|
||||
findCutPoint(entries, startIndex, endIndex, keepRecentTokens)
|
||||
│
|
||||
│ Walk BACKWARD from endIndex
|
||||
│
|
||||
├── Accumulate estimated tokens per message
|
||||
├── Stop when accumulated ≥ keepRecentTokens
|
||||
├── Snap to nearest valid cut point
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ Valid cut points (safe to split): │
|
||||
│ - user message │
|
||||
│ - assistant message │
|
||||
│ - custom message │
|
||||
│ - branch_summary │
|
||||
│ │
|
||||
│ NOT valid (tool results stay with their call): │
|
||||
│ - toolResult message (skipped) │
|
||||
│ │
|
||||
│ Example: │
|
||||
│ │
|
||||
│ ┌────┐ ┌────┐ ┌────┐ ┌────┐ ┌────┐ ┌────┐ ┌────┐ │
|
||||
│ │ u1 │ │ a1 │ │ tr1│ │ u2 │ │ a2 │ │ tr2│ │ u3 │ │
|
||||
│ └────┘ └────┘ └────┘ └────┘ └────┘ └────┘ └────┘ │
|
||||
│ ▲ ▲ ▲ │
|
||||
│ │ │ │ │
|
||||
│ └── kept └── cut └── discarded │
|
||||
│ (~20K tokens) point history │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### 5.3 The Compaction Process
|
||||
|
||||
```
|
||||
prepareCompaction(branchEntries, settings)
|
||||
│
|
||||
├── Find previous compaction (if any) → previousSummary
|
||||
├── Estimate tokens of current context
|
||||
├── findCutPoint() → firstKeptEntryId
|
||||
│
|
||||
├── Split into 3 groups:
|
||||
│ │
|
||||
│ ├── messagesToSummarize: entries BEFORE cut point
|
||||
│ │ (these become the summary)
|
||||
│ │
|
||||
│ ├── retainedTail: entries AFTER cut point
|
||||
│ │ (these stay verbatim)
|
||||
│ │
|
||||
│ └── turnPrefixMessages: if cut splits a turn
|
||||
│ (the beginning of an interrupted turn)
|
||||
│
|
||||
└── Extract file operations from messagesToSummarize:
|
||||
→ readFiles, modifiedFiles
|
||||
|
||||
compact(preparation, model, models)
|
||||
│
|
||||
├── If isSplitTurn:
|
||||
│ ├── generateSummary(messagesToSummarize) → history summary
|
||||
│ ├── generateTurnPrefixSummary(turnPrefixMessages) → turn context
|
||||
│ └── Combine: history + "---" + turn prefix
|
||||
│
|
||||
├── Else (normal):
|
||||
│ ├── Has previousSummary?
|
||||
│ │ ├── YES → UPDATE_SUMMARIZATION_PROMPT (iterative)
|
||||
│ │ └── NO → SUMMARIZATION_PROMPT (fresh)
|
||||
│ └── Call LLM with conversation text + prompt
|
||||
│
|
||||
├── Append file operations:
|
||||
│ "Files read: [...]\nFiles modified: [...]"
|
||||
│
|
||||
└── Return:
|
||||
{
|
||||
summary: "## Goal...\n## Progress...\n...",
|
||||
firstKeptEntryId: "entry-uuid",
|
||||
tokensBefore: 185000,
|
||||
retainedTail: [msg3, msg4, ...],
|
||||
details: { readFiles: [...], modifiedFiles: [...] }
|
||||
}
|
||||
```
|
||||
|
||||
### 5.4 Summary Format
|
||||
|
||||
```
|
||||
The LLM generates a structured summary:
|
||||
|
||||
## Goal
|
||||
- [What is the user trying to accomplish?]
|
||||
|
||||
## Constraints & Preferences
|
||||
- [Any constraints, preferences, or requirements]
|
||||
|
||||
## Progress
|
||||
### Done
|
||||
- [x] [Completed tasks]
|
||||
### In Progress
|
||||
- [ ] [Current work]
|
||||
### Blocked
|
||||
- [Issues preventing progress]
|
||||
|
||||
## Key Decisions
|
||||
- **[Decision]**: [Brief rationale]
|
||||
|
||||
## Next Steps
|
||||
1. [Ordered list of what should happen next]
|
||||
|
||||
## Critical Context
|
||||
- [Any data, examples, or references needed to continue]
|
||||
|
||||
Files read: [src/index.ts, package.json]
|
||||
Files modified: [src/index.ts]
|
||||
```
|
||||
|
||||
### 5.5 Iterative Compaction
|
||||
|
||||
Successive compact calls **update** the existing summary rather than replacing it:
|
||||
|
||||
```
|
||||
Compaction 1 (at ~185K tokens):
|
||||
Summary: "## Goal: Build a login page..."
|
||||
firstKeptEntryId: "entry-003"
|
||||
|
||||
Compaction 2 (at ~185K tokens again):
|
||||
previousSummary: "## Goal: Build a login page..."
|
||||
→ UPDATE_SUMMARIZATION_PROMPT
|
||||
→ PRESERVES existing information
|
||||
→ ADDS new progress (move "In Progress" → "Done")
|
||||
→ NEW summary: "## Goal: Build a login page... Add OAuth..."
|
||||
firstKeptEntryId: "entry-003" (same boundary)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. Session Storage — JSONL Format
|
||||
|
||||
```
|
||||
Session file: .pi/sessions/--home-user--/2024-01-15T10-30-00_abc123.jsonl
|
||||
|
||||
Line 1 (header):
|
||||
{"type":"session","version":3,"id":"abc123","timestamp":"2024-01-15T10:30:00.000Z",
|
||||
"cwd":"/home/user/project","parentSession":"...","metadata":{}}
|
||||
|
||||
Line 2+ (entries, one per line):
|
||||
{"type":"message","id":"e001","parentId":null,"timestamp":"...","message":{...}}
|
||||
{"type":"message","id":"e002","parentId":"e001","timestamp":"...","message":{...}}
|
||||
{"type":"compaction","id":"e003","parentId":"e002","timestamp":"...",
|
||||
"summary":"## Goal: ...\n...","firstKeptEntryId":"e001",
|
||||
"tokensBefore":185000}
|
||||
{"type":"leaf","id":"e004","parentId":"e003","timestamp":"...",
|
||||
"targetId":"e002"}
|
||||
```
|
||||
|
||||
### Entry Types
|
||||
|
||||
| Type | LLM Context? | Purpose |
|
||||
|------|-------------|---------|
|
||||
| `message` | Yes | User, assistant, toolResult |
|
||||
| `compaction` | Yes (as summary message) | Replaces compacted history |
|
||||
| `branch_summary` | Yes (as summary message) | Summary of diverged branch |
|
||||
| `leaf` | No | Points to current tree leaf |
|
||||
| `thinking_level_change` | No | Tracking thinking level changes |
|
||||
| `model_change` | No | Tracking model changes |
|
||||
| `active_tools_change` | No | Tracking tool enable/disable |
|
||||
| `custom` | No (unless projector configured) | App-defined data |
|
||||
| `custom_message` | Yes | App-defined messages |
|
||||
| `label` | No | Human-readable labels |
|
||||
| `session_info` | No | Session name history |
|
||||
|
||||
---
|
||||
|
||||
## 7. Session Tree (Branching)
|
||||
|
||||
Sessions form a **tree**, not a linear log. This lets users "go back" and try a different approach.
|
||||
|
||||
```
|
||||
Session tree:
|
||||
|
||||
┌───[e01]───┐
|
||||
│ user: "a" │
|
||||
└─────┬──────┘
|
||||
▼
|
||||
┌──────────┐
|
||||
│ assist 1 │
|
||||
└─────┬────┘
|
||||
▼
|
||||
┌──────────┐
|
||||
│ toolCall │
|
||||
└─────┬────┘
|
||||
▼
|
||||
┌──────────┐
|
||||
│ toolRes 1│
|
||||
└─────┬────┘
|
||||
▼
|
||||
┌──────────┐
|
||||
│ user: "b"│ ← user goes back here
|
||||
└─────┬────┘
|
||||
│
|
||||
┌─────┴─────┐
|
||||
│ │
|
||||
┌──────────┐ ┌──────────┐
|
||||
│ user: "c" │ │ user: "d" │ ← branch point
|
||||
└────┬─────┘ └────┬─────┘
|
||||
│ │
|
||||
┌────▼─────┐ ┌────▼─────┐
|
||||
│ assist 2 │ │ assist 3 │ ← current leaf (d)
|
||||
└──────────┘ └──────────┘
|
||||
|
||||
When user navigates to "user: b":
|
||||
- Leaf moves from "d" back to "b"
|
||||
- Branch summary generated for diverged work ("c" → "assist 2")
|
||||
- New work branches from "b":
|
||||
┌─────┐
|
||||
│ user: "e" │ ← new branch
|
||||
└─────┬─────┘
|
||||
▼
|
||||
┌──────────┐
|
||||
│ assist 4 │
|
||||
└──────────┘
|
||||
|
||||
Context sent to LLM:
|
||||
[compaction summary, user:b, user:e, assist:4]
|
||||
→ The old "c"/"assist 2" branch is replaced by its summary
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 8. Pending Writes — Batching Session Persistence
|
||||
|
||||
To avoid writing every message individually during a run:
|
||||
|
||||
```
|
||||
Agent loop events
|
||||
│
|
||||
▼
|
||||
handleAgentEvent(event)
|
||||
│
|
||||
├── message_end → pendingSessionWrites.push({ type: "message", message })
|
||||
├── turn_end → flushPendingSessionWrites() (save_point)
|
||||
├── agent_end → flushPendingSessionWrites()
|
||||
│
|
||||
▼
|
||||
flushPendingSessionWrites()
|
||||
│
|
||||
├── Iterate pendingSessionWrites[]
|
||||
│ ├── message → session.appendMessage(msg)
|
||||
│ ├── model_change → session.appendModelChange(provider, id)
|
||||
│ ├── thinking_level_change → session.appendThinkingLevelChange(level)
|
||||
│ ├── active_tools_change → session.appendActiveToolsChange(names)
|
||||
│ ├── custom → session.appendCustomEntry(type, data)
|
||||
│ ├── custom_message → session.appendCustomMessageEntry(...)
|
||||
│ ├── label → session.appendLabel(targetId, label)
|
||||
│ ├── session_info → session.appendSessionName(name)
|
||||
│ └── leaf → session.getStorage().setLeafId(targetId)
|
||||
│
|
||||
└── Shift all writes → empty pending list
|
||||
```
|
||||
|
||||
During the run, messages are accumulated in `pendingSessionWrites` and only flushed to disk at `save_point` (end of each turn) or `agent_end`.
|
||||
|
||||
---
|
||||
|
||||
## 9. Hooks & Extensibility — Context Control Points
|
||||
|
||||
The harness exposes hooks at every memory management boundary:
|
||||
|
||||
```
|
||||
┌────────────────────────────────────────────────────────────────┐
|
||||
│ Hooks │
|
||||
├────────────────────────┬─────────────────────────────────────┤
|
||||
│ Hook │ When │
|
||||
├────────────────────────┼─────────────────────────────────────┤
|
||||
│ before_agent_start │ Before each prompt, can add │
|
||||
│ │ messages or modify system prompt │
|
||||
├────────────────────────┼─────────────────────────────────────┤
|
||||
│ context │ Before each LLM call, can prune/ │
|
||||
│ │ modify AgentMessage[] │
|
||||
├────────────────────────┼─────────────────────────────────────┤
|
||||
│ before_provider_request│ Before each provider API call, can │
|
||||
│ │ modify headers, retries, timeout │
|
||||
├────────────────────────┼─────────────────────────────────────┤
|
||||
│ before_provider_payload│ Before sending payload to provider,│
|
||||
│ │ can modify the request body │
|
||||
├────────────────────────┼─────────────────────────────────────┤
|
||||
│ after_provider_response│ After receiving response, for │
|
||||
│ │ logging/metrics │
|
||||
├────────────────────────┼─────────────────────────────────────┤
|
||||
│ tool_call │ Before tool execution, can │
|
||||
│ │ return { block: true } │
|
||||
├────────────────────────┼─────────────────────────────────────┤
|
||||
│ tool_result │ After tool execution, can override │
|
||||
│ │ content, details, isError, terminate│
|
||||
├────────────────────────┼─────────────────────────────────────┤
|
||||
│ session_before_compact│ Before compaction, can cancel or │
|
||||
│ │ provide custom compaction result │
|
||||
├────────────────────────┼─────────────────────────────────────┤
|
||||
│ session_before_tree │ Before branch navigation, can │
|
||||
│ │ cancel or provide custom summary │
|
||||
├────────────────────────┼─────────────────────────────────────┤
|
||||
│ prepareNextTurn │ Between turns, can replace context, │
|
||||
│ │ model, or thinkingLevel │
|
||||
├────────────────────────┼─────────────────────────────────────┤
|
||||
│ shouldStopAfterTurn │ After a turn, if true the loop │
|
||||
│ │ exits (agent_end, no more LLM calls)│
|
||||
└────────────────────────┴─────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 10. Complete Lifecycle: Long Session
|
||||
|
||||
```
|
||||
Session starts empty
|
||||
│
|
||||
▼
|
||||
Turn 1: "Create a React component"
|
||||
Context: [compaction summary (empty)]
|
||||
Messages exchanged: ~2K tokens
|
||||
└─ Session: [user1, assist1, toolCall, toolRes1, assist2]
|
||||
│
|
||||
▼
|
||||
Turn 2-10: Iterative development
|
||||
Context: growing with each turn
|
||||
Total context: ~50K tokens
|
||||
└─ Session: [user1..assist2, user2..assist20]
|
||||
│
|
||||
▼
|
||||
Turn 15: Context approaching limit (~170K tokens)
|
||||
shouldCompact() → true
|
||||
└─ Compaction 1:
|
||||
- Summarizes turns 1-12
|
||||
- Keeps turns 13-15 verbatim
|
||||
- Summary: "## Goal: React component ## Progress: built X, Y"
|
||||
│
|
||||
▼
|
||||
Turn 20: Context ~180K tokens
|
||||
shouldCompact() → true
|
||||
└─ Compaction 2 (iterative update):
|
||||
- Updates existing summary with new progress
|
||||
- "## Goal: React component ## Done: built X,Y ## New: added auth"
|
||||
│
|
||||
▼
|
||||
Turn 25: Context ~186K tokens → triggers compaction
|
||||
└─ Compaction 3:
|
||||
- Summary now covers ~22 turns of history
|
||||
- Retained tail: last 20K tokens (~5 turns)
|
||||
- Context window freed: ~186K → ~25K tokens
|
||||
│
|
||||
▼
|
||||
User navigates to Turn 8:
|
||||
- Branch summary generated for Turns 9-25
|
||||
- Leaf moves back to Turn 8
|
||||
- Context: [compaction, branch_summary, turns 1-8]
|
||||
│
|
||||
▼
|
||||
User continues from Turn 8:
|
||||
- New branch grows from Turn 8
|
||||
- Old branch (9-25) replaced by branch_summary
|
||||
│
|
||||
▼
|
||||
Session ends, JSONL file persists on disk
|
||||
Next session: loads from JSONL, rebuilds context
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 11. Token Budget Summary
|
||||
|
||||
```
|
||||
Example: Claude Sonnet (200K context window)
|
||||
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ Context Window: 200,000 tokens │
|
||||
├─────────────────────────────────────────────────────────────┤
|
||||
│ Reserved for summary: 16,384 tokens │
|
||||
├─────────────────────────────────────────────────────────────┤
|
||||
│ Keep recent: 20,000 tokens │
|
||||
├─────────────────────────────────────────────────────────────┤
|
||||
│ Max context before compaction: 183,616 tokens │
|
||||
│ (= 200000 - 16384) │
|
||||
├─────────────────────────────────────────────────────────────┤
|
||||
│ After compaction: ~25,000 tokens │
|
||||
│ (20,000 tail + ~5,000 summary) │
|
||||
│ → ~158,616 tokens freed │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 12. Key Files Reference
|
||||
|
||||
| File | Responsibility |
|
||||
|------|---------------|
|
||||
| `agent-loop.ts` | Core loop, tool execution, streaming |
|
||||
| `agent.ts` | Stateful `Agent` class, event system |
|
||||
| `agent-harness.ts` | High-level harness, hooks, session management |
|
||||
| `compaction/compaction.ts` | Token estimation, cut point, LLM summarization |
|
||||
| `compaction/branch-summarization.ts` | Branch divergence summarization |
|
||||
| `session/session.ts` | Session tree, entry appending, context building |
|
||||
| `session/jsonl-storage.ts` | JSONL file read/write |
|
||||
| `session/jsonl-repo.ts` | Session repo: create/open/list/delete/fork |
|
||||
| `types.ts` | All type definitions |
|
||||
| `messages.ts` | `convertToLlm()`, custom message helpers |
|
||||
| `system-prompt.ts` | System prompt building |
|
||||
| `skills.ts` | Skill management |
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@earendil-works/pi-agent-core",
|
||||
"version": "0.81.1",
|
||||
"version": "0.82.1",
|
||||
"description": "General-purpose agent with transport abstraction, state management, and attachment support",
|
||||
"type": "module",
|
||||
"main": "./dist/index.js",
|
||||
@@ -29,7 +29,7 @@
|
||||
"prepublishOnly": "npm run build"
|
||||
},
|
||||
"dependencies": {
|
||||
"@earendil-works/pi-ai": "^0.81.1",
|
||||
"@earendil-works/pi-ai": "^0.82.1",
|
||||
"diff": "8.0.4",
|
||||
"ignore": "7.0.5",
|
||||
"typebox": "1.1.38",
|
||||
|
||||
@@ -11,6 +11,7 @@ import {
|
||||
type SimpleStreamOptions,
|
||||
type TextContent,
|
||||
type Usage,
|
||||
uuidv7,
|
||||
} from "@earendil-works/pi-ai";
|
||||
import type { AgentMessage, ThinkingLevel } from "../../types.ts";
|
||||
import {
|
||||
@@ -122,7 +123,18 @@ export async function completeSimpleWithRetries(
|
||||
retry?: RetryPolicy,
|
||||
callbacks?: RetryCallbacks,
|
||||
): Promise<AssistantMessage> {
|
||||
return retryAssistantCall(() => models.completeSimple(model, context, options), retry, options.signal, callbacks);
|
||||
// Summaries are standalone requests, so isolate routing and avoid cache writes that cannot be reused.
|
||||
const requestOptions: SimpleStreamOptions = {
|
||||
...options,
|
||||
cacheRetention: "none",
|
||||
sessionId: uuidv7(),
|
||||
};
|
||||
return retryAssistantCall(
|
||||
() => models.completeSimple(model, context, requestOptions),
|
||||
retry,
|
||||
requestOptions.signal,
|
||||
callbacks,
|
||||
);
|
||||
}
|
||||
|
||||
function combineUsage(first: Usage, second: Usage): Usage {
|
||||
|
||||
@@ -605,6 +605,9 @@ describe("harness compaction", () => {
|
||||
getOrThrow(await compact(preparation, models, model));
|
||||
|
||||
expect(seenOptions.map((options) => options?.maxTokens)).toEqual([128000, 128000]);
|
||||
expect(seenOptions.map((options) => options?.cacheRetention)).toEqual(["none", "none"]);
|
||||
const sessionIds = seenOptions.map((options) => options?.sessionId);
|
||||
expect(sessionIds[0]).not.toBe(sessionIds[1]);
|
||||
});
|
||||
|
||||
it("returns compaction error results without throwing", async () => {
|
||||
|
||||
@@ -2,8 +2,44 @@
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.82.1] - 2026-07-25
|
||||
|
||||
### Added
|
||||
|
||||
- Added `ModelsStoreEntry.etag` so persisted provider catalogs can carry the remote ETag validator for conditional refreshes.
|
||||
- Added `ANTHROPIC_AUTH_TOKEN` bearer authentication for Anthropic-compatible gateways ([#5871](https://github.com/earendil-works/pi/issues/5871))
|
||||
- Added Claude Opus 5 support for Anthropic and Amazon Bedrock with adaptive thinking, inference profiles, prompt caching, and preserved AWS validation messages ([#7081](https://github.com/earendil-works/pi/pull/7081) by [@unexge](https://github.com/unexge), [#7083](https://github.com/earendil-works/pi/pull/7083) by [@davidbrai](https://github.com/davidbrai)).
|
||||
|
||||
### Changed
|
||||
|
||||
- Changed Radius OAuth device authorization, token exchange, and refresh requests to use the configured gateway directly.
|
||||
- Changed `ModelsError` messages to append the underlying cause, so auth failures such as `OAuth refresh failed for openai-codex` report the provider response instead of a bare wrapper message.
|
||||
|
||||
## [0.82.0] - 2026-07-24
|
||||
|
||||
### Breaking Changes
|
||||
|
||||
- Replaced `getBuiltinModelDataUrl(provider)` with `getBuiltinModelDataGeneratedAt()` so built-in catalog freshness uses its recorded generation time instead of installation-dependent file metadata ([#7016](https://github.com/earendil-works/pi/pull/7016) by [@davidbrai](https://github.com/davidbrai)).
|
||||
|
||||
### Added
|
||||
|
||||
- Added Kimi Code subscription OAuth login for the `kimi-coding` provider, with device authorization, token refresh, and OAuth host overrides ([#6935](https://github.com/earendil-works/pi/pull/6935) by [@zaycruz](https://github.com/zaycruz)).
|
||||
- Added OpenRouter OAuth PKCE login that mints a user-controlled API key for chat and image providers ([#6927](https://github.com/earendil-works/pi/pull/6927) by [@rsaryev](https://github.com/rsaryev)).
|
||||
- Added `Tool.constrainedSampling` with strict JSON Schema (`prefer`/`require`) and OpenAI Lark/regex grammar variants, enforcing provider-side constrained tool sampling across OpenAI, Anthropic, Amazon Bedrock, Google Gemini, and Mistral. See [Constrained Sampling for Tools](README.md#constrained-sampling-for-tools).
|
||||
- Added `supportsGrammarTools` and `supportsStrictTools` compatibility flags, expanded `supportsStrictMode` to Responses and Bedrock models, and generated model capability metadata to gate constrained sampling.
|
||||
|
||||
### Changed
|
||||
|
||||
- Changed generated model catalogs to expose only provider-verified reasoning effort levels from models.dev ([#6928](https://github.com/earendil-works/pi/pull/6928) by [@davidbrai](https://github.com/davidbrai)).
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed OpenAI Codex cached WebSocket continuations after grammar tool calls to send only the real tool-result delta.
|
||||
- Fixed constrained tool sampling across Google, Amazon Bedrock, Mistral, and Azure OpenAI Responses adapters, including model-aware strict-tool capabilities, grammar configuration validation, and malformed grammar-call replay errors.
|
||||
- Fixed `cacheRetention: "none"` to disable implicit prompt-cache writes for supported OpenAI models and session-based caching for OpenAI Codex ([#6618](https://github.com/earendil-works/pi/pull/6618) by [@tmustier](https://github.com/tmustier)).
|
||||
- Fixed DNS lookup failures such as `getaddrinfo`, `ENOTFOUND`, and `EAI_AGAIN` to trigger automatic assistant retries ([#6946](https://github.com/earendil-works/pi/pull/6946) by [@christianklotz](https://github.com/christianklotz)).
|
||||
- Fixed OpenAI Codex WebSocket sessions to retry once without a missing previous-response continuation after `previous_response_not_found` errors ([#6955](https://github.com/earendil-works/pi/pull/6955) by [@davidbrai](https://github.com/davidbrai)).
|
||||
- Fixed OpenAI and Anthropic provider retry waits to honor abort signals and configured delay limits ([#6980](https://github.com/earendil-works/pi/pull/6980) by [@petrroll](https://github.com/petrroll)).
|
||||
- Fixed OpenRouter Anthropic cache breakpoints to advance through tool results and enabled cache control for `~anthropic/*-latest` aliases ([#6941](https://github.com/earendil-works/pi/pull/6941) by [@mteam88](https://github.com/mteam88)).
|
||||
|
||||
## [0.81.1] - 2026-07-21
|
||||
@@ -24,7 +60,6 @@
|
||||
- Added `contentText` for extracting joined text from message content ([#6840](https://github.com/earendil-works/pi/pull/6840) by [@xl0](https://github.com/xl0)).
|
||||
- Added a shared `uuidv7` utility for time-ordered identifiers ([#6834](https://github.com/earendil-works/pi/pull/6834) by [@xl0](https://github.com/xl0)).
|
||||
- Added optional usage metadata to tool result messages ([#6671](https://github.com/earendil-works/pi/pull/6671) by [@davidbrai](https://github.com/davidbrai)).
|
||||
- Added Kimi Code subscription OAuth login (device authorization grant) for the `kimi-coding` provider, with token refresh and `KIMI_CODE_OAUTH_HOST`/`KIMI_OAUTH_HOST` host overrides.
|
||||
|
||||
### Changed
|
||||
|
||||
@@ -167,6 +202,7 @@
|
||||
### Added
|
||||
|
||||
- Added OpenAI GPT-5.6 model metadata for `gpt-5.6`, `gpt-5.6-sol`, `gpt-5.6-terra`, and `gpt-5.6-luna`, plus verified `openai-codex` support for `gpt-5.6-sol`, `gpt-5.6-terra`, and `gpt-5.6-luna`.
|
||||
- Added provider-side constrained sampling for tools via `Tool.constrainedSampling`: strict JSON-schema enforcement for OpenAI and Anthropic tool calls, and OpenAI custom grammar tools (Lark/regex). Grammar tool capability comes from the model catalog's `supportsGrammarTools` compat flag, enabled for GPT-5+ models on OpenAI, OpenAI Codex, Azure OpenAI, GitHub Copilot, opencode, and Cloudflare AI Gateway ([#6341](https://github.com/earendil-works/pi/pull/6341)).
|
||||
- Refreshed generated model catalogs from models.dev, adding newly listed models including Kimi K2.7 Code for GitHub Copilot and Fable 5 to several providers ([#6256](https://github.com/earendil-works/pi/issues/6256)).
|
||||
- Added Claude Sonnet 5 to the GitHub Copilot model catalog ([#6200](https://github.com/earendil-works/pi/issues/6200)).
|
||||
- Added zstd request-body compression for the OpenAI Codex Responses SSE transport. Requests are sent with `Content-Encoding: zstd` when Node/Bun zstd support is available; the WebSocket transport is unchanged.
|
||||
|
||||
+38
-1
@@ -478,6 +478,40 @@ const bookMeetingTool: Tool = {
|
||||
};
|
||||
```
|
||||
|
||||
### Constrained Sampling for Tools
|
||||
|
||||
Tools can opt in to provider-side constrained sampling. For JSON-schema tools, `strict: 'prefer'` uses provider-side strict schema enforcement when supported and otherwise falls back to normal tool calling. `strict: 'require'` fails the request when the active provider/model cannot honor it. Set `constrainedSampling: false` to explicitly opt out; it behaves the same as omitting the field.
|
||||
|
||||
```typescript
|
||||
const strictTool: Tool = {
|
||||
name: 'edit_file',
|
||||
description: 'Edit a file',
|
||||
parameters: Type.Object({
|
||||
path: Type.String(),
|
||||
content: Type.String()
|
||||
}, { additionalProperties: false }),
|
||||
constrainedSampling: { type: 'json_schema', strict: 'prefer' }
|
||||
};
|
||||
```
|
||||
|
||||
Strict JSON-schema constrained sampling is supported for OpenAI, Anthropic, supported Amazon Bedrock Converse models, Mistral, and Gemini 3 tool calls through the Google Generative AI and Vertex adapters. Google uses `VALIDATED` function-calling mode (or `ANY` when explicitly requested); earlier Gemini versions fall back for `strict: 'prefer'` and reject `strict: 'require'` because they do not enforce required parameters. Bedrock strict-tool capability is generated from model structured-output metadata; custom Bedrock models can override `compat.supportsStrictMode`. OpenAI Responses and Chat Completions can also emit grammar-constrained custom tools with OpenAI Lark or regex grammar variants. If multiple OpenAI variants are supplied, Lark is preferred over regex. Grammar constraints are enforced when the active model supports grammar tools; otherwise the tool falls back to normal function/JSON-schema handling. Grammar tool capability is model metadata: the generated catalog sets `compat.supportsOpenAIGrammarTools` for GPT-5+ models on endpoints that pass OpenAI custom tools through (OpenAI, OpenAI Codex, Azure OpenAI Responses, GitHub Copilot, opencode, and Cloudflare AI Gateway). OpenAI rejects `type: "custom"` tools for pre-GPT-5 models, and gateways that normalize tool schemas (e.g. OpenRouter) mangle them, so the flag stays off elsewhere. Custom model definitions can opt in via `compat`. Grammar-capable models reject grammar configurations without a non-empty supported variant. Native grammar tools must have an object parameter schema with exactly one required string property:
|
||||
|
||||
```typescript
|
||||
const patchTool: Tool = {
|
||||
name: 'apply_patch',
|
||||
description: 'Apply a patch',
|
||||
parameters: Type.Object({
|
||||
input: Type.String()
|
||||
}, { additionalProperties: false }),
|
||||
constrainedSampling: {
|
||||
type: 'grammar',
|
||||
variants: {
|
||||
openai_lark: 'start: /.+/s'
|
||||
}
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
### Handling Tool Calls
|
||||
|
||||
Tool results use content blocks and can include both text and images:
|
||||
@@ -1124,6 +1158,7 @@ interface OpenAICompletionsCompat {
|
||||
supportsReasoningEffort?: boolean; // Whether provider supports `reasoning_effort` (default: true)
|
||||
supportsUsageInStreaming?: boolean; // Whether provider supports `stream_options: { include_usage: true }` (default: true)
|
||||
supportsStrictMode?: boolean; // Whether provider supports `strict` in tool definitions (default: true)
|
||||
supportsOpenAIGrammarTools?: boolean; // Whether to emit OpenAI custom Lark/regex grammar tools; false falls back to normal function tools (default: false; the generated catalog enables it for capable models)
|
||||
sendSessionAffinityHeaders?: boolean; // Send session-affinity data from `sessionId` (default: false)
|
||||
sessionAffinityFormat?: 'openai' | 'openai-nosession' | 'openrouter'; // Format for session affinity: 'openai' uses `prompt_cache_key`, `session_id`, `x-client-request-id`, and `x-session-affinity`; 'openai-nosession' uses `prompt_cache_key`, `x-client-request-id`, and `x-session-affinity`; 'openrouter' uses `x-session-id` (default: auto-detected)
|
||||
maxTokensField?: 'max_completion_tokens' | 'max_tokens'; // Which field name to use (default: max_completion_tokens)
|
||||
@@ -1142,6 +1177,8 @@ interface OpenAIResponsesCompat {
|
||||
supportsDeveloperRole?: boolean; // Whether provider supports `developer` role vs `system` (default: true)
|
||||
sessionAffinityFormat?: 'openai' | 'openai-nosession' | 'openrouter'; // Session-affinity header format: 'openai' sends `session_id` and `x-client-request-id`; 'openai-nosession' sends `x-client-request-id`; 'openrouter' sends `x-session-id`. Does not affect the `prompt_cache_key` body param (default: auto-detected)
|
||||
supportsLongCacheRetention?: boolean; // Whether provider supports `prompt_cache_retention: "24h"` (default: true)
|
||||
supportsStrictMode?: boolean; // Whether provider supports strict JSON-schema function tools (default: false; enabled in metadata for built-in OpenAI models)
|
||||
supportsOpenAIGrammarTools?: boolean; // Whether to emit OpenAI custom Lark/regex grammar tools; false falls back to normal function tools (default: false; the generated catalog enables it for capable models)
|
||||
}
|
||||
```
|
||||
|
||||
@@ -1500,7 +1537,7 @@ Built-in login and refresh flows are private provider implementations. Use provi
|
||||
|
||||
Provider notes:
|
||||
|
||||
**OpenAI Codex**: Requires a ChatGPT Plus or Pro subscription. Provides access to GPT-5.x Codex models with extended context windows and reasoning capabilities. The library automatically handles session-based prompt caching when `sessionId` is provided in stream options. You can set `transport` in stream options to `"sse"`, `"websocket"`, or `"auto"` for Codex Responses transport selection. When using WebSocket with a `sessionId`, connections are reused per session and expire after 5 minutes of inactivity.
|
||||
**OpenAI Codex**: Requires a ChatGPT Plus or Pro subscription. Provides access to GPT-5.x Codex models with extended context windows and reasoning capabilities. The library automatically handles session-based prompt caching when `sessionId` is provided in stream options unless `cacheRetention` is `"none"`. You can set `transport` in stream options to `"sse"`, `"websocket"`, or `"auto"` for Codex Responses transport selection. When using WebSocket with a `sessionId` and cache retention enabled, connections are reused per session and expire after 5 minutes of inactivity.
|
||||
|
||||
**Azure OpenAI (Responses)**: Uses the Responses API only. Set `AZURE_OPENAI_API_KEY` and either `AZURE_OPENAI_BASE_URL` or `AZURE_OPENAI_RESOURCE_NAME`. `AZURE_OPENAI_BASE_URL` supports both `https://<resource>.openai.azure.com` and `https://<resource>.cognitiveservices.azure.com`; root endpoints are normalized to `.../openai/v1` automatically. Use `AZURE_OPENAI_API_VERSION` (defaults to `v1`) to override the API version if needed. Deployment names are treated as model IDs by default, override with `azureDeploymentName` or `AZURE_OPENAI_DEPLOYMENT_NAME_MAP` using comma-separated `model-id=deployment` pairs (for example `gpt-4o-mini=my-deployment,gpt-4o=prod`). Legacy deployment-based URLs are intentionally unsupported.
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@earendil-works/pi-ai",
|
||||
"version": "0.81.1",
|
||||
"version": "0.82.1",
|
||||
"description": "Unified LLM API with automatic model discovery and provider configuration",
|
||||
"type": "module",
|
||||
"main": "./dist/index.js",
|
||||
|
||||
@@ -83,6 +83,7 @@ interface ModelsDevModel {
|
||||
id: string;
|
||||
name: string;
|
||||
tool_call?: boolean;
|
||||
structured_output?: boolean;
|
||||
reasoning?: boolean;
|
||||
reasoning_options?: ModelsDevReasoningOption[];
|
||||
limit?: {
|
||||
@@ -291,6 +292,7 @@ const ANT_LING_RING_THINKING_LEVEL_MAP = {
|
||||
xhigh: "xhigh",
|
||||
} as const;
|
||||
|
||||
const BEDROCK_INFERENCE_PROFILE_ONLY_MODEL_IDS = new Set(["anthropic.claude-opus-5"]);
|
||||
const MODELS_DEV_OPENAI_UNSUPPORTED_MODEL_IDS = new Set(["gpt-5.6"]);
|
||||
const OPENAI_TOOL_SEARCH_MODEL_IDS = new Set([
|
||||
"gpt-5.4",
|
||||
@@ -485,6 +487,8 @@ function isAnthropicAdaptiveThinkingModel(modelId: string): boolean {
|
||||
modelId.includes("opus-4.7") ||
|
||||
modelId.includes("opus-4-8") ||
|
||||
modelId.includes("opus-4.8") ||
|
||||
modelId.includes("opus-5") ||
|
||||
modelId.includes("opus.5") ||
|
||||
modelId.includes("sonnet-4-6") ||
|
||||
modelId.includes("sonnet-4.6") ||
|
||||
modelId.includes("sonnet-5") ||
|
||||
@@ -495,7 +499,14 @@ function isAnthropicAdaptiveThinkingModel(modelId: string): boolean {
|
||||
|
||||
function isAnthropicTemperatureUnsupportedModel(modelId: string): boolean {
|
||||
const id = modelId.toLowerCase();
|
||||
return id.includes("opus-4-7") || id.includes("opus-4.7") || id.includes("opus-4-8") || id.includes("opus-4.8");
|
||||
return (
|
||||
id.includes("opus-4-7") ||
|
||||
id.includes("opus-4.7") ||
|
||||
id.includes("opus-4-8") ||
|
||||
id.includes("opus-4.8") ||
|
||||
id.includes("opus-5") ||
|
||||
id.includes("opus.5")
|
||||
);
|
||||
}
|
||||
|
||||
const OPENAI_COMPLETIONS_DEFAULT_COMPAT = {
|
||||
@@ -514,6 +525,7 @@ const OPENAI_COMPLETIONS_DEFAULT_COMPAT = {
|
||||
chatTemplateKwargs: {},
|
||||
zaiToolStream: false,
|
||||
supportsStrictMode: true,
|
||||
supportsOpenAIGrammarTools: false,
|
||||
sendSessionAffinityHeaders: false,
|
||||
supportsLongCacheRetention: true,
|
||||
} satisfies Required<Omit<OpenAICompletionsCompat, "cacheControlFormat" | "deferredToolsMode">> & {
|
||||
@@ -602,6 +614,7 @@ function detectOpenAICompletionsCompat(model: Model<"openai-completions">): Open
|
||||
chatTemplateKwargs: {},
|
||||
zaiToolStream: false,
|
||||
supportsStrictMode: !isMoonshot && !isTogether && !isCloudflareAiGateway && !isNvidia,
|
||||
supportsOpenAIGrammarTools: false,
|
||||
...(cacheControlFormat ? { cacheControlFormat } : {}),
|
||||
sendSessionAffinityHeaders: false,
|
||||
supportsLongCacheRetention: !(
|
||||
@@ -643,6 +656,39 @@ function applyOpenAICompletionsCompatMetadata(model: Model<Api>): void {
|
||||
}
|
||||
}
|
||||
|
||||
function applyStrictToolCompatMetadata(model: Model<Api>): void {
|
||||
if (model.provider === "openai" && model.api === "openai-responses") {
|
||||
model.compat = { ...(model.compat as OpenAIResponsesCompat | undefined), supportsStrictMode: true };
|
||||
} else if (model.provider === "anthropic" && model.api === "anthropic-messages") {
|
||||
mergeAnthropicMessagesCompat(model, { supportsStrictTools: true });
|
||||
}
|
||||
}
|
||||
|
||||
// Responses endpoints verified (OpenAI, ChatGPT Codex backend, GitHub Copilot,
|
||||
// opencode zen) or documented (Azure OpenAI, Cloudflare AI Gateway) to pass
|
||||
// OpenAI custom grammar tools through. OpenAI rejects `type: "custom"` tools
|
||||
// for pre-GPT-5 models (gpt-4.x, gpt-4o, o-series).
|
||||
const OPENAI_GRAMMAR_TOOL_PROVIDERS = new Set([
|
||||
"openai",
|
||||
"openai-codex",
|
||||
"azure-openai-responses",
|
||||
"github-copilot",
|
||||
"opencode",
|
||||
"cloudflare-ai-gateway",
|
||||
]);
|
||||
const OPENAI_GRAMMAR_TOOL_APIS = new Set<Api>([
|
||||
"openai-responses",
|
||||
"azure-openai-responses",
|
||||
"openai-codex-responses",
|
||||
]);
|
||||
|
||||
function applyOpenAIGrammarToolCompatMetadata(model: Model<Api>): void {
|
||||
if (!OPENAI_GRAMMAR_TOOL_APIS.has(model.api) || !OPENAI_GRAMMAR_TOOL_PROVIDERS.has(model.provider)) return;
|
||||
const match = /^gpt-(\d+)/.exec(model.id);
|
||||
if (!match || Number(match[1]) < 5) return;
|
||||
model.compat = { ...(model.compat as OpenAIResponsesCompat | undefined), supportsOpenAIGrammarTools: true };
|
||||
}
|
||||
|
||||
function applyOpenAIToolSearchMetadata(model: Model<Api>): void {
|
||||
const isOpenAIResponses = model.provider === "openai" && model.api === "openai-responses";
|
||||
const isOpenAICodex = model.provider === "openai-codex" && model.api === "openai-codex-responses";
|
||||
@@ -653,6 +699,18 @@ function applyOpenAIToolSearchMetadata(model: Model<Api>): void {
|
||||
};
|
||||
}
|
||||
|
||||
// OpenAI charges prompt-cache writes starting with the GPT-5.6 family, and exactly
|
||||
// those models accept `prompt_cache_options`; older models reject the parameter.
|
||||
// https://developers.openai.com/api/docs/guides/prompt-caching
|
||||
function applyOpenAIExplicitPromptCacheMetadata(model: Model<Api>): void {
|
||||
if (model.provider !== "openai" || model.api !== "openai-responses") return;
|
||||
if (!(model.cost.cacheWrite > 0)) return;
|
||||
model.compat = {
|
||||
...(model.compat as OpenAIResponsesCompat | undefined),
|
||||
supportsExplicitPromptCacheMode: true,
|
||||
};
|
||||
}
|
||||
|
||||
function isGemini3ProModel(modelId: string): boolean {
|
||||
return /gemini-3(?:\.\d+)?-pro/.test(modelId.toLowerCase());
|
||||
}
|
||||
@@ -700,7 +758,7 @@ function applyThinkingLevelMetadata(model: Model<any>): void {
|
||||
}
|
||||
// Anthropic adaptive-thinking effort support (per Anthropic adaptive thinking docs):
|
||||
// - "max" is available on all adaptive-thinking Claude models.
|
||||
// - "xhigh" is only available on Opus 4.7/4.8, Sonnet 5, and Fable 5.
|
||||
// - "xhigh" is only available on Opus 4.7/4.8/5, Sonnet 5, and Fable 5.
|
||||
if (
|
||||
model.id.includes("opus-4-6") ||
|
||||
model.id.includes("opus-4.6") ||
|
||||
@@ -714,6 +772,8 @@ function applyThinkingLevelMetadata(model: Model<any>): void {
|
||||
model.id.includes("opus-4.7") ||
|
||||
model.id.includes("opus-4-8") ||
|
||||
model.id.includes("opus-4.8") ||
|
||||
model.id.includes("opus-5") ||
|
||||
model.id.includes("opus.5") ||
|
||||
model.id.includes("sonnet-5") ||
|
||||
model.id.includes("sonnet.5")
|
||||
) {
|
||||
@@ -1004,6 +1064,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
|
||||
for (const [modelId, model] of Object.entries(data["amazon-bedrock"].models)) {
|
||||
const m = model as ModelsDevModel;
|
||||
if (m.tool_call !== true) continue;
|
||||
if (BEDROCK_INFERENCE_PROFILE_ONLY_MODEL_IDS.has(modelId)) continue;
|
||||
|
||||
let id = modelId;
|
||||
|
||||
@@ -1033,6 +1094,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
|
||||
},
|
||||
contextWindow: m.limit?.context || 4096,
|
||||
maxTokens: m.limit?.output || 4096,
|
||||
...(m.structured_output === true && { compat: { supportsStrictMode: true } }),
|
||||
});
|
||||
recordModelsDevReasoningOptions("amazon-bedrock" as const, id, m);
|
||||
}
|
||||
@@ -2455,7 +2517,10 @@ async function generateModels() {
|
||||
applyOpenAICompletionsCompatMetadata(model);
|
||||
applyModelsDevReasoningOptionMetadata(model);
|
||||
applyThinkingLevelMetadata(model);
|
||||
applyStrictToolCompatMetadata(model);
|
||||
applyOpenAIGrammarToolCompatMetadata(model);
|
||||
applyOpenAIToolSearchMetadata(model);
|
||||
applyOpenAIExplicitPromptCacheMetadata(model);
|
||||
}
|
||||
|
||||
// Group by provider and deduplicate by model ID
|
||||
@@ -2508,6 +2573,8 @@ async function generateModels() {
|
||||
}
|
||||
}
|
||||
|
||||
const generatedAt = new Date().toISOString();
|
||||
|
||||
if (!generatorOptions.jsonOnly) {
|
||||
// Stage and validate all provider values before replacing the current generated data.
|
||||
const providersDir = join(packageRoot, "src/providers");
|
||||
@@ -2527,7 +2594,7 @@ async function generateModels() {
|
||||
}
|
||||
writeJson(
|
||||
join(stagedDataDir, MODEL_DATA_MANIFEST_FILE),
|
||||
createModelDataManifest(modelDataStructure, fileContents),
|
||||
createModelDataManifest(modelDataStructure, fileContents, generatedAt),
|
||||
);
|
||||
validateModelDataDirectory(modelDataStructure, stagedDataDir);
|
||||
|
||||
|
||||
@@ -2,13 +2,14 @@ import { createHash } from "node:crypto";
|
||||
import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
export const MODEL_DATA_SCHEMA_VERSION = 2;
|
||||
export const MODEL_DATA_SCHEMA_VERSION = 3;
|
||||
export const MODEL_DATA_MANIFEST_FILE = ".manifest.json";
|
||||
|
||||
export type ModelDataStructure = Record<string, Record<string, string>>;
|
||||
|
||||
export interface ModelDataManifest {
|
||||
schemaVersion: number;
|
||||
generatedAt: string;
|
||||
structureHash: string;
|
||||
files: Record<string, string>;
|
||||
}
|
||||
@@ -119,9 +120,11 @@ export function modelDataStructureHash(structure: ModelDataStructure): string {
|
||||
export function createModelDataManifest(
|
||||
structure: ModelDataStructure,
|
||||
fileContents: Readonly<Record<string, string>>,
|
||||
generatedAt: string,
|
||||
): ModelDataManifest {
|
||||
return {
|
||||
schemaVersion: MODEL_DATA_SCHEMA_VERSION,
|
||||
generatedAt,
|
||||
structureHash: modelDataStructureHash(structure),
|
||||
files: sortedRecord(Object.entries(fileContents).map(([file, content]) => [file, sha256(content)] as const)),
|
||||
};
|
||||
@@ -203,6 +206,9 @@ export function validateModelDataDirectory(structure: ModelDataStructure, dataDi
|
||||
`model data schema is ${JSON.stringify(manifest?.schemaVersion)}, expected ${MODEL_DATA_SCHEMA_VERSION}`,
|
||||
);
|
||||
}
|
||||
if (typeof manifest?.generatedAt !== "string" || Number.isNaN(Date.parse(manifest.generatedAt))) {
|
||||
errors.push("model data manifest has an invalid generation timestamp");
|
||||
}
|
||||
const expectedStructureHash = modelDataStructureHash(structure);
|
||||
if (manifest?.structureHash !== expectedStructureHash) {
|
||||
errors.push("model data generation stamp does not match the generated catalog");
|
||||
|
||||
@@ -34,8 +34,10 @@ import { AssistantMessageEventStream } from "../utils/event-stream.ts";
|
||||
import { headersToRecord } from "../utils/headers.ts";
|
||||
import { parseJsonWithRepair, parseStreamingJson } from "../utils/json-parse.ts";
|
||||
import { getProviderEnvValue } from "../utils/provider-env.ts";
|
||||
import { retryProviderRequest } from "../utils/provider-retry.ts";
|
||||
import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts";
|
||||
|
||||
import { resolveJsonSchemaStrictSampling } from "./constrained-sampling.ts";
|
||||
import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.ts";
|
||||
import { adjustMaxTokensForThinking, buildBaseOptions, clampMaxTokensToContext } from "./simple-options.ts";
|
||||
import { transformMessages } from "./transform-messages.ts";
|
||||
@@ -178,6 +180,7 @@ function getAnthropicCompat(
|
||||
supportsCacheControlOnTools: model.compat?.supportsCacheControlOnTools ?? true,
|
||||
supportsTemperature: model.compat?.supportsTemperature ?? true,
|
||||
allowEmptySignature: model.compat?.allowEmptySignature ?? false,
|
||||
supportsStrictTools: model.compat?.supportsStrictTools ?? false,
|
||||
supportsToolReferences: model.compat?.supportsToolReferences ?? defaultSupportsToolReferences(model),
|
||||
};
|
||||
}
|
||||
@@ -550,9 +553,16 @@ export const stream: StreamFunction<"anthropic-messages", AnthropicOptions> = (
|
||||
const requestOptions = {
|
||||
...(options?.signal ? { signal: options.signal } : {}),
|
||||
...(options?.timeoutMs !== undefined ? { timeout: options.timeoutMs } : {}),
|
||||
maxRetries: options?.maxRetries ?? 0,
|
||||
maxRetries: 0,
|
||||
};
|
||||
const response = await client.messages.create({ ...params, stream: true }, requestOptions).asResponse();
|
||||
const response = await retryProviderRequest(
|
||||
() => client.messages.create({ ...params, stream: true }, requestOptions).asResponse(),
|
||||
{
|
||||
maxRetries: options?.maxRetries,
|
||||
maxRetryDelayMs: options?.maxRetryDelayMs,
|
||||
signal: options?.signal,
|
||||
},
|
||||
);
|
||||
await options?.onResponse?.({ status: response.status, headers: headersToRecord(response.headers) }, model);
|
||||
stream.push({ type: "start", partial: output });
|
||||
|
||||
@@ -991,9 +1001,17 @@ function buildParams(
|
||||
immediateTools,
|
||||
isOAuthToken,
|
||||
compat.supportsEagerToolInputStreaming,
|
||||
compat.supportsStrictTools,
|
||||
compat.supportsCacheControlOnTools ? cacheControl : undefined,
|
||||
),
|
||||
...convertTools(deferredTools, isOAuthToken, compat.supportsEagerToolInputStreaming, undefined, true),
|
||||
...convertTools(
|
||||
deferredTools,
|
||||
isOAuthToken,
|
||||
compat.supportsEagerToolInputStreaming,
|
||||
compat.supportsStrictTools,
|
||||
undefined,
|
||||
true,
|
||||
),
|
||||
];
|
||||
}
|
||||
|
||||
@@ -1261,23 +1279,34 @@ function convertTools(
|
||||
tools: Tool[],
|
||||
isOAuthToken: boolean,
|
||||
supportsEagerToolInputStreaming: boolean,
|
||||
supportsStrictTools: boolean,
|
||||
cacheControl?: CacheControlEphemeral,
|
||||
deferLoading = false,
|
||||
): Anthropic.Messages.Tool[] {
|
||||
if (!tools) return [];
|
||||
|
||||
return tools.map((tool, index) => {
|
||||
const strict = resolveJsonSchemaStrictSampling(tool, supportsStrictTools);
|
||||
const schema = tool.parameters as { properties?: unknown; required?: string[] };
|
||||
const legacyInputSchema = {
|
||||
type: "object" as const,
|
||||
properties: schema.properties ?? {},
|
||||
required: schema.required ?? [],
|
||||
};
|
||||
const inputSchema =
|
||||
strict === true
|
||||
? {
|
||||
...(tool.parameters as Record<string, unknown>),
|
||||
...legacyInputSchema,
|
||||
}
|
||||
: legacyInputSchema;
|
||||
|
||||
return {
|
||||
name: isOAuthToken ? toClaudeCodeName(tool.name) : tool.name,
|
||||
description: tool.description,
|
||||
...(supportsEagerToolInputStreaming ? { eager_input_streaming: true } : {}),
|
||||
input_schema: {
|
||||
type: "object",
|
||||
properties: schema.properties ?? {},
|
||||
required: schema.required ?? [],
|
||||
},
|
||||
...(strict === true ? { strict: true } : {}),
|
||||
input_schema: inputSchema,
|
||||
...(deferLoading ? { defer_loading: true } : {}),
|
||||
...(cacheControl && index === tools.length - 1 ? { cache_control: cacheControl } : {}),
|
||||
};
|
||||
|
||||
@@ -14,6 +14,8 @@ import { formatProviderError, normalizeProviderError } from "../utils/error-body
|
||||
import { AssistantMessageEventStream } from "../utils/event-stream.ts";
|
||||
import { headersToRecord } from "../utils/headers.ts";
|
||||
import { getProviderEnvValue } from "../utils/provider-env.ts";
|
||||
import { retryProviderRequest } from "../utils/provider-retry.ts";
|
||||
import { createGrammarToolInputProperties } from "./constrained-sampling.ts";
|
||||
import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts";
|
||||
import { convertResponsesMessages, convertResponsesTools, processResponsesStream } from "./openai-responses-shared.ts";
|
||||
import { buildBaseOptions } from "./simple-options.ts";
|
||||
@@ -99,7 +101,11 @@ export const stream: StreamFunction<"azure-openai-responses", AzureOpenAIRespons
|
||||
throw new Error(`No API key for provider: ${model.provider}`);
|
||||
}
|
||||
const client = createClient(model, apiKey, options);
|
||||
let params = buildParams(model, context, options, deploymentName);
|
||||
const grammarToolInputProperties = createGrammarToolInputProperties(
|
||||
context.tools,
|
||||
model.compat?.supportsOpenAIGrammarTools ?? false,
|
||||
);
|
||||
let params = buildParams(model, context, options, deploymentName, grammarToolInputProperties);
|
||||
const nextParams = await options?.onPayload?.(params, model);
|
||||
if (nextParams !== undefined) {
|
||||
params = nextParams as ResponseCreateParamsStreaming;
|
||||
@@ -107,13 +113,20 @@ export const stream: StreamFunction<"azure-openai-responses", AzureOpenAIRespons
|
||||
const requestOptions = {
|
||||
...(options?.signal ? { signal: options.signal } : {}),
|
||||
...(options?.timeoutMs !== undefined ? { timeout: options.timeoutMs } : {}),
|
||||
maxRetries: options?.maxRetries ?? 0,
|
||||
maxRetries: 0,
|
||||
};
|
||||
const { data: openaiStream, response } = await client.responses.create(params, requestOptions).withResponse();
|
||||
const { data: openaiStream, response } = await retryProviderRequest(
|
||||
() => client.responses.create(params, requestOptions).withResponse(),
|
||||
{
|
||||
maxRetries: options?.maxRetries,
|
||||
maxRetryDelayMs: options?.maxRetryDelayMs,
|
||||
signal: options?.signal,
|
||||
},
|
||||
);
|
||||
await options?.onResponse?.({ status: response.status, headers: headersToRecord(response.headers) }, model);
|
||||
stream.push({ type: "start", partial: output });
|
||||
|
||||
await processResponsesStream(openaiStream, output, stream, model);
|
||||
await processResponsesStream(openaiStream, output, stream, model, { grammarToolInputProperties });
|
||||
|
||||
if (options?.signal?.aborted) {
|
||||
throw new Error("Request was aborted");
|
||||
@@ -128,8 +141,9 @@ export const stream: StreamFunction<"azure-openai-responses", AzureOpenAIRespons
|
||||
} catch (error) {
|
||||
for (const block of output.content) {
|
||||
delete (block as { index?: number }).index;
|
||||
// partialJson is only a streaming scratch buffer; never persist it.
|
||||
// Streaming scratch buffers are only used during parsing; never persist them.
|
||||
delete (block as { partialJson?: string }).partialJson;
|
||||
delete (block as { customInput?: unknown }).customInput;
|
||||
}
|
||||
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
|
||||
output.errorMessage = formatAzureOpenAIError(error);
|
||||
@@ -254,8 +268,14 @@ function buildParams(
|
||||
context: Context,
|
||||
options: AzureOpenAIResponsesOptions | undefined,
|
||||
deploymentName: string,
|
||||
grammarToolInputProperties: ReadonlyMap<string, string> = createGrammarToolInputProperties(
|
||||
context.tools,
|
||||
model.compat?.supportsOpenAIGrammarTools ?? false,
|
||||
),
|
||||
) {
|
||||
const messages = convertResponsesMessages(model, context, AZURE_TOOL_CALL_PROVIDERS);
|
||||
const messages = convertResponsesMessages(model, context, AZURE_TOOL_CALL_PROVIDERS, {
|
||||
grammarToolInputProperties,
|
||||
});
|
||||
|
||||
const params: ResponseCreateParamsStreaming = {
|
||||
model: deploymentName,
|
||||
@@ -274,7 +294,10 @@ function buildParams(
|
||||
}
|
||||
|
||||
if (context.tools && context.tools.length > 0) {
|
||||
params.tools = convertResponsesTools(context.tools);
|
||||
params.tools = convertResponsesTools(context.tools, {
|
||||
supportsStrictMode: model.compat?.supportsStrictMode ?? true,
|
||||
supportsOpenAIGrammarTools: model.compat?.supportsOpenAIGrammarTools ?? false,
|
||||
});
|
||||
}
|
||||
|
||||
if (model.reasoning) {
|
||||
|
||||
@@ -54,6 +54,7 @@ import { parseStreamingJson } from "../utils/json-parse.ts";
|
||||
import { resolveHttpProxyUrlForTarget } from "../utils/node-http-proxy.ts";
|
||||
import { getProviderEnvValue } from "../utils/provider-env.ts";
|
||||
import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts";
|
||||
import { resolveJsonSchemaStrictSampling } from "./constrained-sampling.ts";
|
||||
import {
|
||||
adjustMaxTokensForThinking,
|
||||
buildBaseOptions,
|
||||
@@ -228,7 +229,7 @@ export const stream: StreamFunction<"bedrock-converse-stream", BedrockOptions> =
|
||||
...(inferenceMaxTokens !== undefined && { maxTokens: inferenceMaxTokens }),
|
||||
...(options.temperature !== undefined && { temperature: options.temperature }),
|
||||
},
|
||||
toolConfig: convertToolConfig(context.tools, options.toolChoice),
|
||||
toolConfig: convertToolConfig(context.tools, options.toolChoice, model.compat?.supportsStrictMode ?? false),
|
||||
additionalModelRequestFields: buildAdditionalModelRequestFields(model, options),
|
||||
...(options.requestMetadata !== undefined && { requestMetadata: options.requestMetadata }),
|
||||
};
|
||||
@@ -581,6 +582,7 @@ function supportsAdaptiveThinking(modelId: string, modelName?: string): boolean
|
||||
s.includes("opus-4-6") ||
|
||||
s.includes("opus-4-7") ||
|
||||
s.includes("opus-4-8") ||
|
||||
s.includes("opus-5") ||
|
||||
s.includes("sonnet-4-6") ||
|
||||
s.includes("sonnet-5") ||
|
||||
s.includes("fable-5"),
|
||||
@@ -590,7 +592,12 @@ function supportsAdaptiveThinking(modelId: string, modelName?: string): boolean
|
||||
function supportsNativeXhighEffort(model: Model<"bedrock-converse-stream">): boolean {
|
||||
const candidates = getModelMatchCandidates(model.id, model.name);
|
||||
return candidates.some(
|
||||
(s) => s.includes("opus-4-7") || s.includes("opus-4-8") || s.includes("sonnet-5") || s.includes("fable-5"),
|
||||
(s) =>
|
||||
s.includes("opus-4-7") ||
|
||||
s.includes("opus-4-8") ||
|
||||
s.includes("opus-5") ||
|
||||
s.includes("sonnet-5") ||
|
||||
s.includes("fable-5"),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -669,8 +676,8 @@ function supportsPromptCaching(model: Model<"bedrock-converse-stream">, env?: Pr
|
||||
if (getProviderEnvValue("AWS_BEDROCK_FORCE_CACHE", env) === "1") return true;
|
||||
return false;
|
||||
}
|
||||
// Claude 5 models (fable-5, sonnet-5)
|
||||
if (candidates.some((s) => s.includes("fable-5") || s.includes("sonnet-5"))) return true;
|
||||
// Claude 5 models (fable-5, opus-5, sonnet-5)
|
||||
if (candidates.some((s) => s.includes("fable-5") || s.includes("opus-5") || s.includes("sonnet-5"))) return true;
|
||||
// Claude 4.x models (opus-4, sonnet-4, haiku-4)
|
||||
if (candidates.some((s) => s.includes("-4-"))) return true;
|
||||
// Claude 3.7 Sonnet
|
||||
@@ -908,16 +915,22 @@ function convertMessages(
|
||||
function convertToolConfig(
|
||||
tools: Tool[] | undefined,
|
||||
toolChoice: BedrockOptions["toolChoice"],
|
||||
supportsStrictMode: boolean,
|
||||
): ToolConfiguration | undefined {
|
||||
if (!tools?.length || toolChoice === "none") return undefined;
|
||||
if (!tools?.length) return undefined;
|
||||
if (toolChoice === "none") return undefined;
|
||||
|
||||
const bedrockTools: BedrockTool[] = tools.map((tool) => ({
|
||||
toolSpec: {
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
inputSchema: { json: tool.parameters as unknown as DocumentType },
|
||||
},
|
||||
}));
|
||||
const bedrockTools: BedrockTool[] = tools.map((tool) => {
|
||||
const strict = resolveJsonSchemaStrictSampling(tool, supportsStrictMode);
|
||||
return {
|
||||
toolSpec: {
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
inputSchema: { json: tool.parameters as unknown as DocumentType },
|
||||
...(strict === true ? { strict: true } : {}),
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
let bedrockToolChoice: ToolChoice | undefined;
|
||||
switch (toolChoice) {
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
import type { Tool } from "../types.ts";
|
||||
|
||||
interface JsonSchemaObject {
|
||||
type?: unknown;
|
||||
properties?: Record<string, JsonSchemaObject | undefined>;
|
||||
required?: unknown;
|
||||
}
|
||||
|
||||
export interface GrammarConstrainedSampling {
|
||||
format: "lark" | "regex";
|
||||
definition: string;
|
||||
inputProperty: string;
|
||||
}
|
||||
|
||||
export interface GrammarToolInputJsonBuffer {
|
||||
input: string;
|
||||
started: boolean;
|
||||
closed: boolean;
|
||||
}
|
||||
|
||||
export function getGrammarToolInput(
|
||||
toolName: string,
|
||||
arguments_: Record<string, unknown>,
|
||||
inputProperty: string,
|
||||
): string {
|
||||
const input = arguments_[inputProperty];
|
||||
if (typeof input !== "string") {
|
||||
throw new Error(`Grammar tool call "${toolName}" requires argument "${inputProperty}" to be a string.`);
|
||||
}
|
||||
return input;
|
||||
}
|
||||
|
||||
export function appendGrammarToolInputJsonDelta(
|
||||
buffer: GrammarToolInputJsonBuffer,
|
||||
inputProperty: string,
|
||||
nextInput: string,
|
||||
close: boolean,
|
||||
): string | undefined {
|
||||
if (buffer.closed) {
|
||||
if (close && nextInput === buffer.input) return undefined;
|
||||
throw new Error(`grammar tool input for property "${inputProperty}" changed after it was closed`);
|
||||
}
|
||||
if (!nextInput.startsWith(buffer.input)) {
|
||||
throw new Error(`grammar tool input for property "${inputProperty}" changed non-monotonically`);
|
||||
}
|
||||
|
||||
const inputDelta = nextInput.slice(buffer.input.length);
|
||||
if (!close && inputDelta.length === 0) return undefined;
|
||||
|
||||
let delta = "";
|
||||
if (!buffer.started) {
|
||||
delta += `{${JSON.stringify(inputProperty)}:"`;
|
||||
buffer.started = true;
|
||||
}
|
||||
delta += JSON.stringify(inputDelta).slice(1, -1);
|
||||
buffer.input = nextInput;
|
||||
|
||||
if (close) {
|
||||
delta += '"}';
|
||||
buffer.closed = true;
|
||||
}
|
||||
return delta;
|
||||
}
|
||||
|
||||
function inferGrammarInputProperty(tool: Tool): string {
|
||||
const schema = tool.parameters as JsonSchemaObject;
|
||||
if (schema.type !== "object") {
|
||||
throw new Error("grammar constrained sampling requires an object parameter schema");
|
||||
}
|
||||
if (!Array.isArray(schema.required) || schema.required.length !== 1 || typeof schema.required[0] !== "string") {
|
||||
throw new Error("grammar constrained sampling requires exactly one required string property");
|
||||
}
|
||||
|
||||
const inputProperty = schema.required[0];
|
||||
if (!schema.properties?.[inputProperty]) {
|
||||
throw new Error(`grammar constrained sampling requires a properties entry for ${inputProperty}`);
|
||||
}
|
||||
if (schema.properties[inputProperty]?.type !== "string") {
|
||||
throw new Error(`grammar constrained sampling property ${inputProperty} must have type string`);
|
||||
}
|
||||
return inputProperty;
|
||||
}
|
||||
|
||||
export function resolveJsonSchemaStrictSampling(tool: Tool, supportsStrictMode: boolean): boolean | undefined {
|
||||
const config = tool.constrainedSampling;
|
||||
if (!config || config.type !== "json_schema") {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (supportsStrictMode) {
|
||||
return true;
|
||||
}
|
||||
if (config.strict === "require") {
|
||||
throw new Error(
|
||||
`Tool "${tool.name}" requires JSON-schema constrained sampling, but strict tools are unsupported.`,
|
||||
);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function resolveGrammarConstrainedSampling(
|
||||
tool: Tool,
|
||||
supportsOpenAIGrammarTools: boolean,
|
||||
): GrammarConstrainedSampling | undefined {
|
||||
const config = tool.constrainedSampling;
|
||||
if (!config || config.type !== "grammar") {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (!supportsOpenAIGrammarTools) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const larkDefinition = config.variants.openai_lark;
|
||||
const regexDefinition = config.variants.openai_regex;
|
||||
const hasLarkDefinition = typeof larkDefinition === "string" && larkDefinition.trim().length > 0;
|
||||
const hasRegexDefinition = typeof regexDefinition === "string" && regexDefinition.trim().length > 0;
|
||||
if (!hasLarkDefinition && !hasRegexDefinition) {
|
||||
throw new Error(
|
||||
`Tool "${tool.name}" cannot use grammar constrained sampling: no supported grammar variant was provided.`,
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
return {
|
||||
format: hasLarkDefinition ? "lark" : "regex",
|
||||
definition: hasLarkDefinition ? larkDefinition : regexDefinition!,
|
||||
inputProperty: inferGrammarInputProperty(tool),
|
||||
};
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
throw new Error(`Tool "${tool.name}" cannot use grammar constrained sampling: ${message}.`);
|
||||
}
|
||||
}
|
||||
|
||||
export function createGrammarToolInputProperties(
|
||||
tools: Tool[] | undefined,
|
||||
supportsOpenAIGrammarTools: boolean,
|
||||
): ReadonlyMap<string, string> {
|
||||
const properties = new Map<string, string>();
|
||||
for (const tool of tools ?? []) {
|
||||
const grammar = resolveGrammarConstrainedSampling(tool, supportsOpenAIGrammarTools);
|
||||
if (grammar) {
|
||||
properties.set(tool.name, grammar.inputProperty);
|
||||
}
|
||||
}
|
||||
return properties;
|
||||
}
|
||||
@@ -30,8 +30,9 @@ import {
|
||||
convertTools,
|
||||
isThinkingPart,
|
||||
mapStopReason,
|
||||
mapToolChoice,
|
||||
resolveGoogleFunctionCallingMode,
|
||||
retainThoughtSignature,
|
||||
supportsGoogleStrictToolSampling,
|
||||
} from "./google-shared.ts";
|
||||
import { buildBaseOptions } from "./simple-options.ts";
|
||||
|
||||
@@ -355,22 +356,18 @@ function buildParams(
|
||||
generationConfig.maxOutputTokens = options.maxTokens;
|
||||
}
|
||||
|
||||
const functionCallingMode = context.tools?.length
|
||||
? resolveGoogleFunctionCallingMode(context.tools, options.toolChoice, supportsGoogleStrictToolSampling(model.id))
|
||||
: undefined;
|
||||
const config: GenerateContentConfig = {
|
||||
...(Object.keys(generationConfig).length > 0 && generationConfig),
|
||||
...(context.systemPrompt && { systemInstruction: sanitizeSurrogates(context.systemPrompt) }),
|
||||
...(context.tools && context.tools.length > 0 && { tools: convertTools(context.tools) }),
|
||||
...(functionCallingMode !== undefined && {
|
||||
toolConfig: { functionCallingConfig: { mode: functionCallingMode } },
|
||||
}),
|
||||
};
|
||||
|
||||
if (context.tools && context.tools.length > 0 && options.toolChoice) {
|
||||
config.toolConfig = {
|
||||
functionCallingConfig: {
|
||||
mode: mapToolChoice(options.toolChoice),
|
||||
},
|
||||
};
|
||||
} else {
|
||||
config.toolConfig = undefined;
|
||||
}
|
||||
|
||||
if (options.thinking?.enabled && model.reasoning) {
|
||||
const thinkingConfig: ThinkingConfig = { includeThoughts: true };
|
||||
if (options.thinking.level !== undefined) {
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
import { type Content, FinishReason, FunctionCallingConfigMode, type Part } from "@google/genai";
|
||||
import type { Context, ImageContent, Model, StopReason, TextContent, Tool } from "../types.ts";
|
||||
import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts";
|
||||
import { resolveJsonSchemaStrictSampling } from "./constrained-sampling.ts";
|
||||
import { transformMessages } from "./transform-messages.ts";
|
||||
|
||||
type GoogleApiType = "google-generative-ai" | "google-vertex";
|
||||
@@ -287,9 +288,13 @@ export function convertTools(
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Map tool choice string to Gemini FunctionCallingConfigMode.
|
||||
*/
|
||||
/** Gemini 3+ enforces required function parameters in validated tool-calling modes. */
|
||||
export function supportsGoogleStrictToolSampling(modelId: string): boolean {
|
||||
const majorVersion = getGeminiMajorVersion(modelId);
|
||||
return majorVersion !== undefined && majorVersion >= 3;
|
||||
}
|
||||
|
||||
/** Map tool choice string to Gemini FunctionCallingConfigMode. */
|
||||
export function mapToolChoice(choice: string): FunctionCallingConfigMode {
|
||||
switch (choice) {
|
||||
case "auto":
|
||||
@@ -303,6 +308,21 @@ export function mapToolChoice(choice: string): FunctionCallingConfigMode {
|
||||
}
|
||||
}
|
||||
|
||||
export function resolveGoogleFunctionCallingMode(
|
||||
tools: Tool[],
|
||||
toolChoice: string | undefined,
|
||||
supportsStrictMode: boolean,
|
||||
): FunctionCallingConfigMode | undefined {
|
||||
const useStrictMode = tools.some((tool) => resolveJsonSchemaStrictSampling(tool, supportsStrictMode) === true);
|
||||
if (toolChoice === "none" || toolChoice === "any") {
|
||||
return mapToolChoice(toolChoice);
|
||||
}
|
||||
if (useStrictMode) {
|
||||
return FunctionCallingConfigMode.VALIDATED;
|
||||
}
|
||||
return toolChoice ? mapToolChoice(toolChoice) : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Map Gemini FinishReason to our StopReason.
|
||||
*/
|
||||
|
||||
@@ -35,8 +35,9 @@ import {
|
||||
convertTools,
|
||||
isThinkingPart,
|
||||
mapStopReason,
|
||||
mapToolChoice,
|
||||
resolveGoogleFunctionCallingMode,
|
||||
retainThoughtSignature,
|
||||
supportsGoogleStrictToolSampling,
|
||||
} from "./google-shared.ts";
|
||||
import { buildBaseOptions } from "./simple-options.ts";
|
||||
|
||||
@@ -454,22 +455,18 @@ function buildParams(
|
||||
generationConfig.maxOutputTokens = options.maxTokens;
|
||||
}
|
||||
|
||||
const functionCallingMode = context.tools?.length
|
||||
? resolveGoogleFunctionCallingMode(context.tools, options.toolChoice, supportsGoogleStrictToolSampling(model.id))
|
||||
: undefined;
|
||||
const config: GenerateContentConfig = {
|
||||
...(Object.keys(generationConfig).length > 0 && generationConfig),
|
||||
...(context.systemPrompt && { systemInstruction: sanitizeSurrogates(context.systemPrompt) }),
|
||||
...(context.tools && context.tools.length > 0 && { tools: convertTools(context.tools) }),
|
||||
...(functionCallingMode !== undefined && {
|
||||
toolConfig: { functionCallingConfig: { mode: functionCallingMode } },
|
||||
}),
|
||||
};
|
||||
|
||||
if (context.tools && context.tools.length > 0 && options.toolChoice) {
|
||||
config.toolConfig = {
|
||||
functionCallingConfig: {
|
||||
mode: mapToolChoice(options.toolChoice),
|
||||
},
|
||||
};
|
||||
} else {
|
||||
config.toolConfig = undefined;
|
||||
}
|
||||
|
||||
if (options.thinking?.enabled && model.reasoning) {
|
||||
const thinkingConfig: ThinkingConfig = { includeThoughts: true };
|
||||
if (options.thinking.level !== undefined) {
|
||||
|
||||
@@ -25,6 +25,7 @@ import { AssistantMessageEventStream } from "../utils/event-stream.ts";
|
||||
import { shortHash } from "../utils/hash.ts";
|
||||
import { parseStreamingJson } from "../utils/json-parse.ts";
|
||||
import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts";
|
||||
import { resolveJsonSchemaStrictSampling } from "./constrained-sampling.ts";
|
||||
import { buildBaseOptions } from "./simple-options.ts";
|
||||
import { transformMessages } from "./transform-messages.ts";
|
||||
|
||||
@@ -483,15 +484,18 @@ async function consumeChatStream(
|
||||
}
|
||||
|
||||
function toFunctionTools(tools: Tool[]): Array<FunctionTool & { type: "function" }> {
|
||||
return tools.map((tool) => ({
|
||||
type: "function",
|
||||
function: {
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: stripSymbolKeys(tool.parameters) as Record<string, unknown>,
|
||||
strict: false,
|
||||
},
|
||||
}));
|
||||
return tools.map((tool) => {
|
||||
const strict = resolveJsonSchemaStrictSampling(tool, true);
|
||||
return {
|
||||
type: "function",
|
||||
function: {
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: stripSymbolKeys(tool.parameters) as Record<string, unknown>,
|
||||
strict: strict ?? false,
|
||||
},
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function stripSymbolKeys(value: unknown): unknown {
|
||||
|
||||
@@ -47,6 +47,7 @@ import { AssistantMessageEventStream } from "../utils/event-stream.ts";
|
||||
import { headersToRecord } from "../utils/headers.ts";
|
||||
import { resolveHttpProxyUrlForTarget } from "../utils/node-http-proxy.ts";
|
||||
import { uuidv7 } from "../utils/uuid.ts";
|
||||
import { createGrammarToolInputProperties } from "./constrained-sampling.ts";
|
||||
import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts";
|
||||
import { convertResponsesMessages, convertResponsesTools, processResponsesStream } from "./openai-responses-shared.ts";
|
||||
import { buildBaseOptions } from "./simple-options.ts";
|
||||
@@ -158,9 +159,16 @@ function getRetryAfterDelayMs(headers: Headers): number | undefined {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function capRetryDelayMs(delayMs: number, options?: StreamOptions): number {
|
||||
class RetryDelayExceededError extends Error {}
|
||||
|
||||
function validateRetryDelayMs(delayMs: number, options?: StreamOptions): number {
|
||||
const maxRetryDelayMs = options?.maxRetryDelayMs ?? DEFAULT_MAX_RETRY_DELAY_MS;
|
||||
return maxRetryDelayMs > 0 ? Math.min(delayMs, maxRetryDelayMs) : delayMs;
|
||||
if (maxRetryDelayMs > 0 && delayMs > maxRetryDelayMs) {
|
||||
throw new RetryDelayExceededError(
|
||||
`Server requested ${Math.ceil(delayMs / 1000)}s retry delay (max: ${Math.ceil(maxRetryDelayMs / 1000)}s)`,
|
||||
);
|
||||
}
|
||||
return delayMs;
|
||||
}
|
||||
|
||||
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
@@ -255,12 +263,17 @@ export const stream: StreamFunction<"openai-codex-responses", OpenAICodexRespons
|
||||
}
|
||||
|
||||
const accountId = extractAccountId(apiKey);
|
||||
let body = buildRequestBody(model, context, options);
|
||||
const grammarToolInputProperties = createGrammarToolInputProperties(
|
||||
context.tools,
|
||||
model.compat?.supportsOpenAIGrammarTools ?? false,
|
||||
);
|
||||
const cacheSessionId = options?.cacheRetention === "none" ? undefined : options?.sessionId;
|
||||
const codexSessionId = clampOpenAIPromptCacheKey(cacheSessionId);
|
||||
let body = buildRequestBody(model, context, options, codexSessionId, grammarToolInputProperties);
|
||||
const nextBody = await options?.onPayload?.(body, model);
|
||||
if (nextBody !== undefined) {
|
||||
body = nextBody as RequestBody;
|
||||
}
|
||||
const codexSessionId = clampOpenAIPromptCacheKey(options?.sessionId);
|
||||
const websocketRequestId = codexSessionId || uuidv7();
|
||||
const sseHeaders = buildSSEHeaders(model.headers, options?.headers, accountId, apiKey, codexSessionId);
|
||||
const websocketHeaders = buildWebSocketHeaders(
|
||||
@@ -275,9 +288,9 @@ export const stream: StreamFunction<"openai-codex-responses", OpenAICodexRespons
|
||||
const websocketConnectTimeoutMs = normalizeTimeoutMs(options?.websocketConnectTimeoutMs);
|
||||
const transport = options?.transport || "auto";
|
||||
let startEmitted = false;
|
||||
const websocketDisabledForSession = transport !== "sse" && isWebSocketSseFallbackActive(options?.sessionId);
|
||||
const websocketDisabledForSession = transport !== "sse" && isWebSocketSseFallbackActive(cacheSessionId);
|
||||
if (websocketDisabledForSession) {
|
||||
recordWebSocketSseFallback(options?.sessionId);
|
||||
recordWebSocketSseFallback(cacheSessionId);
|
||||
}
|
||||
|
||||
if (transport !== "sse" && !websocketDisabledForSession) {
|
||||
@@ -303,6 +316,8 @@ export const stream: StreamFunction<"openai-codex-responses", OpenAICodexRespons
|
||||
},
|
||||
httpTimeoutMs,
|
||||
websocketConnectTimeoutMs,
|
||||
cacheSessionId,
|
||||
grammarToolInputProperties,
|
||||
options,
|
||||
);
|
||||
|
||||
@@ -341,11 +356,11 @@ export const stream: StreamFunction<"openai-codex-responses", OpenAICodexRespons
|
||||
requestBytes: new TextEncoder().encode(bodyJson).byteLength,
|
||||
}),
|
||||
);
|
||||
recordWebSocketFailure(options?.sessionId, error);
|
||||
recordWebSocketFailure(cacheSessionId, error);
|
||||
if (websocketStarted) {
|
||||
throw error;
|
||||
}
|
||||
recordWebSocketSseFallback(options?.sessionId);
|
||||
recordWebSocketSseFallback(cacheSessionId);
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -404,9 +419,7 @@ export const stream: StreamFunction<"openai-codex-responses", OpenAICodexRespons
|
||||
const delayMs =
|
||||
retryAfterDelayMs === undefined
|
||||
? BASE_DELAY_MS * 2 ** attempt
|
||||
: response.status === 429
|
||||
? capRetryDelayMs(retryAfterDelayMs, options)
|
||||
: retryAfterDelayMs;
|
||||
: validateRetryDelayMs(retryAfterDelayMs, options);
|
||||
|
||||
await sleep(delayMs, options?.signal);
|
||||
continue;
|
||||
@@ -427,7 +440,11 @@ export const stream: StreamFunction<"openai-codex-responses", OpenAICodexRespons
|
||||
}
|
||||
lastError = error instanceof Error ? error : new Error(String(error));
|
||||
// Network errors are retryable
|
||||
if (attempt < maxRetries && !lastError.message.includes("usage limit")) {
|
||||
if (
|
||||
attempt < maxRetries &&
|
||||
!(lastError instanceof RetryDelayExceededError) &&
|
||||
!lastError.message.includes("usage limit")
|
||||
) {
|
||||
const delayMs = BASE_DELAY_MS * 2 ** attempt;
|
||||
await sleep(delayMs, options?.signal);
|
||||
continue;
|
||||
@@ -448,7 +465,7 @@ export const stream: StreamFunction<"openai-codex-responses", OpenAICodexRespons
|
||||
startEmitted = true;
|
||||
stream.push({ type: "start", partial: output });
|
||||
}
|
||||
await processStream(response, output, stream, model, options);
|
||||
await processStream(response, output, stream, model, grammarToolInputProperties, options);
|
||||
|
||||
if (options?.signal?.aborted) {
|
||||
throw new Error("Request was aborted");
|
||||
@@ -458,8 +475,9 @@ export const stream: StreamFunction<"openai-codex-responses", OpenAICodexRespons
|
||||
stream.end();
|
||||
} catch (error) {
|
||||
for (const block of output.content) {
|
||||
// partialJson is only a streaming scratch buffer; never persist it.
|
||||
// Streaming scratch buffers are only used during parsing; never persist them.
|
||||
delete (block as { partialJson?: string }).partialJson;
|
||||
delete (block as { customInput?: unknown }).customInput;
|
||||
}
|
||||
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
|
||||
output.errorMessage = formatProviderError(normalizeProviderError(error));
|
||||
@@ -498,12 +516,25 @@ export const streamSimple: StreamFunction<"openai-codex-responses", SimpleStream
|
||||
function buildRequestBody(
|
||||
model: Model<"openai-codex-responses">,
|
||||
context: Context,
|
||||
options?: OpenAICodexResponsesOptions,
|
||||
options: OpenAICodexResponsesOptions | undefined,
|
||||
cacheSessionId: string | undefined,
|
||||
grammarToolInputProperties: ReadonlyMap<string, string> = createGrammarToolInputProperties(
|
||||
context.tools,
|
||||
model.compat?.supportsOpenAIGrammarTools ?? false,
|
||||
),
|
||||
): RequestBody {
|
||||
const supportsStrictMode = model.compat?.supportsStrictMode ?? true;
|
||||
const supportsOpenAIGrammarTools = model.compat?.supportsOpenAIGrammarTools ?? false;
|
||||
const toolPlacement = splitDeferredTools(context, model.compat?.supportsToolSearch ?? false);
|
||||
const messages = convertResponsesMessages(model, context, CODEX_TOOL_CALL_PROVIDERS, {
|
||||
includeSystemPrompt: false,
|
||||
grammarToolInputProperties,
|
||||
deferredTools: toolPlacement.deferred,
|
||||
toolOptions: {
|
||||
strict: null,
|
||||
supportsStrictMode,
|
||||
supportsOpenAIGrammarTools,
|
||||
},
|
||||
});
|
||||
|
||||
const body: RequestBody = {
|
||||
@@ -514,7 +545,7 @@ function buildRequestBody(
|
||||
input: messages,
|
||||
text: { verbosity: options?.textVerbosity || "low" },
|
||||
include: ["reasoning.encrypted_content"],
|
||||
prompt_cache_key: clampOpenAIPromptCacheKey(options?.sessionId),
|
||||
prompt_cache_key: cacheSessionId,
|
||||
tool_choice: options?.toolChoice ?? "auto",
|
||||
parallel_tool_calls: true,
|
||||
};
|
||||
@@ -528,7 +559,11 @@ function buildRequestBody(
|
||||
}
|
||||
|
||||
if (toolPlacement.immediate.length > 0) {
|
||||
body.tools = convertResponsesTools(toolPlacement.immediate, { strict: null });
|
||||
body.tools = convertResponsesTools(toolPlacement.immediate, {
|
||||
strict: null,
|
||||
supportsStrictMode,
|
||||
supportsOpenAIGrammarTools,
|
||||
});
|
||||
}
|
||||
|
||||
if (options?.reasoningEffort !== undefined) {
|
||||
@@ -610,10 +645,12 @@ async function processStream(
|
||||
output: AssistantMessage,
|
||||
stream: AssistantMessageEventStream,
|
||||
model: Model<"openai-codex-responses">,
|
||||
grammarToolInputProperties: ReadonlyMap<string, string>,
|
||||
options?: OpenAICodexResponsesOptions,
|
||||
): Promise<void> {
|
||||
await processResponsesStream(mapCodexEvents(parseSSE(response, options?.signal)), output, stream, model, {
|
||||
serviceTier: options?.serviceTier,
|
||||
grammarToolInputProperties,
|
||||
resolveServiceTier: resolveCodexServiceTier,
|
||||
applyServiceTierPricing: (usage, serviceTier) => applyServiceTierPricing(usage, serviceTier, model),
|
||||
});
|
||||
@@ -1399,12 +1436,14 @@ async function processWebSocketStream(
|
||||
onStart: () => void,
|
||||
idleTimeoutMs: number | undefined,
|
||||
websocketConnectTimeoutMs: number | undefined,
|
||||
cacheSessionId: string | undefined,
|
||||
grammarToolInputProperties: ReadonlyMap<string, string>,
|
||||
options?: OpenAICodexResponsesOptions,
|
||||
): Promise<void> {
|
||||
const { socket, entry, reused, release } = await acquireWebSocket(
|
||||
url,
|
||||
headers,
|
||||
options?.sessionId,
|
||||
cacheSessionId,
|
||||
options?.signal,
|
||||
websocketConnectTimeoutMs,
|
||||
options?.env,
|
||||
@@ -1415,7 +1454,7 @@ async function processWebSocketStream(
|
||||
// WebSocket continuation still works via connection-scoped previous_response_id state.
|
||||
const fullBody = body;
|
||||
const requestBody = useCachedContext && entry ? buildCachedWebSocketRequestBody(entry, fullBody) : fullBody;
|
||||
const stats = options?.sessionId ? getOrCreateWebSocketDebugStats(options.sessionId) : undefined;
|
||||
const stats = cacheSessionId ? getOrCreateWebSocketDebugStats(cacheSessionId) : undefined;
|
||||
if (stats) {
|
||||
stats.requests++;
|
||||
if (reused) stats.connectionsReused++;
|
||||
@@ -1445,6 +1484,7 @@ async function processWebSocketStream(
|
||||
model,
|
||||
{
|
||||
serviceTier: options?.serviceTier,
|
||||
grammarToolInputProperties,
|
||||
resolveServiceTier: resolveCodexServiceTier,
|
||||
applyServiceTierPricing: (usage, serviceTier) => applyServiceTierPricing(usage, serviceTier, model),
|
||||
},
|
||||
@@ -1454,7 +1494,8 @@ async function processWebSocketStream(
|
||||
} else if (useCachedContext && entry && output.responseId) {
|
||||
const responseItems = convertResponsesMessages(model, { messages: [output] }, CODEX_TOOL_CALL_PROVIDERS, {
|
||||
includeSystemPrompt: false,
|
||||
}).filter((item) => item.type !== "function_call_output");
|
||||
grammarToolInputProperties,
|
||||
}).filter((item) => item.type !== "function_call_output" && item.type !== "custom_tool_call_output");
|
||||
entry.continuation = {
|
||||
lastRequestBody: fullBody,
|
||||
lastResponseId: output.responseId,
|
||||
|
||||
@@ -7,6 +7,7 @@ import type {
|
||||
ChatCompletionContentPartText,
|
||||
ChatCompletionDeveloperMessageParam,
|
||||
ChatCompletionMessageParam,
|
||||
ChatCompletionMessageToolCall,
|
||||
ChatCompletionSystemMessageParam,
|
||||
ChatCompletionToolMessageParam,
|
||||
} from "openai/resources/chat/completions.js";
|
||||
@@ -38,7 +39,16 @@ import { shortHash } from "../utils/hash.ts";
|
||||
import { headersToRecord } from "../utils/headers.ts";
|
||||
import { parseStreamingJson } from "../utils/json-parse.ts";
|
||||
import { getProviderEnvValue } from "../utils/provider-env.ts";
|
||||
import { retryProviderRequest } from "../utils/provider-retry.ts";
|
||||
import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts";
|
||||
import {
|
||||
appendGrammarToolInputJsonDelta,
|
||||
createGrammarToolInputProperties,
|
||||
type GrammarToolInputJsonBuffer,
|
||||
getGrammarToolInput,
|
||||
resolveGrammarConstrainedSampling,
|
||||
resolveJsonSchemaStrictSampling,
|
||||
} from "./constrained-sampling.ts";
|
||||
import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.ts";
|
||||
import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts";
|
||||
import { buildBaseOptions } from "./simple-options.ts";
|
||||
@@ -129,10 +139,14 @@ function isEncryptedReasoningDetail(detail: unknown): detail is OpenAIEncryptedR
|
||||
}
|
||||
|
||||
export interface OpenAICompletionsOptions extends StreamOptions {
|
||||
toolChoice?: "auto" | "none" | "required" | { type: "function"; function: { name: string } };
|
||||
toolChoice?: OpenAI.Chat.Completions.ChatCompletionToolChoiceOption;
|
||||
reasoningEffort?: "minimal" | "low" | "medium" | "high" | "xhigh" | "max";
|
||||
}
|
||||
|
||||
export interface ConvertCompletionsMessagesOptions {
|
||||
grammarToolInputProperties?: ReadonlyMap<string, string>;
|
||||
}
|
||||
|
||||
interface OpenAICompatCacheControl {
|
||||
type: "ephemeral";
|
||||
ttl?: string;
|
||||
@@ -208,10 +222,14 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio
|
||||
try {
|
||||
const apiKey = getClientApiKey(model.provider, options?.apiKey, options?.headers);
|
||||
const compat = getCompat(model);
|
||||
const grammarToolInputProperties = createGrammarToolInputProperties(
|
||||
context.tools,
|
||||
compat.supportsOpenAIGrammarTools,
|
||||
);
|
||||
const cacheRetention = resolveCacheRetention(options?.cacheRetention, options?.env);
|
||||
const cacheSessionId = cacheRetention === "none" ? undefined : options?.sessionId;
|
||||
const client = createClient(model, context, apiKey, options?.headers, cacheSessionId, compat);
|
||||
let params = buildParams(model, context, options, compat, cacheRetention);
|
||||
let params = buildParams(model, context, options, compat, cacheRetention, grammarToolInputProperties);
|
||||
const nextParams = await options?.onPayload?.(params, model);
|
||||
if (nextParams !== undefined) {
|
||||
params = nextParams as OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming;
|
||||
@@ -219,20 +237,35 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio
|
||||
const requestOptions = {
|
||||
...(options?.signal ? { signal: options.signal } : {}),
|
||||
...(options?.timeoutMs !== undefined ? { timeout: options.timeoutMs } : {}),
|
||||
maxRetries: options?.maxRetries ?? 0,
|
||||
maxRetries: 0,
|
||||
};
|
||||
const { data: openaiStream, response } = await client.chat.completions
|
||||
.create(params, requestOptions)
|
||||
.withResponse();
|
||||
const { data: openaiStream, response } = await retryProviderRequest(
|
||||
() => client.chat.completions.create(params, requestOptions).withResponse(),
|
||||
{
|
||||
maxRetries: options?.maxRetries,
|
||||
maxRetryDelayMs: options?.maxRetryDelayMs,
|
||||
signal: options?.signal,
|
||||
},
|
||||
);
|
||||
await options?.onResponse?.({ status: response.status, headers: headersToRecord(response.headers) }, model);
|
||||
stream.push({ type: "start", partial: output });
|
||||
|
||||
interface StreamingToolCallBlock extends ToolCall {
|
||||
partialArgs?: string;
|
||||
customInput?: {
|
||||
property: string;
|
||||
jsonBuffer: GrammarToolInputJsonBuffer;
|
||||
};
|
||||
streamIndex?: number;
|
||||
}
|
||||
type StreamingBlock = TextContent | ThinkingContent | StreamingToolCallBlock;
|
||||
type StreamingToolCallDelta = NonNullable<ChatCompletionChunk.Choice.Delta["tool_calls"]>[number];
|
||||
type StreamingToolCallDelta = {
|
||||
index?: number;
|
||||
id?: string;
|
||||
type?: string;
|
||||
function?: { name?: string; arguments?: string };
|
||||
custom?: { name?: string; input?: string };
|
||||
};
|
||||
|
||||
let textBlock: TextContent | null = null;
|
||||
let thinkingBlock: ThinkingContent | null = null;
|
||||
@@ -242,6 +275,28 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio
|
||||
const pendingReasoningDetailsByToolCallId = new Map<string, string>();
|
||||
const blocks = output.content as StreamingBlock[];
|
||||
const getContentIndex = (block: StreamingBlock) => blocks.indexOf(block);
|
||||
const getCustomToolCallInput = (block: StreamingToolCallBlock): string => {
|
||||
const property = block.customInput?.property;
|
||||
if (property === undefined) return "";
|
||||
const value = block.arguments[property];
|
||||
return typeof value === "string" ? value : "";
|
||||
};
|
||||
const appendCustomToolCallInput = (
|
||||
block: StreamingToolCallBlock,
|
||||
nextInput: string,
|
||||
close: boolean,
|
||||
): string | undefined => {
|
||||
const customInput = block.customInput;
|
||||
if (!customInput) return undefined;
|
||||
const delta = appendGrammarToolInputJsonDelta(
|
||||
customInput.jsonBuffer,
|
||||
customInput.property,
|
||||
nextInput,
|
||||
close,
|
||||
);
|
||||
block.arguments = { [customInput.property]: nextInput };
|
||||
return delta;
|
||||
};
|
||||
const finishBlock = (block: StreamingBlock) => {
|
||||
const contentIndex = getContentIndex(block);
|
||||
if (contentIndex === -1) {
|
||||
@@ -262,10 +317,23 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio
|
||||
partial: output,
|
||||
});
|
||||
} else if (block.type === "toolCall") {
|
||||
block.arguments = parseStreamingJson(block.partialArgs);
|
||||
if (block.customInput) {
|
||||
const delta = appendCustomToolCallInput(block, getCustomToolCallInput(block), true);
|
||||
if (delta !== undefined) {
|
||||
stream.push({
|
||||
type: "toolcall_delta",
|
||||
contentIndex,
|
||||
delta,
|
||||
partial: output,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
block.arguments = parseStreamingJson(block.partialArgs);
|
||||
}
|
||||
// Finalize in-place and strip the scratch buffers so replay only
|
||||
// carries parsed arguments.
|
||||
delete block.partialArgs;
|
||||
delete block.customInput;
|
||||
delete block.streamIndex;
|
||||
stream.push({
|
||||
type: "toolcall_end",
|
||||
@@ -307,17 +375,27 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio
|
||||
};
|
||||
const ensureToolCallBlock = (toolCall: StreamingToolCallDelta) => {
|
||||
const streamIndex = typeof toolCall.index === "number" ? toolCall.index : undefined;
|
||||
const name = toolCall.function?.name ?? toolCall.custom?.name ?? "";
|
||||
let block = streamIndex !== undefined ? toolCallBlocksByIndex.get(streamIndex) : undefined;
|
||||
if (!block && toolCall.id) {
|
||||
block = toolCallBlocksById.get(toolCall.id);
|
||||
}
|
||||
if (!block) {
|
||||
// Note: the "input" fallback here should/must not be taken. in case the LLM makes up
|
||||
// a tool we don't knwo about, we at least have a place to stash our stuff.
|
||||
const customInputProperty = toolCall.custom
|
||||
? (grammarToolInputProperties.get(name) ?? "input")
|
||||
: undefined;
|
||||
const hasCustomInput = customInputProperty !== undefined;
|
||||
block = {
|
||||
type: "toolCall",
|
||||
id: toolCall.id || "",
|
||||
name: toolCall.function?.name || "",
|
||||
arguments: {},
|
||||
partialArgs: "",
|
||||
name,
|
||||
arguments: hasCustomInput ? { [customInputProperty]: "" } : {},
|
||||
partialArgs: hasCustomInput ? undefined : "",
|
||||
customInput: hasCustomInput
|
||||
? { property: customInputProperty, jsonBuffer: { input: "", started: false, closed: false } }
|
||||
: undefined,
|
||||
streamIndex,
|
||||
};
|
||||
if (streamIndex !== undefined) {
|
||||
@@ -340,6 +418,18 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio
|
||||
if (toolCall.id) {
|
||||
toolCallBlocksById.set(toolCall.id, block);
|
||||
}
|
||||
if (!block.name && name) {
|
||||
block.name = name;
|
||||
}
|
||||
if (toolCall.custom && !block.customInput) {
|
||||
const customInputProperty = grammarToolInputProperties.get(block.name) ?? "input";
|
||||
block.arguments = { [customInputProperty]: "" };
|
||||
block.customInput = {
|
||||
property: customInputProperty,
|
||||
jsonBuffer: { input: "", started: false, closed: false },
|
||||
};
|
||||
delete block.partialArgs;
|
||||
}
|
||||
applyPendingReasoningDetail(block);
|
||||
return block;
|
||||
};
|
||||
@@ -425,14 +515,15 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio
|
||||
}
|
||||
|
||||
if (choice?.delta?.tool_calls) {
|
||||
for (const toolCall of choice.delta.tool_calls) {
|
||||
for (const toolCall of choice.delta.tool_calls as StreamingToolCallDelta[]) {
|
||||
const block = ensureToolCallBlock(toolCall);
|
||||
if (!block.id && toolCall.id) {
|
||||
block.id = toolCall.id;
|
||||
toolCallBlocksById.set(toolCall.id, block);
|
||||
}
|
||||
if (!block.name && toolCall.function?.name) {
|
||||
block.name = toolCall.function.name;
|
||||
const name = toolCall.function?.name ?? toolCall.custom?.name;
|
||||
if (!block.name && name) {
|
||||
block.name = name;
|
||||
}
|
||||
|
||||
let delta = "";
|
||||
@@ -440,6 +531,9 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio
|
||||
delta = toolCall.function.arguments;
|
||||
block.partialArgs = (block.partialArgs ?? "") + toolCall.function.arguments;
|
||||
block.arguments = parseStreamingJson(block.partialArgs);
|
||||
} else if (toolCall.custom?.input) {
|
||||
const nextInput = getCustomToolCallInput(block) + toolCall.custom.input;
|
||||
delta = appendCustomToolCallInput(block, nextInput, false) ?? "";
|
||||
}
|
||||
stream.push({
|
||||
type: "toolcall_delta",
|
||||
@@ -491,6 +585,7 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio
|
||||
delete (block as { index?: number }).index;
|
||||
// Streaming scratch buffers are only used during parsing; never persist them.
|
||||
delete (block as { partialArgs?: string }).partialArgs;
|
||||
delete (block as { customInput?: unknown }).customInput;
|
||||
delete (block as { streamIndex?: number }).streamIndex;
|
||||
}
|
||||
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
|
||||
@@ -579,8 +674,12 @@ function buildParams(
|
||||
options?: OpenAICompletionsOptions,
|
||||
compat: ResolvedOpenAICompletionsCompat = getCompat(model),
|
||||
cacheRetention: CacheRetention = resolveCacheRetention(options?.cacheRetention, options?.env),
|
||||
grammarToolInputProperties: ReadonlyMap<string, string> = createGrammarToolInputProperties(
|
||||
context.tools,
|
||||
compat.supportsOpenAIGrammarTools,
|
||||
),
|
||||
) {
|
||||
const messages = convertMessages(model, context, compat);
|
||||
const messages = convertMessages(model, context, compat, { grammarToolInputProperties });
|
||||
const cacheControl = getCompatCacheControl(compat, cacheRetention);
|
||||
|
||||
const params: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = {
|
||||
@@ -889,6 +988,7 @@ export function convertMessages(
|
||||
model: Model<"openai-completions">,
|
||||
context: Context,
|
||||
compat: ResolvedOpenAICompletionsCompat,
|
||||
options?: ConvertCompletionsMessagesOptions,
|
||||
): ChatCompletionMessageParam[] {
|
||||
const params: ChatCompletionMessageParam[] = [];
|
||||
|
||||
@@ -1026,14 +1126,27 @@ export function convertMessages(
|
||||
|
||||
const toolCalls = msg.content.filter(isToolCallBlock);
|
||||
if (toolCalls.length > 0) {
|
||||
assistantMsg.tool_calls = toolCalls.map((tc) => ({
|
||||
id: tc.id,
|
||||
type: "function" as const,
|
||||
function: {
|
||||
name: tc.name,
|
||||
arguments: JSON.stringify(tc.arguments),
|
||||
},
|
||||
}));
|
||||
assistantMsg.tool_calls = toolCalls.map((tc): ChatCompletionMessageToolCall => {
|
||||
const customInputProperty = options?.grammarToolInputProperties?.get(tc.name);
|
||||
if (customInputProperty !== undefined) {
|
||||
return {
|
||||
id: tc.id,
|
||||
type: "custom",
|
||||
custom: {
|
||||
name: tc.name,
|
||||
input: sanitizeSurrogates(getGrammarToolInput(tc.name, tc.arguments, customInputProperty)),
|
||||
},
|
||||
};
|
||||
}
|
||||
return {
|
||||
id: tc.id,
|
||||
type: "function",
|
||||
function: {
|
||||
name: tc.name,
|
||||
arguments: JSON.stringify(tc.arguments),
|
||||
},
|
||||
};
|
||||
});
|
||||
const reasoningDetails = toolCalls
|
||||
.filter((tc) => tc.thoughtSignature)
|
||||
.map((tc) => {
|
||||
@@ -1166,16 +1279,37 @@ function convertTools(
|
||||
tools: Tool[],
|
||||
compat: ResolvedOpenAICompletionsCompat,
|
||||
): OpenAI.Chat.Completions.ChatCompletionTool[] {
|
||||
return tools.map((tool) => ({
|
||||
type: "function",
|
||||
function: {
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: tool.parameters as any, // TypeBox already generates JSON Schema
|
||||
// Only include strict if provider supports it. Some reject unknown fields.
|
||||
...(compat.supportsStrictMode !== false && { strict: false }),
|
||||
},
|
||||
}));
|
||||
return tools.map((tool) => {
|
||||
const grammar = resolveGrammarConstrainedSampling(tool, compat.supportsOpenAIGrammarTools);
|
||||
if (grammar) {
|
||||
return {
|
||||
type: "custom",
|
||||
custom: {
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
format: {
|
||||
type: "grammar",
|
||||
grammar: {
|
||||
syntax: grammar.format,
|
||||
definition: grammar.definition,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const strict = resolveJsonSchemaStrictSampling(tool, compat.supportsStrictMode !== false);
|
||||
return {
|
||||
type: "function",
|
||||
function: {
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: tool.parameters as Record<string, unknown>, // TypeBox already generates JSON Schema
|
||||
// Only include strict if provider supports it. Some reject unknown fields.
|
||||
...(compat.supportsStrictMode !== false && { strict: strict ?? false }),
|
||||
},
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function parseChunkUsage(
|
||||
@@ -1318,6 +1452,7 @@ function detectCompat(model: Model<"openai-completions">): ResolvedOpenAIComplet
|
||||
chatTemplateKwargs: {},
|
||||
zaiToolStream: false,
|
||||
supportsStrictMode: !isMoonshot && !isTogether && !isCloudflareAiGateway && !isNvidia,
|
||||
supportsOpenAIGrammarTools: false,
|
||||
cacheControlFormat,
|
||||
sendSessionAffinityHeaders: false,
|
||||
deferredToolsMode: undefined,
|
||||
@@ -1359,6 +1494,7 @@ function getCompat(model: Model<"openai-completions">): ResolvedOpenAICompletion
|
||||
chatTemplateKwargs: model.compat.chatTemplateKwargs ?? detected.chatTemplateKwargs,
|
||||
zaiToolStream: model.compat.zaiToolStream ?? detected.zaiToolStream,
|
||||
supportsStrictMode: model.compat.supportsStrictMode ?? detected.supportsStrictMode,
|
||||
supportsOpenAIGrammarTools: model.compat.supportsOpenAIGrammarTools ?? detected.supportsOpenAIGrammarTools,
|
||||
cacheControlFormat: model.compat.cacheControlFormat ?? detected.cacheControlFormat,
|
||||
sendSessionAffinityHeaders: model.compat.sendSessionAffinityHeaders ?? detected.sendSessionAffinityHeaders,
|
||||
deferredToolsMode: model.compat.deferredToolsMode ?? detected.deferredToolsMode,
|
||||
|
||||
@@ -2,7 +2,6 @@ import type OpenAI from "openai";
|
||||
import type {
|
||||
Tool as OpenAITool,
|
||||
ResponseCreateParamsStreaming,
|
||||
ResponseFunctionCallOutputItemList,
|
||||
ResponseInput,
|
||||
ResponseInputContent,
|
||||
ResponseInputImage,
|
||||
@@ -33,6 +32,13 @@ import type { AssistantMessageEventStream } from "../utils/event-stream.ts";
|
||||
import { shortHash } from "../utils/hash.ts";
|
||||
import { parseStreamingJson } from "../utils/json-parse.ts";
|
||||
import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts";
|
||||
import {
|
||||
appendGrammarToolInputJsonDelta,
|
||||
type GrammarToolInputJsonBuffer,
|
||||
getGrammarToolInput,
|
||||
resolveGrammarConstrainedSampling,
|
||||
resolveJsonSchemaStrictSampling,
|
||||
} from "./constrained-sampling.ts";
|
||||
import { transformMessages } from "./transform-messages.ts";
|
||||
|
||||
// =============================================================================
|
||||
@@ -65,8 +71,40 @@ function parseTextSignature(
|
||||
return { id: signature };
|
||||
}
|
||||
|
||||
type ToolResultOutputContent = Array<ResponseInputText | ResponseInputImage>;
|
||||
|
||||
function convertToolResultOutput<TApi extends Api>(
|
||||
model: Model<TApi>,
|
||||
content: readonly (TextContent | ImageContent)[],
|
||||
): string | ToolResultOutputContent {
|
||||
const textResult = content
|
||||
.filter((c): c is TextContent => c.type === "text")
|
||||
.map((c) => c.text)
|
||||
.join("\n");
|
||||
const images = content.filter((c): c is ImageContent => c.type === "image");
|
||||
const hasText = textResult.length > 0;
|
||||
|
||||
if (images.length === 0 || !model.input.includes("image")) {
|
||||
return sanitizeSurrogates(hasText ? textResult : images.length > 0 ? "(see attached image)" : "(no tool output)");
|
||||
}
|
||||
|
||||
const output: ToolResultOutputContent = [];
|
||||
if (hasText) {
|
||||
output.push({ type: "input_text", text: sanitizeSurrogates(textResult) });
|
||||
}
|
||||
for (const image of images) {
|
||||
output.push({
|
||||
type: "input_image",
|
||||
detail: "auto",
|
||||
image_url: `data:${image.mimeType};base64,${image.data}`,
|
||||
});
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
export interface OpenAIResponsesStreamOptions {
|
||||
serviceTier?: ResponseCreateParamsStreaming["service_tier"];
|
||||
grammarToolInputProperties?: ReadonlyMap<string, string>;
|
||||
resolveServiceTier?: (
|
||||
responseServiceTier: ResponseCreateParamsStreaming["service_tier"] | undefined,
|
||||
requestServiceTier: ResponseCreateParamsStreaming["service_tier"] | undefined,
|
||||
@@ -79,16 +117,18 @@ export interface OpenAIResponsesStreamOptions {
|
||||
|
||||
export interface ConvertResponsesMessagesOptions {
|
||||
includeSystemPrompt?: boolean;
|
||||
grammarToolInputProperties?: ReadonlyMap<string, string>;
|
||||
deferredTools?: ReadonlyMap<string, Tool>;
|
||||
toolOptions?: ConvertResponsesToolsOptions;
|
||||
}
|
||||
|
||||
export interface ConvertResponsesToolsOptions {
|
||||
strict?: boolean | null;
|
||||
supportsStrictMode?: boolean;
|
||||
supportsOpenAIGrammarTools?: boolean;
|
||||
deferLoading?: boolean;
|
||||
}
|
||||
|
||||
type OpenAIFunctionTool = Extract<OpenAITool, { type: "function" }>;
|
||||
|
||||
// =============================================================================
|
||||
// Message conversion
|
||||
// =============================================================================
|
||||
@@ -206,67 +246,62 @@ export function convertResponsesMessages<TApi extends Api>(
|
||||
} else if (block.type === "toolCall") {
|
||||
const toolCall = block as ToolCall;
|
||||
const [callId, itemIdRaw] = toolCall.id.split("|");
|
||||
const customInputProperty = options?.grammarToolInputProperties?.get(toolCall.name);
|
||||
let itemId: string | undefined = itemIdRaw;
|
||||
|
||||
// For different-model messages, set id to undefined to avoid pairing validation.
|
||||
// OpenAI tracks which fc_xxx IDs were paired with rs_xxx reasoning items.
|
||||
// By omitting the id, we avoid triggering that validation (like cross-provider does).
|
||||
if (isDifferentModel && itemId?.startsWith("fc_")) {
|
||||
// When replaying custom-tool calls as a function_call, also drop non-fc_* ids such as
|
||||
// ctc_* custom-tool ids because function_call item ids must be fc_*.
|
||||
if (
|
||||
(isDifferentModel && itemId?.startsWith("fc_")) ||
|
||||
(customInputProperty === undefined && !itemId?.startsWith("fc_"))
|
||||
) {
|
||||
itemId = undefined;
|
||||
}
|
||||
|
||||
output.push({
|
||||
type: "function_call",
|
||||
id: itemId,
|
||||
call_id: callId,
|
||||
name: toolCall.name,
|
||||
arguments: JSON.stringify(toolCall.arguments),
|
||||
});
|
||||
if (customInputProperty !== undefined) {
|
||||
output.push({
|
||||
type: "custom_tool_call",
|
||||
id: itemId,
|
||||
call_id: callId,
|
||||
name: toolCall.name,
|
||||
input: sanitizeSurrogates(
|
||||
getGrammarToolInput(toolCall.name, toolCall.arguments, customInputProperty),
|
||||
),
|
||||
} satisfies ResponseOutputItem);
|
||||
} else {
|
||||
output.push({
|
||||
type: "function_call",
|
||||
id: itemId,
|
||||
call_id: callId,
|
||||
name: toolCall.name,
|
||||
arguments: JSON.stringify(toolCall.arguments),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
if (output.length === 0) continue;
|
||||
messages.push(...output);
|
||||
} else if (msg.role === "toolResult") {
|
||||
const textResult = msg.content
|
||||
.filter((c): c is TextContent => c.type === "text")
|
||||
.map((c) => c.text)
|
||||
.join("\n");
|
||||
const hasImages = msg.content.some((c): c is ImageContent => c.type === "image");
|
||||
const hasText = textResult.length > 0;
|
||||
const [callId] = msg.toolCallId.split("|");
|
||||
const output = convertToolResultOutput(model, msg.content);
|
||||
|
||||
let output: string | ResponseFunctionCallOutputItemList;
|
||||
if (hasImages && model.input.includes("image")) {
|
||||
const contentParts: ResponseFunctionCallOutputItemList = [];
|
||||
|
||||
if (hasText) {
|
||||
contentParts.push({
|
||||
type: "input_text",
|
||||
text: sanitizeSurrogates(textResult),
|
||||
});
|
||||
}
|
||||
|
||||
for (const block of msg.content) {
|
||||
if (block.type === "image") {
|
||||
contentParts.push({
|
||||
type: "input_image",
|
||||
detail: "auto",
|
||||
image_url: `data:${block.mimeType};base64,${block.data}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
output = contentParts;
|
||||
if (options?.grammarToolInputProperties?.has(msg.toolName)) {
|
||||
messages.push({
|
||||
type: "custom_tool_call_output",
|
||||
call_id: callId,
|
||||
output,
|
||||
});
|
||||
} else {
|
||||
output = sanitizeSurrogates(hasText ? textResult : hasImages ? "(see attached image)" : "(no tool output)");
|
||||
messages.push({
|
||||
type: "function_call_output",
|
||||
call_id: callId,
|
||||
output,
|
||||
});
|
||||
}
|
||||
|
||||
messages.push({
|
||||
type: "function_call_output",
|
||||
call_id: callId,
|
||||
output,
|
||||
});
|
||||
|
||||
const deferredTools: Tool[] = [];
|
||||
for (const name of msg.addedToolNames ?? []) {
|
||||
const tool = options?.deferredTools?.get(name);
|
||||
@@ -289,7 +324,10 @@ export function convertResponsesMessages<TApi extends Api>(
|
||||
call_id: searchCallId,
|
||||
execution: "client",
|
||||
status: "completed",
|
||||
tools: convertResponsesTools(deferredTools, { deferLoading: true }),
|
||||
tools: convertResponsesTools(deferredTools, {
|
||||
...options?.toolOptions,
|
||||
deferLoading: true,
|
||||
}),
|
||||
} satisfies ResponseToolSearchOutputItemParam);
|
||||
}
|
||||
}
|
||||
@@ -304,30 +342,77 @@ export function convertResponsesMessages<TApi extends Api>(
|
||||
// =============================================================================
|
||||
|
||||
export function convertResponsesTools(tools: readonly Tool[], options?: ConvertResponsesToolsOptions): OpenAITool[] {
|
||||
const strict = options?.strict === undefined ? false : options.strict;
|
||||
return tools.map(
|
||||
(tool): OpenAIFunctionTool => ({
|
||||
const defaultStrict = options?.strict === undefined ? false : options.strict;
|
||||
const supportsStrictMode = options?.supportsStrictMode ?? true;
|
||||
const supportsOpenAIGrammarTools = options?.supportsOpenAIGrammarTools ?? false;
|
||||
|
||||
return tools.map((tool) => {
|
||||
const grammar = resolveGrammarConstrainedSampling(tool, supportsOpenAIGrammarTools);
|
||||
if (grammar) {
|
||||
return {
|
||||
type: "custom",
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
format: {
|
||||
type: "grammar",
|
||||
syntax: grammar.format,
|
||||
definition: grammar.definition,
|
||||
},
|
||||
...(options?.deferLoading ? { defer_loading: true } : {}),
|
||||
} satisfies OpenAITool;
|
||||
}
|
||||
|
||||
const constrainedStrict = resolveJsonSchemaStrictSampling(tool, supportsStrictMode);
|
||||
const functionTool: Omit<Extract<OpenAITool, { type: "function" }>, "strict"> & {
|
||||
strict?: Extract<OpenAITool, { type: "function" }>["strict"];
|
||||
} = {
|
||||
type: "function",
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: tool.parameters as Record<string, unknown>, // TypeBox already generates JSON Schema
|
||||
strict,
|
||||
...(options?.deferLoading ? { defer_loading: true } : {}),
|
||||
}),
|
||||
);
|
||||
};
|
||||
if (supportsStrictMode) {
|
||||
functionTool.strict = constrainedStrict ?? defaultStrict;
|
||||
}
|
||||
return functionTool as OpenAITool;
|
||||
});
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Stream processing
|
||||
// =============================================================================
|
||||
|
||||
type StreamingToolCall = ToolCall & { partialJson: string };
|
||||
type StreamingToolCall = ToolCall & {
|
||||
partialJson?: string;
|
||||
customInput?: {
|
||||
property: string;
|
||||
jsonBuffer: GrammarToolInputJsonBuffer;
|
||||
};
|
||||
};
|
||||
|
||||
function getCustomToolCallInput(block: StreamingToolCall): string {
|
||||
const property = block.customInput?.property;
|
||||
if (property === undefined) return "";
|
||||
const value = block.arguments[property];
|
||||
return typeof value === "string" ? value : "";
|
||||
}
|
||||
|
||||
function appendCustomToolCallInput(block: StreamingToolCall, nextInput: string, close: boolean): string | undefined {
|
||||
const customInput = block.customInput;
|
||||
if (!customInput) return undefined;
|
||||
const delta = appendGrammarToolInputJsonDelta(customInput.jsonBuffer, customInput.property, nextInput, close);
|
||||
block.arguments = { [customInput.property]: nextInput };
|
||||
return delta;
|
||||
}
|
||||
|
||||
type ResponsesOutputSlot =
|
||||
| { type: "thinking"; block: ThinkingContent; contentIndex: number }
|
||||
| { type: "text"; block: TextContent; contentIndex: number }
|
||||
| { type: "toolCall"; block: StreamingToolCall; contentIndex: number };
|
||||
|
||||
type ToolCallOutputSlot = Extract<ResponsesOutputSlot, { type: "toolCall" }>;
|
||||
|
||||
export async function processResponsesStream<TApi extends Api>(
|
||||
openaiStream: AsyncIterable<ResponseStreamEvent>,
|
||||
output: AssistantMessage,
|
||||
@@ -345,6 +430,15 @@ export async function processResponsesStream<TApi extends Api>(
|
||||
const slot = outputSlots.get(outputIndex);
|
||||
return slot?.type === type ? (slot as Extract<ResponsesOutputSlot, { type: TType }>) : undefined;
|
||||
};
|
||||
const pushToolCallDelta = (slot: ToolCallOutputSlot, delta: string | undefined): void => {
|
||||
if (delta === undefined) return;
|
||||
stream.push({
|
||||
type: "toolcall_delta",
|
||||
contentIndex: slot.contentIndex,
|
||||
delta,
|
||||
partial: output,
|
||||
});
|
||||
};
|
||||
const createSlot = (outputIndex: number, item: ResponseOutputItem): ResponsesOutputSlot | undefined => {
|
||||
if (item.type === "reasoning") {
|
||||
const block: ThinkingContent = { type: "thinking", thinking: "" };
|
||||
@@ -384,6 +478,29 @@ export async function processResponsesStream<TApi extends Api>(
|
||||
stream.push({ type: "toolcall_start", contentIndex: slot.contentIndex, partial: output });
|
||||
return slot;
|
||||
}
|
||||
if (item.type === "custom_tool_call") {
|
||||
const inputProperty = options?.grammarToolInputProperties?.get(item.name) ?? "input";
|
||||
const input = item.input || "";
|
||||
const block: StreamingToolCall = {
|
||||
type: "toolCall",
|
||||
id: `${item.call_id}|${item.id}`,
|
||||
name: item.name,
|
||||
arguments: { [inputProperty]: input },
|
||||
customInput: {
|
||||
property: inputProperty,
|
||||
jsonBuffer: { input: "", started: false, closed: false },
|
||||
},
|
||||
};
|
||||
output.content.push(block);
|
||||
const slot = {
|
||||
type: "toolCall",
|
||||
block,
|
||||
contentIndex: output.content.length - 1,
|
||||
} satisfies ResponsesOutputSlot;
|
||||
outputSlots.set(outputIndex, slot);
|
||||
stream.push({ type: "toolcall_start", contentIndex: slot.contentIndex, partial: output });
|
||||
return slot;
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
const getOrCreateSlot = (outputIndex: number, item: ResponseOutputItem): ResponsesOutputSlot | undefined => {
|
||||
@@ -503,33 +620,32 @@ export async function processResponsesStream<TApi extends Api>(
|
||||
});
|
||||
} else if (event.type === "response.function_call_arguments.delta") {
|
||||
const slot = getSlot(event.output_index, "toolCall");
|
||||
if (!slot) continue;
|
||||
if (!slot || slot.block.partialJson === undefined) continue;
|
||||
slot.block.partialJson += event.delta;
|
||||
slot.block.arguments = parseStreamingJson(slot.block.partialJson);
|
||||
stream.push({
|
||||
type: "toolcall_delta",
|
||||
contentIndex: slot.contentIndex,
|
||||
delta: event.delta,
|
||||
partial: output,
|
||||
});
|
||||
pushToolCallDelta(slot, event.delta);
|
||||
} else if (event.type === "response.function_call_arguments.done") {
|
||||
const slot = getSlot(event.output_index, "toolCall");
|
||||
if (!slot) continue;
|
||||
if (!slot || slot.block.partialJson === undefined) continue;
|
||||
const previousPartialJson = slot.block.partialJson;
|
||||
slot.block.partialJson = event.arguments;
|
||||
slot.block.arguments = parseStreamingJson(slot.block.partialJson);
|
||||
|
||||
if (event.arguments.startsWith(previousPartialJson)) {
|
||||
const delta = event.arguments.slice(previousPartialJson.length);
|
||||
if (delta.length > 0) {
|
||||
stream.push({
|
||||
type: "toolcall_delta",
|
||||
contentIndex: slot.contentIndex,
|
||||
delta,
|
||||
partial: output,
|
||||
});
|
||||
}
|
||||
if (delta.length > 0) pushToolCallDelta(slot, delta);
|
||||
}
|
||||
} else if (event.type === "response.custom_tool_call_input.delta") {
|
||||
const slot = getSlot(event.output_index, "toolCall");
|
||||
if (!slot || !slot.block.customInput) continue;
|
||||
pushToolCallDelta(
|
||||
slot,
|
||||
appendCustomToolCallInput(slot.block, getCustomToolCallInput(slot.block) + event.delta, false),
|
||||
);
|
||||
} else if (event.type === "response.custom_tool_call_input.done") {
|
||||
const slot = getSlot(event.output_index, "toolCall");
|
||||
if (!slot || !slot.block.customInput) continue;
|
||||
pushToolCallDelta(slot, appendCustomToolCallInput(slot.block, event.input, true));
|
||||
} else if (event.type === "response.output_item.done") {
|
||||
const item = event.item;
|
||||
const slot = getOrCreateSlot(event.output_index, item);
|
||||
@@ -557,11 +673,28 @@ export async function processResponsesStream<TApi extends Api>(
|
||||
partial: output,
|
||||
});
|
||||
outputSlots.delete(event.output_index);
|
||||
} else if (item.type === "function_call" && slot?.type === "toolCall") {
|
||||
} else if (
|
||||
item.type === "function_call" &&
|
||||
slot?.type === "toolCall" &&
|
||||
slot.block.partialJson !== undefined
|
||||
) {
|
||||
slot.block.arguments = parseStreamingJson(item.arguments || slot.block.partialJson || "{}");
|
||||
// Finalize in-place and strip the scratch buffer so replay only
|
||||
// carries parsed arguments.
|
||||
delete (slot.block as { partialJson?: string }).partialJson;
|
||||
delete slot.block.partialJson;
|
||||
stream.push({
|
||||
type: "toolcall_end",
|
||||
contentIndex: slot.contentIndex,
|
||||
toolCall: slot.block,
|
||||
partial: output,
|
||||
});
|
||||
outputSlots.delete(event.output_index);
|
||||
} else if (item.type === "custom_tool_call" && slot?.type === "toolCall" && slot.block.customInput) {
|
||||
pushToolCallDelta(
|
||||
slot,
|
||||
appendCustomToolCallInput(slot.block, item.input ?? getCustomToolCallInput(slot.block), true),
|
||||
);
|
||||
delete slot.block.customInput;
|
||||
stream.push({
|
||||
type: "toolcall_end",
|
||||
contentIndex: slot.contentIndex,
|
||||
|
||||
@@ -20,6 +20,8 @@ import { formatProviderError, normalizeProviderError } from "../utils/error-body
|
||||
import { AssistantMessageEventStream } from "../utils/event-stream.ts";
|
||||
import { headersToRecord } from "../utils/headers.ts";
|
||||
import { getProviderEnvValue } from "../utils/provider-env.ts";
|
||||
import { retryProviderRequest } from "../utils/provider-retry.ts";
|
||||
import { createGrammarToolInputProperties } from "./constrained-sampling.ts";
|
||||
import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.ts";
|
||||
import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts";
|
||||
import { convertResponsesMessages, convertResponsesTools, processResponsesStream } from "./openai-responses-shared.ts";
|
||||
@@ -67,7 +69,10 @@ function getCompat(model: Model<"openai-responses">): Required<OpenAIResponsesCo
|
||||
supportsDeveloperRole: model.compat?.supportsDeveloperRole ?? true,
|
||||
sessionAffinityFormat: model.compat?.sessionAffinityFormat ?? detectSessionAffinityFormat(model),
|
||||
supportsLongCacheRetention: model.compat?.supportsLongCacheRetention ?? true,
|
||||
supportsStrictMode: model.compat?.supportsStrictMode ?? false,
|
||||
supportsOpenAIGrammarTools: model.compat?.supportsOpenAIGrammarTools ?? false,
|
||||
supportsToolSearch: model.compat?.supportsToolSearch ?? false,
|
||||
supportsExplicitPromptCacheMode: model.compat?.supportsExplicitPromptCacheMode ?? false,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -125,8 +130,13 @@ export const stream: StreamFunction<"openai-responses", OpenAIResponsesOptions>
|
||||
const apiKey = getClientApiKey(model.provider, options?.apiKey, options?.headers);
|
||||
const cacheRetention = resolveCacheRetention(options?.cacheRetention, options?.env);
|
||||
const cacheSessionId = cacheRetention === "none" ? undefined : options?.sessionId;
|
||||
const compat = getCompat(model);
|
||||
const grammarToolInputProperties = createGrammarToolInputProperties(
|
||||
context.tools,
|
||||
compat.supportsOpenAIGrammarTools,
|
||||
);
|
||||
const client = createClient(model, context, apiKey, options?.headers, cacheSessionId);
|
||||
let params = buildParams(model, context, options);
|
||||
let params = buildParams(model, context, options, compat, grammarToolInputProperties);
|
||||
const nextParams = await options?.onPayload?.(params, model);
|
||||
if (nextParams !== undefined) {
|
||||
params = nextParams as ResponseCreateParamsStreaming;
|
||||
@@ -134,14 +144,22 @@ export const stream: StreamFunction<"openai-responses", OpenAIResponsesOptions>
|
||||
const requestOptions = {
|
||||
...(options?.signal ? { signal: options.signal } : {}),
|
||||
...(options?.timeoutMs !== undefined ? { timeout: options.timeoutMs } : {}),
|
||||
maxRetries: options?.maxRetries ?? 0,
|
||||
maxRetries: 0,
|
||||
};
|
||||
const { data: openaiStream, response } = await client.responses.create(params, requestOptions).withResponse();
|
||||
const { data: openaiStream, response } = await retryProviderRequest(
|
||||
() => client.responses.create(params, requestOptions).withResponse(),
|
||||
{
|
||||
maxRetries: options?.maxRetries,
|
||||
maxRetryDelayMs: options?.maxRetryDelayMs,
|
||||
signal: options?.signal,
|
||||
},
|
||||
);
|
||||
await options?.onResponse?.({ status: response.status, headers: headersToRecord(response.headers) }, model);
|
||||
stream.push({ type: "start", partial: output });
|
||||
|
||||
await processResponsesStream(openaiStream, output, stream, model, {
|
||||
serviceTier: options?.serviceTier,
|
||||
grammarToolInputProperties,
|
||||
applyServiceTierPricing: (usage, serviceTier) => applyServiceTierPricing(usage, serviceTier, model),
|
||||
});
|
||||
|
||||
@@ -158,8 +176,9 @@ export const stream: StreamFunction<"openai-responses", OpenAIResponsesOptions>
|
||||
} catch (error) {
|
||||
for (const block of output.content) {
|
||||
delete (block as { index?: number }).index;
|
||||
// partialJson is only a streaming scratch buffer; never persist it.
|
||||
// Streaming scratch buffers are only used during parsing; never persist them.
|
||||
delete (block as { partialJson?: string }).partialJson;
|
||||
delete (block as { customInput?: unknown }).customInput;
|
||||
}
|
||||
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
|
||||
output.errorMessage = formatOpenAIResponsesError(error);
|
||||
@@ -230,20 +249,35 @@ function createClient(
|
||||
});
|
||||
}
|
||||
|
||||
function buildParams(model: Model<"openai-responses">, context: Context, options?: OpenAIResponsesOptions) {
|
||||
const compat = getCompat(model);
|
||||
function buildParams(
|
||||
model: Model<"openai-responses">,
|
||||
context: Context,
|
||||
options: OpenAIResponsesOptions | undefined,
|
||||
compat: Required<OpenAIResponsesCompat> = getCompat(model),
|
||||
grammarToolInputProperties: ReadonlyMap<string, string> = createGrammarToolInputProperties(
|
||||
context.tools,
|
||||
compat.supportsOpenAIGrammarTools,
|
||||
),
|
||||
) {
|
||||
const toolPlacement = splitDeferredTools(context, compat.supportsToolSearch);
|
||||
const messages = convertResponsesMessages(model, context, OPENAI_TOOL_CALL_PROVIDERS, {
|
||||
grammarToolInputProperties,
|
||||
deferredTools: toolPlacement.deferred,
|
||||
toolOptions: {
|
||||
supportsStrictMode: compat.supportsStrictMode,
|
||||
supportsOpenAIGrammarTools: compat.supportsOpenAIGrammarTools,
|
||||
},
|
||||
});
|
||||
|
||||
const cacheRetention = resolveCacheRetention(options?.cacheRetention, options?.env);
|
||||
const params: ResponseCreateParamsStreaming = {
|
||||
const disableImplicitPromptCache = cacheRetention === "none" && compat.supportsExplicitPromptCacheMode;
|
||||
const params: ResponseCreateParamsStreaming & { prompt_cache_options?: { mode: "explicit" } } = {
|
||||
model: model.id,
|
||||
input: messages,
|
||||
stream: true,
|
||||
prompt_cache_key: cacheRetention === "none" ? undefined : clampOpenAIPromptCacheKey(options?.sessionId),
|
||||
prompt_cache_retention: getPromptCacheRetention(compat, cacheRetention),
|
||||
prompt_cache_options: disableImplicitPromptCache ? { mode: "explicit" } : undefined,
|
||||
store: false,
|
||||
};
|
||||
|
||||
@@ -260,7 +294,10 @@ function buildParams(model: Model<"openai-responses">, context: Context, options
|
||||
}
|
||||
|
||||
if (toolPlacement.immediate.length > 0) {
|
||||
params.tools = convertResponsesTools(toolPlacement.immediate);
|
||||
params.tools = convertResponsesTools(toolPlacement.immediate, {
|
||||
supportsStrictMode: compat.supportsStrictMode,
|
||||
supportsOpenAIGrammarTools: compat.supportsOpenAIGrammarTools,
|
||||
});
|
||||
}
|
||||
|
||||
if (options?.toolChoice !== undefined) {
|
||||
|
||||
@@ -18,6 +18,7 @@ import type {
|
||||
} from "../types.ts";
|
||||
import { formatProviderError, normalizeProviderError } from "../utils/error-body.ts";
|
||||
import { headersToRecord, providerHeadersToRecord } from "../utils/headers.ts";
|
||||
import { retryProviderRequest } from "../utils/provider-retry.ts";
|
||||
import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts";
|
||||
|
||||
interface OpenRouterGeneratedImage {
|
||||
@@ -64,11 +65,19 @@ export const generateImages: ImagesFunction<"openrouter-images", ImagesOptions>
|
||||
const requestOptions = {
|
||||
...(options?.signal ? { signal: options.signal } : {}),
|
||||
...(options?.timeoutMs !== undefined ? { timeout: options.timeoutMs } : {}),
|
||||
maxRetries: options?.maxRetries ?? 0,
|
||||
maxRetries: 0,
|
||||
};
|
||||
const { data: response, response: rawResponse } = await client.chat.completions
|
||||
.create(params as unknown as ChatCompletionCreateParamsNonStreaming, requestOptions)
|
||||
.withResponse();
|
||||
const { data: response, response: rawResponse } = await retryProviderRequest(
|
||||
() =>
|
||||
client.chat.completions
|
||||
.create(params as unknown as ChatCompletionCreateParamsNonStreaming, requestOptions)
|
||||
.withResponse(),
|
||||
{
|
||||
maxRetries: options?.maxRetries,
|
||||
maxRetryDelayMs: options?.maxRetryDelayMs,
|
||||
signal: options?.signal,
|
||||
},
|
||||
);
|
||||
await options?.onResponse?.({ status: rawResponse.status, headers: headersToRecord(rawResponse.headers) }, model);
|
||||
|
||||
const imageResponse = response as OpenRouterImageGenerationResponse;
|
||||
|
||||
@@ -47,7 +47,7 @@ export const loadOpenRouterOAuth = async (): Promise<OAuthAuth> => {
|
||||
if (bundledLoaders) return bundledLoaders.openrouter();
|
||||
return ((await importOAuthModule("./openrouter.ts")) as { openRouterOAuth: OAuthAuth }).openRouterOAuth;
|
||||
};
|
||||
|
||||
|
||||
export const loadKimiCodingOAuth = async (): Promise<OAuthAuth> => {
|
||||
if (bundledLoaders) return bundledLoaders.kimiCoding();
|
||||
return ((await importOAuthModule("./kimi-coding.ts")) as { kimiCodingOAuth: OAuthAuth }).kimiCodingOAuth;
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
/**
|
||||
* Radius gateway OAuth flow.
|
||||
*
|
||||
* Radius is a pi-messages gateway. OAuth endpoints are discovered from the
|
||||
* gateway (`/v1/oauth`); model catalog loading is owned by the Radius provider.
|
||||
* Radius is a pi-messages gateway. OAuth client APIs live on the configured
|
||||
* gateway; only the interactive browser authorization endpoint is discovered.
|
||||
* Model catalog loading is owned by the Radius provider.
|
||||
*
|
||||
* NOTE: This module uses node:http for the OAuth callback server.
|
||||
* It is only intended for CLI use, not browser environments.
|
||||
@@ -29,29 +30,23 @@ const REDIRECT_URI = `http://${CALLBACK_HOST}:${CALLBACK_PORT}${CALLBACK_PATH}`;
|
||||
const TOKEN_EXPIRY_SKEW_MS = 60_000;
|
||||
const LOGIN_METHOD_BROWSER = "browser";
|
||||
const LOGIN_METHOD_DEVICE_CODE = "device-code";
|
||||
const OAUTH_CLIENT_ID = "pi-gateway";
|
||||
const OAUTH_SCOPE = "gateway offline_access";
|
||||
const OAUTH_DEVICE_CODE_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:device_code";
|
||||
|
||||
type RadiusOAuthConfig = {
|
||||
issuer: string;
|
||||
type RadiusOAuthDiscovery = {
|
||||
authorizationEndpoint: string;
|
||||
tokenEndpoint: string;
|
||||
deviceAuthorizationEndpoint: string;
|
||||
deviceAuthorizationEventsEndpoint: string;
|
||||
verificationEndpoint: string;
|
||||
clientId: string;
|
||||
scope: string;
|
||||
deviceCodeGrantType: string;
|
||||
};
|
||||
|
||||
type DeviceAuthorizationResponse = {
|
||||
device_code: string;
|
||||
user_code: string;
|
||||
verification_uri?: string;
|
||||
verification_uri_complete?: string;
|
||||
verification_uri: string;
|
||||
expires_in: number;
|
||||
interval?: number;
|
||||
};
|
||||
|
||||
async function loadRadiusOAuthConfig(gateway: string): Promise<RadiusOAuthConfig> {
|
||||
async function loadRadiusOAuthDiscovery(gateway: string): Promise<RadiusOAuthDiscovery> {
|
||||
const response = await fetch(new URL("/v1/oauth", gateway), {
|
||||
headers: { accept: "application/json" },
|
||||
});
|
||||
@@ -62,7 +57,11 @@ async function loadRadiusOAuthConfig(gateway: string): Promise<RadiusOAuthConfig
|
||||
);
|
||||
}
|
||||
|
||||
return (await response.json()) as RadiusOAuthConfig;
|
||||
const discovery = (await response.json()) as Partial<RadiusOAuthDiscovery>;
|
||||
if (typeof discovery.authorizationEndpoint !== "string") {
|
||||
throw new Error(`Invalid Radius OAuth config from ${gateway}`);
|
||||
}
|
||||
return { authorizationEndpoint: discovery.authorizationEndpoint };
|
||||
}
|
||||
|
||||
class OAuthResponseError extends Error {
|
||||
@@ -100,13 +99,13 @@ async function readOAuthResponseError(response: Response, message: string): Prom
|
||||
}
|
||||
|
||||
async function requestOAuthToken(
|
||||
oauth: RadiusOAuthConfig,
|
||||
gateway: string,
|
||||
body: URLSearchParams,
|
||||
signal?: AbortSignal,
|
||||
): Promise<OAuthCredential> {
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(oauth.tokenEndpoint, {
|
||||
response = await fetch(new URL("/v1/oauth/token", gateway), {
|
||||
method: "POST",
|
||||
headers: { accept: "application/json", "content-type": "application/x-www-form-urlencoded" },
|
||||
body,
|
||||
@@ -220,15 +219,19 @@ function startOAuthCallbackServer(
|
||||
});
|
||||
}
|
||||
|
||||
async function loginWithBrowser(oauth: RadiusOAuthConfig, interaction: AuthInteraction): Promise<OAuthCredential> {
|
||||
async function loginWithBrowser(
|
||||
gateway: string,
|
||||
authorizationEndpoint: string,
|
||||
interaction: AuthInteraction,
|
||||
): Promise<OAuthCredential> {
|
||||
const { verifier, challenge } = await generatePKCE();
|
||||
const state = crypto.randomUUID();
|
||||
const authorizeUrl = new URL(oauth.authorizationEndpoint);
|
||||
const authorizeUrl = new URL(authorizationEndpoint);
|
||||
authorizeUrl.search = new URLSearchParams({
|
||||
response_type: "code",
|
||||
client_id: oauth.clientId,
|
||||
client_id: OAUTH_CLIENT_ID,
|
||||
redirect_uri: REDIRECT_URI,
|
||||
scope: oauth.scope,
|
||||
scope: OAUTH_SCOPE,
|
||||
code_challenge: challenge,
|
||||
code_challenge_method: "S256",
|
||||
handoff: "url",
|
||||
@@ -252,10 +255,10 @@ async function loginWithBrowser(oauth: RadiusOAuthConfig, interaction: AuthInter
|
||||
throw new Error("OAuth callback did not complete.");
|
||||
}
|
||||
return await requestOAuthToken(
|
||||
oauth,
|
||||
gateway,
|
||||
new URLSearchParams({
|
||||
grant_type: "authorization_code",
|
||||
client_id: oauth.clientId,
|
||||
client_id: OAUTH_CLIENT_ID,
|
||||
redirect_uri: REDIRECT_URI,
|
||||
code,
|
||||
code_verifier: verifier,
|
||||
@@ -268,15 +271,15 @@ async function loginWithBrowser(oauth: RadiusOAuthConfig, interaction: AuthInter
|
||||
}
|
||||
|
||||
async function requestDeviceAuthorization(
|
||||
oauth: RadiusOAuthConfig,
|
||||
gateway: string,
|
||||
signal: AbortSignal | undefined,
|
||||
): Promise<DeviceAuthorizationResponse> {
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(oauth.deviceAuthorizationEndpoint, {
|
||||
response = await fetch(new URL("/v1/oauth/device", gateway), {
|
||||
method: "POST",
|
||||
headers: { accept: "application/json", "content-type": "application/x-www-form-urlencoded" },
|
||||
body: new URLSearchParams({ client_id: oauth.clientId, scope: oauth.scope }),
|
||||
body: new URLSearchParams({ client_id: OAUTH_CLIENT_ID, scope: OAUTH_SCOPE }),
|
||||
signal,
|
||||
});
|
||||
} catch (error) {
|
||||
@@ -291,7 +294,7 @@ async function requestDeviceAuthorization(
|
||||
}
|
||||
|
||||
const data = (await response.json()) as Partial<DeviceAuthorizationResponse>;
|
||||
if (!data.device_code || !data.user_code || !data.expires_in) {
|
||||
if (!data.device_code || !data.user_code || !data.verification_uri || !data.expires_in) {
|
||||
throw new Error("Radius OAuth device authorization response is missing required fields");
|
||||
}
|
||||
|
||||
@@ -299,18 +302,17 @@ async function requestDeviceAuthorization(
|
||||
device_code: data.device_code,
|
||||
user_code: data.user_code,
|
||||
verification_uri: data.verification_uri,
|
||||
verification_uri_complete: data.verification_uri_complete,
|
||||
expires_in: data.expires_in,
|
||||
interval: data.interval,
|
||||
};
|
||||
}
|
||||
|
||||
async function loginWithDeviceCode(oauth: RadiusOAuthConfig, interaction: AuthInteraction): Promise<OAuthCredential> {
|
||||
const device = await requestDeviceAuthorization(oauth, interaction.signal);
|
||||
async function loginWithDeviceCode(gateway: string, interaction: AuthInteraction): Promise<OAuthCredential> {
|
||||
const device = await requestDeviceAuthorization(gateway, interaction.signal);
|
||||
interaction.notify({
|
||||
type: "device_code",
|
||||
userCode: device.user_code,
|
||||
verificationUri: device.verification_uri || oauth.verificationEndpoint,
|
||||
verificationUri: device.verification_uri,
|
||||
intervalSeconds: device.interval,
|
||||
expiresInSeconds: device.expires_in,
|
||||
});
|
||||
@@ -322,10 +324,10 @@ async function loginWithDeviceCode(oauth: RadiusOAuthConfig, interaction: AuthIn
|
||||
poll: async () => {
|
||||
try {
|
||||
const credentials = await requestOAuthToken(
|
||||
oauth,
|
||||
gateway,
|
||||
new URLSearchParams({
|
||||
grant_type: oauth.deviceCodeGrantType,
|
||||
client_id: oauth.clientId,
|
||||
grant_type: OAUTH_DEVICE_CODE_GRANT_TYPE,
|
||||
client_id: OAUTH_CLIENT_ID,
|
||||
device_code: device.device_code,
|
||||
}),
|
||||
interaction.signal,
|
||||
@@ -364,7 +366,6 @@ export function createRadiusOAuth(options: RadiusOAuthOptions): OAuthAuth {
|
||||
name: options.name,
|
||||
|
||||
async login(interaction): Promise<OAuthCredential> {
|
||||
const oauth = await loadRadiusOAuthConfig(gateway);
|
||||
const loginMethod = await interaction.prompt({
|
||||
type: "select",
|
||||
message: `Sign in to ${options.name}:`,
|
||||
@@ -377,25 +378,22 @@ export function createRadiusOAuth(options: RadiusOAuthOptions): OAuthAuth {
|
||||
],
|
||||
});
|
||||
|
||||
let credential: OAuthCredential;
|
||||
if (loginMethod === LOGIN_METHOD_DEVICE_CODE) {
|
||||
credential = await loginWithDeviceCode(oauth, interaction);
|
||||
} else if (loginMethod === LOGIN_METHOD_BROWSER) {
|
||||
credential = await loginWithBrowser(oauth, interaction);
|
||||
} else {
|
||||
throw new Error(`Unknown ${options.name} sign-in method: ${loginMethod}`);
|
||||
return loginWithDeviceCode(gateway, interaction);
|
||||
}
|
||||
|
||||
return credential;
|
||||
if (loginMethod === LOGIN_METHOD_BROWSER) {
|
||||
const discovery = await loadRadiusOAuthDiscovery(gateway);
|
||||
return loginWithBrowser(gateway, discovery.authorizationEndpoint, interaction);
|
||||
}
|
||||
throw new Error(`Unknown ${options.name} sign-in method: ${loginMethod}`);
|
||||
},
|
||||
|
||||
async refresh(credential, signal): Promise<OAuthCredential> {
|
||||
const oauth = await loadRadiusOAuthConfig(gateway);
|
||||
const refreshed = await requestOAuthToken(
|
||||
oauth,
|
||||
gateway,
|
||||
new URLSearchParams({
|
||||
grant_type: "refresh_token",
|
||||
client_id: oauth.clientId,
|
||||
client_id: OAUTH_CLIENT_ID,
|
||||
refresh_token: credential.refresh,
|
||||
}),
|
||||
signal,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { ProviderEnv } from "../types.ts";
|
||||
import { formatThrownValue } from "../utils/diagnostics.ts";
|
||||
import type {
|
||||
ApiKeyAuth,
|
||||
ApiKeyCredential,
|
||||
@@ -22,12 +23,20 @@ export class ModelsError extends Error {
|
||||
readonly code: ModelsErrorCode;
|
||||
|
||||
constructor(code: ModelsErrorCode, message: string, options?: { cause?: unknown }) {
|
||||
super(message, options);
|
||||
super(withCauseDetail(message, options?.cause), options);
|
||||
this.name = "ModelsError";
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
|
||||
/** Callers surface `error.message` only, so keep the underlying reason in it. */
|
||||
function withCauseDetail(message: string, cause: unknown): string {
|
||||
if (cause === undefined || cause === null) return message;
|
||||
const detail = formatThrownValue(cause).trim();
|
||||
if (!detail || message.includes(detail)) return message;
|
||||
return `${message}: ${detail}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Auth resolution shared by the `Models` and `ImagesModels` collections.
|
||||
* A stored credential owns the provider: ambient/env is consulted only when
|
||||
|
||||
@@ -26,6 +26,10 @@ if (typeof process !== "undefined" && (process.versions?.node || process.version
|
||||
import type { KnownProvider, ProviderEnv } from "./types.ts";
|
||||
import { getProviderEnvValue } from "./utils/provider-env.ts";
|
||||
|
||||
export const ANTHROPIC_AUTH_TOKEN_ENV = "ANTHROPIC_AUTH_TOKEN";
|
||||
export const ANTHROPIC_OAUTH_TOKEN_ENV = "ANTHROPIC_OAUTH_TOKEN";
|
||||
export const ANTHROPIC_API_KEY_ENV = "ANTHROPIC_API_KEY";
|
||||
|
||||
let cachedVertexAdcCredentialsExists: boolean | null = null;
|
||||
|
||||
function hasVertexAdcCredentials(env?: ProviderEnv): boolean {
|
||||
@@ -66,9 +70,10 @@ function getApiKeyEnvVars(provider: string): readonly string[] | undefined {
|
||||
return ["COPILOT_GITHUB_TOKEN"];
|
||||
}
|
||||
|
||||
// ANTHROPIC_OAUTH_TOKEN takes precedence over ANTHROPIC_API_KEY
|
||||
// ANTHROPIC_AUTH_TOKEN participates in env discovery/status, but
|
||||
// getEnvApiKey() skips it because requests must pass it as Authorization: Bearer.
|
||||
if (provider === "anthropic") {
|
||||
return ["ANTHROPIC_OAUTH_TOKEN", "ANTHROPIC_API_KEY"];
|
||||
return [ANTHROPIC_AUTH_TOKEN_ENV, ANTHROPIC_OAUTH_TOKEN_ENV, ANTHROPIC_API_KEY_ENV];
|
||||
}
|
||||
|
||||
const envMap: Record<string, string> = {
|
||||
@@ -139,7 +144,8 @@ export function getEnvApiKey(provider: string, env?: ProviderEnv): string | unde
|
||||
export function getEnvApiKey(provider: string, env?: ProviderEnv): string | undefined {
|
||||
const envKeys = findEnvKeys(provider, env);
|
||||
if (envKeys?.[0]) {
|
||||
return getProviderEnvValue(envKeys[0], env);
|
||||
const apiKeyEnv = provider === "anthropic" ? envKeys.find((key) => key !== ANTHROPIC_AUTH_TOKEN_ENV) : envKeys[0];
|
||||
if (apiKeyEnv) return getProviderEnvValue(apiKeyEnv, env);
|
||||
}
|
||||
|
||||
// Vertex AI supports either an explicit API key or Application Default Credentials.
|
||||
|
||||
@@ -230,6 +230,21 @@ export const IMAGE_MODELS = {
|
||||
cacheWrite: 0,
|
||||
},
|
||||
} satisfies ImagesModel<"openrouter-images">,
|
||||
"microsoft/mai-image-2.5-pro": {
|
||||
id: "microsoft/mai-image-2.5-pro",
|
||||
name: "Microsoft: MAI-Image-2.5 Pro",
|
||||
api: "openrouter-images",
|
||||
provider: "openrouter",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
input: ["text", "image"],
|
||||
output: ["image"],
|
||||
cost: {
|
||||
input: 5,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
} satisfies ImagesModel<"openrouter-images">,
|
||||
"openai/gpt-5-image": {
|
||||
id: "openai/gpt-5-image",
|
||||
name: "OpenAI: GPT-5 Image",
|
||||
|
||||
@@ -6,6 +6,11 @@ export interface ModelsStoreEntry {
|
||||
lastModified?: number;
|
||||
/** Unix timestamp of the last completed remote check. */
|
||||
checkedAt?: number;
|
||||
/**
|
||||
* Opaque validator from the remote catalog's ETag header, stored verbatim
|
||||
* (quotes included) and echoed back as If-None-Match.
|
||||
*/
|
||||
etag?: string;
|
||||
}
|
||||
|
||||
/** Persistent model catalogs keyed by provider ID. */
|
||||
|
||||
@@ -9,6 +9,7 @@ import { azureOpenAIResponsesProvider } from "./azure-openai-responses.ts";
|
||||
import { cerebrasProvider } from "./cerebras.ts";
|
||||
import { cloudflareAIGatewayProvider } from "./cloudflare-ai-gateway.ts";
|
||||
import { cloudflareWorkersAIProvider } from "./cloudflare-workers-ai.ts";
|
||||
import modelDataManifest from "./data/.manifest.json" with { type: "json" };
|
||||
import { deepseekProvider } from "./deepseek.ts";
|
||||
import { fireworksProvider } from "./fireworks.ts";
|
||||
import { githubCopilotProvider } from "./github-copilot.ts";
|
||||
@@ -67,9 +68,10 @@ export function getBuiltinProviders(): BuiltinProvider[] {
|
||||
return Object.keys(MODELS) as BuiltinProvider[];
|
||||
}
|
||||
|
||||
/** URL of a generated provider catalog, used to compare its mtime with remote catalogs during development. */
|
||||
export function getBuiltinModelDataUrl(provider: BuiltinProvider): URL {
|
||||
return new URL(`./data/${provider}.json`, import.meta.url);
|
||||
/** Generation timestamp shared by all built-in provider catalogs. */
|
||||
export function getBuiltinModelDataGeneratedAt(): number | undefined {
|
||||
const generatedAt = Date.parse(modelDataManifest.generatedAt);
|
||||
return Number.isNaN(generatedAt) ? undefined : generatedAt;
|
||||
}
|
||||
|
||||
export function getBuiltinModels<TProvider extends BuiltinProvider>(
|
||||
|
||||
@@ -1,17 +1,47 @@
|
||||
import { anthropicMessagesApi } from "../api/anthropic-messages.lazy.ts";
|
||||
import { envApiKeyAuth, lazyOAuth } from "../auth/helpers.ts";
|
||||
import { lazyOAuth } from "../auth/helpers.ts";
|
||||
import { loadAnthropicOAuth } from "../auth/oauth/load.ts";
|
||||
import type { ApiKeyAuth } from "../auth/types.ts";
|
||||
import { ANTHROPIC_API_KEY_ENV, ANTHROPIC_AUTH_TOKEN_ENV, ANTHROPIC_OAUTH_TOKEN_ENV } from "../env-api-keys.ts";
|
||||
import { createProvider, type Provider } from "../models.ts";
|
||||
import { ANTHROPIC_MODELS } from "./anthropic.models.ts";
|
||||
|
||||
function anthropicApiKeyAuth(): ApiKeyAuth {
|
||||
return {
|
||||
name: "Anthropic API key",
|
||||
login: async (interaction) => ({
|
||||
type: "api_key",
|
||||
key: await interaction.prompt({ type: "secret", message: "Enter Anthropic API key" }),
|
||||
}),
|
||||
resolve: async ({ ctx, credential }) => {
|
||||
if (credential?.key) {
|
||||
return { auth: { apiKey: credential.key }, env: credential.env, source: "stored credential" };
|
||||
}
|
||||
|
||||
const authToken = await ctx.env(ANTHROPIC_AUTH_TOKEN_ENV);
|
||||
if (authToken) {
|
||||
return {
|
||||
auth: { headers: { Authorization: `Bearer ${authToken}` } },
|
||||
source: ANTHROPIC_AUTH_TOKEN_ENV,
|
||||
};
|
||||
}
|
||||
|
||||
for (const envVar of [ANTHROPIC_OAUTH_TOKEN_ENV, ANTHROPIC_API_KEY_ENV]) {
|
||||
const apiKey = await ctx.env(envVar);
|
||||
if (apiKey) return { auth: { apiKey }, source: envVar };
|
||||
}
|
||||
return undefined;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function anthropicProvider(): Provider<"anthropic-messages"> {
|
||||
return createProvider({
|
||||
id: "anthropic",
|
||||
name: "Anthropic",
|
||||
baseUrl: "https://api.anthropic.com",
|
||||
auth: {
|
||||
// ANTHROPIC_OAUTH_TOKEN takes precedence over ANTHROPIC_API_KEY
|
||||
apiKey: envApiKeyAuth("Anthropic API key", ["ANTHROPIC_OAUTH_TOKEN", "ANTHROPIC_API_KEY"]),
|
||||
apiKey: anthropicApiKeyAuth(),
|
||||
oauth: lazyOAuth({ name: "Anthropic (Claude Pro/Max)", load: loadAnthropicOAuth }),
|
||||
},
|
||||
models: Object.values(ANTHROPIC_MODELS),
|
||||
|
||||
@@ -445,10 +445,33 @@ export interface AssistantImages {
|
||||
|
||||
import type { TSchema } from "typebox";
|
||||
|
||||
/** OpenAI grammar variants for constrained sampling. */
|
||||
export type GrammarFormat = "openai_lark" | "openai_regex";
|
||||
|
||||
export type GrammarVariants = Partial<Record<GrammarFormat, string>>;
|
||||
|
||||
/**
|
||||
* Optional provider-side constrained sampling configs for a tool.
|
||||
*
|
||||
* The `json_schema` value roughly maps to the concept of `strict` in APIs which is
|
||||
* implemented as json-schema constrained sampling by APIs. Grammar variants let
|
||||
* callers provide provider-specific encodings of the same intended language.
|
||||
*/
|
||||
export type ConstrainedSamplingConfig =
|
||||
| {
|
||||
type: "json_schema";
|
||||
strict: "prefer" | "require";
|
||||
}
|
||||
| {
|
||||
type: "grammar";
|
||||
variants: GrammarVariants;
|
||||
};
|
||||
|
||||
export interface Tool<TParameters extends TSchema = TSchema> {
|
||||
name: string;
|
||||
description: string;
|
||||
parameters: TParameters;
|
||||
constrainedSampling?: false | ConstrainedSamplingConfig;
|
||||
}
|
||||
|
||||
export interface Context {
|
||||
@@ -522,6 +545,8 @@ export interface OpenAICompletionsCompat {
|
||||
vercelGatewayRouting?: VercelGatewayRouting;
|
||||
/** Whether z.ai supports top-level `tool_stream: true` for streaming tool call deltas. Default: false. */
|
||||
zaiToolStream?: boolean;
|
||||
/** Whether the provider supports OpenAI custom tools with Lark/regex grammar formats. When false, grammar-constrained tools fall back to normal function tools. Default: false; the generated model catalog enables it for capable models. */
|
||||
supportsOpenAIGrammarTools?: boolean;
|
||||
/** Whether the provider supports the `strict` field in tool definitions. Default: true. */
|
||||
supportsStrictMode?: boolean;
|
||||
/** Cache control convention for prompt caching. "anthropic" applies Anthropic-style `cache_control` markers to the system prompt, last tool definition, and last user, assistant, or tool-result text content. */
|
||||
@@ -544,8 +569,14 @@ export interface OpenAIResponsesCompat {
|
||||
sessionAffinityFormat?: SessionAffinityFormat;
|
||||
/** Whether the provider supports `prompt_cache_retention: "24h"`. Default: true. */
|
||||
supportsLongCacheRetention?: boolean;
|
||||
/** Whether the provider supports strict JSON-schema function tools. Defaults are API-specific; generated OpenAI models enable it explicitly. */
|
||||
supportsStrictMode?: boolean;
|
||||
/** Whether to emit OpenAI custom tools with Lark/regex grammar formats. When false, grammar-constrained tools fall back to normal function tools. Default: false; the generated model catalog enables it for capable models. */
|
||||
supportsOpenAIGrammarTools?: boolean;
|
||||
/** Whether the model supports client-executed tool search for deferred tools. Default: false. */
|
||||
supportsToolSearch?: boolean;
|
||||
/** Whether the model accepts `prompt_cache_options` (OpenAI GPT-5.6+ explicit prompt caching). Older OpenAI models reject the parameter. Default: false. */
|
||||
supportsExplicitPromptCacheMode?: boolean;
|
||||
}
|
||||
|
||||
/** Compatibility settings for Anthropic Messages-compatible APIs. */
|
||||
@@ -594,6 +625,8 @@ export interface AnthropicMessagesCompat {
|
||||
forceAdaptiveThinking?: boolean;
|
||||
/** Whether to replay empty thinking signatures as `signature: ""` instead of converting thinking to text. Default: false. */
|
||||
allowEmptySignature?: boolean;
|
||||
/** Whether the provider supports Anthropic strict tool schemas. Default: false; generated Anthropic models enable it explicitly. */
|
||||
supportsStrictTools?: boolean;
|
||||
/**
|
||||
* Whether the provider supports deferred tools loaded by `tool_reference`
|
||||
* blocks in tool results. Default: true for first-party Anthropic models
|
||||
@@ -602,6 +635,12 @@ export interface AnthropicMessagesCompat {
|
||||
supportsToolReferences?: boolean;
|
||||
}
|
||||
|
||||
/** Compatibility settings for Amazon Bedrock models. */
|
||||
export interface BedrockCompat {
|
||||
/** Whether the model supports Bedrock strict tool schemas. Default: false. */
|
||||
supportsStrictMode?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* OpenRouter provider routing preferences.
|
||||
* Controls which upstream providers OpenRouter routes requests to.
|
||||
@@ -727,11 +766,13 @@ export interface Model<TApi extends Api> {
|
||||
/** Compatibility overrides for OpenAI-compatible APIs. If not set, auto-detected from baseUrl. */
|
||||
compat?: TApi extends "openai-completions"
|
||||
? OpenAICompletionsCompat
|
||||
: TApi extends "openai-responses" | "openai-codex-responses"
|
||||
: TApi extends "openai-responses" | "azure-openai-responses" | "openai-codex-responses"
|
||||
? OpenAIResponsesCompat
|
||||
: TApi extends "anthropic-messages"
|
||||
? AnthropicMessagesCompat
|
||||
: never;
|
||||
: TApi extends "bedrock-converse-stream"
|
||||
? BedrockCompat
|
||||
: never;
|
||||
}
|
||||
|
||||
export interface ImagesModel<TApi extends ImagesApi>
|
||||
|
||||
@@ -69,9 +69,9 @@ function extractStatus(error: SdkErrorShape): number | undefined {
|
||||
/**
|
||||
* Probe the raw body reason, first usable hit wins, in SDK-field order:
|
||||
* `body` string (Mistral) → `error` parsed JSON body object (`openai` SDK's
|
||||
* `this.error`) → `$response.body` (Bedrock). Empty objects are treated as no
|
||||
* body so an empty parsed body does not surface as `"{}"`. The chosen body is
|
||||
* truncated to the cap.
|
||||
* `this.error`) → `$response.body` (Bedrock). Empty objects and unread response
|
||||
* streams are treated as no body so they do not surface as `"{}"` or serialized
|
||||
* stream internals. The chosen body is truncated to the cap.
|
||||
*/
|
||||
function extractBody(error: SdkErrorShape): string | undefined {
|
||||
const bodyText = pickBodyText(error);
|
||||
@@ -86,10 +86,15 @@ function pickBodyText(error: SdkErrorShape): string | undefined {
|
||||
if (isNonEmptyObject(error.error)) return safeJsonStringify(error.error);
|
||||
const responseBody = error.$response?.body;
|
||||
if (typeof responseBody === "string") return responseBody;
|
||||
if (isReadableStreamLike(responseBody)) return undefined;
|
||||
if (isNonEmptyObject(responseBody)) return safeJsonStringify(responseBody);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function isReadableStreamLike(value: unknown): boolean {
|
||||
return typeof value === "object" && value !== null && "pipe" in value && typeof value.pipe === "function";
|
||||
}
|
||||
|
||||
function isNonEmptyObject(value: unknown): boolean {
|
||||
return typeof value === "object" && value !== null && Object.keys(value).length > 0;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
const DEFAULT_MAX_RETRY_DELAY_MS = 60_000;
|
||||
|
||||
interface ProviderRetryOptions {
|
||||
maxRetries?: number;
|
||||
maxRetryDelayMs?: number;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
interface ProviderError extends Error {
|
||||
status: number | undefined;
|
||||
headers: Headers | undefined;
|
||||
}
|
||||
|
||||
function isProviderError(error: unknown): error is ProviderError {
|
||||
if (!(error instanceof Error) || !("status" in error) || !("headers" in error)) return false;
|
||||
return (
|
||||
(error.status === undefined || typeof error.status === "number") &&
|
||||
(error.headers === undefined || error.headers instanceof Headers)
|
||||
);
|
||||
}
|
||||
|
||||
/** Mirrors the pinned OpenAI/Anthropic SDK retry policy; review when either SDK is upgraded. */
|
||||
function isRetryableProviderError(error: ProviderError): boolean {
|
||||
const shouldRetry = error.headers?.get("x-should-retry");
|
||||
if (shouldRetry === "true") return true;
|
||||
if (shouldRetry === "false") return false;
|
||||
|
||||
if (error.status === undefined) return true;
|
||||
return (
|
||||
error.status === 408 ||
|
||||
error.status === 409 ||
|
||||
error.status === 429 ||
|
||||
(typeof error.status === "number" && error.status >= 500)
|
||||
);
|
||||
}
|
||||
|
||||
function validateServerRetryDelayMs(
|
||||
delayMs: number,
|
||||
maxRetryDelayMs: number | undefined,
|
||||
providerErrorMessage: string,
|
||||
): number {
|
||||
const maxDelayMs = maxRetryDelayMs ?? DEFAULT_MAX_RETRY_DELAY_MS;
|
||||
if (maxDelayMs > 0 && delayMs > maxDelayMs) {
|
||||
throw new Error(
|
||||
`Server requested ${Math.ceil(delayMs / 1000)}s retry delay (max: ${Math.ceil(maxDelayMs / 1000)}s). ${providerErrorMessage}`,
|
||||
);
|
||||
}
|
||||
return delayMs;
|
||||
}
|
||||
|
||||
function getRetryDelayMs(error: ProviderError, retryIndex: number, maxRetryDelayMs: number | undefined): number {
|
||||
const retryAfterMs = error.headers?.get("retry-after-ms");
|
||||
if (retryAfterMs) {
|
||||
const value = Number.parseFloat(retryAfterMs);
|
||||
if (!Number.isNaN(value)) return validateServerRetryDelayMs(value, maxRetryDelayMs, error.message);
|
||||
}
|
||||
|
||||
const retryAfter = error.headers?.get("retry-after");
|
||||
if (retryAfter) {
|
||||
const seconds = Number.parseFloat(retryAfter);
|
||||
const delayMs = Number.isNaN(seconds) ? Date.parse(retryAfter) - Date.now() : seconds * 1000;
|
||||
return validateServerRetryDelayMs(delayMs, maxRetryDelayMs, error.message);
|
||||
}
|
||||
|
||||
const exponentialDelay = Math.min(0.5 * 2 ** retryIndex, 8) * 1000;
|
||||
return exponentialDelay * (1 - Math.random() * 0.25);
|
||||
}
|
||||
|
||||
function createAbortError(): Error {
|
||||
const error = new Error("Request aborted");
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
}
|
||||
|
||||
function abortableSleep(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (signal?.aborted) {
|
||||
reject(createAbortError());
|
||||
return;
|
||||
}
|
||||
|
||||
const onAbort = () => {
|
||||
clearTimeout(timeout);
|
||||
reject(createAbortError());
|
||||
};
|
||||
const timeout = setTimeout(
|
||||
() => {
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
resolve();
|
||||
},
|
||||
Math.max(0, ms),
|
||||
);
|
||||
signal?.addEventListener("abort", onAbort, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Reproduce the retry behavior used by the OpenAI and Anthropic SDKs while making
|
||||
* their backoff sleep interruptible. Their built-in retry timers ignore the
|
||||
* request AbortSignal, so callers must invoke the SDK with `maxRetries: 0` and
|
||||
* wrap the request with this helper. Provider-requested delays above
|
||||
* `maxRetryDelayMs` fail immediately (60 seconds by default); set it to zero to
|
||||
* disable the limit.
|
||||
*/
|
||||
export async function retryProviderRequest<T>(
|
||||
request: () => Promise<T>,
|
||||
options: ProviderRetryOptions = {},
|
||||
): Promise<T> {
|
||||
const maxRetries = options.maxRetries ?? 0;
|
||||
let retriesRemaining = maxRetries;
|
||||
|
||||
for (;;) {
|
||||
try {
|
||||
// Each retry is a fresh SDK request, so X-Stainless-Retry-Count remains zero.
|
||||
return await request();
|
||||
} catch (error) {
|
||||
if (options.signal?.aborted) throw createAbortError();
|
||||
if (retriesRemaining <= 0 || !isProviderError(error) || !isRetryableProviderError(error)) throw error;
|
||||
|
||||
const retryIndex = maxRetries - retriesRemaining;
|
||||
retriesRemaining--;
|
||||
await abortableSleep(getRetryDelayMs(error, retryIndex, options.maxRetryDelayMs), options.signal);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,13 +5,17 @@ import type { Api, Model } from "../src/types.ts";
|
||||
const EXPECTED_CURRENT_ADAPTIVE_THINKING_MODELS = [
|
||||
"anthropic/claude-fable-5",
|
||||
"anthropic/claude-opus-4-8",
|
||||
"anthropic/claude-opus-5",
|
||||
"anthropic/claude-sonnet-5",
|
||||
"cloudflare-ai-gateway/claude-fable-5",
|
||||
"kimi-coding/kimi-for-coding",
|
||||
"kimi-coding/k3",
|
||||
"kimi-coding/kimi-for-coding-highspeed",
|
||||
"opencode/claude-opus-4-8",
|
||||
"opencode/claude-opus-5",
|
||||
"vercel-ai-gateway/anthropic/claude-opus-4.8",
|
||||
"vercel-ai-gateway/anthropic/claude-opus-5",
|
||||
"vercel-ai-gateway/anthropic/claude-opus-5-fast",
|
||||
"vercel-ai-gateway/anthropic/claude-sonnet-5",
|
||||
];
|
||||
|
||||
@@ -30,7 +34,7 @@ describe("Anthropic adaptive thinking model metadata", () => {
|
||||
expect(flaggedModels).toEqual(expect.arrayContaining([...EXPECTED_CURRENT_ADAPTIVE_THINKING_MODELS].sort()));
|
||||
expect(flaggedModels).toEqual(
|
||||
flaggedModels.filter((modelId) =>
|
||||
/(opus[-.]4[-.][678]|sonnet[-.]4[-.]6|sonnet[-.]5|fable[-.]5|kimi-coding\/)/.test(modelId),
|
||||
/(opus[-.](4[-.][678]|5)|sonnet[-.]4[-.]6|sonnet[-.]5|fable[-.]5|kimi-coding\/)/.test(modelId),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { stream as streamAnthropic } from "../src/api/anthropic-messages.ts";
|
||||
import { ANTHROPIC_AUTH_TOKEN_ENV, ANTHROPIC_OAUTH_TOKEN_ENV } from "../src/env-api-keys.ts";
|
||||
import { createModels } from "../src/models.ts";
|
||||
import { anthropicProvider } from "../src/providers/anthropic.ts";
|
||||
import type { Context, Model } from "../src/types.ts";
|
||||
|
||||
const mockState = vi.hoisted(() => ({
|
||||
constructorOpts: undefined as Record<string, unknown> | undefined,
|
||||
createParams: undefined as Record<string, unknown> | undefined,
|
||||
}));
|
||||
|
||||
vi.mock("@anthropic-ai/sdk", () => {
|
||||
function createSseResponse(): Response {
|
||||
const body = [
|
||||
`event: message_start\ndata: ${JSON.stringify({
|
||||
type: "message_start",
|
||||
message: {
|
||||
id: "msg_test",
|
||||
usage: { input_tokens: 1, output_tokens: 0 },
|
||||
},
|
||||
})}\n`,
|
||||
`event: message_delta\ndata: ${JSON.stringify({
|
||||
type: "message_delta",
|
||||
delta: { stop_reason: "end_turn" },
|
||||
usage: { output_tokens: 1 },
|
||||
})}\n`,
|
||||
`event: message_stop\ndata: ${JSON.stringify({ type: "message_stop" })}\n`,
|
||||
].join("\n");
|
||||
|
||||
return new Response(body, {
|
||||
status: 200,
|
||||
headers: { "content-type": "text/event-stream" },
|
||||
});
|
||||
}
|
||||
|
||||
class FakeAnthropic {
|
||||
constructor(opts: Record<string, unknown>) {
|
||||
mockState.constructorOpts = opts;
|
||||
}
|
||||
messages = {
|
||||
create: (params: Record<string, unknown>) => {
|
||||
mockState.createParams = params;
|
||||
return {
|
||||
asResponse: async () => createSseResponse(),
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return { default: FakeAnthropic };
|
||||
});
|
||||
|
||||
const context: Context = {
|
||||
systemPrompt: "System prompt.",
|
||||
messages: [{ role: "user", content: "Hello", timestamp: Date.now() }],
|
||||
};
|
||||
|
||||
const anthropicModel: Model<"anthropic-messages"> = {
|
||||
id: "claude-test",
|
||||
name: "Claude Test",
|
||||
api: "anthropic-messages",
|
||||
provider: "anthropic",
|
||||
baseUrl: "https://api.anthropic.com",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 100000,
|
||||
maxTokens: 4096,
|
||||
};
|
||||
|
||||
afterEach(() => {
|
||||
mockState.constructorOpts = undefined;
|
||||
mockState.createParams = undefined;
|
||||
});
|
||||
|
||||
describe("Anthropic auth token env", () => {
|
||||
it("resolves ANTHROPIC_AUTH_TOKEN as a bearer Authorization header", async () => {
|
||||
const provider = anthropicProvider();
|
||||
const auth = await provider.auth.apiKey?.resolve({
|
||||
ctx: {
|
||||
env: async (name) =>
|
||||
({
|
||||
ANTHROPIC_AUTH_TOKEN: "auth-token",
|
||||
ANTHROPIC_OAUTH_TOKEN: "oauth-token",
|
||||
ANTHROPIC_API_KEY: "api-key",
|
||||
})[name],
|
||||
fileExists: async () => false,
|
||||
},
|
||||
});
|
||||
|
||||
expect(auth).toEqual({
|
||||
auth: { headers: { Authorization: "Bearer auth-token" } },
|
||||
source: ANTHROPIC_AUTH_TOKEN_ENV,
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves ANTHROPIC_OAUTH_TOKEN as OAuth-shaped API auth", async () => {
|
||||
const provider = anthropicProvider();
|
||||
const auth = await provider.auth.apiKey?.resolve({
|
||||
ctx: {
|
||||
env: async (name) =>
|
||||
({
|
||||
ANTHROPIC_OAUTH_TOKEN: "oauth-token",
|
||||
ANTHROPIC_API_KEY: "api-key",
|
||||
})[name],
|
||||
fileExists: async () => false,
|
||||
},
|
||||
});
|
||||
|
||||
expect(auth).toEqual({
|
||||
auth: { apiKey: "oauth-token" },
|
||||
source: ANTHROPIC_OAUTH_TOKEN_ENV,
|
||||
});
|
||||
});
|
||||
|
||||
it("uses Authorization headers without OAuth-mode request shaping", async () => {
|
||||
const stream = streamAnthropic(anthropicModel, context, {
|
||||
headers: { Authorization: "Bearer gateway-token" },
|
||||
});
|
||||
await stream.result();
|
||||
|
||||
expect(mockState.constructorOpts?.apiKey).toBeNull();
|
||||
expect(mockState.constructorOpts?.authToken).toBeNull();
|
||||
const headers = mockState.constructorOpts?.defaultHeaders as Record<string, string | null>;
|
||||
expect(headers.Authorization).toBe("Bearer gateway-token");
|
||||
expect(headers["anthropic-beta"] ?? "").not.toContain("oauth-2025-04-20");
|
||||
expect(mockState.createParams?.system).toEqual([expect.objectContaining({ text: "System prompt." })]);
|
||||
});
|
||||
|
||||
it("threads authContext ANTHROPIC_AUTH_TOKEN through request headers", async () => {
|
||||
const models = createModels({
|
||||
authContext: {
|
||||
env: async (name) => (name === "ANTHROPIC_AUTH_TOKEN" ? "ctx-token" : undefined),
|
||||
fileExists: async () => false,
|
||||
},
|
||||
});
|
||||
models.setProvider(anthropicProvider());
|
||||
|
||||
await models.streamSimple(anthropicModel, context).result();
|
||||
|
||||
expect(mockState.constructorOpts?.apiKey).toBeNull();
|
||||
expect(mockState.constructorOpts?.authToken).toBeNull();
|
||||
const headers = mockState.constructorOpts?.defaultHeaders as Record<string, string>;
|
||||
expect(headers.Authorization).toBe("Bearer ctx-token");
|
||||
expect(headers["anthropic-beta"] ?? "").not.toContain("oauth-2025-04-20");
|
||||
expect(mockState.createParams?.system).toEqual([expect.objectContaining({ text: "System prompt." })]);
|
||||
});
|
||||
|
||||
it("preserves OAuth request shaping for ANTHROPIC_OAUTH_TOKEN", async () => {
|
||||
const models = createModels({
|
||||
authContext: {
|
||||
env: async (name) => (name === "ANTHROPIC_OAUTH_TOKEN" ? "sk-ant-oat-test" : undefined),
|
||||
fileExists: async () => false,
|
||||
},
|
||||
});
|
||||
models.setProvider(anthropicProvider());
|
||||
|
||||
await models.streamSimple(anthropicModel, context).result();
|
||||
|
||||
expect(mockState.constructorOpts?.apiKey).toBeNull();
|
||||
expect(mockState.constructorOpts?.authToken).toBe("sk-ant-oat-test");
|
||||
const headers = mockState.constructorOpts?.defaultHeaders as Record<string, string>;
|
||||
expect(headers["anthropic-beta"]).toContain("oauth-2025-04-20");
|
||||
});
|
||||
|
||||
it("lets explicit request headers override ANTHROPIC_AUTH_TOKEN", async () => {
|
||||
const models = createModels({
|
||||
authContext: {
|
||||
env: async (name) => (name === "ANTHROPIC_AUTH_TOKEN" ? "ctx-token" : undefined),
|
||||
fileExists: async () => false,
|
||||
},
|
||||
});
|
||||
models.setProvider(anthropicProvider());
|
||||
|
||||
await models
|
||||
.streamSimple(anthropicModel, context, { headers: { Authorization: "Bearer explicit-token" } })
|
||||
.result();
|
||||
|
||||
const headers = mockState.constructorOpts?.defaultHeaders as Record<string, string>;
|
||||
expect(headers.Authorization).toBe("Bearer explicit-token");
|
||||
});
|
||||
});
|
||||
@@ -32,6 +32,17 @@ const tool: Tool = {
|
||||
parameters: Type.Object({ value: Type.String() }),
|
||||
};
|
||||
|
||||
const schemaCompatibilityTool: Tool = {
|
||||
...tool,
|
||||
parameters: Type.Object({ value: Type.String() }, { additionalProperties: false, title: "LookupInput" }),
|
||||
};
|
||||
|
||||
const strictTool: Tool = {
|
||||
...tool,
|
||||
parameters: Type.Object({ value: Type.String() }, { additionalProperties: false, title: "StrictLookupInput" }),
|
||||
constrainedSampling: { type: "json_schema", strict: "prefer" },
|
||||
};
|
||||
|
||||
function createContext(tools: Tool[] = [tool]): Context {
|
||||
return {
|
||||
messages: [{ role: "user", content: "Use the tool", timestamp: Date.now() }],
|
||||
@@ -98,6 +109,14 @@ function getFirstTool(body: Record<string, unknown>): Record<string, unknown> {
|
||||
return tools[0] as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function getFirstToolInputSchema(body: Record<string, unknown>): Record<string, unknown> {
|
||||
const inputSchema = getFirstTool(body).input_schema;
|
||||
if (typeof inputSchema !== "object" || inputSchema === null || Array.isArray(inputSchema)) {
|
||||
throw new Error("Expected first tool input schema in request body");
|
||||
}
|
||||
return inputSchema as Record<string, unknown>;
|
||||
}
|
||||
|
||||
describe("Anthropic eager tool input streaming compatibility", () => {
|
||||
it("sends per-tool eager_input_streaming by default", async () => {
|
||||
const request = await captureAnthropicRequest(undefined, createContext());
|
||||
@@ -119,4 +138,24 @@ describe("Anthropic eager tool input streaming compatibility", () => {
|
||||
expect(request.body.tools).toBeUndefined();
|
||||
expect(request.headers["anthropic-beta"]).toBeUndefined();
|
||||
});
|
||||
|
||||
it("only sends the full input schema for strict JSON-schema tools", async () => {
|
||||
const legacyRequest = await captureAnthropicRequest(
|
||||
{ supportsStrictTools: true },
|
||||
createContext([schemaCompatibilityTool]),
|
||||
);
|
||||
const parameters = schemaCompatibilityTool.parameters as { properties?: unknown; required?: unknown };
|
||||
expect(getFirstToolInputSchema(legacyRequest.body)).toEqual({
|
||||
type: "object",
|
||||
properties: parameters.properties,
|
||||
required: parameters.required,
|
||||
});
|
||||
|
||||
const strictRequest = await captureAnthropicRequest({ supportsStrictTools: true }, createContext([strictTool]));
|
||||
expect(getFirstTool(strictRequest.body).strict).toBe(true);
|
||||
expect(getFirstToolInputSchema(strictRequest.body)).toMatchObject({
|
||||
additionalProperties: false,
|
||||
title: "StrictLookupInput",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import { Type } from "typebox";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { stream as streamAzureOpenAIResponses } from "../src/api/azure-openai-responses.ts";
|
||||
import { getModel } from "../src/compat.ts";
|
||||
import type { Context } from "../src/types.ts";
|
||||
import type { Context, Model } from "../src/types.ts";
|
||||
|
||||
interface CapturedAzureClientOptions {
|
||||
apiKey: string;
|
||||
@@ -14,6 +15,7 @@ interface CapturedAzureClientOptions {
|
||||
interface CapturedAzureResponsesPayload {
|
||||
prompt_cache_key?: string;
|
||||
store?: boolean;
|
||||
tools?: Array<{ strict?: boolean }>;
|
||||
}
|
||||
|
||||
const azureMock = vi.hoisted(() => ({
|
||||
@@ -165,6 +167,32 @@ describe("azure-openai-responses base URL normalization", () => {
|
||||
expect(azureMock.lastParams?.store).toBe(false);
|
||||
});
|
||||
|
||||
it("honors supportsStrictMode: false", async () => {
|
||||
const baseModel = getModel("azure-openai-responses", "gpt-4o-mini");
|
||||
const model: Model<"azure-openai-responses"> = {
|
||||
...baseModel,
|
||||
compat: { ...baseModel.compat, supportsStrictMode: false },
|
||||
};
|
||||
|
||||
await streamAzureOpenAIResponses(
|
||||
model,
|
||||
{
|
||||
...context,
|
||||
tools: [
|
||||
{
|
||||
name: "preferred",
|
||||
description: "Preferred constrained tool",
|
||||
parameters: Type.Object({ value: Type.String() }),
|
||||
constrainedSampling: { type: "json_schema", strict: "prefer" },
|
||||
},
|
||||
],
|
||||
},
|
||||
{ apiKey: "test-api-key", azureBaseUrl: "https://my-resource.openai.azure.com" },
|
||||
).result();
|
||||
|
||||
expect(azureMock.lastParams?.tools?.[0]).not.toHaveProperty("strict");
|
||||
});
|
||||
|
||||
it("builds correct default URL from AZURE_OPENAI_RESOURCE_NAME", async () => {
|
||||
process.env.AZURE_OPENAI_RESOURCE_NAME = "my-resource";
|
||||
const model = getModel("azure-openai-responses", "gpt-4o-mini");
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { Type } from "typebox";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
const bedrockMock = vi.hoisted(() => ({
|
||||
@@ -50,9 +51,9 @@ import type { Context, Message } from "../src/types.ts";
|
||||
|
||||
const baseModel = getModel("amazon-bedrock", "us.anthropic.claude-sonnet-4-5-20250929-v1:0");
|
||||
|
||||
async function capturePayload(context: Context): Promise<unknown> {
|
||||
async function capturePayload(context: Context, model = baseModel): Promise<unknown> {
|
||||
let capturedPayload: unknown;
|
||||
const s = streamBedrock(baseModel, context, {
|
||||
const s = streamBedrock(model, context, {
|
||||
cacheRetention: "none",
|
||||
signal: AbortSignal.abort(),
|
||||
onPayload: (payload) => {
|
||||
@@ -66,6 +67,34 @@ async function capturePayload(context: Context): Promise<unknown> {
|
||||
return capturedPayload;
|
||||
}
|
||||
|
||||
describe("Bedrock constrained sampling", () => {
|
||||
it("gates native strict tool use by model capability", async () => {
|
||||
const context: Context = {
|
||||
messages: [{ role: "user", content: "Use the tool", timestamp: Date.now() }],
|
||||
tools: [
|
||||
{
|
||||
name: "lookup",
|
||||
description: "Look up a value",
|
||||
parameters: Type.Object({ value: Type.String() }),
|
||||
constrainedSampling: { type: "json_schema", strict: "require" },
|
||||
},
|
||||
],
|
||||
};
|
||||
const payload = await capturePayload(context);
|
||||
const toolConfig = (payload as { toolConfig: { tools: Array<{ toolSpec: { strict?: boolean } }> } }).toolConfig;
|
||||
expect(toolConfig.tools[0].toolSpec.strict).toBe(true);
|
||||
|
||||
context.tools![0].constrainedSampling = { type: "json_schema", strict: "prefer" };
|
||||
const novaPayload = await capturePayload(context, getModel("amazon-bedrock", "amazon.nova-lite-v1:0"));
|
||||
const novaToolConfig = (
|
||||
novaPayload as {
|
||||
toolConfig: { tools: Array<{ toolSpec: { strict?: boolean } }> };
|
||||
}
|
||||
).toolConfig;
|
||||
expect(novaToolConfig.tools[0].toolSpec.strict).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("bedrock convertMessages skips unknown content types", () => {
|
||||
it("skips unknown user content blocks instead of throwing", async () => {
|
||||
const messages: Message[] = [
|
||||
|
||||
@@ -29,6 +29,11 @@ describe("Amazon Bedrock Models", () => {
|
||||
console.log(`Found ${models.length} Bedrock models`);
|
||||
});
|
||||
|
||||
it("exposes Claude Opus 5 through an inference profile only", () => {
|
||||
expect(models.some((model) => model.id === "global.anthropic.claude-opus-5")).toBe(true);
|
||||
expect(models.some((model) => model.id === "anthropic.claude-opus-5")).toBe(false);
|
||||
});
|
||||
|
||||
if (hasBedrockCredentials() && process.env.BEDROCK_EXTENSIVE_MODEL_TEST) {
|
||||
for (const model of models) {
|
||||
it(`should make a simple request with ${model.id}`, { timeout: 10_000 }, async () => {
|
||||
|
||||
@@ -103,6 +103,26 @@ describe("Bedrock thinking payload", () => {
|
||||
expect(payload.additionalModelRequestFields?.anthropic_beta).toBeUndefined();
|
||||
});
|
||||
|
||||
it("uses adaptive thinking for Claude Opus 5 when reasoning is enabled", async () => {
|
||||
const model = getModel("amazon-bedrock", "global.anthropic.claude-opus-5");
|
||||
|
||||
const payload = await capturePayload(model);
|
||||
|
||||
expect(payload.additionalModelRequestFields?.thinking).toEqual({ type: "adaptive", display: "summarized" });
|
||||
expect(payload.additionalModelRequestFields?.output_config).toEqual({ effort: "high" });
|
||||
expect(payload.additionalModelRequestFields?.anthropic_beta).toBeUndefined();
|
||||
});
|
||||
|
||||
it("maps xhigh reasoning to effort=xhigh for Claude Opus 5", async () => {
|
||||
const model = getModel("amazon-bedrock", "global.anthropic.claude-opus-5");
|
||||
|
||||
const payload = await capturePayload(model, { reasoning: "xhigh" });
|
||||
|
||||
expect(payload.additionalModelRequestFields?.thinking).toEqual({ type: "adaptive", display: "summarized" });
|
||||
expect(payload.additionalModelRequestFields?.output_config).toEqual({ effort: "xhigh" });
|
||||
expect(payload.additionalModelRequestFields?.anthropic_beta).toBeUndefined();
|
||||
});
|
||||
|
||||
it("maps xhigh reasoning to effort=xhigh for Claude Fable 5", async () => {
|
||||
const model = getModel("amazon-bedrock", "global.anthropic.claude-fable-5");
|
||||
|
||||
|
||||
@@ -18,6 +18,10 @@ interface OpenAICompletionsCachePayload {
|
||||
prompt_cache_retention?: string;
|
||||
}
|
||||
|
||||
interface OpenAIResponsesCachePayload extends OpenAICompletionsCachePayload {
|
||||
prompt_cache_options?: { mode: "explicit" };
|
||||
}
|
||||
|
||||
function stopAfterPayload<TPayload>(capture: (payload: TPayload) => void): (payload: unknown) => never {
|
||||
return (payload: unknown): never => {
|
||||
capture(payload as TPayload);
|
||||
@@ -341,16 +345,16 @@ describe("Cache Retention (PI_CACHE_RETENTION)", () => {
|
||||
expect(capturedPayload.prompt_cache_retention).toBeUndefined();
|
||||
});
|
||||
|
||||
it("should omit prompt_cache_key when cacheRetention is none", async () => {
|
||||
const model = getModel("openai", "gpt-4o-mini");
|
||||
let capturedPayload: any = null;
|
||||
it("should omit prompt_cache_key and disable implicit writes when cacheRetention is none", async () => {
|
||||
const model = getModel("openai", "gpt-5.6-sol");
|
||||
let capturedPayload: OpenAIResponsesCachePayload | undefined;
|
||||
|
||||
try {
|
||||
const s = streamOpenAIResponses(model, context, {
|
||||
apiKey: "fake-key",
|
||||
cacheRetention: "none",
|
||||
sessionId: "session-1",
|
||||
onPayload: stopAfterPayload((payload) => {
|
||||
onPayload: stopAfterPayload<OpenAIResponsesCachePayload>((payload) => {
|
||||
capturedPayload = payload;
|
||||
}),
|
||||
});
|
||||
@@ -362,9 +366,36 @@ describe("Cache Retention (PI_CACHE_RETENTION)", () => {
|
||||
// Expected to fail
|
||||
}
|
||||
|
||||
expect(capturedPayload).not.toBeNull();
|
||||
expect(capturedPayload.prompt_cache_key).toBeUndefined();
|
||||
expect(capturedPayload.prompt_cache_retention).toBeUndefined();
|
||||
expect(capturedPayload).toBeDefined();
|
||||
expect(capturedPayload?.prompt_cache_key).toBeUndefined();
|
||||
expect(capturedPayload?.prompt_cache_retention).toBeUndefined();
|
||||
expect(capturedPayload?.prompt_cache_options).toEqual({ mode: "explicit" });
|
||||
});
|
||||
|
||||
it("should omit prompt_cache_options for models that reject it", async () => {
|
||||
const model = getModel("openai", "gpt-4o-mini");
|
||||
let capturedPayload: OpenAIResponsesCachePayload | undefined;
|
||||
|
||||
try {
|
||||
const s = streamOpenAIResponses(model, context, {
|
||||
apiKey: "fake-key",
|
||||
cacheRetention: "none",
|
||||
sessionId: "session-1",
|
||||
onPayload: stopAfterPayload<OpenAIResponsesCachePayload>((payload) => {
|
||||
capturedPayload = payload;
|
||||
}),
|
||||
});
|
||||
|
||||
for await (const event of s) {
|
||||
if (event.type === "error") break;
|
||||
}
|
||||
} catch {
|
||||
// Expected to fail
|
||||
}
|
||||
|
||||
expect(capturedPayload).toBeDefined();
|
||||
expect(capturedPayload?.prompt_cache_key).toBeUndefined();
|
||||
expect(capturedPayload?.prompt_cache_options).toBeUndefined();
|
||||
});
|
||||
|
||||
it("should set prompt_cache_retention when cacheRetention is long", async () => {
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
import type { ResponseStreamEvent } from "openai/resources/responses/responses.js";
|
||||
import { Type } from "typebox";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { appendGrammarToolInputJsonDelta } from "../src/api/constrained-sampling.ts";
|
||||
import {
|
||||
convertResponsesMessages,
|
||||
convertResponsesTools,
|
||||
processResponsesStream,
|
||||
} from "../src/api/openai-responses-shared.ts";
|
||||
import type { AssistantMessage, Context, Model, Tool, ToolCall } from "../src/types.ts";
|
||||
import { AssistantMessageEventStream } from "../src/utils/event-stream.ts";
|
||||
|
||||
function makeModel(): Model<"openai-responses"> {
|
||||
return {
|
||||
id: "gpt-test",
|
||||
name: "GPT Test",
|
||||
api: "openai-responses",
|
||||
provider: "openai",
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
};
|
||||
}
|
||||
|
||||
function makeUsage(): AssistantMessage["usage"] {
|
||||
return {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 0,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
};
|
||||
}
|
||||
|
||||
function makeOutput(): AssistantMessage {
|
||||
return {
|
||||
role: "assistant",
|
||||
content: [],
|
||||
api: "openai-responses",
|
||||
provider: "openai",
|
||||
model: "gpt-test",
|
||||
usage: makeUsage(),
|
||||
stopReason: "stop",
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
async function* iterateEvents(events: ResponseStreamEvent[]): AsyncGenerator<ResponseStreamEvent> {
|
||||
yield* events;
|
||||
}
|
||||
|
||||
function makeTool(overrides: Partial<Tool> = {}): Tool {
|
||||
return {
|
||||
name: "sample_tool",
|
||||
description: "Sample tool",
|
||||
parameters: Type.Object({ payload: Type.String() }, { additionalProperties: false }),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function captureToolCallDeltas(stream: AssistantMessageEventStream): string[] {
|
||||
const deltas: string[] = [];
|
||||
const originalPush = stream.push.bind(stream);
|
||||
stream.push = (event) => {
|
||||
if (event.type === "toolcall_delta") {
|
||||
deltas.push(event.delta);
|
||||
}
|
||||
originalPush(event);
|
||||
};
|
||||
return deltas;
|
||||
}
|
||||
|
||||
describe("constrained tool sampling", () => {
|
||||
it("converts supported constraints and falls back when unsupported", () => {
|
||||
expect(
|
||||
convertResponsesTools([makeTool({ constrainedSampling: { type: "json_schema", strict: "prefer" } })])[0],
|
||||
).toMatchObject({ type: "function", name: "sample_tool", strict: true });
|
||||
|
||||
expect(() =>
|
||||
convertResponsesTools([makeTool({ constrainedSampling: { type: "json_schema", strict: "require" } })], {
|
||||
supportsStrictMode: false,
|
||||
}),
|
||||
).toThrow('Tool "sample_tool" requires JSON-schema constrained sampling');
|
||||
|
||||
const grammarTool = makeTool({
|
||||
constrainedSampling: { type: "grammar", variants: { openai_lark: "start: /[a-z]+/" } },
|
||||
});
|
||||
expect(convertResponsesTools([grammarTool], { supportsOpenAIGrammarTools: true })[0]).toMatchObject({
|
||||
type: "custom",
|
||||
name: "sample_tool",
|
||||
format: { type: "grammar", syntax: "lark", definition: "start: /[a-z]+/" },
|
||||
});
|
||||
expect(() =>
|
||||
convertResponsesTools([makeTool({ constrainedSampling: { type: "grammar", variants: {} } })], {
|
||||
supportsOpenAIGrammarTools: true,
|
||||
}),
|
||||
).toThrow(
|
||||
'Tool "sample_tool" cannot use grammar constrained sampling: no supported grammar variant was provided',
|
||||
);
|
||||
|
||||
const fallback = convertResponsesTools([grammarTool], {
|
||||
supportsOpenAIGrammarTools: false,
|
||||
supportsStrictMode: false,
|
||||
})[0];
|
||||
expect(fallback).toMatchObject({ type: "function", name: "sample_tool" });
|
||||
expect("strict" in (fallback as object)).toBe(false);
|
||||
|
||||
expect(convertResponsesTools([makeTool({ constrainedSampling: false })])).toEqual(
|
||||
convertResponsesTools([makeTool()]),
|
||||
);
|
||||
});
|
||||
|
||||
it("replays grammar calls as custom Responses items", () => {
|
||||
const replayedToolCall: ToolCall = {
|
||||
type: "toolCall",
|
||||
id: "call_1|ctc_1",
|
||||
name: "sample_tool",
|
||||
arguments: { payload: "abc" },
|
||||
};
|
||||
const context: Context = {
|
||||
messages: [
|
||||
{
|
||||
role: "assistant",
|
||||
api: "openai-responses",
|
||||
provider: "openai",
|
||||
model: "gpt-test",
|
||||
content: [replayedToolCall],
|
||||
usage: makeUsage(),
|
||||
stopReason: "toolUse",
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: "call_1|ctc_1",
|
||||
toolName: "sample_tool",
|
||||
content: [{ type: "text", text: "done" }],
|
||||
isError: false,
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
],
|
||||
};
|
||||
for (const invalidArguments of [{}, { payload: 42 }]) {
|
||||
replayedToolCall.arguments = invalidArguments;
|
||||
expect(() =>
|
||||
convertResponsesMessages(makeModel(), context, new Set(["openai"]), {
|
||||
grammarToolInputProperties: new Map([["sample_tool", "payload"]]),
|
||||
}),
|
||||
).toThrow('Grammar tool call "sample_tool" requires argument "payload" to be a string');
|
||||
}
|
||||
|
||||
replayedToolCall.arguments = { payload: "abc" };
|
||||
const messages = convertResponsesMessages(makeModel(), context, new Set(["openai"]), {
|
||||
grammarToolInputProperties: new Map([["sample_tool", "payload"]]),
|
||||
});
|
||||
|
||||
expect(messages).toContainEqual({
|
||||
type: "custom_tool_call",
|
||||
id: "ctc_1",
|
||||
call_id: "call_1",
|
||||
name: "sample_tool",
|
||||
input: "abc",
|
||||
});
|
||||
expect(messages).toContainEqual({
|
||||
type: "custom_tool_call_output",
|
||||
call_id: "call_1",
|
||||
output: "done",
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps grammar input JSON deltas append-only", () => {
|
||||
const buffer = { input: "", started: false, closed: false };
|
||||
const first = appendGrammarToolInputJsonDelta(buffer, "payload", 'a"', false);
|
||||
const second = appendGrammarToolInputJsonDelta(buffer, "payload", 'a"\nb', true);
|
||||
|
||||
expect(JSON.parse(`${first}${second}`)).toEqual({ payload: 'a"\nb' });
|
||||
expect(appendGrammarToolInputJsonDelta(buffer, "payload", 'a"\nb', true)).toBeUndefined();
|
||||
expect(() => appendGrammarToolInputJsonDelta(buffer, "payload", "changed", true)).toThrow(
|
||||
'grammar tool input for property "payload" changed after it was closed',
|
||||
);
|
||||
});
|
||||
|
||||
it("streams custom Responses tool calls as string arguments", async () => {
|
||||
const output = makeOutput();
|
||||
const stream = new AssistantMessageEventStream();
|
||||
const deltas = captureToolCallDeltas(stream);
|
||||
const events = [
|
||||
{
|
||||
type: "response.output_item.added",
|
||||
output_index: 0,
|
||||
item: { type: "custom_tool_call", call_id: "call_1", id: "ctc_1", name: "sample_tool", input: "" },
|
||||
},
|
||||
{
|
||||
type: "response.custom_tool_call_input.delta",
|
||||
output_index: 0,
|
||||
item_id: "ctc_1",
|
||||
delta: "ab",
|
||||
},
|
||||
{
|
||||
type: "response.custom_tool_call_input.done",
|
||||
output_index: 0,
|
||||
item_id: "ctc_1",
|
||||
input: "abc",
|
||||
},
|
||||
{
|
||||
type: "response.output_item.done",
|
||||
output_index: 0,
|
||||
item: { type: "custom_tool_call", call_id: "call_1", id: "ctc_1", name: "sample_tool", input: "abc" },
|
||||
},
|
||||
{
|
||||
type: "response.completed",
|
||||
response: { status: "completed", usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 } },
|
||||
},
|
||||
] as ResponseStreamEvent[];
|
||||
|
||||
await processResponsesStream(iterateEvents(events), output, stream, makeModel(), {
|
||||
grammarToolInputProperties: new Map([["sample_tool", "payload"]]),
|
||||
});
|
||||
|
||||
expect(output.stopReason).toBe("toolUse");
|
||||
expect(output.content).toEqual([
|
||||
{ type: "toolCall", id: "call_1|ctc_1", name: "sample_tool", arguments: { payload: "abc" } },
|
||||
]);
|
||||
expect(JSON.parse(deltas.join(""))).toEqual({ payload: "abc" });
|
||||
});
|
||||
});
|
||||
@@ -369,6 +369,7 @@ describe("deferred tools", () => {
|
||||
chatTemplateKwargs: {},
|
||||
zaiToolStream: false,
|
||||
supportsStrictMode: false,
|
||||
supportsOpenAIGrammarTools: false,
|
||||
cacheControlFormat: undefined,
|
||||
sendSessionAffinityHeaders: false,
|
||||
deferredToolsMode: "kimi",
|
||||
|
||||
@@ -5,6 +5,9 @@ const originalCopilotGitHubToken = process.env.COPILOT_GITHUB_TOKEN;
|
||||
const originalGhToken = process.env.GH_TOKEN;
|
||||
const originalGitHubToken = process.env.GITHUB_TOKEN;
|
||||
const originalZaiCodingCnApiKey = process.env.ZAI_CODING_CN_API_KEY;
|
||||
const originalAnthropicAuthToken = process.env.ANTHROPIC_AUTH_TOKEN;
|
||||
const originalAnthropicOauthToken = process.env.ANTHROPIC_OAUTH_TOKEN;
|
||||
const originalAnthropicApiKey = process.env.ANTHROPIC_API_KEY;
|
||||
|
||||
afterEach(() => {
|
||||
if (originalCopilotGitHubToken === undefined) {
|
||||
@@ -30,6 +33,24 @@ afterEach(() => {
|
||||
} else {
|
||||
process.env.ZAI_CODING_CN_API_KEY = originalZaiCodingCnApiKey;
|
||||
}
|
||||
|
||||
if (originalAnthropicAuthToken === undefined) {
|
||||
delete process.env.ANTHROPIC_AUTH_TOKEN;
|
||||
} else {
|
||||
process.env.ANTHROPIC_AUTH_TOKEN = originalAnthropicAuthToken;
|
||||
}
|
||||
|
||||
if (originalAnthropicOauthToken === undefined) {
|
||||
delete process.env.ANTHROPIC_OAUTH_TOKEN;
|
||||
} else {
|
||||
process.env.ANTHROPIC_OAUTH_TOKEN = originalAnthropicOauthToken;
|
||||
}
|
||||
|
||||
if (originalAnthropicApiKey === undefined) {
|
||||
delete process.env.ANTHROPIC_API_KEY;
|
||||
} else {
|
||||
process.env.ANTHROPIC_API_KEY = originalAnthropicApiKey;
|
||||
}
|
||||
});
|
||||
|
||||
describe("environment API keys", () => {
|
||||
@@ -57,4 +78,39 @@ describe("environment API keys", () => {
|
||||
expect(findEnvKeys("zai-coding-cn")).toEqual(["ZAI_CODING_CN_API_KEY"]);
|
||||
expect(getEnvApiKey("zai-coding-cn")).toBe("zai-coding-cn-token");
|
||||
});
|
||||
|
||||
it("reports ANTHROPIC_AUTH_TOKEN but preserves OAuth token API key lookup", () => {
|
||||
process.env.ANTHROPIC_AUTH_TOKEN = "auth-token";
|
||||
process.env.ANTHROPIC_OAUTH_TOKEN = "oauth-token";
|
||||
process.env.ANTHROPIC_API_KEY = "api-key";
|
||||
|
||||
expect(findEnvKeys("anthropic")).toEqual(["ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_OAUTH_TOKEN", "ANTHROPIC_API_KEY"]);
|
||||
expect(getEnvApiKey("anthropic")).toBe("oauth-token");
|
||||
});
|
||||
|
||||
it("does not return ANTHROPIC_AUTH_TOKEN as an API key", () => {
|
||||
process.env.ANTHROPIC_AUTH_TOKEN = "auth-token";
|
||||
delete process.env.ANTHROPIC_OAUTH_TOKEN;
|
||||
delete process.env.ANTHROPIC_API_KEY;
|
||||
|
||||
expect(findEnvKeys("anthropic")).toEqual(["ANTHROPIC_AUTH_TOKEN"]);
|
||||
expect(getEnvApiKey("anthropic")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("preserves ANTHROPIC_OAUTH_TOKEN as an API key", () => {
|
||||
delete process.env.ANTHROPIC_AUTH_TOKEN;
|
||||
process.env.ANTHROPIC_OAUTH_TOKEN = "oauth-token";
|
||||
delete process.env.ANTHROPIC_API_KEY;
|
||||
|
||||
expect(findEnvKeys("anthropic")).toEqual(["ANTHROPIC_OAUTH_TOKEN"]);
|
||||
expect(getEnvApiKey("anthropic")).toBe("oauth-token");
|
||||
});
|
||||
|
||||
it("falls back to ANTHROPIC_API_KEY for API key lookup", () => {
|
||||
delete process.env.ANTHROPIC_AUTH_TOKEN;
|
||||
delete process.env.ANTHROPIC_OAUTH_TOKEN;
|
||||
process.env.ANTHROPIC_API_KEY = "api-key";
|
||||
|
||||
expect(getEnvApiKey("anthropic")).toBe("api-key");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -64,6 +64,27 @@ describe("normalizeProviderError", () => {
|
||||
expect(norm.messageCarriesBody).toBe(false);
|
||||
});
|
||||
|
||||
it("ignores a Bedrock response stream instead of serializing its internals", () => {
|
||||
const error = Object.assign(
|
||||
new Error("Invocation of model ID anthropic.claude-opus-5 with on-demand throughput isn't supported."),
|
||||
{
|
||||
name: "ValidationException",
|
||||
$metadata: { httpStatusCode: 400 },
|
||||
$response: {
|
||||
statusCode: 400,
|
||||
body: { pipe: () => undefined, _events: { close: [null, null] } },
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const norm = normalizeProviderError(error);
|
||||
|
||||
expect(norm.status).toBe(400);
|
||||
expect(norm.body).toBeUndefined();
|
||||
expect(norm.message).toContain("on-demand throughput isn't supported");
|
||||
expect(norm.messageCarriesBody).toBe(true);
|
||||
});
|
||||
|
||||
it("JSON-stringifies a non-Error thrown value", () => {
|
||||
const norm = normalizeProviderError({ reason: "boom" });
|
||||
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { convertTools } from "../src/api/google-shared.ts";
|
||||
import {
|
||||
convertTools,
|
||||
resolveGoogleFunctionCallingMode,
|
||||
supportsGoogleStrictToolSampling,
|
||||
} from "../src/api/google-shared.ts";
|
||||
import type { Tool } from "../src/types.ts";
|
||||
|
||||
function makeTool(parameters: Record<string, unknown>): Tool {
|
||||
@@ -180,6 +184,18 @@ describe("google-shared convertTools", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("uses validated function calling for strict tools on Gemini 3", () => {
|
||||
const tool = makeTool({ type: "object", properties: {} });
|
||||
tool.constrainedSampling = { type: "json_schema", strict: "require" };
|
||||
|
||||
expect(supportsGoogleStrictToolSampling("gemini-3.1-pro-preview")).toBe(true);
|
||||
expect(supportsGoogleStrictToolSampling("gemini-2.5-pro")).toBe(false);
|
||||
expect(resolveGoogleFunctionCallingMode([tool], undefined, true)).toBe("VALIDATED");
|
||||
expect(() => resolveGoogleFunctionCallingMode([tool], undefined, false)).toThrow(
|
||||
'Tool "test_tool" requires JSON-schema constrained sampling',
|
||||
);
|
||||
});
|
||||
|
||||
it("returns undefined for empty tool list", () => {
|
||||
expect(convertTools([])).toBeUndefined();
|
||||
expect(convertTools([], true)).toBeUndefined();
|
||||
|
||||
@@ -9,6 +9,7 @@ interface MistralToolPayload {
|
||||
function: {
|
||||
name: string;
|
||||
parameters: Record<string, unknown>;
|
||||
strict?: boolean;
|
||||
};
|
||||
}>;
|
||||
}
|
||||
@@ -31,6 +32,7 @@ describe("Mistral tool schema serialization", () => {
|
||||
name: "inspect_schema",
|
||||
description: "Inspect the schema",
|
||||
parameters,
|
||||
constrainedSampling: { type: "json_schema", strict: "require" },
|
||||
},
|
||||
],
|
||||
};
|
||||
@@ -45,6 +47,7 @@ describe("Mistral tool schema serialization", () => {
|
||||
});
|
||||
|
||||
expect(capturedPayload?.tools).toHaveLength(1);
|
||||
expect(capturedPayload?.tools?.[0]?.function.strict).toBe(true);
|
||||
const payloadParameters = capturedPayload?.tools?.[0]?.function.parameters;
|
||||
expect(payloadParameters).toBeDefined();
|
||||
expect(Object.getOwnPropertySymbols(payloadParameters ?? {})).toHaveLength(0);
|
||||
|
||||
@@ -11,6 +11,7 @@ import {
|
||||
validateModelDataDirectory,
|
||||
} from "../scripts/model-data.ts";
|
||||
|
||||
const GENERATED_AT = "2026-07-23T10:00:00.000Z";
|
||||
const temporaryRoots: string[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
@@ -70,7 +71,7 @@ function writeFixtureData(
|
||||
const filename = "test-provider.json";
|
||||
const content = `${JSON.stringify({ [apiGroup]: values })}\n`;
|
||||
writeFileSync(join(dataDir, filename), content);
|
||||
const manifest = createModelDataManifest(structure, { [filename]: content });
|
||||
const manifest = createModelDataManifest(structure, { [filename]: content }, GENERATED_AT);
|
||||
manifest.schemaVersion = manifestSchemaVersion;
|
||||
writeFileSync(join(dataDir, MODEL_DATA_MANIFEST_FILE), `${JSON.stringify(manifest)}\n`);
|
||||
}
|
||||
@@ -120,7 +121,7 @@ describe("generated model data validation", () => {
|
||||
"anthropic-messages": fixture.values,
|
||||
})}\n`;
|
||||
writeFileSync(join(fixture.dataDir, filename), content);
|
||||
const manifest = createModelDataManifest(fixture.structure, { [filename]: content });
|
||||
const manifest = createModelDataManifest(fixture.structure, { [filename]: content }, GENERATED_AT);
|
||||
writeFileSync(join(fixture.dataDir, MODEL_DATA_MANIFEST_FILE), `${JSON.stringify(manifest)}\n`);
|
||||
expect(() => validateModelDataDirectory(fixture.structure, fixture.dataDir)).toThrow("more than one API group");
|
||||
});
|
||||
@@ -143,6 +144,15 @@ describe("generated model data validation", () => {
|
||||
expect(() => validateModelDataDirectory(fixture.structure, fixture.dataDir)).toThrow("generation stamp");
|
||||
});
|
||||
|
||||
it("rejects an invalid generation timestamp", () => {
|
||||
const fixture = createFixture();
|
||||
const manifestPath = join(fixture.dataDir, MODEL_DATA_MANIFEST_FILE);
|
||||
const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as Record<string, unknown>;
|
||||
manifest.generatedAt = "invalid";
|
||||
writeFileSync(manifestPath, `${JSON.stringify(manifest)}\n`);
|
||||
expect(() => validateModelDataDirectory(fixture.structure, fixture.dataDir)).toThrow("generation timestamp");
|
||||
});
|
||||
|
||||
it("rejects missing provider shards imported by the aggregator", () => {
|
||||
const { packageRoot } = createFixture();
|
||||
writeFileSync(
|
||||
|
||||
@@ -567,6 +567,28 @@ describe("Models runtime", () => {
|
||||
await expect(oauthModels.getAuth("p1")).rejects.toMatchObject({ code: "auth" });
|
||||
});
|
||||
|
||||
it("keeps the underlying reason in wrapped oauth refresh errors", async () => {
|
||||
const credentials = new InMemoryCredentialStore();
|
||||
await credentials.modify("p1", async () => ({ type: "oauth", access: "old", refresh: "r", expires: 0 }));
|
||||
const models = createModels({ credentials });
|
||||
models.setProvider(
|
||||
testProvider({
|
||||
id: "p1",
|
||||
auth: {
|
||||
oauth: testOAuth({
|
||||
refresh: async () => {
|
||||
throw new Error("token refresh failed (400): invalid_grant");
|
||||
},
|
||||
}),
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(models.getAuth("p1")).rejects.toThrow(
|
||||
"OAuth refresh failed for p1: token refresh failed (400): invalid_grant",
|
||||
);
|
||||
});
|
||||
|
||||
it("wraps api-key auth failures in ModelsError", async () => {
|
||||
const failing: ApiKeyAuth = {
|
||||
name: "Failing",
|
||||
|
||||
@@ -590,6 +590,57 @@ describe("openai-codex streaming", () => {
|
||||
await streamResult.result();
|
||||
});
|
||||
|
||||
it("omits SSE cache affinity when cacheRetention is none", async () => {
|
||||
const token = mockToken();
|
||||
const encoder = new TextEncoder();
|
||||
let capturedHeaders: Headers | undefined;
|
||||
let capturedBody: Record<string, unknown> | null = null;
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async (_input: string | URL, init?: RequestInit) => {
|
||||
capturedHeaders = init?.headers instanceof Headers ? init.headers : undefined;
|
||||
capturedBody = decodeCodexRequestBody(init?.body);
|
||||
return new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(encoder.encode(buildSSEPayload({ status: "completed" })));
|
||||
controller.close();
|
||||
},
|
||||
}),
|
||||
{ status: 200, headers: { "content-type": "text/event-stream" } },
|
||||
);
|
||||
}),
|
||||
);
|
||||
|
||||
const model: Model<"openai-codex-responses"> = {
|
||||
id: "gpt-5.1-codex",
|
||||
name: "GPT-5.1 Codex",
|
||||
api: "openai-codex-responses",
|
||||
provider: "openai-codex",
|
||||
baseUrl: "https://chatgpt.com/backend-api",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 400000,
|
||||
maxTokens: 128000,
|
||||
};
|
||||
const context: Context = {
|
||||
systemPrompt: "You are a helpful assistant.",
|
||||
messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }],
|
||||
};
|
||||
|
||||
await streamOpenAICodexResponses(model, context, {
|
||||
apiKey: token,
|
||||
cacheRetention: "none",
|
||||
sessionId: "one-off-summary",
|
||||
transport: "sse",
|
||||
}).result();
|
||||
|
||||
expect(capturedHeaders?.has("session-id")).toBe(false);
|
||||
expect(capturedHeaders?.has("x-client-request-id")).toBe(false);
|
||||
expect(capturedBody).not.toHaveProperty("prompt_cache_key");
|
||||
});
|
||||
|
||||
it("clamps prompt_cache_key to OpenAI's 64-character limit", async () => {
|
||||
const token = mockToken();
|
||||
const sessionId = "x".repeat(67);
|
||||
@@ -804,6 +855,75 @@ describe("openai-codex streaming", () => {
|
||||
expect(requestedToolChoice).toBe("required");
|
||||
});
|
||||
|
||||
it("sets Codex strict mode explicitly and honors constrained sampling", async () => {
|
||||
const token = mockToken();
|
||||
const encoder = new TextEncoder();
|
||||
const sse = buildSSEPayload({ status: "completed" });
|
||||
let requestedTools: Array<{ type?: string; name?: string; strict?: boolean | null }> | undefined;
|
||||
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(
|
||||
async () =>
|
||||
new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(encoder.encode(sse));
|
||||
controller.close();
|
||||
},
|
||||
}),
|
||||
{ status: 200, headers: { "content-type": "text/event-stream" } },
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
const model: Model<"openai-codex-responses"> = {
|
||||
id: "gpt-5.5",
|
||||
name: "GPT-5.5",
|
||||
api: "openai-codex-responses",
|
||||
provider: "openai-codex",
|
||||
baseUrl: "https://chatgpt.com/backend-api",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 400000,
|
||||
maxTokens: 128000,
|
||||
};
|
||||
|
||||
await streamOpenAICodexResponses(
|
||||
model,
|
||||
{
|
||||
messages: [{ role: "user", content: "Use a tool", timestamp: Date.now() }],
|
||||
tools: [
|
||||
{
|
||||
name: "optional",
|
||||
description: "Optional constrained sampling",
|
||||
parameters: Type.Object({ value: Type.String() }),
|
||||
constrainedSampling: false,
|
||||
},
|
||||
{
|
||||
name: "strict",
|
||||
description: "Strict constrained sampling",
|
||||
parameters: Type.Object({ value: Type.String() }, { additionalProperties: false }),
|
||||
constrainedSampling: { type: "json_schema", strict: "prefer" },
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
apiKey: token,
|
||||
transport: "sse",
|
||||
onPayload: (payload) => {
|
||||
requestedTools = (payload as { tools?: typeof requestedTools }).tools;
|
||||
},
|
||||
},
|
||||
).result();
|
||||
|
||||
expect(requestedTools).toMatchObject([
|
||||
{ type: "function", name: "optional", strict: null },
|
||||
{ type: "function", name: "strict", strict: true },
|
||||
]);
|
||||
});
|
||||
|
||||
it.each(["gpt-5.3-codex", "gpt-5.4", "gpt-5.5"])("clamps %s minimal reasoning effort to low", async (modelId) => {
|
||||
const tempDir = mkdtempSync(join(tmpdir(), "pi-codex-stream-"));
|
||||
process.env.PI_CODING_AGENT_DIR = tempDir;
|
||||
@@ -1214,6 +1334,100 @@ describe("openai-codex streaming", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("closes one-shot websockets when cacheRetention is none", async () => {
|
||||
const token = mockToken();
|
||||
const sentBodies: Array<{ prompt_cache_key?: string }> = [];
|
||||
let connections = 0;
|
||||
let closedConnections = 0;
|
||||
|
||||
class MockWebSocket {
|
||||
private listeners = new Map<string, Set<(event: unknown) => void>>();
|
||||
|
||||
constructor() {
|
||||
connections++;
|
||||
queueMicrotask(() => this.dispatch("open", {}));
|
||||
}
|
||||
|
||||
addEventListener(type: string, listener: (event: unknown) => void): void {
|
||||
let listeners = this.listeners.get(type);
|
||||
if (!listeners) {
|
||||
listeners = new Set();
|
||||
this.listeners.set(type, listeners);
|
||||
}
|
||||
listeners.add(listener);
|
||||
}
|
||||
|
||||
removeEventListener(type: string, listener: (event: unknown) => void): void {
|
||||
this.listeners.get(type)?.delete(listener);
|
||||
}
|
||||
|
||||
send(data: string): void {
|
||||
sentBodies.push(JSON.parse(data) as { prompt_cache_key?: string });
|
||||
queueMicrotask(() => {
|
||||
this.dispatch("message", {
|
||||
data: JSON.stringify({
|
||||
type: "response.completed",
|
||||
response: {
|
||||
id: `resp_${connections}`,
|
||||
status: "completed",
|
||||
usage: { input_tokens: 5, output_tokens: 3, total_tokens: 8 },
|
||||
},
|
||||
}),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
close(): void {
|
||||
closedConnections++;
|
||||
}
|
||||
|
||||
private dispatch(type: string, event: unknown): void {
|
||||
for (const listener of this.listeners.get(type) ?? []) {
|
||||
listener(event);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
vi.stubGlobal("WebSocket", MockWebSocket);
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async () => new Response("unexpected fetch", { status: 500 })),
|
||||
);
|
||||
|
||||
const model: Model<"openai-codex-responses"> = {
|
||||
id: "gpt-5.1-codex",
|
||||
name: "GPT-5.1 Codex",
|
||||
api: "openai-codex-responses",
|
||||
provider: "openai-codex",
|
||||
baseUrl: "https://chatgpt.com/backend-api",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 400000,
|
||||
maxTokens: 128000,
|
||||
};
|
||||
const context: Context = {
|
||||
systemPrompt: "You are a helpful assistant.",
|
||||
messages: [{ role: "user", content: "Say hello", timestamp: 1 }],
|
||||
};
|
||||
const options = {
|
||||
apiKey: token,
|
||||
cacheRetention: "none" as const,
|
||||
sessionId: "one-off-summary",
|
||||
transport: "auto" as const,
|
||||
};
|
||||
|
||||
await streamOpenAICodexResponses(model, context, options).result();
|
||||
await streamOpenAICodexResponses(model, context, options).result();
|
||||
|
||||
expect(connections).toBe(2);
|
||||
expect(closedConnections).toBe(2);
|
||||
expect(sentBodies).toHaveLength(2);
|
||||
expect(sentBodies.every((body) => body.prompt_cache_key === undefined)).toBe(true);
|
||||
expect(getOpenAICodexWebSocketDebugStats("one-off-summary")).toBeUndefined();
|
||||
expect(global.fetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("falls back to SSE when websocket connect does not open before the connect timeout", async () => {
|
||||
vi.useFakeTimers();
|
||||
const token = mockToken();
|
||||
@@ -1658,10 +1872,6 @@ describe("openai-codex streaming", () => {
|
||||
it("sends only response input deltas in websocket-cached mode", async () => {
|
||||
const token = mockToken();
|
||||
const sentBodies: unknown[] = [];
|
||||
const responses = [
|
||||
{ responseId: "resp_1", messageId: "msg_1", text: "Hello" },
|
||||
{ responseId: "resp_2", messageId: "msg_2", text: "Done" },
|
||||
];
|
||||
|
||||
class MockWebSocket {
|
||||
static OPEN = 1;
|
||||
@@ -1687,36 +1897,41 @@ describe("openai-codex streaming", () => {
|
||||
|
||||
send(data: string): void {
|
||||
sentBodies.push(JSON.parse(data));
|
||||
const response = responses.shift();
|
||||
if (!response) throw new Error("unexpected websocket request");
|
||||
const responseId = `resp_${sentBodies.length}`;
|
||||
const outputEvents =
|
||||
sentBodies.length === 1
|
||||
? [
|
||||
{
|
||||
type: "response.output_item.added",
|
||||
item: {
|
||||
type: "custom_tool_call",
|
||||
id: "ctc_1",
|
||||
call_id: "call_1",
|
||||
name: "sample_tool",
|
||||
input: "",
|
||||
},
|
||||
},
|
||||
{ type: "response.custom_tool_call_input.delta", item_id: "ctc_1", delta: "abc" },
|
||||
{ type: "response.custom_tool_call_input.done", item_id: "ctc_1", input: "abc" },
|
||||
{
|
||||
type: "response.output_item.done",
|
||||
item: {
|
||||
type: "custom_tool_call",
|
||||
id: "ctc_1",
|
||||
call_id: "call_1",
|
||||
name: "sample_tool",
|
||||
input: "abc",
|
||||
},
|
||||
},
|
||||
]
|
||||
: [];
|
||||
const events = [
|
||||
{ type: "response.created", response: { id: response.responseId } },
|
||||
{
|
||||
type: "response.output_item.added",
|
||||
item: {
|
||||
type: "message",
|
||||
id: response.messageId,
|
||||
role: "assistant",
|
||||
status: "in_progress",
|
||||
content: [],
|
||||
},
|
||||
},
|
||||
{ type: "response.content_part.added", part: { type: "output_text", text: "" } },
|
||||
{ type: "response.output_text.delta", delta: response.text },
|
||||
{
|
||||
type: "response.output_item.done",
|
||||
item: {
|
||||
type: "message",
|
||||
id: response.messageId,
|
||||
role: "assistant",
|
||||
status: "completed",
|
||||
content: [{ type: "output_text", text: response.text }],
|
||||
},
|
||||
},
|
||||
{ type: "response.created", response: { id: responseId } },
|
||||
...outputEvents,
|
||||
{
|
||||
type: "response.completed",
|
||||
response: {
|
||||
id: response.responseId,
|
||||
id: responseId,
|
||||
status: "completed",
|
||||
usage: {
|
||||
input_tokens: 5,
|
||||
@@ -1758,10 +1973,19 @@ describe("openai-codex streaming", () => {
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 400000,
|
||||
maxTokens: 128000,
|
||||
compat: { supportsOpenAIGrammarTools: true },
|
||||
};
|
||||
const firstContext: Context = {
|
||||
systemPrompt: "You are a helpful assistant.",
|
||||
messages: [{ role: "user", content: "Say hello", timestamp: 1 }],
|
||||
messages: [{ role: "user", content: "Use the tool", timestamp: 1 }],
|
||||
tools: [
|
||||
{
|
||||
name: "sample_tool",
|
||||
description: "Sample tool",
|
||||
parameters: Type.Object({ payload: Type.String() }),
|
||||
constrainedSampling: { type: "grammar", variants: { openai_lark: "start: /[a-z]+/" } },
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const first = await streamOpenAICodexResponses(model, firstContext, {
|
||||
@@ -1771,8 +1995,20 @@ describe("openai-codex streaming", () => {
|
||||
}).result();
|
||||
|
||||
const secondContext: Context = {
|
||||
systemPrompt: "You are a helpful assistant.",
|
||||
messages: [...firstContext.messages, first, { role: "user", content: "Now finish", timestamp: 2 }],
|
||||
...firstContext,
|
||||
messages: [
|
||||
...firstContext.messages,
|
||||
first,
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: "call_1|ctc_1",
|
||||
toolName: "sample_tool",
|
||||
content: [{ type: "text", text: "real result" }],
|
||||
isError: false,
|
||||
timestamp: 2,
|
||||
},
|
||||
{ role: "user", content: "Now finish", timestamp: 3 },
|
||||
],
|
||||
};
|
||||
await streamOpenAICodexResponses(model, secondContext, {
|
||||
apiKey: token,
|
||||
@@ -1785,10 +2021,13 @@ describe("openai-codex streaming", () => {
|
||||
const secondBody = sentBodies[1] as { input: unknown[]; previous_response_id?: string; store?: boolean };
|
||||
expect(firstBody.store).toBe(false);
|
||||
expect(firstBody.previous_response_id).toBeUndefined();
|
||||
expect(firstBody.input).toEqual([{ role: "user", content: [{ type: "input_text", text: "Say hello" }] }]);
|
||||
expect(firstBody.input).toEqual([{ role: "user", content: [{ type: "input_text", text: "Use the tool" }] }]);
|
||||
expect(secondBody.store).toBe(false);
|
||||
expect(secondBody.previous_response_id).toBe("resp_1");
|
||||
expect(secondBody.input).toEqual([{ role: "user", content: [{ type: "input_text", text: "Now finish" }] }]);
|
||||
expect(secondBody.input).toEqual([
|
||||
{ type: "custom_tool_call_output", call_id: "call_1", output: "real result" },
|
||||
{ role: "user", content: [{ type: "input_text", text: "Now finish" }] },
|
||||
]);
|
||||
expect(getOpenAICodexWebSocketDebugStats("session-1")).toMatchObject({
|
||||
requests: 2,
|
||||
connectionsCreated: 1,
|
||||
@@ -1797,7 +2036,7 @@ describe("openai-codex streaming", () => {
|
||||
storeTrueRequests: 0,
|
||||
fullContextRequests: 1,
|
||||
deltaRequests: 1,
|
||||
lastDeltaInputItems: 1,
|
||||
lastDeltaInputItems: 2,
|
||||
lastPreviousResponseId: "resp_1",
|
||||
});
|
||||
});
|
||||
@@ -2094,6 +2333,46 @@ describe("openai-codex streaming", () => {
|
||||
expect(codexRequests).toBe(2);
|
||||
});
|
||||
|
||||
it.each([429, 503])("fails immediately when a %i retry delay exceeds the limit", async (status) => {
|
||||
const token = mockToken();
|
||||
const fetchMock = vi.fn(
|
||||
async () =>
|
||||
new Response(JSON.stringify({ error: { code: "temporarily_unavailable", message: "retry later" } }), {
|
||||
status,
|
||||
headers: { "content-type": "application/json", "retry-after": "2" },
|
||||
}),
|
||||
);
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const model: Model<"openai-codex-responses"> = {
|
||||
id: "gpt-5.1-codex",
|
||||
name: "GPT-5.1 Codex",
|
||||
api: "openai-codex-responses",
|
||||
provider: "openai-codex",
|
||||
baseUrl: "https://chatgpt.com/backend-api",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 400000,
|
||||
maxTokens: 128000,
|
||||
};
|
||||
const context: Context = {
|
||||
systemPrompt: "You are a helpful assistant.",
|
||||
messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }],
|
||||
};
|
||||
|
||||
const result = await streamOpenAICodexResponses(model, context, {
|
||||
apiKey: token,
|
||||
transport: "sse",
|
||||
maxRetries: 3,
|
||||
maxRetryDelayMs: 1000,
|
||||
}).result();
|
||||
|
||||
expect(result.stopReason).toBe("error");
|
||||
expect(result.errorMessage).toBe("Server requested 2s retry delay (max: 1s)");
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("zstd-compresses SSE request bodies", async () => {
|
||||
const token = mockToken();
|
||||
const encoder = new TextEncoder();
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { stream as streamOpenAICompletions } from "../src/api/openai-completions.ts";
|
||||
import type { Context, Model } from "../src/types.ts";
|
||||
|
||||
const mockState = vi.hoisted(() => ({
|
||||
requestOptions: [] as unknown[],
|
||||
requestErrors: [] as Error[],
|
||||
}));
|
||||
|
||||
vi.mock("openai", () => {
|
||||
@@ -30,10 +31,14 @@ vi.mock("openai", () => {
|
||||
response: { status: number; headers: Headers };
|
||||
}>;
|
||||
};
|
||||
promise.withResponse = async () => ({
|
||||
data: stream,
|
||||
response: { status: 200, headers: new Headers() },
|
||||
});
|
||||
promise.withResponse = async () => {
|
||||
const error = mockState.requestErrors.shift();
|
||||
if (error) throw error;
|
||||
return {
|
||||
data: stream,
|
||||
response: { status: 200, headers: new Headers() },
|
||||
};
|
||||
};
|
||||
return promise;
|
||||
},
|
||||
},
|
||||
@@ -61,7 +66,7 @@ const context: Context = {
|
||||
tools: [],
|
||||
};
|
||||
|
||||
async function consume(options?: { maxRetries?: number }) {
|
||||
async function consume(options?: { maxRetries?: number; maxRetryDelayMs?: number }) {
|
||||
const stream = streamOpenAICompletions(model, context, { apiKey: "test", ...options });
|
||||
for await (const _event of stream) {
|
||||
void _event;
|
||||
@@ -72,6 +77,11 @@ async function consume(options?: { maxRetries?: number }) {
|
||||
describe("openai-completions provider retries", () => {
|
||||
beforeEach(() => {
|
||||
mockState.requestOptions = [];
|
||||
mockState.requestErrors = [];
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("disables SDK retries by default", async () => {
|
||||
@@ -79,8 +89,51 @@ describe("openai-completions provider retries", () => {
|
||||
expect(mockState.requestOptions).toEqual([expect.objectContaining({ maxRetries: 0 })]);
|
||||
});
|
||||
|
||||
it("honors explicit provider retry settings", async () => {
|
||||
await consume({ maxRetries: 2 });
|
||||
expect(mockState.requestOptions).toEqual([expect.objectContaining({ maxRetries: 2 })]);
|
||||
it("honors provider retries while keeping SDK retries disabled", async () => {
|
||||
vi.useFakeTimers();
|
||||
mockState.requestErrors = [
|
||||
Object.assign(new Error("rate limited"), {
|
||||
status: 429,
|
||||
headers: new Headers({ "retry-after-ms": "100" }),
|
||||
}),
|
||||
Object.assign(new Error("server error"), {
|
||||
status: 500,
|
||||
headers: new Headers({ "retry-after-ms": "100" }),
|
||||
}),
|
||||
];
|
||||
|
||||
const result = consume({ maxRetries: 2, maxRetryDelayMs: 100 });
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(mockState.requestOptions).toHaveLength(1);
|
||||
await vi.advanceTimersByTimeAsync(99);
|
||||
expect(mockState.requestOptions).toHaveLength(1);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(mockState.requestOptions).toHaveLength(2);
|
||||
await vi.advanceTimersByTimeAsync(99);
|
||||
expect(mockState.requestOptions).toHaveLength(2);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
await result;
|
||||
|
||||
expect(mockState.requestOptions).toEqual([
|
||||
expect.objectContaining({ maxRetries: 0 }),
|
||||
expect.objectContaining({ maxRetries: 0 }),
|
||||
expect.objectContaining({ maxRetries: 0 }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("fails immediately when a provider-requested retry delay exceeds the limit", async () => {
|
||||
mockState.requestErrors = [
|
||||
Object.assign(new Error("rate limited"), {
|
||||
status: 429,
|
||||
headers: new Headers({ "retry-after": "277403" }),
|
||||
}),
|
||||
];
|
||||
|
||||
const result = await consume({ maxRetries: 2, maxRetryDelayMs: 1000 });
|
||||
|
||||
expect(result.stopReason).toBe("error");
|
||||
expect(result.errorMessage).toContain("Server requested 277403s retry delay (max: 1s)");
|
||||
expect(result.errorMessage).toContain("rate limited");
|
||||
expect(mockState.requestOptions).toEqual([expect.objectContaining({ maxRetries: 0 })]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -37,6 +37,7 @@ const compat = {
|
||||
chatTemplateKwargs: {},
|
||||
zaiToolStream: false,
|
||||
supportsStrictMode: true,
|
||||
supportsOpenAIGrammarTools: false,
|
||||
cacheControlFormat: undefined,
|
||||
sendSessionAffinityHeaders: false,
|
||||
sessionAffinityFormat: "openai",
|
||||
|
||||
@@ -1258,6 +1258,7 @@ describe("openai-completions tool_choice", () => {
|
||||
chatTemplateKwargs: {},
|
||||
zaiToolStream: false,
|
||||
supportsStrictMode: true,
|
||||
supportsOpenAIGrammarTools: false,
|
||||
sendSessionAffinityHeaders: false,
|
||||
sessionAffinityFormat: "openai",
|
||||
supportsLongCacheRetention: true,
|
||||
|
||||
@@ -37,6 +37,7 @@ const compat: Omit<Required<OpenAICompletionsCompat>, "deferredToolsMode"> & {
|
||||
chatTemplateKwargs: {},
|
||||
zaiToolStream: false,
|
||||
supportsStrictMode: true,
|
||||
supportsOpenAIGrammarTools: false,
|
||||
cacheControlFormat: "anthropic",
|
||||
sendSessionAffinityHeaders: false,
|
||||
sessionAffinityFormat: "openai",
|
||||
|
||||
@@ -90,7 +90,7 @@ describe.skipIf(!process.env.OPENAI_API_KEY || !process.env.ANTHROPIC_API_KEY)(
|
||||
// 6. With fix: tool calls/results converted to text, conversation continues
|
||||
|
||||
const modelA = getModel("openai", "gpt-5-mini");
|
||||
const modelB = getModel("openai", "gpt-5.2-codex");
|
||||
const modelB = getModel("openai", "gpt-5.5");
|
||||
|
||||
const apiKey = getEnvApiKey("openai");
|
||||
if (!apiKey) {
|
||||
@@ -189,7 +189,7 @@ describe.skipIf(!process.env.OPENAI_API_KEY || !process.env.ANTHROPIC_API_KEY)(
|
||||
// 5. Should work because foreign IDs have no pairing expectation
|
||||
|
||||
const anthropicModel = getModel("anthropic", "claude-sonnet-4-5");
|
||||
const codexModel = getModel("openai", "gpt-5.2-codex");
|
||||
const codexModel = getModel("openai", "gpt-5.5");
|
||||
|
||||
const anthropicApiKey = getEnvApiKey("anthropic");
|
||||
const openaiApiKey = getEnvApiKey("openai");
|
||||
|
||||
@@ -186,4 +186,28 @@ describe("provider error body passthrough (per-tier regression)", () => {
|
||||
expect(output.errorMessage).toContain("blocked by gateway WAF");
|
||||
expect(output.errorMessage).not.toContain("Unknown: UnknownError");
|
||||
});
|
||||
|
||||
it("bedrock preserves the SDK validation message when the response body is a stream", async () => {
|
||||
bedrockMock.sendError = Object.assign(
|
||||
new Error(
|
||||
"Invocation of model ID anthropic.claude-opus-5 with on-demand throughput isn't supported. Retry with an inference profile.",
|
||||
),
|
||||
{
|
||||
name: "ValidationException",
|
||||
$metadata: { httpStatusCode: 400 },
|
||||
$response: {
|
||||
statusCode: 400,
|
||||
body: { pipe: () => undefined, _readableState: { buffer: [], length: 0 } },
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const model = getModel("amazon-bedrock", "global.anthropic.claude-opus-5");
|
||||
const output = await drainResult(streamSimpleBedrock(model, { messages: context.messages }, {}));
|
||||
|
||||
expect(output.stopReason).toBe("error");
|
||||
expect(output.errorMessage).toContain("on-demand throughput isn't supported");
|
||||
expect(output.errorMessage).toContain("inference profile");
|
||||
expect(output.errorMessage).not.toContain("_readableState");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { retryProviderRequest } from "../src/utils/provider-retry.ts";
|
||||
|
||||
function providerError(status: number | undefined, headers?: Record<string, string>): Error {
|
||||
return Object.assign(new Error(`Provider error: ${status}`), {
|
||||
status,
|
||||
headers: new Headers(headers),
|
||||
});
|
||||
}
|
||||
|
||||
describe("provider request retries", () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("retries retryable provider errors", async () => {
|
||||
vi.useFakeTimers();
|
||||
const request = vi
|
||||
.fn<() => Promise<string>>()
|
||||
.mockRejectedValueOnce(providerError(429, { "retry-after-ms": "1000" }))
|
||||
.mockResolvedValue("ok");
|
||||
|
||||
const result = retryProviderRequest(request, { maxRetries: 1 });
|
||||
await vi.advanceTimersByTimeAsync(999);
|
||||
expect(request).toHaveBeenCalledTimes(1);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
|
||||
await expect(result).resolves.toBe("ok");
|
||||
expect(request).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("does not retry errors the provider marks as non-retryable", async () => {
|
||||
const error = providerError(429, { "x-should-retry": "false" });
|
||||
const request = vi.fn<() => Promise<string>>().mockRejectedValue(error);
|
||||
|
||||
await expect(retryProviderRequest(request, { maxRetries: 2 })).rejects.toBe(error);
|
||||
expect(request).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("rejects a provider-requested retry delay above the limit", async () => {
|
||||
const request = vi.fn<() => Promise<string>>().mockRejectedValue(providerError(429, { "retry-after": "277403" }));
|
||||
|
||||
await expect(retryProviderRequest(request, { maxRetries: 1, maxRetryDelayMs: 1000 })).rejects.toThrow(
|
||||
"Server requested 277403s retry delay (max: 1s)",
|
||||
);
|
||||
expect(request).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("allows disabling the provider-requested retry delay cap", async () => {
|
||||
vi.useFakeTimers();
|
||||
const request = vi
|
||||
.fn<() => Promise<string>>()
|
||||
.mockRejectedValueOnce(providerError(429, { "retry-after": "2" }))
|
||||
.mockResolvedValue("ok");
|
||||
|
||||
const result = retryProviderRequest(request, { maxRetries: 1, maxRetryDelayMs: 0 });
|
||||
await vi.advanceTimersByTimeAsync(1999);
|
||||
expect(request).toHaveBeenCalledTimes(1);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
|
||||
await expect(result).resolves.toBe("ok");
|
||||
expect(request).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("aborts a provider-requested retry delay", async () => {
|
||||
vi.useFakeTimers();
|
||||
const controller = new AbortController();
|
||||
const request = vi.fn<() => Promise<string>>().mockRejectedValue(providerError(429, { "retry-after": "277403" }));
|
||||
|
||||
const result = retryProviderRequest(request, { maxRetries: 2, maxRetryDelayMs: 0, signal: controller.signal });
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(request).toHaveBeenCalledTimes(1);
|
||||
expect(vi.getTimerCount()).toBe(1);
|
||||
|
||||
controller.abort();
|
||||
|
||||
await expect(result).rejects.toMatchObject({ name: "AbortError" });
|
||||
expect(request).toHaveBeenCalledTimes(1);
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -3,7 +3,7 @@ import { envApiKeyAuth } from "../src/auth/helpers.ts";
|
||||
import type { AuthContext, AuthEvent } from "../src/auth/types.ts";
|
||||
import { createModels, createProvider } from "../src/models.ts";
|
||||
import { InMemoryModelsStore, type ModelsStoreEntry } from "../src/models-store.ts";
|
||||
import { builtinModels, builtinProviders } from "../src/providers/all.ts";
|
||||
import { builtinModels, builtinProviders, getBuiltinModel } from "../src/providers/all.ts";
|
||||
import { amazonBedrockProvider } from "../src/providers/amazon-bedrock.ts";
|
||||
import { anthropicProvider } from "../src/providers/anthropic.ts";
|
||||
import { cloudflareAIGatewayProvider } from "../src/providers/cloudflare-ai-gateway.ts";
|
||||
@@ -44,6 +44,17 @@ describe("builtin providers", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("stores native constrained-sampling capabilities in model metadata", () => {
|
||||
const gpt4o = getBuiltinModel("openai", "gpt-4o");
|
||||
expect(gpt4o.compat?.supportsStrictMode).toBe(true);
|
||||
expect(gpt4o.compat?.supportsOpenAIGrammarTools).toBeUndefined();
|
||||
expect(getBuiltinModel("openai", "gpt-5.4").compat).toMatchObject({
|
||||
supportsStrictMode: true,
|
||||
supportsOpenAIGrammarTools: true,
|
||||
});
|
||||
expect(getBuiltinModel("anthropic", "claude-haiku-4-5").compat?.supportsStrictTools).toBe(true);
|
||||
});
|
||||
|
||||
it("uses official Kimi K3 pricing for Moonshot providers", () => {
|
||||
const models = builtinModels();
|
||||
for (const provider of ["moonshotai", "moonshotai-cn"]) {
|
||||
@@ -68,14 +79,29 @@ describe("builtin providers", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("resolves anthropic auth from env with OAuth token precedence", async () => {
|
||||
it("resolves Anthropic bearer auth from env with auth token precedence", async () => {
|
||||
const models = createModels({
|
||||
authContext: fakeAuthContext({
|
||||
ANTHROPIC_AUTH_TOKEN: "auth-token",
|
||||
ANTHROPIC_OAUTH_TOKEN: "oauth-token",
|
||||
ANTHROPIC_API_KEY: "api-key",
|
||||
}),
|
||||
});
|
||||
models.setProvider(anthropicProvider());
|
||||
|
||||
expect(await models.getAuth("anthropic")).toEqual({
|
||||
auth: { headers: { Authorization: "Bearer auth-token" } },
|
||||
source: "ANTHROPIC_AUTH_TOKEN",
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves Anthropic OAuth token precedence over the API key", async () => {
|
||||
const models = createModels({
|
||||
authContext: fakeAuthContext({ ANTHROPIC_API_KEY: "key", ANTHROPIC_OAUTH_TOKEN: "oauth-token" }),
|
||||
});
|
||||
models.setProvider(anthropicProvider());
|
||||
const model = models.getModel("anthropic", "claude-haiku-4-5")!;
|
||||
|
||||
const result = await models.getAuth(model.provider);
|
||||
const result = await models.getAuth("anthropic");
|
||||
expect(result?.auth.apiKey).toBe("oauth-token");
|
||||
expect(result?.source).toBe("ANTHROPIC_OAUTH_TOKEN");
|
||||
});
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { createRadiusOAuth } from "../src/auth/oauth/radius.ts";
|
||||
import type { AuthEvent, AuthInteraction } from "../src/auth/types.ts";
|
||||
|
||||
const GATEWAY = "https://radius.example";
|
||||
|
||||
function jsonResponse(body: unknown, status = 200): Response {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}
|
||||
|
||||
function requestUrl(input: unknown): string {
|
||||
if (typeof input === "string") return input;
|
||||
if (input instanceof URL) return input.toString();
|
||||
if (input instanceof Request) return input.url;
|
||||
throw new Error(`Unsupported request input: ${String(input)}`);
|
||||
}
|
||||
|
||||
function interaction(loginMethod: "browser" | "device-code", events: AuthEvent[] = []): AuthInteraction {
|
||||
return {
|
||||
prompt: async () => loginMethod,
|
||||
notify: (event) => events.push(event),
|
||||
};
|
||||
}
|
||||
|
||||
describe("Radius OAuth", () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllGlobals();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("uses gateway endpoints directly for device login", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-07-24T00:00:00Z"));
|
||||
const events: AuthEvent[] = [];
|
||||
const urls: string[] = [];
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async (input: unknown, init?: RequestInit) => {
|
||||
const url = requestUrl(input);
|
||||
urls.push(url);
|
||||
const form = new URLSearchParams(String(init?.body));
|
||||
if (url === `${GATEWAY}/v1/oauth/device`) {
|
||||
expect(form.get("client_id")).toBe("pi-gateway");
|
||||
expect(form.get("scope")).toBe("gateway offline_access");
|
||||
return jsonResponse({
|
||||
device_code: "device-code",
|
||||
user_code: "ABCD-1234",
|
||||
verification_uri: "https://radius-ui.example/pair",
|
||||
expires_in: 600,
|
||||
interval: 5,
|
||||
});
|
||||
}
|
||||
if (url === `${GATEWAY}/v1/oauth/token`) {
|
||||
expect(form.get("grant_type")).toBe("urn:ietf:params:oauth:grant-type:device_code");
|
||||
expect(form.get("client_id")).toBe("pi-gateway");
|
||||
expect(form.get("device_code")).toBe("device-code");
|
||||
return jsonResponse({
|
||||
access_token: "access-token",
|
||||
refresh_token: "refresh-token",
|
||||
expires_in: 3600,
|
||||
scope: "gateway offline_access",
|
||||
});
|
||||
}
|
||||
throw new Error(`Unexpected request: ${url}`);
|
||||
}),
|
||||
);
|
||||
|
||||
const oauth = createRadiusOAuth({ name: "Radius", gateway: GATEWAY });
|
||||
await expect(oauth.login(interaction("device-code", events))).resolves.toEqual({
|
||||
type: "oauth",
|
||||
access: "access-token",
|
||||
refresh: "refresh-token",
|
||||
expires: Date.now() + 3600 * 1000 - 60_000,
|
||||
scope: "gateway offline_access",
|
||||
});
|
||||
expect(events).toEqual([
|
||||
{
|
||||
type: "device_code",
|
||||
userCode: "ABCD-1234",
|
||||
verificationUri: "https://radius-ui.example/pair",
|
||||
intervalSeconds: 5,
|
||||
expiresInSeconds: 600,
|
||||
},
|
||||
]);
|
||||
expect(urls).toEqual([`${GATEWAY}/v1/oauth/device`, `${GATEWAY}/v1/oauth/token`]);
|
||||
});
|
||||
|
||||
it("refreshes directly through the gateway without discovery", async () => {
|
||||
const fetchMock = vi.fn(async (input: unknown, init?: RequestInit) => {
|
||||
expect(requestUrl(input)).toBe(`${GATEWAY}/v1/oauth/token`);
|
||||
const form = new URLSearchParams(String(init?.body));
|
||||
expect(form.get("grant_type")).toBe("refresh_token");
|
||||
expect(form.get("client_id")).toBe("pi-gateway");
|
||||
expect(form.get("refresh_token")).toBe("old-refresh");
|
||||
return jsonResponse({
|
||||
access_token: "new-access",
|
||||
refresh_token: "new-refresh",
|
||||
expires_in: 3600,
|
||||
});
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const oauth = createRadiusOAuth({ name: "Radius", gateway: GATEWAY });
|
||||
await expect(
|
||||
oauth.refresh({ type: "oauth", access: "old-access", refresh: "old-refresh", expires: 0 }),
|
||||
).resolves.toMatchObject({ access: "new-access", refresh: "new-refresh" });
|
||||
expect(fetchMock).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("discovers only the interactive browser authorization endpoint", async () => {
|
||||
const fetchMock = vi.fn(async (input: unknown) => {
|
||||
expect(requestUrl(input)).toBe(`${GATEWAY}/v1/oauth`);
|
||||
return jsonResponse({ issuer: "https://radius-ui.example" });
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const oauth = createRadiusOAuth({ name: "Radius", gateway: GATEWAY });
|
||||
await expect(oauth.login(interaction("browser"))).rejects.toThrow(`Invalid Radius OAuth config from ${GATEWAY}`);
|
||||
expect(fetchMock).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
@@ -16,6 +16,13 @@ describe("getSupportedThinkingLevels", () => {
|
||||
expect(getSupportedThinkingLevels(model!)).toContain("max");
|
||||
});
|
||||
|
||||
it("includes xhigh and max for Anthropic Opus 5 on anthropic-messages API", () => {
|
||||
const model = getModel("anthropic", "claude-opus-5");
|
||||
expect(model).toBeDefined();
|
||||
expect(getSupportedThinkingLevels(model!)).toContain("xhigh");
|
||||
expect(getSupportedThinkingLevels(model!)).toContain("max");
|
||||
});
|
||||
|
||||
it("includes max but not xhigh for Anthropic Sonnet 4.6 on anthropic-messages API", () => {
|
||||
const model = getModel("anthropic", "claude-sonnet-4-6");
|
||||
expect(model).toBeDefined();
|
||||
@@ -133,6 +140,13 @@ describe("getSupportedThinkingLevels", () => {
|
||||
expect(getSupportedThinkingLevels(model!)).not.toContain("xhigh");
|
||||
});
|
||||
|
||||
it("includes xhigh and max for Bedrock Claude Opus 5", () => {
|
||||
const model = getModel("amazon-bedrock", "global.anthropic.claude-opus-5");
|
||||
expect(model).toBeDefined();
|
||||
expect(getSupportedThinkingLevels(model!)).toContain("xhigh");
|
||||
expect(getSupportedThinkingLevels(model!)).toContain("max");
|
||||
});
|
||||
|
||||
it("includes xhigh and max but not off for Bedrock Claude Fable 5", () => {
|
||||
const model = getModel("amazon-bedrock", "global.anthropic.claude-fable-5");
|
||||
expect(model).toBeDefined();
|
||||
|
||||
@@ -15,10 +15,10 @@ function makeContext(): Context {
|
||||
}
|
||||
|
||||
describe.skipIf(!process.env.OPENAI_API_KEY)("xhigh reasoning", () => {
|
||||
describe("codex-max (supports xhigh)", () => {
|
||||
describe("gpt 5.5 (supports xhigh)", () => {
|
||||
// Note: codex models only support the responses API, not chat completions
|
||||
it("should work with openai-responses", async () => {
|
||||
const model = getModel("openai", "gpt-5.1-codex-max");
|
||||
const model = getModel("openai", "gpt-5.5");
|
||||
const s = stream(model, makeContext(), { reasoningEffort: "xhigh" });
|
||||
let hasThinking = false;
|
||||
|
||||
|
||||
@@ -2,14 +2,73 @@
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.82.1] - 2026-07-25
|
||||
|
||||
### New Features
|
||||
|
||||
- **Claude Opus 5** — Available on Anthropic and Amazon Bedrock with adaptive thinking (including `xhigh`), inference profiles, and prompt caching. See [Providers](docs/providers.md#api-keys).
|
||||
- **Anthropic gateway bearer auth** — `ANTHROPIC_AUTH_TOKEN` authenticates against Anthropic-compatible gateways that require `Authorization: Bearer`, including compaction and branch summaries. See [Environment Variables or Auth File](docs/providers.md#environment-variables-or-auth-file).
|
||||
- **Faster, more resilient model catalogs** — pi.dev catalogs revalidate with `If-None-Match` so unchanged providers answer with an empty `304`, and llama.cpp models stay listed across restarts. See [llama.cpp](docs/llama-cpp.md).
|
||||
|
||||
### Added
|
||||
|
||||
- Exposed `PI_SESSION_ID`, `PI_SESSION_FILE`, `PI_PROVIDER`, `PI_MODEL`, and `PI_REASONING_LEVEL` to commands run by built-in and factory-created bash tools.
|
||||
- Exposed the `outputPad` setting to custom message renderers. See [Extensions](docs/extensions.md) ([#7045](https://github.com/earendil-works/pi/pull/7045) by [@xl0](https://github.com/xl0)).
|
||||
- Added inherited `ANTHROPIC_AUTH_TOKEN` bearer authentication for Anthropic-compatible gateways. See [Providers](docs/providers.md#environment-variables-or-auth-file) ([#5871](https://github.com/earendil-works/pi/issues/5871)).
|
||||
- Added inherited Claude Opus 5 support for Anthropic and Amazon Bedrock with adaptive thinking, inference profiles, prompt caching, and preserved AWS validation messages ([#7081](https://github.com/earendil-works/pi/pull/7081) by [@unexge](https://github.com/unexge), [#7083](https://github.com/earendil-works/pi/pull/7083) by [@davidbrai](https://github.com/davidbrai)).
|
||||
|
||||
### Changed
|
||||
|
||||
- Changed pi.dev model catalog refreshes to revalidate with `If-None-Match`, so unchanged provider catalogs answer with an empty `304` instead of a full download.
|
||||
- Changed inherited Radius OAuth device authorization, token exchange, and refresh requests to use the configured gateway directly.
|
||||
- Changed inherited model loading errors to append the underlying cause, so auth failures such as `OAuth refresh failed for openai-codex` report the provider response instead of a bare wrapper message.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed compaction and branch-summary requests to use fresh routing session IDs with prompt caching disabled where supported.
|
||||
- Fixed compaction and branch summaries for providers whose authentication resolves entirely to request headers ([#5871](https://github.com/earendil-works/pi/issues/5871))
|
||||
- Fixed unavailable scoped models being hidden from `/models`, allowing them to be removed without editing settings manually ([#6949](https://github.com/earendil-works/pi/issues/6949), [#7032](https://github.com/earendil-works/pi/pull/7032) by [@christianklotz](https://github.com/christianklotz)).
|
||||
- Fixed startup context file discovery to skip directories that match context file names such as `AGENTS.md`, which produced `EISDIR` warnings ([#7106](https://github.com/earendil-works/pi/pull/7106) by [@mrexodia](https://github.com/mrexodia)).
|
||||
- Fixed the llama.cpp extension to persist its model catalog, so llama.cpp models stay listed before the first successful refresh. See [llama.cpp](docs/llama-cpp.md) ([#7072](https://github.com/earendil-works/pi/pull/7072) by [@davidbrai](https://github.com/davidbrai)).
|
||||
|
||||
## [0.82.0] - 2026-07-24
|
||||
|
||||
### New Features
|
||||
|
||||
- **Constrained tool sampling** — Tools can prefer or require strict JSON Schema sampling or use OpenAI Lark/regex grammars, with model capability metadata preventing unsupported requests. See [Constrained Sampling for Tools](../ai/README.md#constrained-sampling-for-tools).
|
||||
- **OpenRouter and Kimi Code sign-in** — Use `/login` to authorize OpenRouter or a Kimi Code subscription without manually configuring API keys. See [OpenRouter](docs/providers.md#openrouter).
|
||||
- **Session-aware, streaming bash integrations** — Bash tools receive current session/model metadata, while direct RPC bash commands stream correlated output. See [Bash Tool Session Environment](docs/environment-variables.md#bash-tool-session-environment) and [RPC bash events](docs/rpc.md#bash_execution_update).
|
||||
|
||||
### Added
|
||||
|
||||
- Added inherited `Tool.constrainedSampling` with strict JSON Schema (`prefer`/`require`) and OpenAI Lark/regex grammar variants across OpenAI, Anthropic, Amazon Bedrock, Google Gemini, and Mistral. See [Constrained Sampling for Tools](../ai/README.md#constrained-sampling-for-tools).
|
||||
- Added inherited `supportsGrammarTools` and `supportsStrictTools` compatibility flags, expanded `supportsStrictMode` coverage, and generated model capability metadata to gate constrained sampling.
|
||||
- Added inherited Kimi Code subscription OAuth login for the Kimi For Coding provider, including device authorization and automatic token refresh ([#6935](https://github.com/earendil-works/pi/pull/6935) by [@zaycruz](https://github.com/zaycruz)).
|
||||
- Added inherited OpenRouter OAuth PKCE login through `/login`, minting a user-controlled API key. See [OpenRouter](docs/providers.md#openrouter) ([#6927](https://github.com/earendil-works/pi/pull/6927) by [@rsaryev](https://github.com/rsaryev)).
|
||||
- Exposed `PI_SESSION_ID`, `PI_SESSION_FILE`, `PI_PROVIDER`, `PI_MODEL`, and `PI_REASONING_LEVEL` to commands run by built-in and factory-created bash tools. See [Bash Tool Session Environment](docs/environment-variables.md#bash-tool-session-environment).
|
||||
- Added streaming `bash_execution_update` events for direct RPC bash commands, correlated with request IDs. See [RPC bash events](docs/rpc.md#bash_execution_update) ([#6971](https://github.com/earendil-works/pi/pull/6971) by [@ananthakumaran](https://github.com/ananthakumaran)).
|
||||
|
||||
### Changed
|
||||
|
||||
- Changed inherited generated model catalogs to expose only provider-verified reasoning effort levels from models.dev ([#6928](https://github.com/earendil-works/pi/pull/6928) by [@davidbrai](https://github.com/davidbrai)).
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed inherited DNS lookup failures such as `getaddrinfo`, `ENOTFOUND`, and `EAI_AGAIN` to trigger automatic assistant retries ([#6946](https://github.com/earendil-works/pi/pull/6946) by [@christianklotz](https://github.com/christianklotz)).
|
||||
- Fixed inherited OpenRouter Anthropic cache breakpoints to advance through tool results and enabled cache control for `~anthropic/*-latest` aliases ([#6941](https://github.com/earendil-works/pi/pull/6941) by [@mteam88](https://github.com/mteam88)).
|
||||
- Fixed inherited OpenAI Codex WebSocket sessions to retry once without a missing previous-response continuation after `previous_response_not_found` errors ([#6955](https://github.com/earendil-works/pi/pull/6955) by [@davidbrai](https://github.com/davidbrai)).
|
||||
- Fixed TUI debug and crash logs to respect custom agent directories instead of always writing under `~/.pi/agent` ([#6958](https://github.com/earendil-works/pi/pull/6958) by [@davidbrai](https://github.com/davidbrai)).
|
||||
- Fixed slow Ctrl+G external-editor startup when the system temporary directory contains many entries ([#6903](https://github.com/earendil-works/pi/pull/6903) by [@christianklotz](https://github.com/christianklotz)).
|
||||
- Fixed startup resource display to preserve relative paths for sibling npm extensions loaded by a package ([#6964](https://github.com/earendil-works/pi/pull/6964) by [@davidbrai](https://github.com/davidbrai)).
|
||||
- Fixed compaction and branch-summary requests to use fresh routing session IDs with prompt caching disabled where supported ([#6618](https://github.com/earendil-works/pi/pull/6618) by [@tmustier](https://github.com/tmustier)).
|
||||
- Fixed explicit self-updates when `PI_SKIP_VERSION_CHECK` is set ([#6977](https://github.com/earendil-works/pi/issues/6977)).
|
||||
- Fixed scoped model IDs containing brackets to resolve as literal exact matches before glob matching ([#6210](https://github.com/earendil-works/pi/issues/6210)).
|
||||
- Fixed inherited OpenAI and Anthropic provider retry waits to honor abort signals and configured delay limits ([#6980](https://github.com/earendil-works/pi/pull/6980) by [@petrroll](https://github.com/petrroll)).
|
||||
- Fixed fresh installs from preferring bundled model catalogs over newer remote catalogs because package file mtimes were newer ([#7016](https://github.com/earendil-works/pi/pull/7016) by [@davidbrai](https://github.com/davidbrai)).
|
||||
- Fixed inherited editor scroll indicators overflowing narrow terminals ([#7015](https://github.com/earendil-works/pi/pull/7015) by [@christianklotz](https://github.com/christianklotz)).
|
||||
- Fixed llama.cpp models to use the loaded context window as their output token limit instead of capping it at 16K ([#7034](https://github.com/earendil-works/pi/pull/7034) by [@christianklotz](https://github.com/christianklotz)).
|
||||
- Fixed release source archives to include the generated provider model data used to build standalone binaries.
|
||||
- Updated the packaged `protobufjs` dependency to 7.6.5 to address GHSA-j3f2-48v5-ccww ([#7005](https://github.com/earendil-works/pi/issues/7005)).
|
||||
- Fixed `/copy` on Wayland to fall back to X11 or OSC 52 when `wl-copy` fails ([#7009](https://github.com/earendil-works/pi/pull/7009) by [@rkfshakti](https://github.com/rkfshakti)).
|
||||
- Fixed `/model` to reload updated `models.json` configuration when opening the model picker ([#6999](https://github.com/earendil-works/pi/issues/6999)).
|
||||
|
||||
## [0.81.1] - 2026-07-21
|
||||
|
||||
@@ -66,7 +125,6 @@
|
||||
- Fixed llama.cpp router download progress updates and removed redundant wording from model action confirmations.
|
||||
- Moved automatic model catalog network refresh out of startup initialization and into the running interactive and RPC modes.
|
||||
- Fixed persisted sessions being read and parsed twice when opened, reducing startup latency for large sessions ([#6793](https://github.com/earendil-works/pi/issues/6793)).
|
||||
- Fixed slow Ctrl+G external-editor startup when the system temporary directory contains many entries ([#6774](https://github.com/earendil-works/pi/issues/6774)).
|
||||
- Fixed prompt-template defaults for all arguments (`${@:-default}` and `${ARGUMENTS:-default}`) ([#6695](https://github.com/earendil-works/pi/issues/6695)).
|
||||
- Fixed obsolete custom UI, custom tool, and custom editor examples in the extension documentation ([#6735](https://github.com/earendil-works/pi/issues/6735)).
|
||||
- Fixed Kimi Coding sessions to show API-equivalent implied costs with the subscription indicator.
|
||||
|
||||
@@ -20,7 +20,7 @@ Pi has two summarization mechanisms:
|
||||
| Compaction | Context exceeds threshold, or `/compact` | Summarize old messages to free up context |
|
||||
| Branch summarization | `/tree` navigation | Preserve context when switching branches |
|
||||
|
||||
Both use the same structured summary format and track file operations cumulatively.
|
||||
Both use the same structured summary format and track file operations cumulatively. Compaction and branch-summary requests use fresh routing session IDs and, where supported by the provider, disable prompt-cache writes because these one-off prompts are unlikely to be reused.
|
||||
|
||||
## Compaction
|
||||
|
||||
|
||||
@@ -737,6 +737,8 @@ interface ProviderModelConfig {
|
||||
supportsDeveloperRole?: boolean;
|
||||
supportsReasoningEffort?: boolean;
|
||||
supportsUsageInStreaming?: boolean;
|
||||
supportsStrictMode?: boolean;
|
||||
supportsOpenAIGrammarTools?: boolean; // openai-completions/openai-responses; false falls back to normal function tools
|
||||
maxTokensField?: "max_completion_tokens" | "max_tokens";
|
||||
requiresToolResultName?: boolean;
|
||||
requiresAssistantAfterToolResult?: boolean;
|
||||
@@ -755,6 +757,7 @@ interface ProviderModelConfig {
|
||||
supportsCacheControlOnTools?: boolean;
|
||||
forceAdaptiveThinking?: boolean;
|
||||
allowEmptySignature?: boolean;
|
||||
supportsStrictTools?: boolean;
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
@@ -2791,7 +2791,7 @@ Register a custom renderer for messages with your `customType`. Use message rend
|
||||
import { Text } from "@earendil-works/pi-tui";
|
||||
|
||||
pi.registerMessageRenderer("my-extension", (message, options, theme) => {
|
||||
const { expanded } = options;
|
||||
const { expanded, outputPad } = options;
|
||||
let text = theme.fg("accent", `[${message.customType}] `);
|
||||
text += message.content;
|
||||
|
||||
@@ -2799,7 +2799,7 @@ pi.registerMessageRenderer("my-extension", (message, options, theme) => {
|
||||
text += "\n" + theme.fg("dim", JSON.stringify(message.details, null, 2));
|
||||
}
|
||||
|
||||
return new Text(text, 0, 0);
|
||||
return new Text(text, outputPad, 0);
|
||||
});
|
||||
```
|
||||
|
||||
|
||||
@@ -375,6 +375,8 @@ Some Anthropic models require adaptive thinking (`thinking.type: "adaptive"` plu
|
||||
|
||||
Some Anthropic-compatible providers emit thinking blocks with empty signatures and still expect them on replay. Set `allowEmptySignature` to `true` only for those providers; real Anthropic rejects empty thinking signatures.
|
||||
|
||||
Built-in Anthropic models enable `supportsStrictTools` in their model metadata. Custom Anthropic-compatible models must set it to `true` when their endpoint accepts strict JSON-schema tool definitions.
|
||||
|
||||
```json
|
||||
{
|
||||
"providers": {
|
||||
@@ -408,6 +410,7 @@ Some Anthropic-compatible providers emit thinking blocks with empty signatures a
|
||||
| `supportsCacheControlOnTools` | Whether the provider accepts Anthropic-style `cache_control` markers on tool definitions. Default: `true`. |
|
||||
| `forceAdaptiveThinking` | Whether to send adaptive thinking (`thinking.type: "adaptive"` plus `output_config.effort`) for this model. Built-in adaptive models set this automatically. Default: `false`. |
|
||||
| `allowEmptySignature` | Whether to replay empty thinking signatures as `signature: ""` instead of converting thinking to text. Default: `false`. |
|
||||
| `supportsStrictTools` | Whether the provider accepts strict JSON-schema tool definitions. Default: `false`; built-in Anthropic models enable it in generated metadata. |
|
||||
|
||||
## OpenAI Compatibility
|
||||
|
||||
@@ -448,7 +451,8 @@ For providers with partial OpenAI compatibility, use the `compat` field.
|
||||
| `cacheControlFormat` | Use Anthropic-style `cache_control` markers on the system prompt, last tool definition, and last user, assistant, or tool-result text content. Currently only `anthropic` is supported. |
|
||||
| `sendSessionAffinityHeaders` | For `openai-completions`, send session-affinity headers from the session id when caching is enabled. Default: `false`. |
|
||||
| `sessionAffinityFormat` | For `openai-completions` and `openai-responses`, the session-affinity header format: `openai` sends `session_id`/`x-client-request-id` (completions also `x-session-affinity`), `openai-nosession` omits the underscore-containing `session_id` header, `openrouter` sends `x-session-id`. Does not affect the `prompt_cache_key` body param. Default: auto-detected. |
|
||||
| `supportsStrictMode` | Include the `strict` field in tool definitions |
|
||||
| `supportsStrictMode` | Whether the provider accepts strict JSON-schema function tool definitions. Defaults depend on the API; built-in OpenAI models carry explicit capability metadata. |
|
||||
| `supportsOpenAIGrammarTools` | Whether OpenAI-compatible APIs emit custom Lark/regex grammar tools. When `false`, grammar-constrained tools fall back to normal function tools. Default: `false`; the built-in model catalog enables it for GPT-5+ models on OpenAI, OpenAI Codex, Azure OpenAI, GitHub Copilot, opencode, and Cloudflare AI Gateway. |
|
||||
| `deferredToolsMode` | Use provider-specific deferred tool serialization. Currently only `"kimi"` is supported for Kimi's OpenAI-compatible Chat Completions format. |
|
||||
| `supportsLongCacheRetention` | Whether the provider accepts long cache retention when cache retention is `long`: `prompt_cache_retention: "24h"` for OpenAI prompt caching, or `cache_control.ttl: "1h"` when `cacheControlFormat` is `anthropic`. Default: `true`. |
|
||||
| `openRouterRouting` | OpenRouter provider routing preferences. This object is sent as-is in the `provider` field of the [OpenRouter API request](https://openrouter.ai/docs/guides/routing/provider-selection). |
|
||||
|
||||
@@ -23,7 +23,7 @@ Common options:
|
||||
- **Responses**: JSON objects with `type: "response"` indicating command success/failure
|
||||
- **Events**: Agent events streamed to stdout as JSON lines
|
||||
|
||||
All commands support an optional `id` field for request/response correlation. If provided, the corresponding response will include the same `id`.
|
||||
All commands support an optional `id` field for request/response correlation. If provided, the corresponding response will include the same `id`. `bash_execution_update` events also include the `id` of their originating `bash` command.
|
||||
|
||||
### Framing
|
||||
|
||||
@@ -455,15 +455,18 @@ Response:
|
||||
|
||||
#### bash
|
||||
|
||||
Execute a shell command and add output to conversation context.
|
||||
Execute a shell command and add output to conversation context. Output streams as `bash_execution_update` events while the command runs; the response contains the final result.
|
||||
|
||||
```json
|
||||
{"type": "bash", "command": "ls -la"}
|
||||
{"id": "req-1", "type": "bash", "command": "ls -la"}
|
||||
```
|
||||
|
||||
Include an `id` to associate streamed `bash_execution_update` events with this command.
|
||||
|
||||
Response:
|
||||
```json
|
||||
{
|
||||
"id": "req-1",
|
||||
"type": "response",
|
||||
"command": "bash",
|
||||
"success": true,
|
||||
@@ -494,7 +497,7 @@ If output was truncated, includes `fullOutputPath`:
|
||||
|
||||
**How bash results reach the LLM:**
|
||||
|
||||
The `bash` command executes immediately and returns a `BashResult`. Internally, a `BashExecutionMessage` is created and stored in the agent's message state. This message does NOT emit an event.
|
||||
The `bash` command executes immediately and returns a `BashResult`. Internally, a `BashExecutionMessage` is created and stored in the agent's message state.
|
||||
|
||||
When the next `prompt` command is sent, all messages (including `BashExecutionMessage`) are transformed before being sent to the LLM. The `BashExecutionMessage` is converted to a `UserMessage` with this format:
|
||||
|
||||
@@ -509,7 +512,6 @@ drwxr-xr-x ...
|
||||
This means:
|
||||
1. Bash output is included in the LLM context on the **next prompt**, not immediately
|
||||
2. Multiple bash commands can be executed before a prompt; all outputs will be included
|
||||
3. No event is emitted for the `BashExecutionMessage` itself
|
||||
|
||||
#### abort_bash
|
||||
|
||||
@@ -829,7 +831,7 @@ Each command has:
|
||||
|
||||
## Events
|
||||
|
||||
Events are streamed to stdout as JSON lines during agent operation. Events do NOT include an `id` field (only responses do).
|
||||
Events are streamed to stdout as JSON lines during agent operation. Events do not generally include an `id` field; `bash_execution_update` includes the `id` of its originating `bash` command when one was provided.
|
||||
|
||||
### Event Types
|
||||
|
||||
@@ -843,6 +845,7 @@ Events are streamed to stdout as JSON lines during agent operation. Events do NO
|
||||
| `message_start` | Message begins |
|
||||
| `message_update` | Streaming update (text/thinking/toolcall deltas) |
|
||||
| `message_end` | Message completes |
|
||||
| `bash_execution_update` | Direct RPC bash command output chunk |
|
||||
| `tool_execution_start` | Tool begins execution |
|
||||
| `tool_execution_update` | Tool execution progress (streaming output) |
|
||||
| `tool_execution_end` | Tool completes |
|
||||
@@ -951,6 +954,20 @@ Example streaming a text response:
|
||||
{"type":"message_update","message":{...},"assistantMessageEvent":{"type":"text_end","contentIndex":0,"content":"Hello world","partial":{...}}}
|
||||
```
|
||||
|
||||
### bash_execution_update
|
||||
|
||||
Emitted once for each output chunk from a direct `bash` command. `id` matches the command's `id`, allowing clients to associate output with the correct command.
|
||||
|
||||
Events stream all output while the command runs, even if the final `bash` response's `output` is truncated.
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "bash_execution_update",
|
||||
"id": "req-1",
|
||||
"delta": "total 48\n"
|
||||
}
|
||||
```
|
||||
|
||||
### tool_execution_start / tool_execution_update / tool_execution_end
|
||||
|
||||
Emitted when a tool begins, streams progress, and completes execution.
|
||||
|
||||
@@ -142,7 +142,7 @@ Set `PI_SKIP_VERSION_CHECK=1` to disable the Pi version update check. Use `--off
|
||||
| `retry.provider.maxRetries` | number | `0` | Provider/SDK retry attempts |
|
||||
| `retry.provider.maxRetryDelayMs` | number | `60000` | Max server-requested delay before failing (60s) |
|
||||
|
||||
When a provider requests a retry delay longer than `retry.provider.maxRetryDelayMs` (e.g., Google's "quota will reset after 5h"), the request fails immediately with an informative error instead of waiting silently. Set to `0` to disable the cap.
|
||||
When a provider requests a retry delay longer than `retry.provider.maxRetryDelayMs`, the request fails immediately with an informative error instead of waiting silently. Set it to `0` to disable the limit.
|
||||
|
||||
Keep `retry.provider.maxRetries` at `0` unless provider-level retries are explicitly needed. Setting it above `0` can make SDK/provider retries handle out-of-usage-limit errors before Pi sees them, which may block the agent until the provider quota resets in some circumstances.
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
* pi --extension examples/extensions/custom-compaction.ts
|
||||
*/
|
||||
|
||||
import { uuidv7 } from "@earendil-works/pi-ai";
|
||||
import { complete } from "@earendil-works/pi-ai/compat";
|
||||
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
||||
import { convertToLlm, serializeConversation } from "@earendil-works/pi-coding-agent";
|
||||
@@ -96,6 +97,8 @@ ${conversationText}
|
||||
env: auth.env,
|
||||
maxTokens: 8192,
|
||||
signal,
|
||||
cacheRetention: "none",
|
||||
sessionId: uuidv7(),
|
||||
},
|
||||
);
|
||||
|
||||
|
||||
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "pi-extension-custom-provider",
|
||||
"version": "0.81.1",
|
||||
"version": "0.82.1",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "pi-extension-custom-provider",
|
||||
"version": "0.81.1",
|
||||
"version": "0.82.1",
|
||||
"dependencies": {
|
||||
"@anthropic-ai/sdk": "^0.52.0"
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "pi-extension-custom-provider-anthropic",
|
||||
"private": true,
|
||||
"version": "0.81.1",
|
||||
"version": "0.82.1",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"clean": "echo 'nothing to clean'",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "pi-extension-custom-provider-gitlab-duo",
|
||||
"private": true,
|
||||
"version": "0.81.1",
|
||||
"version": "0.82.1",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"clean": "echo 'nothing to clean'",
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "pi-extension-gondolin",
|
||||
"version": "0.81.1",
|
||||
"version": "0.82.1",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "pi-extension-gondolin",
|
||||
"version": "0.81.1",
|
||||
"version": "0.82.1",
|
||||
"dependencies": {
|
||||
"@earendil-works/gondolin": "0.12.0"
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "pi-extension-gondolin",
|
||||
"private": true,
|
||||
"version": "0.81.1",
|
||||
"version": "0.82.1",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"clean": "echo 'nothing to clean'",
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
*/
|
||||
|
||||
import type { AgentMessage } from "@earendil-works/pi-agent-core";
|
||||
import { uuidv7 } from "@earendil-works/pi-ai";
|
||||
import { complete, type Message } from "@earendil-works/pi-ai/compat";
|
||||
import type { ExtensionAPI, SessionEntry } from "@earendil-works/pi-coding-agent";
|
||||
import { BorderedLoader, convertToLlm, serializeConversation } from "@earendil-works/pi-coding-agent";
|
||||
@@ -136,7 +137,14 @@ export default function (pi: ExtensionAPI) {
|
||||
const response = await complete(
|
||||
ctx.model!,
|
||||
{ systemPrompt: SYSTEM_PROMPT, messages: [userMessage] },
|
||||
{ apiKey: auth.apiKey, headers: auth.headers, env: auth.env, signal: loader.signal },
|
||||
{
|
||||
apiKey: auth.apiKey,
|
||||
headers: auth.headers,
|
||||
env: auth.env,
|
||||
signal: loader.signal,
|
||||
cacheRetention: "none",
|
||||
sessionId: uuidv7(),
|
||||
},
|
||||
);
|
||||
|
||||
if (response.stopReason === "aborted") {
|
||||
|
||||
@@ -12,7 +12,7 @@ import { Box, Text } from "@earendil-works/pi-tui";
|
||||
|
||||
export default function (pi: ExtensionAPI) {
|
||||
// Register custom renderer for "status-update" messages
|
||||
pi.registerMessageRenderer("status-update", (message, { expanded }, theme) => {
|
||||
pi.registerMessageRenderer("status-update", (message, { expanded, outputPad }, theme) => {
|
||||
const details = message.details as { level: string; timestamp: number } | undefined;
|
||||
const level = details?.level ?? "info";
|
||||
|
||||
@@ -29,7 +29,7 @@ export default function (pi: ExtensionAPI) {
|
||||
}
|
||||
|
||||
// Use Box with customMessageBg for consistent styling
|
||||
const box = new Box(1, 1, (t) => theme.bg("customMessageBg", t));
|
||||
const box = new Box(outputPad, 1, (t) => theme.bg("customMessageBg", t));
|
||||
box.addChild(new Text(text, 0, 0));
|
||||
return box;
|
||||
});
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "pi-extension-sandbox",
|
||||
"version": "1.11.1",
|
||||
"version": "1.12.1",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "pi-extension-sandbox",
|
||||
"version": "1.11.1",
|
||||
"version": "1.12.1",
|
||||
"dependencies": {
|
||||
"@anthropic-ai/sandbox-runtime": "^0.0.26"
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "pi-extension-sandbox",
|
||||
"private": true,
|
||||
"version": "1.11.1",
|
||||
"version": "1.12.1",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"clean": "echo 'nothing to clean'",
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { uuidv7 } from "@earendil-works/pi-ai";
|
||||
import { complete, getModel } from "@earendil-works/pi-ai/compat";
|
||||
import type { ExtensionAPI, ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
|
||||
import { DynamicBorder, getMarkdownTheme } from "@earendil-works/pi-coding-agent";
|
||||
@@ -193,6 +194,8 @@ export default function (pi: ExtensionAPI) {
|
||||
headers: auth.headers,
|
||||
env: auth.env,
|
||||
reasoningEffort: "high",
|
||||
cacheRetention: "none",
|
||||
sessionId: uuidv7(),
|
||||
},
|
||||
);
|
||||
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "pi-extension-with-deps",
|
||||
"version": "0.81.1",
|
||||
"version": "0.82.1",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "pi-extension-with-deps",
|
||||
"version": "0.81.1",
|
||||
"version": "0.82.1",
|
||||
"dependencies": {
|
||||
"ms": "^2.1.3"
|
||||
},
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "pi-extension-with-deps",
|
||||
"private": true,
|
||||
"version": "0.81.1",
|
||||
"version": "0.82.1",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"clean": "echo 'nothing to clean'",
|
||||
|
||||
+18
-18
@@ -1,14 +1,14 @@
|
||||
{
|
||||
"name": "@earendil-works/pi-coding-agent-install",
|
||||
"version": "0.81.1",
|
||||
"version": "0.82.1",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@earendil-works/pi-coding-agent-install",
|
||||
"version": "0.81.1",
|
||||
"version": "0.82.1",
|
||||
"dependencies": {
|
||||
"@earendil-works/pi-coding-agent": "0.81.1"
|
||||
"@earendil-works/pi-coding-agent": "0.82.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=22.19.0"
|
||||
@@ -450,11 +450,11 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@earendil-works/pi-agent-core": {
|
||||
"version": "0.81.1",
|
||||
"resolved": "https://registry.npmjs.org/@earendil-works/pi-agent-core/-/pi-agent-core-0.81.1.tgz",
|
||||
"version": "0.82.1",
|
||||
"resolved": "https://registry.npmjs.org/@earendil-works/pi-agent-core/-/pi-agent-core-0.82.1.tgz",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@earendil-works/pi-ai": "^0.81.1",
|
||||
"@earendil-works/pi-ai": "^0.82.1",
|
||||
"diff": "8.0.4",
|
||||
"ignore": "7.0.5",
|
||||
"typebox": "1.1.38",
|
||||
@@ -465,8 +465,8 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@earendil-works/pi-ai": {
|
||||
"version": "0.81.1",
|
||||
"resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-0.81.1.tgz",
|
||||
"version": "0.82.1",
|
||||
"resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-0.82.1.tgz",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@anthropic-ai/sdk": "0.91.1",
|
||||
@@ -489,13 +489,13 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@earendil-works/pi-coding-agent": {
|
||||
"version": "0.81.1",
|
||||
"resolved": "https://registry.npmjs.org/@earendil-works/pi-coding-agent/-/pi-coding-agent-0.81.1.tgz",
|
||||
"version": "0.82.1",
|
||||
"resolved": "https://registry.npmjs.org/@earendil-works/pi-coding-agent/-/pi-coding-agent-0.82.1.tgz",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@earendil-works/pi-agent-core": "^0.81.1",
|
||||
"@earendil-works/pi-ai": "^0.81.1",
|
||||
"@earendil-works/pi-tui": "^0.81.1",
|
||||
"@earendil-works/pi-agent-core": "^0.82.1",
|
||||
"@earendil-works/pi-ai": "^0.82.1",
|
||||
"@earendil-works/pi-tui": "^0.82.1",
|
||||
"@silvia-odwyer/photon-node": "0.3.4",
|
||||
"chalk": "5.6.2",
|
||||
"cross-spawn": "7.0.6",
|
||||
@@ -523,8 +523,8 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@earendil-works/pi-tui": {
|
||||
"version": "0.81.1",
|
||||
"resolved": "https://registry.npmjs.org/@earendil-works/pi-tui/-/pi-tui-0.81.1.tgz",
|
||||
"version": "0.82.1",
|
||||
"resolved": "https://registry.npmjs.org/@earendil-works/pi-tui/-/pi-tui-0.82.1.tgz",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"get-east-asian-width": "1.6.0",
|
||||
@@ -1603,9 +1603,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/protobufjs": {
|
||||
"version": "7.6.4",
|
||||
"resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.4.tgz",
|
||||
"integrity": "sha512-RJJPTTpvFfHcWLkIa2JFWK4XvtSzS0yEWDmunqHXli1h3JlkbcQZXDZdcWxv+JK3Xsl5/UFDPZ0iGm7DAengYw==",
|
||||
"version": "7.6.5",
|
||||
"resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.5.tgz",
|
||||
"integrity": "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==",
|
||||
"license": "BSD-3-Clause",
|
||||
"dependencies": {
|
||||
"@protobufjs/aspromise": "^1.1.2",
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
{
|
||||
"name": "@earendil-works/pi-coding-agent-install",
|
||||
"version": "0.81.1",
|
||||
"version": "0.82.1",
|
||||
"private": true,
|
||||
"description": "Lockfile root used by the Pi installer and updater.",
|
||||
"dependencies": {
|
||||
"@earendil-works/pi-coding-agent": "0.81.1"
|
||||
"@earendil-works/pi-coding-agent": "0.82.1"
|
||||
},
|
||||
"overrides": {
|
||||
"protobufjs": "7.6.5",
|
||||
"rimraf": "6.1.2",
|
||||
"gaxios": {
|
||||
"rimraf": "6.1.2"
|
||||
|
||||
+15
-15
@@ -1,17 +1,17 @@
|
||||
{
|
||||
"name": "@earendil-works/pi-coding-agent",
|
||||
"version": "0.81.1",
|
||||
"version": "0.82.1",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@earendil-works/pi-coding-agent",
|
||||
"version": "0.81.1",
|
||||
"version": "0.82.1",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@earendil-works/pi-agent-core": "^0.81.1",
|
||||
"@earendil-works/pi-ai": "^0.81.1",
|
||||
"@earendil-works/pi-tui": "^0.81.1",
|
||||
"@earendil-works/pi-agent-core": "^0.82.1",
|
||||
"@earendil-works/pi-ai": "^0.82.1",
|
||||
"@earendil-works/pi-tui": "^0.82.1",
|
||||
"@silvia-odwyer/photon-node": "0.3.4",
|
||||
"chalk": "5.6.2",
|
||||
"cross-spawn": "7.0.6",
|
||||
@@ -474,11 +474,11 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@earendil-works/pi-agent-core": {
|
||||
"version": "0.81.1",
|
||||
"resolved": "https://registry.npmjs.org/@earendil-works/pi-agent-core/-/pi-agent-core-0.81.1.tgz",
|
||||
"version": "0.82.1",
|
||||
"resolved": "https://registry.npmjs.org/@earendil-works/pi-agent-core/-/pi-agent-core-0.82.1.tgz",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@earendil-works/pi-ai": "^0.81.1",
|
||||
"@earendil-works/pi-ai": "^0.82.1",
|
||||
"diff": "8.0.4",
|
||||
"ignore": "7.0.5",
|
||||
"typebox": "1.1.38",
|
||||
@@ -489,8 +489,8 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@earendil-works/pi-ai": {
|
||||
"version": "0.81.1",
|
||||
"resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-0.81.1.tgz",
|
||||
"version": "0.82.1",
|
||||
"resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-0.82.1.tgz",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@anthropic-ai/sdk": "0.91.1",
|
||||
@@ -513,8 +513,8 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@earendil-works/pi-tui": {
|
||||
"version": "0.81.1",
|
||||
"resolved": "https://registry.npmjs.org/@earendil-works/pi-tui/-/pi-tui-0.81.1.tgz",
|
||||
"version": "0.82.1",
|
||||
"resolved": "https://registry.npmjs.org/@earendil-works/pi-tui/-/pi-tui-0.82.1.tgz",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"get-east-asian-width": "1.6.0",
|
||||
@@ -1593,9 +1593,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/protobufjs": {
|
||||
"version": "7.6.4",
|
||||
"resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.4.tgz",
|
||||
"integrity": "sha512-RJJPTTpvFfHcWLkIa2JFWK4XvtSzS0yEWDmunqHXli1h3JlkbcQZXDZdcWxv+JK3Xsl5/UFDPZ0iGm7DAengYw==",
|
||||
"version": "7.6.5",
|
||||
"resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.5.tgz",
|
||||
"integrity": "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==",
|
||||
"license": "BSD-3-Clause",
|
||||
"dependencies": {
|
||||
"@protobufjs/aspromise": "^1.1.2",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@earendil-works/pi-coding-agent",
|
||||
"version": "0.81.1",
|
||||
"version": "0.82.1",
|
||||
"description": "Coding agent CLI with read, bash, edit, write tools and session management",
|
||||
"type": "module",
|
||||
"piConfig": {
|
||||
@@ -39,9 +39,9 @@
|
||||
"prepublishOnly": "npm run clean && npm run build && npm run shrinkwrap"
|
||||
},
|
||||
"dependencies": {
|
||||
"@earendil-works/pi-agent-core": "^0.81.1",
|
||||
"@earendil-works/pi-ai": "^0.81.1",
|
||||
"@earendil-works/pi-tui": "^0.81.1",
|
||||
"@earendil-works/pi-agent-core": "^0.82.1",
|
||||
"@earendil-works/pi-ai": "^0.82.1",
|
||||
"@earendil-works/pi-tui": "^0.82.1",
|
||||
"@silvia-odwyer/photon-node": "0.3.4",
|
||||
"chalk": "5.6.2",
|
||||
"cross-spawn": "7.0.6",
|
||||
@@ -59,6 +59,7 @@
|
||||
"yaml": "2.9.0"
|
||||
},
|
||||
"overrides": {
|
||||
"protobufjs": "7.6.5",
|
||||
"rimraf": "6.1.2",
|
||||
"gaxios": {
|
||||
"rimraf": "6.1.2"
|
||||
|
||||
@@ -333,6 +333,7 @@ ${chalk.bold("Examples:")}
|
||||
${APP_NAME} --export session.jsonl output.html
|
||||
|
||||
${chalk.bold("Environment Variables:")}
|
||||
ANTHROPIC_AUTH_TOKEN - Anthropic bearer auth token
|
||||
ANTHROPIC_API_KEY - Anthropic Claude API key
|
||||
ANTHROPIC_OAUTH_TOKEN - Anthropic OAuth token (alternative to API key)
|
||||
ANT_LING_API_KEY - Ant Ling API key
|
||||
|
||||
@@ -176,7 +176,9 @@ export type AgentSessionEvent =
|
||||
source: "compaction";
|
||||
reason: "manual" | "threshold" | "overflow";
|
||||
}
|
||||
| { type: "summarization_retry_finished" };
|
||||
| { type: "summarization_retry_finished" }
|
||||
| { type: "auto_retry_end"; success: boolean; attempt: number; finalError?: string }
|
||||
| { type: "bash_execution_update"; id?: string; delta: string };
|
||||
|
||||
/** Listener function for agent session events */
|
||||
export type AgentSessionEventListener = (event: AgentSessionEvent) => void;
|
||||
@@ -403,7 +405,7 @@ export class AgentSession {
|
||||
}
|
||||
|
||||
private async _getRequiredRequestAuth(model: Model<any>): Promise<{
|
||||
apiKey: string;
|
||||
apiKey?: string;
|
||||
headers?: Record<string, string>;
|
||||
env?: Record<string, string>;
|
||||
}> {
|
||||
@@ -417,7 +419,7 @@ export class AgentSession {
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (result?.auth.apiKey) {
|
||||
if (result && (result.auth.apiKey || result.auth.headers)) {
|
||||
return {
|
||||
apiKey: result.auth.apiKey,
|
||||
headers: withoutDeletedHeaders(result.auth.headers),
|
||||
@@ -2055,11 +2057,7 @@ export class AgentSession {
|
||||
let headers: Record<string, string> | undefined;
|
||||
let env: Record<string, string> | undefined;
|
||||
if (this.agent.streamFunction === streamSimple) {
|
||||
const authResult = await this._modelRuntime.getAuth(this.model);
|
||||
if (!authResult?.auth.apiKey) return false;
|
||||
apiKey = authResult.auth.apiKey;
|
||||
headers = withoutDeletedHeaders(authResult.auth.headers);
|
||||
env = authResult.env;
|
||||
({ apiKey, headers, env } = await this._getRequiredRequestAuth(this.model));
|
||||
} else {
|
||||
({ apiKey, headers, env } = await this._getSummarizationRequestAuth(this.model));
|
||||
}
|
||||
@@ -2760,12 +2758,13 @@ export class AgentSession {
|
||||
* @param command The bash command to execute
|
||||
* @param onChunk Optional streaming callback for output
|
||||
* @param options.excludeFromContext If true, command output won't be sent to LLM (!! prefix)
|
||||
* @param options.id Optional identifier included in bash execution update events
|
||||
* @param options.operations Custom BashOperations for remote execution
|
||||
*/
|
||||
async executeBash(
|
||||
command: string,
|
||||
onChunk?: (chunk: string) => void,
|
||||
options?: { excludeFromContext?: boolean; operations?: BashOperations },
|
||||
options?: { excludeFromContext?: boolean; id?: string; operations?: BashOperations },
|
||||
): Promise<BashResult> {
|
||||
this._bashAbortController = new AbortController();
|
||||
|
||||
@@ -2780,7 +2779,10 @@ export class AgentSession {
|
||||
this.sessionManager.getCwd(),
|
||||
options?.operations ?? createLocalBashOperations({ shellPath }),
|
||||
{
|
||||
onChunk,
|
||||
onChunk: (delta) => {
|
||||
onChunk?.(delta);
|
||||
this._emit({ type: "bash_execution_update", id: options?.id, delta });
|
||||
},
|
||||
signal: this._bashAbortController.signal,
|
||||
},
|
||||
);
|
||||
|
||||
@@ -19,6 +19,7 @@ import type {
|
||||
Api,
|
||||
AssistantMessageEvent,
|
||||
AssistantMessageEventStream,
|
||||
ConstrainedSamplingConfig,
|
||||
Context,
|
||||
ImageContent,
|
||||
Model,
|
||||
@@ -452,6 +453,8 @@ export interface ToolDefinition<TParams extends TSchema = TSchema, TDetails = un
|
||||
promptGuidelines?: string[];
|
||||
/** Parameter schema (TypeBox) */
|
||||
parameters: TParams;
|
||||
/** Optional provider-side constrained sampling request for this tool. Set false to explicitly disable it, equivalent to leaving it undefined. */
|
||||
constrainedSampling?: false | ConstrainedSamplingConfig;
|
||||
/** Controls whether ToolExecutionComponent renders the standard colored shell or the tool renders its own framing. */
|
||||
renderShell?: "default" | "self";
|
||||
|
||||
@@ -1126,6 +1129,8 @@ export interface SessionBeforeTreeResult {
|
||||
|
||||
export interface MessageRenderOptions {
|
||||
expanded: boolean;
|
||||
/** Horizontal padding configured by the outputPad setting. */
|
||||
outputPad: number;
|
||||
}
|
||||
|
||||
export interface EntryRenderOptions {
|
||||
|
||||
@@ -97,6 +97,7 @@ const OpenAICompletionsCompatSchema = Type.Object({
|
||||
cacheControlFormat: Type.Optional(Type.Literal("anthropic")),
|
||||
openRouterRouting: Type.Optional(OpenRouterRoutingSchema),
|
||||
vercelGatewayRouting: Type.Optional(VercelGatewayRoutingSchema),
|
||||
supportsOpenAIGrammarTools: Type.Optional(Type.Boolean()),
|
||||
supportsStrictMode: Type.Optional(Type.Boolean()),
|
||||
sendSessionAffinityHeaders: Type.Optional(Type.Boolean()),
|
||||
deferredToolsMode: Type.Optional(Type.Literal("kimi")),
|
||||
@@ -112,6 +113,8 @@ const OpenAIResponsesCompatSchema = Type.Object({
|
||||
Type.Union([Type.Literal("openai"), Type.Literal("openai-nosession"), Type.Literal("openrouter")]),
|
||||
),
|
||||
supportsLongCacheRetention: Type.Optional(Type.Boolean()),
|
||||
supportsStrictMode: Type.Optional(Type.Boolean()),
|
||||
supportsOpenAIGrammarTools: Type.Optional(Type.Boolean()),
|
||||
supportsToolSearch: Type.Optional(Type.Boolean()),
|
||||
});
|
||||
|
||||
@@ -120,7 +123,10 @@ const AnthropicMessagesCompatSchema = Type.Object({
|
||||
supportsLongCacheRetention: Type.Optional(Type.Boolean()),
|
||||
sendSessionAffinityHeaders: Type.Optional(Type.Boolean()),
|
||||
supportsCacheControlOnTools: Type.Optional(Type.Boolean()),
|
||||
supportsTemperature: Type.Optional(Type.Boolean()),
|
||||
forceAdaptiveThinking: Type.Optional(Type.Boolean()),
|
||||
allowEmptySignature: Type.Optional(Type.Boolean()),
|
||||
supportsStrictTools: Type.Optional(Type.Boolean()),
|
||||
supportsToolReferences: Type.Optional(Type.Boolean()),
|
||||
});
|
||||
|
||||
|
||||
@@ -25,8 +25,8 @@ export class ModelRegistry {
|
||||
}
|
||||
|
||||
/** Reload models.json asynchronously. Await before making synchronous registry reads. */
|
||||
refresh(): Promise<void> {
|
||||
return this.runtime.reloadConfig();
|
||||
async refresh(): Promise<void> {
|
||||
await this.runtime.refresh();
|
||||
}
|
||||
|
||||
getError(): string | undefined {
|
||||
|
||||
@@ -260,6 +260,7 @@ export function parseModelPattern(
|
||||
*/
|
||||
export interface ModelScopeDiagnostic {
|
||||
type: "warning";
|
||||
code: "no-match" | "invalid-thinking-level";
|
||||
message: string;
|
||||
pattern: string;
|
||||
}
|
||||
@@ -293,6 +294,14 @@ export async function resolveModelScopeWithDiagnostics(
|
||||
}
|
||||
}
|
||||
|
||||
const exactMatch = findExactModelReferenceMatch(globPattern, availableModels);
|
||||
if (exactMatch) {
|
||||
if (!scopedModels.find((sm) => modelsAreEqual(sm.model, exactMatch))) {
|
||||
scopedModels.push({ model: exactMatch, thinkingLevel });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Match against "provider/modelId" format OR just model ID
|
||||
// This allows "*sonnet*" to match without requiring "anthropic/*sonnet*"
|
||||
const matchingModels = availableModels.filter((m) => {
|
||||
@@ -301,7 +310,12 @@ export async function resolveModelScopeWithDiagnostics(
|
||||
});
|
||||
|
||||
if (matchingModels.length === 0) {
|
||||
diagnostics.push({ type: "warning", message: `No models match pattern "${pattern}"`, pattern });
|
||||
diagnostics.push({
|
||||
type: "warning",
|
||||
code: "no-match",
|
||||
message: `No models match pattern "${pattern}"`,
|
||||
pattern,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -316,11 +330,16 @@ export async function resolveModelScopeWithDiagnostics(
|
||||
const { model, thinkingLevel, warning } = parseModelPattern(pattern, availableModels);
|
||||
|
||||
if (warning) {
|
||||
diagnostics.push({ type: "warning", message: warning, pattern });
|
||||
diagnostics.push({ type: "warning", code: "invalid-thinking-level", message: warning, pattern });
|
||||
}
|
||||
|
||||
if (!model) {
|
||||
diagnostics.push({ type: "warning", message: `No models match pattern "${pattern}"`, pattern });
|
||||
diagnostics.push({
|
||||
type: "warning",
|
||||
code: "no-match",
|
||||
message: `No models match pattern "${pattern}"`,
|
||||
pattern,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
@@ -140,18 +140,13 @@ export class ModelRuntime implements Models {
|
||||
(modelsPath
|
||||
? new FileModelsStore(options.modelsStorePath ?? join(dirname(modelsPath), "models-store.json"))
|
||||
: new InMemoryCodingAgentModelsStore());
|
||||
const builtinModelDataGeneratedAt = builtinProviderCatalog.getBuiltinModelDataGeneratedAt();
|
||||
const providers = builtinProviderCatalog
|
||||
.builtinProviders()
|
||||
.map((provider) =>
|
||||
provider.id === "radius"
|
||||
? provider
|
||||
: withRemoteCatalog(
|
||||
provider,
|
||||
options.catalogBaseUrl,
|
||||
builtinProviderCatalog.getBuiltinModelDataUrl(
|
||||
provider.id as builtinProviderCatalog.BuiltinProvider,
|
||||
),
|
||||
),
|
||||
: withRemoteCatalog(provider, options.catalogBaseUrl, builtinModelDataGeneratedAt),
|
||||
);
|
||||
const runtime = new ModelRuntime(
|
||||
credentials,
|
||||
@@ -518,14 +513,10 @@ export class ModelRuntime implements Models {
|
||||
await this.refresh({ allowNetwork: this.modelNetworkEnabled });
|
||||
}
|
||||
|
||||
async reloadConfig(): Promise<void> {
|
||||
async refresh(options: ModelsRefreshOptions = {}): Promise<ModelsRefreshResult> {
|
||||
this.config = await ModelConfig.load(this.modelsPath);
|
||||
this.configureRadiusProviders();
|
||||
this.rebuildProviders();
|
||||
await this.refresh({ allowNetwork: this.modelNetworkEnabled });
|
||||
}
|
||||
|
||||
async refresh(options: ModelsRefreshOptions = {}): Promise<ModelsRefreshResult> {
|
||||
const refreshOptions = {
|
||||
...options,
|
||||
allowNetwork: options.allowNetwork ?? this.modelNetworkEnabled,
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import { stat } from "node:fs/promises";
|
||||
import type { Api, Model, ModelsStoreEntry, Provider } from "@earendil-works/pi-ai";
|
||||
import { VERSION } from "../config.ts";
|
||||
import { getPiUserAgent } from "../utils/pi-user-agent.ts";
|
||||
@@ -32,13 +31,10 @@ function parseCatalog(providerId: string, value: unknown): Model<Api>[] {
|
||||
|
||||
function remoteModels(
|
||||
entry: ModelsStoreEntry | undefined,
|
||||
localLastModified: number | undefined,
|
||||
localGeneratedAt: number | undefined,
|
||||
): readonly Model<Api>[] {
|
||||
if (!entry) return [];
|
||||
if (
|
||||
localLastModified !== undefined &&
|
||||
(entry.lastModified === undefined || entry.lastModified <= localLastModified)
|
||||
) {
|
||||
if (localGeneratedAt !== undefined && (entry.lastModified === undefined || entry.lastModified <= localGeneratedAt)) {
|
||||
return [];
|
||||
}
|
||||
return entry.models;
|
||||
@@ -48,7 +44,7 @@ function remoteModels(
|
||||
export function withRemoteCatalog(
|
||||
provider: Provider,
|
||||
catalogBaseUrl: string = DEFAULT_CATALOG_BASE_URL,
|
||||
localCatalogUrl?: URL,
|
||||
localGeneratedAt?: number,
|
||||
): Provider {
|
||||
let dynamicModels: readonly Model<Api>[] = [];
|
||||
let inflightRefresh: Promise<void> | undefined;
|
||||
@@ -59,16 +55,8 @@ export function withRemoteCatalog(
|
||||
refreshModels: (context) => {
|
||||
inflightRefresh ??= (async () => {
|
||||
try {
|
||||
const localLastModified = localCatalogUrl
|
||||
? await stat(localCatalogUrl).then(
|
||||
(value) => value.mtimeMs,
|
||||
() => undefined,
|
||||
)
|
||||
: undefined;
|
||||
const stored = await context.store.read();
|
||||
dynamicModels = remoteModels(stored, localLastModified).filter(
|
||||
(model) => model.provider === provider.id,
|
||||
);
|
||||
dynamicModels = remoteModels(stored, localGeneratedAt).filter((model) => model.provider === provider.id);
|
||||
if (!context.allowNetwork || context.signal?.aborted) return;
|
||||
if (
|
||||
!context.force &&
|
||||
@@ -79,21 +67,38 @@ export function withRemoteCatalog(
|
||||
return;
|
||||
}
|
||||
|
||||
// Only revalidate when a cached body backs the validator, so a 304 can never
|
||||
// leave the overlay empty.
|
||||
const validator = stored?.models.length ? stored.etag : undefined;
|
||||
const url = new URL(`/api/models/providers/${encodeURIComponent(provider.id)}`, catalogBaseUrl);
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
accept: "application/json",
|
||||
"User-Agent": getPiUserAgent(VERSION),
|
||||
...(validator ? { "if-none-match": validator } : {}),
|
||||
},
|
||||
signal: context.signal,
|
||||
});
|
||||
if (context.signal?.aborted) return;
|
||||
const checkedAt = Date.now();
|
||||
// Unchanged: dynamicModels already holds the stored overlay, so only the
|
||||
// freshness window moves.
|
||||
if (response.status === 304 && stored) {
|
||||
await context.store.write({ ...stored, checkedAt });
|
||||
return;
|
||||
}
|
||||
if (response.status === 404 || response.status === 501) {
|
||||
await context.store.write({ ...(stored ?? { models: [] }), checkedAt, lastModified: 0 });
|
||||
await context.store.write({
|
||||
...(stored ?? { models: [] }),
|
||||
checkedAt,
|
||||
lastModified: 0,
|
||||
etag: undefined,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (!response.ok) {
|
||||
// Transient failure: the cached body and its validator stay valid, so keep the
|
||||
// etag and let the next refresh revalidate instead of downloading the catalog.
|
||||
await context.store.write({ ...(stored ?? { models: [] }), checkedAt });
|
||||
throw new Error(`Model catalog request failed for ${provider.id}: ${response.status}`);
|
||||
}
|
||||
@@ -104,8 +109,9 @@ export function withRemoteCatalog(
|
||||
models: refreshed,
|
||||
checkedAt,
|
||||
lastModified: Number.isNaN(lastModified) ? 0 : lastModified,
|
||||
etag: response.headers.get("etag") ?? undefined,
|
||||
};
|
||||
dynamicModels = remoteModels(entry, localLastModified);
|
||||
dynamicModels = remoteModels(entry, localGeneratedAt);
|
||||
await context.store.write(entry);
|
||||
} finally {
|
||||
inflightRefresh = undefined;
|
||||
|
||||
@@ -70,6 +70,9 @@ function loadContextFileFromDir(dir: string): { path: string; content: string }
|
||||
const filePath = join(dir, filename);
|
||||
if (existsSync(filePath)) {
|
||||
try {
|
||||
if (!statSync(filePath).isFile()) {
|
||||
continue;
|
||||
}
|
||||
return {
|
||||
path: filePath,
|
||||
content: readFileSync(filePath, "utf-8"),
|
||||
|
||||
@@ -11,6 +11,7 @@ export function wrapToolDefinition<TDetails = unknown>(
|
||||
label: definition.label,
|
||||
description: definition.description,
|
||||
parameters: definition.parameters,
|
||||
constrainedSampling: definition.constrainedSampling,
|
||||
prepareArguments: definition.prepareArguments,
|
||||
executionMode: definition.executionMode,
|
||||
execute: (toolCallId, params, signal, onUpdate, ctx?: ExtensionContext) =>
|
||||
@@ -38,6 +39,7 @@ export function createToolDefinitionFromAgentTool(tool: AgentTool<any>): ToolDef
|
||||
label: tool.label,
|
||||
description: tool.description,
|
||||
parameters: tool.parameters as any,
|
||||
constrainedSampling: tool.constrainedSampling,
|
||||
prepareArguments: tool.prepareArguments,
|
||||
executionMode: tool.executionMode,
|
||||
execute: async (toolCallId, params, signal, onUpdate) => tool.execute(toolCallId, params, signal, onUpdate),
|
||||
|
||||
@@ -12,8 +12,6 @@ import { LlamaClient, type LlamaModelInfo, llamaInferenceUrl, normalizeLlamaServ
|
||||
|
||||
export const LLAMA_PROVIDER_ID = "llama.cpp";
|
||||
export const DEFAULT_LLAMA_SERVER_URL = "http://127.0.0.1:8080";
|
||||
const DEFAULT_MAX_TOKENS = 16384;
|
||||
|
||||
function credentialServerUrl(credential: ApiKeyCredential | undefined): string | undefined {
|
||||
const value = credential?.env?.LLAMA_BASE_URL;
|
||||
return typeof value === "string" && value.trim() ? normalizeLlamaServerUrl(value) : undefined;
|
||||
@@ -40,7 +38,7 @@ function toPiModel(model: LlamaModelInfo, serverUrl: string): Model<"openai-comp
|
||||
input: model.architecture?.input_modalities?.includes("image") ? ["text", "image"] : ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow,
|
||||
maxTokens: Math.min(DEFAULT_MAX_TOKENS, contextWindow),
|
||||
maxTokens: contextWindow,
|
||||
compat: {
|
||||
supportsStore: false,
|
||||
supportsDeveloperRole: false,
|
||||
@@ -113,11 +111,20 @@ export function createLlamaProvider(): LlamaProviderController {
|
||||
},
|
||||
getModels: () => models,
|
||||
refreshModels: async (context: RefreshModelsContext): Promise<void> => {
|
||||
const stored = await context.store.read();
|
||||
if (stored) {
|
||||
models = stored.models.filter(
|
||||
(model): model is Model<"openai-completions"> =>
|
||||
model.provider === LLAMA_PROVIDER_ID && model.api === "openai-completions",
|
||||
);
|
||||
}
|
||||
|
||||
if (!context.allowNetwork || context.signal?.aborted || context.credential?.type !== "api_key") return;
|
||||
const serverUrl = credentialServerUrl(context.credential);
|
||||
if (!serverUrl) return;
|
||||
const catalog = await new LlamaClient(serverUrl, context.credential.key).list({ signal: context.signal });
|
||||
setCatalog(catalog, serverUrl);
|
||||
if (!context.signal?.aborted) await context.store.write({ models, checkedAt: Date.now() });
|
||||
},
|
||||
stream: (model, context, options) => stream(model, context, options as ProviderStreamOptions | undefined),
|
||||
streamSimple: (model, context, options) => streamSimple(model, context, options),
|
||||
|
||||
@@ -16,16 +16,19 @@ export class CustomMessageComponent extends Container {
|
||||
private customComponent?: Component;
|
||||
private markdownTheme: MarkdownTheme;
|
||||
private _expanded = false;
|
||||
private outputPad: number;
|
||||
|
||||
constructor(
|
||||
message: CustomMessage<unknown>,
|
||||
customRenderer?: MessageRenderer,
|
||||
markdownTheme: MarkdownTheme = getMarkdownTheme(),
|
||||
outputPad = 1,
|
||||
) {
|
||||
super();
|
||||
this.message = message;
|
||||
this.customRenderer = customRenderer;
|
||||
this.markdownTheme = markdownTheme;
|
||||
this.outputPad = outputPad;
|
||||
|
||||
this.addChild(new Spacer(1));
|
||||
|
||||
@@ -42,6 +45,13 @@ export class CustomMessageComponent extends Container {
|
||||
}
|
||||
}
|
||||
|
||||
setOutputPad(outputPad: number): void {
|
||||
if (this.outputPad !== outputPad) {
|
||||
this.outputPad = outputPad;
|
||||
this.rebuild();
|
||||
}
|
||||
}
|
||||
|
||||
override invalidate(): void {
|
||||
super.invalidate();
|
||||
this.rebuild();
|
||||
@@ -58,7 +68,11 @@ export class CustomMessageComponent extends Container {
|
||||
// Try custom renderer first - it handles its own styling
|
||||
if (this.customRenderer) {
|
||||
try {
|
||||
const component = this.customRenderer(this.message, { expanded: this._expanded }, theme);
|
||||
const component = this.customRenderer(
|
||||
this.message,
|
||||
{ expanded: this._expanded, outputPad: this.outputPad },
|
||||
theme,
|
||||
);
|
||||
if (component) {
|
||||
// Custom renderer provides its own styled component
|
||||
this.customComponent = component;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user