diff --git a/.github/workflows/approve-contributor.yml b/.github/workflows/approve-contributor.yml index 794efad6..d2d8a58d 100644 --- a/.github/workflows/approve-contributor.yml +++ b/.github/workflows/approve-contributor.yml @@ -13,13 +13,13 @@ jobs: issues: write steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.repository.default_branch }} - name: Update contributor approval id: update - uses: actions/github-script@v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const fs = require('fs'); @@ -146,7 +146,7 @@ jobs: - name: Comment on issue if: steps.update.outputs.status == 'added' || steps.update.outputs.status == 'updated' || steps.update.outputs.status == 'already' - uses: actions/github-script@v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const issueAuthor = context.payload.issue.user.login; diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index e77ab650..93e781d8 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -33,7 +33,7 @@ jobs: SOURCE_REF: ${{ github.event.inputs.source_ref || github.event.inputs.tag || github.ref_name }} steps: - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ env.SOURCE_REF }} persist-credentials: false @@ -44,7 +44,7 @@ jobs: bun-version: 1.3.14 - name: Setup Node.js - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: '22' registry-url: 'https://registry.npmjs.org' @@ -201,13 +201,13 @@ jobs: SOURCE_REF: ${{ github.event.inputs.source_ref || github.event.inputs.tag || github.ref_name }} steps: - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ env.SOURCE_REF }} persist-credentials: false - name: Setup Node.js - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: '22' registry-url: 'https://registry.npmjs.org' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a40a894c..ce0cfef9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,10 +15,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 22 cache: npm diff --git a/.github/workflows/issue-analysis.yml b/.github/workflows/issue-analysis.yml index c4d0a1fb..679fb3f4 100644 --- a/.github/workflows/issue-analysis.yml +++ b/.github/workflows/issue-analysis.yml @@ -61,7 +61,7 @@ jobs: steps: - name: Verify sender permission id: verify - uses: actions/github-script@v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: ORG_READ_TOKEN: ${{ secrets.EARENDIL_ORG_READ_TOKEN }} with: @@ -254,19 +254,19 @@ jobs: steps: - name: Create high-entropy working directory name id: workdir - uses: actions/github-script@v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const crypto = require('crypto'); core.setOutput('name', `pi-ci-${crypto.randomBytes(16).toString('hex')}`); - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: path: ${{ steps.workdir.outputs.name }} - name: Setup Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 22 cache: npm @@ -319,7 +319,7 @@ jobs: - name: Write auth.json id: write_auth - uses: actions/github-script@v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: PI_AUTH_JSON: ${{ secrets.PI_AUTH_JSON }} with: @@ -341,7 +341,7 @@ jobs: } - name: Run pi /is - uses: actions/github-script@v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: PI_CODING_AGENT_DIR: ${{ runner.temp }}/pi-agent GH_TOKEN: ${{ github.token }} @@ -404,7 +404,7 @@ jobs: - name: Persist refreshed auth.json if: always() && steps.write_auth.outcome == 'success' - uses: actions/github-script@v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_TOKEN: ${{ secrets.PI_AUTH_UPDATE_TOKEN }} PI_CODING_AGENT_DIR: ${{ runner.temp }}/pi-agent @@ -457,7 +457,7 @@ jobs: - name: Export session files id: export_session_files if: always() - uses: actions/github-script@v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: PI_CODING_AGENT_DIR: ${{ runner.temp }}/pi-agent WORKDIR: ${{ steps.workdir.outputs.name }} @@ -509,7 +509,7 @@ jobs: - name: Upload session gist id: gist if: always() && steps.export_session_files.outcome == 'success' - uses: actions/github-script@v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: PI_GIST_TOKEN: ${{ secrets.PI_GIST_TOKEN }} with: @@ -540,7 +540,7 @@ jobs: - name: Comment with session import instructions if: always() && steps.gist.outcome == 'success' - uses: actions/github-script@v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GIST_URL: ${{ steps.gist.outputs.url }} GIST_ID: ${{ steps.gist.outputs.id }} @@ -619,7 +619,7 @@ jobs: - name: Remove trigger label if: always() - uses: actions/github-script@v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | try { diff --git a/.github/workflows/issue-gate.yml b/.github/workflows/issue-gate.yml index 6fcbfa21..02d4000d 100644 --- a/.github/workflows/issue-gate.yml +++ b/.github/workflows/issue-gate.yml @@ -12,7 +12,7 @@ jobs: issues: write steps: - name: Check issue author - uses: actions/github-script@v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const APPROVED_FILE = '.github/APPROVED_CONTRIBUTORS'; diff --git a/.github/workflows/issue-triage-labels.yml b/.github/workflows/issue-triage-labels.yml index 0e6204d5..1a44253f 100644 --- a/.github/workflows/issue-triage-labels.yml +++ b/.github/workflows/issue-triage-labels.yml @@ -11,7 +11,7 @@ jobs: issues: write steps: - name: Update triage labels - uses: actions/github-script@v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const UNTRIAGED_LABEL = 'untriaged'; diff --git a/.github/workflows/npm-audit.yml b/.github/workflows/npm-audit.yml index 021b81ca..bca7b0e5 100644 --- a/.github/workflows/npm-audit.yml +++ b/.github/workflows/npm-audit.yml @@ -13,10 +13,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 22 cache: npm diff --git a/.github/workflows/pr-gate.yml b/.github/workflows/pr-gate.yml index 669fd95b..48e0745f 100644 --- a/.github/workflows/pr-gate.yml +++ b/.github/workflows/pr-gate.yml @@ -13,7 +13,7 @@ jobs: pull-requests: write steps: - name: Check if contributor is approved - uses: actions/github-script@v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const APPROVED_FILE = '.github/APPROVED_CONTRIBUTORS'; diff --git a/.github/workflows/publish-model-catalog.yml b/.github/workflows/publish-model-catalog.yml index 14571104..729defbe 100644 --- a/.github/workflows/publish-model-catalog.yml +++ b/.github/workflows/publish-model-catalog.yml @@ -42,13 +42,13 @@ jobs: SOURCE_REF: ${{ github.event.workflow_run.head_sha || inputs.source_ref || github.sha }} steps: - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ env.SOURCE_REF }} persist-credentials: false - name: Setup Node.js - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: '22' cache: npm @@ -87,13 +87,13 @@ jobs: R2_ENDPOINT: https://67c0d357268b0fca6e0b465bb9d01b84.r2.cloudflarestorage.com steps: - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ env.SOURCE_REF }} persist-credentials: false - name: Setup Node.js - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: '22' diff --git a/.github/workflows/remove-inprogress-on-close.yml b/.github/workflows/remove-inprogress-on-close.yml index e94b10e2..4904067e 100644 --- a/.github/workflows/remove-inprogress-on-close.yml +++ b/.github/workflows/remove-inprogress-on-close.yml @@ -11,7 +11,7 @@ jobs: issues: write steps: - name: Remove inprogress label - uses: actions/github-script@v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const labelName = 'inprogress'; diff --git a/biome.json b/biome.json index b5451402..ef3cec42 100644 --- a/biome.json +++ b/biome.json @@ -27,6 +27,8 @@ "includes": [ "packages/*/src/**/*.ts", "packages/*/test/**/*.ts", + "packages/storage/*/src/**/*.ts", + "packages/storage/*/test/**/*.ts", "packages/coding-agent/examples/**/*.ts", "!**/node_modules/**/*", "!**/test-sessions.ts", diff --git a/package-lock.json b/package-lock.json index 1190eb57..55a541c0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.0.3", "workspaces": [ "packages/*", + "packages/storage/*", "packages/coding-agent/examples/extensions/with-deps", "packages/coding-agent/examples/extensions/custom-provider-anthropic", "packages/coding-agent/examples/extensions/custom-provider-gitlab-duo", @@ -793,48 +794,18 @@ "resolved": "packages/coding-agent", "link": true }, - "node_modules/@earendil-works/pi-orchestrator": { - "resolved": "packages/orchestrator", + "node_modules/@earendil-works/pi-server": { + "resolved": "packages/server", + "link": true + }, + "node_modules/@earendil-works/pi-storage-sqlite-node": { + "resolved": "packages/storage/sqlite-node", "link": true }, "node_modules/@earendil-works/pi-tui": { "resolved": "packages/tui", "link": true }, - "node_modules/@emnapi/core": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", - "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/wasi-threads": "1.2.1", - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/runtime": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", - "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/wasi-threads": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", - "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, "node_modules/@esbuild/aix-ppc64": { "version": "0.28.1", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", @@ -1403,9 +1374,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1422,9 +1390,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1441,9 +1406,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1460,9 +1422,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1479,9 +1438,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1796,9 +1752,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1816,9 +1769,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1836,9 +1786,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1856,9 +1803,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1876,9 +1820,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1896,9 +1837,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3607,9 +3545,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -3631,9 +3566,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -3655,9 +3587,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -3679,9 +3608,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -5115,10 +5041,10 @@ }, "packages/agent": { "name": "@earendil-works/pi-agent-core", - "version": "0.80.10", + "version": "0.81.0", "license": "MIT", "dependencies": { - "@earendil-works/pi-ai": "^0.80.10", + "@earendil-works/pi-ai": "^0.81.0", "ignore": "7.0.5", "typebox": "1.1.38", "yaml": "2.9.0" @@ -5467,7 +5393,7 @@ }, "packages/ai": { "name": "@earendil-works/pi-ai", - "version": "0.80.10", + "version": "0.81.0", "license": "MIT", "dependencies": { "@anthropic-ai/sdk": "0.91.1", @@ -5773,12 +5699,12 @@ }, "packages/coding-agent": { "name": "@earendil-works/pi-coding-agent", - "version": "0.80.10", + "version": "0.81.0", "license": "MIT", "dependencies": { - "@earendil-works/pi-agent-core": "^0.80.10", - "@earendil-works/pi-ai": "^0.80.10", - "@earendil-works/pi-tui": "^0.80.10", + "@earendil-works/pi-agent-core": "^0.81.0", + "@earendil-works/pi-ai": "^0.81.0", + "@earendil-works/pi-tui": "^0.81.0", "@silvia-odwyer/photon-node": "0.3.4", "chalk": "5.6.2", "cross-spawn": "7.0.6", @@ -5819,32 +5745,32 @@ }, "packages/coding-agent/examples/extensions/custom-provider-anthropic": { "name": "pi-extension-custom-provider-anthropic", - "version": "0.80.10", + "version": "0.81.0", "dependencies": { "@anthropic-ai/sdk": "0.52.0" } }, "packages/coding-agent/examples/extensions/custom-provider-gitlab-duo": { "name": "pi-extension-custom-provider-gitlab-duo", - "version": "0.80.10" + "version": "0.81.0" }, "packages/coding-agent/examples/extensions/gondolin": { "name": "pi-extension-gondolin", - "version": "0.80.10", + "version": "0.81.0", "dependencies": { "@earendil-works/gondolin": "0.12.0" } }, "packages/coding-agent/examples/extensions/sandbox": { "name": "pi-extension-sandbox", - "version": "1.10.10", + "version": "1.11.0", "dependencies": { "@anthropic-ai/sandbox-runtime": "0.0.26" } }, "packages/coding-agent/examples/extensions/with-deps": { "name": "pi-extension-with-deps", - "version": "0.80.10", + "version": "0.81.0", "dependencies": { "ms": "2.1.3" }, @@ -6138,12 +6064,15 @@ } } }, - "packages/orchestrator": { - "name": "@earendil-works/pi-orchestrator", - "version": "0.80.10", + "packages/server": { + "name": "@earendil-works/pi-server", + "version": "0.81.0", "license": "MIT", "dependencies": { - "@earendil-works/pi-coding-agent": "^0.80.10" + "@earendil-works/pi-coding-agent": "^0.81.0" + }, + "bin": { + "server": "dist/cli.js" }, "devDependencies": { "shx": "0.4.0" @@ -6152,9 +6081,21 @@ "node": ">=22.19.0" } }, + "packages/storage/sqlite-node": { + "name": "@earendil-works/pi-storage-sqlite-node", + "version": "0.81.0", + "license": "MIT", + "dependencies": { + "@earendil-works/pi-agent-core": "^0.81.0", + "@earendil-works/pi-ai": "^0.81.0" + }, + "engines": { + "node": ">=22.19.0" + } + }, "packages/tui": { "name": "@earendil-works/pi-tui", - "version": "0.80.10", + "version": "0.81.0", "license": "MIT", "dependencies": { "get-east-asian-width": "1.6.0", diff --git a/package.json b/package.json index fc3dd5db..4f710af9 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,7 @@ "type": "module", "workspaces": [ "packages/*", + "packages/storage/*", "packages/coding-agent/examples/extensions/with-deps", "packages/coding-agent/examples/extensions/custom-provider-anthropic", "packages/coding-agent/examples/extensions/custom-provider-gitlab-duo", @@ -12,8 +13,8 @@ ], "scripts": { "clean": "npm run clean --workspaces", - "build": "cd packages/tui && npm run build && cd ../ai && npm run build && cd ../agent && npm run build && cd ../coding-agent && npm run build && cd ../orchestrator && npm run build", - "build:offline": "cd packages/tui && npm run build && cd ../ai && npm run build:offline && cd ../agent && npm run build && cd ../coding-agent && npm run build && cd ../orchestrator && npm run build", + "build": "cd packages/tui && npm run build && cd ../ai && npm run build && cd ../agent && npm run build && cd ../storage/sqlite-node && npm run build && cd ../../coding-agent && npm run build && cd ../server && npm run build", + "build:offline": "cd packages/tui && npm run build && cd ../ai && npm run build:offline && cd ../agent && npm run build && cd ../storage/sqlite-node && npm run build && cd ../../coding-agent && npm run build && cd ../server && npm run build", "check": "biome check --write --error-on-warnings . && npm run check:pinned-deps && npm run check:ts-imports && npm run check:shrinkwrap && npm run check:install-lock:coding-agent && tsgo --noEmit && npm run check:browser-smoke", "check:browser-smoke": "node scripts/check-browser-smoke.mjs", "check:pinned-deps": "node scripts/check-pinned-deps.mjs", diff --git a/packages/agent/CHANGELOG.md b/packages/agent/CHANGELOG.md index 57bd9ad8..0e578ff6 100644 --- a/packages/agent/CHANGELOG.md +++ b/packages/agent/CHANGELOG.md @@ -2,9 +2,12 @@ ## [Unreleased] +## [0.81.0] - 2026-07-21 + ### Breaking Changes -- Moved the `uuidv7` export to `@earendil-works/pi-ai`. +- Changed `SessionStorage` to use `getPathToRootOrCompaction()`, require session name and statistics methods, support cursor-based entry reads, and store retained compaction tails as self-contained checkpoints ([#6594](https://github.com/earendil-works/pi/pull/6594) by [@cristinaponcela](https://github.com/cristinaponcela)). +- Moved the `uuidv7` export to `@earendil-works/pi-ai` ([#6834](https://github.com/earendil-works/pi/pull/6834) by [@xl0](https://github.com/xl0)). - Replaced the optional `Agent` `streamFn` fallback with a required `streamFunction` and made low-level loop stream functions required, preventing `@earendil-works/pi-ai/compat` and all built-in providers from entering selective-provider bundles ([#6851](https://github.com/earendil-works/pi/issues/6851)). ### Added diff --git a/packages/agent/README.md b/packages/agent/README.md index 3c935f44..543f4c8f 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -8,6 +8,10 @@ Stateful agent with tool execution and event streaming. Built on `@earendil-work npm install @earendil-works/pi-agent-core ``` +### SQLite session backends + +The SQLite session backend and the `node:sqlite` adapter live in a separate package, `@earendil-works/pi-storage-sqlite-node`, so the core package does not pull in runtime builtins or native SQLite dependencies by default. The backend accepts a runtime-specific SQLite factory, allowing other storage backends to ship as their own packages in the future. + ## Quick Start ```typescript diff --git a/packages/agent/package.json b/packages/agent/package.json index 7334e3a6..81384d19 100644 --- a/packages/agent/package.json +++ b/packages/agent/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-agent-core", - "version": "0.80.10", + "version": "0.81.0", "description": "General-purpose agent with transport abstraction, state management, and attachment support", "type": "module", "main": "./dist/index.js", @@ -26,10 +26,10 @@ "test": "vitest --run", "test:harness": "vitest --run --config vitest.harness.config.ts", "coverage:harness": "vitest --run --config vitest.harness.config.ts --coverage", - "prepublishOnly": "npm run clean && npm run build" + "prepublishOnly": "npm run build" }, "dependencies": { - "@earendil-works/pi-ai": "^0.80.10", + "@earendil-works/pi-ai": "^0.81.0", "ignore": "7.0.5", "typebox": "1.1.38", "yaml": "2.9.0" diff --git a/packages/agent/src/harness/agent-harness.ts b/packages/agent/src/harness/agent-harness.ts index 2273a9ed..1d361e92 100644 --- a/packages/agent/src/harness/agent-harness.ts +++ b/packages/agent/src/harness/agent-harness.ts @@ -752,6 +752,7 @@ export class AgentHarness< result.details, provided !== undefined, result.usage, + result.retainedTail, ); const entry = await this.session.getEntry(entryId); if (entry?.type === "compaction") { diff --git a/packages/agent/src/harness/compaction/compaction.ts b/packages/agent/src/harness/compaction/compaction.ts index 8ff0794b..6a5e8257 100644 --- a/packages/agent/src/harness/compaction/compaction.ts +++ b/packages/agent/src/harness/compaction/compaction.ts @@ -102,12 +102,14 @@ function getMessageFromEntryForCompaction(entry: SessionTreeEntry): AgentMessage export interface CompactionResult { /** Summary text that replaces compacted history in future context. */ summary: string; - /** Entry id where retained history starts. */ - firstKeptEntryId: string; + /** Entry id where retained history starts. Optional during Pi 2.0 transition. */ + firstKeptEntryId?: string; /** Estimated context tokens before compaction. */ tokensBefore: number; /** Usage from the LLM call(s) that generated this summary, if available. */ usage?: Usage; + /** Retained recent messages stored directly on the compaction entry. Optional during Pi 2.0 transition. */ + retainedTail?: AgentMessage[]; /** Optional implementation-specific details stored with the compaction entry. */ details?: T; } @@ -608,6 +610,8 @@ export interface CompactionPreparation { messagesToSummarize: AgentMessage[]; /** Prefix messages summarized separately when compaction splits a turn. */ turnPrefixMessages: AgentMessage[]; + /** Recent messages retained after compaction and stored on the compaction entry. */ + retainedTail: AgentMessage[]; /** Whether compaction splits a turn. */ isSplitTurn: boolean; /** Estimated context tokens before compaction. */ @@ -642,7 +646,9 @@ export function prepareCompaction( if (prevCompactionIndex >= 0) { const prevCompaction = pathEntries[prevCompactionIndex] as CompactionEntry; previousSummary = prevCompaction.summary; - const firstKeptEntryIndex = pathEntries.findIndex((entry) => entry.id === prevCompaction.firstKeptEntryId); + const firstKeptEntryIndex = prevCompaction.firstKeptEntryId + ? pathEntries.findIndex((entry) => entry.id === prevCompaction.firstKeptEntryId) + : -1; boundaryStart = firstKeptEntryIndex >= 0 ? firstKeptEntryIndex : prevCompactionIndex + 1; } const boundaryEnd = pathEntries.length; @@ -669,6 +675,11 @@ export function prepareCompaction( if (msg) turnPrefixMessages.push(msg); } } + const retainedTail: AgentMessage[] = []; + for (let i = cutPoint.firstKeptEntryIndex; i < boundaryEnd; i++) { + const msg = getMessageFromEntryForCompaction(pathEntries[i]); + if (msg) retainedTail.push(msg); + } const fileOps = extractFileOperations(messagesToSummarize, pathEntries, prevCompactionIndex); if (cutPoint.isSplitTurn) { for (const msg of turnPrefixMessages) { @@ -680,6 +691,7 @@ export function prepareCompaction( firstKeptEntryId, messagesToSummarize, turnPrefixMessages, + retainedTail, isSplitTurn: cutPoint.isSplitTurn, tokensBefore, previousSummary, @@ -720,6 +732,7 @@ export async function compact( firstKeptEntryId, messagesToSummarize, turnPrefixMessages, + retainedTail, isSplitTurn, tokensBefore, previousSummary, @@ -795,6 +808,7 @@ export async function compact( firstKeptEntryId, tokensBefore, usage: summaryUsage, + retainedTail, details: { readFiles, modifiedFiles } as CompactionDetails, }); } diff --git a/packages/agent/src/harness/session/jsonl-storage.ts b/packages/agent/src/harness/session/jsonl-storage.ts index d57398d5..fea97fbd 100644 --- a/packages/agent/src/harness/session/jsonl-storage.ts +++ b/packages/agent/src/harness/session/jsonl-storage.ts @@ -1,5 +1,12 @@ import { uuidv7 } from "@earendil-works/pi-ai"; -import type { FileSystem, JsonlSessionMetadata, LeafEntry, SessionStorage, SessionTreeEntry } from "../types.ts"; +import type { + FileSystem, + JsonlSessionMetadata, + LeafEntry, + SessionEntryCursorOptions, + SessionStorage, + SessionTreeEntry, +} from "../types.ts"; import { SessionError, toError } from "../types.ts"; import { getFileSystemResultOrThrow } from "./repo-utils.ts"; @@ -293,13 +300,66 @@ export class JsonlSessionStorage implements SessionStorage return this.labelsById.get(id); } - async getPathToRoot(leafId: string | null): Promise { + async getSessionName(): Promise { + const entries = await this.findEntries("session_info"); + return entries[entries.length - 1]?.name?.trim() || undefined; + } + + async getSessionStats() { + let messageCount = 0; + let cachedTokens = 0; + let uncachedTokens = 0; + let totalTokens = 0; + let costTotal = 0; + for (const entry of this.entries) { + if (entry.type === "message") { + messageCount += 1; + } + const usage = + entry.type === "message" + ? entry.message.role === "assistant" + ? entry.message.usage + : undefined + : entry.type === "compaction" || entry.type === "branch_summary" + ? entry.usage + : undefined; + if ( + !usage || + typeof usage.input !== "number" || + typeof usage.output !== "number" || + typeof usage.cacheRead !== "number" || + typeof usage.cacheWrite !== "number" || + typeof usage.cost?.total !== "number" + ) { + continue; + } + cachedTokens += usage.cacheRead; + uncachedTokens += usage.input + usage.cacheWrite; + totalTokens += usage.input + usage.output + usage.cacheRead + usage.cacheWrite; + costTotal += usage.cost.total; + } + return { + messageCount, + cachedTokens, + uncachedTokens, + totalTokens, + costTotal, + }; + } + + async getPathToRootOrCompaction(leafId: string | null): Promise { if (leafId === null) return []; const path: SessionTreeEntry[] = []; + let stopAtEntryId: string | null = null; let current = this.byId.get(leafId); if (!current) throw new SessionError("not_found", `Entry ${leafId} not found`); while (current) { path.unshift(current); + if (stopAtEntryId !== null && current.id === stopAtEntryId) break; + if (current.type === "compaction") { + if (current.retainedTail) break; + stopAtEntryId = current.firstKeptEntryId ?? null; + } if (!current.parentId) break; const parent = this.byId.get(current.parentId); if (!parent) throw new SessionError("invalid_session", `Entry ${current.parentId} not found`); @@ -308,7 +368,9 @@ export class JsonlSessionStorage implements SessionStorage return path; } - async getEntries(): Promise { - return [...this.entries]; + async getEntries(options?: SessionEntryCursorOptions): Promise { + const start = options?.afterEntrySeq ?? 0; + const end = options?.limit === undefined ? undefined : start + options.limit; + return this.entries.slice(start, end); } } diff --git a/packages/agent/src/harness/session/memory-storage.ts b/packages/agent/src/harness/session/memory-storage.ts index 68ec7623..1bc24f04 100644 --- a/packages/agent/src/harness/session/memory-storage.ts +++ b/packages/agent/src/harness/session/memory-storage.ts @@ -1,6 +1,7 @@ import { uuidv7 } from "@earendil-works/pi-ai"; import { type LeafEntry, + type SessionEntryCursorOptions, SessionError, type SessionMetadata, type SessionStorage, @@ -112,13 +113,66 @@ export class InMemorySessionStorage { + async getSessionName(): Promise { + const entries = await this.findEntries("session_info"); + return entries[entries.length - 1]?.name?.trim() || undefined; + } + + async getSessionStats() { + let messageCount = 0; + let cachedTokens = 0; + let uncachedTokens = 0; + let totalTokens = 0; + let costTotal = 0; + for (const entry of this.entries) { + if (entry.type === "message") { + messageCount += 1; + } + const usage = + entry.type === "message" + ? entry.message.role === "assistant" + ? entry.message.usage + : undefined + : entry.type === "compaction" || entry.type === "branch_summary" + ? entry.usage + : undefined; + if ( + !usage || + typeof usage.input !== "number" || + typeof usage.output !== "number" || + typeof usage.cacheRead !== "number" || + typeof usage.cacheWrite !== "number" || + typeof usage.cost?.total !== "number" + ) { + continue; + } + cachedTokens += usage.cacheRead; + uncachedTokens += usage.input + usage.cacheWrite; + totalTokens += usage.input + usage.output + usage.cacheRead + usage.cacheWrite; + costTotal += usage.cost.total; + } + return { + messageCount, + cachedTokens, + uncachedTokens, + totalTokens, + costTotal, + }; + } + + async getPathToRootOrCompaction(leafId: string | null): Promise { if (leafId === null) return []; const path: SessionTreeEntry[] = []; + let stopAtEntryId: string | null = null; let current = this.byId.get(leafId); if (!current) throw new SessionError("not_found", `Entry ${leafId} not found`); while (current) { path.unshift(current); + if (stopAtEntryId !== null && current.id === stopAtEntryId) break; + if (current.type === "compaction") { + if (current.retainedTail) break; + stopAtEntryId = current.firstKeptEntryId ?? null; + } if (!current.parentId) break; const parent = this.byId.get(current.parentId); if (!parent) throw new SessionError("invalid_session", `Entry ${current.parentId} not found`); @@ -127,7 +181,9 @@ export class InMemorySessionStorage { - return [...this.entries]; + async getEntries(options?: SessionEntryCursorOptions): Promise { + const start = options?.afterEntrySeq ?? 0; + const end = options?.limit === undefined ? undefined : start + options.limit; + return this.entries.slice(start, end); } } diff --git a/packages/agent/src/harness/session/repo-utils.ts b/packages/agent/src/harness/session/repo-utils.ts index dd2e0f9a..fd6ee4f2 100644 --- a/packages/agent/src/harness/session/repo-utils.ts +++ b/packages/agent/src/harness/session/repo-utils.ts @@ -47,5 +47,5 @@ export async function getEntriesToFork( } effectiveLeafId = target.parentId; } - return storage.getPathToRoot(effectiveLeafId); + return storage.getPathToRootOrCompaction(effectiveLeafId); } diff --git a/packages/agent/src/harness/session/session.ts b/packages/agent/src/harness/session/session.ts index bff41fdd..13007bc6 100644 --- a/packages/agent/src/harness/session/session.ts +++ b/packages/agent/src/harness/session/session.ts @@ -11,8 +11,10 @@ import type { MessageEntry, ModelChangeEntry, SessionContext, + SessionEntryCursorOptions, SessionInfoEntry, SessionMetadata, + SessionStats, SessionStorage, SessionTreeEntry, ThinkingLevelChangeEntry, @@ -67,11 +69,19 @@ export function defaultContextEntryTransform(pathEntries: readonly SessionTreeEn const entries: SessionTreeEntry[] = [compaction]; const compactionIdx = pathEntries.findIndex((entry) => entry.type === "compaction" && entry.id === compaction.id); - let foundFirstKept = false; - for (let i = 0; i < compactionIdx; i++) { - const entry = pathEntries[i]!; - if (entry.id === compaction.firstKeptEntryId) foundFirstKept = true; - if (foundFirstKept) entries.push(entry); + if (compaction.retainedTail) { + for (let i = compactionIdx + 1; i < pathEntries.length; i++) { + entries.push(pathEntries[i]!); + } + return entries; + } + if (compaction.firstKeptEntryId) { + let foundFirstKept = false; + for (let i = 0; i < compactionIdx; i++) { + const entry = pathEntries[i]!; + if (entry.id === compaction.firstKeptEntryId) foundFirstKept = true; + if (foundFirstKept) entries.push(entry); + } } for (let i = compactionIdx + 1; i < pathEntries.length; i++) { entries.push(pathEntries[i]!); @@ -111,7 +121,10 @@ export function sessionEntryToContextMessages( ]; } if (entry.type === "compaction") { - return [createCompactionSummaryMessage(entry.summary, entry.tokensBefore, entry.timestamp)]; + return [ + createCompactionSummaryMessage(entry.summary, entry.tokensBefore, entry.timestamp), + ...(entry.retainedTail ?? []), + ]; } if (entry.type === "branch_summary" && entry.summary) { return [createBranchSummaryMessage(entry.summary, entry.fromId, entry.timestamp)]; @@ -159,13 +172,13 @@ export class Session { return this.storage.getEntry(id); } - getEntries(): Promise { - return this.storage.getEntries(); + getEntries(options?: SessionEntryCursorOptions): Promise { + return this.storage.getEntries(options); } async getBranch(fromId?: string): Promise { const leafId = fromId ?? (await this.storage.getLeafId()); - return this.storage.getPathToRoot(leafId); + return this.storage.getPathToRootOrCompaction(leafId); } async buildContextEntries(options: SessionContextBuildOptions = {}): Promise { @@ -190,9 +203,12 @@ export class Session { return this.storage.getLabel(id); } + getSessionStats(): Promise { + return this.storage.getSessionStats(); + } + async getSessionName(): Promise { - const entries = await this.storage.findEntries("session_info"); - return entries[entries.length - 1]?.name?.trim() || undefined; + return this.storage.getSessionName(); } private async appendTypedEntry(entry: TEntry): Promise { @@ -243,11 +259,12 @@ export class Session { async appendCompaction( summary: string, - firstKeptEntryId: string, + firstKeptEntryId: string | undefined, tokensBefore: number, details?: T, fromHook?: boolean, usage?: Usage, + retainedTail?: AgentMessage[], ): Promise { return this.appendTypedEntry({ type: "compaction", @@ -257,6 +274,7 @@ export class Session { summary, firstKeptEntryId, tokensBefore, + retainedTail, details, usage, fromHook, diff --git a/packages/agent/src/harness/types.ts b/packages/agent/src/harness/types.ts index 58d13451..da7e7ba2 100644 --- a/packages/agent/src/harness/types.ts +++ b/packages/agent/src/harness/types.ts @@ -371,8 +371,9 @@ export interface ActiveToolsChangeEntry extends SessionTreeEntryBase { export interface CompactionEntry extends SessionTreeEntryBase { type: "compaction"; summary: string; - firstKeptEntryId: string; + firstKeptEntryId?: string; tokensBefore: number; + retainedTail?: AgentMessage[]; details?: T; usage?: Usage; fromHook?: boolean; @@ -437,6 +438,14 @@ export interface SessionContext { activeToolNames: string[] | null; } +export interface SessionStats { + messageCount: number; + cachedTokens: number; + uncachedTokens: number; + totalTokens: number; + costTotal: number; +} + export interface SessionMetadata { id: string; createdAt: string; @@ -449,6 +458,11 @@ export interface JsonlSessionMetadata extends SessionMetadata { metadata?: Record; } +export interface SessionEntryCursorOptions { + afterEntrySeq?: number; + limit?: number; +} + export interface SessionStorage { getMetadata(): Promise; getLeafId(): Promise; @@ -461,8 +475,10 @@ export interface SessionStorage>>; getLabel(id: string): Promise; - getPathToRoot(leafId: string | null): Promise; - getEntries(): Promise; + getSessionName(): Promise; + getSessionStats(): Promise; + getPathToRootOrCompaction(leafId: string | null): Promise; + getEntries(options?: SessionEntryCursorOptions): Promise; } export type { Session } from "./session/session.ts"; @@ -779,10 +795,11 @@ export interface AbortResult { export interface CompactResult { summary: string; - firstKeptEntryId: string; + firstKeptEntryId?: string; tokensBefore: number; /** Usage from the LLM call(s) that generated this summary, if available. */ usage?: Usage; + retainedTail?: AgentMessage[]; details?: unknown; } @@ -802,6 +819,7 @@ export interface CompactionPreparation { firstKeptEntryId: string; messagesToSummarize: AgentMessage[]; turnPrefixMessages: AgentMessage[]; + retainedTail: AgentMessage[]; isSplitTurn: boolean; tokensBefore: number; previousSummary?: string; diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index b5ec7807..b98023b0 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -1,4 +1,5 @@ // Core Agent +export { uuidv7 } from "@earendil-works/pi-ai"; export * from "./agent.ts"; // Loop functions export * from "./agent-loop.ts"; diff --git a/packages/agent/test/harness/compaction.test.ts b/packages/agent/test/harness/compaction.test.ts index c8160307..e55be45a 100644 --- a/packages/agent/test/harness/compaction.test.ts +++ b/packages/agent/test/harness/compaction.test.ts @@ -91,6 +91,7 @@ function createCompactionEntry( summary: string, firstKeptEntryId: string, parentId: string | null = null, + retainedTail?: AgentMessage[], ): CompactionEntry { return { type: "compaction", @@ -100,6 +101,7 @@ function createCompactionEntry( summary, firstKeptEntryId, tokensBefore: 1234, + retainedTail, }; } @@ -343,12 +345,38 @@ describe("harness compaction", () => { const a1 = createMessageEntry(createAssistantMessage("a"), u1.id); const u2 = createMessageEntry(createUserMessage("2"), a1.id); const a2 = createMessageEntry(createAssistantMessage("b"), u2.id); - const compaction = createCompactionEntry("Summary of 1,a,2,b", u2.id, a2.id); + const compaction = createCompactionEntry("Summary of 1,a,2,b", u2.id, a2.id, [ + createUserMessage("2"), + createAssistantMessage("b"), + ]); const u3 = createMessageEntry(createUserMessage("3"), compaction.id); const a3 = createMessageEntry(createAssistantMessage("c"), u3.id); const loaded = buildSessionContext([u1, a1, u2, a2, compaction, u3, a3]); expect(loaded.messages).toHaveLength(5); expect(loaded.messages[0]?.role).toBe("compactionSummary"); + expect(loaded.messages.map((message) => message.role)).toEqual([ + "compactionSummary", + "user", + "assistant", + "user", + "assistant", + ]); + }); + + it("falls back to firstKeptEntryId when a compaction has no retained tail", () => { + const u1 = createMessageEntry(createUserMessage("1")); + const a1 = createMessageEntry(createAssistantMessage("a"), u1.id); + const u2 = createMessageEntry(createUserMessage("2"), a1.id); + const a2 = createMessageEntry(createAssistantMessage("b"), u2.id); + const compaction = createCompactionEntry("Summary of 1,a,2,b", u2.id, a2.id); + const u3 = createMessageEntry(createUserMessage("3"), compaction.id); + const loaded = buildSessionContext([u1, a1, u2, a2, compaction, u3]); + expect(loaded.messages.map((message) => message.role)).toEqual([ + "compactionSummary", + "user", + "assistant", + "user", + ]); }); it("tracks model and thinking level changes in built context", () => { @@ -374,6 +402,7 @@ describe("harness compaction", () => { expect(preparation).toBeDefined(); expect(preparation?.previousSummary).toBe("First summary"); expect(preparation?.firstKeptEntryId).toBeTruthy(); + expect(preparation?.retainedTail.length).toBeGreaterThan(0); expect(preparation?.tokensBefore).toBe(estimateContextTokens(buildSessionContext(pathEntries).messages).tokens); }); @@ -566,6 +595,7 @@ describe("harness compaction", () => { firstKeptEntryId: "entry-keep", messagesToSummarize: messages, turnPrefixMessages: messages, + retainedTail: messages, isSplitTurn: true, tokensBefore: 600000, fileOps: { read: new Set(), written: new Set(), edited: new Set() }, @@ -583,6 +613,7 @@ describe("harness compaction", () => { firstKeptEntryId: "entry-keep", messagesToSummarize: messages, turnPrefixMessages: [], + retainedTail: messages, isSplitTurn: false, tokensBefore: 100, fileOps: { read: new Set(), written: new Set(), edited: new Set() }, @@ -619,6 +650,7 @@ describe("harness compaction", () => { turnPrefixMessages: messages, isSplitTurn: true, tokensBefore: 100, + retainedTail: messages, fileOps: { read: new Set(), written: new Set(), edited: new Set() }, settings: { enabled: true, reserveTokens: 2000, keepRecentTokens: 20 }, }; @@ -642,6 +674,7 @@ describe("harness compaction", () => { firstKeptEntryId: "entry-keep", messagesToSummarize: [], turnPrefixMessages: messages, + retainedTail: messages, isSplitTurn: true, tokensBefore: 100, fileOps: { read: new Set(), written: new Set(), edited: new Set() }, @@ -659,6 +692,7 @@ describe("harness compaction", () => { firstKeptEntryId: "entry-keep", messagesToSummarize: [], turnPrefixMessages: messages, + retainedTail: messages, isSplitTurn: true, tokensBefore: 100, fileOps: { read: new Set(), written: new Set(), edited: new Set() }, @@ -697,6 +731,7 @@ describe("harness compaction", () => { expect(result.summary.length).toBeGreaterThan(0); expect(result.firstKeptEntryId).toBeTruthy(); expect(result.usage?.totalTokens).toBeGreaterThan(0); + expect(result.retainedTail?.length).toBeGreaterThan(0); expect(result.details).toBeDefined(); }); }); diff --git a/packages/agent/test/harness/session.test.ts b/packages/agent/test/harness/session.test.ts index 26af52e2..70adbb38 100644 --- a/packages/agent/test/harness/session.test.ts +++ b/packages/agent/test/harness/session.test.ts @@ -68,11 +68,20 @@ async function runSessionSuite( await session.appendMessage(createAssistantMessage("two")); const user2 = await session.appendMessage(createUserMessage("three")); await session.appendMessage(createAssistantMessage("four")); - await session.appendCompaction("summary", user2, 1234); + await session.appendCompaction("summary", user2, 1234, undefined, undefined, undefined, [ + createUserMessage("three"), + createAssistantMessage("four"), + ]); await session.appendMessage(createUserMessage("five")); const context = await session.buildContext(); expect(context.messages[0]?.role).toBe("compactionSummary"); expect(context.messages).toHaveLength(4); + expect(context.messages.map((message) => message.role)).toEqual([ + "compactionSummary", + "user", + "assistant", + "user", + ]); }); it("supports moving with branch summary entries in context", async () => { diff --git a/packages/agent/test/harness/sqlite-migrations.test.ts b/packages/agent/test/harness/sqlite-migrations.test.ts new file mode 100644 index 00000000..0cf74938 --- /dev/null +++ b/packages/agent/test/harness/sqlite-migrations.test.ts @@ -0,0 +1,441 @@ +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { + applyMigrations, + createNodeSqliteFactory, + type SqliteDatabase, + type SqliteDatabaseFactory, + type SqliteRunResult, + type SqliteSessionMetadata, + SqliteSessionRepo, + SqliteSessionStorage, + type SqliteStatement, +} from "../../../storage/sqlite-node/src/index.ts"; +import { NodeExecutionEnv } from "../../src/harness/env/nodejs.ts"; +import { createAssistantMessage, createUserMessage } from "./session-test-utils.ts"; + +function createTempDir(): string { + return mkdtempSync(join(tmpdir(), "pi-agent-sqlite-")); +} + +class ThrowingStatement implements SqliteStatement { + private readonly onRun: () => Promise; + + constructor(onRun: () => Promise) { + this.onRun = onRun; + } + + async run(..._params: unknown[]): Promise { + return this.onRun(); + } + + async get(..._params: unknown[]): Promise { + return undefined; + } + + async all(..._params: unknown[]): Promise { + return []; + } +} + +class CountingDatabase implements SqliteDatabase { + closeCount = 0; + private readonly statementFactory: (sql: string) => SqliteStatement; + + constructor(statementFactory: (sql: string) => SqliteStatement) { + this.statementFactory = statementFactory; + } + + async exec(_sql: string): Promise {} + + prepare(sql: string): SqliteStatement { + return this.statementFactory(sql); + } + + async transaction(fn: () => Promise): Promise { + return fn(); + } + + async close(): Promise { + this.closeCount += 1; + } +} + +describe("SQLite migrations", () => { + it("applies file-based migrations and records them", async () => { + const root = createTempDir(); + const databasePath = join(root, "sessions.sqlite"); + const env = new NodeExecutionEnv({ cwd: root }); + const sqlite = createNodeSqliteFactory(); + const repo = new SqliteSessionRepo({ env, sqlite, databasePath }); + await repo.create({ cwd: root, id: "session-1" }); + + const db = await sqlite.open(databasePath); + try { + const rows = await db.prepare("SELECT id FROM migrations ORDER BY id").all<{ id: string }>(); + expect(rows.map((row) => row.id)).toEqual(["001_initial.sql"]); + const tables = await db + .prepare("SELECT name, sql FROM sqlite_master WHERE type = 'table' ORDER BY name") + .all<{ name: string; sql: string | null }>(); + expect(tables.map((row) => row.name)).toEqual( + expect.arrayContaining([ + "migrations", + "sessions", + "session_entries", + "session_sequences", + "branch_entries", + "session_materialized", + "entry_materialized", + ]), + ); + const sessionColumns = await db.prepare("PRAGMA table_info(sessions)").all<{ name: string }>(); + expect(sessionColumns.map((column) => column.name)).toContain("active_leaf_id"); + for (const tableName of [ + "sessions", + "session_sequences", + "branch_entries", + "session_materialized", + "entry_materialized", + ]) { + const table = tables.find((row) => row.name === tableName); + expect(table?.sql).toContain("WITHOUT ROWID"); + } + } finally { + await db.close(); + } + }); + + it("persists session metadata through create, list, open, and fork", async () => { + const root = createTempDir(); + const databasePath = join(root, "sessions.sqlite"); + const env = new NodeExecutionEnv({ cwd: root }); + const repo = new SqliteSessionRepo({ env, sqlite: createNodeSqliteFactory(), databasePath }); + const source = await repo.create({ + cwd: root, + id: "session-1", + metadata: { profile: "reviewer" }, + }); + const sourceMetadata = await source.getMetadata(); + expect(sourceMetadata.metadata).toEqual({ profile: "reviewer" }); + expect((await repo.list({ cwd: root })).map((listed) => listed.metadata)).toEqual([{ profile: "reviewer" }]); + expect((await (await repo.open(sourceMetadata)).getMetadata()).metadata).toEqual({ profile: "reviewer" }); + const fork = await repo.fork(sourceMetadata, { cwd: root, id: "session-2" }); + expect((await fork.getMetadata()).metadata).toEqual({ profile: "reviewer" }); + const overridden = await repo.fork(sourceMetadata, { + cwd: root, + id: "session-3", + metadata: { profile: "writer" }, + }); + expect((await overridden.getMetadata()).metadata).toEqual({ profile: "writer" }); + }); + + it("materializes active leaf id in sessions transactionally", async () => { + const root = createTempDir(); + const databasePath = join(root, "sessions.sqlite"); + const env = new NodeExecutionEnv({ cwd: root }); + const sqlite = createNodeSqliteFactory(); + const repo = new SqliteSessionRepo({ env, sqlite, databasePath }); + const session = await repo.create({ cwd: root, id: "session-1" }); + const rootId = await session.appendMessage(createUserMessage("root")); + const childId = await session.appendMessage(createAssistantMessage("child")); + await session.getStorage().setLeafId(rootId); + + const db = await sqlite.open(databasePath); + try { + const row = await db + .prepare("SELECT active_leaf_id FROM sessions WHERE id = ?") + .get<{ active_leaf_id: string | null }>("session-1"); + expect(row?.active_leaf_id).toBe(rootId); + const latestBranchRow = await db + .prepare( + "SELECT branch_id, entry_id, entry_seq FROM branch_entries WHERE session_id = ? ORDER BY entry_seq DESC LIMIT 1", + ) + .get<{ branch_id: string; entry_id: string; entry_seq: number }>("session-1"); + const latestSessionEntry = await db + .prepare("SELECT id, type FROM session_entries WHERE session_id = ? ORDER BY entry_seq DESC LIMIT 1") + .get<{ id: string; type: string }>("session-1"); + expect(latestSessionEntry?.type).toBe("leaf"); + expect(latestBranchRow?.entry_id).toBe(latestSessionEntry?.id); + } finally { + await db.close(); + } + + const reopened = await repo.open(await session.getMetadata()); + expect(await reopened.getLeafId()).toBe(rootId); + expect(childId).not.toBe(rootId); + }); + + it("materializes a new branch when appending from a parent with an existing child", async () => { + const root = createTempDir(); + const databasePath = join(root, "sessions.sqlite"); + const env = new NodeExecutionEnv({ cwd: root }); + const sqlite = createNodeSqliteFactory(); + const repo = new SqliteSessionRepo({ env, sqlite, databasePath }); + const session = await repo.create({ cwd: root, id: "session-1" }); + const rootId = await session.appendMessage(createUserMessage("root")); + const firstChildId = await session.appendMessage(createAssistantMessage("first child")); + await session.getStorage().setLeafId(rootId); + const secondChildId = await session.appendMessage(createAssistantMessage("second child")); + + const db = await sqlite.open(databasePath); + try { + const branchRows = await db + .prepare( + "SELECT branch_id, entry_id, entry_seq FROM branch_entries WHERE session_id = ? ORDER BY branch_id, entry_seq", + ) + .all<{ branch_id: string; entry_id: string; entry_seq: number }>("session-1"); + const branchIds = [...new Set(branchRows.map((row) => row.branch_id))]; + expect(branchIds).toHaveLength(3); + expect(branchRows.filter((row) => row.entry_id === rootId)).toHaveLength(3); + expect(branchRows.filter((row) => row.entry_id === firstChildId)).toHaveLength(1); + expect(branchRows.filter((row) => row.entry_id === secondChildId)).toHaveLength(1); + } finally { + await db.close(); + } + }); + + it("reopens using branch materialization and session summary state", async () => { + const root = createTempDir(); + const databasePath = join(root, "sessions.sqlite"); + const env = new NodeExecutionEnv({ cwd: root }); + const repo = new SqliteSessionRepo({ env, sqlite: createNodeSqliteFactory(), databasePath }); + const session = await repo.create({ cwd: root, id: "session-1" }); + const rootId = await session.appendMessage(createUserMessage("root")); + await session.appendMessage(createAssistantMessage("first child")); + await session.appendSessionName(" Reopened Session "); + await session.getStorage().setLeafId(rootId); + await session.appendMessage(createAssistantMessage("branched child")); + + const reopened = await repo.open(await session.getMetadata()); + expect(await reopened.getSessionName()).toBe("Reopened Session"); + expect((await reopened.buildContext()).messages.map((message) => message.role)).toEqual(["user", "assistant"]); + expect((await reopened.buildContext()).messages.at(-1)).toMatchObject({ + content: [{ type: "text", text: "branched child" }], + }); + }); + + it("pages entries by entry_seq cursor", async () => { + const root = createTempDir(); + const databasePath = join(root, "sessions.sqlite"); + const env = new NodeExecutionEnv({ cwd: root }); + const repo = new SqliteSessionRepo({ env, sqlite: createNodeSqliteFactory(), databasePath }); + const session = await repo.create({ cwd: root, id: "session-1" }); + await session.appendMessage(createUserMessage("one")); + await session.appendMessage(createAssistantMessage("two")); + await session.appendMessage(createUserMessage("three")); + + expect((await session.getEntries({ limit: 2 })).map((entry) => entry.type)).toEqual(["message", "message"]); + expect((await session.getEntries({ afterEntrySeq: 2, limit: 2 })).map((entry) => entry.type)).toEqual([ + "message", + "message", + ]); + }); + + it("closes the database when create fails after openDatabase succeeds", async () => { + const root = createTempDir(); + const db = new CountingDatabase((sql) => { + if (sql.startsWith("INSERT INTO sessions")) { + return new ThrowingStatement(async () => { + throw new Error("insert failed"); + }); + } + return new ThrowingStatement(async () => ({ changes: 1 })); + }); + const sqlite: SqliteDatabaseFactory = { + open: async () => db, + }; + const env = new NodeExecutionEnv({ cwd: root }); + const repo = new SqliteSessionRepo({ env, sqlite, databasePath: join(root, "sessions.sqlite") }); + + await expect(repo.create({ cwd: root, id: "session-1" })).rejects.toThrow("insert failed"); + expect(db.closeCount).toBe(1); + }); + + it("closes the database when open fails after openDatabase succeeds", async () => { + const root = createTempDir(); + const db = new CountingDatabase((sql) => { + if (sql.includes("FROM sessions WHERE id = ?")) { + return new ThrowingStatement(async () => ({ changes: 0 })); + } + return new ThrowingStatement(async () => ({ changes: 1 })); + }); + const sqlite: SqliteDatabaseFactory = { + open: async () => db, + }; + const env = new NodeExecutionEnv({ cwd: root }); + const repo = new SqliteSessionRepo({ env, sqlite, databasePath: join(root, "sessions.sqlite") }); + const metadata: SqliteSessionMetadata = { + id: "missing", + createdAt: new Date().toISOString(), + cwd: root, + path: join(root, "sessions.sqlite"), + }; + writeFileSync(metadata.path, ""); + + await expect(repo.open(metadata)).rejects.toThrow("Session not found: missing"); + expect(db.closeCount).toBe(1); + }); + + it("closes the source storage after fork reads its entries", async () => { + const root = createTempDir(); + const databasePath = join(root, "sessions.sqlite"); + const env = new NodeExecutionEnv({ cwd: root }); + const repo = new SqliteSessionRepo({ env, sqlite: createNodeSqliteFactory(), databasePath }); + let cleanupCount = 0; + const sourceStorage = { + async getEntries() { + return []; + }, + async getPathToRootOrCompaction() { + return []; + }, + async cleanup() { + cleanupCount += 1; + }, + } as const; + const originalOpen = repo.open.bind(repo); + repo.open = async () => + ({ + getStorage() { + return sourceStorage; + }, + }) as never; + + try { + await repo.fork( + { + id: "session-1", + createdAt: new Date().toISOString(), + cwd: root, + path: databasePath, + }, + { cwd: root, id: "session-2" }, + ); + } finally { + repo.open = originalOpen; + } + + expect(cleanupCount).toBe(1); + }); + + it("restores in-memory state when appendEntry fails after mutating caches", async () => { + const root = createTempDir(); + const databasePath = join(root, "sessions.sqlite"); + const sqlite = createNodeSqliteFactory(); + const db = await sqlite.open(databasePath); + await applyMigrations(db); + const storage = await SqliteSessionStorage.create(db, databasePath, { + cwd: root, + sessionId: "session-1", + }); + const originalPrepare = db.prepare.bind(db); + db.prepare = (sql: string) => { + if (sql.startsWith("UPDATE sessions SET active_leaf_id = ?")) { + return new ThrowingStatement(async () => { + throw new Error("active leaf update failed"); + }); + } + return originalPrepare(sql); + }; + + await expect( + storage.appendEntry({ + type: "message", + id: "root", + parentId: null, + timestamp: new Date().toISOString(), + message: createUserMessage("root"), + }), + ).rejects.toMatchObject({ code: "storage" }); + expect(await storage.getLeafId()).toBeNull(); + expect(await storage.getEntry("root")).toBeUndefined(); + expect(await storage.getEntries()).toEqual([]); + await db.close(); + }); + + it("materializes session summary fields transactionally", async () => { + const root = createTempDir(); + const databasePath = join(root, "sessions.sqlite"); + const env = new NodeExecutionEnv({ cwd: root }); + const sqlite = createNodeSqliteFactory(); + const repo = new SqliteSessionRepo({ env, sqlite, databasePath }); + const session = await repo.create({ cwd: root, id: "session-1" }); + const userId = await session.appendMessage(createUserMessage("one")); + await session.appendThinkingLevelChange("high"); + await session.appendModelChange("anthropic", "claude-sonnet-4-5"); + const assistant = { + ...createAssistantMessage("two"), + provider: "anthropic", + model: "claude-sonnet-4-5", + usage: { + input: 100, + output: 25, + cacheRead: 40, + cacheWrite: 10, + totalTokens: 175, + cost: { input: 0.1, output: 0.2, cacheRead: 0.03, cacheWrite: 0.04, total: 0.37 }, + }, + }; + await session.appendMessage(assistant); + await session.appendCompaction("summary", userId, 200, undefined, false, { + input: 1, + output: 2, + cacheRead: 3, + cacheWrite: 4, + totalTokens: 10, + cost: { input: 0.01, output: 0.02, cacheRead: 0.03, cacheWrite: 0.04, total: 0.1 }, + }); + await session.moveTo(userId, { + summary: "branch summary", + usage: { + input: 5, + output: 6, + cacheRead: 7, + cacheWrite: 8, + totalTokens: 26, + cost: { input: 0.05, output: 0.06, cacheRead: 0.07, cacheWrite: 0.08, total: 0.26 }, + }, + }); + await session.appendSessionName(" My Session "); + await session.appendLabel(userId, "checkpoint"); + + const db = await sqlite.open(databasePath); + try { + const row = await db.prepare("SELECT session_id, payload FROM session_materialized WHERE session_id = ?").get<{ + session_id: string; + payload: string; + }>("session-1"); + expect(row).toBeDefined(); + expect(row?.session_id).toBe("session-1"); + expect(JSON.parse(row?.payload ?? "null")).toMatchObject({ + name: "My Session", + messageCount: 2, + cachedTokens: 50, + uncachedTokens: 128, + totalTokens: 211, + costTotal: 0.73, + currentModel: { provider: "anthropic", modelId: "claude-sonnet-4-5" }, + currentThinkingLevel: "high", + }); + const entryRows = await db + .prepare( + "SELECT session_id, entry_seq, type, payload FROM entry_materialized WHERE session_id = ? ORDER BY entry_seq, type", + ) + .all<{ + session_id: string; + entry_seq: number; + type: string; + payload: string; + }>("session-1"); + expect( + entryRows.some((entryRow) => entryRow.type === "label" && JSON.parse(entryRow.payload).targetId === userId), + ).toBe(true); + expect(entryRows.some((entryRow) => entryRow.type === "thinking")).toBe(false); + expect(entryRows.some((entryRow) => entryRow.type === "model")).toBe(false); + } finally { + await db.close(); + } + }); +}); diff --git a/packages/agent/test/harness/sqlite-node.test.ts b/packages/agent/test/harness/sqlite-node.test.ts new file mode 100644 index 00000000..a94d3785 --- /dev/null +++ b/packages/agent/test/harness/sqlite-node.test.ts @@ -0,0 +1,21 @@ +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { createNodeSqliteFactory } from "../../../storage/sqlite-node/src/index.ts"; +import { createTempDir } from "./session-test-utils.ts"; + +describe("sqlite-node adapter", () => { + it("supports node:sqlite-style named parameters", async () => { + const root = createTempDir(); + const databasePath = join(root, "adapter.sqlite"); + const sqlite = createNodeSqliteFactory(); + const db = await sqlite.open(databasePath); + try { + await db.exec("CREATE TABLE items (id INTEGER PRIMARY KEY, text TEXT NOT NULL)"); + await db.prepare("INSERT INTO items (id, text) VALUES ($id, $text)").run({ $id: 1, $text: "hello" }); + const row = await db.prepare("SELECT text FROM items WHERE id = $id").get<{ text: string }>({ $id: 1 }); + expect(row).toEqual({ text: "hello" }); + } finally { + await db.close(); + } + }); +}); diff --git a/packages/agent/test/harness/storage.test.ts b/packages/agent/test/harness/storage.test.ts index 07f07df2..50f4fc59 100644 --- a/packages/agent/test/harness/storage.test.ts +++ b/packages/agent/test/harness/storage.test.ts @@ -4,7 +4,13 @@ import { describe, expect, it } from "vitest"; import { NodeExecutionEnv } from "../../src/harness/env/nodejs.ts"; import { JsonlSessionStorage, loadJsonlSessionMetadata } from "../../src/harness/session/jsonl-storage.ts"; import { InMemorySessionStorage } from "../../src/harness/session/memory-storage.ts"; -import { type MessageEntry, ok, type SessionMetadata } from "../../src/harness/types.ts"; +import { + type BranchSummaryEntry, + type CompactionEntry, + type MessageEntry, + ok, + type SessionMetadata, +} from "../../src/harness/types.ts"; import { createAssistantMessage, createTempDir, createUserMessage } from "./session-test-utils.ts"; describe("InMemorySessionStorage", () => { @@ -80,7 +86,74 @@ describe("InMemorySessionStorage", () => { expect(await storage.getLabel("entry-1")).toBeUndefined(); }); - it("walks paths to root", async () => { + it("includes summary-entry usage in session stats", async () => { + const assistant: MessageEntry = { + type: "message", + id: "assistant", + parentId: null, + timestamp: "2026-01-01T00:00:00.000Z", + message: { + role: "assistant", + content: [{ type: "text", text: "reply" }], + api: "anthropic-messages", + provider: "anthropic", + model: "claude-sonnet-4-5", + usage: { + input: 10, + output: 20, + cacheRead: 30, + cacheWrite: 40, + totalTokens: 100, + cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 }, + }, + stopReason: "stop", + timestamp: 0, + }, + }; + const compaction: CompactionEntry = { + type: "compaction", + id: "compaction", + parentId: "assistant", + timestamp: "2026-01-01T00:00:01.000Z", + summary: "summary", + firstKeptEntryId: "assistant", + tokensBefore: 1234, + usage: { + input: 1, + output: 2, + cacheRead: 3, + cacheWrite: 4, + totalTokens: 10, + cost: { input: 0.01, output: 0.02, cacheRead: 0.03, cacheWrite: 0.04, total: 0.1 }, + }, + }; + const branchSummary: BranchSummaryEntry = { + type: "branch_summary", + id: "branch-summary", + parentId: "compaction", + timestamp: "2026-01-01T00:00:02.000Z", + fromId: "assistant", + summary: "branch", + usage: { + input: 5, + output: 6, + cacheRead: 7, + cacheWrite: 8, + totalTokens: 26, + cost: { input: 0.05, output: 0.06, cacheRead: 0.07, cacheWrite: 0.08, total: 0.26 }, + }, + }; + const storage = new InMemorySessionStorage({ entries: [assistant, compaction, branchSummary] }); + expect(await storage.getSessionStats()).toEqual({ + messageCount: 1, + cachedTokens: 40, + uncachedTokens: 68, + totalTokens: 136, + costTotal: 1.36, + }); + }); + + it("walks paths to root or retained-tail compaction", async () => { const root: MessageEntry = { type: "message", id: "root", @@ -94,9 +167,29 @@ describe("InMemorySessionStorage", () => { parentId: "root", message: createAssistantMessage("child"), }; - const storage = new InMemorySessionStorage({ entries: [root, child] }); - expect((await storage.getPathToRoot("child")).map((entry) => entry.id)).toEqual(["root", "child"]); - expect(await storage.getPathToRoot(null)).toEqual([]); + const compaction: CompactionEntry = { + type: "compaction", + id: "compaction", + parentId: "child", + timestamp: "2026-01-01T00:00:01.000Z", + summary: "summary", + firstKeptEntryId: "child", + tokensBefore: 1234, + retainedTail: [createAssistantMessage("child")], + }; + const afterCompaction: MessageEntry = { + ...root, + id: "after-compaction", + parentId: "compaction", + message: createUserMessage("after"), + }; + const storage = new InMemorySessionStorage({ entries: [root, child, compaction, afterCompaction] }); + expect((await storage.getPathToRootOrCompaction("child")).map((entry) => entry.id)).toEqual(["root", "child"]); + expect((await storage.getPathToRootOrCompaction("after-compaction")).map((entry) => entry.id)).toEqual([ + "compaction", + "after-compaction", + ]); + expect(await storage.getPathToRootOrCompaction(null)).toEqual([]); }); }); @@ -255,7 +348,7 @@ describe("JsonlSessionStorage", () => { const reloaded = await JsonlSessionStorage.open(env, filePath); expect(await reloaded.getLeafId()).toBe("root"); expect((await reloaded.getEntries()).at(-1)).toMatchObject({ type: "leaf", targetId: "root" }); - expect((await loaded.getPathToRoot("child")).map((entry) => entry.id)).toEqual(["root", "child"]); + expect((await loaded.getPathToRootOrCompaction("child")).map((entry) => entry.id)).toEqual(["root", "child"]); }); it("finds entries by type", async () => { @@ -309,6 +402,76 @@ describe("JsonlSessionStorage", () => { expect(await loaded.getLabel("entry-1")).toBeUndefined(); }); + it("includes summary-entry usage in session stats", async () => { + const dir = createTempDir(); + const env = new NodeExecutionEnv({ cwd: dir }); + const filePath = join(dir, "session.jsonl"); + const storage = await JsonlSessionStorage.create(env, filePath, { cwd: dir, sessionId: "session-1" }); + await storage.appendEntry({ + type: "message", + id: "assistant", + parentId: null, + timestamp: "2026-01-01T00:00:00.000Z", + message: { + role: "assistant", + content: [{ type: "text", text: "reply" }], + api: "anthropic-messages", + provider: "anthropic", + model: "claude-sonnet-4-5", + usage: { + input: 10, + output: 20, + cacheRead: 30, + cacheWrite: 40, + totalTokens: 100, + cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 }, + }, + stopReason: "stop", + timestamp: 0, + }, + }); + await storage.appendEntry({ + type: "compaction", + id: "compaction", + parentId: "assistant", + timestamp: "2026-01-01T00:00:01.000Z", + summary: "summary", + firstKeptEntryId: "assistant", + tokensBefore: 1234, + usage: { + input: 1, + output: 2, + cacheRead: 3, + cacheWrite: 4, + totalTokens: 10, + cost: { input: 0.01, output: 0.02, cacheRead: 0.03, cacheWrite: 0.04, total: 0.1 }, + }, + }); + await storage.appendEntry({ + type: "branch_summary", + id: "branch-summary", + parentId: "compaction", + timestamp: "2026-01-01T00:00:02.000Z", + fromId: "assistant", + summary: "branch", + usage: { + input: 5, + output: 6, + cacheRead: 7, + cacheWrite: 8, + totalTokens: 26, + cost: { input: 0.05, output: 0.06, cacheRead: 0.07, cacheWrite: 0.08, total: 0.26 }, + }, + }); + expect(await storage.getSessionStats()).toEqual({ + messageCount: 1, + cachedTokens: 40, + uncachedTokens: 68, + totalTokens: 136, + costTotal: 1.36, + }); + }); + it("reads session metadata through the line-reading filesystem operation", async () => { const dir = createTempDir(); const filePath = join(dir, "session.jsonl"); diff --git a/packages/agent/vitest.harness.config.ts b/packages/agent/vitest.harness.config.ts index 91c0d471..8045bb1a 100644 --- a/packages/agent/vitest.harness.config.ts +++ b/packages/agent/vitest.harness.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from "vitest/config"; const aiSrcIndex = fileURLToPath(new URL("../ai/src/index.ts", import.meta.url)); const aiSrcCompat = fileURLToPath(new URL("../ai/src/compat.ts", import.meta.url)); +const agentSrcIndex = fileURLToPath(new URL("../agent/src/index.ts", import.meta.url)); export default defineConfig({ test: { @@ -21,6 +22,7 @@ export default defineConfig({ }, resolve: { alias: [ + { find: /^@earendil-works\/pi-agent-core$/, replacement: agentSrcIndex }, { find: /^@earendil-works\/pi-ai$/, replacement: aiSrcIndex }, { find: /^@earendil-works\/pi-ai\/compat$/, replacement: aiSrcCompat }, ], diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index b3a89736..c559aed0 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -2,23 +2,32 @@ ## [Unreleased] +## [0.81.0] - 2026-07-21 + ### Added -- Added `contentText` for extracting joined text from message content. -- Added a shared `uuidv7` utility for time-ordered identifiers. +- Added Qwen Token Plan and Qwen Token Plan China as built-in providers with regional endpoints, API-key authentication, and generated model catalogs ([#6858](https://github.com/earendil-works/pi/pull/6858) by [@QuintinShaw](https://github.com/QuintinShaw)). +- Added `contentText` for extracting joined text from message content ([#6840](https://github.com/earendil-works/pi/pull/6840) by [@xl0](https://github.com/xl0)). +- Added a shared `uuidv7` utility for time-ordered identifiers ([#6834](https://github.com/earendil-works/pi/pull/6834) by [@xl0](https://github.com/xl0)). - Added optional usage metadata to tool result messages ([#6671](https://github.com/earendil-works/pi/pull/6671) by [@davidbrai](https://github.com/davidbrai)). ### Changed +- Changed generated model catalogs to keep TypeScript model shapes separate from ignored JSON model values, reducing generated source churn ([#6765](https://github.com/earendil-works/pi/pull/6765) by [@mitsuhiko](https://github.com/mitsuhiko)). - Changed model generation to validate ignored provider data before compilation; `npm run build` refreshes model data as before, while `npm run build:offline` reuses existing data without network access. ### Fixed -- Fixed sessionless OpenAI Codex WebSocket requests to use UUIDv7 request IDs, enabling models that reject UUIDv4 IDs. +- Fixed stored API-key credentials to apply their provider-scoped `env` values during auth resolution, including Amazon Bedrock profiles ([#6864](https://github.com/earendil-works/pi/pull/6864) by [@cristinaponcela](https://github.com/cristinaponcela)). +- Fixed OpenAI-compatible cross-provider replay to preserve unique tool call IDs when multiple calls share a provider call ID ([#6854](https://github.com/earendil-works/pi/pull/6854) by [@cristinaponcela](https://github.com/cristinaponcela)). +- Fixed Kimi K3 to expose its supported low, high, and max thinking levels, and normalized the `k2p7` alias to the canonical `kimi-for-coding` model. +- Fixed the OpenCode Go provider to support models routed through the OpenAI Responses API. +- Fixed the `pi-ai` executable path to match npm registry metadata, avoiding repeated consumer lockfile changes ([#6812](https://github.com/earendil-works/pi/pull/6812) by [@jmfederico](https://github.com/jmfederico)). +- Fixed sessionless OpenAI Codex WebSocket requests to use UUIDv7 request IDs, enabling models that reject UUIDv4 IDs ([#6834](https://github.com/earendil-works/pi/pull/6834) by [@xl0](https://github.com/xl0)). - Fixed GitHub Copilot long-context pricing tiers in generated model metadata ([#6668](https://github.com/earendil-works/pi/issues/6668)). - Fixed Kimi Coding subscription models to report API-equivalent implied costs when models.dev reports zero pricing. - Fixed OpenAI Responses early stream endings to be classified as retryable provider errors ([#6727](https://github.com/earendil-works/pi/issues/6727)). -- Fixed GPT-5.6 Codex models to default to the 272K context window, avoiding automatic long-context pricing ([#6838](https://github.com/earendil-works/pi/issues/6838)). +- Fixed GPT-5.6 Codex models to default to the 272K context window, avoiding automatic long-context pricing ([#6853](https://github.com/earendil-works/pi/pull/6853) by [@aadishv](https://github.com/aadishv)). ## [0.80.10] - 2026-07-16 diff --git a/packages/ai/package.json b/packages/ai/package.json index ce936c9e..f7627894 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-ai", - "version": "0.80.10", + "version": "0.81.0", "description": "Unified LLM API with automatic model discovery and provider configuration", "type": "module", "main": "./dist/index.js", diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index 66e64e5a..1892c11a 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -1761,6 +1761,8 @@ async function loadModelsDevData(): Promise[]> { if (isKimiK3) { compat.requiresReasoningContentOnAssistantMessages = true; compat.deferredToolsMode = "kimi"; + compat.thinkingFormat = "openai"; + compat.supportsReasoningEffort = true; } models.push({ id: modelId, diff --git a/packages/ai/src/image-models.generated.ts b/packages/ai/src/image-models.generated.ts index 57338661..f89ae02b 100644 --- a/packages/ai/src/image-models.generated.ts +++ b/packages/ai/src/image-models.generated.ts @@ -170,6 +170,51 @@ export const IMAGE_MODELS = { cacheWrite: 0, }, } satisfies ImagesModel<"openrouter-images">, + "krea/krea-2-large": { + id: "krea/krea-2-large", + name: "Krea: Krea 2 Large", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["image"], + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "krea/krea-2-medium": { + id: "krea/krea-2-medium", + name: "Krea: Krea 2 Medium", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["image"], + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "krea/krea-2-medium-turbo": { + id: "krea/krea-2-medium-turbo", + name: "Krea: Krea 2 Medium Turbo", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["image"], + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, "microsoft/mai-image-2.5": { id: "microsoft/mai-image-2.5", name: "Microsoft: MAI-Image-2.5", @@ -290,6 +335,21 @@ export const IMAGE_MODELS = { cacheWrite: 0, }, } satisfies ImagesModel<"openrouter-images">, + "openrouter/auto-beta": { + id: "openrouter/auto-beta", + name: "Auto Router (Beta)", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["text", "image"], + cost: { + input: -1000000, + output: -1000000, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, "recraft/recraft-v3": { id: "recraft/recraft-v3", name: "Recraft: Recraft V3", diff --git a/packages/ai/src/models-store.ts b/packages/ai/src/models-store.ts index 020edb1f..3bb2b57e 100644 --- a/packages/ai/src/models-store.ts +++ b/packages/ai/src/models-store.ts @@ -2,6 +2,8 @@ import type { Api, Model } from "./types.ts"; export interface ModelsStoreEntry { models: readonly Model[]; + /** Unix timestamp from the remote catalog's Last-Modified header. */ + lastModified?: number; /** Unix timestamp of the last completed remote check. */ checkedAt?: number; } diff --git a/packages/ai/src/providers/all.ts b/packages/ai/src/providers/all.ts index fd7bea58..549880e3 100644 --- a/packages/ai/src/providers/all.ts +++ b/packages/ai/src/providers/all.ts @@ -67,6 +67,11 @@ export function getBuiltinProviders(): BuiltinProvider[] { return Object.keys(MODELS) as BuiltinProvider[]; } +/** URL of a generated provider catalog, used to compare its mtime with remote catalogs during development. */ +export function getBuiltinModelDataUrl(provider: BuiltinProvider): URL { + return new URL(`./data/${provider}.json`, import.meta.url); +} + export function getBuiltinModels( provider: TProvider, ): Model>[] { diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index f12db68e..f3670a31 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -2,14 +2,41 @@ ## [Unreleased] +## [0.81.0] - 2026-07-21 + +### New Features + +- **Local llama.cpp model management** — Connect to a llama.cpp router, search and download Hugging Face models, and explicitly load or unload models with live progress. See [llama.cpp](docs/llama-cpp.md). +- **Full provider extensions** — Extensions can register complete pi-ai providers with authentication, model refresh, filtering, and custom streaming. See [Register New Provider](docs/custom-provider.md#register-new-provider). +- **Qwen Token Plan providers** — Use the built-in international and China subscription providers with regional endpoints and API-key authentication. See [API Keys](docs/providers.md#api-keys). +- **Expanded usage accounting** — Tool, compaction, and branch-summary usage is persisted and included in session totals. See [Compaction & Branch Summarization](docs/compaction.md). + ### Added +- Added Qwen Token Plan and Qwen Token Plan China to built-in provider setup, default model resolution, and provider documentation ([#6858](https://github.com/earendil-works/pi/pull/6858) by [@QuintinShaw](https://github.com/QuintinShaw)). +- Added the `get_available_thinking_levels` RPC command and `RpcClient.getAvailableThinkingLevels()` method ([#6865](https://github.com/earendil-works/pi/pull/6865) by [@cristinaponcela](https://github.com/cristinaponcela)). +- Exported message and tool execution lifecycle event types from the package root ([#6772](https://github.com/earendil-works/pi/pull/6772) by [@davidbrai](https://github.com/davidbrai)). - Added built-in llama.cpp router support with `/login` connection setup and `/llama` Hugging Face model search and downloads, explicit loading, unloading, and live progress. See [llama.cpp](docs/llama-cpp.md). - Added extension registration for complete pi-ai providers, including native authentication, model refresh, filtering, and streaming behavior. - Added usage accounting for tools, compaction, and branch summaries in persisted sessions, footer totals, and session statistics ([#6671](https://github.com/earendil-works/pi/pull/6671) by [@davidbrai](https://github.com/davidbrai)). ### Fixed +- Updated the packaged `brace-expansion` dependency to 5.0.7 ([#6896](https://github.com/earendil-works/pi/pull/6896) by [@davidbrai](https://github.com/davidbrai)). +- Fixed persisted remote model catalogs from overriding newer bundled catalogs after an upgrade. +- Fixed inherited stored API-key credentials to apply their provider-scoped `env` values, including Amazon Bedrock profiles ([#6864](https://github.com/earendil-works/pi/pull/6864) by [@cristinaponcela](https://github.com/cristinaponcela)). +- Fixed inherited OpenAI-compatible cross-provider replay to keep tool call IDs unique when multiple calls share a provider call ID ([#6854](https://github.com/earendil-works/pi/pull/6854) by [@cristinaponcela](https://github.com/cristinaponcela)). +- Fixed inherited Kimi K3 thinking levels to expose low, high, and max, and normalized the `k2p7` alias to `kimi-for-coding`. +- Fixed inherited OpenCode Go models routed through the OpenAI Responses API. +- Fixed inherited `pi-ai` package metadata to avoid repeated consumer lockfile changes ([#6812](https://github.com/earendil-works/pi/pull/6812) by [@jmfederico](https://github.com/jmfederico)). +- Fixed inherited terminal shutdown to clear the editor's inverted software cursor before restoring the hardware cursor ([#6790](https://github.com/earendil-works/pi/pull/6790) by [@dam9000](https://github.com/dam9000)). +- Fixed inherited ANSI-aware text wrapping to recognize CRLF and CR line endings while preserving styles ([#6764](https://github.com/earendil-works/pi/pull/6764) by [@xz-dev](https://github.com/xz-dev)). +- Fixed inherited editor paste registry corruption after deleting and undoing paste markers, preventing literal or mismatched paste markers in submitted prompts ([#6844](https://github.com/earendil-works/pi/issues/6844)). +- Fixed sessionless OpenAI Codex WebSocket requests to use UUIDv7 request IDs ([#6834](https://github.com/earendil-works/pi/pull/6834) by [@xl0](https://github.com/xl0)). +- Fixed inherited GPT-5.6 Codex models to default to the 272K context window, avoiding automatic long-context pricing ([#6853](https://github.com/earendil-works/pi/pull/6853) by [@aadishv](https://github.com/aadishv)). +- Fixed messages queued during compaction to preserve steering and follow-up delivery behavior ([#6730](https://github.com/earendil-works/pi/pull/6730) by [@dannote](https://github.com/dannote)). +- Fixed read tool errors being syntax-highlighted as if they were file contents ([#6731](https://github.com/earendil-works/pi/pull/6731) by [@dannote](https://github.com/dannote)). +- Fixed llama.cpp router download progress updates and removed redundant wording from model action confirmations. - Moved automatic model catalog network refresh out of startup initialization and into the running interactive and RPC modes. - Fixed persisted sessions being read and parsed twice when opened, reducing startup latency for large sessions ([#6793](https://github.com/earendil-works/pi/issues/6793)). - Fixed prompt-template defaults for all arguments (`${@:-default}` and `${ARGUMENTS:-default}`) ([#6695](https://github.com/earendil-works/pi/issues/6695)). diff --git a/packages/coding-agent/docs/session-format.md b/packages/coding-agent/docs/session-format.md index b27b8b70..e2c49f1d 100644 --- a/packages/coding-agent/docs/session-format.md +++ b/packages/coding-agent/docs/session-format.md @@ -232,10 +232,18 @@ Created when context is compacted. Stores a summary of earlier messages. {"type":"compaction","id":"f6g7h8i9","parentId":"e5f6g7h8","timestamp":"2024-12-03T14:10:00.000Z","summary":"User discussed X, Y, Z...","firstKeptEntryId":"c3d4e5f6","tokensBefore":50000} ``` +Newer harness-generated compactions embed the retained post-compaction context directly on the entry, instead of `firstKeptEntryId`: + +```json +{"type":"compaction","id":"f6g7h8i9","parentId":"e5f6g7h8","timestamp":"2024-12-03T14:10:00.000Z","summary":"User discussed X, Y, Z...","tokensBefore":50000,"retainedTail":[{"role":"user","content":"latest request"},{"role":"assistant","content":[{"type":"text","text":"latest reply"}],"provider":"anthropic","model":"claude-sonnet-4-5","usage":{...},"stopReason":"stop"}]} +``` + Optional fields: - `usage`: LLM usage from generating the summary; included in session token and cost totals +- `retainedTail`: Materialized `AgentMessage[]` kept after compaction. This is optional only for backward compatibility with older sessions. Newer harness-generated compactions include it so we can rebuild context from this checkpoint without walking older entries before the compaction entry. - `details`: Implementation-specific data (e.g., `{ readFiles: string[], modifiedFiles: string[] }` for default, or custom data for extensions) - `fromHook`: `true` if generated by an extension, `false`/`undefined` if pi-generated (legacy field name) +- `firstKeptEntryId`: for compatibility with old entry format. ### BranchSummaryEntry @@ -314,8 +322,9 @@ Entries form a tree: 1. Collects all entries on the path 2. If a `CompactionEntry` is on the path: - Includes the compaction entry first - - Then entries from `firstKeptEntryId` to compaction - - Then entries after compaction + - If `retainedTail` is present, it acts as a self-contained checkpoint and entries after the compaction are included + - Otherwise entries from `firstKeptEntryId` to the compaction are included + - Then entries after compaction are included 3. Preserves non-message entries in the selected range so interactive mode can render them `buildSessionContext()` builds on that entry list to produce the message list for the LLM: @@ -323,11 +332,13 @@ Entries form a tree: 1. Extracts current model and thinking level settings from the full path 2. Converts selected entries to messages: - `message` -> stored `AgentMessage` - - `compaction` -> `compactionSummary` + - `compaction` -> `compactionSummary` plus `retainedTail` when present - `branch_summary` -> `branchSummary` - `custom_message` -> `CustomMessage` - `custom` -> no context message +This makes newer compactions act like self-contained checkpoints. `retainedTail` is optional only so older sessions that only store `firstKeptEntryId` continue to load correctly. + ## Parsing Example ```typescript diff --git a/packages/coding-agent/examples/extensions/custom-provider-anthropic/package-lock.json b/packages/coding-agent/examples/extensions/custom-provider-anthropic/package-lock.json index 7809bdd8..48b9b039 100644 --- a/packages/coding-agent/examples/extensions/custom-provider-anthropic/package-lock.json +++ b/packages/coding-agent/examples/extensions/custom-provider-anthropic/package-lock.json @@ -1,12 +1,12 @@ { "name": "pi-extension-custom-provider", - "version": "0.80.10", + "version": "0.81.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pi-extension-custom-provider", - "version": "0.80.10", + "version": "0.81.0", "dependencies": { "@anthropic-ai/sdk": "^0.52.0" } diff --git a/packages/coding-agent/examples/extensions/custom-provider-anthropic/package.json b/packages/coding-agent/examples/extensions/custom-provider-anthropic/package.json index 7043a3c8..2aaa62c9 100644 --- a/packages/coding-agent/examples/extensions/custom-provider-anthropic/package.json +++ b/packages/coding-agent/examples/extensions/custom-provider-anthropic/package.json @@ -1,7 +1,7 @@ { "name": "pi-extension-custom-provider-anthropic", "private": true, - "version": "0.80.10", + "version": "0.81.0", "type": "module", "scripts": { "clean": "echo 'nothing to clean'", diff --git a/packages/coding-agent/examples/extensions/custom-provider-gitlab-duo/package.json b/packages/coding-agent/examples/extensions/custom-provider-gitlab-duo/package.json index d4bdb4f3..2eec0eaf 100644 --- a/packages/coding-agent/examples/extensions/custom-provider-gitlab-duo/package.json +++ b/packages/coding-agent/examples/extensions/custom-provider-gitlab-duo/package.json @@ -1,7 +1,7 @@ { "name": "pi-extension-custom-provider-gitlab-duo", "private": true, - "version": "0.80.10", + "version": "0.81.0", "type": "module", "scripts": { "clean": "echo 'nothing to clean'", diff --git a/packages/coding-agent/examples/extensions/gondolin/package-lock.json b/packages/coding-agent/examples/extensions/gondolin/package-lock.json index 669695cb..a4b1dc37 100644 --- a/packages/coding-agent/examples/extensions/gondolin/package-lock.json +++ b/packages/coding-agent/examples/extensions/gondolin/package-lock.json @@ -1,12 +1,12 @@ { "name": "pi-extension-gondolin", - "version": "0.80.10", + "version": "0.81.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pi-extension-gondolin", - "version": "0.80.10", + "version": "0.81.0", "dependencies": { "@earendil-works/gondolin": "0.12.0" } diff --git a/packages/coding-agent/examples/extensions/gondolin/package.json b/packages/coding-agent/examples/extensions/gondolin/package.json index 2737ed01..7c1bd234 100644 --- a/packages/coding-agent/examples/extensions/gondolin/package.json +++ b/packages/coding-agent/examples/extensions/gondolin/package.json @@ -1,7 +1,7 @@ { "name": "pi-extension-gondolin", "private": true, - "version": "0.80.10", + "version": "0.81.0", "type": "module", "scripts": { "clean": "echo 'nothing to clean'", diff --git a/packages/coding-agent/examples/extensions/sandbox/package-lock.json b/packages/coding-agent/examples/extensions/sandbox/package-lock.json index afc412e1..9c0e254a 100644 --- a/packages/coding-agent/examples/extensions/sandbox/package-lock.json +++ b/packages/coding-agent/examples/extensions/sandbox/package-lock.json @@ -1,12 +1,12 @@ { "name": "pi-extension-sandbox", - "version": "1.10.10", + "version": "1.11.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pi-extension-sandbox", - "version": "1.10.10", + "version": "1.11.0", "dependencies": { "@anthropic-ai/sandbox-runtime": "^0.0.26" } diff --git a/packages/coding-agent/examples/extensions/sandbox/package.json b/packages/coding-agent/examples/extensions/sandbox/package.json index f096c010..b341b732 100644 --- a/packages/coding-agent/examples/extensions/sandbox/package.json +++ b/packages/coding-agent/examples/extensions/sandbox/package.json @@ -1,7 +1,7 @@ { "name": "pi-extension-sandbox", "private": true, - "version": "1.10.10", + "version": "1.11.0", "type": "module", "scripts": { "clean": "echo 'nothing to clean'", diff --git a/packages/coding-agent/examples/extensions/with-deps/package-lock.json b/packages/coding-agent/examples/extensions/with-deps/package-lock.json index 860a256c..7ad0e0ce 100644 --- a/packages/coding-agent/examples/extensions/with-deps/package-lock.json +++ b/packages/coding-agent/examples/extensions/with-deps/package-lock.json @@ -1,12 +1,12 @@ { "name": "pi-extension-with-deps", - "version": "0.80.10", + "version": "0.81.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pi-extension-with-deps", - "version": "0.80.10", + "version": "0.81.0", "dependencies": { "ms": "^2.1.3" }, diff --git a/packages/coding-agent/examples/extensions/with-deps/package.json b/packages/coding-agent/examples/extensions/with-deps/package.json index 8b3d953c..b414081a 100644 --- a/packages/coding-agent/examples/extensions/with-deps/package.json +++ b/packages/coding-agent/examples/extensions/with-deps/package.json @@ -1,7 +1,7 @@ { "name": "pi-extension-with-deps", "private": true, - "version": "0.80.10", + "version": "0.81.0", "type": "module", "scripts": { "clean": "echo 'nothing to clean'", diff --git a/packages/coding-agent/install-lock/package-lock.json b/packages/coding-agent/install-lock/package-lock.json index f66fd9b3..4b624b36 100644 --- a/packages/coding-agent/install-lock/package-lock.json +++ b/packages/coding-agent/install-lock/package-lock.json @@ -1,14 +1,14 @@ { "name": "@earendil-works/pi-coding-agent-install", - "version": "0.80.10", + "version": "0.81.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@earendil-works/pi-coding-agent-install", - "version": "0.80.10", + "version": "0.81.0", "dependencies": { - "@earendil-works/pi-coding-agent": "0.80.10" + "@earendil-works/pi-coding-agent": "0.81.0" }, "engines": { "node": ">=22.19.0" @@ -450,11 +450,11 @@ } }, "node_modules/@earendil-works/pi-agent-core": { - "version": "0.80.10", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-agent-core/-/pi-agent-core-0.80.10.tgz", + "version": "0.81.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-agent-core/-/pi-agent-core-0.81.0.tgz", "license": "MIT", "dependencies": { - "@earendil-works/pi-ai": "^0.80.10", + "@earendil-works/pi-ai": "^0.81.0", "ignore": "7.0.5", "typebox": "1.1.38", "yaml": "2.9.0" @@ -464,8 +464,8 @@ } }, "node_modules/@earendil-works/pi-ai": { - "version": "0.80.10", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-0.80.10.tgz", + "version": "0.81.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-0.81.0.tgz", "license": "MIT", "dependencies": { "@anthropic-ai/sdk": "0.91.1", @@ -488,13 +488,13 @@ } }, "node_modules/@earendil-works/pi-coding-agent": { - "version": "0.80.10", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-coding-agent/-/pi-coding-agent-0.80.10.tgz", + "version": "0.81.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-coding-agent/-/pi-coding-agent-0.81.0.tgz", "license": "MIT", "dependencies": { - "@earendil-works/pi-agent-core": "^0.80.10", - "@earendil-works/pi-ai": "^0.80.10", - "@earendil-works/pi-tui": "^0.80.10", + "@earendil-works/pi-agent-core": "^0.81.0", + "@earendil-works/pi-ai": "^0.81.0", + "@earendil-works/pi-tui": "^0.81.0", "@silvia-odwyer/photon-node": "0.3.4", "chalk": "5.6.2", "cross-spawn": "7.0.6", @@ -522,8 +522,8 @@ } }, "node_modules/@earendil-works/pi-tui": { - "version": "0.80.10", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-tui/-/pi-tui-0.80.10.tgz", + "version": "0.81.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-tui/-/pi-tui-0.81.0.tgz", "license": "MIT", "dependencies": { "get-east-asian-width": "1.6.0", @@ -638,9 +638,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "optional": true }, "node_modules/@mariozechner/clipboard-linux-arm64-musl": { @@ -657,9 +654,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "optional": true }, "node_modules/@mariozechner/clipboard-linux-riscv64-gnu": { @@ -676,9 +670,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "optional": true }, "node_modules/@mariozechner/clipboard-linux-x64-gnu": { @@ -695,9 +686,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "optional": true }, "node_modules/@mariozechner/clipboard-linux-x64-musl": { @@ -714,9 +702,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "optional": true }, "node_modules/@mariozechner/clipboard-win32-arm64-msvc": { diff --git a/packages/coding-agent/install-lock/package.json b/packages/coding-agent/install-lock/package.json index 1ff7e478..c06b3105 100644 --- a/packages/coding-agent/install-lock/package.json +++ b/packages/coding-agent/install-lock/package.json @@ -1,10 +1,10 @@ { "name": "@earendil-works/pi-coding-agent-install", - "version": "0.80.10", + "version": "0.81.0", "private": true, "description": "Lockfile root used by the Pi installer and updater.", "dependencies": { - "@earendil-works/pi-coding-agent": "0.80.10" + "@earendil-works/pi-coding-agent": "0.81.0" }, "overrides": { "rimraf": "6.1.2", diff --git a/packages/coding-agent/npm-shrinkwrap.json b/packages/coding-agent/npm-shrinkwrap.json index 96c93de8..235bb97d 100644 --- a/packages/coding-agent/npm-shrinkwrap.json +++ b/packages/coding-agent/npm-shrinkwrap.json @@ -1,17 +1,17 @@ { "name": "@earendil-works/pi-coding-agent", - "version": "0.80.10", + "version": "0.81.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@earendil-works/pi-coding-agent", - "version": "0.80.10", + "version": "0.81.0", "license": "MIT", "dependencies": { - "@earendil-works/pi-agent-core": "^0.80.10", - "@earendil-works/pi-ai": "^0.80.10", - "@earendil-works/pi-tui": "^0.80.10", + "@earendil-works/pi-agent-core": "^0.81.0", + "@earendil-works/pi-ai": "^0.81.0", + "@earendil-works/pi-tui": "^0.81.0", "@silvia-odwyer/photon-node": "0.3.4", "chalk": "5.6.2", "cross-spawn": "7.0.6", @@ -474,11 +474,11 @@ } }, "node_modules/@earendil-works/pi-agent-core": { - "version": "0.80.10", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-agent-core/-/pi-agent-core-0.80.10.tgz", + "version": "0.81.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-agent-core/-/pi-agent-core-0.81.0.tgz", "license": "MIT", "dependencies": { - "@earendil-works/pi-ai": "^0.80.10", + "@earendil-works/pi-ai": "^0.81.0", "ignore": "7.0.5", "typebox": "1.1.38", "yaml": "2.9.0" @@ -488,8 +488,8 @@ } }, "node_modules/@earendil-works/pi-ai": { - "version": "0.80.10", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-0.80.10.tgz", + "version": "0.81.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-0.81.0.tgz", "license": "MIT", "dependencies": { "@anthropic-ai/sdk": "0.91.1", @@ -512,8 +512,8 @@ } }, "node_modules/@earendil-works/pi-tui": { - "version": "0.80.10", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-tui/-/pi-tui-0.80.10.tgz", + "version": "0.81.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-tui/-/pi-tui-0.81.0.tgz", "license": "MIT", "dependencies": { "get-east-asian-width": "1.6.0", @@ -628,9 +628,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "optional": true }, "node_modules/@mariozechner/clipboard-linux-arm64-musl": { @@ -647,9 +644,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "optional": true }, "node_modules/@mariozechner/clipboard-linux-riscv64-gnu": { @@ -666,9 +660,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "optional": true }, "node_modules/@mariozechner/clipboard-linux-x64-gnu": { @@ -685,9 +676,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "optional": true }, "node_modules/@mariozechner/clipboard-linux-x64-musl": { @@ -704,9 +692,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "optional": true }, "node_modules/@mariozechner/clipboard-win32-arm64-msvc": { diff --git a/packages/coding-agent/package.json b/packages/coding-agent/package.json index 26e8513f..6c454520 100644 --- a/packages/coding-agent/package.json +++ b/packages/coding-agent/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-coding-agent", - "version": "0.80.10", + "version": "0.81.0", "description": "Coding agent CLI with read, bash, edit, write tools and session management", "type": "module", "piConfig": { @@ -39,9 +39,9 @@ "prepublishOnly": "npm run clean && npm run build && npm run shrinkwrap" }, "dependencies": { - "@earendil-works/pi-agent-core": "^0.80.10", - "@earendil-works/pi-ai": "^0.80.10", - "@earendil-works/pi-tui": "^0.80.10", + "@earendil-works/pi-agent-core": "^0.81.0", + "@earendil-works/pi-ai": "^0.81.0", + "@earendil-works/pi-tui": "^0.81.0", "@silvia-odwyer/photon-node": "0.3.4", "chalk": "5.6.2", "cross-spawn": "7.0.6", diff --git a/packages/coding-agent/src/core/model-runtime.ts b/packages/coding-agent/src/core/model-runtime.ts index 64501f73..2cd85b8d 100644 --- a/packages/coding-agent/src/core/model-runtime.ts +++ b/packages/coding-agent/src/core/model-runtime.ts @@ -143,7 +143,15 @@ export class ModelRuntime implements Models { const providers = builtinProviderCatalog .builtinProviders() .map((provider) => - provider.id === "radius" ? provider : withRemoteCatalog(provider, options.catalogBaseUrl), + provider.id === "radius" + ? provider + : withRemoteCatalog( + provider, + options.catalogBaseUrl, + builtinProviderCatalog.getBuiltinModelDataUrl( + provider.id as builtinProviderCatalog.BuiltinProvider, + ), + ), ); const runtime = new ModelRuntime( credentials, diff --git a/packages/coding-agent/src/core/remote-catalog-provider.ts b/packages/coding-agent/src/core/remote-catalog-provider.ts index 6667916d..b823cf7d 100644 --- a/packages/coding-agent/src/core/remote-catalog-provider.ts +++ b/packages/coding-agent/src/core/remote-catalog-provider.ts @@ -1,4 +1,5 @@ -import type { Api, Model, Provider } from "@earendil-works/pi-ai"; +import { stat } from "node:fs/promises"; +import type { Api, Model, ModelsStoreEntry, Provider } from "@earendil-works/pi-ai"; import { VERSION } from "../config.ts"; import { getPiUserAgent } from "../utils/pi-user-agent.ts"; @@ -29,8 +30,26 @@ function parseCatalog(providerId: string, value: unknown): Model[] { .map((model) => ({ ...model, provider: providerId })); } +function remoteModels( + entry: ModelsStoreEntry | undefined, + localLastModified: number | undefined, +): readonly Model[] { + if (!entry) return []; + if ( + localLastModified !== undefined && + (entry.lastModified === undefined || entry.lastModified <= localLastModified) + ) { + return []; + } + return entry.models; +} + /** Add a persisted pi.dev catalog overlay to a static built-in provider. */ -export function withRemoteCatalog(provider: Provider, catalogBaseUrl: string = DEFAULT_CATALOG_BASE_URL): Provider { +export function withRemoteCatalog( + provider: Provider, + catalogBaseUrl: string = DEFAULT_CATALOG_BASE_URL, + localCatalogUrl?: URL, +): Provider { let dynamicModels: readonly Model[] = []; let inflightRefresh: Promise | undefined; @@ -40,12 +59,21 @@ export function withRemoteCatalog(provider: Provider, catalogBaseUrl: string = D refreshModels: (context) => { inflightRefresh ??= (async () => { try { + const localLastModified = localCatalogUrl + ? await stat(localCatalogUrl).then( + (value) => value.mtimeMs, + () => undefined, + ) + : undefined; const stored = await context.store.read(); - if (stored) dynamicModels = stored.models.filter((model) => model.provider === provider.id); + dynamicModels = remoteModels(stored, localLastModified).filter( + (model) => model.provider === provider.id, + ); if (!context.allowNetwork || context.signal?.aborted) return; if ( !context.force && stored?.checkedAt !== undefined && + stored.lastModified !== undefined && Date.now() - stored.checkedAt < REMOTE_CATALOG_REFRESH_INTERVAL_MS ) { return; @@ -62,17 +90,23 @@ export function withRemoteCatalog(provider: Provider, catalogBaseUrl: string = D if (context.signal?.aborted) return; const checkedAt = Date.now(); if (response.status === 404 || response.status === 501) { - await context.store.write({ models: dynamicModels, checkedAt }); + await context.store.write({ ...(stored ?? { models: [] }), checkedAt, lastModified: 0 }); return; } if (!response.ok) { - await context.store.write({ models: dynamicModels, checkedAt }); + await context.store.write({ ...(stored ?? { models: [] }), checkedAt }); throw new Error(`Model catalog request failed for ${provider.id}: ${response.status}`); } const refreshed = parseCatalog(provider.id, await response.json()); + const lastModified = Date.parse(response.headers.get("last-modified") ?? ""); if (context.signal?.aborted) return; - dynamicModels = refreshed; - await context.store.write({ models: refreshed, checkedAt }); + const entry = { + models: refreshed, + checkedAt, + lastModified: Number.isNaN(lastModified) ? 0 : lastModified, + }; + dynamicModels = remoteModels(entry, localLastModified); + await context.store.write(entry); } finally { inflightRefresh = undefined; } diff --git a/packages/coding-agent/src/extensions/llama/client.ts b/packages/coding-agent/src/extensions/llama/client.ts index 0e58e1bd..45cd8f3a 100644 --- a/packages/coding-agent/src/extensions/llama/client.ts +++ b/packages/coding-agent/src/extensions/llama/client.ts @@ -107,9 +107,11 @@ function parseLoadProgress(data: unknown): LlamaProgress | undefined { function parseDownloadProgress(data: unknown): LlamaProgress | undefined { if (typeof data !== "object" || data === null) return undefined; + const nested = (data as { progress?: unknown }).progress; + const files = typeof nested === "object" && nested !== null ? nested : data; let done = 0; let total = 0; - for (const value of Object.values(data as Record)) { + for (const value of Object.values(files as Record)) { if (typeof value !== "object" || value === null) continue; const entry = value as { done?: unknown; total?: unknown }; if (typeof entry.done !== "number" || typeof entry.total !== "number") continue; diff --git a/packages/coding-agent/src/extensions/llama/index.ts b/packages/coding-agent/src/extensions/llama/index.ts index 4a38b3c6..5fc17b99 100644 --- a/packages/coding-agent/src/extensions/llama/index.ts +++ b/packages/coding-agent/src/extensions/llama/index.ts @@ -88,7 +88,7 @@ export default function llamaExtension(pi: ExtensionAPI): void { model: target.id, initialMessage: "Starting…", cancelTitle: "Stop loading?", - cancelMessage: `Stop loading ${target.id}?`, + cancelMessage: target.id, run: (signal, update) => client.loadAndWait(target.id, update, signal), cancel: () => client.unload(target.id), }); @@ -119,7 +119,7 @@ export default function llamaExtension(pi: ExtensionAPI): void { client: LlamaClient, model: LlamaModelInfo, ): Promise => { - if (!(await ui.confirm("Unload model?", `Unload ${model.id}?`))) return; + if (!(await ui.confirm("Unload model?", model.id))) return; await client.unloadAndWait(model.id); await syncCatalog(ctx, client); ctx.ui.notify(`Unloaded ${model.id}`); @@ -162,7 +162,7 @@ export default function llamaExtension(pi: ExtensionAPI): void { model, initialMessage: "Starting…", cancelTitle: "Stop download?", - cancelMessage: `Stop downloading ${model}?`, + cancelMessage: model, run: (signal, update) => client.downloadAndWait(model, update, signal), cancel: () => client.unload(model), }); diff --git a/packages/coding-agent/test/llama-extension.test.ts b/packages/coding-agent/test/llama-extension.test.ts index 3841af53..19dbdee7 100644 --- a/packages/coding-agent/test/llama-extension.test.ts +++ b/packages/coding-agent/test/llama-extension.test.ts @@ -220,7 +220,7 @@ describe("llama.cpp extension", () => { send({ model: "owner/repo:Q4_K_M", event: "download_progress", - data: { "https://example/model.gguf": { done: 512, total: 1024 } }, + data: { progress: { "https://example/model.gguf": { done: 512, total: 1024 } } }, }); status = "unloaded"; send({ model: "owner/repo:Q4_K_M", event: "download_finished", data: {} }); diff --git a/packages/coding-agent/test/model-registry.test.ts b/packages/coding-agent/test/model-registry.test.ts index ef4b133b..70a8fe9e 100644 --- a/packages/coding-agent/test/model-registry.test.ts +++ b/packages/coding-agent/test/model-registry.test.ts @@ -652,7 +652,7 @@ describe("ModelRegistry", () => { expect(anthropicModels.some((m) => m.id === "claude-custom")).toBe(false); expect(anthropicModels.some((m) => m.id === "claude-custom-2")).toBe(true); expect(anthropicModels.some((m) => m.id.includes("claude"))).toBe(true); - }); + }, 60_000); test("removing custom models from models.json keeps built-in provider models", async () => { writeModelsJson({ diff --git a/packages/coding-agent/test/remote-catalog-provider.test.ts b/packages/coding-agent/test/remote-catalog-provider.test.ts index c8c5f869..8b43a4f3 100644 --- a/packages/coding-agent/test/remote-catalog-provider.test.ts +++ b/packages/coding-agent/test/remote-catalog-provider.test.ts @@ -1,4 +1,11 @@ -import { createProvider, InMemoryModelsStore, type Model } from "@earendil-works/pi-ai"; +import { statSync } from "node:fs"; +import { + createProvider, + InMemoryModelsStore, + type Model, + type ModelsStoreEntry, + type ProviderModelsStore, +} from "@earendil-works/pi-ai"; import { afterEach, describe, expect, it, vi } from "vitest"; import { VERSION } from "../src/config.ts"; import { withRemoteCatalog } from "../src/core/remote-catalog-provider.ts"; @@ -18,6 +25,34 @@ function model(id: string): Model<"openai-completions"> { }; } +function testProvider(localCatalogUrl?: URL) { + return withRemoteCatalog( + createProvider({ + id: "test-provider", + auth: { apiKey: { name: "Test", resolve: async () => ({ auth: {} }) } }, + models: [model("static")], + api: { + stream: () => { + throw new Error("not used"); + }, + streamSimple: () => { + throw new Error("not used"); + }, + }, + }), + "https://pi.dev", + localCatalogUrl, + ); +} + +function scopedStore(store: InMemoryModelsStore): ProviderModelsStore { + return { + read: () => store.read("test-provider"), + write: (entry: ModelsStoreEntry) => store.write("test-provider", entry), + delete: () => store.delete("test-provider"), + }; +} + afterEach(() => vi.restoreAllMocks()); describe("remote catalog provider", () => { @@ -29,50 +64,12 @@ describe("remote catalog provider", () => { headers: { "content-type": "application/json" }, }), ); - const provider = withRemoteCatalog( - createProvider({ - id: "test-provider", - auth: { apiKey: { name: "Test", resolve: async () => ({ auth: {} }) } }, - models: [model("static")], - api: { - stream: () => { - throw new Error("not used"); - }, - streamSimple: () => { - throw new Error("not used"); - }, - }, - }), - ); + const provider = testProvider(); const store = new InMemoryModelsStore(); - await provider.refreshModels?.({ - credential: { type: "api_key" }, - store: { - read: () => store.read(provider.id), - write: (entry) => store.write(provider.id, entry), - delete: () => store.delete(provider.id), - }, - allowNetwork: true, - }); - await provider.refreshModels?.({ - credential: { type: "api_key" }, - store: { - read: () => store.read(provider.id), - write: (entry) => store.write(provider.id, entry), - delete: () => store.delete(provider.id), - }, - allowNetwork: true, - }); - await provider.refreshModels?.({ - credential: { type: "api_key" }, - store: { - read: () => store.read(provider.id), - write: (entry) => store.write(provider.id, entry), - delete: () => store.delete(provider.id), - }, - allowNetwork: true, - force: true, - }); + const refresh = { credential: { type: "api_key" } as const, store: scopedStore(store), allowNetwork: true }; + await provider.refreshModels?.(refresh); + await provider.refreshModels?.(refresh); + await provider.refreshModels?.({ ...refresh, force: true }); expect(provider.getModels().map((entry) => entry.id)).toEqual(["static", "dynamic"]); expect((await store.read(provider.id))?.models.map((entry) => entry.id)).toEqual(["dynamic"]); @@ -82,33 +79,40 @@ describe("remote catalog provider", () => { }); }); + it("prefers the newer of the generated and remote catalogs", async () => { + const localCatalogUrl = new URL(import.meta.url); + const localMtime = statSync(localCatalogUrl).mtimeMs; + const newerHeader = new Date(localMtime + 60_000).toUTCString(); + const responses = [ + new Response(JSON.stringify({ old: model("old") }), { + headers: { "last-modified": new Date(localMtime - 60_000).toUTCString() }, + }), + new Response(JSON.stringify({ newer: model("newer") }), { + headers: { "last-modified": newerHeader }, + }), + ]; + vi.spyOn(globalThis, "fetch").mockImplementation(async () => responses.shift() as Response); + const provider = testProvider(localCatalogUrl); + const store = new InMemoryModelsStore(); + const refresh = { credential: { type: "api_key" } as const, store: scopedStore(store), allowNetwork: true }; + + await provider.refreshModels?.(refresh); + expect(provider.getModels().map((entry) => entry.id)).toEqual(["static"]); + + await provider.refreshModels?.({ ...refresh, force: true }); + expect(provider.getModels().map((entry) => entry.id)).toEqual(["static", "newer"]); + expect(await store.read(provider.id)).toMatchObject({ lastModified: Date.parse(newerHeader) }); + }); + it("treats unimplemented pi.dev catalog routes as an unavailable overlay", async () => { vi.spyOn(globalThis, "fetch").mockResolvedValue(new Response("not implemented", { status: 501 })); - const provider = withRemoteCatalog( - createProvider({ - id: "test-provider", - auth: { apiKey: { name: "Test", resolve: async () => ({ auth: {} }) } }, - models: [model("static")], - api: { - stream: () => { - throw new Error("not used"); - }, - streamSimple: () => { - throw new Error("not used"); - }, - }, - }), - ); + const provider = testProvider(); const store = new InMemoryModelsStore(); await expect( provider.refreshModels?.({ credential: { type: "api_key" }, - store: { - read: () => store.read(provider.id), - write: (entry) => store.write(provider.id, entry), - delete: () => store.delete(provider.id), - }, + store: scopedStore(store), allowNetwork: true, }), ).resolves.toBeUndefined(); diff --git a/packages/orchestrator/CHANGELOG.md b/packages/orchestrator/CHANGELOG.md deleted file mode 100644 index 9762afa6..00000000 --- a/packages/orchestrator/CHANGELOG.md +++ /dev/null @@ -1,19 +0,0 @@ -# Changelog - -## [Unreleased] - -## [0.80.10] - 2026-07-16 - -## [0.80.9] - 2026-07-16 - -## [0.80.8] - 2026-07-16 - -## [0.80.7] - 2026-07-14 - -## [0.80.6] - 2026-07-09 - -## [0.80.5] - 2026-07-09 - -## [0.80.4] - 2026-07-09 - -## [0.80.3] - 2026-06-30 diff --git a/packages/server/CHANGELOG.md b/packages/server/CHANGELOG.md new file mode 100644 index 00000000..7f82e7c3 --- /dev/null +++ b/packages/server/CHANGELOG.md @@ -0,0 +1,25 @@ +# Changelog + +## [Unreleased] + +## [0.81.0] - 2026-07-21 + +### Changed + +- Renamed the orchestrator workspace package and internal server references to server ([#6898](https://github.com/earendil-works/pi/pull/6898) by [@cristinaponcela](https://github.com/cristinaponcela)). + +## [0.80.10] - 2026-07-16 + +## [0.80.9] - 2026-07-16 + +## [0.80.8] - 2026-07-16 + +## [0.80.7] - 2026-07-14 + +## [0.80.6] - 2026-07-09 + +## [0.80.5] - 2026-07-09 + +## [0.80.4] - 2026-07-09 + +## [0.80.3] - 2026-06-30 diff --git a/packages/orchestrator/README.md b/packages/server/README.md similarity index 67% rename from packages/orchestrator/README.md rename to packages/server/README.md index aabd7243..2e8e877b 100644 --- a/packages/orchestrator/README.md +++ b/packages/server/README.md @@ -1,11 +1,11 @@ -# @earendil-works/pi-orchestrator +# @earendil-works/pi-server Experimental. This package is under active development and may change or be removed without notice. Its CLI, APIs, and behavior are not yet stable. -Orchestrator package for pi. +Server package for pi. ## CLI ```bash -orchestrator --help +server --help ``` diff --git a/packages/orchestrator/package.json b/packages/server/package.json similarity index 75% rename from packages/orchestrator/package.json rename to packages/server/package.json index c1d7e95a..c5b6d8a0 100644 --- a/packages/orchestrator/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { - "name": "@earendil-works/pi-orchestrator", - "version": "0.80.10", - "description": "experimental orchestrator package for pi", + "name": "@earendil-works/pi-server", + "version": "0.81.0", + "description": "experimental server package for pi", "type": "module", "main": "./dist/index.js", "types": "./dist/index.d.ts", @@ -11,6 +11,9 @@ "import": "./dist/index.js" } }, + "bin": { + "server": "./dist/cli.js" + }, "files": [ "dist", "README.md", @@ -24,20 +27,20 @@ }, "keywords": [ "pi", - "orchestrator" + "server" ], "author": "Earendil Works", "license": "MIT", "repository": { "type": "git", "url": "git+https://github.com/earendil-works/pi.git", - "directory": "packages/orchestrator" + "directory": "packages/server" }, "engines": { "node": ">=22.19.0" }, "dependencies": { - "@earendil-works/pi-coding-agent": "^0.80.10" + "@earendil-works/pi-coding-agent": "^0.81.0" }, "devDependencies": { "shx": "0.4.0" diff --git a/packages/orchestrator/src/cli.ts b/packages/server/src/cli.ts similarity index 84% rename from packages/orchestrator/src/cli.ts rename to packages/server/src/cli.ts index 856dd658..738d416e 100644 --- a/packages/orchestrator/src/cli.ts +++ b/packages/server/src/cli.ts @@ -18,7 +18,7 @@ const packageJson = JSON.parse(readFileSync(join(__dirname, "../package.json"), function printHelp(): void { console.log( - `orchestrator v${packageJson.version}\n\nUsage:\n orchestrator serve\n orchestrator list\n orchestrator spawn [--cwd ] [--label