feat: sqlite session storage (#6594)

This PR:

- Adds retainedTail to compaction entries in the new agent harness so we don't have to walk up the tree for the 2000 tokens before compaction,
- Changes getPathToRoot to getPathToRootOrCompaction to only load until last compaction, as unnecessary to access all nodes where it is called,
- Adds a SQLite storage backend, in a separate packages/session-backend-sqlite, with a migration system and schemas as per on-site discussions: sessions to match session header messages (except for metadata, which I couldn't understand what it's used for or where it gets written, so I omitted it), session_entries for shared entry types as columns plus payload as a json for what remains, session_sequences to represent the append-only, serialized nature of the jsonl files, branch_entries to attribute nodes to branches (relationship one-to-many), and session_materialized with the session info (see /session in TUI) to act as a "cache" or quick-access for costs, message count, token info, labels, session name, and model-thinking-level config (e.g. for fast resume).
- This is compatible with the new agent harness Session abstraction.
This commit is contained in:
Cristina Poncela Cubeiro
2026-07-21 11:36:31 +02:00
committed by GitHub
parent 54fad505b9
commit 9e7582aa03
40 changed files with 2659 additions and 145 deletions
+4
View File
@@ -8,6 +8,10 @@ Stateful agent with tool execution and event streaming. Built on `@earendil-work
npm install @earendil-works/pi-agent-core
```
### SQLite session backends
The SQLite session backend and the `node:sqlite` adapter live in a separate package, `@earendil-works/pi-agent-sqlite-node`, so the core package does not pull in runtime builtins or native SQLite dependencies by default. The backend accepts a runtime-specific SQLite factory, allowing other storage backends to ship as their own packages in the future.
## Quick Start
```typescript
+1 -2
View File
@@ -21,12 +21,11 @@
"README.md"
],
"scripts": {
"clean": "shx rm -rf dist",
"build": "tsgo -p tsconfig.build.json",
"test": "vitest --run",
"test:harness": "vitest --run --config vitest.harness.config.ts",
"coverage:harness": "vitest --run --config vitest.harness.config.ts --coverage",
"prepublishOnly": "npm run clean && npm run build"
"prepublishOnly": "npm run build"
},
"dependencies": {
"@earendil-works/pi-ai": "^0.80.10",
@@ -730,6 +730,7 @@ export class AgentHarness<
result.details,
provided !== undefined,
result.usage,
result.retainedTail,
);
const entry = await this.session.getEntry(entryId);
if (entry?.type === "compaction") {
@@ -97,12 +97,14 @@ function getMessageFromEntryForCompaction(entry: SessionTreeEntry): AgentMessage
export interface CompactionResult<T = unknown> {
/** Summary text that replaces compacted history in future context. */
summary: string;
/** Entry id where retained history starts. */
firstKeptEntryId: string;
/** Entry id where retained history starts. Optional during Pi 2.0 transition. */
firstKeptEntryId?: string;
/** Estimated context tokens before compaction. */
tokensBefore: number;
/** Usage from the LLM call(s) that generated this summary, if available. */
usage?: Usage;
/** Retained recent messages stored directly on the compaction entry. Optional during Pi 2.0 transition. */
retainedTail?: AgentMessage[];
/** Optional implementation-specific details stored with the compaction entry. */
details?: T;
}
@@ -583,6 +585,8 @@ export interface CompactionPreparation {
messagesToSummarize: AgentMessage[];
/** Prefix messages summarized separately when compaction splits a turn. */
turnPrefixMessages: AgentMessage[];
/** Recent messages retained after compaction and stored on the compaction entry. */
retainedTail: AgentMessage[];
/** Whether compaction splits a turn. */
isSplitTurn: boolean;
/** Estimated context tokens before compaction. */
@@ -617,7 +621,9 @@ export function prepareCompaction(
if (prevCompactionIndex >= 0) {
const prevCompaction = pathEntries[prevCompactionIndex] as CompactionEntry;
previousSummary = prevCompaction.summary;
const firstKeptEntryIndex = pathEntries.findIndex((entry) => entry.id === prevCompaction.firstKeptEntryId);
const firstKeptEntryIndex = prevCompaction.firstKeptEntryId
? pathEntries.findIndex((entry) => entry.id === prevCompaction.firstKeptEntryId)
: -1;
boundaryStart = firstKeptEntryIndex >= 0 ? firstKeptEntryIndex : prevCompactionIndex + 1;
}
const boundaryEnd = pathEntries.length;
@@ -644,6 +650,11 @@ export function prepareCompaction(
if (msg) turnPrefixMessages.push(msg);
}
}
const retainedTail: AgentMessage[] = [];
for (let i = cutPoint.firstKeptEntryIndex; i < boundaryEnd; i++) {
const msg = getMessageFromEntryForCompaction(pathEntries[i]);
if (msg) retainedTail.push(msg);
}
const fileOps = extractFileOperations(messagesToSummarize, pathEntries, prevCompactionIndex);
if (cutPoint.isSplitTurn) {
for (const msg of turnPrefixMessages) {
@@ -655,6 +666,7 @@ export function prepareCompaction(
firstKeptEntryId,
messagesToSummarize,
turnPrefixMessages,
retainedTail,
isSplitTurn: cutPoint.isSplitTurn,
tokensBefore,
previousSummary,
@@ -693,6 +705,7 @@ export async function compact(
firstKeptEntryId,
messagesToSummarize,
turnPrefixMessages,
retainedTail,
isSplitTurn,
tokensBefore,
previousSummary,
@@ -762,6 +775,7 @@ export async function compact(
firstKeptEntryId,
tokensBefore,
usage: summaryUsage,
retainedTail,
details: { readFiles, modifiedFiles } as CompactionDetails,
});
}
@@ -1,5 +1,12 @@
import { uuidv7 } from "@earendil-works/pi-ai";
import type { FileSystem, JsonlSessionMetadata, LeafEntry, SessionStorage, SessionTreeEntry } from "../types.ts";
import type {
FileSystem,
JsonlSessionMetadata,
LeafEntry,
SessionEntryCursorOptions,
SessionStorage,
SessionTreeEntry,
} from "../types.ts";
import { SessionError, toError } from "../types.ts";
import { getFileSystemResultOrThrow } from "./repo-utils.ts";
@@ -293,13 +300,66 @@ export class JsonlSessionStorage implements SessionStorage<JsonlSessionMetadata>
return this.labelsById.get(id);
}
async getPathToRoot(leafId: string | null): Promise<SessionTreeEntry[]> {
async getSessionName(): Promise<string | undefined> {
const entries = await this.findEntries("session_info");
return entries[entries.length - 1]?.name?.trim() || undefined;
}
async getSessionStats() {
let messageCount = 0;
let cachedTokens = 0;
let uncachedTokens = 0;
let totalTokens = 0;
let costTotal = 0;
for (const entry of this.entries) {
if (entry.type === "message") {
messageCount += 1;
}
const usage =
entry.type === "message"
? entry.message.role === "assistant"
? entry.message.usage
: undefined
: entry.type === "compaction" || entry.type === "branch_summary"
? entry.usage
: undefined;
if (
!usage ||
typeof usage.input !== "number" ||
typeof usage.output !== "number" ||
typeof usage.cacheRead !== "number" ||
typeof usage.cacheWrite !== "number" ||
typeof usage.cost?.total !== "number"
) {
continue;
}
cachedTokens += usage.cacheRead;
uncachedTokens += usage.input + usage.cacheWrite;
totalTokens += usage.input + usage.output + usage.cacheRead + usage.cacheWrite;
costTotal += usage.cost.total;
}
return {
messageCount,
cachedTokens,
uncachedTokens,
totalTokens,
costTotal,
};
}
async getPathToRootOrCompaction(leafId: string | null): Promise<SessionTreeEntry[]> {
if (leafId === null) return [];
const path: SessionTreeEntry[] = [];
let stopAtEntryId: string | null = null;
let current = this.byId.get(leafId);
if (!current) throw new SessionError("not_found", `Entry ${leafId} not found`);
while (current) {
path.unshift(current);
if (stopAtEntryId !== null && current.id === stopAtEntryId) break;
if (current.type === "compaction") {
if (current.retainedTail) break;
stopAtEntryId = current.firstKeptEntryId ?? null;
}
if (!current.parentId) break;
const parent = this.byId.get(current.parentId);
if (!parent) throw new SessionError("invalid_session", `Entry ${current.parentId} not found`);
@@ -308,7 +368,9 @@ export class JsonlSessionStorage implements SessionStorage<JsonlSessionMetadata>
return path;
}
async getEntries(): Promise<SessionTreeEntry[]> {
return [...this.entries];
async getEntries(options?: SessionEntryCursorOptions): Promise<SessionTreeEntry[]> {
const start = options?.afterEntrySeq ?? 0;
const end = options?.limit === undefined ? undefined : start + options.limit;
return this.entries.slice(start, end);
}
}
@@ -1,6 +1,7 @@
import { uuidv7 } from "@earendil-works/pi-ai";
import {
type LeafEntry,
type SessionEntryCursorOptions,
SessionError,
type SessionMetadata,
type SessionStorage,
@@ -112,13 +113,66 @@ export class InMemorySessionStorage<TMetadata extends SessionMetadata = SessionM
return this.labelsById.get(id);
}
async getPathToRoot(leafId: string | null): Promise<SessionTreeEntry[]> {
async getSessionName(): Promise<string | undefined> {
const entries = await this.findEntries("session_info");
return entries[entries.length - 1]?.name?.trim() || undefined;
}
async getSessionStats() {
let messageCount = 0;
let cachedTokens = 0;
let uncachedTokens = 0;
let totalTokens = 0;
let costTotal = 0;
for (const entry of this.entries) {
if (entry.type === "message") {
messageCount += 1;
}
const usage =
entry.type === "message"
? entry.message.role === "assistant"
? entry.message.usage
: undefined
: entry.type === "compaction" || entry.type === "branch_summary"
? entry.usage
: undefined;
if (
!usage ||
typeof usage.input !== "number" ||
typeof usage.output !== "number" ||
typeof usage.cacheRead !== "number" ||
typeof usage.cacheWrite !== "number" ||
typeof usage.cost?.total !== "number"
) {
continue;
}
cachedTokens += usage.cacheRead;
uncachedTokens += usage.input + usage.cacheWrite;
totalTokens += usage.input + usage.output + usage.cacheRead + usage.cacheWrite;
costTotal += usage.cost.total;
}
return {
messageCount,
cachedTokens,
uncachedTokens,
totalTokens,
costTotal,
};
}
async getPathToRootOrCompaction(leafId: string | null): Promise<SessionTreeEntry[]> {
if (leafId === null) return [];
const path: SessionTreeEntry[] = [];
let stopAtEntryId: string | null = null;
let current = this.byId.get(leafId);
if (!current) throw new SessionError("not_found", `Entry ${leafId} not found`);
while (current) {
path.unshift(current);
if (stopAtEntryId !== null && current.id === stopAtEntryId) break;
if (current.type === "compaction") {
if (current.retainedTail) break;
stopAtEntryId = current.firstKeptEntryId ?? null;
}
if (!current.parentId) break;
const parent = this.byId.get(current.parentId);
if (!parent) throw new SessionError("invalid_session", `Entry ${current.parentId} not found`);
@@ -127,7 +181,9 @@ export class InMemorySessionStorage<TMetadata extends SessionMetadata = SessionM
return path;
}
async getEntries(): Promise<SessionTreeEntry[]> {
return [...this.entries];
async getEntries(options?: SessionEntryCursorOptions): Promise<SessionTreeEntry[]> {
const start = options?.afterEntrySeq ?? 0;
const end = options?.limit === undefined ? undefined : start + options.limit;
return this.entries.slice(start, end);
}
}
@@ -47,5 +47,5 @@ export async function getEntriesToFork(
}
effectiveLeafId = target.parentId;
}
return storage.getPathToRoot(effectiveLeafId);
return storage.getPathToRootOrCompaction(effectiveLeafId);
}
+30 -12
View File
@@ -11,8 +11,10 @@ import type {
MessageEntry,
ModelChangeEntry,
SessionContext,
SessionEntryCursorOptions,
SessionInfoEntry,
SessionMetadata,
SessionStats,
SessionStorage,
SessionTreeEntry,
ThinkingLevelChangeEntry,
@@ -67,11 +69,19 @@ export function defaultContextEntryTransform(pathEntries: readonly SessionTreeEn
const entries: SessionTreeEntry[] = [compaction];
const compactionIdx = pathEntries.findIndex((entry) => entry.type === "compaction" && entry.id === compaction.id);
let foundFirstKept = false;
for (let i = 0; i < compactionIdx; i++) {
const entry = pathEntries[i]!;
if (entry.id === compaction.firstKeptEntryId) foundFirstKept = true;
if (foundFirstKept) entries.push(entry);
if (compaction.retainedTail) {
for (let i = compactionIdx + 1; i < pathEntries.length; i++) {
entries.push(pathEntries[i]!);
}
return entries;
}
if (compaction.firstKeptEntryId) {
let foundFirstKept = false;
for (let i = 0; i < compactionIdx; i++) {
const entry = pathEntries[i]!;
if (entry.id === compaction.firstKeptEntryId) foundFirstKept = true;
if (foundFirstKept) entries.push(entry);
}
}
for (let i = compactionIdx + 1; i < pathEntries.length; i++) {
entries.push(pathEntries[i]!);
@@ -111,7 +121,10 @@ export function sessionEntryToContextMessages(
];
}
if (entry.type === "compaction") {
return [createCompactionSummaryMessage(entry.summary, entry.tokensBefore, entry.timestamp)];
return [
createCompactionSummaryMessage(entry.summary, entry.tokensBefore, entry.timestamp),
...(entry.retainedTail ?? []),
];
}
if (entry.type === "branch_summary" && entry.summary) {
return [createBranchSummaryMessage(entry.summary, entry.fromId, entry.timestamp)];
@@ -159,13 +172,13 @@ export class Session<TMetadata extends SessionMetadata = SessionMetadata> {
return this.storage.getEntry(id);
}
getEntries(): Promise<SessionTreeEntry[]> {
return this.storage.getEntries();
getEntries(options?: SessionEntryCursorOptions): Promise<SessionTreeEntry[]> {
return this.storage.getEntries(options);
}
async getBranch(fromId?: string): Promise<SessionTreeEntry[]> {
const leafId = fromId ?? (await this.storage.getLeafId());
return this.storage.getPathToRoot(leafId);
return this.storage.getPathToRootOrCompaction(leafId);
}
async buildContextEntries(options: SessionContextBuildOptions = {}): Promise<SessionTreeEntry[]> {
@@ -190,9 +203,12 @@ export class Session<TMetadata extends SessionMetadata = SessionMetadata> {
return this.storage.getLabel(id);
}
getSessionStats(): Promise<SessionStats> {
return this.storage.getSessionStats();
}
async getSessionName(): Promise<string | undefined> {
const entries = await this.storage.findEntries("session_info");
return entries[entries.length - 1]?.name?.trim() || undefined;
return this.storage.getSessionName();
}
private async appendTypedEntry<TEntry extends SessionTreeEntry>(entry: TEntry): Promise<string> {
@@ -243,11 +259,12 @@ export class Session<TMetadata extends SessionMetadata = SessionMetadata> {
async appendCompaction<T = unknown>(
summary: string,
firstKeptEntryId: string,
firstKeptEntryId: string | undefined,
tokensBefore: number,
details?: T,
fromHook?: boolean,
usage?: Usage,
retainedTail?: AgentMessage[],
): Promise<string> {
return this.appendTypedEntry({
type: "compaction",
@@ -257,6 +274,7 @@ export class Session<TMetadata extends SessionMetadata = SessionMetadata> {
summary,
firstKeptEntryId,
tokensBefore,
retainedTail,
details,
usage,
fromHook,
+22 -4
View File
@@ -370,8 +370,9 @@ export interface ActiveToolsChangeEntry extends SessionTreeEntryBase {
export interface CompactionEntry<T = unknown> extends SessionTreeEntryBase {
type: "compaction";
summary: string;
firstKeptEntryId: string;
firstKeptEntryId?: string;
tokensBefore: number;
retainedTail?: AgentMessage[];
details?: T;
usage?: Usage;
fromHook?: boolean;
@@ -436,6 +437,14 @@ export interface SessionContext {
activeToolNames: string[] | null;
}
export interface SessionStats {
messageCount: number;
cachedTokens: number;
uncachedTokens: number;
totalTokens: number;
costTotal: number;
}
export interface SessionMetadata {
id: string;
createdAt: string;
@@ -448,6 +457,11 @@ export interface JsonlSessionMetadata extends SessionMetadata {
metadata?: Record<string, unknown>;
}
export interface SessionEntryCursorOptions {
afterEntrySeq?: number;
limit?: number;
}
export interface SessionStorage<TMetadata extends SessionMetadata = SessionMetadata> {
getMetadata(): Promise<TMetadata>;
getLeafId(): Promise<string | null>;
@@ -460,8 +474,10 @@ export interface SessionStorage<TMetadata extends SessionMetadata = SessionMetad
type: TType,
): Promise<Array<Extract<SessionTreeEntry, { type: TType }>>>;
getLabel(id: string): Promise<string | undefined>;
getPathToRoot(leafId: string | null): Promise<SessionTreeEntry[]>;
getEntries(): Promise<SessionTreeEntry[]>;
getSessionName(): Promise<string | undefined>;
getSessionStats(): Promise<SessionStats>;
getPathToRootOrCompaction(leafId: string | null): Promise<SessionTreeEntry[]>;
getEntries(options?: SessionEntryCursorOptions): Promise<SessionTreeEntry[]>;
}
export type { Session } from "./session/session.ts";
@@ -753,10 +769,11 @@ export interface AbortResult {
export interface CompactResult {
summary: string;
firstKeptEntryId: string;
firstKeptEntryId?: string;
tokensBefore: number;
/** Usage from the LLM call(s) that generated this summary, if available. */
usage?: Usage;
retainedTail?: AgentMessage[];
details?: unknown;
}
@@ -776,6 +793,7 @@ export interface CompactionPreparation {
firstKeptEntryId: string;
messagesToSummarize: AgentMessage[];
turnPrefixMessages: AgentMessage[];
retainedTail: AgentMessage[];
isSplitTurn: boolean;
tokensBefore: number;
previousSummary?: string;
+1
View File
@@ -1,4 +1,5 @@
// Core Agent
export { uuidv7 } from "@earendil-works/pi-ai";
export * from "./agent.ts";
// Loop functions
export * from "./agent-loop.ts";
+36 -1
View File
@@ -91,6 +91,7 @@ function createCompactionEntry(
summary: string,
firstKeptEntryId: string,
parentId: string | null = null,
retainedTail?: AgentMessage[],
): CompactionEntry {
return {
type: "compaction",
@@ -100,6 +101,7 @@ function createCompactionEntry(
summary,
firstKeptEntryId,
tokensBefore: 1234,
retainedTail,
};
}
@@ -343,12 +345,38 @@ describe("harness compaction", () => {
const a1 = createMessageEntry(createAssistantMessage("a"), u1.id);
const u2 = createMessageEntry(createUserMessage("2"), a1.id);
const a2 = createMessageEntry(createAssistantMessage("b"), u2.id);
const compaction = createCompactionEntry("Summary of 1,a,2,b", u2.id, a2.id);
const compaction = createCompactionEntry("Summary of 1,a,2,b", u2.id, a2.id, [
createUserMessage("2"),
createAssistantMessage("b"),
]);
const u3 = createMessageEntry(createUserMessage("3"), compaction.id);
const a3 = createMessageEntry(createAssistantMessage("c"), u3.id);
const loaded = buildSessionContext([u1, a1, u2, a2, compaction, u3, a3]);
expect(loaded.messages).toHaveLength(5);
expect(loaded.messages[0]?.role).toBe("compactionSummary");
expect(loaded.messages.map((message) => message.role)).toEqual([
"compactionSummary",
"user",
"assistant",
"user",
"assistant",
]);
});
it("falls back to firstKeptEntryId when a compaction has no retained tail", () => {
const u1 = createMessageEntry(createUserMessage("1"));
const a1 = createMessageEntry(createAssistantMessage("a"), u1.id);
const u2 = createMessageEntry(createUserMessage("2"), a1.id);
const a2 = createMessageEntry(createAssistantMessage("b"), u2.id);
const compaction = createCompactionEntry("Summary of 1,a,2,b", u2.id, a2.id);
const u3 = createMessageEntry(createUserMessage("3"), compaction.id);
const loaded = buildSessionContext([u1, a1, u2, a2, compaction, u3]);
expect(loaded.messages.map((message) => message.role)).toEqual([
"compactionSummary",
"user",
"assistant",
"user",
]);
});
it("tracks model and thinking level changes in built context", () => {
@@ -374,6 +402,7 @@ describe("harness compaction", () => {
expect(preparation).toBeDefined();
expect(preparation?.previousSummary).toBe("First summary");
expect(preparation?.firstKeptEntryId).toBeTruthy();
expect(preparation?.retainedTail.length).toBeGreaterThan(0);
expect(preparation?.tokensBefore).toBe(estimateContextTokens(buildSessionContext(pathEntries).messages).tokens);
});
@@ -566,6 +595,7 @@ describe("harness compaction", () => {
firstKeptEntryId: "entry-keep",
messagesToSummarize: messages,
turnPrefixMessages: messages,
retainedTail: messages,
isSplitTurn: true,
tokensBefore: 600000,
fileOps: { read: new Set(), written: new Set(), edited: new Set() },
@@ -583,6 +613,7 @@ describe("harness compaction", () => {
firstKeptEntryId: "entry-keep",
messagesToSummarize: messages,
turnPrefixMessages: [],
retainedTail: messages,
isSplitTurn: false,
tokensBefore: 100,
fileOps: { read: new Set(), written: new Set(), edited: new Set() },
@@ -619,6 +650,7 @@ describe("harness compaction", () => {
turnPrefixMessages: messages,
isSplitTurn: true,
tokensBefore: 100,
retainedTail: messages,
fileOps: { read: new Set(), written: new Set(), edited: new Set() },
settings: { enabled: true, reserveTokens: 2000, keepRecentTokens: 20 },
};
@@ -642,6 +674,7 @@ describe("harness compaction", () => {
firstKeptEntryId: "entry-keep",
messagesToSummarize: [],
turnPrefixMessages: messages,
retainedTail: messages,
isSplitTurn: true,
tokensBefore: 100,
fileOps: { read: new Set(), written: new Set(), edited: new Set() },
@@ -659,6 +692,7 @@ describe("harness compaction", () => {
firstKeptEntryId: "entry-keep",
messagesToSummarize: [],
turnPrefixMessages: messages,
retainedTail: messages,
isSplitTurn: true,
tokensBefore: 100,
fileOps: { read: new Set(), written: new Set(), edited: new Set() },
@@ -697,6 +731,7 @@ describe("harness compaction", () => {
expect(result.summary.length).toBeGreaterThan(0);
expect(result.firstKeptEntryId).toBeTruthy();
expect(result.usage?.totalTokens).toBeGreaterThan(0);
expect(result.retainedTail?.length).toBeGreaterThan(0);
expect(result.details).toBeDefined();
});
});
+10 -1
View File
@@ -68,11 +68,20 @@ async function runSessionSuite(
await session.appendMessage(createAssistantMessage("two"));
const user2 = await session.appendMessage(createUserMessage("three"));
await session.appendMessage(createAssistantMessage("four"));
await session.appendCompaction("summary", user2, 1234);
await session.appendCompaction("summary", user2, 1234, undefined, undefined, undefined, [
createUserMessage("three"),
createAssistantMessage("four"),
]);
await session.appendMessage(createUserMessage("five"));
const context = await session.buildContext();
expect(context.messages[0]?.role).toBe("compactionSummary");
expect(context.messages).toHaveLength(4);
expect(context.messages.map((message) => message.role)).toEqual([
"compactionSummary",
"user",
"assistant",
"user",
]);
});
it("supports moving with branch summary entries in context", async () => {
@@ -0,0 +1,441 @@
import { mkdtempSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import {
applyMigrations,
createNodeSqliteFactory,
type SqliteDatabase,
type SqliteDatabaseFactory,
type SqliteRunResult,
type SqliteSessionMetadata,
SqliteSessionRepo,
SqliteSessionStorage,
type SqliteStatement,
} from "../../../storage/sqlite-node/src/index.ts";
import { NodeExecutionEnv } from "../../src/harness/env/nodejs.ts";
import { createAssistantMessage, createUserMessage } from "./session-test-utils.ts";
function createTempDir(): string {
return mkdtempSync(join(tmpdir(), "pi-agent-sqlite-"));
}
class ThrowingStatement implements SqliteStatement {
private readonly onRun: () => Promise<SqliteRunResult>;
constructor(onRun: () => Promise<SqliteRunResult>) {
this.onRun = onRun;
}
async run(..._params: unknown[]): Promise<SqliteRunResult> {
return this.onRun();
}
async get<TRow extends object>(..._params: unknown[]): Promise<TRow | undefined> {
return undefined;
}
async all<TRow extends object>(..._params: unknown[]): Promise<TRow[]> {
return [];
}
}
class CountingDatabase implements SqliteDatabase {
closeCount = 0;
private readonly statementFactory: (sql: string) => SqliteStatement;
constructor(statementFactory: (sql: string) => SqliteStatement) {
this.statementFactory = statementFactory;
}
async exec(_sql: string): Promise<void> {}
prepare(sql: string): SqliteStatement {
return this.statementFactory(sql);
}
async transaction<T>(fn: () => Promise<T>): Promise<T> {
return fn();
}
async close(): Promise<void> {
this.closeCount += 1;
}
}
describe("SQLite migrations", () => {
it("applies file-based migrations and records them", async () => {
const root = createTempDir();
const databasePath = join(root, "sessions.sqlite");
const env = new NodeExecutionEnv({ cwd: root });
const sqlite = createNodeSqliteFactory();
const repo = new SqliteSessionRepo({ env, sqlite, databasePath });
await repo.create({ cwd: root, id: "session-1" });
const db = await sqlite.open(databasePath);
try {
const rows = await db.prepare("SELECT id FROM migrations ORDER BY id").all<{ id: string }>();
expect(rows.map((row) => row.id)).toEqual(["001_initial.sql"]);
const tables = await db
.prepare("SELECT name, sql FROM sqlite_master WHERE type = 'table' ORDER BY name")
.all<{ name: string; sql: string | null }>();
expect(tables.map((row) => row.name)).toEqual(
expect.arrayContaining([
"migrations",
"sessions",
"session_entries",
"session_sequences",
"branch_entries",
"session_materialized",
"entry_materialized",
]),
);
const sessionColumns = await db.prepare("PRAGMA table_info(sessions)").all<{ name: string }>();
expect(sessionColumns.map((column) => column.name)).toContain("active_leaf_id");
for (const tableName of [
"sessions",
"session_sequences",
"branch_entries",
"session_materialized",
"entry_materialized",
]) {
const table = tables.find((row) => row.name === tableName);
expect(table?.sql).toContain("WITHOUT ROWID");
}
} finally {
await db.close();
}
});
it("persists session metadata through create, list, open, and fork", async () => {
const root = createTempDir();
const databasePath = join(root, "sessions.sqlite");
const env = new NodeExecutionEnv({ cwd: root });
const repo = new SqliteSessionRepo({ env, sqlite: createNodeSqliteFactory(), databasePath });
const source = await repo.create({
cwd: root,
id: "session-1",
metadata: { profile: "reviewer" },
});
const sourceMetadata = await source.getMetadata();
expect(sourceMetadata.metadata).toEqual({ profile: "reviewer" });
expect((await repo.list({ cwd: root })).map((listed) => listed.metadata)).toEqual([{ profile: "reviewer" }]);
expect((await (await repo.open(sourceMetadata)).getMetadata()).metadata).toEqual({ profile: "reviewer" });
const fork = await repo.fork(sourceMetadata, { cwd: root, id: "session-2" });
expect((await fork.getMetadata()).metadata).toEqual({ profile: "reviewer" });
const overridden = await repo.fork(sourceMetadata, {
cwd: root,
id: "session-3",
metadata: { profile: "writer" },
});
expect((await overridden.getMetadata()).metadata).toEqual({ profile: "writer" });
});
it("materializes active leaf id in sessions transactionally", async () => {
const root = createTempDir();
const databasePath = join(root, "sessions.sqlite");
const env = new NodeExecutionEnv({ cwd: root });
const sqlite = createNodeSqliteFactory();
const repo = new SqliteSessionRepo({ env, sqlite, databasePath });
const session = await repo.create({ cwd: root, id: "session-1" });
const rootId = await session.appendMessage(createUserMessage("root"));
const childId = await session.appendMessage(createAssistantMessage("child"));
await session.getStorage().setLeafId(rootId);
const db = await sqlite.open(databasePath);
try {
const row = await db
.prepare("SELECT active_leaf_id FROM sessions WHERE id = ?")
.get<{ active_leaf_id: string | null }>("session-1");
expect(row?.active_leaf_id).toBe(rootId);
const latestBranchRow = await db
.prepare(
"SELECT branch_id, entry_id, entry_seq FROM branch_entries WHERE session_id = ? ORDER BY entry_seq DESC LIMIT 1",
)
.get<{ branch_id: string; entry_id: string; entry_seq: number }>("session-1");
const latestSessionEntry = await db
.prepare("SELECT id, type FROM session_entries WHERE session_id = ? ORDER BY entry_seq DESC LIMIT 1")
.get<{ id: string; type: string }>("session-1");
expect(latestSessionEntry?.type).toBe("leaf");
expect(latestBranchRow?.entry_id).toBe(latestSessionEntry?.id);
} finally {
await db.close();
}
const reopened = await repo.open(await session.getMetadata());
expect(await reopened.getLeafId()).toBe(rootId);
expect(childId).not.toBe(rootId);
});
it("materializes a new branch when appending from a parent with an existing child", async () => {
const root = createTempDir();
const databasePath = join(root, "sessions.sqlite");
const env = new NodeExecutionEnv({ cwd: root });
const sqlite = createNodeSqliteFactory();
const repo = new SqliteSessionRepo({ env, sqlite, databasePath });
const session = await repo.create({ cwd: root, id: "session-1" });
const rootId = await session.appendMessage(createUserMessage("root"));
const firstChildId = await session.appendMessage(createAssistantMessage("first child"));
await session.getStorage().setLeafId(rootId);
const secondChildId = await session.appendMessage(createAssistantMessage("second child"));
const db = await sqlite.open(databasePath);
try {
const branchRows = await db
.prepare(
"SELECT branch_id, entry_id, entry_seq FROM branch_entries WHERE session_id = ? ORDER BY branch_id, entry_seq",
)
.all<{ branch_id: string; entry_id: string; entry_seq: number }>("session-1");
const branchIds = [...new Set(branchRows.map((row) => row.branch_id))];
expect(branchIds).toHaveLength(3);
expect(branchRows.filter((row) => row.entry_id === rootId)).toHaveLength(3);
expect(branchRows.filter((row) => row.entry_id === firstChildId)).toHaveLength(1);
expect(branchRows.filter((row) => row.entry_id === secondChildId)).toHaveLength(1);
} finally {
await db.close();
}
});
it("reopens using branch materialization and session summary state", async () => {
const root = createTempDir();
const databasePath = join(root, "sessions.sqlite");
const env = new NodeExecutionEnv({ cwd: root });
const repo = new SqliteSessionRepo({ env, sqlite: createNodeSqliteFactory(), databasePath });
const session = await repo.create({ cwd: root, id: "session-1" });
const rootId = await session.appendMessage(createUserMessage("root"));
await session.appendMessage(createAssistantMessage("first child"));
await session.appendSessionName(" Reopened Session ");
await session.getStorage().setLeafId(rootId);
await session.appendMessage(createAssistantMessage("branched child"));
const reopened = await repo.open(await session.getMetadata());
expect(await reopened.getSessionName()).toBe("Reopened Session");
expect((await reopened.buildContext()).messages.map((message) => message.role)).toEqual(["user", "assistant"]);
expect((await reopened.buildContext()).messages.at(-1)).toMatchObject({
content: [{ type: "text", text: "branched child" }],
});
});
it("pages entries by entry_seq cursor", async () => {
const root = createTempDir();
const databasePath = join(root, "sessions.sqlite");
const env = new NodeExecutionEnv({ cwd: root });
const repo = new SqliteSessionRepo({ env, sqlite: createNodeSqliteFactory(), databasePath });
const session = await repo.create({ cwd: root, id: "session-1" });
await session.appendMessage(createUserMessage("one"));
await session.appendMessage(createAssistantMessage("two"));
await session.appendMessage(createUserMessage("three"));
expect((await session.getEntries({ limit: 2 })).map((entry) => entry.type)).toEqual(["message", "message"]);
expect((await session.getEntries({ afterEntrySeq: 2, limit: 2 })).map((entry) => entry.type)).toEqual([
"message",
"message",
]);
});
it("closes the database when create fails after openDatabase succeeds", async () => {
const root = createTempDir();
const db = new CountingDatabase((sql) => {
if (sql.startsWith("INSERT INTO sessions")) {
return new ThrowingStatement(async () => {
throw new Error("insert failed");
});
}
return new ThrowingStatement(async () => ({ changes: 1 }));
});
const sqlite: SqliteDatabaseFactory = {
open: async () => db,
};
const env = new NodeExecutionEnv({ cwd: root });
const repo = new SqliteSessionRepo({ env, sqlite, databasePath: join(root, "sessions.sqlite") });
await expect(repo.create({ cwd: root, id: "session-1" })).rejects.toThrow("insert failed");
expect(db.closeCount).toBe(1);
});
it("closes the database when open fails after openDatabase succeeds", async () => {
const root = createTempDir();
const db = new CountingDatabase((sql) => {
if (sql.includes("FROM sessions WHERE id = ?")) {
return new ThrowingStatement(async () => ({ changes: 0 }));
}
return new ThrowingStatement(async () => ({ changes: 1 }));
});
const sqlite: SqliteDatabaseFactory = {
open: async () => db,
};
const env = new NodeExecutionEnv({ cwd: root });
const repo = new SqliteSessionRepo({ env, sqlite, databasePath: join(root, "sessions.sqlite") });
const metadata: SqliteSessionMetadata = {
id: "missing",
createdAt: new Date().toISOString(),
cwd: root,
path: join(root, "sessions.sqlite"),
};
writeFileSync(metadata.path, "");
await expect(repo.open(metadata)).rejects.toThrow("Session not found: missing");
expect(db.closeCount).toBe(1);
});
it("closes the source storage after fork reads its entries", async () => {
const root = createTempDir();
const databasePath = join(root, "sessions.sqlite");
const env = new NodeExecutionEnv({ cwd: root });
const repo = new SqliteSessionRepo({ env, sqlite: createNodeSqliteFactory(), databasePath });
let cleanupCount = 0;
const sourceStorage = {
async getEntries() {
return [];
},
async getPathToRootOrCompaction() {
return [];
},
async cleanup() {
cleanupCount += 1;
},
} as const;
const originalOpen = repo.open.bind(repo);
repo.open = async () =>
({
getStorage() {
return sourceStorage;
},
}) as never;
try {
await repo.fork(
{
id: "session-1",
createdAt: new Date().toISOString(),
cwd: root,
path: databasePath,
},
{ cwd: root, id: "session-2" },
);
} finally {
repo.open = originalOpen;
}
expect(cleanupCount).toBe(1);
});
it("restores in-memory state when appendEntry fails after mutating caches", async () => {
const root = createTempDir();
const databasePath = join(root, "sessions.sqlite");
const sqlite = createNodeSqliteFactory();
const db = await sqlite.open(databasePath);
await applyMigrations(db);
const storage = await SqliteSessionStorage.create(db, databasePath, {
cwd: root,
sessionId: "session-1",
});
const originalPrepare = db.prepare.bind(db);
db.prepare = (sql: string) => {
if (sql.startsWith("UPDATE sessions SET active_leaf_id = ?")) {
return new ThrowingStatement(async () => {
throw new Error("active leaf update failed");
});
}
return originalPrepare(sql);
};
await expect(
storage.appendEntry({
type: "message",
id: "root",
parentId: null,
timestamp: new Date().toISOString(),
message: createUserMessage("root"),
}),
).rejects.toMatchObject({ code: "storage" });
expect(await storage.getLeafId()).toBeNull();
expect(await storage.getEntry("root")).toBeUndefined();
expect(await storage.getEntries()).toEqual([]);
await db.close();
});
it("materializes session summary fields transactionally", async () => {
const root = createTempDir();
const databasePath = join(root, "sessions.sqlite");
const env = new NodeExecutionEnv({ cwd: root });
const sqlite = createNodeSqliteFactory();
const repo = new SqliteSessionRepo({ env, sqlite, databasePath });
const session = await repo.create({ cwd: root, id: "session-1" });
const userId = await session.appendMessage(createUserMessage("one"));
await session.appendThinkingLevelChange("high");
await session.appendModelChange("anthropic", "claude-sonnet-4-5");
const assistant = {
...createAssistantMessage("two"),
provider: "anthropic",
model: "claude-sonnet-4-5",
usage: {
input: 100,
output: 25,
cacheRead: 40,
cacheWrite: 10,
totalTokens: 175,
cost: { input: 0.1, output: 0.2, cacheRead: 0.03, cacheWrite: 0.04, total: 0.37 },
},
};
await session.appendMessage(assistant);
await session.appendCompaction("summary", userId, 200, undefined, false, {
input: 1,
output: 2,
cacheRead: 3,
cacheWrite: 4,
totalTokens: 10,
cost: { input: 0.01, output: 0.02, cacheRead: 0.03, cacheWrite: 0.04, total: 0.1 },
});
await session.moveTo(userId, {
summary: "branch summary",
usage: {
input: 5,
output: 6,
cacheRead: 7,
cacheWrite: 8,
totalTokens: 26,
cost: { input: 0.05, output: 0.06, cacheRead: 0.07, cacheWrite: 0.08, total: 0.26 },
},
});
await session.appendSessionName(" My Session ");
await session.appendLabel(userId, "checkpoint");
const db = await sqlite.open(databasePath);
try {
const row = await db.prepare("SELECT session_id, payload FROM session_materialized WHERE session_id = ?").get<{
session_id: string;
payload: string;
}>("session-1");
expect(row).toBeDefined();
expect(row?.session_id).toBe("session-1");
expect(JSON.parse(row?.payload ?? "null")).toMatchObject({
name: "My Session",
messageCount: 2,
cachedTokens: 50,
uncachedTokens: 128,
totalTokens: 211,
costTotal: 0.73,
currentModel: { provider: "anthropic", modelId: "claude-sonnet-4-5" },
currentThinkingLevel: "high",
});
const entryRows = await db
.prepare(
"SELECT session_id, entry_seq, type, payload FROM entry_materialized WHERE session_id = ? ORDER BY entry_seq, type",
)
.all<{
session_id: string;
entry_seq: number;
type: string;
payload: string;
}>("session-1");
expect(
entryRows.some((entryRow) => entryRow.type === "label" && JSON.parse(entryRow.payload).targetId === userId),
).toBe(true);
expect(entryRows.some((entryRow) => entryRow.type === "thinking")).toBe(false);
expect(entryRows.some((entryRow) => entryRow.type === "model")).toBe(false);
} finally {
await db.close();
}
});
});
@@ -0,0 +1,21 @@
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { createNodeSqliteFactory } from "../../../storage/sqlite-node/src/index.ts";
import { createTempDir } from "./session-test-utils.ts";
describe("sqlite-node adapter", () => {
it("supports node:sqlite-style named parameters", async () => {
const root = createTempDir();
const databasePath = join(root, "adapter.sqlite");
const sqlite = createNodeSqliteFactory();
const db = await sqlite.open(databasePath);
try {
await db.exec("CREATE TABLE items (id INTEGER PRIMARY KEY, text TEXT NOT NULL)");
await db.prepare("INSERT INTO items (id, text) VALUES ($id, $text)").run({ $id: 1, $text: "hello" });
const row = await db.prepare("SELECT text FROM items WHERE id = $id").get<{ text: string }>({ $id: 1 });
expect(row).toEqual({ text: "hello" });
} finally {
await db.close();
}
});
});
+169 -6
View File
@@ -4,7 +4,13 @@ import { describe, expect, it } from "vitest";
import { NodeExecutionEnv } from "../../src/harness/env/nodejs.ts";
import { JsonlSessionStorage, loadJsonlSessionMetadata } from "../../src/harness/session/jsonl-storage.ts";
import { InMemorySessionStorage } from "../../src/harness/session/memory-storage.ts";
import { type MessageEntry, ok, type SessionMetadata } from "../../src/harness/types.ts";
import {
type BranchSummaryEntry,
type CompactionEntry,
type MessageEntry,
ok,
type SessionMetadata,
} from "../../src/harness/types.ts";
import { createAssistantMessage, createTempDir, createUserMessage } from "./session-test-utils.ts";
describe("InMemorySessionStorage", () => {
@@ -80,7 +86,74 @@ describe("InMemorySessionStorage", () => {
expect(await storage.getLabel("entry-1")).toBeUndefined();
});
it("walks paths to root", async () => {
it("includes summary-entry usage in session stats", async () => {
const assistant: MessageEntry = {
type: "message",
id: "assistant",
parentId: null,
timestamp: "2026-01-01T00:00:00.000Z",
message: {
role: "assistant",
content: [{ type: "text", text: "reply" }],
api: "anthropic-messages",
provider: "anthropic",
model: "claude-sonnet-4-5",
usage: {
input: 10,
output: 20,
cacheRead: 30,
cacheWrite: 40,
totalTokens: 100,
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
},
stopReason: "stop",
timestamp: 0,
},
};
const compaction: CompactionEntry = {
type: "compaction",
id: "compaction",
parentId: "assistant",
timestamp: "2026-01-01T00:00:01.000Z",
summary: "summary",
firstKeptEntryId: "assistant",
tokensBefore: 1234,
usage: {
input: 1,
output: 2,
cacheRead: 3,
cacheWrite: 4,
totalTokens: 10,
cost: { input: 0.01, output: 0.02, cacheRead: 0.03, cacheWrite: 0.04, total: 0.1 },
},
};
const branchSummary: BranchSummaryEntry = {
type: "branch_summary",
id: "branch-summary",
parentId: "compaction",
timestamp: "2026-01-01T00:00:02.000Z",
fromId: "assistant",
summary: "branch",
usage: {
input: 5,
output: 6,
cacheRead: 7,
cacheWrite: 8,
totalTokens: 26,
cost: { input: 0.05, output: 0.06, cacheRead: 0.07, cacheWrite: 0.08, total: 0.26 },
},
};
const storage = new InMemorySessionStorage({ entries: [assistant, compaction, branchSummary] });
expect(await storage.getSessionStats()).toEqual({
messageCount: 1,
cachedTokens: 40,
uncachedTokens: 68,
totalTokens: 136,
costTotal: 1.36,
});
});
it("walks paths to root or retained-tail compaction", async () => {
const root: MessageEntry = {
type: "message",
id: "root",
@@ -94,9 +167,29 @@ describe("InMemorySessionStorage", () => {
parentId: "root",
message: createAssistantMessage("child"),
};
const storage = new InMemorySessionStorage({ entries: [root, child] });
expect((await storage.getPathToRoot("child")).map((entry) => entry.id)).toEqual(["root", "child"]);
expect(await storage.getPathToRoot(null)).toEqual([]);
const compaction: CompactionEntry = {
type: "compaction",
id: "compaction",
parentId: "child",
timestamp: "2026-01-01T00:00:01.000Z",
summary: "summary",
firstKeptEntryId: "child",
tokensBefore: 1234,
retainedTail: [createAssistantMessage("child")],
};
const afterCompaction: MessageEntry = {
...root,
id: "after-compaction",
parentId: "compaction",
message: createUserMessage("after"),
};
const storage = new InMemorySessionStorage({ entries: [root, child, compaction, afterCompaction] });
expect((await storage.getPathToRootOrCompaction("child")).map((entry) => entry.id)).toEqual(["root", "child"]);
expect((await storage.getPathToRootOrCompaction("after-compaction")).map((entry) => entry.id)).toEqual([
"compaction",
"after-compaction",
]);
expect(await storage.getPathToRootOrCompaction(null)).toEqual([]);
});
});
@@ -255,7 +348,7 @@ describe("JsonlSessionStorage", () => {
const reloaded = await JsonlSessionStorage.open(env, filePath);
expect(await reloaded.getLeafId()).toBe("root");
expect((await reloaded.getEntries()).at(-1)).toMatchObject({ type: "leaf", targetId: "root" });
expect((await loaded.getPathToRoot("child")).map((entry) => entry.id)).toEqual(["root", "child"]);
expect((await loaded.getPathToRootOrCompaction("child")).map((entry) => entry.id)).toEqual(["root", "child"]);
});
it("finds entries by type", async () => {
@@ -309,6 +402,76 @@ describe("JsonlSessionStorage", () => {
expect(await loaded.getLabel("entry-1")).toBeUndefined();
});
it("includes summary-entry usage in session stats", async () => {
const dir = createTempDir();
const env = new NodeExecutionEnv({ cwd: dir });
const filePath = join(dir, "session.jsonl");
const storage = await JsonlSessionStorage.create(env, filePath, { cwd: dir, sessionId: "session-1" });
await storage.appendEntry({
type: "message",
id: "assistant",
parentId: null,
timestamp: "2026-01-01T00:00:00.000Z",
message: {
role: "assistant",
content: [{ type: "text", text: "reply" }],
api: "anthropic-messages",
provider: "anthropic",
model: "claude-sonnet-4-5",
usage: {
input: 10,
output: 20,
cacheRead: 30,
cacheWrite: 40,
totalTokens: 100,
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
},
stopReason: "stop",
timestamp: 0,
},
});
await storage.appendEntry({
type: "compaction",
id: "compaction",
parentId: "assistant",
timestamp: "2026-01-01T00:00:01.000Z",
summary: "summary",
firstKeptEntryId: "assistant",
tokensBefore: 1234,
usage: {
input: 1,
output: 2,
cacheRead: 3,
cacheWrite: 4,
totalTokens: 10,
cost: { input: 0.01, output: 0.02, cacheRead: 0.03, cacheWrite: 0.04, total: 0.1 },
},
});
await storage.appendEntry({
type: "branch_summary",
id: "branch-summary",
parentId: "compaction",
timestamp: "2026-01-01T00:00:02.000Z",
fromId: "assistant",
summary: "branch",
usage: {
input: 5,
output: 6,
cacheRead: 7,
cacheWrite: 8,
totalTokens: 26,
cost: { input: 0.05, output: 0.06, cacheRead: 0.07, cacheWrite: 0.08, total: 0.26 },
},
});
expect(await storage.getSessionStats()).toEqual({
messageCount: 1,
cachedTokens: 40,
uncachedTokens: 68,
totalTokens: 136,
costTotal: 1.36,
});
});
it("reads session metadata through the line-reading filesystem operation", async () => {
const dir = createTempDir();
const filePath = join(dir, "session.jsonl");
+2
View File
@@ -3,6 +3,7 @@ import { defineConfig } from "vitest/config";
const aiSrcIndex = fileURLToPath(new URL("../ai/src/index.ts", import.meta.url));
const aiSrcCompat = fileURLToPath(new URL("../ai/src/compat.ts", import.meta.url));
const agentSrcIndex = fileURLToPath(new URL("../agent/src/index.ts", import.meta.url));
export default defineConfig({
test: {
@@ -21,6 +22,7 @@ export default defineConfig({
},
resolve: {
alias: [
{ find: /^@earendil-works\/pi-agent-core$/, replacement: agentSrcIndex },
{ find: /^@earendil-works\/pi-ai$/, replacement: aiSrcIndex },
{ find: /^@earendil-works\/pi-ai\/compat$/, replacement: aiSrcCompat },
],