feat: sqlite session storage (#6594)

This PR:

- Adds retainedTail to compaction entries in the new agent harness so we don't have to walk up the tree for the 2000 tokens before compaction,
- Changes getPathToRoot to getPathToRootOrCompaction to only load until last compaction, as unnecessary to access all nodes where it is called,
- Adds a SQLite storage backend, in a separate packages/session-backend-sqlite, with a migration system and schemas as per on-site discussions: sessions to match session header messages (except for metadata, which I couldn't understand what it's used for or where it gets written, so I omitted it), session_entries for shared entry types as columns plus payload as a json for what remains, session_sequences to represent the append-only, serialized nature of the jsonl files, branch_entries to attribute nodes to branches (relationship one-to-many), and session_materialized with the session info (see /session in TUI) to act as a "cache" or quick-access for costs, message count, token info, labels, session name, and model-thinking-level config (e.g. for fast resume).
- This is compatible with the new agent harness Session abstraction.

packages/agent/test/harness/compaction.test.ts

@@ -91,6 +91,7 @@ function createCompactionEntry(
 	summary: string,
 	firstKeptEntryId: string,
 	parentId: string | null = null,
+	retainedTail?: AgentMessage[],
 ): CompactionEntry {
 	return {
 		type: "compaction",
@@ -100,6 +101,7 @@ function createCompactionEntry(
 		summary,
 		firstKeptEntryId,
 		tokensBefore: 1234,
+		retainedTail,
 	};
 }
 
@@ -343,12 +345,38 @@ describe("harness compaction", () => {
 		const a1 = createMessageEntry(createAssistantMessage("a"), u1.id);
 		const u2 = createMessageEntry(createUserMessage("2"), a1.id);
 		const a2 = createMessageEntry(createAssistantMessage("b"), u2.id);
-		const compaction = createCompactionEntry("Summary of 1,a,2,b", u2.id, a2.id);
+		const compaction = createCompactionEntry("Summary of 1,a,2,b", u2.id, a2.id, [
+			createUserMessage("2"),
+			createAssistantMessage("b"),
+		]);
 		const u3 = createMessageEntry(createUserMessage("3"), compaction.id);
 		const a3 = createMessageEntry(createAssistantMessage("c"), u3.id);
 		const loaded = buildSessionContext([u1, a1, u2, a2, compaction, u3, a3]);
 		expect(loaded.messages).toHaveLength(5);
 		expect(loaded.messages[0]?.role).toBe("compactionSummary");
+		expect(loaded.messages.map((message) => message.role)).toEqual([
+			"compactionSummary",
+			"user",
+			"assistant",
+			"user",
+			"assistant",
+		]);
+	});
+
+	it("falls back to firstKeptEntryId when a compaction has no retained tail", () => {
+		const u1 = createMessageEntry(createUserMessage("1"));
+		const a1 = createMessageEntry(createAssistantMessage("a"), u1.id);
+		const u2 = createMessageEntry(createUserMessage("2"), a1.id);
+		const a2 = createMessageEntry(createAssistantMessage("b"), u2.id);
+		const compaction = createCompactionEntry("Summary of 1,a,2,b", u2.id, a2.id);
+		const u3 = createMessageEntry(createUserMessage("3"), compaction.id);
+		const loaded = buildSessionContext([u1, a1, u2, a2, compaction, u3]);
+		expect(loaded.messages.map((message) => message.role)).toEqual([
+			"compactionSummary",
+			"user",
+			"assistant",
+			"user",
+		]);
 	});
 
 	it("tracks model and thinking level changes in built context", () => {
@@ -374,6 +402,7 @@ describe("harness compaction", () => {
 		expect(preparation).toBeDefined();
 		expect(preparation?.previousSummary).toBe("First summary");
 		expect(preparation?.firstKeptEntryId).toBeTruthy();
+		expect(preparation?.retainedTail.length).toBeGreaterThan(0);
 		expect(preparation?.tokensBefore).toBe(estimateContextTokens(buildSessionContext(pathEntries).messages).tokens);
 	});
 
@@ -566,6 +595,7 @@ describe("harness compaction", () => {
 			firstKeptEntryId: "entry-keep",
 			messagesToSummarize: messages,
 			turnPrefixMessages: messages,
+			retainedTail: messages,
 			isSplitTurn: true,
 			tokensBefore: 600000,
 			fileOps: { read: new Set(), written: new Set(), edited: new Set() },
@@ -583,6 +613,7 @@ describe("harness compaction", () => {
 			firstKeptEntryId: "entry-keep",
 			messagesToSummarize: messages,
 			turnPrefixMessages: [],
+			retainedTail: messages,
 			isSplitTurn: false,
 			tokensBefore: 100,
 			fileOps: { read: new Set(), written: new Set(), edited: new Set() },
@@ -619,6 +650,7 @@ describe("harness compaction", () => {
 			turnPrefixMessages: messages,
 			isSplitTurn: true,
 			tokensBefore: 100,
+			retainedTail: messages,
 			fileOps: { read: new Set(), written: new Set(), edited: new Set() },
 			settings: { enabled: true, reserveTokens: 2000, keepRecentTokens: 20 },
 		};
@@ -642,6 +674,7 @@ describe("harness compaction", () => {
 			firstKeptEntryId: "entry-keep",
 			messagesToSummarize: [],
 			turnPrefixMessages: messages,
+			retainedTail: messages,
 			isSplitTurn: true,
 			tokensBefore: 100,
 			fileOps: { read: new Set(), written: new Set(), edited: new Set() },
@@ -659,6 +692,7 @@ describe("harness compaction", () => {
 			firstKeptEntryId: "entry-keep",
 			messagesToSummarize: [],
 			turnPrefixMessages: messages,
+			retainedTail: messages,
 			isSplitTurn: true,
 			tokensBefore: 100,
 			fileOps: { read: new Set(), written: new Set(), edited: new Set() },
@@ -697,6 +731,7 @@ describe("harness compaction", () => {
 		expect(result.summary.length).toBeGreaterThan(0);
 		expect(result.firstKeptEntryId).toBeTruthy();
 		expect(result.usage?.totalTokens).toBeGreaterThan(0);
+		expect(result.retainedTail?.length).toBeGreaterThan(0);
 		expect(result.details).toBeDefined();
 	});
 });