feat: add real tmux session manager

packages/run-engine/src/engine.test.ts

@@ -15,7 +15,7 @@ import { join, resolve } from "node:path";
 import { and, eq, inArray } from "drizzle-orm";
 import { afterEach, describe, expect, it } from "vitest";
 
-import { loadPersonaFiles, loadTemplateFiles, validateArtifact } from "@devflow/core";
+import { DevflowError, loadPersonaFiles, loadTemplateFiles, validateArtifact } from "@devflow/core";
 import {
   type DbClient,
   agentPersonas,
@@ -30,7 +30,14 @@ import {
   tuiSessions,
   workflowTemplates,
 } from "@devflow/db";
-import { FakeSessionAdapter, type SessionAdapter, SessionManager } from "@devflow/session";
+import {
+  FakeSessionAdapter,
+  type SessionAdapter,
+  type SessionHandle,
+  SessionManager,
+  type SessionRuntime,
+  type TranscriptChunk,
+} from "@devflow/session";
 
 import { DbRunEngine, sweepM4ProcessRestart } from "./engine.js";
 
@@ -94,6 +101,110 @@ class DisposeCountingFakeAdapter extends FakeSessionAdapter {
   }
 }
 
+class DisposeFailsFakeAdapter extends FakeSessionAdapter {
+  override async dispose(handle: Parameters<FakeSessionAdapter["dispose"]>[0]): Promise<void> {
+    throw new DevflowError("dispose failed", {
+      class: "recoverable",
+      code: "pane_briefly_unresponsive",
+      recoveryHint: `session=${handle.sessionId}`,
+    });
+  }
+}
+
+class CaptureOrderingFakeAdapter extends FakeSessionAdapter {
+  events: string[] = [];
+  failCapture = false;
+
+  override async *capture(
+    handle: Parameters<FakeSessionAdapter["capture"]>[0],
+    fromSeq: bigint,
+  ): AsyncIterable<TranscriptChunk> {
+    this.events.push("capture");
+    if (this.failCapture) {
+      throw new DevflowError("transcript capture failed", {
+        class: "recoverable",
+        code: "pane_briefly_unresponsive",
+      });
+    }
+    yield* super.capture(handle, fromSeq);
+  }
+
+  override async dispose(handle: Parameters<FakeSessionAdapter["dispose"]>[0]): Promise<void> {
+    this.events.push("dispose");
+    await super.dispose(handle);
+  }
+}
+
+class CaptureFailsAfterDisposeFakeAdapter extends FakeSessionAdapter {
+  readonly disposedSessionIds = new Set<string>();
+  readonly events: string[] = [];
+
+  override async *capture(
+    handle: Parameters<FakeSessionAdapter["capture"]>[0],
+    fromSeq: bigint,
+  ): AsyncIterable<TranscriptChunk> {
+    this.events.push("capture");
+    if (this.disposedSessionIds.has(handle.sessionId)) {
+      throw new DevflowError("tmux session already disposed", {
+        class: "recoverable",
+        code: "pane_briefly_unresponsive",
+        recoveryHint: `session=${handle.sessionId}`,
+      });
+    }
+    yield* super.capture(handle, fromSeq);
+  }
+
+  override async dispose(handle: Parameters<FakeSessionAdapter["dispose"]>[0]): Promise<void> {
+    this.events.push("dispose");
+    this.disposedSessionIds.add(handle.sessionId);
+    await super.dispose(handle);
+  }
+}
+
+class TerminalHandleRecordingRuntime implements SessionRuntime {
+  readonly adapter = new FakeSessionAdapter({ writeDelayMs: 0 });
+  readonly captureHandles: SessionHandle[] = [];
+  readonly disposeHandles: SessionHandle[] = [];
+
+  trackOperation<T>(operation: Promise<T>): Promise<T> {
+    return operation;
+  }
+
+  start(...args: Parameters<SessionRuntime["start"]>): ReturnType<SessionRuntime["start"]> {
+    return this.adapter.start(...args);
+  }
+
+  sendPrompt(
+    ...args: Parameters<SessionRuntime["sendPrompt"]>
+  ): ReturnType<SessionRuntime["sendPrompt"]> {
+    return this.adapter.sendPrompt(...args);
+  }
+
+  probe(...args: Parameters<SessionRuntime["probe"]>): ReturnType<SessionRuntime["probe"]> {
+    return this.adapter.probe(...args);
+  }
+
+  resume(...args: Parameters<SessionRuntime["resume"]>): ReturnType<SessionRuntime["resume"]> {
+    return this.adapter.resume(...args);
+  }
+
+  rebootstrap(
+    ...args: Parameters<SessionRuntime["rebootstrap"]>
+  ): ReturnType<SessionRuntime["rebootstrap"]> {
+    return this.adapter.rebootstrap(...args);
+  }
+
+  async *capture(handle: SessionHandle, fromSeq: bigint): ReturnType<SessionRuntime["capture"]> {
+    this.captureHandles.push(handle);
+    yield* this.adapter.capture(handle, fromSeq);
+  }
+
+  async dispose(handle: SessionHandle): Promise<void> {
+    this.disposeHandles.push(handle);
+    await this.adapter.dispose(handle);
+  }
+}
+
 describe("DbRunEngine", () => {
   let client: DbClient | undefined;
   const runIds: string[] = [];
@@ -857,6 +968,43 @@ describe("DbRunEngine", () => {
     expect(sessions.every((session) => session.state === "FAILED_NEEDS_HUMAN")).toBe(true);
   });
 
+  it("repairs final reports when direct advance sees a terminalized fatal phase", async () => {
+    client = createDbClient(databaseUrl);
+    await seedDevelopmentRegistry(client.db);
+    const workspaceRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-engine-workspace-")));
+    const repoPath = createGitRepo();
+    tempRoots.push(workspaceRoot, repoPath);
+    const engine = new DbRunEngine({
+      db: client.db,
+      sessions: sessionRuntime(client.db, new FakeSessionAdapter({ writeDelayMs: 0 })),
+      workspaceRoot,
+      maxConcurrentRuns: 100,
+      wait: { pollIntervalMs: 1, stableMs: 0, timeoutMs: 500 },
+    });
+
+    const { runId } = await engine.startRun({
+      requirementsMd: "Direct advance should repair terminal reports.",
+      repoPath,
+      baseBranch: "main",
+      scenarios: {
+        phase_plan: "unknown-schema",
+      },
+    });
+    runIds.push(runId);
+    const specApproval = pendingApproval(await engine.getStatus(runId), "spec_approved");
+    await engine.signalApproval(runId, specApproval.id, "approve", randomUUID());
+    const phasePlanApproval = pendingApproval(await engine.getStatus(runId), "phase_plan_approved");
+    await engine.signalApprovalForWorkflow(runId, phasePlanApproval.id, "approve", randomUUID());
+
+    await expect(engine.advanceRunUntilBlocked(runId)).rejects.toMatchObject({
+      code: "fake_fixture_missing",
+    });
+
+    const failed = await engine.getStatus(runId);
+    expect(failed.run.state).toBe("failed");
+    expect(failed.run.finalReportPath).toMatch(/\.report\.md$/);
+  });
+
   it("does not start another pending phase when approval replay sees active work", async () => {
     client = createDbClient(databaseUrl);
     await seedDevelopmentRegistry(client.db);
@@ -1061,6 +1209,278 @@ describe("DbRunEngine", () => {
     expect((await engine.getStatus(runId)).run.state).toBe("aborted");
   });
 
+  it("surfaces session dispose failures during abort", async () => {
+    client = createDbClient(databaseUrl);
+    await seedDevelopmentRegistry(client.db);
+    const workspaceRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-engine-workspace-")));
+    const repoPath = createGitRepo();
+    tempRoots.push(workspaceRoot, repoPath);
+    const engine = new DbRunEngine({
+      db: client.db,
+      sessions: sessionRuntime(client.db, new DisposeFailsFakeAdapter({ writeDelayMs: 0 })),
+      workspaceRoot,
+      maxConcurrentRuns: 100,
+      wait: { pollIntervalMs: 1, stableMs: 0, timeoutMs: 500 },
+    });
+
+    const { runId } = await engine.startRun({
+      requirementsMd: "Abort while waiting for approval.",
+      repoPath,
+      baseBranch: "main",
+    });
+    runIds.push(runId);
+
+    await expect(engine.abortRun(runId, "user requested abort")).rejects.toMatchObject({
+      code: "pane_briefly_unresponsive",
+    });
+
+    const aborted = await engine.getStatus(runId);
+    expect(aborted.run.state).toBe("aborted");
+    const [run] = await client.db
+      .select({ finalReportPath: runs.finalReportPath })
+      .from(runs)
+      .where(eq(runs.id, runId));
+    expect(run?.finalReportPath).toMatch(/\.report\.md$/);
+  });
+
+  it("captures terminal session transcripts before abort disposal", async () => {
+    client = createDbClient(databaseUrl);
+    await seedDevelopmentRegistry(client.db);
+    const workspaceRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-engine-workspace-")));
+    const repoPath = createGitRepo();
+    tempRoots.push(workspaceRoot, repoPath);
+    const adapter = new CaptureOrderingFakeAdapter({ writeDelayMs: 0 });
+    const engine = new DbRunEngine({
+      db: client.db,
+      sessions: sessionRuntime(client.db, adapter),
+      workspaceRoot,
+      maxConcurrentRuns: 100,
+      wait: { pollIntervalMs: 1, stableMs: 0, timeoutMs: 500 },
+    });
+
+    const { runId } = await engine.startRun({
+      requirementsMd: "Abort after capturing terminal transcript.",
+      repoPath,
+      baseBranch: "main",
+    });
+    runIds.push(runId);
+    adapter.events.length = 0;
+
+    await engine.abortRun(runId, "user requested abort");
+
+    expect(adapter.events).toEqual(["capture", "dispose"]);
+    const [session] = await client.db
+      .select({ lastCaptureSeq: tuiSessions.lastCaptureSeq })
+      .from(tuiSessions)
+      .where(eq(tuiSessions.runId, runId));
+    expect(session?.lastCaptureSeq).toBeGreaterThan(0n);
+  });
+
+  it("retries terminal approval cleanup idempotently when a decision is replayed", async () => {
+    client = createDbClient(databaseUrl);
+    await seedDevelopmentRegistry(client.db);
+    const workspaceRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-engine-workspace-")));
+    const repoPath = createGitRepo();
+    tempRoots.push(workspaceRoot, repoPath);
+    const adapter = new CaptureFailsAfterDisposeFakeAdapter({ writeDelayMs: 0 });
+    const engine = new DbRunEngine({
+      db: client.db,
+      sessions: sessionRuntime(client.db, adapter),
+      workspaceRoot,
+      maxConcurrentRuns: 100,
+      wait: { pollIntervalMs: 1, stableMs: 0, timeoutMs: 500 },
+    });
+
+    const { runId } = await engine.startRun({
+      requirementsMd: "Replay a terminal approval decision.",
+      repoPath,
+      baseBranch: "main",
+    });
+    runIds.push(runId);
+    const approvalId = (await engine.getStatus(runId)).approvals[0]?.id;
+    expect(approvalId).toBeDefined();
+    if (approvalId === undefined) {
+      throw new Error("approval id missing");
+    }
+    const clientToken = randomUUID();
+    adapter.events.length = 0;
+
+    await engine.signalApproval(runId, approvalId, "reject", clientToken);
+    expect(adapter.events).toEqual(["capture", "dispose"]);
+    adapter.events.length = 0;
+
+    await expect(
+      engine.signalApproval(runId, approvalId, "reject", clientToken),
+    ).resolves.toBeUndefined();
+    expect(adapter.events).toEqual(["capture", "dispose"]);
+    expect((await engine.getStatus(runId)).run.state).toBe("failed");
+  });
+
+  it("uses persisted tmux handles when capturing and disposing terminal sessions", async () => {
+    client = createDbClient(databaseUrl);
+    await seedDevelopmentRegistry(client.db);
+    const workspaceRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-engine-workspace-")));
+    const repoPath = createGitRepo();
+    tempRoots.push(workspaceRoot, repoPath);
+    const sessions = new TerminalHandleRecordingRuntime();
+    const engine = new DbRunEngine({
+      db: client.db,
+      sessions,
+      workspaceRoot,
+      maxConcurrentRuns: 100,
+      wait: { pollIntervalMs: 1, stableMs: 0, timeoutMs: 500 },
+    });
+
+    const { runId } = await engine.startRun({
+      requirementsMd: "Abort after persisting tmux handle fields.",
+      repoPath,
+      baseBranch: "main",
+    });
+    runIds.push(runId);
+    await client.db
+      .update(tuiSessions)
+      .set({
+        lastCaptureSeq: 1n,
+        lastKnownPanePid: 777,
+        tmuxSession: "persisted-session",
+        tmuxWindow: "persisted-window",
+      })
+      .where(eq(tuiSessions.runId, runId));
+
+    await engine.abortRun(runId, "user requested abort");
+
+    expect(sessions.captureHandles).toContainEqual({
+      sessionId: expect.any(String),
+      pid: 777,
+      transcriptBaseline: {
+        startSeq: 1n,
+        lines: expect.arrayContaining([expect.any(String)]),
+      },
+      tmuxSession: "persisted-session",
+      tmuxWindow: "persisted-window",
+    });
+    expect(sessions.disposeHandles).toContainEqual({
+      sessionId: expect.any(String),
+      pid: 777,
+      tmuxSession: "persisted-session",
+      tmuxWindow: "persisted-window",
+    });
+  });
+
+  it("attempts disposal when transcript capture fails during cleanup", async () => {
+    client = createDbClient(databaseUrl);
+    await seedDevelopmentRegistry(client.db);
+    const workspaceRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-engine-workspace-")));
+    const repoPath = createGitRepo();
+    tempRoots.push(workspaceRoot, repoPath);
+    const adapter = new CaptureOrderingFakeAdapter({ writeDelayMs: 0 });
+    const engine = new DbRunEngine({
+      db: client.db,
+      sessions: sessionRuntime(client.db, adapter),
+      workspaceRoot,
+      maxConcurrentRuns: 100,
+      wait: { pollIntervalMs: 1, stableMs: 0, timeoutMs: 500 },
+    });
+
+    const { runId } = await engine.startRun({
+      requirementsMd: "Abort with failed transcript capture.",
+      repoPath,
+      baseBranch: "main",
+    });
+    runIds.push(runId);
+    adapter.events.length = 0;
+    adapter.failCapture = true;
+
+    await expect(engine.abortRun(runId, "user requested abort")).rejects.toMatchObject({
+      code: "pane_briefly_unresponsive",
+    });
+
+    expect(adapter.events).toEqual(["capture", "dispose"]);
+    const [run] = await client.db
+      .select({ finalReportPath: runs.finalReportPath, state: runs.state })
+      .from(runs)
+      .where(eq(runs.id, runId));
+    expect(run).toMatchObject({ state: "aborted" });
+    expect(run?.finalReportPath).toMatch(/\.report\.md$/);
+
+    adapter.events.length = 0;
+    adapter.failCapture = false;
+    await engine.abortRun(runId, "retry abort cleanup");
+    expect(adapter.events).toEqual(["capture", "dispose"]);
+  });
+
+  it("writes a failed final report before surfacing approval reject dispose failures", async () => {
+    client = createDbClient(databaseUrl);
+    await seedDevelopmentRegistry(client.db);
+    const workspaceRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-engine-workspace-")));
+    const repoPath = createGitRepo();
+    tempRoots.push(workspaceRoot, repoPath);
+    const engine = new DbRunEngine({
+      db: client.db,
+      sessions: sessionRuntime(client.db, new DisposeFailsFakeAdapter({ writeDelayMs: 0 })),
+      workspaceRoot,
+      maxConcurrentRuns: 100,
+      wait: { pollIntervalMs: 1, stableMs: 0, timeoutMs: 500 },
+    });
+
+    const { runId } = await engine.startRun({
+      requirementsMd: "Reject while waiting for approval.",
+      repoPath,
+      baseBranch: "main",
+    });
+    runIds.push(runId);
+    const request = pendingApproval(await engine.getStatus(runId), "spec_approved");
+
+    await expect(
+      engine.signalApproval(runId, request.id, "reject", randomUUID()),
+    ).rejects.toMatchObject({
+      code: "pane_briefly_unresponsive",
+    });
+
+    const [run] = await client.db
+      .select({ finalReportPath: runs.finalReportPath, state: runs.state })
+      .from(runs)
+      .where(eq(runs.id, runId));
+    expect(run?.state).toBe("failed");
+    expect(run?.finalReportPath).toMatch(/\.report\.md$/);
+  });
+
+  it("writes a failed final report before surfacing workflow approval reject dispose failures", async () => {
+    client = createDbClient(databaseUrl);
+    await seedDevelopmentRegistry(client.db);
+    const workspaceRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-engine-workspace-")));
+    const repoPath = createGitRepo();
+    tempRoots.push(workspaceRoot, repoPath);
+    const engine = new DbRunEngine({
+      db: client.db,
+      sessions: sessionRuntime(client.db, new DisposeFailsFakeAdapter({ writeDelayMs: 0 })),
+      workspaceRoot,
+      maxConcurrentRuns: 100,
+      wait: { pollIntervalMs: 1, stableMs: 0, timeoutMs: 500 },
+    });
+
+    const { runId } = await engine.startRun({
+      requirementsMd: "Reject through workflow approval path.",
+      repoPath,
+      baseBranch: "main",
+    });
+    runIds.push(runId);
+    const request = pendingApproval(await engine.getStatus(runId), "spec_approved");
+
+    await expect(
+      engine.signalApprovalForWorkflow(runId, request.id, "reject", randomUUID()),
+    ).rejects.toMatchObject({
+      code: "pane_briefly_unresponsive",
+    });
+
+    const [run] = await client.db
+      .select({ finalReportPath: runs.finalReportPath, state: runs.state })
+      .from(runs)
+      .where(eq(runs.id, runId));
+    expect(run?.state).toBe("failed");
+    expect(run?.finalReportPath).toMatch(/\.report\.md$/);
+  });
+
   it("sweeps non-terminal M4 runs on API startup recovery", async () => {
     client = createDbClient(databaseUrl);
     await seedDevelopmentRegistry(client.db);
@@ -1226,7 +1646,7 @@ describe("DbRunEngine", () => {
     });
   });
 
-  it("replays terminal approval disposal side effects for duplicate decisions", async () => {
+  it("replays terminal approval cleanup side effects idempotently", async () => {
     client = createDbClient(databaseUrl);
     await seedDevelopmentRegistry(client.db);
     const workspaceRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-engine-workspace-")));
@@ -1309,6 +1729,60 @@ describe("DbRunEngine", () => {
       ),
     ).toMatchObject({ runId, status: "aborted" });
   });
+
+  it("repairs terminal final reports before surfacing approval replay dispose failures", async () => {
+    client = createDbClient(databaseUrl);
+    await seedDevelopmentRegistry(client.db);
+    const workspaceRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-engine-workspace-")));
+    const repoPath = createGitRepo();
+    const worktreeRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-engine-worktree-")));
+    tempRoots.push(workspaceRoot, repoPath, worktreeRoot);
+    const [template] = await client.db
+      .select({ hash: workflowTemplates.hash, id: workflowTemplates.id })
+      .from(workflowTemplates)
+      .where(eq(workflowTemplates.name, "development"))
+      .limit(1);
+    if (template === undefined) {
+      throw new Error("development template missing");
+    }
+    const runId = randomUUID();
+    runIds.push(runId);
+    await client.db.insert(runs).values({
+      id: runId,
+      templateId: template.id,
+      templateHash: template.hash,
+      state: "aborted",
+      repoPath,
+      baseBranch: "main",
+      worktreeRoot,
+      endedAt: new Date(),
+      finalReportPath: null,
+    });
+    await client.db.insert(tuiSessions).values({
+      id: randomUUID(),
+      runId,
+      roleId: "implementer",
+      backend: "fake",
+      cwd: worktreeRoot,
+      state: "FAILED_NEEDS_HUMAN",
+    });
+    const engine = new DbRunEngine({
+      db: client.db,
+      sessions: sessionRuntime(client.db, new DisposeFailsFakeAdapter({ writeDelayMs: 0 })),
+      workspaceRoot,
+      maxConcurrentRuns: 100,
+    });
+
+    await expect(engine.replayAppliedApprovalSideEffects(runId, "abort")).rejects.toMatchObject({
+      code: "pane_briefly_unresponsive",
+    });
+
+    const [run] = await client.db
+      .select({ finalReportPath: runs.finalReportPath })
+      .from(runs)
+      .where(eq(runs.id, runId));
+    expect(run?.finalReportPath).toMatch(/\.report\.md$/);
+  });
 });
 
 function pendingApproval(status: Awaited<ReturnType<DbRunEngine["getStatus"]>>, gateKey: string) {