import { execFileSync } from "node:child_process"; import { randomUUID } from "node:crypto"; import { mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { DevflowError, type PromptEnvelope, loadPersonaFiles, loadTemplateFiles, } from "@devflow/core"; import { type DbClient, agentPersonas, approvalDecisions, approvalRequests, createDbClient, runEvents, runs, tuiSessions, workflowTemplates, } from "@devflow/db"; import { DbRunEngine } from "@devflow/run-engine"; import { FakeSessionAdapter, type SessionHandle, SessionManager } from "@devflow/session"; import type { WorkflowClient, WorkflowHandle } from "@temporalio/client"; import { and, eq, inArray } from "drizzle-orm"; import { afterEach, describe, expect, it } from "vitest"; import { startApi, startM4Api } from "./index.js"; const databaseUrl = process.env.DATABASE_URL ?? "postgres://devflow:devflow@127.0.0.1:55432/devflow"; class ResumeFailsFakeSessionAdapter extends FakeSessionAdapter { resumeAttempts = 0; override async resume(_handle: SessionHandle): Promise { this.resumeAttempts += 1; throw new DevflowError("resume failed", { class: "recoverable", code: "pane_briefly_unresponsive", recoveryHint: "resume failed", }); } } class ResumeSucceedsAfterTwoFailuresFakeSessionAdapter extends FakeSessionAdapter { resumeAttempts = 0; override async resume(handle: SessionHandle): Promise { this.resumeAttempts += 1; if (this.resumeAttempts <= 2) { throw new DevflowError("resume failed transiently", { class: "recoverable", code: "pane_briefly_unresponsive", recoveryHint: "resume failed transiently", }); } return super.resume(handle); } } class DelayedSendPromptFakeSessionAdapter extends FakeSessionAdapter { readonly promptStarted = deferred(); readonly releasePrompt = deferred(); override async sendPrompt( handle: SessionHandle, envelope: PromptEnvelope, ): Promise<{ promptId: string }> { this.promptStarted.resolve(); await this.releasePrompt.promise; return super.sendPrompt(handle, envelope); } } class FakeWorkflowClient { started: { workflowId: string; taskQueue: string; args: unknown[] } | undefined; async start( _workflow: unknown, options: { workflowId: string; taskQueue: string; args: unknown[] }, ) { this.started = { workflowId: options.workflowId, taskQueue: options.taskQueue, args: options.args, }; } getHandle(_workflowId: string): Pick { return { signal: async () => undefined, }; } } function deferred() { let resolve!: (value: T | PromiseLike) => void; let reject!: (reason?: unknown) => void; const promise = new Promise((promiseResolve, promiseReject) => { resolve = promiseResolve; reject = promiseReject; }); return { promise, reject, resolve }; } function createGitRepo(): string { const repoPath = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-repo-"))); execFileSync("git", ["init", "-b", "main"], { cwd: repoPath, stdio: "ignore" }); writeFileSync(join(repoPath, "README.md"), "# API fixture\n"); execFileSync("git", ["add", "README.md"], { cwd: repoPath, stdio: "ignore" }); execFileSync( "git", [ "-c", "user.name=Devflow Test", "-c", "user.email=devflow@example.test", "commit", "-m", "initial", ], { cwd: repoPath, stdio: "ignore" }, ); return repoPath; } async function seedDevelopmentRegistry(db: DbClient["db"]) { const [templateEntry] = loadTemplateFiles(resolve("docs/schemas/templates")).filter( (entry) => entry.name === "development" && entry.version === 1, ); if (templateEntry === undefined) { throw new Error("development@1 template fixture is missing"); } await db .insert(workflowTemplates) .values({ name: templateEntry.name, version: templateEntry.version, hash: templateEntry.hash, definition: templateEntry.definition, }) .onConflictDoUpdate({ target: [workflowTemplates.name, workflowTemplates.version], set: { hash: templateEntry.hash, definition: templateEntry.definition }, }); for (const personaEntry of loadPersonaFiles(resolve("docs/schemas/personas"))) { await db .insert(agentPersonas) .values({ name: personaEntry.name, version: personaEntry.version, hash: personaEntry.hash, definition: personaEntry.definition, }) .onConflictDoNothing({ target: [agentPersonas.name, agentPersonas.version] }); } } async function waitForRunEventType(db: DbClient["db"], runId: string, type: string) { const deadline = Date.now() + 2_000; while (Date.now() < deadline) { const [event] = await db .select({ id: runEvents.id }) .from(runEvents) .where(and(eq(runEvents.runId, runId), eq(runEvents.type, type))) .limit(1); if (event !== undefined) { return; } await new Promise((resolveWait) => setTimeout(resolveWait, 10)); } throw new Error(`timed out waiting for ${type}`); } describe("startApi", () => { let client: DbClient | undefined; const runIds: string[] = []; const templateIds: string[] = []; const tempRoots: string[] = []; function createApiWorkspaceRoot(): string { const workspaceRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-workspace-"))); tempRoots.push(workspaceRoot); return workspaceRoot; } function startTestM4Api(options: Parameters[0] = {}) { return startM4ApiWhenLockFree({ workspaceRoot: createApiWorkspaceRoot(), maxConcurrentRuns: 100, ...options, }); } afterEach(async () => { if (client !== undefined) { if (runIds.length > 0) { const requests = await client.db .select({ id: approvalRequests.id }) .from(approvalRequests) .where(inArray(approvalRequests.runId, [...runIds])); if (requests.length > 0) { await client.db.delete(approvalDecisions).where( inArray( approvalDecisions.approvalRequestId, requests.map((request) => request.id), ), ); } await client.db .delete(approvalRequests) .where(inArray(approvalRequests.runId, [...runIds])); await client.db.delete(runs).where(inArray(runs.id, [...runIds])); } if (templateIds.length > 0) { await client.db .delete(workflowTemplates) .where(inArray(workflowTemplates.id, [...templateIds])); } await client.close(); client = undefined; } for (const root of tempRoots.splice(0)) { rmSync(root, { recursive: true, force: true }); } runIds.length = 0; templateIds.length = 0; }); it("runs M4 restart recovery before startup completes", async () => { client = createDbClient(databaseUrl); const templateId = randomUUID(); const runId = randomUUID(); const sessionId = randomUUID(); const repoPath = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-repo-"))); const worktreeRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-worktree-"))); tempRoots.push(repoPath, worktreeRoot); templateIds.push(templateId); runIds.push(runId); await client.db.insert(workflowTemplates).values({ id: templateId, name: `api-startup-${templateId}`, version: 1, hash: "a".repeat(64), definition: { name: "api-startup", version: 1, roles: [], phases: [], defaultGates: [] }, }); await client.db.insert(runs).values({ id: runId, templateId, templateHash: "a".repeat(64), state: "executing", repoPath, baseBranch: "main", worktreeRoot, }); await client.db.insert(tuiSessions).values({ id: sessionId, runId, roleId: "spec_writer", backend: "fake", cwd: worktreeRoot, state: "READY", }); const result = await startTestM4Api({ dbClient: client, recoveryRunIds: [runId] }); try { expect(result.recovery).toEqual({ failedSessionIds: [sessionId], sweptRunIds: [runId], }); expect(result.sessionRecovery).toEqual({ failedSessionIds: [], recoveredSessionIds: [], }); } finally { await result.stop(); } const [run] = await client.db .select({ state: runs.state }) .from(runs) .where(eq(runs.id, runId)); expect(run).toEqual({ state: "failed" }); const [session] = await client.db .select({ state: tuiSessions.state }) .from(tuiSessions) .where(eq(tuiSessions.id, sessionId)); expect(session).toEqual({ state: "FAILED_NEEDS_HUMAN" }); const events = await client.db .select({ type: runEvents.type }) .from(runEvents) .where(eq(runEvents.runId, runId)) .orderBy(runEvents.seq); expect(events.map((event) => event.type)).toEqual(["run.failed", "session.failed"]); }); it("holds the SessionManager singleton lock until stopped", async () => { client = createDbClient(databaseUrl); const recoveryRunIds = [randomUUID()]; const first = await startTestM4Api({ dbClient: client, recoveryRunIds }); try { await expect( startM4Api({ dbClient: client, workspaceRoot: createApiWorkspaceRoot(), recoveryRunIds, }), ).rejects.toMatchObject({ code: "session_manager_already_running", }); } finally { await first.stop(); } }); it("hosts the M4 run engine behind the API startup boundary", async () => { client = createDbClient(databaseUrl); await seedDevelopmentRegistry(client.db); const workspaceRoot = createApiWorkspaceRoot(); const repoPath = createGitRepo(); tempRoots.push(repoPath); const api = await startM4ApiWhenLockFree({ dbClient: client, workspaceRoot, recoveryRunIds: [], maxConcurrentRuns: 100, }); try { expect(api.engine).toBeInstanceOf(DbRunEngine); const { runId } = await api.engine.startRun({ requirementsMd: "Start a fake development run through the API-owned engine.", repoPath, baseBranch: "main", scenarios: { spec: "ok" }, }); runIds.push(runId); const status = await api.engine.getStatus(runId); expect(status.run.state).toBe("awaiting_approval"); expect(status.run.worktreeRoot).toBe(resolve(workspaceRoot, runId, "main")); expect(status.approvals).toMatchObject([{ gateKey: "spec_approved", state: "pending" }]); } finally { await api.stop(); } }); it("uses the Temporal RunEngine by default without acquiring the SessionManager lock", async () => { client = createDbClient(databaseUrl); const first = await startM4ApiWhenLockFree({ dbClient: client, workspaceRoot: createApiWorkspaceRoot(), recoveryRunIds: [], maxConcurrentRuns: 100, }); const temporalClient = new FakeWorkflowClient(); try { const temporalApi = await startApi({ dbClient: client, temporalClient: temporalClient as unknown as WorkflowClient, taskQueue: "devflow-runs-test", workspaceRoot: createApiWorkspaceRoot(), awaitRunStart: false, }); const runId = randomUUID(); await temporalApi.engine.startRun({ runId, requirementsMd: "Temporal API should only dispatch workflow commands.", repoPath: "/repo", baseBranch: "main", }); expect(temporalClient.started).toMatchObject({ taskQueue: "devflow-runs-test", workflowId: `devflow-run:${runId}`, }); await temporalApi.stop(); } finally { await first.stop(); } }); it("wires Temporal approval replay side effects through the API boundary", async () => { client = createDbClient(databaseUrl); await seedDevelopmentRegistry(client.db); const workspaceRoot = createApiWorkspaceRoot(); const template = ( await client.db .select({ hash: workflowTemplates.hash, id: workflowTemplates.id }) .from(workflowTemplates) .where(eq(workflowTemplates.name, "development")) .limit(1) )[0]; if (template === undefined) { throw new Error("development template missing"); } const runId = randomUUID(); const approvalRequestId = randomUUID(); const clientToken = randomUUID(); const repoPath = createGitRepo(); const worktreeRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-worktree-"))); tempRoots.push(repoPath, worktreeRoot); runIds.push(runId); await client.db.insert(runs).values({ id: runId, templateId: template.id, templateHash: template.hash, state: "completed", repoPath, baseBranch: "main", worktreeRoot, endedAt: new Date(), finalReportPath: null, }); await client.db.insert(approvalRequests).values({ id: approvalRequestId, runId, gateKey: "spec_approved", state: "approved", idempotencyKey: `${runId}:spec_approved::1`, payload: { replay: true }, }); await client.db.insert(approvalDecisions).values({ approvalRequestId, action: "approve", idempotencyKey: `${approvalRequestId}:approve:${clientToken}`, }); const temporalApi = await startApi({ dbClient: client, temporalClient: new FakeWorkflowClient() as unknown as WorkflowClient, taskQueue: "devflow-runs-test", workspaceRoot, awaitRunStart: false, }); try { await temporalApi.engine.signalApproval(runId, approvalRequestId, "approve", clientToken); const [run] = await client.db .select({ finalReportPath: runs.finalReportPath }) .from(runs) .where(eq(runs.id, runId)); expect(run?.finalReportPath).toMatch(/\.report\.md$/); } finally { await temporalApi.stop(); } }); it.each([ { action: "reject" as const, approvalState: "rejected", runState: "failed" }, { action: "abort" as const, approvalState: "aborted", runState: "aborted" }, ])( "repairs $runState approval replay reports without mutating sessions through the API", async ({ action, approvalState, runState }) => { client = createDbClient(databaseUrl); await seedDevelopmentRegistry(client.db); const workspaceRoot = createApiWorkspaceRoot(); const template = ( await client.db .select({ hash: workflowTemplates.hash, id: workflowTemplates.id }) .from(workflowTemplates) .where(eq(workflowTemplates.name, "development")) .limit(1) )[0]; if (template === undefined) { throw new Error("development template missing"); } const runId = randomUUID(); const approvalRequestId = randomUUID(); const clientToken = randomUUID(); const sessionId = randomUUID(); const repoPath = createGitRepo(); const worktreeRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-worktree-"))); tempRoots.push(repoPath, worktreeRoot); runIds.push(runId); await client.db.insert(runs).values({ id: runId, templateId: template.id, templateHash: template.hash, state: runState, repoPath, baseBranch: "main", worktreeRoot, endedAt: new Date(), finalReportPath: null, }); await client.db.insert(tuiSessions).values({ id: sessionId, runId, roleId: "implementer", backend: "fake", cwd: worktreeRoot, state: "READY", }); await client.db.insert(approvalRequests).values({ id: approvalRequestId, runId, gateKey: "spec_approved", state: approvalState, idempotencyKey: `${runId}:spec_approved::1`, payload: { replay: true }, }); await client.db.insert(approvalDecisions).values({ approvalRequestId, action, idempotencyKey: `${approvalRequestId}:${action}:${clientToken}`, }); const temporalApi = await startApi({ dbClient: client, temporalClient: new FakeWorkflowClient() as unknown as WorkflowClient, taskQueue: "devflow-runs-test", workspaceRoot, awaitRunStart: false, }); try { await temporalApi.engine.signalApproval(runId, approvalRequestId, action, clientToken); const [run] = await client.db .select({ finalReportPath: runs.finalReportPath }) .from(runs) .where(eq(runs.id, runId)); expect(run?.finalReportPath).toMatch(/\.report\.md$/); const [session] = await client.db .select({ state: tuiSessions.state }) .from(tuiSessions) .where(eq(tuiSessions.id, sessionId)); expect(session).toEqual({ state: "READY" }); } finally { await temporalApi.stop(); } }, ); it("repairs missing terminal final reports during API startup", async () => { client = createDbClient(databaseUrl); await seedDevelopmentRegistry(client.db); const workspaceRoot = createApiWorkspaceRoot(); const template = ( await client.db .select({ hash: workflowTemplates.hash, id: workflowTemplates.id }) .from(workflowTemplates) .where(eq(workflowTemplates.name, "development")) .limit(1) )[0]; if (template === undefined) { throw new Error("development template missing"); } const runId = randomUUID(); const repoPath = createGitRepo(); const worktreeRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-worktree-"))); tempRoots.push(repoPath, worktreeRoot); runIds.push(runId); await client.db.insert(runs).values({ id: runId, templateId: template.id, templateHash: template.hash, state: "completed", repoPath, baseBranch: "main", worktreeRoot, endedAt: new Date(), finalReportPath: null, }); const api = await startM4ApiWhenLockFree({ dbClient: client, workspaceRoot, recoveryRunIds: [runId], maxConcurrentRuns: 100, }); try { expect(api.finalReportRecovery).toEqual([runId]); const [run] = await client.db .select({ finalReportPath: runs.finalReportPath }) .from(runs) .where(eq(runs.id, runId)); expect(run?.finalReportPath).toMatch(/\.report\.md$/); } finally { await api.stop(); } }); it("does not sweep active runs when a second API instance fails the singleton lock", async () => { client = createDbClient(databaseUrl); const first = await startTestM4Api({ dbClient: client, recoveryRunIds: [] }); const templateId = randomUUID(); const runId = randomUUID(); const sessionId = randomUUID(); const repoPath = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-repo-"))); const worktreeRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-worktree-"))); tempRoots.push(repoPath, worktreeRoot); templateIds.push(templateId); runIds.push(runId); try { await client.db.insert(workflowTemplates).values({ id: templateId, name: `api-lock-order-${templateId}`, version: 1, hash: "c".repeat(64), definition: { name: "api-lock-order", version: 1, roles: [], phases: [] }, }); await client.db.insert(runs).values({ id: runId, templateId, templateHash: "c".repeat(64), state: "executing", repoPath, baseBranch: "main", worktreeRoot, }); await client.db.insert(tuiSessions).values({ id: sessionId, runId, roleId: "spec_writer", backend: "fake", cwd: worktreeRoot, state: "READY", }); await expect( startM4Api({ dbClient: client, workspaceRoot: createApiWorkspaceRoot(), recoveryRunIds: [runId], }), ).rejects.toMatchObject({ code: "session_manager_already_running", }); const [run] = await client.db .select({ state: runs.state }) .from(runs) .where(eq(runs.id, runId)); expect(run).toEqual({ state: "executing" }); const [session] = await client.db .select({ state: tuiSessions.state }) .from(tuiSessions) .where(eq(tuiSessions.id, sessionId)); expect(session).toEqual({ state: "READY" }); const events = await client.db.select().from(runEvents).where(eq(runEvents.runId, runId)); expect(events).toEqual([]); } finally { await first.stop(); } }); it("ignores terminal-run sessions during SessionManager startup recovery", async () => { client = createDbClient(databaseUrl); const templateId = randomUUID(); const runId = randomUUID(); const sessionId = randomUUID(); const repoPath = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-repo-"))); const worktreeRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-worktree-"))); tempRoots.push(repoPath, worktreeRoot); templateIds.push(templateId); runIds.push(runId); const adapter = new FakeSessionAdapter({ sessionIdFactory: () => sessionId, writeDelayMs: 0, }); await adapter.start({ runId, roleId: "spec_writer", backend: "fake", cwd: worktreeRoot, }); await client.db.insert(workflowTemplates).values({ id: templateId, name: `api-session-recovery-${templateId}`, version: 1, hash: "b".repeat(64), definition: { name: "api-session-recovery", version: 1, roles: [], phases: [] }, }); await client.db.insert(runs).values({ id: runId, templateId, templateHash: "b".repeat(64), state: "completed", repoPath, baseBranch: "main", worktreeRoot, }); await client.db.insert(tuiSessions).values({ id: sessionId, runId, roleId: "spec_writer", backend: "fake", cwd: worktreeRoot, state: "READY", }); const result = await startTestM4Api({ dbClient: client, recoveryRunIds: [runId], sessionAdapter: adapter, }); try { expect(result.recovery).toEqual({ failedSessionIds: [], sweptRunIds: [] }); expect(result.sessionRecovery).toEqual({ failedSessionIds: [], recoveredSessionIds: [], }); } finally { await result.stop(); } const [session] = await client.db .select({ state: tuiSessions.state }) .from(tuiSessions) .where(eq(tuiSessions.id, sessionId)); expect(session).toEqual({ state: "READY" }); const approvals = await client.db .select() .from(approvalRequests) .where(eq(approvalRequests.runId, runId)); expect(approvals).toEqual([]); const events = await client.db.select().from(runEvents).where(eq(runEvents.runId, runId)); expect(events).toEqual([]); }); it("fails CREATED session reservations during SessionManager startup recovery", async () => { client = createDbClient(databaseUrl); const templateId = randomUUID(); const runId = randomUUID(); const sessionId = randomUUID(); const repoPath = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-repo-"))); const worktreeRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-worktree-"))); tempRoots.push(repoPath, worktreeRoot); templateIds.push(templateId); runIds.push(runId); await client.db.insert(workflowTemplates).values({ id: templateId, name: `api-session-created-${templateId}`, version: 1, hash: "f".repeat(64), definition: { name: "api-session-created", version: 1, roles: [], phases: [] }, }); await client.db.insert(runs).values({ id: runId, templateId, templateHash: "f".repeat(64), state: "executing", repoPath, baseBranch: "main", worktreeRoot, }); await client.db.insert(tuiSessions).values({ id: sessionId, runId, roleId: "spec_writer", backend: "fake", cwd: worktreeRoot, state: "CREATED", }); const adapter = new ResumeFailsFakeSessionAdapter(); const manager = new SessionManager({ dbClient: client, adapter, recoveryRunIds: [runId], }); const recovery = await initializeManagerWhenLockFree(manager); try { expect(adapter.resumeAttempts).toBe(3); expect(recovery).toEqual({ failedSessionIds: [sessionId], recoveredSessionIds: [] }); } finally { await manager.shutdown(); } const [run] = await client.db .select({ pausedFromState: runs.pausedFromState, state: runs.state }) .from(runs) .where(eq(runs.id, runId)); expect(run).toEqual({ pausedFromState: "executing", state: "paused" }); const [session] = await client.db .select({ recoveryAttempts: tuiSessions.recoveryAttempts, state: tuiSessions.state }) .from(tuiSessions) .where(eq(tuiSessions.id, sessionId)); expect(session).toEqual({ recoveryAttempts: 1, state: "FAILED_NEEDS_HUMAN" }); const approvals = await client.db .select({ gateKey: approvalRequests.gateKey, state: approvalRequests.state }) .from(approvalRequests) .where(eq(approvalRequests.runId, runId)); expect(approvals).toEqual([{ gateKey: "session_recovery_required", state: "pending" }]); const events = await client.db .select({ type: runEvents.type }) .from(runEvents) .where(eq(runEvents.runId, runId)) .orderBy(runEvents.seq); expect(events.map((event) => event.type)).toEqual([ "session.failed", "run.paused", "approval.requested", ]); }); it("retries transient session resume failures during startup recovery", async () => { client = createDbClient(databaseUrl); const templateId = randomUUID(); const runId = randomUUID(); const sessionId = randomUUID(); const repoPath = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-repo-"))); const worktreeRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-worktree-"))); tempRoots.push(repoPath, worktreeRoot); templateIds.push(templateId); runIds.push(runId); const adapter = new ResumeSucceedsAfterTwoFailuresFakeSessionAdapter({ sessionIdFactory: () => sessionId, writeDelayMs: 0, }); await adapter.start({ runId, roleId: "spec_writer", backend: "fake", cwd: worktreeRoot, }); await client.db.insert(workflowTemplates).values({ id: templateId, name: `api-session-retry-${templateId}`, version: 1, hash: "e".repeat(64), definition: { name: "api-session-retry", version: 1, roles: [], phases: [] }, }); await client.db.insert(runs).values({ id: runId, templateId, templateHash: "e".repeat(64), state: "executing", repoPath, baseBranch: "main", worktreeRoot, }); await client.db.insert(tuiSessions).values({ id: sessionId, runId, roleId: "spec_writer", backend: "fake", cwd: worktreeRoot, state: "READY", }); const manager = new SessionManager({ dbClient: client, adapter, recoveryRunIds: [runId], }); const recovery = await initializeManagerWhenLockFree(manager); try { expect(adapter.resumeAttempts).toBe(3); expect(recovery).toEqual({ failedSessionIds: [], recoveredSessionIds: [sessionId] }); const approvals = await client.db .select() .from(approvalRequests) .where(eq(approvalRequests.runId, runId)); expect(approvals).toEqual([]); } finally { await manager.shutdown(); } }); it("pauses a non-terminal run when SessionManager startup recovery cannot resume a session", async () => { client = createDbClient(databaseUrl); const templateId = randomUUID(); const runId = randomUUID(); const sessionId = randomUUID(); const repoPath = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-repo-"))); const worktreeRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-worktree-"))); tempRoots.push(repoPath, worktreeRoot); templateIds.push(templateId); runIds.push(runId); await client.db.insert(workflowTemplates).values({ id: templateId, name: `api-session-recovery-failure-${templateId}`, version: 1, hash: "d".repeat(64), definition: { name: "api-session-recovery-failure", version: 1, roles: [], phases: [] }, }); await client.db.insert(runs).values({ id: runId, templateId, templateHash: "d".repeat(64), state: "executing", repoPath, baseBranch: "main", worktreeRoot, }); await client.db.insert(tuiSessions).values({ id: sessionId, runId, roleId: "spec_writer", backend: "fake", cwd: worktreeRoot, state: "READY", }); const adapter = new ResumeFailsFakeSessionAdapter(); const manager = new SessionManager({ dbClient: client, adapter, recoveryRunIds: [runId], }); const recovery = await initializeManagerWhenLockFree(manager); try { expect(adapter.resumeAttempts).toBe(3); expect(recovery).toEqual({ failedSessionIds: [sessionId], recoveredSessionIds: [] }); const [run] = await client.db .select({ pausedFromState: runs.pausedFromState, state: runs.state }) .from(runs) .where(eq(runs.id, runId)); expect(run).toEqual({ pausedFromState: "executing", state: "paused" }); const [session] = await client.db .select({ recoveryAttempts: tuiSessions.recoveryAttempts, state: tuiSessions.state }) .from(tuiSessions) .where(eq(tuiSessions.id, sessionId)); expect(session).toEqual({ recoveryAttempts: 1, state: "FAILED_NEEDS_HUMAN" }); const [approval] = await client.db .select({ gateKey: approvalRequests.gateKey, phaseId: approvalRequests.phaseId, state: approvalRequests.state, }) .from(approvalRequests) .where(eq(approvalRequests.runId, runId)); expect(approval).toEqual({ gateKey: "session_recovery_required", phaseId: null, state: "pending", }); const events = await client.db .select({ type: runEvents.type }) .from(runEvents) .where(eq(runEvents.runId, runId)) .orderBy(runEvents.seq); expect(events.map((event) => event.type)).toEqual([ "session.failed", "run.paused", "approval.requested", ]); } finally { await manager.shutdown(); } }); it("keeps the singleton lock while shutdown drains in-flight session operations", async () => { client = createDbClient(databaseUrl); const adapter = new DelayedSendPromptFakeSessionAdapter({ writeDelayMs: 0 }); const manager = new SessionManager({ dbClient: client, adapter, recoveryRunIds: [], shutdownDrainMs: 5_000, }); await initializeManagerWhenLockFree(manager); const runId = randomUUID(); const cwd = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-session-"))); tempRoots.push(cwd); const handle = await manager.start({ runId, roleId: "spec_writer", backend: "fake", cwd, }); const envelope: PromptEnvelope = { uuid: randomUUID(), runId, roleId: "spec_writer", phaseKey: "spec", attempt: 0, expectedArtifact: join(tmpdir(), `${randomUUID()}.json`), expectedSchema: "dev/spec@1", dedupKey: `dedup-${randomUUID()}`, instructions: "Scenario: timeout", }; const promptPromise = manager.sendPrompt(handle, envelope); await adapter.promptStarted.promise; const shutdownPromise = manager.shutdown(); await expect( new SessionManager({ dbClient: client, adapter: new FakeSessionAdapter(), recoveryRunIds: [], }).initialize(), ).rejects.toMatchObject({ code: "session_manager_already_running" }); adapter.releasePrompt.resolve(undefined); await expect(promptPromise).resolves.toEqual({ promptId: envelope.dedupKey }); await shutdownPromise; const nextManager = new SessionManager({ dbClient: client, adapter: new FakeSessionAdapter(), recoveryRunIds: [], }); await expect(initializeManagerWhenLockFree(nextManager)).resolves.toEqual({ failedSessionIds: [], recoveredSessionIds: [], }); await nextManager.shutdown(); }); it("keeps the singleton lock while shutdown drains in-flight artifact polling", async () => { client = createDbClient(databaseUrl); await seedDevelopmentRegistry(client.db); const workspaceRoot = createApiWorkspaceRoot(); const repoPath = createGitRepo(); tempRoots.push(repoPath); const runId = randomUUID(); runIds.push(runId); const api = await startM4ApiWhenLockFree({ dbClient: client, workspaceRoot, recoveryRunIds: [], maxConcurrentRuns: 100, sessionAdapter: new FakeSessionAdapter({ writeDelayMs: 1_000 }), }); const startPromise = api.engine.startRun({ runId, requirementsMd: "Keep artifact polling in flight during shutdown.", repoPath, baseBranch: "main", scenarios: { spec: "ok" }, }); await waitForRunEventType(client.db, runId, "artifact.expected"); const stopPromise = api.stop(); await expect( new SessionManager({ dbClient: client, adapter: new FakeSessionAdapter(), recoveryRunIds: [], }).initialize(), ).rejects.toMatchObject({ code: "session_manager_already_running" }); await expect(startPromise).resolves.toEqual({ runId }); await stopPromise; const nextManager = new SessionManager({ dbClient: client, adapter: new FakeSessionAdapter(), recoveryRunIds: [], }); await expect(initializeManagerWhenLockFree(nextManager)).resolves.toEqual({ failedSessionIds: [], recoveredSessionIds: [], }); await nextManager.shutdown(); }); }); async function startM4ApiWhenLockFree(options: Parameters[0]) { const deadline = Date.now() + 6_000; let lastError: unknown; while (Date.now() < deadline) { try { return await startM4Api(options); } catch (error) { lastError = error; if (!(error instanceof DevflowError) || error.code !== "session_manager_already_running") { throw error; } await new Promise((resolveWait) => setTimeout(resolveWait, 50)); } } throw lastError; } async function initializeManagerWhenLockFree(manager: SessionManager) { const deadline = Date.now() + 6_000; let lastError: unknown; while (Date.now() < deadline) { try { return await manager.initialize(); } catch (error) { lastError = error; if (!(error instanceof DevflowError) || error.code !== "session_manager_already_running") { throw error; } await new Promise((resolveWait) => setTimeout(resolveWait, 50)); } } throw lastError; }