1088 lines
35 KiB
TypeScript
1088 lines
35 KiB
TypeScript
import { execFileSync } from "node:child_process";
|
|
import { randomUUID } from "node:crypto";
|
|
import { mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join, resolve } from "node:path";
|
|
|
|
import {
|
|
DevflowError,
|
|
type PromptEnvelope,
|
|
loadPersonaFiles,
|
|
loadTemplateFiles,
|
|
} from "@devflow/core";
|
|
import {
|
|
type DbClient,
|
|
agentPersonas,
|
|
approvalDecisions,
|
|
approvalRequests,
|
|
createDbClient,
|
|
runEvents,
|
|
runs,
|
|
tuiSessions,
|
|
workflowTemplates,
|
|
} from "@devflow/db";
|
|
import { DbRunEngine } from "@devflow/run-engine";
|
|
import { FakeSessionAdapter, type SessionHandle, SessionManager } from "@devflow/session";
|
|
import type { WorkflowClient, WorkflowHandle } from "@temporalio/client";
|
|
import { and, eq, inArray } from "drizzle-orm";
|
|
import { afterEach, describe, expect, it } from "vitest";
|
|
|
|
import { startApi, startM4Api } from "./index.js";
|
|
|
|
const databaseUrl =
|
|
process.env.DATABASE_URL ?? "postgres://devflow:devflow@127.0.0.1:55432/devflow";
|
|
|
|
class ResumeFailsFakeSessionAdapter extends FakeSessionAdapter {
|
|
resumeAttempts = 0;
|
|
|
|
override async resume(_handle: SessionHandle): Promise<SessionHandle> {
|
|
this.resumeAttempts += 1;
|
|
throw new DevflowError("resume failed", {
|
|
class: "recoverable",
|
|
code: "pane_briefly_unresponsive",
|
|
recoveryHint: "resume failed",
|
|
});
|
|
}
|
|
}
|
|
|
|
class ResumeSucceedsAfterTwoFailuresFakeSessionAdapter extends FakeSessionAdapter {
|
|
resumeAttempts = 0;
|
|
|
|
override async resume(handle: SessionHandle): Promise<SessionHandle> {
|
|
this.resumeAttempts += 1;
|
|
if (this.resumeAttempts <= 2) {
|
|
throw new DevflowError("resume failed transiently", {
|
|
class: "recoverable",
|
|
code: "pane_briefly_unresponsive",
|
|
recoveryHint: "resume failed transiently",
|
|
});
|
|
}
|
|
return super.resume(handle);
|
|
}
|
|
}
|
|
|
|
class DelayedSendPromptFakeSessionAdapter extends FakeSessionAdapter {
|
|
readonly promptStarted = deferred<void>();
|
|
readonly releasePrompt = deferred<void>();
|
|
|
|
override async sendPrompt(
|
|
handle: SessionHandle,
|
|
envelope: PromptEnvelope,
|
|
): Promise<{ promptId: string }> {
|
|
this.promptStarted.resolve();
|
|
await this.releasePrompt.promise;
|
|
return super.sendPrompt(handle, envelope);
|
|
}
|
|
}
|
|
|
|
class FakeWorkflowClient {
|
|
started: { workflowId: string; taskQueue: string; args: unknown[] } | undefined;
|
|
|
|
async start(
|
|
_workflow: unknown,
|
|
options: { workflowId: string; taskQueue: string; args: unknown[] },
|
|
) {
|
|
this.started = {
|
|
workflowId: options.workflowId,
|
|
taskQueue: options.taskQueue,
|
|
args: options.args,
|
|
};
|
|
}
|
|
|
|
getHandle(_workflowId: string): Pick<WorkflowHandle, "signal"> {
|
|
return {
|
|
signal: async () => undefined,
|
|
};
|
|
}
|
|
}
|
|
|
|
function deferred<T>() {
|
|
let resolve!: (value: T | PromiseLike<T>) => void;
|
|
let reject!: (reason?: unknown) => void;
|
|
const promise = new Promise<T>((promiseResolve, promiseReject) => {
|
|
resolve = promiseResolve;
|
|
reject = promiseReject;
|
|
});
|
|
return { promise, reject, resolve };
|
|
}
|
|
|
|
function createGitRepo(): string {
|
|
const repoPath = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-repo-")));
|
|
execFileSync("git", ["init", "-b", "main"], { cwd: repoPath, stdio: "ignore" });
|
|
writeFileSync(join(repoPath, "README.md"), "# API fixture\n");
|
|
execFileSync("git", ["add", "README.md"], { cwd: repoPath, stdio: "ignore" });
|
|
execFileSync(
|
|
"git",
|
|
[
|
|
"-c",
|
|
"user.name=Devflow Test",
|
|
"-c",
|
|
"user.email=devflow@example.test",
|
|
"commit",
|
|
"-m",
|
|
"initial",
|
|
],
|
|
{ cwd: repoPath, stdio: "ignore" },
|
|
);
|
|
return repoPath;
|
|
}
|
|
|
|
async function seedDevelopmentRegistry(db: DbClient["db"]) {
|
|
const [templateEntry] = loadTemplateFiles(resolve("docs/schemas/templates")).filter(
|
|
(entry) => entry.name === "development" && entry.version === 1,
|
|
);
|
|
if (templateEntry === undefined) {
|
|
throw new Error("development@1 template fixture is missing");
|
|
}
|
|
await db
|
|
.insert(workflowTemplates)
|
|
.values({
|
|
name: templateEntry.name,
|
|
version: templateEntry.version,
|
|
hash: templateEntry.hash,
|
|
definition: templateEntry.definition,
|
|
})
|
|
.onConflictDoUpdate({
|
|
target: [workflowTemplates.name, workflowTemplates.version],
|
|
set: { hash: templateEntry.hash, definition: templateEntry.definition },
|
|
});
|
|
|
|
for (const personaEntry of loadPersonaFiles(resolve("docs/schemas/personas"))) {
|
|
await db
|
|
.insert(agentPersonas)
|
|
.values({
|
|
name: personaEntry.name,
|
|
version: personaEntry.version,
|
|
hash: personaEntry.hash,
|
|
definition: personaEntry.definition,
|
|
})
|
|
.onConflictDoNothing({ target: [agentPersonas.name, agentPersonas.version] });
|
|
}
|
|
}
|
|
|
|
async function waitForRunEventType(db: DbClient["db"], runId: string, type: string) {
|
|
const deadline = Date.now() + 2_000;
|
|
while (Date.now() < deadline) {
|
|
const [event] = await db
|
|
.select({ id: runEvents.id })
|
|
.from(runEvents)
|
|
.where(and(eq(runEvents.runId, runId), eq(runEvents.type, type)))
|
|
.limit(1);
|
|
if (event !== undefined) {
|
|
return;
|
|
}
|
|
await new Promise((resolveWait) => setTimeout(resolveWait, 10));
|
|
}
|
|
throw new Error(`timed out waiting for ${type}`);
|
|
}
|
|
|
|
describe("startApi", () => {
|
|
let client: DbClient | undefined;
|
|
const runIds: string[] = [];
|
|
const templateIds: string[] = [];
|
|
const tempRoots: string[] = [];
|
|
|
|
function createApiWorkspaceRoot(): string {
|
|
const workspaceRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-workspace-")));
|
|
tempRoots.push(workspaceRoot);
|
|
return workspaceRoot;
|
|
}
|
|
|
|
function startTestM4Api(options: Parameters<typeof startM4Api>[0] = {}) {
|
|
return startM4ApiWhenLockFree({
|
|
workspaceRoot: createApiWorkspaceRoot(),
|
|
maxConcurrentRuns: 100,
|
|
...options,
|
|
});
|
|
}
|
|
|
|
afterEach(async () => {
|
|
if (client !== undefined) {
|
|
if (runIds.length > 0) {
|
|
const requests = await client.db
|
|
.select({ id: approvalRequests.id })
|
|
.from(approvalRequests)
|
|
.where(inArray(approvalRequests.runId, [...runIds]));
|
|
if (requests.length > 0) {
|
|
await client.db.delete(approvalDecisions).where(
|
|
inArray(
|
|
approvalDecisions.approvalRequestId,
|
|
requests.map((request) => request.id),
|
|
),
|
|
);
|
|
}
|
|
await client.db
|
|
.delete(approvalRequests)
|
|
.where(inArray(approvalRequests.runId, [...runIds]));
|
|
await client.db.delete(runs).where(inArray(runs.id, [...runIds]));
|
|
}
|
|
if (templateIds.length > 0) {
|
|
await client.db
|
|
.delete(workflowTemplates)
|
|
.where(inArray(workflowTemplates.id, [...templateIds]));
|
|
}
|
|
await client.close();
|
|
client = undefined;
|
|
}
|
|
|
|
for (const root of tempRoots.splice(0)) {
|
|
rmSync(root, { recursive: true, force: true });
|
|
}
|
|
runIds.length = 0;
|
|
templateIds.length = 0;
|
|
});
|
|
|
|
it("runs M4 restart recovery before startup completes", async () => {
|
|
client = createDbClient(databaseUrl);
|
|
const templateId = randomUUID();
|
|
const runId = randomUUID();
|
|
const sessionId = randomUUID();
|
|
const repoPath = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-repo-")));
|
|
const worktreeRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-worktree-")));
|
|
tempRoots.push(repoPath, worktreeRoot);
|
|
templateIds.push(templateId);
|
|
runIds.push(runId);
|
|
|
|
await client.db.insert(workflowTemplates).values({
|
|
id: templateId,
|
|
name: `api-startup-${templateId}`,
|
|
version: 1,
|
|
hash: "a".repeat(64),
|
|
definition: { name: "api-startup", version: 1, roles: [], phases: [], defaultGates: [] },
|
|
});
|
|
await client.db.insert(runs).values({
|
|
id: runId,
|
|
templateId,
|
|
templateHash: "a".repeat(64),
|
|
state: "executing",
|
|
repoPath,
|
|
baseBranch: "main",
|
|
worktreeRoot,
|
|
});
|
|
await client.db.insert(tuiSessions).values({
|
|
id: sessionId,
|
|
runId,
|
|
roleId: "spec_writer",
|
|
backend: "fake",
|
|
cwd: worktreeRoot,
|
|
state: "READY",
|
|
});
|
|
|
|
const result = await startTestM4Api({ dbClient: client, recoveryRunIds: [runId] });
|
|
try {
|
|
expect(result.recovery).toEqual({
|
|
failedSessionIds: [sessionId],
|
|
sweptRunIds: [runId],
|
|
});
|
|
expect(result.sessionRecovery).toEqual({
|
|
failedSessionIds: [],
|
|
recoveredSessionIds: [],
|
|
});
|
|
} finally {
|
|
await result.stop();
|
|
}
|
|
const [run] = await client.db
|
|
.select({ state: runs.state })
|
|
.from(runs)
|
|
.where(eq(runs.id, runId));
|
|
expect(run).toEqual({ state: "failed" });
|
|
const [session] = await client.db
|
|
.select({ state: tuiSessions.state })
|
|
.from(tuiSessions)
|
|
.where(eq(tuiSessions.id, sessionId));
|
|
expect(session).toEqual({ state: "FAILED_NEEDS_HUMAN" });
|
|
const events = await client.db
|
|
.select({ type: runEvents.type })
|
|
.from(runEvents)
|
|
.where(eq(runEvents.runId, runId))
|
|
.orderBy(runEvents.seq);
|
|
expect(events.map((event) => event.type)).toEqual(["run.failed", "session.failed"]);
|
|
});
|
|
|
|
it("holds the SessionManager singleton lock until stopped", async () => {
|
|
client = createDbClient(databaseUrl);
|
|
const recoveryRunIds = [randomUUID()];
|
|
const first = await startTestM4Api({ dbClient: client, recoveryRunIds });
|
|
try {
|
|
await expect(
|
|
startM4Api({
|
|
dbClient: client,
|
|
workspaceRoot: createApiWorkspaceRoot(),
|
|
recoveryRunIds,
|
|
}),
|
|
).rejects.toMatchObject({
|
|
code: "session_manager_already_running",
|
|
});
|
|
} finally {
|
|
await first.stop();
|
|
}
|
|
});
|
|
|
|
it("hosts the M4 run engine behind the API startup boundary", async () => {
|
|
client = createDbClient(databaseUrl);
|
|
await seedDevelopmentRegistry(client.db);
|
|
const workspaceRoot = createApiWorkspaceRoot();
|
|
const repoPath = createGitRepo();
|
|
tempRoots.push(repoPath);
|
|
|
|
const api = await startM4ApiWhenLockFree({
|
|
dbClient: client,
|
|
workspaceRoot,
|
|
recoveryRunIds: [],
|
|
maxConcurrentRuns: 100,
|
|
});
|
|
try {
|
|
expect(api.engine).toBeInstanceOf(DbRunEngine);
|
|
const { runId } = await api.engine.startRun({
|
|
requirementsMd: "Start a fake development run through the API-owned engine.",
|
|
repoPath,
|
|
baseBranch: "main",
|
|
scenarios: { spec: "ok" },
|
|
});
|
|
runIds.push(runId);
|
|
|
|
const status = await api.engine.getStatus(runId);
|
|
expect(status.run.state).toBe("awaiting_approval");
|
|
expect(status.run.worktreeRoot).toBe(resolve(workspaceRoot, runId, "main"));
|
|
expect(status.approvals).toMatchObject([{ gateKey: "spec_approved", state: "pending" }]);
|
|
} finally {
|
|
await api.stop();
|
|
}
|
|
});
|
|
|
|
it("uses the Temporal RunEngine by default without acquiring the SessionManager lock", async () => {
|
|
client = createDbClient(databaseUrl);
|
|
const first = await startM4ApiWhenLockFree({
|
|
dbClient: client,
|
|
workspaceRoot: createApiWorkspaceRoot(),
|
|
recoveryRunIds: [],
|
|
maxConcurrentRuns: 100,
|
|
});
|
|
const temporalClient = new FakeWorkflowClient();
|
|
try {
|
|
const temporalApi = await startApi({
|
|
dbClient: client,
|
|
temporalClient: temporalClient as unknown as WorkflowClient,
|
|
taskQueue: "devflow-runs-test",
|
|
workspaceRoot: createApiWorkspaceRoot(),
|
|
awaitRunStart: false,
|
|
});
|
|
const runId = randomUUID();
|
|
await temporalApi.engine.startRun({
|
|
runId,
|
|
requirementsMd: "Temporal API should only dispatch workflow commands.",
|
|
repoPath: "/repo",
|
|
baseBranch: "main",
|
|
});
|
|
expect(temporalClient.started).toMatchObject({
|
|
taskQueue: "devflow-runs-test",
|
|
workflowId: `devflow-run:${runId}`,
|
|
});
|
|
await temporalApi.stop();
|
|
} finally {
|
|
await first.stop();
|
|
}
|
|
});
|
|
|
|
it("wires Temporal approval replay side effects through the API boundary", async () => {
|
|
client = createDbClient(databaseUrl);
|
|
await seedDevelopmentRegistry(client.db);
|
|
const workspaceRoot = createApiWorkspaceRoot();
|
|
const template = (
|
|
await client.db
|
|
.select({ hash: workflowTemplates.hash, id: workflowTemplates.id })
|
|
.from(workflowTemplates)
|
|
.where(eq(workflowTemplates.name, "development"))
|
|
.limit(1)
|
|
)[0];
|
|
if (template === undefined) {
|
|
throw new Error("development template missing");
|
|
}
|
|
const runId = randomUUID();
|
|
const approvalRequestId = randomUUID();
|
|
const clientToken = randomUUID();
|
|
const repoPath = createGitRepo();
|
|
const worktreeRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-worktree-")));
|
|
tempRoots.push(repoPath, worktreeRoot);
|
|
runIds.push(runId);
|
|
await client.db.insert(runs).values({
|
|
id: runId,
|
|
templateId: template.id,
|
|
templateHash: template.hash,
|
|
state: "completed",
|
|
repoPath,
|
|
baseBranch: "main",
|
|
worktreeRoot,
|
|
endedAt: new Date(),
|
|
finalReportPath: null,
|
|
});
|
|
await client.db.insert(approvalRequests).values({
|
|
id: approvalRequestId,
|
|
runId,
|
|
gateKey: "spec_approved",
|
|
state: "approved",
|
|
idempotencyKey: `${runId}:spec_approved::1`,
|
|
payload: { replay: true },
|
|
});
|
|
await client.db.insert(approvalDecisions).values({
|
|
approvalRequestId,
|
|
action: "approve",
|
|
idempotencyKey: `${approvalRequestId}:approve:${clientToken}`,
|
|
});
|
|
|
|
const temporalApi = await startApi({
|
|
dbClient: client,
|
|
temporalClient: new FakeWorkflowClient() as unknown as WorkflowClient,
|
|
taskQueue: "devflow-runs-test",
|
|
workspaceRoot,
|
|
awaitRunStart: false,
|
|
});
|
|
try {
|
|
await temporalApi.engine.signalApproval(runId, approvalRequestId, "approve", clientToken);
|
|
const [run] = await client.db
|
|
.select({ finalReportPath: runs.finalReportPath })
|
|
.from(runs)
|
|
.where(eq(runs.id, runId));
|
|
expect(run?.finalReportPath).toMatch(/\.report\.md$/);
|
|
} finally {
|
|
await temporalApi.stop();
|
|
}
|
|
});
|
|
|
|
it.each([
|
|
{ action: "reject" as const, approvalState: "rejected", runState: "failed" },
|
|
{ action: "abort" as const, approvalState: "aborted", runState: "aborted" },
|
|
])(
|
|
"repairs $runState approval replay reports without mutating sessions through the API",
|
|
async ({ action, approvalState, runState }) => {
|
|
client = createDbClient(databaseUrl);
|
|
await seedDevelopmentRegistry(client.db);
|
|
const workspaceRoot = createApiWorkspaceRoot();
|
|
const template = (
|
|
await client.db
|
|
.select({ hash: workflowTemplates.hash, id: workflowTemplates.id })
|
|
.from(workflowTemplates)
|
|
.where(eq(workflowTemplates.name, "development"))
|
|
.limit(1)
|
|
)[0];
|
|
if (template === undefined) {
|
|
throw new Error("development template missing");
|
|
}
|
|
const runId = randomUUID();
|
|
const approvalRequestId = randomUUID();
|
|
const clientToken = randomUUID();
|
|
const sessionId = randomUUID();
|
|
const repoPath = createGitRepo();
|
|
const worktreeRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-worktree-")));
|
|
tempRoots.push(repoPath, worktreeRoot);
|
|
runIds.push(runId);
|
|
await client.db.insert(runs).values({
|
|
id: runId,
|
|
templateId: template.id,
|
|
templateHash: template.hash,
|
|
state: runState,
|
|
repoPath,
|
|
baseBranch: "main",
|
|
worktreeRoot,
|
|
endedAt: new Date(),
|
|
finalReportPath: null,
|
|
});
|
|
await client.db.insert(tuiSessions).values({
|
|
id: sessionId,
|
|
runId,
|
|
roleId: "implementer",
|
|
backend: "fake",
|
|
cwd: worktreeRoot,
|
|
state: "READY",
|
|
});
|
|
await client.db.insert(approvalRequests).values({
|
|
id: approvalRequestId,
|
|
runId,
|
|
gateKey: "spec_approved",
|
|
state: approvalState,
|
|
idempotencyKey: `${runId}:spec_approved::1`,
|
|
payload: { replay: true },
|
|
});
|
|
await client.db.insert(approvalDecisions).values({
|
|
approvalRequestId,
|
|
action,
|
|
idempotencyKey: `${approvalRequestId}:${action}:${clientToken}`,
|
|
});
|
|
|
|
const temporalApi = await startApi({
|
|
dbClient: client,
|
|
temporalClient: new FakeWorkflowClient() as unknown as WorkflowClient,
|
|
taskQueue: "devflow-runs-test",
|
|
workspaceRoot,
|
|
awaitRunStart: false,
|
|
});
|
|
try {
|
|
await temporalApi.engine.signalApproval(runId, approvalRequestId, action, clientToken);
|
|
const [run] = await client.db
|
|
.select({ finalReportPath: runs.finalReportPath })
|
|
.from(runs)
|
|
.where(eq(runs.id, runId));
|
|
expect(run?.finalReportPath).toMatch(/\.report\.md$/);
|
|
const [session] = await client.db
|
|
.select({ state: tuiSessions.state })
|
|
.from(tuiSessions)
|
|
.where(eq(tuiSessions.id, sessionId));
|
|
expect(session).toEqual({ state: "READY" });
|
|
} finally {
|
|
await temporalApi.stop();
|
|
}
|
|
},
|
|
);
|
|
|
|
it("repairs missing terminal final reports during API startup", async () => {
|
|
client = createDbClient(databaseUrl);
|
|
await seedDevelopmentRegistry(client.db);
|
|
const workspaceRoot = createApiWorkspaceRoot();
|
|
const template = (
|
|
await client.db
|
|
.select({ hash: workflowTemplates.hash, id: workflowTemplates.id })
|
|
.from(workflowTemplates)
|
|
.where(eq(workflowTemplates.name, "development"))
|
|
.limit(1)
|
|
)[0];
|
|
if (template === undefined) {
|
|
throw new Error("development template missing");
|
|
}
|
|
const runId = randomUUID();
|
|
const repoPath = createGitRepo();
|
|
const worktreeRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-worktree-")));
|
|
tempRoots.push(repoPath, worktreeRoot);
|
|
runIds.push(runId);
|
|
await client.db.insert(runs).values({
|
|
id: runId,
|
|
templateId: template.id,
|
|
templateHash: template.hash,
|
|
state: "completed",
|
|
repoPath,
|
|
baseBranch: "main",
|
|
worktreeRoot,
|
|
endedAt: new Date(),
|
|
finalReportPath: null,
|
|
});
|
|
|
|
const api = await startM4ApiWhenLockFree({
|
|
dbClient: client,
|
|
workspaceRoot,
|
|
recoveryRunIds: [runId],
|
|
maxConcurrentRuns: 100,
|
|
});
|
|
try {
|
|
expect(api.finalReportRecovery).toEqual([runId]);
|
|
const [run] = await client.db
|
|
.select({ finalReportPath: runs.finalReportPath })
|
|
.from(runs)
|
|
.where(eq(runs.id, runId));
|
|
expect(run?.finalReportPath).toMatch(/\.report\.md$/);
|
|
} finally {
|
|
await api.stop();
|
|
}
|
|
});
|
|
|
|
it("does not sweep active runs when a second API instance fails the singleton lock", async () => {
|
|
client = createDbClient(databaseUrl);
|
|
const first = await startTestM4Api({ dbClient: client, recoveryRunIds: [] });
|
|
const templateId = randomUUID();
|
|
const runId = randomUUID();
|
|
const sessionId = randomUUID();
|
|
const repoPath = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-repo-")));
|
|
const worktreeRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-worktree-")));
|
|
tempRoots.push(repoPath, worktreeRoot);
|
|
templateIds.push(templateId);
|
|
runIds.push(runId);
|
|
try {
|
|
await client.db.insert(workflowTemplates).values({
|
|
id: templateId,
|
|
name: `api-lock-order-${templateId}`,
|
|
version: 1,
|
|
hash: "c".repeat(64),
|
|
definition: { name: "api-lock-order", version: 1, roles: [], phases: [] },
|
|
});
|
|
await client.db.insert(runs).values({
|
|
id: runId,
|
|
templateId,
|
|
templateHash: "c".repeat(64),
|
|
state: "executing",
|
|
repoPath,
|
|
baseBranch: "main",
|
|
worktreeRoot,
|
|
});
|
|
await client.db.insert(tuiSessions).values({
|
|
id: sessionId,
|
|
runId,
|
|
roleId: "spec_writer",
|
|
backend: "fake",
|
|
cwd: worktreeRoot,
|
|
state: "READY",
|
|
});
|
|
|
|
await expect(
|
|
startM4Api({
|
|
dbClient: client,
|
|
workspaceRoot: createApiWorkspaceRoot(),
|
|
recoveryRunIds: [runId],
|
|
}),
|
|
).rejects.toMatchObject({
|
|
code: "session_manager_already_running",
|
|
});
|
|
|
|
const [run] = await client.db
|
|
.select({ state: runs.state })
|
|
.from(runs)
|
|
.where(eq(runs.id, runId));
|
|
expect(run).toEqual({ state: "executing" });
|
|
const [session] = await client.db
|
|
.select({ state: tuiSessions.state })
|
|
.from(tuiSessions)
|
|
.where(eq(tuiSessions.id, sessionId));
|
|
expect(session).toEqual({ state: "READY" });
|
|
const events = await client.db.select().from(runEvents).where(eq(runEvents.runId, runId));
|
|
expect(events).toEqual([]);
|
|
} finally {
|
|
await first.stop();
|
|
}
|
|
});
|
|
|
|
it("ignores terminal-run sessions during SessionManager startup recovery", async () => {
|
|
client = createDbClient(databaseUrl);
|
|
const templateId = randomUUID();
|
|
const runId = randomUUID();
|
|
const sessionId = randomUUID();
|
|
const repoPath = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-repo-")));
|
|
const worktreeRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-worktree-")));
|
|
tempRoots.push(repoPath, worktreeRoot);
|
|
templateIds.push(templateId);
|
|
runIds.push(runId);
|
|
const adapter = new FakeSessionAdapter({
|
|
sessionIdFactory: () => sessionId,
|
|
writeDelayMs: 0,
|
|
});
|
|
await adapter.start({
|
|
runId,
|
|
roleId: "spec_writer",
|
|
backend: "fake",
|
|
cwd: worktreeRoot,
|
|
});
|
|
|
|
await client.db.insert(workflowTemplates).values({
|
|
id: templateId,
|
|
name: `api-session-recovery-${templateId}`,
|
|
version: 1,
|
|
hash: "b".repeat(64),
|
|
definition: { name: "api-session-recovery", version: 1, roles: [], phases: [] },
|
|
});
|
|
await client.db.insert(runs).values({
|
|
id: runId,
|
|
templateId,
|
|
templateHash: "b".repeat(64),
|
|
state: "completed",
|
|
repoPath,
|
|
baseBranch: "main",
|
|
worktreeRoot,
|
|
});
|
|
await client.db.insert(tuiSessions).values({
|
|
id: sessionId,
|
|
runId,
|
|
roleId: "spec_writer",
|
|
backend: "fake",
|
|
cwd: worktreeRoot,
|
|
state: "READY",
|
|
});
|
|
|
|
const result = await startTestM4Api({
|
|
dbClient: client,
|
|
recoveryRunIds: [runId],
|
|
sessionAdapter: adapter,
|
|
});
|
|
try {
|
|
expect(result.recovery).toEqual({ failedSessionIds: [], sweptRunIds: [] });
|
|
expect(result.sessionRecovery).toEqual({
|
|
failedSessionIds: [],
|
|
recoveredSessionIds: [],
|
|
});
|
|
} finally {
|
|
await result.stop();
|
|
}
|
|
const [session] = await client.db
|
|
.select({ state: tuiSessions.state })
|
|
.from(tuiSessions)
|
|
.where(eq(tuiSessions.id, sessionId));
|
|
expect(session).toEqual({ state: "READY" });
|
|
const approvals = await client.db
|
|
.select()
|
|
.from(approvalRequests)
|
|
.where(eq(approvalRequests.runId, runId));
|
|
expect(approvals).toEqual([]);
|
|
const events = await client.db.select().from(runEvents).where(eq(runEvents.runId, runId));
|
|
expect(events).toEqual([]);
|
|
});
|
|
|
|
it("fails CREATED session reservations during SessionManager startup recovery", async () => {
|
|
client = createDbClient(databaseUrl);
|
|
const templateId = randomUUID();
|
|
const runId = randomUUID();
|
|
const sessionId = randomUUID();
|
|
const repoPath = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-repo-")));
|
|
const worktreeRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-worktree-")));
|
|
tempRoots.push(repoPath, worktreeRoot);
|
|
templateIds.push(templateId);
|
|
runIds.push(runId);
|
|
|
|
await client.db.insert(workflowTemplates).values({
|
|
id: templateId,
|
|
name: `api-session-created-${templateId}`,
|
|
version: 1,
|
|
hash: "f".repeat(64),
|
|
definition: { name: "api-session-created", version: 1, roles: [], phases: [] },
|
|
});
|
|
await client.db.insert(runs).values({
|
|
id: runId,
|
|
templateId,
|
|
templateHash: "f".repeat(64),
|
|
state: "executing",
|
|
repoPath,
|
|
baseBranch: "main",
|
|
worktreeRoot,
|
|
});
|
|
await client.db.insert(tuiSessions).values({
|
|
id: sessionId,
|
|
runId,
|
|
roleId: "spec_writer",
|
|
backend: "fake",
|
|
cwd: worktreeRoot,
|
|
state: "CREATED",
|
|
});
|
|
|
|
const adapter = new ResumeFailsFakeSessionAdapter();
|
|
const manager = new SessionManager({
|
|
dbClient: client,
|
|
adapter,
|
|
recoveryRunIds: [runId],
|
|
});
|
|
const recovery = await initializeManagerWhenLockFree(manager);
|
|
try {
|
|
expect(adapter.resumeAttempts).toBe(3);
|
|
expect(recovery).toEqual({ failedSessionIds: [sessionId], recoveredSessionIds: [] });
|
|
} finally {
|
|
await manager.shutdown();
|
|
}
|
|
const [run] = await client.db
|
|
.select({ pausedFromState: runs.pausedFromState, state: runs.state })
|
|
.from(runs)
|
|
.where(eq(runs.id, runId));
|
|
expect(run).toEqual({ pausedFromState: "executing", state: "paused" });
|
|
const [session] = await client.db
|
|
.select({ recoveryAttempts: tuiSessions.recoveryAttempts, state: tuiSessions.state })
|
|
.from(tuiSessions)
|
|
.where(eq(tuiSessions.id, sessionId));
|
|
expect(session).toEqual({ recoveryAttempts: 1, state: "FAILED_NEEDS_HUMAN" });
|
|
const approvals = await client.db
|
|
.select({ gateKey: approvalRequests.gateKey, state: approvalRequests.state })
|
|
.from(approvalRequests)
|
|
.where(eq(approvalRequests.runId, runId));
|
|
expect(approvals).toEqual([{ gateKey: "session_recovery_required", state: "pending" }]);
|
|
const events = await client.db
|
|
.select({ type: runEvents.type })
|
|
.from(runEvents)
|
|
.where(eq(runEvents.runId, runId))
|
|
.orderBy(runEvents.seq);
|
|
expect(events.map((event) => event.type)).toEqual([
|
|
"session.failed",
|
|
"run.paused",
|
|
"approval.requested",
|
|
]);
|
|
});
|
|
|
|
it("retries transient session resume failures during startup recovery", async () => {
|
|
client = createDbClient(databaseUrl);
|
|
const templateId = randomUUID();
|
|
const runId = randomUUID();
|
|
const sessionId = randomUUID();
|
|
const repoPath = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-repo-")));
|
|
const worktreeRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-worktree-")));
|
|
tempRoots.push(repoPath, worktreeRoot);
|
|
templateIds.push(templateId);
|
|
runIds.push(runId);
|
|
const adapter = new ResumeSucceedsAfterTwoFailuresFakeSessionAdapter({
|
|
sessionIdFactory: () => sessionId,
|
|
writeDelayMs: 0,
|
|
});
|
|
await adapter.start({
|
|
runId,
|
|
roleId: "spec_writer",
|
|
backend: "fake",
|
|
cwd: worktreeRoot,
|
|
});
|
|
|
|
await client.db.insert(workflowTemplates).values({
|
|
id: templateId,
|
|
name: `api-session-retry-${templateId}`,
|
|
version: 1,
|
|
hash: "e".repeat(64),
|
|
definition: { name: "api-session-retry", version: 1, roles: [], phases: [] },
|
|
});
|
|
await client.db.insert(runs).values({
|
|
id: runId,
|
|
templateId,
|
|
templateHash: "e".repeat(64),
|
|
state: "executing",
|
|
repoPath,
|
|
baseBranch: "main",
|
|
worktreeRoot,
|
|
});
|
|
await client.db.insert(tuiSessions).values({
|
|
id: sessionId,
|
|
runId,
|
|
roleId: "spec_writer",
|
|
backend: "fake",
|
|
cwd: worktreeRoot,
|
|
state: "READY",
|
|
});
|
|
|
|
const manager = new SessionManager({
|
|
dbClient: client,
|
|
adapter,
|
|
recoveryRunIds: [runId],
|
|
});
|
|
const recovery = await initializeManagerWhenLockFree(manager);
|
|
try {
|
|
expect(adapter.resumeAttempts).toBe(3);
|
|
expect(recovery).toEqual({ failedSessionIds: [], recoveredSessionIds: [sessionId] });
|
|
const approvals = await client.db
|
|
.select()
|
|
.from(approvalRequests)
|
|
.where(eq(approvalRequests.runId, runId));
|
|
expect(approvals).toEqual([]);
|
|
} finally {
|
|
await manager.shutdown();
|
|
}
|
|
});
|
|
|
|
it("pauses a non-terminal run when SessionManager startup recovery cannot resume a session", async () => {
|
|
client = createDbClient(databaseUrl);
|
|
const templateId = randomUUID();
|
|
const runId = randomUUID();
|
|
const sessionId = randomUUID();
|
|
const repoPath = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-repo-")));
|
|
const worktreeRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-worktree-")));
|
|
tempRoots.push(repoPath, worktreeRoot);
|
|
templateIds.push(templateId);
|
|
runIds.push(runId);
|
|
|
|
await client.db.insert(workflowTemplates).values({
|
|
id: templateId,
|
|
name: `api-session-recovery-failure-${templateId}`,
|
|
version: 1,
|
|
hash: "d".repeat(64),
|
|
definition: { name: "api-session-recovery-failure", version: 1, roles: [], phases: [] },
|
|
});
|
|
await client.db.insert(runs).values({
|
|
id: runId,
|
|
templateId,
|
|
templateHash: "d".repeat(64),
|
|
state: "executing",
|
|
repoPath,
|
|
baseBranch: "main",
|
|
worktreeRoot,
|
|
});
|
|
await client.db.insert(tuiSessions).values({
|
|
id: sessionId,
|
|
runId,
|
|
roleId: "spec_writer",
|
|
backend: "fake",
|
|
cwd: worktreeRoot,
|
|
state: "READY",
|
|
});
|
|
|
|
const adapter = new ResumeFailsFakeSessionAdapter();
|
|
const manager = new SessionManager({
|
|
dbClient: client,
|
|
adapter,
|
|
recoveryRunIds: [runId],
|
|
});
|
|
const recovery = await initializeManagerWhenLockFree(manager);
|
|
try {
|
|
expect(adapter.resumeAttempts).toBe(3);
|
|
expect(recovery).toEqual({ failedSessionIds: [sessionId], recoveredSessionIds: [] });
|
|
const [run] = await client.db
|
|
.select({ pausedFromState: runs.pausedFromState, state: runs.state })
|
|
.from(runs)
|
|
.where(eq(runs.id, runId));
|
|
expect(run).toEqual({ pausedFromState: "executing", state: "paused" });
|
|
const [session] = await client.db
|
|
.select({ recoveryAttempts: tuiSessions.recoveryAttempts, state: tuiSessions.state })
|
|
.from(tuiSessions)
|
|
.where(eq(tuiSessions.id, sessionId));
|
|
expect(session).toEqual({ recoveryAttempts: 1, state: "FAILED_NEEDS_HUMAN" });
|
|
const [approval] = await client.db
|
|
.select({
|
|
gateKey: approvalRequests.gateKey,
|
|
phaseId: approvalRequests.phaseId,
|
|
state: approvalRequests.state,
|
|
})
|
|
.from(approvalRequests)
|
|
.where(eq(approvalRequests.runId, runId));
|
|
expect(approval).toEqual({
|
|
gateKey: "session_recovery_required",
|
|
phaseId: null,
|
|
state: "pending",
|
|
});
|
|
const events = await client.db
|
|
.select({ type: runEvents.type })
|
|
.from(runEvents)
|
|
.where(eq(runEvents.runId, runId))
|
|
.orderBy(runEvents.seq);
|
|
expect(events.map((event) => event.type)).toEqual([
|
|
"session.failed",
|
|
"run.paused",
|
|
"approval.requested",
|
|
]);
|
|
} finally {
|
|
await manager.shutdown();
|
|
}
|
|
});
|
|
|
|
it("keeps the singleton lock while shutdown drains in-flight session operations", async () => {
|
|
client = createDbClient(databaseUrl);
|
|
const adapter = new DelayedSendPromptFakeSessionAdapter({ writeDelayMs: 0 });
|
|
const manager = new SessionManager({
|
|
dbClient: client,
|
|
adapter,
|
|
recoveryRunIds: [],
|
|
shutdownDrainMs: 5_000,
|
|
});
|
|
await initializeManagerWhenLockFree(manager);
|
|
const runId = randomUUID();
|
|
const cwd = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-session-")));
|
|
tempRoots.push(cwd);
|
|
const handle = await manager.start({
|
|
runId,
|
|
roleId: "spec_writer",
|
|
backend: "fake",
|
|
cwd,
|
|
});
|
|
const envelope: PromptEnvelope = {
|
|
uuid: randomUUID(),
|
|
runId,
|
|
roleId: "spec_writer",
|
|
phaseKey: "spec",
|
|
attempt: 0,
|
|
expectedArtifact: join(tmpdir(), `${randomUUID()}.json`),
|
|
expectedSchema: "dev/spec@1",
|
|
dedupKey: `dedup-${randomUUID()}`,
|
|
instructions: "Scenario: timeout",
|
|
};
|
|
const promptPromise = manager.sendPrompt(handle, envelope);
|
|
await adapter.promptStarted.promise;
|
|
|
|
const shutdownPromise = manager.shutdown();
|
|
await expect(
|
|
new SessionManager({
|
|
dbClient: client,
|
|
adapter: new FakeSessionAdapter(),
|
|
recoveryRunIds: [],
|
|
}).initialize(),
|
|
).rejects.toMatchObject({ code: "session_manager_already_running" });
|
|
|
|
adapter.releasePrompt.resolve(undefined);
|
|
await expect(promptPromise).resolves.toEqual({ promptId: envelope.dedupKey });
|
|
await shutdownPromise;
|
|
|
|
const nextManager = new SessionManager({
|
|
dbClient: client,
|
|
adapter: new FakeSessionAdapter(),
|
|
recoveryRunIds: [],
|
|
});
|
|
await expect(initializeManagerWhenLockFree(nextManager)).resolves.toEqual({
|
|
failedSessionIds: [],
|
|
recoveredSessionIds: [],
|
|
});
|
|
await nextManager.shutdown();
|
|
});
|
|
|
|
it("keeps the singleton lock while shutdown drains in-flight artifact polling", async () => {
|
|
client = createDbClient(databaseUrl);
|
|
await seedDevelopmentRegistry(client.db);
|
|
const workspaceRoot = createApiWorkspaceRoot();
|
|
const repoPath = createGitRepo();
|
|
tempRoots.push(repoPath);
|
|
const runId = randomUUID();
|
|
runIds.push(runId);
|
|
const api = await startM4ApiWhenLockFree({
|
|
dbClient: client,
|
|
workspaceRoot,
|
|
recoveryRunIds: [],
|
|
maxConcurrentRuns: 100,
|
|
sessionAdapter: new FakeSessionAdapter({ writeDelayMs: 1_000 }),
|
|
});
|
|
const startPromise = api.engine.startRun({
|
|
runId,
|
|
requirementsMd: "Keep artifact polling in flight during shutdown.",
|
|
repoPath,
|
|
baseBranch: "main",
|
|
scenarios: { spec: "ok" },
|
|
});
|
|
await waitForRunEventType(client.db, runId, "artifact.expected");
|
|
|
|
const stopPromise = api.stop();
|
|
await expect(
|
|
new SessionManager({
|
|
dbClient: client,
|
|
adapter: new FakeSessionAdapter(),
|
|
recoveryRunIds: [],
|
|
}).initialize(),
|
|
).rejects.toMatchObject({ code: "session_manager_already_running" });
|
|
|
|
await expect(startPromise).resolves.toEqual({ runId });
|
|
await stopPromise;
|
|
const nextManager = new SessionManager({
|
|
dbClient: client,
|
|
adapter: new FakeSessionAdapter(),
|
|
recoveryRunIds: [],
|
|
});
|
|
await expect(initializeManagerWhenLockFree(nextManager)).resolves.toEqual({
|
|
failedSessionIds: [],
|
|
recoveredSessionIds: [],
|
|
});
|
|
await nextManager.shutdown();
|
|
});
|
|
});
|
|
|
|
async function startM4ApiWhenLockFree(options: Parameters<typeof startM4Api>[0]) {
|
|
const deadline = Date.now() + 6_000;
|
|
let lastError: unknown;
|
|
while (Date.now() < deadline) {
|
|
try {
|
|
return await startM4Api(options);
|
|
} catch (error) {
|
|
lastError = error;
|
|
if (!(error instanceof DevflowError) || error.code !== "session_manager_already_running") {
|
|
throw error;
|
|
}
|
|
await new Promise((resolveWait) => setTimeout(resolveWait, 50));
|
|
}
|
|
}
|
|
throw lastError;
|
|
}
|
|
|
|
async function initializeManagerWhenLockFree(manager: SessionManager) {
|
|
const deadline = Date.now() + 6_000;
|
|
let lastError: unknown;
|
|
while (Date.now() < deadline) {
|
|
try {
|
|
return await manager.initialize();
|
|
} catch (error) {
|
|
lastError = error;
|
|
if (!(error instanceof DevflowError) || error.code !== "session_manager_already_running") {
|
|
throw error;
|
|
}
|
|
await new Promise((resolveWait) => setTimeout(resolveWait, 50));
|
|
}
|
|
}
|
|
throw lastError;
|
|
}
|