chungyeong/dev-puppeteer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add temporal run engine integration

Browse Source
This commit is contained in:
chungyeong
2026-05-13 08:39:19 +09:00
parent 78ebd5ef78
commit aa3033771a
37 changed files with 7338 additions and 224 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
apps/api/package.json
View File

@@ -12,6 +12,8 @@
"@devflow/core": "workspace:*",
"@devflow/db": "workspace:*",
"@devflow/run-engine": "workspace:*",
"@devflow/session": "workspace:*"
"@devflow/session": "workspace:*",
"@devflow/workflows": "workspace:*",
"@temporalio/client": "^1.17.1"
}
}

375
apps/api/src/index.test.ts
View File

@@ -23,10 +23,11 @@ import {
} from "@devflow/db";
import { DbRunEngine } from "@devflow/run-engine";
import { FakeSessionAdapter, type SessionHandle, SessionManager } from "@devflow/session";
import type { WorkflowClient, WorkflowHandle } from "@temporalio/client";
import { and, eq, inArray } from "drizzle-orm";
import { afterEach, describe, expect, it } from "vitest";
import { startApi } from "./index.js";
import { startApi, startM4Api } from "./index.js";
const databaseUrl =
process.env.DATABASE_URL ?? "postgres://devflow:devflow@127.0.0.1:55432/devflow";
@@ -74,6 +75,27 @@ class DelayedSendPromptFakeSessionAdapter extends FakeSessionAdapter {
}
}
class FakeWorkflowClient {
started: { workflowId: string; taskQueue: string; args: unknown[] } | undefined;
async start(
_workflow: unknown,
options: { workflowId: string; taskQueue: string; args: unknown[] },
) {
this.started = {
workflowId: options.workflowId,
taskQueue: options.taskQueue,
args: options.args,
};
}
getHandle(_workflowId: string): Pick<WorkflowHandle, "signal"> {
return {
signal: async () => undefined,
};
}
}
function deferred<T>() {
let resolve!: (value: T | PromiseLike<T>) => void;
let reject!: (reason?: unknown) => void;
@@ -166,8 +188,12 @@ describe("startApi", () => {
return workspaceRoot;
}
function startTestApi(options: Parameters<typeof startApi>[0] = {}) {
return startApi({ workspaceRoot: createApiWorkspaceRoot(), ...options });
function startTestM4Api(options: Parameters<typeof startM4Api>[0] = {}) {
return startM4ApiWhenLockFree({
workspaceRoot: createApiWorkspaceRoot(),
maxConcurrentRuns: 100,
...options,
});
}
afterEach(async () => {
@@ -242,7 +268,7 @@ describe("startApi", () => {
state: "READY",
});
const result = await startTestApi({ dbClient: client, recoveryRunIds: [runId] });
const result = await startTestM4Api({ dbClient: client, recoveryRunIds: [runId] });
try {
expect(result.recovery).toEqual({
failedSessionIds: [sessionId],
@@ -276,9 +302,15 @@ describe("startApi", () => {
it("holds the SessionManager singleton lock until stopped", async () => {
client = createDbClient(databaseUrl);
const recoveryRunIds = [randomUUID()];
const first = await startTestApi({ dbClient: client, recoveryRunIds });
const first = await startTestM4Api({ dbClient: client, recoveryRunIds });
try {
await expect(startTestApi({ dbClient: client, recoveryRunIds })).rejects.toMatchObject({
await expect(
startM4Api({
dbClient: client,
workspaceRoot: createApiWorkspaceRoot(),
recoveryRunIds,
}),
).rejects.toMatchObject({
code: "session_manager_already_running",
});
} finally {
@@ -293,7 +325,12 @@ describe("startApi", () => {
const repoPath = createGitRepo();
tempRoots.push(repoPath);
const api = await startApi({ dbClient: client, workspaceRoot, recoveryRunIds: [] });
const api = await startM4ApiWhenLockFree({
dbClient: client,
workspaceRoot,
recoveryRunIds: [],
maxConcurrentRuns: 100,
});
try {
expect(api.engine).toBeInstanceOf(DbRunEngine);
const { runId } = await api.engine.startRun({
@@ -313,6 +350,190 @@ describe("startApi", () => {
}
});
it("uses the Temporal RunEngine by default without acquiring the SessionManager lock", async () => {
client = createDbClient(databaseUrl);
const first = await startM4ApiWhenLockFree({
dbClient: client,
workspaceRoot: createApiWorkspaceRoot(),
recoveryRunIds: [],
maxConcurrentRuns: 100,
});
const temporalClient = new FakeWorkflowClient();
try {
const temporalApi = await startApi({
dbClient: client,
temporalClient: temporalClient as unknown as WorkflowClient,
taskQueue: "devflow-runs-test",
workspaceRoot: createApiWorkspaceRoot(),
awaitRunStart: false,
});
const runId = randomUUID();
await temporalApi.engine.startRun({
runId,
requirementsMd: "Temporal API should only dispatch workflow commands.",
repoPath: "/repo",
baseBranch: "main",
});
expect(temporalClient.started).toMatchObject({
taskQueue: "devflow-runs-test",
workflowId: `devflow-run:${runId}`,
});
await temporalApi.stop();
} finally {
await first.stop();
}
});
it("wires Temporal approval replay side effects through the API boundary", async () => {
client = createDbClient(databaseUrl);
await seedDevelopmentRegistry(client.db);
const workspaceRoot = createApiWorkspaceRoot();
const template = (
await client.db
.select({ hash: workflowTemplates.hash, id: workflowTemplates.id })
.from(workflowTemplates)
.where(eq(workflowTemplates.name, "development"))
.limit(1)
)[0];
if (template === undefined) {
throw new Error("development template missing");
}
const runId = randomUUID();
const approvalRequestId = randomUUID();
const clientToken = randomUUID();
const repoPath = createGitRepo();
const worktreeRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-worktree-")));
tempRoots.push(repoPath, worktreeRoot);
runIds.push(runId);
await client.db.insert(runs).values({
id: runId,
templateId: template.id,
templateHash: template.hash,
state: "completed",
repoPath,
baseBranch: "main",
worktreeRoot,
endedAt: new Date(),
finalReportPath: null,
});
await client.db.insert(approvalRequests).values({
id: approvalRequestId,
runId,
gateKey: "spec_approved",
state: "approved",
idempotencyKey: `${runId}:spec_approved::1`,
payload: { replay: true },
});
await client.db.insert(approvalDecisions).values({
approvalRequestId,
action: "approve",
idempotencyKey: `${approvalRequestId}:approve:${clientToken}`,
});
const temporalApi = await startApi({
dbClient: client,
temporalClient: new FakeWorkflowClient() as unknown as WorkflowClient,
taskQueue: "devflow-runs-test",
workspaceRoot,
awaitRunStart: false,
});
try {
await temporalApi.engine.signalApproval(runId, approvalRequestId, "approve", clientToken);
const [run] = await client.db
.select({ finalReportPath: runs.finalReportPath })
.from(runs)
.where(eq(runs.id, runId));
expect(run?.finalReportPath).toMatch(/\.report\.md$/);
} finally {
await temporalApi.stop();
}
});
it.each([
{ action: "reject" as const, approvalState: "rejected", runState: "failed" },
{ action: "abort" as const, approvalState: "aborted", runState: "aborted" },
])(
"repairs $runState approval replay reports without mutating sessions through the API",
async ({ action, approvalState, runState }) => {
client = createDbClient(databaseUrl);
await seedDevelopmentRegistry(client.db);
const workspaceRoot = createApiWorkspaceRoot();
const template = (
await client.db
.select({ hash: workflowTemplates.hash, id: workflowTemplates.id })
.from(workflowTemplates)
.where(eq(workflowTemplates.name, "development"))
.limit(1)
)[0];
if (template === undefined) {
throw new Error("development template missing");
}
const runId = randomUUID();
const approvalRequestId = randomUUID();
const clientToken = randomUUID();
const sessionId = randomUUID();
const repoPath = createGitRepo();
const worktreeRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-worktree-")));
tempRoots.push(repoPath, worktreeRoot);
runIds.push(runId);
await client.db.insert(runs).values({
id: runId,
templateId: template.id,
templateHash: template.hash,
state: runState,
repoPath,
baseBranch: "main",
worktreeRoot,
endedAt: new Date(),
finalReportPath: null,
});
await client.db.insert(tuiSessions).values({
id: sessionId,
runId,
roleId: "implementer",
backend: "fake",
cwd: worktreeRoot,
state: "READY",
});
await client.db.insert(approvalRequests).values({
id: approvalRequestId,
runId,
gateKey: "spec_approved",
state: approvalState,
idempotencyKey: `${runId}:spec_approved::1`,
payload: { replay: true },
});
await client.db.insert(approvalDecisions).values({
approvalRequestId,
action,
idempotencyKey: `${approvalRequestId}:${action}:${clientToken}`,
});
const temporalApi = await startApi({
dbClient: client,
temporalClient: new FakeWorkflowClient() as unknown as WorkflowClient,
taskQueue: "devflow-runs-test",
workspaceRoot,
awaitRunStart: false,
});
try {
await temporalApi.engine.signalApproval(runId, approvalRequestId, action, clientToken);
const [run] = await client.db
.select({ finalReportPath: runs.finalReportPath })
.from(runs)
.where(eq(runs.id, runId));
expect(run?.finalReportPath).toMatch(/\.report\.md$/);
const [session] = await client.db
.select({ state: tuiSessions.state })
.from(tuiSessions)
.where(eq(tuiSessions.id, sessionId));
expect(session).toEqual({ state: "READY" });
} finally {
await temporalApi.stop();
}
},
);
it("repairs missing terminal final reports during API startup", async () => {
client = createDbClient(databaseUrl);
await seedDevelopmentRegistry(client.db);
@@ -344,7 +565,12 @@ describe("startApi", () => {
finalReportPath: null,
});
const api = await startApi({ dbClient: client, workspaceRoot, recoveryRunIds: [runId] });
const api = await startM4ApiWhenLockFree({
dbClient: client,
workspaceRoot,
recoveryRunIds: [runId],
maxConcurrentRuns: 100,
});
try {
expect(api.finalReportRecovery).toEqual([runId]);
const [run] = await client.db
@@ -359,7 +585,7 @@ describe("startApi", () => {
it("does not sweep active runs when a second API instance fails the singleton lock", async () => {
client = createDbClient(databaseUrl);
const first = await startTestApi({ dbClient: client, recoveryRunIds: [] });
const first = await startTestM4Api({ dbClient: client, recoveryRunIds: [] });
const templateId = randomUUID();
const runId = randomUUID();
const sessionId = randomUUID();
@@ -395,7 +621,11 @@ describe("startApi", () => {
});
await expect(
startTestApi({ dbClient: client, recoveryRunIds: [runId] }),
startM4Api({
dbClient: client,
workspaceRoot: createApiWorkspaceRoot(),
recoveryRunIds: [runId],
}),
).rejects.toMatchObject({
code: "session_manager_already_running",
});
@@ -463,7 +693,7 @@ describe("startApi", () => {
state: "READY",
});
const result = await startTestApi({
const result = await startTestM4Api({
dbClient: client,
recoveryRunIds: [runId],
sessionAdapter: adapter,
@@ -491,6 +721,82 @@ describe("startApi", () => {
expect(events).toEqual([]);
});
it("fails CREATED session reservations during SessionManager startup recovery", async () => {
client = createDbClient(databaseUrl);
const templateId = randomUUID();
const runId = randomUUID();
const sessionId = randomUUID();
const repoPath = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-repo-")));
const worktreeRoot = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-worktree-")));
tempRoots.push(repoPath, worktreeRoot);
templateIds.push(templateId);
runIds.push(runId);
await client.db.insert(workflowTemplates).values({
id: templateId,
name: `api-session-created-${templateId}`,
version: 1,
hash: "f".repeat(64),
definition: { name: "api-session-created", version: 1, roles: [], phases: [] },
});
await client.db.insert(runs).values({
id: runId,
templateId,
templateHash: "f".repeat(64),
state: "executing",
repoPath,
baseBranch: "main",
worktreeRoot,
});
await client.db.insert(tuiSessions).values({
id: sessionId,
runId,
roleId: "spec_writer",
backend: "fake",
cwd: worktreeRoot,
state: "CREATED",
});
const adapter = new ResumeFailsFakeSessionAdapter();
const manager = new SessionManager({
dbClient: client,
adapter,
recoveryRunIds: [runId],
});
const recovery = await initializeManagerWhenLockFree(manager);
try {
expect(adapter.resumeAttempts).toBe(3);
expect(recovery).toEqual({ failedSessionIds: [sessionId], recoveredSessionIds: [] });
} finally {
await manager.shutdown();
}
const [run] = await client.db
.select({ pausedFromState: runs.pausedFromState, state: runs.state })
.from(runs)
.where(eq(runs.id, runId));
expect(run).toEqual({ pausedFromState: "executing", state: "paused" });
const [session] = await client.db
.select({ recoveryAttempts: tuiSessions.recoveryAttempts, state: tuiSessions.state })
.from(tuiSessions)
.where(eq(tuiSessions.id, sessionId));
expect(session).toEqual({ recoveryAttempts: 1, state: "FAILED_NEEDS_HUMAN" });
const approvals = await client.db
.select({ gateKey: approvalRequests.gateKey, state: approvalRequests.state })
.from(approvalRequests)
.where(eq(approvalRequests.runId, runId));
expect(approvals).toEqual([{ gateKey: "session_recovery_required", state: "pending" }]);
const events = await client.db
.select({ type: runEvents.type })
.from(runEvents)
.where(eq(runEvents.runId, runId))
.orderBy(runEvents.seq);
expect(events.map((event) => event.type)).toEqual([
"session.failed",
"run.paused",
"approval.requested",
]);
});
it("retries transient session resume failures during startup recovery", async () => {
client = createDbClient(databaseUrl);
const templateId = randomUUID();
@@ -542,7 +848,7 @@ describe("startApi", () => {
adapter,
recoveryRunIds: [runId],
});
const recovery = await manager.initialize();
const recovery = await initializeManagerWhenLockFree(manager);
try {
expect(adapter.resumeAttempts).toBe(3);
expect(recovery).toEqual({ failedSessionIds: [], recoveredSessionIds: [sessionId] });
@@ -598,7 +904,7 @@ describe("startApi", () => {
adapter,
recoveryRunIds: [runId],
});
const recovery = await manager.initialize();
const recovery = await initializeManagerWhenLockFree(manager);
try {
expect(adapter.resumeAttempts).toBe(3);
expect(recovery).toEqual({ failedSessionIds: [sessionId], recoveredSessionIds: [] });
@@ -649,7 +955,7 @@ describe("startApi", () => {
recoveryRunIds: [],
shutdownDrainMs: 5_000,
});
await manager.initialize();
await initializeManagerWhenLockFree(manager);
const runId = randomUUID();
const cwd = realpathSync(mkdtempSync(join(tmpdir(), "devflow-api-session-")));
tempRoots.push(cwd);
@@ -691,7 +997,7 @@ describe("startApi", () => {
adapter: new FakeSessionAdapter(),
recoveryRunIds: [],
});
await expect(nextManager.initialize()).resolves.toEqual({
await expect(initializeManagerWhenLockFree(nextManager)).resolves.toEqual({
failedSessionIds: [],
recoveredSessionIds: [],
});
@@ -706,10 +1012,11 @@ describe("startApi", () => {
tempRoots.push(repoPath);
const runId = randomUUID();
runIds.push(runId);
const api = await startApi({
const api = await startM4ApiWhenLockFree({
dbClient: client,
workspaceRoot,
recoveryRunIds: [],
maxConcurrentRuns: 100,
sessionAdapter: new FakeSessionAdapter({ writeDelayMs: 1_000 }),
});
const startPromise = api.engine.startRun({
@@ -737,10 +1044,44 @@ describe("startApi", () => {
adapter: new FakeSessionAdapter(),
recoveryRunIds: [],
});
await expect(nextManager.initialize()).resolves.toEqual({
await expect(initializeManagerWhenLockFree(nextManager)).resolves.toEqual({
failedSessionIds: [],
recoveredSessionIds: [],
});
await nextManager.shutdown();
});
});
async function startM4ApiWhenLockFree(options: Parameters<typeof startM4Api>[0]) {
const deadline = Date.now() + 6_000;
let lastError: unknown;
while (Date.now() < deadline) {
try {
return await startM4Api(options);
} catch (error) {
lastError = error;
if (!(error instanceof DevflowError) || error.code !== "session_manager_already_running") {
throw error;
}
await new Promise((resolveWait) => setTimeout(resolveWait, 50));
}
}
throw lastError;
}
async function initializeManagerWhenLockFree(manager: SessionManager) {
const deadline = Date.now() + 6_000;
let lastError: unknown;
while (Date.now() < deadline) {
try {
return await manager.initialize();
} catch (error) {
lastError = error;
if (!(error instanceof DevflowError) || error.code !== "session_manager_already_running") {
throw error;
}
await new Promise((resolveWait) => setTimeout(resolveWait, 50));
}
}
throw lastError;
}

151
apps/api/src/index.ts
View File

@@ -4,19 +4,22 @@ import { fileURLToPath } from "node:url";
import { type BackendConfig, getConfig } from "@devflow/core";
import { DevflowError } from "@devflow/core";
import { type DbClient, createDbClient } from "@devflow/db";
import { DbRunEngine, type RunEngine } from "@devflow/run-engine";
import { DbRunEngine, type RunEngine, readRunStatus } from "@devflow/run-engine";
import {
FakeSessionAdapter,
type SessionAdapter,
SessionManager,
type SessionManagerRecoveryResult,
type SessionRuntime,
} from "@devflow/session";
import { TemporalRunEngine, temporalNamespace } from "@devflow/workflows";
import { Connection, WorkflowClient } from "@temporalio/client";
import { recoverM4ApiStartup, startM4SessionManager } from "./startup.js";
export * from "./startup.js";
export interface StartApiOptions {
export interface StartM4ApiOptions {
dbClient?: DbClient;
workspaceRoot?: string;
availableBackends?: readonly BackendConfig[];
@@ -24,9 +27,10 @@ export interface StartApiOptions {
sessionAdapter?: SessionAdapter;
sessionManager?: SessionManager;
runEngine?: RunEngine;
maxConcurrentRuns?: number;
}
export interface StartApiResult {
export interface StartM4ApiResult {
recovery: Awaited<ReturnType<typeof recoverM4ApiStartup>>;
sessionRecovery: SessionManagerRecoveryResult;
sessionManager: SessionManager;
@@ -35,7 +39,32 @@ export interface StartApiResult {
stop(): Promise<void>;
}
export interface StartTemporalApiOptions {
dbClient?: DbClient;
temporalClient?: WorkflowClient;
temporalAddress?: string;
taskQueue?: string;
workflowIdPrefix?: string;
awaitRunStart?: boolean;
awaitSignals?: boolean;
availableBackends?: readonly BackendConfig[];
maxConcurrentRuns?: number;
workspaceRoot?: string;
}
export interface StartTemporalApiResult {
engine: RunEngine;
stop(): Promise<void>;
}
export type StartApiOptions = StartTemporalApiOptions;
export type StartApiResult = StartTemporalApiResult;
export async function startApi(options: StartApiOptions = {}): Promise<StartApiResult> {
return startTemporalApi(options);
}
export async function startM4Api(options: StartM4ApiOptions = {}): Promise<StartM4ApiResult> {
const ownedClient = options.dbClient === undefined;
const config = ownedClient || options.workspaceRoot === undefined ? getConfig() : undefined;
const dbClient =
@@ -58,6 +87,9 @@ export async function startApi(options: StartApiOptions = {}): Promise<StartApiR
? {}
: { availableBackends: config.backends }
: { availableBackends: options.availableBackends }),
...(options.maxConcurrentRuns === undefined
? {}
: { maxConcurrentRuns: options.maxConcurrentRuns }),
});
try {
@@ -97,6 +129,119 @@ export async function startApi(options: StartApiOptions = {}): Promise<StartApiR
}
}
export async function startTemporalApi(
options: StartTemporalApiOptions = {},
): Promise<StartTemporalApiResult> {
const ownedClient = options.dbClient === undefined;
const config =
options.dbClient === undefined || options.temporalClient === undefined
? getConfig()
: undefined;
const dbClient =
options.dbClient ?? createDbClient(config?.DATABASE_URL ?? getConfig().DATABASE_URL);
const ownedTemporalClient = options.temporalClient === undefined;
let connection: Connection | undefined;
let temporalClient: WorkflowClient;
if (options.temporalClient === undefined) {
connection = await Connection.connect({
address: options.temporalAddress ?? config?.TEMPORAL_ADDRESS ?? getConfig().TEMPORAL_ADDRESS,
});
temporalClient = new WorkflowClient({ connection, namespace: temporalNamespace });
} else {
temporalClient = options.temporalClient;
}
const replayValidationWorkspaceRoot =
options.workspaceRoot ?? config?.WORKSPACE_ROOT ?? getConfig().WORKSPACE_ROOT;
const replayValidationBackends = options.availableBackends ?? config?.backends;
const replayValidationMaxConcurrentRuns =
options.maxConcurrentRuns ?? config?.MAX_CONCURRENT_RUNS;
const replayValidationEngine = new DbRunEngine({
db: dbClient.db,
sessions: dbOnlySessionRuntime(),
workspaceRoot: replayValidationWorkspaceRoot,
...(replayValidationBackends === undefined
? {}
: { availableBackends: replayValidationBackends }),
...(replayValidationMaxConcurrentRuns === undefined
? {}
: { maxConcurrentRuns: replayValidationMaxConcurrentRuns }),
});
const engine = new TemporalRunEngine({
client: temporalClient,
startReplayValidator: {
validateStartReplay: (input) => replayValidationEngine.validatePreparedRunInput(input),
},
approvalSignalReader: {
readApprovalSignalResult: (runId, approvalRequestId, action, clientToken) =>
replayValidationEngine.readApprovalSignalResult(
runId,
approvalRequestId,
action,
clientToken,
),
validateApprovalSignalInput: (runId, approvalRequestId, action, clientToken) =>
replayValidationEngine.validateApprovalSignalInput(
runId,
approvalRequestId,
action,
clientToken,
),
replayAppliedApprovalSideEffects: (runId, action) =>
replayValidationEngine.replayAppliedApprovalSideEffects(runId, action, {
disposeSessions: false,
}),
},
controlValidator: {
validateResumeSignalInput: (runId) => replayValidationEngine.validateResumeSignalInput(runId),
},
statusReader: {
getStatus: (runId) => readRunStatus(dbClient.db, runId),
},
...(options.taskQueue === undefined ? {} : { taskQueue: options.taskQueue }),
...(options.workflowIdPrefix === undefined
? {}
: { workflowIdPrefix: options.workflowIdPrefix }),
...(options.awaitRunStart === undefined ? {} : { awaitRunStart: options.awaitRunStart }),
...(options.awaitSignals === undefined ? {} : { awaitSignals: options.awaitSignals }),
});
return {
engine,
async stop() {
if (ownedTemporalClient) {
await connection?.close();
}
if (ownedClient) {
await dbClient.close();
}
},
};
}
function dbOnlySessionRuntime(): SessionRuntime {
const rejectMutation = (operation: string) =>
Promise.reject(
new DevflowError("API replay validation cannot mutate TUI sessions", {
class: "fatal",
code: "internal_state_corruption",
recoveryHint: operation,
}),
);
return {
trackOperation: (operation) => operation,
start: () => rejectMutation("start"),
sendPrompt: () => rejectMutation("sendPrompt"),
probe: () => rejectMutation("probe"),
resume: () => rejectMutation("resume"),
rebootstrap: () => rejectMutation("rebootstrap"),
async *capture() {
yield await rejectMutation("capture");
},
dispose: () => rejectMutation("dispose"),
};
}
if (isDirectEntry(import.meta.url, process.argv)) {
startApi()
.then(async (api) => {

3
apps/api/tsconfig.json
View File

@@ -10,6 +10,7 @@
{ "path": "../../packages/core" },
{ "path": "../../packages/db" },
{ "path": "../../packages/run-engine" },
{ "path": "../../packages/session" }
{ "path": "../../packages/session" },
{ "path": "../../packages/workflows" }
]
}