feat: add tui recovery state machine

apps/api/src/index.ts

@@ -28,6 +28,7 @@ export interface StartM4ApiOptions {
   sessionManager?: SessionManager;
   runEngine?: RunEngine;
   maxConcurrentRuns?: number;
+  sessionMaxHungMs?: number;
 }
 
 export interface StartM4ApiResult {
@@ -50,6 +51,7 @@ export interface StartTemporalApiOptions {
   availableBackends?: readonly BackendConfig[];
   maxConcurrentRuns?: number;
   workspaceRoot?: string;
+  sessionMaxHungMs?: number;
 }
 
 export interface StartTemporalApiResult {
@@ -69,6 +71,7 @@ export async function startM4Api(options: StartM4ApiOptions = {}): Promise<Start
   const config = ownedClient || options.workspaceRoot === undefined ? getConfig() : undefined;
   const dbClient =
     options.dbClient ?? createDbClient(config?.DATABASE_URL ?? getConfig().DATABASE_URL);
+  const sessionMaxHungMs = options.sessionMaxHungMs ?? config?.SESSION_MAX_HUNG_MS;
   const sessionManager =
     options.sessionManager ??
     new SessionManager({
@@ -90,6 +93,7 @@ export async function startM4Api(options: StartM4ApiOptions = {}): Promise<Start
       ...(options.maxConcurrentRuns === undefined
         ? {}
         : { maxConcurrentRuns: options.maxConcurrentRuns }),
+      ...(sessionMaxHungMs === undefined ? {} : { recovery: { maxHungMs: sessionMaxHungMs } }),
     });
 
   try {
@@ -158,6 +162,7 @@ export async function startTemporalApi(
   const replayValidationBackends = options.availableBackends ?? config?.backends;
   const replayValidationMaxConcurrentRuns =
     options.maxConcurrentRuns ?? config?.MAX_CONCURRENT_RUNS;
+  const replayValidationSessionMaxHungMs = options.sessionMaxHungMs ?? config?.SESSION_MAX_HUNG_MS;
   const replayValidationEngine = new DbRunEngine({
     db: dbClient.db,
     sessions: dbOnlySessionRuntime(),
@@ -168,6 +173,9 @@ export async function startTemporalApi(
     ...(replayValidationMaxConcurrentRuns === undefined
       ? {}
       : { maxConcurrentRuns: replayValidationMaxConcurrentRuns }),
+    ...(replayValidationSessionMaxHungMs === undefined
+      ? {}
+      : { recovery: { maxHungMs: replayValidationSessionMaxHungMs } }),
   });
   const engine = new TemporalRunEngine({
     client: temporalClient,

apps/worker/src/index.test.ts

@@ -110,6 +110,7 @@ describe("startWorker", () => {
         TEMPORAL_ADDRESS: "localhost:7233",
         WORKSPACE_ROOT: worktreeRoot,
         MAX_CONCURRENT_RUNS: 4,
+        SESSION_MAX_HUNG_MS: 20 * 60 * 1000,
         backends: [{ id: "fake", enabled: true }],
       },
       dbClient: client,
@@ -148,6 +149,7 @@ describe("startWorker", () => {
         TEMPORAL_ADDRESS: "localhost:7233",
         WORKSPACE_ROOT: realpathSync(mkdtempSync(join(tmpdir(), "devflow-worker-workspace-"))),
         MAX_CONCURRENT_RUNS: 4,
+        SESSION_MAX_HUNG_MS: 20 * 60 * 1000,
         backends: [{ id: "fake", enabled: true }],
       },
       dbClient: client,
@@ -165,6 +167,7 @@ describe("startWorker", () => {
             TEMPORAL_ADDRESS: "localhost:7233",
             WORKSPACE_ROOT: realpathSync(mkdtempSync(join(tmpdir(), "devflow-worker-workspace-"))),
             MAX_CONCURRENT_RUNS: 4,
+            SESSION_MAX_HUNG_MS: 20 * 60 * 1000,
             backends: [{ id: "fake", enabled: true }],
           },
           dbClient: client,
@@ -191,6 +194,7 @@ describe("startWorker", () => {
         TEMPORAL_ADDRESS: "localhost:7233",
         WORKSPACE_ROOT: workspaceRoot,
         MAX_CONCURRENT_RUNS: 4,
+        SESSION_MAX_HUNG_MS: 20 * 60 * 1000,
         backends: [{ id: "fake", enabled: true }],
       },
       dbClient: client,
@@ -211,6 +215,7 @@ describe("startWorker", () => {
         TEMPORAL_ADDRESS: "localhost:7233",
         WORKSPACE_ROOT: workspaceRoot,
         MAX_CONCURRENT_RUNS: 4,
+        SESSION_MAX_HUNG_MS: 20 * 60 * 1000,
         backends: [{ id: "fake", enabled: true }],
       },
       dbClient: client,
@@ -233,6 +238,7 @@ describe("startWorker", () => {
         TEMPORAL_ADDRESS: "localhost:7233",
         WORKSPACE_ROOT: workspaceRoot,
         MAX_CONCURRENT_RUNS: 4,
+        SESSION_MAX_HUNG_MS: 20 * 60 * 1000,
         backends: [{ id: "fake", enabled: true }],
       },
       dbClient: client,
@@ -251,6 +257,7 @@ describe("startWorker", () => {
         TEMPORAL_ADDRESS: "localhost:7233",
         WORKSPACE_ROOT: workspaceRoot,
         MAX_CONCURRENT_RUNS: 4,
+        SESSION_MAX_HUNG_MS: 20 * 60 * 1000,
         backends: [{ id: "fake", enabled: true }],
       },
       dbClient: client,

apps/worker/src/index.ts

@@ -51,6 +51,7 @@ export async function startWorker(options: StartWorkerOptions = {}) {
         workspaceRoot: config.WORKSPACE_ROOT,
         availableBackends: config.backends,
         maxConcurrentRuns: config.MAX_CONCURRENT_RUNS,
+        recovery: { maxHungMs: config.SESSION_MAX_HUNG_MS },
       }),
       connection: connection as NativeConnection,
       namespace: "devflow",