Improve heartbeat execution, run tracking, and agent detail display

Enhance heartbeat service with better process adapter error recovery and run state management. Expand heartbeat-run CLI with additional output and diagnostics. Improve AgentDetail page run history and status display. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-18 13:17:03 -06:00
parent d6024b3ca5
commit 3a91ecbae3
3 changed files with 318 additions and 75 deletions
--- a/cli/src/commands/heartbeat-run.ts
+++ b/cli/src/commands/heartbeat-run.ts
@@ -26,6 +26,29 @@ interface HeartbeatRunOptions {
  debug?: boolean;
 }

+function asRecord(value: unknown): Record<string, unknown> | null {
+  return typeof value === "object" && value !== null && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : null;
+}
+
+function asErrorText(value: unknown): string {
+  if (typeof value === "string") return value;
+  const obj = asRecord(value);
+  if (!obj) return "";
+  const message =
+    (typeof obj.message === "string" && obj.message) ||
+    (typeof obj.error === "string" && obj.error) ||
+    (typeof obj.code === "string" && obj.code) ||
+    "";
+  if (message) return message;
+  try {
+    return JSON.stringify(obj);
+  } catch {
+    return "";
+  }
+}
+
 export async function heartbeatRun(opts: HeartbeatRunOptions): Promise<void> {
  const debug = Boolean(opts.debug);
  const parsedTimeout = Number.parseInt(opts.timeoutMs, 10);
@@ -185,11 +208,20 @@ export async function heartbeatRun(opts: HeartbeatRunOptions): Promise<void> {
      const output = Number(usage.output_tokens ?? 0);
      const cached = Number(usage.cache_read_input_tokens ?? 0);
      const cost = Number(parsed.total_cost_usd ?? 0);
+      const subtype = typeof parsed.subtype === "string" ? parsed.subtype : "";
+      const isError = parsed.is_error === true;
      const resultText = typeof parsed.result === "string" ? parsed.result : "";
      if (resultText) {
        console.log(pc.green("result:"));
        console.log(resultText);
      }
+      const errors = Array.isArray(parsed.errors) ? parsed.errors.map(asErrorText).filter(Boolean) : [];
+      if (subtype.startsWith("error") || isError || errors.length > 0) {
+        console.log(pc.red(`claude_result: subtype=${subtype || "unknown"} is_error=${isError ? "true" : "false"}`));
+        if (errors.length > 0) {
+          console.log(pc.red(`claude_errors: ${errors.join(" | ")}`));
+        }
+      }
      console.log(
        pc.blue(
          `tokens: in=${Number.isFinite(input) ? input : 0} out=${Number.isFinite(output) ? output : 0} cached=${Number.isFinite(cached) ? cached : 0} cost=$${Number.isFinite(cost) ? cost.toFixed(6) : "0.000000"}`,
@@ -255,6 +287,7 @@ export async function heartbeatRun(opts: HeartbeatRunOptions): Promise<void> {
  activeRunId = runId;
  let finalStatus: string | null = null;
  let finalError: string | null = null;
+  let finalRun: HeartbeatRun | null = null;

  const deadline = timeoutMs > 0 ? Date.now() + timeoutMs : null;
  if (!activeRunId) {
@@ -290,6 +323,7 @@ export async function heartbeatRun(opts: HeartbeatRunOptions): Promise<void> {
    if (currentStatus && TERMINAL_STATUSES.has(currentStatus)) {
      finalStatus = currentRun.status;
      finalError = currentRun.error;
+      finalRun = currentRun;
      break;
    }

@@ -337,6 +371,33 @@ export async function heartbeatRun(opts: HeartbeatRunOptions): Promise<void> {
    if (finalError) {
      console.log(pc.red(`Error: ${finalError}`));
    }
+    if (finalRun) {
+      const resultObj = asRecord(finalRun.resultJson);
+      if (resultObj) {
+        const subtype = typeof resultObj.subtype === "string" ? resultObj.subtype : "";
+        const isError = resultObj.is_error === true;
+        const errors = Array.isArray(resultObj.errors) ? resultObj.errors.map(asErrorText).filter(Boolean) : [];
+        const resultText = typeof resultObj.result === "string" ? resultObj.result.trim() : "";
+        if (subtype || isError || errors.length > 0 || resultText) {
+          console.log(pc.red("Claude result details:"));
+          if (subtype) console.log(pc.red(`  subtype: ${subtype}`));
+          if (isError) console.log(pc.red("  is_error: true"));
+          if (errors.length > 0) console.log(pc.red(`  errors: ${errors.join(" | ")}`));
+          if (resultText) console.log(pc.red(`  result: ${resultText}`));
+        }
+      }
+
+      const stderrExcerpt = typeof finalRun.stderrExcerpt === "string" ? finalRun.stderrExcerpt.trim() : "";
+      const stdoutExcerpt = typeof finalRun.stdoutExcerpt === "string" ? finalRun.stdoutExcerpt.trim() : "";
+      if (stderrExcerpt) {
+        console.log(pc.red("stderr excerpt:"));
+        console.log(stderrExcerpt);
+      }
+      if (stdoutExcerpt && (debug || !stderrExcerpt)) {
+        console.log(pc.gray("stdout excerpt:"));
+        console.log(stdoutExcerpt);
+      }
+    }
    process.exitCode = 1;
  } else {
    process.exitCode = 1;
--- a/server/src/services/heartbeat.ts
+++ b/server/src/services/heartbeat.ts
@@ -47,6 +47,7 @@ interface AdapterExecutionResult {
  costUsd?: number | null;
  resultJson?: Record<string, unknown> | null;
  summary?: string | null;
+  clearSession?: boolean;
 }

 interface AdapterInvocationMeta {
@@ -188,6 +189,65 @@ function parseCodexJsonl(stdout: string) {
  };
 }

+function describeClaudeFailure(parsed: Record<string, unknown>): string | null {
+  const subtype = asString(parsed.subtype, "");
+  const resultText = asString(parsed.result, "").trim();
+  const errors = extractClaudeErrorMessages(parsed);
+
+  let detail = resultText;
+  if (!detail && errors.length > 0) {
+    detail = errors[0] ?? "";
+  }
+
+  const parts = ["Claude run failed"];
+  if (subtype) parts.push(`subtype=${subtype}`);
+  if (detail) parts.push(detail);
+  return parts.length > 1 ? parts.join(": ") : null;
+}
+
+function extractClaudeErrorMessages(parsed: Record<string, unknown>): string[] {
+  const raw = Array.isArray(parsed.errors) ? parsed.errors : [];
+  const messages: string[] = [];
+
+  for (const entry of raw) {
+    if (typeof entry === "string") {
+      const msg = entry.trim();
+      if (msg) messages.push(msg);
+      continue;
+    }
+
+    if (typeof entry !== "object" || entry === null || Array.isArray(entry)) {
+      continue;
+    }
+
+    const obj = entry as Record<string, unknown>;
+    const msg = asString(obj.message, "") || asString(obj.error, "") || asString(obj.code, "");
+    if (msg) {
+      messages.push(msg);
+      continue;
+    }
+
+    try {
+      messages.push(JSON.stringify(obj));
+    } catch {
+      // skip non-serializable entry
+    }
+  }
+
+  return messages;
+}
+
+function isClaudeUnknownSessionError(parsed: Record<string, unknown>): boolean {
+  const resultText = asString(parsed.result, "").trim();
+  const allMessages = [resultText, ...extractClaudeErrorMessages(parsed)]
+    .map((msg) => msg.trim())
+    .filter(Boolean);
+
+  return allMessages.some((msg) =>
+    /no conversation found with session id|unknown session|session .* not found/i.test(msg),
+  );
+}
+
 function parseClaudeStreamJson(stdout: string) {
  let sessionId: string | null = null;
  let model = "";
@@ -623,7 +683,7 @@ export function heartbeatService(db: Db) {
      .update(agentRuntimeState)
      .set({
        adapterType: agent.adapterType,
-        sessionId: result.sessionId ?? existing.sessionId,
+        sessionId: result.clearSession ? null : (result.sessionId ?? existing.sessionId),
        lastRunId: run.id,
        lastRunStatus: run.status,
        lastError: result.errorMessage ?? null,
@@ -819,86 +879,143 @@ export function heartbeatService(db: Db) {
      context,
    });

-    const args = ["--print", prompt, "--output-format", "stream-json", "--verbose"];
-    if (sessionId) args.push("--resume", sessionId);
-    if (dangerouslySkipPermissions) args.push("--dangerously-skip-permissions");
-    if (model) args.push("--model", model);
-    if (maxTurns > 0) args.push("--max-turns", String(maxTurns));
-    if (extraArgs.length > 0) args.push(...extraArgs);
+    const buildClaudeArgs = (resumeSessionId: string | null) => {
+      const args = ["--print", prompt, "--output-format", "stream-json", "--verbose"];
+      if (resumeSessionId) args.push("--resume", resumeSessionId);
+      if (dangerouslySkipPermissions) args.push("--dangerously-skip-permissions");
+      if (model) args.push("--model", model);
+      if (maxTurns > 0) args.push("--max-turns", String(maxTurns));
+      if (extraArgs.length > 0) args.push(...extraArgs);
+      return args;
+    };

-    if (onMeta) {
-      await onMeta({
-        adapterType: "claude_local",
-        command,
+    const parseFallbackErrorMessage = (proc: RunProcessResult) => {
+      const stderrLine =
+        proc.stderr
+          .split(/\r?\n/)
+          .map((line) => line.trim())
+          .find(Boolean) ?? "";
+
+      if ((proc.exitCode ?? 0) === 0) {
+        return "Failed to parse claude JSON output";
+      }
+
+      return stderrLine
+        ? `Claude exited with code ${proc.exitCode ?? -1}: ${stderrLine}`
+        : `Claude exited with code ${proc.exitCode ?? -1}`;
+    };
+
+    const runAttempt = async (resumeSessionId: string | null) => {
+      const args = buildClaudeArgs(resumeSessionId);
+      if (onMeta) {
+        await onMeta({
+          adapterType: "claude_local",
+          command,
+          cwd,
+          commandArgs: args.map((value, idx) => (idx === 1 ? `<prompt ${prompt.length} chars>` : value)),
+          env: redactEnvForLogs(env),
+          prompt,
+          context,
+        });
+      }
+
+      const proc = await runChildProcess(runId, command, args, {
        cwd,
-        commandArgs: args.map((value, idx) => (idx === 1 ? `<prompt ${prompt.length} chars>` : value)),
-        env: redactEnvForLogs(env),
-        prompt,
-        context,
+        env,
+        timeoutSec,
+        graceSec,
+        onLog,
      });
-    }

-    const proc = await runChildProcess(runId, command, args, {
-      cwd,
-      env,
-      timeoutSec,
-      graceSec,
-      onLog,
-    });
+      const parsedStream = parseClaudeStreamJson(proc.stdout);
+      const parsed = parsedStream.resultJson ?? parseJson(proc.stdout);
+      return { proc, parsedStream, parsed };
+    };

-    if (proc.timedOut) {
-      return {
-        exitCode: proc.exitCode,
-        signal: proc.signal,
-        timedOut: true,
-        errorMessage: `Timed out after ${timeoutSec}s`,
-      };
-    }
+    const toAdapterResult = (
+      attempt: {
+        proc: RunProcessResult;
+        parsedStream: ReturnType<typeof parseClaudeStreamJson>;
+        parsed: Record<string, unknown> | null;
+      },
+      opts: { fallbackSessionId: string | null; clearSessionOnMissingSession?: boolean },
+    ): AdapterExecutionResult => {
+      const { proc, parsedStream, parsed } = attempt;
+      if (proc.timedOut) {
+        return {
+          exitCode: proc.exitCode,
+          signal: proc.signal,
+          timedOut: true,
+          errorMessage: `Timed out after ${timeoutSec}s`,
+          clearSession: Boolean(opts.clearSessionOnMissingSession),
+        };
+      }
+
+      if (!parsed) {
+        return {
+          exitCode: proc.exitCode,
+          signal: proc.signal,
+          timedOut: false,
+          errorMessage: parseFallbackErrorMessage(proc),
+          resultJson: {
+            stdout: proc.stdout,
+            stderr: proc.stderr,
+          },
+          clearSession: Boolean(opts.clearSessionOnMissingSession),
+        };
+      }
+
+      const usage =
+        parsedStream.usage ??
+        (() => {
+          const usageObj = parseObject(parsed.usage);
+          return {
+            inputTokens: asNumber(usageObj.input_tokens, 0),
+            cachedInputTokens: asNumber(usageObj.cache_read_input_tokens, 0),
+            outputTokens: asNumber(usageObj.output_tokens, 0),
+          };
+        })();
+
+      const resolvedSessionId =
+        parsedStream.sessionId ??
+        (asString(parsed.session_id, opts.fallbackSessionId ?? "") || opts.fallbackSessionId);

-    const parsedStream = parseClaudeStreamJson(proc.stdout);
-    const parsed = parsedStream.resultJson ?? parseJson(proc.stdout);
-    if (!parsed) {
      return {
        exitCode: proc.exitCode,
        signal: proc.signal,
        timedOut: false,
        errorMessage:
          (proc.exitCode ?? 0) === 0
-            ? "Failed to parse claude JSON output"
-            : `Claude exited with code ${proc.exitCode ?? -1}`,
-        resultJson: {
-          stdout: proc.stdout,
-          stderr: proc.stderr,
-        },
+            ? null
+            : describeClaudeFailure(parsed) ?? `Claude exited with code ${proc.exitCode ?? -1}`,
+        usage,
+        sessionId: resolvedSessionId,
+        provider: "anthropic",
+        model: parsedStream.model || asString(parsed.model, model),
+        costUsd: parsedStream.costUsd ?? asNumber(parsed.total_cost_usd, 0),
+        resultJson: parsed,
+        summary: parsedStream.summary || asString(parsed.result, ""),
+        clearSession: Boolean(opts.clearSessionOnMissingSession && !resolvedSessionId),
      };
+    };
+
+    const initial = await runAttempt(sessionId ?? null);
+    if (
+      sessionId &&
+      !initial.proc.timedOut &&
+      (initial.proc.exitCode ?? 0) !== 0 &&
+      initial.parsed &&
+      isClaudeUnknownSessionError(initial.parsed)
+    ) {
+      await onLog(
+        "stderr",
+        `[paperclip] Claude resume session "${sessionId}" is unavailable; retrying with a fresh session.\n`,
+      );
+      const retry = await runAttempt(null);
+      return toAdapterResult(retry, { fallbackSessionId: null, clearSessionOnMissingSession: true });
    }

-    const usage =
-      parsedStream.usage ??
-      (() => {
-        const usageObj = parseObject(parsed.usage);
-        return {
-          inputTokens: asNumber(usageObj.input_tokens, 0),
-          cachedInputTokens: asNumber(usageObj.cache_read_input_tokens, 0),
-          outputTokens: asNumber(usageObj.output_tokens, 0),
-        };
-      })();
-
-    return {
-      exitCode: proc.exitCode,
-      signal: proc.signal,
-      timedOut: false,
-      errorMessage: (proc.exitCode ?? 0) === 0 ? null : `Claude exited with code ${proc.exitCode ?? -1}`,
-      usage,
-      sessionId:
-        parsedStream.sessionId ??
-        (asString(parsed.session_id, runtime.sessionId ?? "") || runtime.sessionId),
-      provider: "anthropic",
-      model: parsedStream.model || asString(parsed.model, model),
-      costUsd: parsedStream.costUsd ?? asNumber(parsed.total_cost_usd, 0),
-      resultJson: parsed,
-      summary: parsedStream.summary || asString(parsed.result, ""),
-    };
+    return toAdapterResult(initial, { fallbackSessionId: runtime.sessionId });
  }

  async function executeCodexLocalRun(
--- a/ui/src/pages/AgentDetail.tsx
+++ b/ui/src/pages/AgentDetail.tsx
@@ -110,7 +110,7 @@ type TranscriptEntry =
  | { kind: "assistant"; ts: string; text: string }
  | { kind: "tool_call"; ts: string; name: string; input: unknown }
  | { kind: "init"; ts: string; model: string; sessionId: string }
-  | { kind: "result"; ts: string; text: string; inputTokens: number; outputTokens: number; cachedTokens: number; costUsd: number }
+  | { kind: "result"; ts: string; text: string; inputTokens: number; outputTokens: number; cachedTokens: number; costUsd: number; subtype: string; isError: boolean; errors: string[] }
  | { kind: "stderr"; ts: string; text: string }
  | { kind: "system"; ts: string; text: string }
  | { kind: "stdout"; ts: string; text: string };
@@ -124,6 +124,23 @@ function asNumber(value: unknown): number {
  return typeof value === "number" && Number.isFinite(value) ? value : 0;
 }

+function errorText(value: unknown): string {
+  if (typeof value === "string") return value;
+  const rec = asRecord(value);
+  if (!rec) return "";
+  const msg =
+    (typeof rec.message === "string" && rec.message) ||
+    (typeof rec.error === "string" && rec.error) ||
+    (typeof rec.code === "string" && rec.code) ||
+    "";
+  if (msg) return msg;
+  try {
+    return JSON.stringify(rec);
+  } catch {
+    return "";
+  }
+}
+
 function parseClaudeStdoutLine(line: string, ts: string): TranscriptEntry[] {
  const parsed = asRecord(safeJsonParse(line));
  if (!parsed) {
@@ -171,6 +188,9 @@ function parseClaudeStdoutLine(line: string, ts: string): TranscriptEntry[] {
    const outputTokens = asNumber(usage.output_tokens);
    const cachedTokens = asNumber(usage.cache_read_input_tokens);
    const costUsd = asNumber(parsed.total_cost_usd);
+    const subtype = typeof parsed.subtype === "string" ? parsed.subtype : "";
+    const isError = parsed.is_error === true;
+    const errors = Array.isArray(parsed.errors) ? parsed.errors.map(errorText).filter(Boolean) : [];
    const text = typeof parsed.result === "string" ? parsed.result : "";
    return [{
      kind: "result",
@@ -180,6 +200,9 @@ function parseClaudeStdoutLine(line: string, ts: string): TranscriptEntry[] {
      outputTokens,
      cachedTokens,
      costUsd,
+      subtype,
+      isError,
+      errors,
    }];
  }

@@ -701,12 +724,12 @@ function RunsTab({ runs, companyId, agentId, selectedRunId }: { runs: HeartbeatR
  const selectedRun = sorted.find((r) => r.id === effectiveRunId) ?? null;

  return (
-    <div className="flex gap-0 border border-border rounded-lg overflow-hidden" style={{ height: "calc(100vh - 220px)" }}>
-      {/* Left: run list */}
+    <div className="flex gap-0">
+      {/* Left: run list — sticky, scrolls independently */}
      <div className={cn(
-        "shrink-0 overflow-y-auto border-r border-border",
+        "shrink-0 border border-border rounded-lg overflow-y-auto sticky top-4 self-start",
        selectedRun ? "w-72" : "w-full",
-      )}>
+      )} style={{ maxHeight: "calc(100vh - 2rem)" }}>
        {sorted.map((run) => {
          const statusInfo = runStatusIcons[run.status] ?? { icon: Clock, color: "text-neutral-400" };
          const StatusIcon = statusInfo.icon;
@@ -759,9 +782,9 @@ function RunsTab({ runs, companyId, agentId, selectedRunId }: { runs: HeartbeatR
        })}
      </div>

-      {/* Right: run detail */}
+      {/* Right: run detail — natural height, page scrolls */}
      {selectedRun && (
-        <div className="flex-1 min-w-0 overflow-y-auto pr-2">
+        <div className="flex-1 min-w-0 pl-4">
          <RunDetail key={selectedRun.id} run={selectedRun} />
        </div>
      )}
@@ -1165,7 +1188,7 @@ function LogViewer({ run }: { run: HeartbeatRun }) {
          </span>
        )}
      </div>
-      <div className="bg-neutral-950 rounded-lg p-3 font-mono text-xs max-h-80 overflow-y-auto space-y-0.5">
+      <div className="bg-neutral-950 rounded-lg p-3 font-mono text-xs space-y-0.5" style={{ maxHeight: "200vh" }}>
        {transcript.length === 0 && !run.logRef && (
          <div className="text-neutral-500">No persisted transcript for this run.</div>
        )}
@@ -1218,6 +1241,12 @@ function LogViewer({ run }: { run: HeartbeatRun }) {
                    tokens in={formatTokens(entry.inputTokens)} out={formatTokens(entry.outputTokens)} cached={formatTokens(entry.cachedTokens)} cost=${entry.costUsd.toFixed(6)}
                  </span>
                </div>
+                {(entry.subtype || entry.isError || entry.errors.length > 0) && (
+                  <div className="ml-[74px] text-red-300 whitespace-pre-wrap break-all">
+                    subtype={entry.subtype || "unknown"} is_error={entry.isError ? "true" : "false"}
+                    {entry.errors.length > 0 ? ` errors=${entry.errors.join(" | ")}` : ""}
+                  </div>
+                )}
                {entry.text && (
                  <div className="ml-[74px] whitespace-pre-wrap break-all text-neutral-100">{entry.text}</div>
                )}
@@ -1252,10 +1281,46 @@ function LogViewer({ run }: { run: HeartbeatRun }) {
        <div ref={logEndRef} />
      </div>

+      {(run.status === "failed" || run.status === "timed_out") && (
+        <div className="rounded-lg border border-red-500/30 bg-red-950/20 p-3 space-y-2">
+          <div className="text-xs font-medium text-red-300">Failure details</div>
+          {run.error && (
+            <div className="text-xs text-red-200">
+              <span className="text-red-300">Error: </span>
+              {run.error}
+            </div>
+          )}
+          {run.stderrExcerpt && run.stderrExcerpt.trim() && (
+            <div>
+              <div className="text-xs text-red-300 mb-1">stderr excerpt</div>
+              <pre className="bg-neutral-950 rounded-md p-2 text-xs overflow-x-auto whitespace-pre-wrap text-red-100">
+                {run.stderrExcerpt}
+              </pre>
+            </div>
+          )}
+          {run.resultJson && (
+            <div>
+              <div className="text-xs text-red-300 mb-1">adapter result JSON</div>
+              <pre className="bg-neutral-950 rounded-md p-2 text-xs overflow-x-auto whitespace-pre-wrap text-red-100">
+                {JSON.stringify(run.resultJson, null, 2)}
+              </pre>
+            </div>
+          )}
+          {run.stdoutExcerpt && run.stdoutExcerpt.trim() && !run.resultJson && (
+            <div>
+              <div className="text-xs text-red-300 mb-1">stdout excerpt</div>
+              <pre className="bg-neutral-950 rounded-md p-2 text-xs overflow-x-auto whitespace-pre-wrap text-red-100">
+                {run.stdoutExcerpt}
+              </pre>
+            </div>
+          )}
+        </div>
+      )}
+
      {events.length > 0 && (
        <div>
          <div className="mb-2 text-xs font-medium text-muted-foreground">Events ({events.length})</div>
-          <div className="bg-neutral-950 rounded-lg p-3 font-mono text-xs max-h-56 overflow-y-auto space-y-0.5">
+          <div className="bg-neutral-950 rounded-lg p-3 font-mono text-xs space-y-0.5" style={{ maxHeight: "100vh" }}>
            {events.map((evt) => {
              const color = evt.color
                ?? (evt.level ? levelColors[evt.level] : null)