diff --git a/cli/src/commands/heartbeat-run.ts b/cli/src/commands/heartbeat-run.ts index 82975076..ecdd7f8c 100644 --- a/cli/src/commands/heartbeat-run.ts +++ b/cli/src/commands/heartbeat-run.ts @@ -26,6 +26,29 @@ interface HeartbeatRunOptions { debug?: boolean; } +function asRecord(value: unknown): Record | null { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record) + : null; +} + +function asErrorText(value: unknown): string { + if (typeof value === "string") return value; + const obj = asRecord(value); + if (!obj) return ""; + const message = + (typeof obj.message === "string" && obj.message) || + (typeof obj.error === "string" && obj.error) || + (typeof obj.code === "string" && obj.code) || + ""; + if (message) return message; + try { + return JSON.stringify(obj); + } catch { + return ""; + } +} + export async function heartbeatRun(opts: HeartbeatRunOptions): Promise { const debug = Boolean(opts.debug); const parsedTimeout = Number.parseInt(opts.timeoutMs, 10); @@ -185,11 +208,20 @@ export async function heartbeatRun(opts: HeartbeatRunOptions): Promise { const output = Number(usage.output_tokens ?? 0); const cached = Number(usage.cache_read_input_tokens ?? 0); const cost = Number(parsed.total_cost_usd ?? 0); + const subtype = typeof parsed.subtype === "string" ? parsed.subtype : ""; + const isError = parsed.is_error === true; const resultText = typeof parsed.result === "string" ? parsed.result : ""; if (resultText) { console.log(pc.green("result:")); console.log(resultText); } + const errors = Array.isArray(parsed.errors) ? parsed.errors.map(asErrorText).filter(Boolean) : []; + if (subtype.startsWith("error") || isError || errors.length > 0) { + console.log(pc.red(`claude_result: subtype=${subtype || "unknown"} is_error=${isError ? "true" : "false"}`)); + if (errors.length > 0) { + console.log(pc.red(`claude_errors: ${errors.join(" | ")}`)); + } + } console.log( pc.blue( `tokens: in=${Number.isFinite(input) ? input : 0} out=${Number.isFinite(output) ? output : 0} cached=${Number.isFinite(cached) ? cached : 0} cost=$${Number.isFinite(cost) ? cost.toFixed(6) : "0.000000"}`, @@ -255,6 +287,7 @@ export async function heartbeatRun(opts: HeartbeatRunOptions): Promise { activeRunId = runId; let finalStatus: string | null = null; let finalError: string | null = null; + let finalRun: HeartbeatRun | null = null; const deadline = timeoutMs > 0 ? Date.now() + timeoutMs : null; if (!activeRunId) { @@ -290,6 +323,7 @@ export async function heartbeatRun(opts: HeartbeatRunOptions): Promise { if (currentStatus && TERMINAL_STATUSES.has(currentStatus)) { finalStatus = currentRun.status; finalError = currentRun.error; + finalRun = currentRun; break; } @@ -337,6 +371,33 @@ export async function heartbeatRun(opts: HeartbeatRunOptions): Promise { if (finalError) { console.log(pc.red(`Error: ${finalError}`)); } + if (finalRun) { + const resultObj = asRecord(finalRun.resultJson); + if (resultObj) { + const subtype = typeof resultObj.subtype === "string" ? resultObj.subtype : ""; + const isError = resultObj.is_error === true; + const errors = Array.isArray(resultObj.errors) ? resultObj.errors.map(asErrorText).filter(Boolean) : []; + const resultText = typeof resultObj.result === "string" ? resultObj.result.trim() : ""; + if (subtype || isError || errors.length > 0 || resultText) { + console.log(pc.red("Claude result details:")); + if (subtype) console.log(pc.red(` subtype: ${subtype}`)); + if (isError) console.log(pc.red(" is_error: true")); + if (errors.length > 0) console.log(pc.red(` errors: ${errors.join(" | ")}`)); + if (resultText) console.log(pc.red(` result: ${resultText}`)); + } + } + + const stderrExcerpt = typeof finalRun.stderrExcerpt === "string" ? finalRun.stderrExcerpt.trim() : ""; + const stdoutExcerpt = typeof finalRun.stdoutExcerpt === "string" ? finalRun.stdoutExcerpt.trim() : ""; + if (stderrExcerpt) { + console.log(pc.red("stderr excerpt:")); + console.log(stderrExcerpt); + } + if (stdoutExcerpt && (debug || !stderrExcerpt)) { + console.log(pc.gray("stdout excerpt:")); + console.log(stdoutExcerpt); + } + } process.exitCode = 1; } else { process.exitCode = 1; diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index c2ff7e89..f42630f2 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -47,6 +47,7 @@ interface AdapterExecutionResult { costUsd?: number | null; resultJson?: Record | null; summary?: string | null; + clearSession?: boolean; } interface AdapterInvocationMeta { @@ -188,6 +189,65 @@ function parseCodexJsonl(stdout: string) { }; } +function describeClaudeFailure(parsed: Record): string | null { + const subtype = asString(parsed.subtype, ""); + const resultText = asString(parsed.result, "").trim(); + const errors = extractClaudeErrorMessages(parsed); + + let detail = resultText; + if (!detail && errors.length > 0) { + detail = errors[0] ?? ""; + } + + const parts = ["Claude run failed"]; + if (subtype) parts.push(`subtype=${subtype}`); + if (detail) parts.push(detail); + return parts.length > 1 ? parts.join(": ") : null; +} + +function extractClaudeErrorMessages(parsed: Record): string[] { + const raw = Array.isArray(parsed.errors) ? parsed.errors : []; + const messages: string[] = []; + + for (const entry of raw) { + if (typeof entry === "string") { + const msg = entry.trim(); + if (msg) messages.push(msg); + continue; + } + + if (typeof entry !== "object" || entry === null || Array.isArray(entry)) { + continue; + } + + const obj = entry as Record; + const msg = asString(obj.message, "") || asString(obj.error, "") || asString(obj.code, ""); + if (msg) { + messages.push(msg); + continue; + } + + try { + messages.push(JSON.stringify(obj)); + } catch { + // skip non-serializable entry + } + } + + return messages; +} + +function isClaudeUnknownSessionError(parsed: Record): boolean { + const resultText = asString(parsed.result, "").trim(); + const allMessages = [resultText, ...extractClaudeErrorMessages(parsed)] + .map((msg) => msg.trim()) + .filter(Boolean); + + return allMessages.some((msg) => + /no conversation found with session id|unknown session|session .* not found/i.test(msg), + ); +} + function parseClaudeStreamJson(stdout: string) { let sessionId: string | null = null; let model = ""; @@ -623,7 +683,7 @@ export function heartbeatService(db: Db) { .update(agentRuntimeState) .set({ adapterType: agent.adapterType, - sessionId: result.sessionId ?? existing.sessionId, + sessionId: result.clearSession ? null : (result.sessionId ?? existing.sessionId), lastRunId: run.id, lastRunStatus: run.status, lastError: result.errorMessage ?? null, @@ -819,86 +879,143 @@ export function heartbeatService(db: Db) { context, }); - const args = ["--print", prompt, "--output-format", "stream-json", "--verbose"]; - if (sessionId) args.push("--resume", sessionId); - if (dangerouslySkipPermissions) args.push("--dangerously-skip-permissions"); - if (model) args.push("--model", model); - if (maxTurns > 0) args.push("--max-turns", String(maxTurns)); - if (extraArgs.length > 0) args.push(...extraArgs); + const buildClaudeArgs = (resumeSessionId: string | null) => { + const args = ["--print", prompt, "--output-format", "stream-json", "--verbose"]; + if (resumeSessionId) args.push("--resume", resumeSessionId); + if (dangerouslySkipPermissions) args.push("--dangerously-skip-permissions"); + if (model) args.push("--model", model); + if (maxTurns > 0) args.push("--max-turns", String(maxTurns)); + if (extraArgs.length > 0) args.push(...extraArgs); + return args; + }; - if (onMeta) { - await onMeta({ - adapterType: "claude_local", - command, + const parseFallbackErrorMessage = (proc: RunProcessResult) => { + const stderrLine = + proc.stderr + .split(/\r?\n/) + .map((line) => line.trim()) + .find(Boolean) ?? ""; + + if ((proc.exitCode ?? 0) === 0) { + return "Failed to parse claude JSON output"; + } + + return stderrLine + ? `Claude exited with code ${proc.exitCode ?? -1}: ${stderrLine}` + : `Claude exited with code ${proc.exitCode ?? -1}`; + }; + + const runAttempt = async (resumeSessionId: string | null) => { + const args = buildClaudeArgs(resumeSessionId); + if (onMeta) { + await onMeta({ + adapterType: "claude_local", + command, + cwd, + commandArgs: args.map((value, idx) => (idx === 1 ? `` : value)), + env: redactEnvForLogs(env), + prompt, + context, + }); + } + + const proc = await runChildProcess(runId, command, args, { cwd, - commandArgs: args.map((value, idx) => (idx === 1 ? `` : value)), - env: redactEnvForLogs(env), - prompt, - context, + env, + timeoutSec, + graceSec, + onLog, }); - } - const proc = await runChildProcess(runId, command, args, { - cwd, - env, - timeoutSec, - graceSec, - onLog, - }); + const parsedStream = parseClaudeStreamJson(proc.stdout); + const parsed = parsedStream.resultJson ?? parseJson(proc.stdout); + return { proc, parsedStream, parsed }; + }; - if (proc.timedOut) { - return { - exitCode: proc.exitCode, - signal: proc.signal, - timedOut: true, - errorMessage: `Timed out after ${timeoutSec}s`, - }; - } + const toAdapterResult = ( + attempt: { + proc: RunProcessResult; + parsedStream: ReturnType; + parsed: Record | null; + }, + opts: { fallbackSessionId: string | null; clearSessionOnMissingSession?: boolean }, + ): AdapterExecutionResult => { + const { proc, parsedStream, parsed } = attempt; + if (proc.timedOut) { + return { + exitCode: proc.exitCode, + signal: proc.signal, + timedOut: true, + errorMessage: `Timed out after ${timeoutSec}s`, + clearSession: Boolean(opts.clearSessionOnMissingSession), + }; + } + + if (!parsed) { + return { + exitCode: proc.exitCode, + signal: proc.signal, + timedOut: false, + errorMessage: parseFallbackErrorMessage(proc), + resultJson: { + stdout: proc.stdout, + stderr: proc.stderr, + }, + clearSession: Boolean(opts.clearSessionOnMissingSession), + }; + } + + const usage = + parsedStream.usage ?? + (() => { + const usageObj = parseObject(parsed.usage); + return { + inputTokens: asNumber(usageObj.input_tokens, 0), + cachedInputTokens: asNumber(usageObj.cache_read_input_tokens, 0), + outputTokens: asNumber(usageObj.output_tokens, 0), + }; + })(); + + const resolvedSessionId = + parsedStream.sessionId ?? + (asString(parsed.session_id, opts.fallbackSessionId ?? "") || opts.fallbackSessionId); - const parsedStream = parseClaudeStreamJson(proc.stdout); - const parsed = parsedStream.resultJson ?? parseJson(proc.stdout); - if (!parsed) { return { exitCode: proc.exitCode, signal: proc.signal, timedOut: false, errorMessage: (proc.exitCode ?? 0) === 0 - ? "Failed to parse claude JSON output" - : `Claude exited with code ${proc.exitCode ?? -1}`, - resultJson: { - stdout: proc.stdout, - stderr: proc.stderr, - }, + ? null + : describeClaudeFailure(parsed) ?? `Claude exited with code ${proc.exitCode ?? -1}`, + usage, + sessionId: resolvedSessionId, + provider: "anthropic", + model: parsedStream.model || asString(parsed.model, model), + costUsd: parsedStream.costUsd ?? asNumber(parsed.total_cost_usd, 0), + resultJson: parsed, + summary: parsedStream.summary || asString(parsed.result, ""), + clearSession: Boolean(opts.clearSessionOnMissingSession && !resolvedSessionId), }; + }; + + const initial = await runAttempt(sessionId ?? null); + if ( + sessionId && + !initial.proc.timedOut && + (initial.proc.exitCode ?? 0) !== 0 && + initial.parsed && + isClaudeUnknownSessionError(initial.parsed) + ) { + await onLog( + "stderr", + `[paperclip] Claude resume session "${sessionId}" is unavailable; retrying with a fresh session.\n`, + ); + const retry = await runAttempt(null); + return toAdapterResult(retry, { fallbackSessionId: null, clearSessionOnMissingSession: true }); } - const usage = - parsedStream.usage ?? - (() => { - const usageObj = parseObject(parsed.usage); - return { - inputTokens: asNumber(usageObj.input_tokens, 0), - cachedInputTokens: asNumber(usageObj.cache_read_input_tokens, 0), - outputTokens: asNumber(usageObj.output_tokens, 0), - }; - })(); - - return { - exitCode: proc.exitCode, - signal: proc.signal, - timedOut: false, - errorMessage: (proc.exitCode ?? 0) === 0 ? null : `Claude exited with code ${proc.exitCode ?? -1}`, - usage, - sessionId: - parsedStream.sessionId ?? - (asString(parsed.session_id, runtime.sessionId ?? "") || runtime.sessionId), - provider: "anthropic", - model: parsedStream.model || asString(parsed.model, model), - costUsd: parsedStream.costUsd ?? asNumber(parsed.total_cost_usd, 0), - resultJson: parsed, - summary: parsedStream.summary || asString(parsed.result, ""), - }; + return toAdapterResult(initial, { fallbackSessionId: runtime.sessionId }); } async function executeCodexLocalRun( diff --git a/ui/src/pages/AgentDetail.tsx b/ui/src/pages/AgentDetail.tsx index c0cf7314..86b9b573 100644 --- a/ui/src/pages/AgentDetail.tsx +++ b/ui/src/pages/AgentDetail.tsx @@ -110,7 +110,7 @@ type TranscriptEntry = | { kind: "assistant"; ts: string; text: string } | { kind: "tool_call"; ts: string; name: string; input: unknown } | { kind: "init"; ts: string; model: string; sessionId: string } - | { kind: "result"; ts: string; text: string; inputTokens: number; outputTokens: number; cachedTokens: number; costUsd: number } + | { kind: "result"; ts: string; text: string; inputTokens: number; outputTokens: number; cachedTokens: number; costUsd: number; subtype: string; isError: boolean; errors: string[] } | { kind: "stderr"; ts: string; text: string } | { kind: "system"; ts: string; text: string } | { kind: "stdout"; ts: string; text: string }; @@ -124,6 +124,23 @@ function asNumber(value: unknown): number { return typeof value === "number" && Number.isFinite(value) ? value : 0; } +function errorText(value: unknown): string { + if (typeof value === "string") return value; + const rec = asRecord(value); + if (!rec) return ""; + const msg = + (typeof rec.message === "string" && rec.message) || + (typeof rec.error === "string" && rec.error) || + (typeof rec.code === "string" && rec.code) || + ""; + if (msg) return msg; + try { + return JSON.stringify(rec); + } catch { + return ""; + } +} + function parseClaudeStdoutLine(line: string, ts: string): TranscriptEntry[] { const parsed = asRecord(safeJsonParse(line)); if (!parsed) { @@ -171,6 +188,9 @@ function parseClaudeStdoutLine(line: string, ts: string): TranscriptEntry[] { const outputTokens = asNumber(usage.output_tokens); const cachedTokens = asNumber(usage.cache_read_input_tokens); const costUsd = asNumber(parsed.total_cost_usd); + const subtype = typeof parsed.subtype === "string" ? parsed.subtype : ""; + const isError = parsed.is_error === true; + const errors = Array.isArray(parsed.errors) ? parsed.errors.map(errorText).filter(Boolean) : []; const text = typeof parsed.result === "string" ? parsed.result : ""; return [{ kind: "result", @@ -180,6 +200,9 @@ function parseClaudeStdoutLine(line: string, ts: string): TranscriptEntry[] { outputTokens, cachedTokens, costUsd, + subtype, + isError, + errors, }]; } @@ -701,12 +724,12 @@ function RunsTab({ runs, companyId, agentId, selectedRunId }: { runs: HeartbeatR const selectedRun = sorted.find((r) => r.id === effectiveRunId) ?? null; return ( -
- {/* Left: run list */} +
+ {/* Left: run list — sticky, scrolls independently */}
+ )} style={{ maxHeight: "calc(100vh - 2rem)" }}> {sorted.map((run) => { const statusInfo = runStatusIcons[run.status] ?? { icon: Clock, color: "text-neutral-400" }; const StatusIcon = statusInfo.icon; @@ -759,9 +782,9 @@ function RunsTab({ runs, companyId, agentId, selectedRunId }: { runs: HeartbeatR })}
- {/* Right: run detail */} + {/* Right: run detail — natural height, page scrolls */} {selectedRun && ( -
+
)} @@ -1165,7 +1188,7 @@ function LogViewer({ run }: { run: HeartbeatRun }) { )}
-
+
{transcript.length === 0 && !run.logRef && (
No persisted transcript for this run.
)} @@ -1218,6 +1241,12 @@ function LogViewer({ run }: { run: HeartbeatRun }) { tokens in={formatTokens(entry.inputTokens)} out={formatTokens(entry.outputTokens)} cached={formatTokens(entry.cachedTokens)} cost=${entry.costUsd.toFixed(6)}
+ {(entry.subtype || entry.isError || entry.errors.length > 0) && ( +
+ subtype={entry.subtype || "unknown"} is_error={entry.isError ? "true" : "false"} + {entry.errors.length > 0 ? ` errors=${entry.errors.join(" | ")}` : ""} +
+ )} {entry.text && (
{entry.text}
)} @@ -1252,10 +1281,46 @@ function LogViewer({ run }: { run: HeartbeatRun }) {
+ {(run.status === "failed" || run.status === "timed_out") && ( +
+
Failure details
+ {run.error && ( +
+ Error: + {run.error} +
+ )} + {run.stderrExcerpt && run.stderrExcerpt.trim() && ( +
+
stderr excerpt
+ 
+                {run.stderrExcerpt}
+
+
+ )} + {run.resultJson && ( +
+
adapter result JSON
+ 
+                {JSON.stringify(run.resultJson, null, 2)}
+
+
+ )} + {run.stdoutExcerpt && run.stdoutExcerpt.trim() && !run.resultJson && ( +
+
stdout excerpt
+ 
+                {run.stdoutExcerpt}
+
+
+ )} +
+ )} + {events.length > 0 && (
Events ({events.length})
-
+
{events.map((evt) => { const color = evt.color ?? (evt.level ? levelColors[evt.level] : null)