aevgarik/paperclip
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve heartbeat execution, run tracking, and agent detail display

Browse Source 
Enhance heartbeat service with better process adapter error recovery
and run state management. Expand heartbeat-run CLI with additional
output and diagnostics. Improve AgentDetail page run history and
status display.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Forgotten
2026-02-18 13:17:03 -06:00
parent d6024b3ca5
commit 3a91ecbae3
3 changed files with 318 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

249
server/src/services/heartbeat.ts
View File

@@ -47,6 +47,7 @@ interface AdapterExecutionResult {
costUsd?: number | null;
resultJson?: Record<string, unknown> | null;
summary?: string | null;
clearSession?: boolean;
}
interface AdapterInvocationMeta {
@@ -188,6 +189,65 @@ function parseCodexJsonl(stdout: string) {
};
}
function describeClaudeFailure(parsed: Record<string, unknown>): string | null {
const subtype = asString(parsed.subtype, "");
const resultText = asString(parsed.result, "").trim();
const errors = extractClaudeErrorMessages(parsed);
let detail = resultText;
if (!detail && errors.length > 0) {
detail = errors[0] ?? "";
}
const parts = ["Claude run failed"];
if (subtype) parts.push(`subtype=${subtype}`);
if (detail) parts.push(detail);
return parts.length > 1 ? parts.join(": ") : null;
}
function extractClaudeErrorMessages(parsed: Record<string, unknown>): string[] {
const raw = Array.isArray(parsed.errors) ? parsed.errors : [];
const messages: string[] = [];
for (const entry of raw) {
if (typeof entry === "string") {
const msg = entry.trim();
if (msg) messages.push(msg);
continue;
}
if (typeof entry !== "object" || entry === null || Array.isArray(entry)) {
continue;
}
const obj = entry as Record<string, unknown>;
const msg = asString(obj.message, "") || asString(obj.error, "") || asString(obj.code, "");
if (msg) {
messages.push(msg);
continue;
}
try {
messages.push(JSON.stringify(obj));
} catch {
// skip non-serializable entry
}
}
return messages;
}
function isClaudeUnknownSessionError(parsed: Record<string, unknown>): boolean {
const resultText = asString(parsed.result, "").trim();
const allMessages = [resultText, ...extractClaudeErrorMessages(parsed)]
.map((msg) => msg.trim())
.filter(Boolean);
return allMessages.some((msg) =>
/no conversation found with session id|unknown session|session .* not found/i.test(msg),
);
}
function parseClaudeStreamJson(stdout: string) {
let sessionId: string | null = null;
let model = "";
@@ -623,7 +683,7 @@ export function heartbeatService(db: Db) {
.update(agentRuntimeState)
.set({
adapterType: agent.adapterType,
sessionId: result.sessionId ?? existing.sessionId,
sessionId: result.clearSession ? null : (result.sessionId ?? existing.sessionId),
lastRunId: run.id,
lastRunStatus: run.status,
lastError: result.errorMessage ?? null,
@@ -819,86 +879,143 @@ export function heartbeatService(db: Db) {
context,
});
const args = ["--print", prompt, "--output-format", "stream-json", "--verbose"];
if (sessionId) args.push("--resume", sessionId);
if (dangerouslySkipPermissions) args.push("--dangerously-skip-permissions");
if (model) args.push("--model", model);
if (maxTurns > 0) args.push("--max-turns", String(maxTurns));
if (extraArgs.length > 0) args.push(...extraArgs);
const buildClaudeArgs = (resumeSessionId: string | null) => {
const args = ["--print", prompt, "--output-format", "stream-json", "--verbose"];
if (resumeSessionId) args.push("--resume", resumeSessionId);
if (dangerouslySkipPermissions) args.push("--dangerously-skip-permissions");
if (model) args.push("--model", model);
if (maxTurns > 0) args.push("--max-turns", String(maxTurns));
if (extraArgs.length > 0) args.push(...extraArgs);
return args;
};
if (onMeta) {
await onMeta({
adapterType: "claude_local",
command,
const parseFallbackErrorMessage = (proc: RunProcessResult) => {
const stderrLine =
proc.stderr
.split(/\r?\n/)
.map((line) => line.trim())
.find(Boolean) ?? "";
if ((proc.exitCode ?? 0) === 0) {
return "Failed to parse claude JSON output";
}
return stderrLine
? `Claude exited with code ${proc.exitCode ?? -1}: ${stderrLine}`
: `Claude exited with code ${proc.exitCode ?? -1}`;
};
const runAttempt = async (resumeSessionId: string | null) => {
const args = buildClaudeArgs(resumeSessionId);
if (onMeta) {
await onMeta({
adapterType: "claude_local",
command,
cwd,
commandArgs: args.map((value, idx) => (idx === 1 ? `<prompt ${prompt.length} chars>` : value)),
env: redactEnvForLogs(env),
prompt,
context,
});
}
const proc = await runChildProcess(runId, command, args, {
cwd,
commandArgs: args.map((value, idx) => (idx === 1 ? `<prompt ${prompt.length} chars>` : value)),
env: redactEnvForLogs(env),
prompt,
context,
env,
timeoutSec,
graceSec,
onLog,
});
}
const proc = await runChildProcess(runId, command, args, {
cwd,
env,
timeoutSec,
graceSec,
onLog,
});
const parsedStream = parseClaudeStreamJson(proc.stdout);
const parsed = parsedStream.resultJson ?? parseJson(proc.stdout);
return { proc, parsedStream, parsed };
};
if (proc.timedOut) {
return {
exitCode: proc.exitCode,
signal: proc.signal,
timedOut: true,
errorMessage: `Timed out after ${timeoutSec}s`,
};
}
const toAdapterResult = (
attempt: {
proc: RunProcessResult;
parsedStream: ReturnType<typeof parseClaudeStreamJson>;
parsed: Record<string, unknown> | null;
},
opts: { fallbackSessionId: string | null; clearSessionOnMissingSession?: boolean },
): AdapterExecutionResult => {
const { proc, parsedStream, parsed } = attempt;
if (proc.timedOut) {
return {
exitCode: proc.exitCode,
signal: proc.signal,
timedOut: true,
errorMessage: `Timed out after ${timeoutSec}s`,
clearSession: Boolean(opts.clearSessionOnMissingSession),
};
}
if (!parsed) {
return {
exitCode: proc.exitCode,
signal: proc.signal,
timedOut: false,
errorMessage: parseFallbackErrorMessage(proc),
resultJson: {
stdout: proc.stdout,
stderr: proc.stderr,
},
clearSession: Boolean(opts.clearSessionOnMissingSession),
};
}
const usage =
parsedStream.usage ??
(() => {
const usageObj = parseObject(parsed.usage);
return {
inputTokens: asNumber(usageObj.input_tokens, 0),
cachedInputTokens: asNumber(usageObj.cache_read_input_tokens, 0),
outputTokens: asNumber(usageObj.output_tokens, 0),
};
})();
const resolvedSessionId =
parsedStream.sessionId ??
(asString(parsed.session_id, opts.fallbackSessionId ?? "") || opts.fallbackSessionId);
const parsedStream = parseClaudeStreamJson(proc.stdout);
const parsed = parsedStream.resultJson ?? parseJson(proc.stdout);
if (!parsed) {
return {
exitCode: proc.exitCode,
signal: proc.signal,
timedOut: false,
errorMessage:
(proc.exitCode ?? 0) === 0
? "Failed to parse claude JSON output"
: `Claude exited with code ${proc.exitCode ?? -1}`,
resultJson: {
stdout: proc.stdout,
stderr: proc.stderr,
},
? null
: describeClaudeFailure(parsed) ?? `Claude exited with code ${proc.exitCode ?? -1}`,
usage,
sessionId: resolvedSessionId,
provider: "anthropic",
model: parsedStream.model || asString(parsed.model, model),
costUsd: parsedStream.costUsd ?? asNumber(parsed.total_cost_usd, 0),
resultJson: parsed,
summary: parsedStream.summary || asString(parsed.result, ""),
clearSession: Boolean(opts.clearSessionOnMissingSession && !resolvedSessionId),
};
};
const initial = await runAttempt(sessionId ?? null);
if (
sessionId &&
!initial.proc.timedOut &&
(initial.proc.exitCode ?? 0) !== 0 &&
initial.parsed &&
isClaudeUnknownSessionError(initial.parsed)
) {
await onLog(
"stderr",
`[paperclip] Claude resume session "${sessionId}" is unavailable; retrying with a fresh session.\n`,
);
const retry = await runAttempt(null);
return toAdapterResult(retry, { fallbackSessionId: null, clearSessionOnMissingSession: true });
}
const usage =
parsedStream.usage ??
(() => {
const usageObj = parseObject(parsed.usage);
return {
inputTokens: asNumber(usageObj.input_tokens, 0),
cachedInputTokens: asNumber(usageObj.cache_read_input_tokens, 0),
outputTokens: asNumber(usageObj.output_tokens, 0),
};
})();
return {
exitCode: proc.exitCode,
signal: proc.signal,
timedOut: false,
errorMessage: (proc.exitCode ?? 0) === 0 ? null : `Claude exited with code ${proc.exitCode ?? -1}`,
usage,
sessionId:
parsedStream.sessionId ??
(asString(parsed.session_id, runtime.sessionId ?? "") || runtime.sessionId),
provider: "anthropic",
model: parsedStream.model || asString(parsed.model, model),
costUsd: parsedStream.costUsd ?? asNumber(parsed.total_cost_usd, 0),
resultJson: parsed,
summary: parsedStream.summary || asString(parsed.result, ""),
};
return toAdapterResult(initial, { fallbackSessionId: runtime.sessionId });
}
async function executeCodexLocalRun(