From 9248881d42d02764751c7e9ca14afd26511a187e Mon Sep 17 00:00:00 2001 From: Jayakrishnan Date: Tue, 10 Mar 2026 12:01:46 +0000 Subject: [PATCH 01/12] fix(adapter-utils): strip Claude Code env vars from child processes When the Paperclip server is started from within a Claude Code session (e.g. `npx paperclipai run` in a Claude Code terminal), the `CLAUDECODE` and related env vars (`CLAUDE_CODE_ENTRYPOINT`, `CLAUDE_CODE_SESSION`, `CLAUDE_CODE_PARENT_SESSION`) leak into `process.env`. Since `runChildProcess()` spreads `process.env` into the child environment, every spawned `claude` CLI process inherits these vars and immediately exits with: "Claude Code cannot be launched inside another Claude Code session." This is particularly disruptive for the `claude-local` adapter, where every agent run spawns a `claude` child process. A single contaminated server start (or cron job that inherits the env) silently breaks all agent executions until the server is restarted in a clean environment. The fix deletes the four known Claude Code nesting-guard env vars from the merged environment before passing it to `spawn()`. Co-Authored-By: Claude Opus 4.6 --- packages/adapter-utils/src/server-utils.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/adapter-utils/src/server-utils.ts b/packages/adapter-utils/src/server-utils.ts index 3d273cd9..8e28fbff 100644 --- a/packages/adapter-utils/src/server-utils.ts +++ b/packages/adapter-utils/src/server-utils.ts @@ -272,7 +272,19 @@ export async function runChildProcess( const onLogError = opts.onLogError ?? ((err, id, msg) => console.warn({ err, runId: id }, msg)); return new Promise((resolve, reject) => { - const mergedEnv = ensurePathInEnv({ ...process.env, ...opts.env }); + const rawMerged: NodeJS.ProcessEnv = { ...process.env, ...opts.env }; + + // Strip Claude Code nesting-guard env vars so spawned `claude` processes + // don't refuse to start with "cannot be launched inside another session". + // These vars leak in when the Paperclip server itself is started from + // within a Claude Code session (e.g. `npx paperclipai run` in a terminal + // owned by Claude Code) or when cron inherits a contaminated shell env. + delete rawMerged.CLAUDECODE; + delete rawMerged.CLAUDE_CODE_ENTRYPOINT; + delete rawMerged.CLAUDE_CODE_SESSION; + delete rawMerged.CLAUDE_CODE_PARENT_SESSION; + + const mergedEnv = ensurePathInEnv(rawMerged); void resolveSpawnTarget(command, args, opts.cwd, mergedEnv) .then((target) => { const child = spawn(target.command, target.args, { From 1a53567cb6195a34f51aeb44979fe2f9aa7e5b9e Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 10 Mar 2026 07:24:48 -0500 Subject: [PATCH 02/12] Apply suggestions from code review Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- packages/adapter-utils/src/server-utils.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/adapter-utils/src/server-utils.ts b/packages/adapter-utils/src/server-utils.ts index 8e28fbff..2b9de31f 100644 --- a/packages/adapter-utils/src/server-utils.ts +++ b/packages/adapter-utils/src/server-utils.ts @@ -279,10 +279,15 @@ export async function runChildProcess( // These vars leak in when the Paperclip server itself is started from // within a Claude Code session (e.g. `npx paperclipai run` in a terminal // owned by Claude Code) or when cron inherits a contaminated shell env. - delete rawMerged.CLAUDECODE; - delete rawMerged.CLAUDE_CODE_ENTRYPOINT; - delete rawMerged.CLAUDE_CODE_SESSION; - delete rawMerged.CLAUDE_CODE_PARENT_SESSION; + const CLAUDE_CODE_NESTING_VARS = [ + "CLAUDECODE", + "CLAUDE_CODE_ENTRYPOINT", + "CLAUDE_CODE_SESSION", + "CLAUDE_CODE_PARENT_SESSION", + ] as const; + for (const key of CLAUDE_CODE_NESTING_VARS) { + delete rawMerged[key]; + } const mergedEnv = ensurePathInEnv(rawMerged); void resolveSpawnTarget(command, args, opts.cwd, mergedEnv) From f6f5fee200ae79fdc62683664eb28a23dd5bcede Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Mar 2026 15:54:31 +0200 Subject: [PATCH 03/12] fix: wire parentId query filter into issues list endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The parentId parameter on GET /api/companies/:companyId/issues was silently ignored — the filter was never extracted from the query string, never passed to the service layer, and the IssueFilters type did not include it. All other filters (status, assigneeAgentId, projectId, etc.) worked correctly. This caused subtask lookups to return every issue in the company instead of only children of the specified parent. Changes: - Add parentId to IssueFilters interface - Add eq(issues.parentId, filters.parentId) condition in list() - Extract parentId from req.query in the route handler Fixes: LAS-101 --- server/src/routes/issues.ts | 1 + server/src/services/issues.ts | 2 ++ 2 files changed, 3 insertions(+) diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index e4035dfc..c5c4e29d 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -230,6 +230,7 @@ export function issueRoutes(db: Db, storage: StorageService) { touchedByUserId, unreadForUserId, projectId: req.query.projectId as string | undefined, + parentId: req.query.parentId as string | undefined, labelId: req.query.labelId as string | undefined, q: req.query.q as string | undefined, }); diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index cb258e23..8f34be18 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -53,6 +53,7 @@ export interface IssueFilters { touchedByUserId?: string; unreadForUserId?: string; projectId?: string; + parentId?: string; labelId?: string; q?: string; } @@ -458,6 +459,7 @@ export function issueService(db: Db) { conditions.push(unreadForUserCondition(companyId, unreadForUserId)); } if (filters?.projectId) conditions.push(eq(issues.projectId, filters.projectId)); + if (filters?.parentId) conditions.push(eq(issues.parentId, filters.parentId)); if (filters?.labelId) { const labeledIssueIds = await db .select({ issueId: issueLabels.issueId }) From dec02225f19d9114246629793e1880575a771dc2 Mon Sep 17 00:00:00 2001 From: Subhendu Kundu Date: Tue, 10 Mar 2026 19:40:22 +0530 Subject: [PATCH 04/12] feat: make attachment content types configurable via env var Add PAPERCLIP_ALLOWED_ATTACHMENT_TYPES env var to configure allowed MIME types for issue attachments and asset uploads. Supports exact types (application/pdf) and wildcard patterns (image/*, text/*). Falls back to the existing image-only defaults when the env var is unset, preserving backward compatibility. - Extract shared module `attachment-types.ts` with `isAllowedContentType()` and `matchesContentType()` (pure, testable) - Update `routes/issues.ts` and `routes/assets.ts` to use shared module - Add unit tests for parsing and wildcard matching Closes #487 --- server/src/__tests__/attachment-types.test.ts | 89 +++++++++++++++++++ server/src/attachment-types.ts | 67 ++++++++++++++ server/src/routes/assets.ts | 16 +--- server/src/routes/issues.ts | 12 +-- 4 files changed, 162 insertions(+), 22 deletions(-) create mode 100644 server/src/__tests__/attachment-types.test.ts create mode 100644 server/src/attachment-types.ts diff --git a/server/src/__tests__/attachment-types.test.ts b/server/src/__tests__/attachment-types.test.ts new file mode 100644 index 00000000..af0a58b3 --- /dev/null +++ b/server/src/__tests__/attachment-types.test.ts @@ -0,0 +1,89 @@ +import { describe, it, expect } from "vitest"; +import { + parseAllowedTypes, + matchesContentType, + DEFAULT_ALLOWED_TYPES, +} from "../attachment-types.js"; + +describe("parseAllowedTypes", () => { + it("returns default image types when input is undefined", () => { + expect(parseAllowedTypes(undefined)).toEqual([...DEFAULT_ALLOWED_TYPES]); + }); + + it("returns default image types when input is empty string", () => { + expect(parseAllowedTypes("")).toEqual([...DEFAULT_ALLOWED_TYPES]); + }); + + it("parses comma-separated types", () => { + expect(parseAllowedTypes("image/*,application/pdf")).toEqual([ + "image/*", + "application/pdf", + ]); + }); + + it("trims whitespace", () => { + expect(parseAllowedTypes(" image/png , application/pdf ")).toEqual([ + "image/png", + "application/pdf", + ]); + }); + + it("lowercases entries", () => { + expect(parseAllowedTypes("Application/PDF")).toEqual(["application/pdf"]); + }); + + it("filters empty segments", () => { + expect(parseAllowedTypes("image/png,,application/pdf,")).toEqual([ + "image/png", + "application/pdf", + ]); + }); +}); + +describe("matchesContentType", () => { + it("matches exact types", () => { + const patterns = ["application/pdf", "image/png"]; + expect(matchesContentType("application/pdf", patterns)).toBe(true); + expect(matchesContentType("image/png", patterns)).toBe(true); + expect(matchesContentType("text/plain", patterns)).toBe(false); + }); + + it("matches /* wildcard patterns", () => { + const patterns = ["image/*"]; + expect(matchesContentType("image/png", patterns)).toBe(true); + expect(matchesContentType("image/jpeg", patterns)).toBe(true); + expect(matchesContentType("image/svg+xml", patterns)).toBe(true); + expect(matchesContentType("application/pdf", patterns)).toBe(false); + }); + + it("matches .* wildcard patterns", () => { + const patterns = ["application/vnd.openxmlformats-officedocument.*"]; + expect( + matchesContentType( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + patterns, + ), + ).toBe(true); + expect( + matchesContentType( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + patterns, + ), + ).toBe(true); + expect(matchesContentType("application/pdf", patterns)).toBe(false); + }); + + it("is case-insensitive", () => { + const patterns = ["application/pdf"]; + expect(matchesContentType("APPLICATION/PDF", patterns)).toBe(true); + expect(matchesContentType("Application/Pdf", patterns)).toBe(true); + }); + + it("combines exact and wildcard patterns", () => { + const patterns = ["image/*", "application/pdf", "text/*"]; + expect(matchesContentType("image/webp", patterns)).toBe(true); + expect(matchesContentType("application/pdf", patterns)).toBe(true); + expect(matchesContentType("text/csv", patterns)).toBe(true); + expect(matchesContentType("application/zip", patterns)).toBe(false); + }); +}); diff --git a/server/src/attachment-types.ts b/server/src/attachment-types.ts new file mode 100644 index 00000000..3f95156b --- /dev/null +++ b/server/src/attachment-types.ts @@ -0,0 +1,67 @@ +/** + * Shared attachment content-type configuration. + * + * By default only image types are allowed. Set the + * `PAPERCLIP_ALLOWED_ATTACHMENT_TYPES` environment variable to a + * comma-separated list of MIME types or wildcard patterns to expand the + * allowed set. + * + * Examples: + * PAPERCLIP_ALLOWED_ATTACHMENT_TYPES=image/*,application/pdf + * PAPERCLIP_ALLOWED_ATTACHMENT_TYPES=image/*,application/pdf,text/* + * + * Supported pattern syntax: + * - Exact types: "application/pdf" + * - Wildcards: "image/*" or "application/vnd.openxmlformats-officedocument.*" + */ + +export const DEFAULT_ALLOWED_TYPES: readonly string[] = [ + "image/png", + "image/jpeg", + "image/jpg", + "image/webp", + "image/gif", +]; + +/** + * Parse a comma-separated list of MIME type patterns into a normalised array. + * Returns the default image-only list when the input is empty or undefined. + */ +export function parseAllowedTypes(raw: string | undefined): string[] { + if (!raw) return [...DEFAULT_ALLOWED_TYPES]; + const parsed = raw + .split(",") + .map((s) => s.trim().toLowerCase()) + .filter((s) => s.length > 0); + return parsed.length > 0 ? parsed : [...DEFAULT_ALLOWED_TYPES]; +} + +/** + * Check whether `contentType` matches any entry in `allowedPatterns`. + * + * Supports exact matches ("application/pdf") and wildcard / prefix + * patterns ("image/*", "application/vnd.openxmlformats-officedocument.*"). + */ +export function matchesContentType(contentType: string, allowedPatterns: string[]): boolean { + const ct = contentType.toLowerCase(); + return allowedPatterns.some((pattern) => { + if (pattern.endsWith("/*") || pattern.endsWith(".*")) { + return ct.startsWith(pattern.slice(0, -1)); + } + return ct === pattern; + }); +} + +// ---------- Module-level singletons read once at startup ---------- + +const allowedPatterns: string[] = parseAllowedTypes( + process.env.PAPERCLIP_ALLOWED_ATTACHMENT_TYPES, +); + +/** Convenience wrapper using the process-level allowed list. */ +export function isAllowedContentType(contentType: string): boolean { + return matchesContentType(contentType, allowedPatterns); +} + +export const MAX_ATTACHMENT_BYTES = + Number(process.env.PAPERCLIP_ATTACHMENT_MAX_BYTES) || 10 * 1024 * 1024; diff --git a/server/src/routes/assets.ts b/server/src/routes/assets.ts index cde29ada..4c9847fe 100644 --- a/server/src/routes/assets.ts +++ b/server/src/routes/assets.ts @@ -5,22 +5,14 @@ import { createAssetImageMetadataSchema } from "@paperclipai/shared"; import type { StorageService } from "../storage/types.js"; import { assetService, logActivity } from "../services/index.js"; import { assertCompanyAccess, getActorInfo } from "./authz.js"; - -const MAX_ASSET_IMAGE_BYTES = Number(process.env.PAPERCLIP_ATTACHMENT_MAX_BYTES) || 10 * 1024 * 1024; -const ALLOWED_IMAGE_CONTENT_TYPES = new Set([ - "image/png", - "image/jpeg", - "image/jpg", - "image/webp", - "image/gif", -]); +import { isAllowedContentType, MAX_ATTACHMENT_BYTES } from "../attachment-types.js"; export function assetRoutes(db: Db, storage: StorageService) { const router = Router(); const svc = assetService(db); const upload = multer({ storage: multer.memoryStorage(), - limits: { fileSize: MAX_ASSET_IMAGE_BYTES, files: 1 }, + limits: { fileSize: MAX_ATTACHMENT_BYTES, files: 1 }, }); async function runSingleFileUpload(req: Request, res: Response) { @@ -41,7 +33,7 @@ export function assetRoutes(db: Db, storage: StorageService) { } catch (err) { if (err instanceof multer.MulterError) { if (err.code === "LIMIT_FILE_SIZE") { - res.status(422).json({ error: `Image exceeds ${MAX_ASSET_IMAGE_BYTES} bytes` }); + res.status(422).json({ error: `Image exceeds ${MAX_ATTACHMENT_BYTES} bytes` }); return; } res.status(400).json({ error: err.message }); @@ -57,7 +49,7 @@ export function assetRoutes(db: Db, storage: StorageService) { } const contentType = (file.mimetype || "").toLowerCase(); - if (!ALLOWED_IMAGE_CONTENT_TYPES.has(contentType)) { + if (!isAllowedContentType(contentType)) { res.status(422).json({ error: `Unsupported image type: ${contentType || "unknown"}` }); return; } diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index e4035dfc..8e398afc 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -26,15 +26,7 @@ import { logger } from "../middleware/logger.js"; import { forbidden, HttpError, unauthorized } from "../errors.js"; import { assertCompanyAccess, getActorInfo } from "./authz.js"; import { shouldWakeAssigneeOnCheckout } from "./issues-checkout-wakeup.js"; - -const MAX_ATTACHMENT_BYTES = Number(process.env.PAPERCLIP_ATTACHMENT_MAX_BYTES) || 10 * 1024 * 1024; -const ALLOWED_ATTACHMENT_CONTENT_TYPES = new Set([ - "image/png", - "image/jpeg", - "image/jpg", - "image/webp", - "image/gif", -]); +import { isAllowedContentType, MAX_ATTACHMENT_BYTES } from "../attachment-types.js"; export function issueRoutes(db: Db, storage: StorageService) { const router = Router(); @@ -1067,7 +1059,7 @@ export function issueRoutes(db: Db, storage: StorageService) { return; } const contentType = (file.mimetype || "").toLowerCase(); - if (!ALLOWED_ATTACHMENT_CONTENT_TYPES.has(contentType)) { + if (!isAllowedContentType(contentType)) { res.status(422).json({ error: `Unsupported attachment type: ${contentType || "unknown"}` }); return; } From 3ff07c23d213eb5b1c8a5841b55e34dd387c4981 Mon Sep 17 00:00:00 2001 From: Subhendu Kundu Date: Tue, 10 Mar 2026 19:54:42 +0530 Subject: [PATCH 05/12] Update server/src/routes/assets.ts Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- server/src/routes/assets.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/routes/assets.ts b/server/src/routes/assets.ts index 4c9847fe..ed8d5944 100644 --- a/server/src/routes/assets.ts +++ b/server/src/routes/assets.ts @@ -50,7 +50,7 @@ export function assetRoutes(db: Db, storage: StorageService) { const contentType = (file.mimetype || "").toLowerCase(); if (!isAllowedContentType(contentType)) { - res.status(422).json({ error: `Unsupported image type: ${contentType || "unknown"}` }); + res.status(422).json({ error: `Unsupported file type: ${contentType || "unknown"}` }); return; } if (file.buffer.length <= 0) { From 1959badde721a07c4e51a8fa05d84602959780d9 Mon Sep 17 00:00:00 2001 From: Subhendu Kundu Date: Tue, 10 Mar 2026 20:01:08 +0530 Subject: [PATCH 06/12] =?UTF-8?q?fix:=20address=20review=20feedback=20?= =?UTF-8?q?=E2=80=94=20stale=20error=20message=20and=20*=20wildcard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - assets.ts: change "Image exceeds" to "File exceeds" in size-limit error - attachment-types.ts: handle plain "*" as allow-all wildcard pattern - Add test for "*" wildcard (12 tests total) --- server/src/__tests__/attachment-types.test.ts | 8 ++++++++ server/src/attachment-types.ts | 1 + server/src/routes/assets.ts | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/server/src/__tests__/attachment-types.test.ts b/server/src/__tests__/attachment-types.test.ts index af0a58b3..5a430102 100644 --- a/server/src/__tests__/attachment-types.test.ts +++ b/server/src/__tests__/attachment-types.test.ts @@ -86,4 +86,12 @@ describe("matchesContentType", () => { expect(matchesContentType("text/csv", patterns)).toBe(true); expect(matchesContentType("application/zip", patterns)).toBe(false); }); + + it("handles plain * as allow-all wildcard", () => { + const patterns = ["*"]; + expect(matchesContentType("image/png", patterns)).toBe(true); + expect(matchesContentType("application/pdf", patterns)).toBe(true); + expect(matchesContentType("text/plain", patterns)).toBe(true); + expect(matchesContentType("application/zip", patterns)).toBe(true); + }); }); diff --git a/server/src/attachment-types.ts b/server/src/attachment-types.ts index 3f95156b..f9625de1 100644 --- a/server/src/attachment-types.ts +++ b/server/src/attachment-types.ts @@ -45,6 +45,7 @@ export function parseAllowedTypes(raw: string | undefined): string[] { export function matchesContentType(contentType: string, allowedPatterns: string[]): boolean { const ct = contentType.toLowerCase(); return allowedPatterns.some((pattern) => { + if (pattern === "*") return true; if (pattern.endsWith("/*") || pattern.endsWith(".*")) { return ct.startsWith(pattern.slice(0, -1)); } diff --git a/server/src/routes/assets.ts b/server/src/routes/assets.ts index ed8d5944..bd2f154d 100644 --- a/server/src/routes/assets.ts +++ b/server/src/routes/assets.ts @@ -33,7 +33,7 @@ export function assetRoutes(db: Db, storage: StorageService) { } catch (err) { if (err instanceof multer.MulterError) { if (err.code === "LIMIT_FILE_SIZE") { - res.status(422).json({ error: `Image exceeds ${MAX_ATTACHMENT_BYTES} bytes` }); + res.status(422).json({ error: `File exceeds ${MAX_ATTACHMENT_BYTES} bytes` }); return; } res.status(400).json({ error: err.message }); From 0704854926fcebf512933ff741fc0788660a4381 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 10 Mar 2026 10:08:13 -0500 Subject: [PATCH 07/12] Add worktree init CLI for isolated development instances --- cli/package.json | 1 + cli/src/__tests__/worktree.test.ts | 110 +++++++++ cli/src/commands/worktree-lib.ts | 172 +++++++++++++ cli/src/commands/worktree.ts | 384 +++++++++++++++++++++++++++++ cli/src/config/env.ts | 36 ++- cli/src/index.ts | 4 + doc/DEVELOPING.md | 36 +++ packages/db/src/backup-lib.ts | 22 +- packages/db/src/index.ts | 2 + 9 files changed, 760 insertions(+), 7 deletions(-) create mode 100644 cli/src/__tests__/worktree.test.ts create mode 100644 cli/src/commands/worktree-lib.ts create mode 100644 cli/src/commands/worktree.ts diff --git a/cli/package.json b/cli/package.json index 21de193a..24a8bf66 100644 --- a/cli/package.json +++ b/cli/package.json @@ -47,6 +47,7 @@ "drizzle-orm": "0.38.4", "dotenv": "^17.0.1", "commander": "^13.1.0", + "embedded-postgres": "^18.1.0-beta.16", "picocolors": "^1.1.1" }, "devDependencies": { diff --git a/cli/src/__tests__/worktree.test.ts b/cli/src/__tests__/worktree.test.ts new file mode 100644 index 00000000..f393c10c --- /dev/null +++ b/cli/src/__tests__/worktree.test.ts @@ -0,0 +1,110 @@ +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + buildWorktreeConfig, + buildWorktreeEnvEntries, + formatShellExports, + resolveWorktreeLocalPaths, + rewriteLocalUrlPort, + sanitizeWorktreeInstanceId, +} from "../commands/worktree-lib.js"; +import type { PaperclipConfig } from "../config/schema.js"; + +function buildSourceConfig(): PaperclipConfig { + return { + $meta: { + version: 1, + updatedAt: "2026-03-09T00:00:00.000Z", + source: "configure", + }, + database: { + mode: "embedded-postgres", + embeddedPostgresDataDir: "/tmp/main/db", + embeddedPostgresPort: 54329, + backup: { + enabled: true, + intervalMinutes: 60, + retentionDays: 30, + dir: "/tmp/main/backups", + }, + }, + logging: { + mode: "file", + logDir: "/tmp/main/logs", + }, + server: { + deploymentMode: "authenticated", + exposure: "private", + host: "127.0.0.1", + port: 3100, + allowedHostnames: ["localhost"], + serveUi: true, + }, + auth: { + baseUrlMode: "explicit", + publicBaseUrl: "http://127.0.0.1:3100", + disableSignUp: false, + }, + storage: { + provider: "local_disk", + localDisk: { + baseDir: "/tmp/main/storage", + }, + s3: { + bucket: "paperclip", + region: "us-east-1", + prefix: "", + forcePathStyle: false, + }, + }, + secrets: { + provider: "local_encrypted", + strictMode: false, + localEncrypted: { + keyFilePath: "/tmp/main/secrets/master.key", + }, + }, + }; +} + +describe("worktree helpers", () => { + it("sanitizes instance ids", () => { + expect(sanitizeWorktreeInstanceId("feature/worktree-support")).toBe("feature-worktree-support"); + expect(sanitizeWorktreeInstanceId(" ")).toBe("worktree"); + }); + + it("rewrites loopback auth URLs to the new port only", () => { + expect(rewriteLocalUrlPort("http://127.0.0.1:3100", 3110)).toBe("http://127.0.0.1:3110/"); + expect(rewriteLocalUrlPort("https://paperclip.example", 3110)).toBe("https://paperclip.example"); + }); + + it("builds isolated config and env paths for a worktree", () => { + const paths = resolveWorktreeLocalPaths({ + cwd: "/tmp/paperclip-feature", + homeDir: "/tmp/paperclip-worktrees", + instanceId: "feature-worktree-support", + }); + const config = buildWorktreeConfig({ + sourceConfig: buildSourceConfig(), + paths, + serverPort: 3110, + databasePort: 54339, + now: new Date("2026-03-09T12:00:00.000Z"), + }); + + expect(config.database.embeddedPostgresDataDir).toBe( + path.resolve("/tmp/paperclip-worktrees", "instances", "feature-worktree-support", "db"), + ); + expect(config.database.embeddedPostgresPort).toBe(54339); + expect(config.server.port).toBe(3110); + expect(config.auth.publicBaseUrl).toBe("http://127.0.0.1:3110/"); + expect(config.storage.localDisk.baseDir).toBe( + path.resolve("/tmp/paperclip-worktrees", "instances", "feature-worktree-support", "data", "storage"), + ); + + const env = buildWorktreeEnvEntries(paths); + expect(env.PAPERCLIP_HOME).toBe(path.resolve("/tmp/paperclip-worktrees")); + expect(env.PAPERCLIP_INSTANCE_ID).toBe("feature-worktree-support"); + expect(formatShellExports(env)).toContain("export PAPERCLIP_INSTANCE_ID='feature-worktree-support'"); + }); +}); diff --git a/cli/src/commands/worktree-lib.ts b/cli/src/commands/worktree-lib.ts new file mode 100644 index 00000000..bc96a700 --- /dev/null +++ b/cli/src/commands/worktree-lib.ts @@ -0,0 +1,172 @@ +import path from "node:path"; +import type { PaperclipConfig } from "../config/schema.js"; +import { expandHomePrefix } from "../config/home.js"; + +export const DEFAULT_WORKTREE_HOME = "~/.paperclip-worktrees"; + +export type WorktreeLocalPaths = { + cwd: string; + repoConfigDir: string; + configPath: string; + envPath: string; + homeDir: string; + instanceId: string; + instanceRoot: string; + contextPath: string; + embeddedPostgresDataDir: string; + backupDir: string; + logDir: string; + secretsKeyFilePath: string; + storageDir: string; +}; + +function nonEmpty(value: string | null | undefined): string | null { + return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; +} + +function isLoopbackHost(hostname: string): boolean { + const value = hostname.trim().toLowerCase(); + return value === "127.0.0.1" || value === "localhost" || value === "::1"; +} + +export function sanitizeWorktreeInstanceId(rawValue: string): string { + const trimmed = rawValue.trim().toLowerCase(); + const normalized = trimmed + .replace(/[^a-z0-9_-]+/g, "-") + .replace(/-+/g, "-") + .replace(/^[-_]+|[-_]+$/g, ""); + return normalized || "worktree"; +} + +export function resolveSuggestedWorktreeName(cwd: string, explicitName?: string): string { + return nonEmpty(explicitName) ?? path.basename(path.resolve(cwd)); +} + +export function resolveWorktreeLocalPaths(opts: { + cwd: string; + homeDir?: string; + instanceId: string; +}): WorktreeLocalPaths { + const cwd = path.resolve(opts.cwd); + const homeDir = path.resolve(expandHomePrefix(opts.homeDir ?? DEFAULT_WORKTREE_HOME)); + const instanceRoot = path.resolve(homeDir, "instances", opts.instanceId); + const repoConfigDir = path.resolve(cwd, ".paperclip"); + return { + cwd, + repoConfigDir, + configPath: path.resolve(repoConfigDir, "config.json"), + envPath: path.resolve(repoConfigDir, ".env"), + homeDir, + instanceId: opts.instanceId, + instanceRoot, + contextPath: path.resolve(homeDir, "context.json"), + embeddedPostgresDataDir: path.resolve(instanceRoot, "db"), + backupDir: path.resolve(instanceRoot, "data", "backups"), + logDir: path.resolve(instanceRoot, "logs"), + secretsKeyFilePath: path.resolve(instanceRoot, "secrets", "master.key"), + storageDir: path.resolve(instanceRoot, "data", "storage"), + }; +} + +export function rewriteLocalUrlPort(rawUrl: string | undefined, port: number): string | undefined { + if (!rawUrl) return undefined; + try { + const parsed = new URL(rawUrl); + if (!isLoopbackHost(parsed.hostname)) return rawUrl; + parsed.port = String(port); + return parsed.toString(); + } catch { + return rawUrl; + } +} + +export function buildWorktreeConfig(input: { + sourceConfig: PaperclipConfig | null; + paths: WorktreeLocalPaths; + serverPort: number; + databasePort: number; + now?: Date; +}): PaperclipConfig { + const { sourceConfig, paths, serverPort, databasePort } = input; + const nowIso = (input.now ?? new Date()).toISOString(); + + const source = sourceConfig; + const authPublicBaseUrl = rewriteLocalUrlPort(source?.auth.publicBaseUrl, serverPort); + + return { + $meta: { + version: 1, + updatedAt: nowIso, + source: "configure", + }, + ...(source?.llm ? { llm: source.llm } : {}), + database: { + mode: "embedded-postgres", + embeddedPostgresDataDir: paths.embeddedPostgresDataDir, + embeddedPostgresPort: databasePort, + backup: { + enabled: source?.database.backup.enabled ?? true, + intervalMinutes: source?.database.backup.intervalMinutes ?? 60, + retentionDays: source?.database.backup.retentionDays ?? 30, + dir: paths.backupDir, + }, + }, + logging: { + mode: source?.logging.mode ?? "file", + logDir: paths.logDir, + }, + server: { + deploymentMode: source?.server.deploymentMode ?? "local_trusted", + exposure: source?.server.exposure ?? "private", + host: source?.server.host ?? "127.0.0.1", + port: serverPort, + allowedHostnames: source?.server.allowedHostnames ?? [], + serveUi: source?.server.serveUi ?? true, + }, + auth: { + baseUrlMode: source?.auth.baseUrlMode ?? "auto", + ...(authPublicBaseUrl ? { publicBaseUrl: authPublicBaseUrl } : {}), + disableSignUp: source?.auth.disableSignUp ?? false, + }, + storage: { + provider: source?.storage.provider ?? "local_disk", + localDisk: { + baseDir: paths.storageDir, + }, + s3: { + bucket: source?.storage.s3.bucket ?? "paperclip", + region: source?.storage.s3.region ?? "us-east-1", + endpoint: source?.storage.s3.endpoint, + prefix: source?.storage.s3.prefix ?? "", + forcePathStyle: source?.storage.s3.forcePathStyle ?? false, + }, + }, + secrets: { + provider: source?.secrets.provider ?? "local_encrypted", + strictMode: source?.secrets.strictMode ?? false, + localEncrypted: { + keyFilePath: paths.secretsKeyFilePath, + }, + }, + }; +} + +export function buildWorktreeEnvEntries(paths: WorktreeLocalPaths): Record { + return { + PAPERCLIP_HOME: paths.homeDir, + PAPERCLIP_INSTANCE_ID: paths.instanceId, + PAPERCLIP_CONFIG: paths.configPath, + PAPERCLIP_CONTEXT: paths.contextPath, + }; +} + +function shellEscape(value: string): string { + return `'${value.replaceAll("'", `'\"'\"'`)}'`; +} + +export function formatShellExports(entries: Record): string { + return Object.entries(entries) + .filter(([, value]) => typeof value === "string" && value.trim().length > 0) + .map(([key, value]) => `export ${key}=${shellEscape(value)}`) + .join("\n"); +} diff --git a/cli/src/commands/worktree.ts b/cli/src/commands/worktree.ts new file mode 100644 index 00000000..6ccba042 --- /dev/null +++ b/cli/src/commands/worktree.ts @@ -0,0 +1,384 @@ +import { existsSync, readFileSync, rmSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { execFileSync } from "node:child_process"; +import { createServer } from "node:net"; +import * as p from "@clack/prompts"; +import pc from "picocolors"; +import { + ensurePostgresDatabase, + formatDatabaseBackupResult, + runDatabaseBackup, + runDatabaseRestore, +} from "@paperclipai/db"; +import type { Command } from "commander"; +import { ensureAgentJwtSecret, loadPaperclipEnvFile, mergePaperclipEnvEntries, readPaperclipEnvEntries, resolvePaperclipEnvFile } from "../config/env.js"; +import { expandHomePrefix } from "../config/home.js"; +import type { PaperclipConfig } from "../config/schema.js"; +import { readConfig, resolveConfigPath, writeConfig } from "../config/store.js"; +import { printPaperclipCliBanner } from "../utils/banner.js"; +import { + buildWorktreeConfig, + buildWorktreeEnvEntries, + DEFAULT_WORKTREE_HOME, + formatShellExports, + resolveSuggestedWorktreeName, + resolveWorktreeLocalPaths, + sanitizeWorktreeInstanceId, + type WorktreeLocalPaths, +} from "./worktree-lib.js"; + +type WorktreeInitOptions = { + name?: string; + instance?: string; + home?: string; + fromConfig?: string; + fromDataDir?: string; + fromInstance?: string; + serverPort?: number; + dbPort?: number; + seed?: boolean; + force?: boolean; +}; + +type WorktreeEnvOptions = { + config?: string; + json?: boolean; +}; + +type EmbeddedPostgresInstance = { + initialise(): Promise; + start(): Promise; + stop(): Promise; +}; + +type EmbeddedPostgresCtor = new (opts: { + databaseDir: string; + user: string; + password: string; + port: number; + persistent: boolean; + onLog?: (message: unknown) => void; + onError?: (message: unknown) => void; +}) => EmbeddedPostgresInstance; + +type EmbeddedPostgresHandle = { + port: number; + startedByThisProcess: boolean; + stop: () => Promise; +}; + +function nonEmpty(value: string | null | undefined): string | null { + return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; +} + +function readPidFilePort(postmasterPidFile: string): number | null { + if (!existsSync(postmasterPidFile)) return null; + try { + const lines = readFileSync(postmasterPidFile, "utf8").split("\n"); + const port = Number(lines[3]?.trim()); + return Number.isInteger(port) && port > 0 ? port : null; + } catch { + return null; + } +} + +function readRunningPostmasterPid(postmasterPidFile: string): number | null { + if (!existsSync(postmasterPidFile)) return null; + try { + const pid = Number(readFileSync(postmasterPidFile, "utf8").split("\n")[0]?.trim()); + if (!Number.isInteger(pid) || pid <= 0) return null; + process.kill(pid, 0); + return pid; + } catch { + return null; + } +} + +async function isPortAvailable(port: number): Promise { + return await new Promise((resolve) => { + const server = createServer(); + server.unref(); + server.once("error", () => resolve(false)); + server.listen(port, "127.0.0.1", () => { + server.close(() => resolve(true)); + }); + }); +} + +async function findAvailablePort(preferredPort: number, reserved = new Set()): Promise { + let port = Math.max(1, Math.trunc(preferredPort)); + while (reserved.has(port) || !(await isPortAvailable(port))) { + port += 1; + } + return port; +} + +function detectGitBranchName(cwd: string): string | null { + try { + const value = execFileSync("git", ["branch", "--show-current"], { + cwd, + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }).trim(); + return nonEmpty(value); + } catch { + return null; + } +} + +function resolveSourceConfigPath(opts: WorktreeInitOptions): string { + if (opts.fromConfig) return path.resolve(opts.fromConfig); + const sourceHome = path.resolve(expandHomePrefix(opts.fromDataDir ?? "~/.paperclip")); + const sourceInstanceId = sanitizeWorktreeInstanceId(opts.fromInstance ?? "default"); + return path.resolve(sourceHome, "instances", sourceInstanceId, "config.json"); +} + +function resolveSourceConnectionString(config: PaperclipConfig, envEntries: Record, portOverride?: number): string { + if (config.database.mode === "postgres") { + const connectionString = nonEmpty(envEntries.DATABASE_URL) ?? nonEmpty(config.database.connectionString); + if (!connectionString) { + throw new Error( + "Source instance uses postgres mode but has no connection string in config or adjacent .env.", + ); + } + return connectionString; + } + + const port = portOverride ?? config.database.embeddedPostgresPort; + return `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`; +} + +async function ensureEmbeddedPostgres(dataDir: string, preferredPort: number): Promise { + const moduleName = "embedded-postgres"; + let EmbeddedPostgres: EmbeddedPostgresCtor; + try { + const mod = await import(moduleName); + EmbeddedPostgres = mod.default as EmbeddedPostgresCtor; + } catch { + throw new Error( + "Embedded PostgreSQL support requires dependency `embedded-postgres`. Reinstall dependencies and try again.", + ); + } + + const postmasterPidFile = path.resolve(dataDir, "postmaster.pid"); + const runningPid = readRunningPostmasterPid(postmasterPidFile); + if (runningPid) { + return { + port: readPidFilePort(postmasterPidFile) ?? preferredPort, + startedByThisProcess: false, + stop: async () => {}, + }; + } + + const port = await findAvailablePort(preferredPort); + const instance = new EmbeddedPostgres({ + databaseDir: dataDir, + user: "paperclip", + password: "paperclip", + port, + persistent: true, + }); + + if (!existsSync(path.resolve(dataDir, "PG_VERSION"))) { + await instance.initialise(); + } + if (existsSync(postmasterPidFile)) { + rmSync(postmasterPidFile, { force: true }); + } + await instance.start(); + + return { + port, + startedByThisProcess: true, + stop: async () => { + await instance.stop(); + }, + }; +} + +async function seedWorktreeDatabase(input: { + sourceConfigPath: string; + sourceConfig: PaperclipConfig; + targetConfig: PaperclipConfig; + targetPaths: WorktreeLocalPaths; + instanceId: string; +}): Promise { + const sourceEnvFile = resolvePaperclipEnvFile(input.sourceConfigPath); + const sourceEnvEntries = readPaperclipEnvEntries(sourceEnvFile); + let sourceHandle: EmbeddedPostgresHandle | null = null; + let targetHandle: EmbeddedPostgresHandle | null = null; + + try { + if (input.sourceConfig.database.mode === "embedded-postgres") { + sourceHandle = await ensureEmbeddedPostgres( + input.sourceConfig.database.embeddedPostgresDataDir, + input.sourceConfig.database.embeddedPostgresPort, + ); + } + const sourceConnectionString = resolveSourceConnectionString( + input.sourceConfig, + sourceEnvEntries, + sourceHandle?.port, + ); + const backup = await runDatabaseBackup({ + connectionString: sourceConnectionString, + backupDir: path.resolve(input.targetPaths.backupDir, "seed"), + retentionDays: 7, + filenamePrefix: `${input.instanceId}-seed`, + includeMigrationJournal: true, + }); + + targetHandle = await ensureEmbeddedPostgres( + input.targetConfig.database.embeddedPostgresDataDir, + input.targetConfig.database.embeddedPostgresPort, + ); + + const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${targetHandle.port}/postgres`; + await ensurePostgresDatabase(adminConnectionString, "paperclip"); + const targetConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${targetHandle.port}/paperclip`; + await runDatabaseRestore({ + connectionString: targetConnectionString, + backupFile: backup.backupFile, + }); + + return formatDatabaseBackupResult(backup); + } finally { + if (targetHandle?.startedByThisProcess) { + await targetHandle.stop(); + } + if (sourceHandle?.startedByThisProcess) { + await sourceHandle.stop(); + } + } +} + +export async function worktreeInitCommand(opts: WorktreeInitOptions): Promise { + printPaperclipCliBanner(); + p.intro(pc.bgCyan(pc.black(" paperclipai worktree init "))); + + const cwd = process.cwd(); + const name = resolveSuggestedWorktreeName( + cwd, + opts.name ?? detectGitBranchName(cwd) ?? undefined, + ); + const instanceId = sanitizeWorktreeInstanceId(opts.instance ?? name); + const paths = resolveWorktreeLocalPaths({ + cwd, + homeDir: opts.home ?? DEFAULT_WORKTREE_HOME, + instanceId, + }); + const sourceConfigPath = resolveSourceConfigPath(opts); + const sourceConfig = existsSync(sourceConfigPath) ? readConfig(sourceConfigPath) : null; + + if ((existsSync(paths.configPath) || existsSync(paths.instanceRoot)) && !opts.force) { + throw new Error( + `Worktree config already exists at ${paths.configPath} or instance data exists at ${paths.instanceRoot}. Re-run with --force to replace it.`, + ); + } + + if (opts.force) { + rmSync(paths.repoConfigDir, { recursive: true, force: true }); + rmSync(paths.instanceRoot, { recursive: true, force: true }); + } + + const preferredServerPort = opts.serverPort ?? ((sourceConfig?.server.port ?? 3100) + 1); + const serverPort = await findAvailablePort(preferredServerPort); + const preferredDbPort = opts.dbPort ?? ((sourceConfig?.database.embeddedPostgresPort ?? 54329) + 1); + const databasePort = await findAvailablePort(preferredDbPort, new Set([serverPort])); + const targetConfig = buildWorktreeConfig({ + sourceConfig, + paths, + serverPort, + databasePort, + }); + + writeConfig(targetConfig, paths.configPath); + mergePaperclipEnvEntries(buildWorktreeEnvEntries(paths), paths.envPath); + ensureAgentJwtSecret(paths.configPath); + loadPaperclipEnvFile(paths.configPath); + + let seedSummary: string | null = null; + if (opts.seed !== false) { + if (!sourceConfig) { + throw new Error( + `Cannot seed worktree database because source config was not found at ${sourceConfigPath}. Use --no-seed or provide --from-config.`, + ); + } + const spinner = p.spinner(); + spinner.start("Seeding isolated worktree database from source instance..."); + try { + seedSummary = await seedWorktreeDatabase({ + sourceConfigPath, + sourceConfig, + targetConfig, + targetPaths: paths, + instanceId, + }); + spinner.stop("Seeded isolated worktree database."); + } catch (error) { + spinner.stop(pc.red("Failed to seed worktree database.")); + throw error; + } + } + + p.log.message(pc.dim(`Repo config: ${paths.configPath}`)); + p.log.message(pc.dim(`Repo env: ${paths.envPath}`)); + p.log.message(pc.dim(`Isolated home: ${paths.homeDir}`)); + p.log.message(pc.dim(`Instance: ${paths.instanceId}`)); + p.log.message(pc.dim(`Server port: ${serverPort} | DB port: ${databasePort}`)); + if (seedSummary) { + p.log.message(pc.dim(`Seed snapshot: ${seedSummary}`)); + } + p.outro( + pc.green( + `Worktree ready. Run Paperclip inside this repo and the CLI/server will use ${paths.instanceId} automatically.`, + ), + ); +} + +export async function worktreeEnvCommand(opts: WorktreeEnvOptions): Promise { + const configPath = resolveConfigPath(opts.config); + const envPath = resolvePaperclipEnvFile(configPath); + const envEntries = readPaperclipEnvEntries(envPath); + const out = { + PAPERCLIP_CONFIG: configPath, + ...(envEntries.PAPERCLIP_HOME ? { PAPERCLIP_HOME: envEntries.PAPERCLIP_HOME } : {}), + ...(envEntries.PAPERCLIP_INSTANCE_ID ? { PAPERCLIP_INSTANCE_ID: envEntries.PAPERCLIP_INSTANCE_ID } : {}), + ...(envEntries.PAPERCLIP_CONTEXT ? { PAPERCLIP_CONTEXT: envEntries.PAPERCLIP_CONTEXT } : {}), + ...envEntries, + }; + + if (opts.json) { + console.log(JSON.stringify(out, null, 2)); + return; + } + + console.log(formatShellExports(out)); +} + +export function registerWorktreeCommands(program: Command): void { + const worktree = program.command("worktree").description("Worktree-local Paperclip instance helpers"); + + worktree + .command("init") + .description("Create repo-local config/env and an isolated instance for this worktree") + .option("--name ", "Display name used to derive the instance id") + .option("--instance ", "Explicit isolated instance id") + .option("--home ", `Home root for worktree instances (default: ${DEFAULT_WORKTREE_HOME})`) + .option("--from-config ", "Source config.json to seed from") + .option("--from-data-dir ", "Source PAPERCLIP_HOME used when deriving the source config") + .option("--from-instance ", "Source instance id when deriving the source config", "default") + .option("--server-port ", "Preferred server port", (value) => Number(value)) + .option("--db-port ", "Preferred embedded Postgres port", (value) => Number(value)) + .option("--no-seed", "Skip database seeding from the source instance") + .option("--force", "Replace existing repo-local config and isolated instance data", false) + .action(worktreeInitCommand); + + worktree + .command("env") + .description("Print shell exports for the current worktree-local Paperclip instance") + .option("-c, --config ", "Path to config file") + .option("--json", "Print JSON instead of shell exports") + .action(worktreeEnvCommand); +} diff --git a/cli/src/config/env.ts b/cli/src/config/env.ts index 0ca4bcc1..4bc8f16e 100644 --- a/cli/src/config/env.ts +++ b/cli/src/config/env.ts @@ -25,13 +25,17 @@ function parseEnvFile(contents: string) { function renderEnvFile(entries: Record) { const lines = [ "# Paperclip environment variables", - "# Generated by `paperclipai onboard`", + "# Generated by Paperclip CLI commands", ...Object.entries(entries).map(([key, value]) => `${key}=${value}`), "", ]; return lines.join("\n"); } +export function resolvePaperclipEnvFile(configPath?: string): string { + return resolveEnvFilePath(configPath); +} + export function resolveAgentJwtEnvFile(configPath?: string): string { return resolveEnvFilePath(configPath); } @@ -82,13 +86,33 @@ export function ensureAgentJwtSecret(configPath?: string): { secret: string; cre } export function writeAgentJwtEnv(secret: string, filePath = resolveEnvFilePath()): void { + mergePaperclipEnvEntries({ [JWT_SECRET_ENV_KEY]: secret }, filePath); +} + +export function readPaperclipEnvEntries(filePath = resolveEnvFilePath()): Record { + if (!fs.existsSync(filePath)) return {}; + return parseEnvFile(fs.readFileSync(filePath, "utf-8")); +} + +export function writePaperclipEnvEntries(entries: Record, filePath = resolveEnvFilePath()): void { const dir = path.dirname(filePath); fs.mkdirSync(dir, { recursive: true }); - - const current = fs.existsSync(filePath) ? parseEnvFile(fs.readFileSync(filePath, "utf-8")) : {}; - current[JWT_SECRET_ENV_KEY] = secret; - - fs.writeFileSync(filePath, renderEnvFile(current), { + fs.writeFileSync(filePath, renderEnvFile(entries), { mode: 0o600, }); } + +export function mergePaperclipEnvEntries( + entries: Record, + filePath = resolveEnvFilePath(), +): Record { + const current = readPaperclipEnvEntries(filePath); + const next = { + ...current, + ...Object.fromEntries( + Object.entries(entries).filter(([, value]) => typeof value === "string" && value.trim().length > 0), + ), + }; + writePaperclipEnvEntries(next, filePath); + return next; +} diff --git a/cli/src/index.ts b/cli/src/index.ts index 9c31f5ae..19ef69f9 100644 --- a/cli/src/index.ts +++ b/cli/src/index.ts @@ -16,6 +16,8 @@ import { registerApprovalCommands } from "./commands/client/approval.js"; import { registerActivityCommands } from "./commands/client/activity.js"; import { registerDashboardCommands } from "./commands/client/dashboard.js"; import { applyDataDirOverride, type DataDirOptionLike } from "./config/data-dir.js"; +import { loadPaperclipEnvFile } from "./config/env.js"; +import { registerWorktreeCommands } from "./commands/worktree.js"; const program = new Command(); const DATA_DIR_OPTION_HELP = @@ -33,6 +35,7 @@ program.hook("preAction", (_thisCommand, actionCommand) => { hasConfigOption: optionNames.has("config"), hasContextOption: optionNames.has("context"), }); + loadPaperclipEnvFile(options.config); }); program @@ -132,6 +135,7 @@ registerAgentCommands(program); registerApprovalCommands(program); registerActivityCommands(program); registerDashboardCommands(program); +registerWorktreeCommands(program); const auth = program.command("auth").description("Authentication and bootstrap utilities"); diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index d3362600..9578ead2 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -124,6 +124,42 @@ When a local agent run has no resolved project/session workspace, Paperclip fall This path honors `PAPERCLIP_HOME` and `PAPERCLIP_INSTANCE_ID` in non-default setups. +## Worktree-local Instances + +When developing from multiple git worktrees, do not point two Paperclip servers at the same embedded PostgreSQL data directory. + +Instead, create a repo-local Paperclip config plus an isolated instance for the worktree: + +```sh +paperclipai worktree init +``` + +This command: + +- writes repo-local files at `.paperclip/config.json` and `.paperclip/.env` +- creates an isolated instance under `~/.paperclip-worktrees/instances//` +- picks a free app port and embedded PostgreSQL port +- by default seeds the isolated DB from your main instance via a logical SQL snapshot + +After `worktree init`, both the server and the CLI auto-load the repo-local `.paperclip/.env` when run inside that worktree, so normal commands like `pnpm dev`, `paperclipai doctor`, and `paperclipai db:backup` stay scoped to the worktree instance. + +Print shell exports explicitly when needed: + +```sh +paperclipai worktree env +# or: +eval "$(paperclipai worktree env)" +``` + +Useful options: + +```sh +paperclipai worktree init --no-seed +paperclipai worktree init --from-instance default +paperclipai worktree init --from-data-dir ~/.paperclip +paperclipai worktree init --force +``` + ## Quick Health Checks In another terminal: diff --git a/packages/db/src/backup-lib.ts b/packages/db/src/backup-lib.ts index 951540b1..de8c290d 100644 --- a/packages/db/src/backup-lib.ts +++ b/packages/db/src/backup-lib.ts @@ -9,6 +9,7 @@ export type RunDatabaseBackupOptions = { retentionDays: number; filenamePrefix?: string; connectTimeoutSeconds?: number; + includeMigrationJournal?: boolean; }; export type RunDatabaseBackupResult = { @@ -17,6 +18,12 @@ export type RunDatabaseBackupResult = { prunedCount: number; }; +export type RunDatabaseRestoreOptions = { + connectionString: string; + backupFile: string; + connectTimeoutSeconds?: number; +}; + function timestamp(date: Date = new Date()): string { const pad = (n: number) => String(n).padStart(2, "0"); return `${date.getFullYear()}${pad(date.getMonth() + 1)}${pad(date.getDate())}-${pad(date.getHours())}${pad(date.getMinutes())}${pad(date.getSeconds())}`; @@ -51,6 +58,7 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise const filenamePrefix = opts.filenamePrefix ?? "paperclip"; const retentionDays = Math.max(1, Math.trunc(opts.retentionDays)); const connectTimeout = Math.max(1, Math.trunc(opts.connectTimeoutSeconds ?? 5)); + const includeMigrationJournal = opts.includeMigrationJournal === true; const sql = postgres(opts.connectionString, { max: 1, connect_timeout: connectTimeout }); try { @@ -89,7 +97,7 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'public' AND c.relkind = 'r' - AND c.relname != '__drizzle_migrations' + AND (${includeMigrationJournal}::boolean OR c.relname != '__drizzle_migrations') ORDER BY c.relname `; @@ -326,6 +334,18 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise } } +export async function runDatabaseRestore(opts: RunDatabaseRestoreOptions): Promise { + const connectTimeout = Math.max(1, Math.trunc(opts.connectTimeoutSeconds ?? 5)); + const sql = postgres(opts.connectionString, { max: 1, connect_timeout: connectTimeout }); + + try { + await sql`SELECT 1`; + await sql.file(opts.backupFile).execute(); + } finally { + await sql.end(); + } +} + export function formatDatabaseBackupResult(result: RunDatabaseBackupResult): string { const size = formatBackupSize(result.sizeBytes); const pruned = result.prunedCount > 0 ? `; pruned ${result.prunedCount} old backup(s)` : ""; diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 3cafa7af..f280cee1 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -12,8 +12,10 @@ export { } from "./client.js"; export { runDatabaseBackup, + runDatabaseRestore, formatDatabaseBackupResult, type RunDatabaseBackupOptions, type RunDatabaseBackupResult, + type RunDatabaseRestoreOptions, } from "./backup-lib.js"; export * from "./schema/index.js"; From 4a67db6a4d1eedf5b96408e99c68b561056673f6 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 10 Mar 2026 07:41:01 -0500 Subject: [PATCH 08/12] Add minimal worktree seed mode --- cli/src/__tests__/worktree.test.ts | 15 ++ cli/src/commands/worktree-lib.ts | 45 ++++++ cli/src/commands/worktree.ts | 21 ++- doc/DEVELOPING.md | 10 +- packages/db/src/backup-lib.ts | 219 ++++++++++++++++++++++++----- 5 files changed, 269 insertions(+), 41 deletions(-) diff --git a/cli/src/__tests__/worktree.test.ts b/cli/src/__tests__/worktree.test.ts index f393c10c..18bdabf1 100644 --- a/cli/src/__tests__/worktree.test.ts +++ b/cli/src/__tests__/worktree.test.ts @@ -4,6 +4,7 @@ import { buildWorktreeConfig, buildWorktreeEnvEntries, formatShellExports, + resolveWorktreeSeedPlan, resolveWorktreeLocalPaths, rewriteLocalUrlPort, sanitizeWorktreeInstanceId, @@ -107,4 +108,18 @@ describe("worktree helpers", () => { expect(env.PAPERCLIP_INSTANCE_ID).toBe("feature-worktree-support"); expect(formatShellExports(env)).toContain("export PAPERCLIP_INSTANCE_ID='feature-worktree-support'"); }); + + it("uses minimal seed mode to keep app state but drop heavy runtime history", () => { + const minimal = resolveWorktreeSeedPlan("minimal"); + const full = resolveWorktreeSeedPlan("full"); + + expect(minimal.excludedTables).toContain("heartbeat_runs"); + expect(minimal.excludedTables).toContain("heartbeat_run_events"); + expect(minimal.excludedTables).toContain("workspace_runtime_services"); + expect(minimal.excludedTables).toContain("agent_task_sessions"); + expect(minimal.nullifyColumns.issues).toEqual(["checkout_run_id", "execution_run_id"]); + + expect(full.excludedTables).toEqual([]); + expect(full.nullifyColumns).toEqual({}); + }); }); diff --git a/cli/src/commands/worktree-lib.ts b/cli/src/commands/worktree-lib.ts index bc96a700..63509371 100644 --- a/cli/src/commands/worktree-lib.ts +++ b/cli/src/commands/worktree-lib.ts @@ -3,6 +3,30 @@ import type { PaperclipConfig } from "../config/schema.js"; import { expandHomePrefix } from "../config/home.js"; export const DEFAULT_WORKTREE_HOME = "~/.paperclip-worktrees"; +export const WORKTREE_SEED_MODES = ["minimal", "full"] as const; + +export type WorktreeSeedMode = (typeof WORKTREE_SEED_MODES)[number]; + +export type WorktreeSeedPlan = { + mode: WorktreeSeedMode; + excludedTables: string[]; + nullifyColumns: Record; +}; + +const MINIMAL_WORKTREE_EXCLUDED_TABLES = [ + "activity_log", + "agent_runtime_state", + "agent_task_sessions", + "agent_wakeup_requests", + "cost_events", + "heartbeat_run_events", + "heartbeat_runs", + "workspace_runtime_services", +]; + +const MINIMAL_WORKTREE_NULLIFIED_COLUMNS: Record = { + issues: ["checkout_run_id", "execution_run_id"], +}; export type WorktreeLocalPaths = { cwd: string; @@ -20,6 +44,27 @@ export type WorktreeLocalPaths = { storageDir: string; }; +export function isWorktreeSeedMode(value: string): value is WorktreeSeedMode { + return (WORKTREE_SEED_MODES as readonly string[]).includes(value); +} + +export function resolveWorktreeSeedPlan(mode: WorktreeSeedMode): WorktreeSeedPlan { + if (mode === "full") { + return { + mode, + excludedTables: [], + nullifyColumns: {}, + }; + } + return { + mode, + excludedTables: [...MINIMAL_WORKTREE_EXCLUDED_TABLES], + nullifyColumns: { + ...MINIMAL_WORKTREE_NULLIFIED_COLUMNS, + }, + }; +} + function nonEmpty(value: string | null | undefined): string | null { return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; } diff --git a/cli/src/commands/worktree.ts b/cli/src/commands/worktree.ts index 6ccba042..3c699dc0 100644 --- a/cli/src/commands/worktree.ts +++ b/cli/src/commands/worktree.ts @@ -22,9 +22,12 @@ import { buildWorktreeEnvEntries, DEFAULT_WORKTREE_HOME, formatShellExports, + isWorktreeSeedMode, resolveSuggestedWorktreeName, + resolveWorktreeSeedPlan, resolveWorktreeLocalPaths, sanitizeWorktreeInstanceId, + type WorktreeSeedMode, type WorktreeLocalPaths, } from "./worktree-lib.js"; @@ -38,6 +41,7 @@ type WorktreeInitOptions = { serverPort?: number; dbPort?: number; seed?: boolean; + seedMode?: string; force?: boolean; }; @@ -178,6 +182,8 @@ async function ensureEmbeddedPostgres(dataDir: string, preferredPort: number): P password: "paperclip", port, persistent: true, + onLog: () => {}, + onError: () => {}, }); if (!existsSync(path.resolve(dataDir, "PG_VERSION"))) { @@ -203,7 +209,9 @@ async function seedWorktreeDatabase(input: { targetConfig: PaperclipConfig; targetPaths: WorktreeLocalPaths; instanceId: string; + seedMode: WorktreeSeedMode; }): Promise { + const seedPlan = resolveWorktreeSeedPlan(input.seedMode); const sourceEnvFile = resolvePaperclipEnvFile(input.sourceConfigPath); const sourceEnvEntries = readPaperclipEnvEntries(sourceEnvFile); let sourceHandle: EmbeddedPostgresHandle | null = null; @@ -227,6 +235,8 @@ async function seedWorktreeDatabase(input: { retentionDays: 7, filenamePrefix: `${input.instanceId}-seed`, includeMigrationJournal: true, + excludeTables: seedPlan.excludedTables, + nullifyColumns: seedPlan.nullifyColumns, }); targetHandle = await ensureEmbeddedPostgres( @@ -262,6 +272,10 @@ export async function worktreeInitCommand(opts: WorktreeInitOptions): Promise", "Source instance id when deriving the source config", "default") .option("--server-port ", "Preferred server port", (value) => Number(value)) .option("--db-port ", "Preferred embedded Postgres port", (value) => Number(value)) + .option("--seed-mode ", "Seed profile: minimal or full (default: minimal)", "minimal") .option("--no-seed", "Skip database seeding from the source instance") .option("--force", "Replace existing repo-local config and isolated instance data", false) .action(worktreeInitCommand); diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 9578ead2..0ce30684 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -139,7 +139,13 @@ This command: - writes repo-local files at `.paperclip/config.json` and `.paperclip/.env` - creates an isolated instance under `~/.paperclip-worktrees/instances//` - picks a free app port and embedded PostgreSQL port -- by default seeds the isolated DB from your main instance via a logical SQL snapshot +- by default seeds the isolated DB in `minimal` mode from your main instance via a logical SQL snapshot + +Seed modes: + +- `minimal` keeps core app state like companies, projects, issues, comments, approvals, and auth state, but drops heavy operational history such as heartbeat runs, wake requests, activity logs, runtime services, and agent session state +- `full` makes a full logical clone of the source instance +- `--no-seed` creates an empty isolated instance After `worktree init`, both the server and the CLI auto-load the repo-local `.paperclip/.env` when run inside that worktree, so normal commands like `pnpm dev`, `paperclipai doctor`, and `paperclipai db:backup` stay scoped to the worktree instance. @@ -155,6 +161,8 @@ Useful options: ```sh paperclipai worktree init --no-seed +paperclipai worktree init --seed-mode minimal +paperclipai worktree init --seed-mode full paperclipai worktree init --from-instance default paperclipai worktree init --from-data-dir ~/.paperclip paperclipai worktree init --force diff --git a/packages/db/src/backup-lib.ts b/packages/db/src/backup-lib.ts index de8c290d..810703a7 100644 --- a/packages/db/src/backup-lib.ts +++ b/packages/db/src/backup-lib.ts @@ -1,6 +1,6 @@ import { existsSync, mkdirSync, readdirSync, statSync, unlinkSync } from "node:fs"; -import { writeFile } from "node:fs/promises"; -import { resolve } from "node:path"; +import { readFile, writeFile } from "node:fs/promises"; +import { basename, resolve } from "node:path"; import postgres from "postgres"; export type RunDatabaseBackupOptions = { @@ -10,6 +10,8 @@ export type RunDatabaseBackupOptions = { filenamePrefix?: string; connectTimeoutSeconds?: number; includeMigrationJournal?: boolean; + excludeTables?: string[]; + nullifyColumns?: Record; }; export type RunDatabaseBackupResult = { @@ -24,6 +26,34 @@ export type RunDatabaseRestoreOptions = { connectTimeoutSeconds?: number; }; +type SequenceDefinition = { + sequence_name: string; + data_type: string; + start_value: string; + minimum_value: string; + maximum_value: string; + increment: string; + cycle_option: "YES" | "NO"; + owner_table: string | null; + owner_column: string | null; +}; + +const STATEMENT_BREAKPOINT = "-- paperclip statement breakpoint 69f6f3f1-42fd-46a6-bf17-d1d85f8f3900"; + +function sanitizeRestoreErrorMessage(error: unknown): string { + if (error && typeof error === "object") { + const record = error as Record; + const firstLine = typeof record.message === "string" + ? record.message.split(/\r?\n/, 1)[0]?.trim() + : ""; + const detail = typeof record.detail === "string" ? record.detail.trim() : ""; + const severity = typeof record.severity === "string" ? record.severity.trim() : ""; + const message = firstLine || detail || (error instanceof Error ? error.message : String(error)); + return severity ? `${severity}: ${message}` : message; + } + return error instanceof Error ? error.message : String(error); +} + function timestamp(date: Date = new Date()): string { const pad = (n: number) => String(n).padStart(2, "0"); return `${date.getFullYear()}${pad(date.getMonth() + 1)}${pad(date.getDate())}-${pad(date.getHours())}${pad(date.getMinutes())}${pad(date.getSeconds())}`; @@ -54,11 +84,48 @@ function formatBackupSize(sizeBytes: number): string { return `${(sizeBytes / (1024 * 1024)).toFixed(1)}M`; } +function formatSqlLiteral(value: string): string { + const sanitized = value.replace(/\u0000/g, ""); + let tag = "$paperclip$"; + while (sanitized.includes(tag)) { + tag = `$paperclip_${Math.random().toString(36).slice(2, 8)}$`; + } + return `${tag}${sanitized}${tag}`; +} + +function normalizeTableNameSet(values: string[] | undefined): Set { + return new Set( + (values ?? []) + .map((value) => value.trim()) + .filter((value) => value.length > 0), + ); +} + +function normalizeNullifyColumnMap(values: Record | undefined): Map> { + const out = new Map>(); + if (!values) return out; + for (const [tableName, columns] of Object.entries(values)) { + const normalizedTable = tableName.trim(); + if (normalizedTable.length === 0) continue; + const normalizedColumns = new Set( + columns + .map((column) => column.trim()) + .filter((column) => column.length > 0), + ); + if (normalizedColumns.size > 0) { + out.set(normalizedTable, normalizedColumns); + } + } + return out; +} + export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise { const filenamePrefix = opts.filenamePrefix ?? "paperclip"; const retentionDays = Math.max(1, Math.trunc(opts.retentionDays)); const connectTimeout = Math.max(1, Math.trunc(opts.connectTimeoutSeconds ?? 5)); const includeMigrationJournal = opts.includeMigrationJournal === true; + const excludedTableNames = normalizeTableNameSet(opts.excludeTables); + const nullifiedColumnsByTable = normalizeNullifyColumnMap(opts.nullifyColumns); const sql = postgres(opts.connectionString, { max: 1, connect_timeout: connectTimeout }); try { @@ -66,13 +133,36 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise const lines: string[] = []; const emit = (line: string) => lines.push(line); + const emitStatement = (statement: string) => { + emit(statement); + emit(STATEMENT_BREAKPOINT); + }; + const emitStatementBoundary = () => { + emit(STATEMENT_BREAKPOINT); + }; emit("-- Paperclip database backup"); emit(`-- Created: ${new Date().toISOString()}`); emit(""); - emit("BEGIN;"); + emitStatement("BEGIN;"); + emitStatement("SET LOCAL session_replication_role = replica;"); + emitStatement("SET LOCAL client_min_messages = warning;"); emit(""); + const allTables = await sql<{ tablename: string }[]>` + SELECT c.relname AS tablename + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' + AND c.relkind = 'r' + ORDER BY c.relname + `; + const tables = allTables.filter(({ tablename }) => { + if (!includeMigrationJournal && tablename === "__drizzle_migrations") return false; + return !excludedTableNames.has(tablename); + }); + const includedTableNames = new Set(tables.map(({ tablename }) => tablename)); + // Get all enums const enums = await sql<{ typname: string; labels: string[] }[]>` SELECT t.typname, array_agg(e.enumlabel ORDER BY e.enumsortorder) AS labels @@ -86,20 +176,42 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise for (const e of enums) { const labels = e.labels.map((l) => `'${l.replace(/'/g, "''")}'`).join(", "); - emit(`CREATE TYPE "public"."${e.typname}" AS ENUM (${labels});`); + emitStatement(`CREATE TYPE "public"."${e.typname}" AS ENUM (${labels});`); } if (enums.length > 0) emit(""); - // Get tables in dependency order (referenced tables first) - const tables = await sql<{ tablename: string }[]>` - SELECT c.relname AS tablename - FROM pg_class c - JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE n.nspname = 'public' - AND c.relkind = 'r' - AND (${includeMigrationJournal}::boolean OR c.relname != '__drizzle_migrations') - ORDER BY c.relname + const allSequences = await sql` + SELECT + s.sequence_name, + s.data_type, + s.start_value, + s.minimum_value, + s.maximum_value, + s.increment, + s.cycle_option, + tbl.relname AS owner_table, + attr.attname AS owner_column + FROM information_schema.sequences s + JOIN pg_class seq ON seq.relname = s.sequence_name + JOIN pg_namespace n ON n.oid = seq.relnamespace AND n.nspname = s.sequence_schema + LEFT JOIN pg_depend dep ON dep.objid = seq.oid AND dep.deptype = 'a' + LEFT JOIN pg_class tbl ON tbl.oid = dep.refobjid + LEFT JOIN pg_attribute attr ON attr.attrelid = tbl.oid AND attr.attnum = dep.refobjsubid + WHERE s.sequence_schema = 'public' + ORDER BY s.sequence_name `; + const sequences = allSequences.filter((seq) => !seq.owner_table || includedTableNames.has(seq.owner_table)); + + if (sequences.length > 0) { + emit("-- Sequences"); + for (const seq of sequences) { + emitStatement(`DROP SEQUENCE IF EXISTS "${seq.sequence_name}" CASCADE;`); + emitStatement( + `CREATE SEQUENCE "${seq.sequence_name}" AS ${seq.data_type} INCREMENT BY ${seq.increment} MINVALUE ${seq.minimum_value} MAXVALUE ${seq.maximum_value} START WITH ${seq.start_value}${seq.cycle_option === "YES" ? " CYCLE" : " NO CYCLE"};`, + ); + } + emit(""); + } // Get full CREATE TABLE DDL via column info for (const { tablename } of tables) { @@ -121,7 +233,7 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise `; emit(`-- Table: ${tablename}`); - emit(`DROP TABLE IF EXISTS "${tablename}" CASCADE;`); + emitStatement(`DROP TABLE IF EXISTS "${tablename}" CASCADE;`); const colDefs: string[] = []; for (const col of columns) { @@ -168,11 +280,23 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise emit(`CREATE TABLE "${tablename}" (`); emit(colDefs.join(",\n")); emit(");"); + emitStatementBoundary(); + emit(""); + } + + const ownedSequences = sequences.filter((seq) => seq.owner_table && seq.owner_column); + if (ownedSequences.length > 0) { + emit("-- Sequence ownership"); + for (const seq of ownedSequences) { + emitStatement( + `ALTER SEQUENCE "${seq.sequence_name}" OWNED BY "${seq.owner_table!}"."${seq.owner_column!}";`, + ); + } emit(""); } // Foreign keys (after all tables created) - const fks = await sql<{ + const allForeignKeys = await sql<{ constraint_name: string; source_table: string; source_columns: string[]; @@ -199,13 +323,16 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise GROUP BY c.conname, src.relname, tgt.relname, c.confupdtype, c.confdeltype ORDER BY src.relname, c.conname `; + const fks = allForeignKeys.filter( + (fk) => includedTableNames.has(fk.source_table) && includedTableNames.has(fk.target_table), + ); if (fks.length > 0) { emit("-- Foreign keys"); for (const fk of fks) { const srcCols = fk.source_columns.map((c) => `"${c}"`).join(", "); const tgtCols = fk.target_columns.map((c) => `"${c}"`).join(", "); - emit( + emitStatement( `ALTER TABLE "${fk.source_table}" ADD CONSTRAINT "${fk.constraint_name}" FOREIGN KEY (${srcCols}) REFERENCES "${fk.target_table}" (${tgtCols}) ON UPDATE ${fk.update_rule} ON DELETE ${fk.delete_rule};`, ); } @@ -213,7 +340,7 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise } // Unique constraints - const uniques = await sql<{ + const allUniqueConstraints = await sql<{ constraint_name: string; tablename: string; column_names: string[]; @@ -229,19 +356,20 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise GROUP BY c.conname, t.relname ORDER BY t.relname, c.conname `; + const uniques = allUniqueConstraints.filter((entry) => includedTableNames.has(entry.tablename)); if (uniques.length > 0) { emit("-- Unique constraints"); for (const u of uniques) { const cols = u.column_names.map((c) => `"${c}"`).join(", "); - emit(`ALTER TABLE "${u.tablename}" ADD CONSTRAINT "${u.constraint_name}" UNIQUE (${cols});`); + emitStatement(`ALTER TABLE "${u.tablename}" ADD CONSTRAINT "${u.constraint_name}" UNIQUE (${cols});`); } emit(""); } // Indexes (non-primary, non-unique-constraint) - const indexes = await sql<{ indexdef: string }[]>` - SELECT indexdef + const allIndexes = await sql<{ tablename: string; indexdef: string }[]>` + SELECT tablename, indexdef FROM pg_indexes WHERE schemaname = 'public' AND indexname NOT IN ( @@ -250,11 +378,12 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise ) ORDER BY tablename, indexname `; + const indexes = allIndexes.filter((entry) => includedTableNames.has(entry.tablename)); if (indexes.length > 0) { emit("-- Indexes"); for (const idx of indexes) { - emit(`${idx.indexdef};`); + emitStatement(`${idx.indexdef};`); } emit(""); } @@ -278,42 +407,38 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise emit(`-- Data for: ${tablename} (${count[0]!.n} rows)`); const rows = await sql`SELECT * FROM ${sql(tablename)}`.values(); + const nullifiedColumns = nullifiedColumnsByTable.get(tablename) ?? new Set(); for (const row of rows) { - const values = row.map((val: unknown) => { + const values = row.map((rawValue: unknown, index) => { + const columnName = cols[index]?.column_name; + const val = columnName && nullifiedColumns.has(columnName) ? null : rawValue; if (val === null || val === undefined) return "NULL"; if (typeof val === "boolean") return val ? "true" : "false"; if (typeof val === "number") return String(val); - if (val instanceof Date) return `'${val.toISOString()}'`; - if (typeof val === "object") return `'${JSON.stringify(val).replace(/'/g, "''")}'`; - return `'${String(val).replace(/'/g, "''")}'`; + if (val instanceof Date) return formatSqlLiteral(val.toISOString()); + if (typeof val === "object") return formatSqlLiteral(JSON.stringify(val)); + return formatSqlLiteral(String(val)); }); - emit(`INSERT INTO "${tablename}" (${colNames}) VALUES (${values.join(", ")});`); + emitStatement(`INSERT INTO "${tablename}" (${colNames}) VALUES (${values.join(", ")});`); } emit(""); } // Sequence values - const sequences = await sql<{ sequence_name: string }[]>` - SELECT sequence_name - FROM information_schema.sequences - WHERE sequence_schema = 'public' - ORDER BY sequence_name - `; - if (sequences.length > 0) { emit("-- Sequence values"); for (const seq of sequences) { - const val = await sql<{ last_value: string }[]>` - SELECT last_value::text FROM ${sql(seq.sequence_name)} + const val = await sql<{ last_value: string; is_called: boolean }[]>` + SELECT last_value::text, is_called FROM ${sql(seq.sequence_name)} `; if (val[0]) { - emit(`SELECT setval('"${seq.sequence_name}"', ${val[0].last_value});`); + emitStatement(`SELECT setval('"${seq.sequence_name}"', ${val[0].last_value}, ${val[0].is_called ? "true" : "false"});`); } } emit(""); } - emit("COMMIT;"); + emitStatement("COMMIT;"); emit(""); // Write the backup file @@ -340,7 +465,25 @@ export async function runDatabaseRestore(opts: RunDatabaseRestoreOptions): Promi try { await sql`SELECT 1`; - await sql.file(opts.backupFile).execute(); + const contents = await readFile(opts.backupFile, "utf8"); + const statements = contents + .split(STATEMENT_BREAKPOINT) + .map((statement) => statement.trim()) + .filter((statement) => statement.length > 0); + + for (const statement of statements) { + await sql.unsafe(statement).execute(); + } + } catch (error) { + const statementPreview = typeof error === "object" && error !== null && typeof (error as Record).query === "string" + ? String((error as Record).query) + .split(/\r?\n/) + .map((line) => line.trim()) + .find((line) => line.length > 0 && !line.startsWith("--")) + : null; + throw new Error( + `Failed to restore ${basename(opts.backupFile)}: ${sanitizeRestoreErrorMessage(error)}${statementPreview ? ` [statement: ${statementPreview.slice(0, 120)}]` : ""}`, + ); } finally { await sql.end(); } From 83738b45cd567248ec0b0461cf3a7ad06bc4fee9 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 10 Mar 2026 10:08:58 -0500 Subject: [PATCH 09/12] Fix worktree minimal clone startup --- cli/src/commands/worktree.ts | 2 + doc/DEVELOPING.md | 2 +- packages/db/src/backup-lib.ts | 174 +++++++++++++++++++++++----------- packages/db/src/client.ts | 16 ++-- server/src/app.ts | 7 ++ server/src/index.ts | 3 +- 6 files changed, 140 insertions(+), 64 deletions(-) diff --git a/cli/src/commands/worktree.ts b/cli/src/commands/worktree.ts index 3c699dc0..f9363ff8 100644 --- a/cli/src/commands/worktree.ts +++ b/cli/src/commands/worktree.ts @@ -6,6 +6,7 @@ import { createServer } from "node:net"; import * as p from "@clack/prompts"; import pc from "picocolors"; import { + applyPendingMigrations, ensurePostgresDatabase, formatDatabaseBackupResult, runDatabaseBackup, @@ -251,6 +252,7 @@ async function seedWorktreeDatabase(input: { connectionString: targetConnectionString, backupFile: backup.backupFile, }); + await applyPendingMigrations(targetConnectionString); return formatDatabaseBackupResult(backup); } finally { diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 0ce30684..334306c2 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -143,7 +143,7 @@ This command: Seed modes: -- `minimal` keeps core app state like companies, projects, issues, comments, approvals, and auth state, but drops heavy operational history such as heartbeat runs, wake requests, activity logs, runtime services, and agent session state +- `minimal` keeps core app state like companies, projects, issues, comments, approvals, and auth state, preserves schema for all tables, but omits row data from heavy operational history such as heartbeat runs, wake requests, activity logs, runtime services, and agent session state - `full` makes a full logical clone of the source instance - `--no-seed` creates an empty isolated instance diff --git a/packages/db/src/backup-lib.ts b/packages/db/src/backup-lib.ts index 810703a7..26f918c3 100644 --- a/packages/db/src/backup-lib.ts +++ b/packages/db/src/backup-lib.ts @@ -27,6 +27,7 @@ export type RunDatabaseRestoreOptions = { }; type SequenceDefinition = { + sequence_schema: string; sequence_name: string; data_type: string; start_value: string; @@ -34,10 +35,19 @@ type SequenceDefinition = { maximum_value: string; increment: string; cycle_option: "YES" | "NO"; + owner_schema: string | null; owner_table: string | null; owner_column: string | null; }; +type TableDefinition = { + schema_name: string; + tablename: string; +}; + +const DRIZZLE_SCHEMA = "drizzle"; +const DRIZZLE_MIGRATIONS_TABLE = "__drizzle_migrations"; + const STATEMENT_BREAKPOINT = "-- paperclip statement breakpoint 69f6f3f1-42fd-46a6-bf17-d1d85f8f3900"; function sanitizeRestoreErrorMessage(error: unknown): string { @@ -119,6 +129,18 @@ function normalizeNullifyColumnMap(values: Record | undefined) return out; } +function quoteIdentifier(value: string): string { + return `"${value.replaceAll("\"", "\"\"")}"`; +} + +function quoteQualifiedName(schemaName: string, objectName: string): string { + return `${quoteIdentifier(schemaName)}.${quoteIdentifier(objectName)}`; +} + +function tableKey(schemaName: string, tableName: string): string { + return `${schemaName}.${tableName}`; +} + export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise { const filenamePrefix = opts.filenamePrefix ?? "paperclip"; const retentionDays = Math.max(1, Math.trunc(opts.retentionDays)); @@ -149,19 +171,18 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise emitStatement("SET LOCAL client_min_messages = warning;"); emit(""); - const allTables = await sql<{ tablename: string }[]>` - SELECT c.relname AS tablename - FROM pg_class c - JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE n.nspname = 'public' - AND c.relkind = 'r' - ORDER BY c.relname + const allTables = await sql` + SELECT table_schema AS schema_name, table_name AS tablename + FROM information_schema.tables + WHERE table_type = 'BASE TABLE' + AND ( + table_schema = 'public' + OR (${includeMigrationJournal}::boolean AND table_schema = ${DRIZZLE_SCHEMA} AND table_name = ${DRIZZLE_MIGRATIONS_TABLE}) + ) + ORDER BY table_schema, table_name `; - const tables = allTables.filter(({ tablename }) => { - if (!includeMigrationJournal && tablename === "__drizzle_migrations") return false; - return !excludedTableNames.has(tablename); - }); - const includedTableNames = new Set(tables.map(({ tablename }) => tablename)); + const tables = allTables; + const includedTableNames = new Set(tables.map(({ schema_name, tablename }) => tableKey(schema_name, tablename))); // Get all enums const enums = await sql<{ typname: string; labels: string[] }[]>` @@ -182,6 +203,7 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise const allSequences = await sql` SELECT + s.sequence_schema, s.sequence_name, s.data_type, s.start_value, @@ -189,6 +211,7 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise s.maximum_value, s.increment, s.cycle_option, + tblns.nspname AS owner_schema, tbl.relname AS owner_table, attr.attname AS owner_column FROM information_schema.sequences s @@ -196,25 +219,43 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise JOIN pg_namespace n ON n.oid = seq.relnamespace AND n.nspname = s.sequence_schema LEFT JOIN pg_depend dep ON dep.objid = seq.oid AND dep.deptype = 'a' LEFT JOIN pg_class tbl ON tbl.oid = dep.refobjid + LEFT JOIN pg_namespace tblns ON tblns.oid = tbl.relnamespace LEFT JOIN pg_attribute attr ON attr.attrelid = tbl.oid AND attr.attnum = dep.refobjsubid WHERE s.sequence_schema = 'public' - ORDER BY s.sequence_name + OR (${includeMigrationJournal}::boolean AND s.sequence_schema = ${DRIZZLE_SCHEMA}) + ORDER BY s.sequence_schema, s.sequence_name `; - const sequences = allSequences.filter((seq) => !seq.owner_table || includedTableNames.has(seq.owner_table)); + const sequences = allSequences.filter( + (seq) => !seq.owner_table || includedTableNames.has(tableKey(seq.owner_schema ?? "public", seq.owner_table)), + ); + + const schemas = new Set(); + for (const table of tables) schemas.add(table.schema_name); + for (const seq of sequences) schemas.add(seq.sequence_schema); + const extraSchemas = [...schemas].filter((schemaName) => schemaName !== "public"); + if (extraSchemas.length > 0) { + emit("-- Schemas"); + for (const schemaName of extraSchemas) { + emitStatement(`CREATE SCHEMA IF NOT EXISTS ${quoteIdentifier(schemaName)};`); + } + emit(""); + } if (sequences.length > 0) { emit("-- Sequences"); for (const seq of sequences) { - emitStatement(`DROP SEQUENCE IF EXISTS "${seq.sequence_name}" CASCADE;`); + const qualifiedSequenceName = quoteQualifiedName(seq.sequence_schema, seq.sequence_name); + emitStatement(`DROP SEQUENCE IF EXISTS ${qualifiedSequenceName} CASCADE;`); emitStatement( - `CREATE SEQUENCE "${seq.sequence_name}" AS ${seq.data_type} INCREMENT BY ${seq.increment} MINVALUE ${seq.minimum_value} MAXVALUE ${seq.maximum_value} START WITH ${seq.start_value}${seq.cycle_option === "YES" ? " CYCLE" : " NO CYCLE"};`, + `CREATE SEQUENCE ${qualifiedSequenceName} AS ${seq.data_type} INCREMENT BY ${seq.increment} MINVALUE ${seq.minimum_value} MAXVALUE ${seq.maximum_value} START WITH ${seq.start_value}${seq.cycle_option === "YES" ? " CYCLE" : " NO CYCLE"};`, ); } emit(""); } // Get full CREATE TABLE DDL via column info - for (const { tablename } of tables) { + for (const { schema_name, tablename } of tables) { + const qualifiedTableName = quoteQualifiedName(schema_name, tablename); const columns = await sql<{ column_name: string; data_type: string; @@ -228,12 +269,12 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise SELECT column_name, data_type, udt_name, is_nullable, column_default, character_maximum_length, numeric_precision, numeric_scale FROM information_schema.columns - WHERE table_schema = 'public' AND table_name = ${tablename} + WHERE table_schema = ${schema_name} AND table_name = ${tablename} ORDER BY ordinal_position `; - emit(`-- Table: ${tablename}`); - emitStatement(`DROP TABLE IF EXISTS "${tablename}" CASCADE;`); + emit(`-- Table: ${schema_name}.${tablename}`); + emitStatement(`DROP TABLE IF EXISTS ${qualifiedTableName} CASCADE;`); const colDefs: string[] = []; for (const col of columns) { @@ -269,7 +310,7 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise JOIN pg_class t ON t.oid = c.conrelid JOIN pg_namespace n ON n.oid = t.relnamespace JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(c.conkey) - WHERE n.nspname = 'public' AND t.relname = ${tablename} AND c.contype = 'p' + WHERE n.nspname = ${schema_name} AND t.relname = ${tablename} AND c.contype = 'p' GROUP BY c.conname `; for (const p of pk) { @@ -277,7 +318,7 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise colDefs.push(` CONSTRAINT "${p.constraint_name}" PRIMARY KEY (${cols})`); } - emit(`CREATE TABLE "${tablename}" (`); + emit(`CREATE TABLE ${qualifiedTableName} (`); emit(colDefs.join(",\n")); emit(");"); emitStatementBoundary(); @@ -289,7 +330,7 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise emit("-- Sequence ownership"); for (const seq of ownedSequences) { emitStatement( - `ALTER SEQUENCE "${seq.sequence_name}" OWNED BY "${seq.owner_table!}"."${seq.owner_column!}";`, + `ALTER SEQUENCE ${quoteQualifiedName(seq.sequence_schema, seq.sequence_name)} OWNED BY ${quoteQualifiedName(seq.owner_schema ?? "public", seq.owner_table!)}.${quoteIdentifier(seq.owner_column!)};`, ); } emit(""); @@ -298,8 +339,10 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise // Foreign keys (after all tables created) const allForeignKeys = await sql<{ constraint_name: string; + source_schema: string; source_table: string; source_columns: string[]; + target_schema: string; target_table: string; target_columns: string[]; update_rule: string; @@ -307,24 +350,31 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise }[]>` SELECT c.conname AS constraint_name, + srcn.nspname AS source_schema, src.relname AS source_table, array_agg(sa.attname ORDER BY array_position(c.conkey, sa.attnum)) AS source_columns, + tgtn.nspname AS target_schema, tgt.relname AS target_table, array_agg(ta.attname ORDER BY array_position(c.confkey, ta.attnum)) AS target_columns, CASE c.confupdtype WHEN 'a' THEN 'NO ACTION' WHEN 'r' THEN 'RESTRICT' WHEN 'c' THEN 'CASCADE' WHEN 'n' THEN 'SET NULL' WHEN 'd' THEN 'SET DEFAULT' END AS update_rule, CASE c.confdeltype WHEN 'a' THEN 'NO ACTION' WHEN 'r' THEN 'RESTRICT' WHEN 'c' THEN 'CASCADE' WHEN 'n' THEN 'SET NULL' WHEN 'd' THEN 'SET DEFAULT' END AS delete_rule FROM pg_constraint c JOIN pg_class src ON src.oid = c.conrelid + JOIN pg_namespace srcn ON srcn.oid = src.relnamespace JOIN pg_class tgt ON tgt.oid = c.confrelid - JOIN pg_namespace n ON n.oid = src.relnamespace + JOIN pg_namespace tgtn ON tgtn.oid = tgt.relnamespace JOIN pg_attribute sa ON sa.attrelid = src.oid AND sa.attnum = ANY(c.conkey) JOIN pg_attribute ta ON ta.attrelid = tgt.oid AND ta.attnum = ANY(c.confkey) - WHERE c.contype = 'f' AND n.nspname = 'public' - GROUP BY c.conname, src.relname, tgt.relname, c.confupdtype, c.confdeltype - ORDER BY src.relname, c.conname + WHERE c.contype = 'f' AND ( + srcn.nspname = 'public' + OR (${includeMigrationJournal}::boolean AND srcn.nspname = ${DRIZZLE_SCHEMA}) + ) + GROUP BY c.conname, srcn.nspname, src.relname, tgtn.nspname, tgt.relname, c.confupdtype, c.confdeltype + ORDER BY srcn.nspname, src.relname, c.conname `; const fks = allForeignKeys.filter( - (fk) => includedTableNames.has(fk.source_table) && includedTableNames.has(fk.target_table), + (fk) => includedTableNames.has(tableKey(fk.source_schema, fk.source_table)) + && includedTableNames.has(tableKey(fk.target_schema, fk.target_table)), ); if (fks.length > 0) { @@ -333,7 +383,7 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise const srcCols = fk.source_columns.map((c) => `"${c}"`).join(", "); const tgtCols = fk.target_columns.map((c) => `"${c}"`).join(", "); emitStatement( - `ALTER TABLE "${fk.source_table}" ADD CONSTRAINT "${fk.constraint_name}" FOREIGN KEY (${srcCols}) REFERENCES "${fk.target_table}" (${tgtCols}) ON UPDATE ${fk.update_rule} ON DELETE ${fk.delete_rule};`, + `ALTER TABLE ${quoteQualifiedName(fk.source_schema, fk.source_table)} ADD CONSTRAINT "${fk.constraint_name}" FOREIGN KEY (${srcCols}) REFERENCES ${quoteQualifiedName(fk.target_schema, fk.target_table)} (${tgtCols}) ON UPDATE ${fk.update_rule} ON DELETE ${fk.delete_rule};`, ); } emit(""); @@ -342,43 +392,52 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise // Unique constraints const allUniqueConstraints = await sql<{ constraint_name: string; + schema_name: string; tablename: string; column_names: string[]; }[]>` SELECT c.conname AS constraint_name, + n.nspname AS schema_name, t.relname AS tablename, array_agg(a.attname ORDER BY array_position(c.conkey, a.attnum)) AS column_names FROM pg_constraint c JOIN pg_class t ON t.oid = c.conrelid JOIN pg_namespace n ON n.oid = t.relnamespace JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(c.conkey) - WHERE n.nspname = 'public' AND c.contype = 'u' - GROUP BY c.conname, t.relname - ORDER BY t.relname, c.conname + WHERE c.contype = 'u' AND ( + n.nspname = 'public' + OR (${includeMigrationJournal}::boolean AND n.nspname = ${DRIZZLE_SCHEMA}) + ) + GROUP BY c.conname, n.nspname, t.relname + ORDER BY n.nspname, t.relname, c.conname `; - const uniques = allUniqueConstraints.filter((entry) => includedTableNames.has(entry.tablename)); + const uniques = allUniqueConstraints.filter((entry) => includedTableNames.has(tableKey(entry.schema_name, entry.tablename))); if (uniques.length > 0) { emit("-- Unique constraints"); for (const u of uniques) { const cols = u.column_names.map((c) => `"${c}"`).join(", "); - emitStatement(`ALTER TABLE "${u.tablename}" ADD CONSTRAINT "${u.constraint_name}" UNIQUE (${cols});`); + emitStatement(`ALTER TABLE ${quoteQualifiedName(u.schema_name, u.tablename)} ADD CONSTRAINT "${u.constraint_name}" UNIQUE (${cols});`); } emit(""); } // Indexes (non-primary, non-unique-constraint) - const allIndexes = await sql<{ tablename: string; indexdef: string }[]>` - SELECT tablename, indexdef + const allIndexes = await sql<{ schema_name: string; tablename: string; indexdef: string }[]>` + SELECT schemaname AS schema_name, tablename, indexdef FROM pg_indexes - WHERE schemaname = 'public' - AND indexname NOT IN ( - SELECT conname FROM pg_constraint - WHERE connamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'public') + WHERE ( + schemaname = 'public' + OR (${includeMigrationJournal}::boolean AND schemaname = ${DRIZZLE_SCHEMA}) ) - ORDER BY tablename, indexname + AND indexname NOT IN ( + SELECT conname FROM pg_constraint c + JOIN pg_namespace n ON n.oid = c.connamespace + WHERE n.nspname = pg_indexes.schemaname + ) + ORDER BY schemaname, tablename, indexname `; - const indexes = allIndexes.filter((entry) => includedTableNames.has(entry.tablename)); + const indexes = allIndexes.filter((entry) => includedTableNames.has(tableKey(entry.schema_name, entry.tablename))); if (indexes.length > 0) { emit("-- Indexes"); @@ -389,24 +448,23 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise } // Dump data for each table - for (const { tablename } of tables) { - const count = await sql<{ n: number }[]>` - SELECT count(*)::int AS n FROM ${sql(tablename)} - `; - if ((count[0]?.n ?? 0) === 0) continue; + for (const { schema_name, tablename } of tables) { + const qualifiedTableName = quoteQualifiedName(schema_name, tablename); + const count = await sql.unsafe<{ n: number }[]>(`SELECT count(*)::int AS n FROM ${qualifiedTableName}`); + if (excludedTableNames.has(tablename) || (count[0]?.n ?? 0) === 0) continue; // Get column info for this table const cols = await sql<{ column_name: string; data_type: string }[]>` SELECT column_name, data_type FROM information_schema.columns - WHERE table_schema = 'public' AND table_name = ${tablename} + WHERE table_schema = ${schema_name} AND table_name = ${tablename} ORDER BY ordinal_position `; const colNames = cols.map((c) => `"${c.column_name}"`).join(", "); - emit(`-- Data for: ${tablename} (${count[0]!.n} rows)`); + emit(`-- Data for: ${schema_name}.${tablename} (${count[0]!.n} rows)`); - const rows = await sql`SELECT * FROM ${sql(tablename)}`.values(); + const rows = await sql.unsafe(`SELECT * FROM ${qualifiedTableName}`).values(); const nullifiedColumns = nullifiedColumnsByTable.get(tablename) ?? new Set(); for (const row of rows) { const values = row.map((rawValue: unknown, index) => { @@ -419,7 +477,7 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise if (typeof val === "object") return formatSqlLiteral(JSON.stringify(val)); return formatSqlLiteral(String(val)); }); - emitStatement(`INSERT INTO "${tablename}" (${colNames}) VALUES (${values.join(", ")});`); + emitStatement(`INSERT INTO ${qualifiedTableName} (${colNames}) VALUES (${values.join(", ")});`); } emit(""); } @@ -428,11 +486,15 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise if (sequences.length > 0) { emit("-- Sequence values"); for (const seq of sequences) { - const val = await sql<{ last_value: string; is_called: boolean }[]>` - SELECT last_value::text, is_called FROM ${sql(seq.sequence_name)} - `; - if (val[0]) { - emitStatement(`SELECT setval('"${seq.sequence_name}"', ${val[0].last_value}, ${val[0].is_called ? "true" : "false"});`); + const qualifiedSequenceName = quoteQualifiedName(seq.sequence_schema, seq.sequence_name); + const val = await sql.unsafe<{ last_value: string; is_called: boolean }[]>( + `SELECT last_value::text, is_called FROM ${qualifiedSequenceName}`, + ); + const skipSequenceValue = + seq.owner_table !== null + && excludedTableNames.has(seq.owner_table); + if (val[0] && !skipSequenceValue) { + emitStatement(`SELECT setval('${qualifiedSequenceName.replaceAll("'", "''")}', ${val[0].last_value}, ${val[0].is_called ? "true" : "false"});`); } } emit(""); diff --git a/packages/db/src/client.ts b/packages/db/src/client.ts index 8fa979d2..c4275dc4 100644 --- a/packages/db/src/client.ts +++ b/packages/db/src/client.ts @@ -10,6 +10,10 @@ const MIGRATIONS_FOLDER = fileURLToPath(new URL("./migrations", import.meta.url) const DRIZZLE_MIGRATIONS_TABLE = "__drizzle_migrations"; const MIGRATIONS_JOURNAL_JSON = fileURLToPath(new URL("./migrations/meta/_journal.json", import.meta.url)); +function createUtilitySql(url: string) { + return postgres(url, { max: 1, onnotice: () => {} }); +} + function isSafeIdentifier(value: string): boolean { return /^[A-Za-z_][A-Za-z0-9_]*$/.test(value); } @@ -223,7 +227,7 @@ async function applyPendingMigrationsManually( journalEntries.map((entry) => [entry.fileName, normalizeFolderMillis(entry.folderMillis)]), ); - const sql = postgres(url, { max: 1 }); + const sql = createUtilitySql(url); try { const { migrationTableSchema, columnNames } = await ensureMigrationJournalTable(sql); const qualifiedTable = `${quoteIdentifier(migrationTableSchema)}.${quoteIdentifier(DRIZZLE_MIGRATIONS_TABLE)}`; @@ -472,7 +476,7 @@ export async function reconcilePendingMigrationHistory( return { repairedMigrations: [], remainingMigrations: [] }; } - const sql = postgres(url, { max: 1 }); + const sql = createUtilitySql(url); const repairedMigrations: string[] = []; try { @@ -579,7 +583,7 @@ async function discoverMigrationTableSchema(sql: ReturnType): P } export async function inspectMigrations(url: string): Promise { - const sql = postgres(url, { max: 1 }); + const sql = createUtilitySql(url); try { const availableMigrations = await listMigrationFiles(); @@ -642,7 +646,7 @@ export async function applyPendingMigrations(url: string): Promise { const initialState = await inspectMigrations(url); if (initialState.status === "upToDate") return; - const sql = postgres(url, { max: 1 }); + const sql = createUtilitySql(url); try { const db = drizzlePg(sql); @@ -680,7 +684,7 @@ export type MigrationBootstrapResult = | { migrated: false; reason: "not-empty-no-migration-journal"; tableCount: number }; export async function migratePostgresIfEmpty(url: string): Promise { - const sql = postgres(url, { max: 1 }); + const sql = createUtilitySql(url); try { const migrationTableSchema = await discoverMigrationTableSchema(sql); @@ -719,7 +723,7 @@ export async function ensurePostgresDatabase( throw new Error(`Unsafe database name: ${databaseName}`); } - const sql = postgres(url, { max: 1 }); + const sql = createUtilitySql(url); try { const existing = await sql<{ one: number }[]>` select 1 as one from pg_database where datname = ${databaseName} limit 1 diff --git a/server/src/app.ts b/server/src/app.ts index b21ec39f..32b3e3bc 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -32,6 +32,7 @@ export async function createApp( db: Db, opts: { uiMode: UiMode; + serverPort: number; storageService: StorageService; deploymentMode: DeploymentMode; deploymentExposure: DeploymentExposure; @@ -146,12 +147,18 @@ export async function createApp( if (opts.uiMode === "vite-dev") { const uiRoot = path.resolve(__dirname, "../../ui"); + const hmrPort = opts.serverPort + 10000; const { createServer: createViteServer } = await import("vite"); const vite = await createViteServer({ root: uiRoot, appType: "spa", server: { middlewareMode: true, + hmr: { + host: opts.bindHost, + port: hmrPort, + clientPort: hmrPort, + }, allowedHosts: privateHostnameGateEnabled ? Array.from(privateHostnameAllowSet) : undefined, }, }); diff --git a/server/src/index.ts b/server/src/index.ts index 71992ce2..5220c4b1 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -460,10 +460,12 @@ export async function startServer(): Promise { authReady = true; } + const listenPort = await detectPort(config.port); const uiMode = config.uiDevMiddleware ? "vite-dev" : config.serveUi ? "static" : "none"; const storageService = createStorageServiceFromConfig(config); const app = await createApp(db as any, { uiMode, + serverPort: listenPort, storageService, deploymentMode: config.deploymentMode, deploymentExposure: config.deploymentExposure, @@ -475,7 +477,6 @@ export async function startServer(): Promise { resolveSession, }); const server = createServer(app as unknown as Parameters[0]); - const listenPort = await detectPort(config.port); if (listenPort !== config.port) { logger.warn(`Requested port is busy; using next free port (requestedPort=${config.port}, selectedPort=${listenPort})`); From d9574fea715d2a19d3ab83bb23357823c10bb741 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 10 Mar 2026 08:09:36 -0500 Subject: [PATCH 10/12] Fix doctor summary after repairs --- cli/src/__tests__/doctor.test.ts | 99 ++++++++++++++++++++++++++++++++ cli/src/commands/doctor.ts | 83 ++++++++++++++++++-------- 2 files changed, 158 insertions(+), 24 deletions(-) create mode 100644 cli/src/__tests__/doctor.test.ts diff --git a/cli/src/__tests__/doctor.test.ts b/cli/src/__tests__/doctor.test.ts new file mode 100644 index 00000000..83a67831 --- /dev/null +++ b/cli/src/__tests__/doctor.test.ts @@ -0,0 +1,99 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { doctor } from "../commands/doctor.js"; +import { writeConfig } from "../config/store.js"; +import type { PaperclipConfig } from "../config/schema.js"; + +const ORIGINAL_ENV = { ...process.env }; + +function createTempConfig(): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-doctor-")); + const configPath = path.join(root, ".paperclip", "config.json"); + const runtimeRoot = path.join(root, "runtime"); + + const config: PaperclipConfig = { + $meta: { + version: 1, + updatedAt: "2026-03-10T00:00:00.000Z", + source: "configure", + }, + database: { + mode: "embedded-postgres", + embeddedPostgresDataDir: path.join(runtimeRoot, "db"), + embeddedPostgresPort: 55432, + backup: { + enabled: true, + intervalMinutes: 60, + retentionDays: 30, + dir: path.join(runtimeRoot, "backups"), + }, + }, + logging: { + mode: "file", + logDir: path.join(runtimeRoot, "logs"), + }, + server: { + deploymentMode: "local_trusted", + exposure: "private", + host: "127.0.0.1", + port: 3199, + allowedHostnames: [], + serveUi: true, + }, + auth: { + baseUrlMode: "auto", + disableSignUp: false, + }, + storage: { + provider: "local_disk", + localDisk: { + baseDir: path.join(runtimeRoot, "storage"), + }, + s3: { + bucket: "paperclip", + region: "us-east-1", + prefix: "", + forcePathStyle: false, + }, + }, + secrets: { + provider: "local_encrypted", + strictMode: false, + localEncrypted: { + keyFilePath: path.join(runtimeRoot, "secrets", "master.key"), + }, + }, + }; + + writeConfig(config, configPath); + return configPath; +} + +describe("doctor", () => { + beforeEach(() => { + process.env = { ...ORIGINAL_ENV }; + delete process.env.PAPERCLIP_AGENT_JWT_SECRET; + delete process.env.PAPERCLIP_SECRETS_MASTER_KEY; + delete process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE; + }); + + afterEach(() => { + process.env = { ...ORIGINAL_ENV }; + }); + + it("re-runs repairable checks so repaired failures do not remain blocking", async () => { + const configPath = createTempConfig(); + + const summary = await doctor({ + config: configPath, + repair: true, + yes: true, + }); + + expect(summary.failed).toBe(0); + expect(summary.warned).toBe(0); + expect(process.env.PAPERCLIP_AGENT_JWT_SECRET).toBeTruthy(); + }); +}); diff --git a/cli/src/commands/doctor.ts b/cli/src/commands/doctor.ts index ab99b012..3ace070e 100644 --- a/cli/src/commands/doctor.ts +++ b/cli/src/commands/doctor.ts @@ -66,28 +66,40 @@ export async function doctor(opts: { printResult(deploymentAuthResult); // 3. Agent JWT check - const jwtResult = agentJwtSecretCheck(opts.config); - results.push(jwtResult); - printResult(jwtResult); - await maybeRepair(jwtResult, opts); + results.push( + await runRepairableCheck({ + run: () => agentJwtSecretCheck(opts.config), + configPath, + opts, + }), + ); // 4. Secrets adapter check - const secretsResult = secretsCheck(config, configPath); - results.push(secretsResult); - printResult(secretsResult); - await maybeRepair(secretsResult, opts); + results.push( + await runRepairableCheck({ + run: () => secretsCheck(config, configPath), + configPath, + opts, + }), + ); // 5. Storage check - const storageResult = storageCheck(config, configPath); - results.push(storageResult); - printResult(storageResult); - await maybeRepair(storageResult, opts); + results.push( + await runRepairableCheck({ + run: () => storageCheck(config, configPath), + configPath, + opts, + }), + ); // 6. Database check - const dbResult = await databaseCheck(config, configPath); - results.push(dbResult); - printResult(dbResult); - await maybeRepair(dbResult, opts); + results.push( + await runRepairableCheck({ + run: () => databaseCheck(config, configPath), + configPath, + opts, + }), + ); // 7. LLM check const llmResult = await llmCheck(config); @@ -95,10 +107,13 @@ export async function doctor(opts: { printResult(llmResult); // 8. Log directory check - const logResult = logCheck(config, configPath); - results.push(logResult); - printResult(logResult); - await maybeRepair(logResult, opts); + results.push( + await runRepairableCheck({ + run: () => logCheck(config, configPath), + configPath, + opts, + }), + ); // 9. Port check const portResult = await portCheck(config); @@ -120,9 +135,9 @@ function printResult(result: CheckResult): void { async function maybeRepair( result: CheckResult, opts: { repair?: boolean; yes?: boolean }, -): Promise { - if (result.status === "pass" || !result.canRepair || !result.repair) return; - if (!opts.repair) return; +): Promise { + if (result.status === "pass" || !result.canRepair || !result.repair) return false; + if (!opts.repair) return false; let shouldRepair = opts.yes; if (!shouldRepair) { @@ -130,7 +145,7 @@ async function maybeRepair( message: `Repair "${result.name}"?`, initialValue: true, }); - if (p.isCancel(answer)) return; + if (p.isCancel(answer)) return false; shouldRepair = answer; } @@ -138,10 +153,30 @@ async function maybeRepair( try { await result.repair(); p.log.success(`Repaired: ${result.name}`); + return true; } catch (err) { p.log.error(`Repair failed: ${err instanceof Error ? err.message : String(err)}`); } } + return false; +} + +async function runRepairableCheck(input: { + run: () => CheckResult | Promise; + configPath: string; + opts: { repair?: boolean; yes?: boolean }; +}): Promise { + let result = await input.run(); + printResult(result); + + const repaired = await maybeRepair(result, input.opts); + if (!repaired) return result; + + // Repairs may create/update the adjacent .env file or other local resources. + loadPaperclipEnvFile(input.configPath); + result = await input.run(); + printResult(result); + return result; } function printSummary(results: CheckResult[]): { passed: number; warned: number; failed: number } { From 9c68c1b80b398b76c7924a092fa59a94bfe00139 Mon Sep 17 00:00:00 2001 From: David Ahmann Date: Tue, 10 Mar 2026 12:00:29 -0400 Subject: [PATCH 11/12] server: make approval retries idempotent (#499) --- .../approval-routes-idempotency.test.ts | 110 +++++++++++ .../src/__tests__/approvals-service.test.ts | 110 +++++++++++ server/src/routes/approvals.ts | 185 ++++++++++-------- server/src/services/approvals.ts | 106 ++++++---- 4 files changed, 384 insertions(+), 127 deletions(-) create mode 100644 server/src/__tests__/approval-routes-idempotency.test.ts create mode 100644 server/src/__tests__/approvals-service.test.ts diff --git a/server/src/__tests__/approval-routes-idempotency.test.ts b/server/src/__tests__/approval-routes-idempotency.test.ts new file mode 100644 index 00000000..51181ff5 --- /dev/null +++ b/server/src/__tests__/approval-routes-idempotency.test.ts @@ -0,0 +1,110 @@ +import express from "express"; +import request from "supertest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { approvalRoutes } from "../routes/approvals.js"; +import { errorHandler } from "../middleware/index.js"; + +const mockApprovalService = vi.hoisted(() => ({ + list: vi.fn(), + getById: vi.fn(), + create: vi.fn(), + approve: vi.fn(), + reject: vi.fn(), + requestRevision: vi.fn(), + resubmit: vi.fn(), + listComments: vi.fn(), + addComment: vi.fn(), +})); + +const mockHeartbeatService = vi.hoisted(() => ({ + wakeup: vi.fn(), +})); + +const mockIssueApprovalService = vi.hoisted(() => ({ + listIssuesForApproval: vi.fn(), + linkManyForApproval: vi.fn(), +})); + +const mockSecretService = vi.hoisted(() => ({ + normalizeHireApprovalPayloadForPersistence: vi.fn(), +})); + +const mockLogActivity = vi.hoisted(() => vi.fn()); + +vi.mock("../services/index.js", () => ({ + approvalService: () => mockApprovalService, + heartbeatService: () => mockHeartbeatService, + issueApprovalService: () => mockIssueApprovalService, + logActivity: mockLogActivity, + secretService: () => mockSecretService, +})); + +function createApp() { + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + (req as any).actor = { + type: "board", + userId: "user-1", + companyIds: ["company-1"], + source: "session", + isInstanceAdmin: false, + }; + next(); + }); + app.use("/api", approvalRoutes({} as any)); + app.use(errorHandler); + return app; +} + +describe("approval routes idempotent retries", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockHeartbeatService.wakeup.mockResolvedValue({ id: "wake-1" }); + mockIssueApprovalService.listIssuesForApproval.mockResolvedValue([{ id: "issue-1" }]); + mockLogActivity.mockResolvedValue(undefined); + }); + + it("does not emit duplicate approval side effects when approve is already resolved", async () => { + mockApprovalService.approve.mockResolvedValue({ + approval: { + id: "approval-1", + companyId: "company-1", + type: "hire_agent", + status: "approved", + payload: {}, + requestedByAgentId: "agent-1", + }, + applied: false, + }); + + const res = await request(createApp()) + .post("/api/approvals/approval-1/approve") + .send({}); + + expect(res.status).toBe(200); + expect(mockIssueApprovalService.listIssuesForApproval).not.toHaveBeenCalled(); + expect(mockHeartbeatService.wakeup).not.toHaveBeenCalled(); + expect(mockLogActivity).not.toHaveBeenCalled(); + }); + + it("does not emit duplicate rejection logs when reject is already resolved", async () => { + mockApprovalService.reject.mockResolvedValue({ + approval: { + id: "approval-1", + companyId: "company-1", + type: "hire_agent", + status: "rejected", + payload: {}, + }, + applied: false, + }); + + const res = await request(createApp()) + .post("/api/approvals/approval-1/reject") + .send({}); + + expect(res.status).toBe(200); + expect(mockLogActivity).not.toHaveBeenCalled(); + }); +}); diff --git a/server/src/__tests__/approvals-service.test.ts b/server/src/__tests__/approvals-service.test.ts new file mode 100644 index 00000000..967fd295 --- /dev/null +++ b/server/src/__tests__/approvals-service.test.ts @@ -0,0 +1,110 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { approvalService } from "../services/approvals.js"; + +const mockAgentService = vi.hoisted(() => ({ + activatePendingApproval: vi.fn(), + create: vi.fn(), + terminate: vi.fn(), +})); + +const mockNotifyHireApproved = vi.hoisted(() => vi.fn()); + +vi.mock("../services/agents.js", () => ({ + agentService: vi.fn(() => mockAgentService), +})); + +vi.mock("../services/hire-hook.js", () => ({ + notifyHireApproved: mockNotifyHireApproved, +})); + +type ApprovalRecord = { + id: string; + companyId: string; + type: string; + status: string; + payload: Record; + requestedByAgentId: string | null; +}; + +function createApproval(status: string): ApprovalRecord { + return { + id: "approval-1", + companyId: "company-1", + type: "hire_agent", + status, + payload: { agentId: "agent-1" }, + requestedByAgentId: "requester-1", + }; +} + +function createDbStub(selectResults: ApprovalRecord[][], updateResults: ApprovalRecord[]) { + const selectWhere = vi.fn(); + for (const result of selectResults) { + selectWhere.mockResolvedValueOnce(result); + } + + const from = vi.fn(() => ({ where: selectWhere })); + const select = vi.fn(() => ({ from })); + + const returning = vi.fn().mockResolvedValue(updateResults); + const updateWhere = vi.fn(() => ({ returning })); + const set = vi.fn(() => ({ where: updateWhere })); + const update = vi.fn(() => ({ set })); + + return { + db: { select, update }, + selectWhere, + returning, + }; +} + +describe("approvalService resolution idempotency", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockAgentService.activatePendingApproval.mockResolvedValue(undefined); + mockAgentService.create.mockResolvedValue({ id: "agent-1" }); + mockAgentService.terminate.mockResolvedValue(undefined); + mockNotifyHireApproved.mockResolvedValue(undefined); + }); + + it("treats repeated approve retries as no-ops after another worker resolves the approval", async () => { + const dbStub = createDbStub( + [[createApproval("pending")], [createApproval("approved")]], + [], + ); + + const svc = approvalService(dbStub.db as any); + const result = await svc.approve("approval-1", "board", "ship it"); + + expect(result.applied).toBe(false); + expect(result.approval.status).toBe("approved"); + expect(mockAgentService.activatePendingApproval).not.toHaveBeenCalled(); + expect(mockNotifyHireApproved).not.toHaveBeenCalled(); + }); + + it("treats repeated reject retries as no-ops after another worker resolves the approval", async () => { + const dbStub = createDbStub( + [[createApproval("pending")], [createApproval("rejected")]], + [], + ); + + const svc = approvalService(dbStub.db as any); + const result = await svc.reject("approval-1", "board", "not now"); + + expect(result.applied).toBe(false); + expect(result.approval.status).toBe("rejected"); + expect(mockAgentService.terminate).not.toHaveBeenCalled(); + }); + + it("still performs side effects when the resolution update is newly applied", async () => { + const approved = createApproval("approved"); + const dbStub = createDbStub([[createApproval("pending")]], [approved]); + + const svc = approvalService(dbStub.db as any); + const result = await svc.approve("approval-1", "board", "ship it"); + + expect(result.applied).toBe(true); + expect(mockAgentService.activatePendingApproval).toHaveBeenCalledWith("agent-1"); + expect(mockNotifyHireApproved).toHaveBeenCalledTimes(1); + }); +}); diff --git a/server/src/routes/approvals.ts b/server/src/routes/approvals.ts index f9903435..99d33abd 100644 --- a/server/src/routes/approvals.ts +++ b/server/src/routes/approvals.ts @@ -121,85 +121,92 @@ export function approvalRoutes(db: Db) { router.post("/approvals/:id/approve", validate(resolveApprovalSchema), async (req, res) => { assertBoard(req); const id = req.params.id as string; - const approval = await svc.approve(id, req.body.decidedByUserId ?? "board", req.body.decisionNote); - const linkedIssues = await issueApprovalsSvc.listIssuesForApproval(approval.id); - const linkedIssueIds = linkedIssues.map((issue) => issue.id); - const primaryIssueId = linkedIssueIds[0] ?? null; + const { approval, applied } = await svc.approve( + id, + req.body.decidedByUserId ?? "board", + req.body.decisionNote, + ); - await logActivity(db, { - companyId: approval.companyId, - actorType: "user", - actorId: req.actor.userId ?? "board", - action: "approval.approved", - entityType: "approval", - entityId: approval.id, - details: { - type: approval.type, - requestedByAgentId: approval.requestedByAgentId, - linkedIssueIds, - }, - }); + if (applied) { + const linkedIssues = await issueApprovalsSvc.listIssuesForApproval(approval.id); + const linkedIssueIds = linkedIssues.map((issue) => issue.id); + const primaryIssueId = linkedIssueIds[0] ?? null; - if (approval.requestedByAgentId) { - try { - const wakeRun = await heartbeat.wakeup(approval.requestedByAgentId, { - source: "automation", - triggerDetail: "system", - reason: "approval_approved", - payload: { - approvalId: approval.id, - approvalStatus: approval.status, - issueId: primaryIssueId, - issueIds: linkedIssueIds, - }, - requestedByActorType: "user", - requestedByActorId: req.actor.userId ?? "board", - contextSnapshot: { - source: "approval.approved", - approvalId: approval.id, - approvalStatus: approval.status, - issueId: primaryIssueId, - issueIds: linkedIssueIds, - taskId: primaryIssueId, - wakeReason: "approval_approved", - }, - }); + await logActivity(db, { + companyId: approval.companyId, + actorType: "user", + actorId: req.actor.userId ?? "board", + action: "approval.approved", + entityType: "approval", + entityId: approval.id, + details: { + type: approval.type, + requestedByAgentId: approval.requestedByAgentId, + linkedIssueIds, + }, + }); - await logActivity(db, { - companyId: approval.companyId, - actorType: "user", - actorId: req.actor.userId ?? "board", - action: "approval.requester_wakeup_queued", - entityType: "approval", - entityId: approval.id, - details: { - requesterAgentId: approval.requestedByAgentId, - wakeRunId: wakeRun?.id ?? null, - linkedIssueIds, - }, - }); - } catch (err) { - logger.warn( - { - err, - approvalId: approval.id, - requestedByAgentId: approval.requestedByAgentId, - }, - "failed to queue requester wakeup after approval", - ); - await logActivity(db, { - companyId: approval.companyId, - actorType: "user", - actorId: req.actor.userId ?? "board", - action: "approval.requester_wakeup_failed", - entityType: "approval", - entityId: approval.id, - details: { - requesterAgentId: approval.requestedByAgentId, - linkedIssueIds, - error: err instanceof Error ? err.message : String(err), - }, - }); + if (approval.requestedByAgentId) { + try { + const wakeRun = await heartbeat.wakeup(approval.requestedByAgentId, { + source: "automation", + triggerDetail: "system", + reason: "approval_approved", + payload: { + approvalId: approval.id, + approvalStatus: approval.status, + issueId: primaryIssueId, + issueIds: linkedIssueIds, + }, + requestedByActorType: "user", + requestedByActorId: req.actor.userId ?? "board", + contextSnapshot: { + source: "approval.approved", + approvalId: approval.id, + approvalStatus: approval.status, + issueId: primaryIssueId, + issueIds: linkedIssueIds, + taskId: primaryIssueId, + wakeReason: "approval_approved", + }, + }); + + await logActivity(db, { + companyId: approval.companyId, + actorType: "user", + actorId: req.actor.userId ?? "board", + action: "approval.requester_wakeup_queued", + entityType: "approval", + entityId: approval.id, + details: { + requesterAgentId: approval.requestedByAgentId, + wakeRunId: wakeRun?.id ?? null, + linkedIssueIds, + }, + }); + } catch (err) { + logger.warn( + { + err, + approvalId: approval.id, + requestedByAgentId: approval.requestedByAgentId, + }, + "failed to queue requester wakeup after approval", + ); + await logActivity(db, { + companyId: approval.companyId, + actorType: "user", + actorId: req.actor.userId ?? "board", + action: "approval.requester_wakeup_failed", + entityType: "approval", + entityId: approval.id, + details: { + requesterAgentId: approval.requestedByAgentId, + linkedIssueIds, + error: err instanceof Error ? err.message : String(err), + }, + }); + } } } @@ -209,17 +216,23 @@ export function approvalRoutes(db: Db) { router.post("/approvals/:id/reject", validate(resolveApprovalSchema), async (req, res) => { assertBoard(req); const id = req.params.id as string; - const approval = await svc.reject(id, req.body.decidedByUserId ?? "board", req.body.decisionNote); + const { approval, applied } = await svc.reject( + id, + req.body.decidedByUserId ?? "board", + req.body.decisionNote, + ); - await logActivity(db, { - companyId: approval.companyId, - actorType: "user", - actorId: req.actor.userId ?? "board", - action: "approval.rejected", - entityType: "approval", - entityId: approval.id, - details: { type: approval.type }, - }); + if (applied) { + await logActivity(db, { + companyId: approval.companyId, + actorType: "user", + actorId: req.actor.userId ?? "board", + action: "approval.rejected", + entityType: "approval", + entityId: approval.id, + details: { type: approval.type }, + }); + } res.json(redactApprovalPayload(approval)); }); diff --git a/server/src/services/approvals.ts b/server/src/services/approvals.ts index ba2890a3..13e57c8a 100644 --- a/server/src/services/approvals.ts +++ b/server/src/services/approvals.ts @@ -1,4 +1,4 @@ -import { and, asc, eq } from "drizzle-orm"; +import { and, asc, eq, inArray } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { approvalComments, approvals } from "@paperclipai/db"; import { notFound, unprocessable } from "../errors.js"; @@ -8,6 +8,9 @@ import { notifyHireApproved } from "./hire-hook.js"; export function approvalService(db: Db) { const agentsSvc = agentService(db); const canResolveStatuses = new Set(["pending", "revision_requested"]); + const resolvableStatuses = Array.from(canResolveStatuses); + type ApprovalRecord = typeof approvals.$inferSelect; + type ResolutionResult = { approval: ApprovalRecord; applied: boolean }; async function getExistingApproval(id: string) { const existing = await db @@ -19,6 +22,50 @@ export function approvalService(db: Db) { return existing; } + async function resolveApproval( + id: string, + targetStatus: "approved" | "rejected", + decidedByUserId: string, + decisionNote: string | null | undefined, + ): Promise { + const existing = await getExistingApproval(id); + if (!canResolveStatuses.has(existing.status)) { + if (existing.status === targetStatus) { + return { approval: existing, applied: false }; + } + throw unprocessable( + `Only pending or revision requested approvals can be ${targetStatus === "approved" ? "approved" : "rejected"}`, + ); + } + + const now = new Date(); + const updated = await db + .update(approvals) + .set({ + status: targetStatus, + decidedByUserId, + decisionNote: decisionNote ?? null, + decidedAt: now, + updatedAt: now, + }) + .where(and(eq(approvals.id, id), inArray(approvals.status, resolvableStatuses))) + .returning() + .then((rows) => rows[0] ?? null); + + if (updated) { + return { approval: updated, applied: true }; + } + + const latest = await getExistingApproval(id); + if (latest.status === targetStatus) { + return { approval: latest, applied: false }; + } + + throw unprocessable( + `Only pending or revision requested approvals can be ${targetStatus === "approved" ? "approved" : "rejected"}`, + ); + } + return { list: (companyId: string, status?: string) => { const conditions = [eq(approvals.companyId, companyId)]; @@ -41,27 +88,16 @@ export function approvalService(db: Db) { .then((rows) => rows[0]), approve: async (id: string, decidedByUserId: string, decisionNote?: string | null) => { - const existing = await getExistingApproval(id); - if (!canResolveStatuses.has(existing.status)) { - throw unprocessable("Only pending or revision requested approvals can be approved"); - } - - const now = new Date(); - const updated = await db - .update(approvals) - .set({ - status: "approved", - decidedByUserId, - decisionNote: decisionNote ?? null, - decidedAt: now, - updatedAt: now, - }) - .where(eq(approvals.id, id)) - .returning() - .then((rows) => rows[0]); + const { approval: updated, applied } = await resolveApproval( + id, + "approved", + decidedByUserId, + decisionNote, + ); let hireApprovedAgentId: string | null = null; - if (updated.type === "hire_agent") { + const now = new Date(); + if (applied && updated.type === "hire_agent") { const payload = updated.payload as Record; const payloadAgentId = typeof payload.agentId === "string" ? payload.agentId : null; if (payloadAgentId) { @@ -103,30 +139,18 @@ export function approvalService(db: Db) { } } - return updated; + return { approval: updated, applied }; }, reject: async (id: string, decidedByUserId: string, decisionNote?: string | null) => { - const existing = await getExistingApproval(id); - if (!canResolveStatuses.has(existing.status)) { - throw unprocessable("Only pending or revision requested approvals can be rejected"); - } + const { approval: updated, applied } = await resolveApproval( + id, + "rejected", + decidedByUserId, + decisionNote, + ); - const now = new Date(); - const updated = await db - .update(approvals) - .set({ - status: "rejected", - decidedByUserId, - decisionNote: decisionNote ?? null, - decidedAt: now, - updatedAt: now, - }) - .where(eq(approvals.id, id)) - .returning() - .then((rows) => rows[0]); - - if (updated.type === "hire_agent") { + if (applied && updated.type === "hire_agent") { const payload = updated.payload as Record; const payloadAgentId = typeof payload.agentId === "string" ? payload.agentId : null; if (payloadAgentId) { @@ -134,7 +158,7 @@ export function approvalService(db: Db) { } } - return updated; + return { approval: updated, applied }; }, requestRevision: async (id: string, decidedByUserId: string, decisionNote?: string | null) => { From deec68ab163a9726bc99307654d65fdcb17ad9c2 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 10 Mar 2026 12:57:53 -0500 Subject: [PATCH 12/12] Copy seeded secrets key into worktree instances --- cli/src/__tests__/worktree.test.ts | 49 +++++++++++++++++++++++++ cli/src/commands/worktree.ts | 57 +++++++++++++++++++++++++++++- 2 files changed, 105 insertions(+), 1 deletion(-) diff --git a/cli/src/__tests__/worktree.test.ts b/cli/src/__tests__/worktree.test.ts index 18bdabf1..9ce48cf7 100644 --- a/cli/src/__tests__/worktree.test.ts +++ b/cli/src/__tests__/worktree.test.ts @@ -1,5 +1,8 @@ +import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; +import { copySeededSecretsKey } from "../commands/worktree.js"; import { buildWorktreeConfig, buildWorktreeEnvEntries, @@ -122,4 +125,50 @@ describe("worktree helpers", () => { expect(full.excludedTables).toEqual([]); expect(full.nullifyColumns).toEqual({}); }); + + it("copies the source local_encrypted secrets key into the seeded worktree instance", () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-worktree-secrets-")); + try { + const sourceConfigPath = path.join(tempRoot, "source", "config.json"); + const sourceKeyPath = path.join(tempRoot, "source", "secrets", "master.key"); + const targetKeyPath = path.join(tempRoot, "target", "secrets", "master.key"); + fs.mkdirSync(path.dirname(sourceKeyPath), { recursive: true }); + fs.writeFileSync(sourceKeyPath, "source-master-key", "utf8"); + + const sourceConfig = buildSourceConfig(); + sourceConfig.secrets.localEncrypted.keyFilePath = sourceKeyPath; + + copySeededSecretsKey({ + sourceConfigPath, + sourceConfig, + sourceEnvEntries: {}, + targetKeyFilePath: targetKeyPath, + }); + + expect(fs.readFileSync(targetKeyPath, "utf8")).toBe("source-master-key"); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } + }); + + it("writes the source inline secrets master key into the seeded worktree instance", () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-worktree-secrets-")); + try { + const sourceConfigPath = path.join(tempRoot, "source", "config.json"); + const targetKeyPath = path.join(tempRoot, "target", "secrets", "master.key"); + + copySeededSecretsKey({ + sourceConfigPath, + sourceConfig: buildSourceConfig(), + sourceEnvEntries: { + PAPERCLIP_SECRETS_MASTER_KEY: "inline-source-master-key", + }, + targetKeyFilePath: targetKeyPath, + }); + + expect(fs.readFileSync(targetKeyPath, "utf8")).toBe("inline-source-master-key"); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } + }); }); diff --git a/cli/src/commands/worktree.ts b/cli/src/commands/worktree.ts index f9363ff8..8306c8ed 100644 --- a/cli/src/commands/worktree.ts +++ b/cli/src/commands/worktree.ts @@ -1,4 +1,4 @@ -import { existsSync, readFileSync, rmSync } from "node:fs"; +import { chmodSync, copyFileSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import os from "node:os"; import path from "node:path"; import { execFileSync } from "node:child_process"; @@ -18,6 +18,7 @@ import { expandHomePrefix } from "../config/home.js"; import type { PaperclipConfig } from "../config/schema.js"; import { readConfig, resolveConfigPath, writeConfig } from "../config/store.js"; import { printPaperclipCliBanner } from "../utils/banner.js"; +import { resolveRuntimeLikePath } from "../utils/path-resolver.js"; import { buildWorktreeConfig, buildWorktreeEnvEntries, @@ -154,6 +155,54 @@ function resolveSourceConnectionString(config: PaperclipConfig, envEntries: Reco return `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`; } +export function copySeededSecretsKey(input: { + sourceConfigPath: string; + sourceConfig: PaperclipConfig; + sourceEnvEntries: Record; + targetKeyFilePath: string; +}): void { + if (input.sourceConfig.secrets.provider !== "local_encrypted") { + return; + } + + mkdirSync(path.dirname(input.targetKeyFilePath), { recursive: true }); + + const sourceInlineMasterKey = + nonEmpty(input.sourceEnvEntries.PAPERCLIP_SECRETS_MASTER_KEY) ?? + nonEmpty(process.env.PAPERCLIP_SECRETS_MASTER_KEY); + if (sourceInlineMasterKey) { + writeFileSync(input.targetKeyFilePath, sourceInlineMasterKey, { + encoding: "utf8", + mode: 0o600, + }); + try { + chmodSync(input.targetKeyFilePath, 0o600); + } catch { + // best effort + } + return; + } + + const sourceKeyFileOverride = + nonEmpty(input.sourceEnvEntries.PAPERCLIP_SECRETS_MASTER_KEY_FILE) ?? + nonEmpty(process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE); + const sourceConfiguredKeyPath = sourceKeyFileOverride ?? input.sourceConfig.secrets.localEncrypted.keyFilePath; + const sourceKeyFilePath = resolveRuntimeLikePath(sourceConfiguredKeyPath, input.sourceConfigPath); + + if (!existsSync(sourceKeyFilePath)) { + throw new Error( + `Cannot seed worktree database because source local_encrypted secrets key was not found at ${sourceKeyFilePath}.`, + ); + } + + copyFileSync(sourceKeyFilePath, input.targetKeyFilePath); + try { + chmodSync(input.targetKeyFilePath, 0o600); + } catch { + // best effort + } +} + async function ensureEmbeddedPostgres(dataDir: string, preferredPort: number): Promise { const moduleName = "embedded-postgres"; let EmbeddedPostgres: EmbeddedPostgresCtor; @@ -215,6 +264,12 @@ async function seedWorktreeDatabase(input: { const seedPlan = resolveWorktreeSeedPlan(input.seedMode); const sourceEnvFile = resolvePaperclipEnvFile(input.sourceConfigPath); const sourceEnvEntries = readPaperclipEnvEntries(sourceEnvFile); + copySeededSecretsKey({ + sourceConfigPath: input.sourceConfigPath, + sourceConfig: input.sourceConfig, + sourceEnvEntries, + targetKeyFilePath: input.targetPaths.secretsKeyFilePath, + }); let sourceHandle: EmbeddedPostgresHandle | null = null; let targetHandle: EmbeddedPostgresHandle | null = null;