diff --git a/doc/DATABASE.md b/doc/DATABASE.md index 8f46f382..d2425d14 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -19,6 +19,14 @@ That's it. On first start the server: Data persists across restarts in `~/.paperclip/instances/default/db/`. To reset local dev data, delete that directory. +If you need to apply pending migrations manually, run: + +```sh +pnpm db:migrate +``` + +When `DATABASE_URL` is unset, this command targets the current embedded PostgreSQL instance for your active Paperclip config/instance. + This mode is ideal for local development and one-command installs. Docker note: the Docker quickstart image also uses embedded PostgreSQL by default. Persist `/paperclip` to keep DB state across container restarts (see `doc/DOCKER.md`). diff --git a/package.json b/package.json index b2e23d55..61f9968e 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "type": "module", "scripts": { "dev": "node scripts/dev-runner.mjs watch", - "dev:watch": "cross-env PAPERCLIP_MIGRATION_PROMPT=never node scripts/dev-runner.mjs watch", + "dev:watch": "node scripts/dev-runner.mjs watch", "dev:once": "node scripts/dev-runner.mjs dev", "dev:server": "pnpm --filter @paperclipai/server dev", "dev:ui": "pnpm --filter @paperclipai/ui dev", diff --git a/packages/db/src/migrate.ts b/packages/db/src/migrate.ts index b4c7b975..f51b629e 100644 --- a/packages/db/src/migrate.ts +++ b/packages/db/src/migrate.ts @@ -1,21 +1,29 @@ import { applyPendingMigrations, inspectMigrations } from "./client.js"; +import { resolveMigrationConnection } from "./migration-runtime.js"; -const url = process.env.DATABASE_URL; +async function main(): Promise { + const resolved = await resolveMigrationConnection(); -if (!url) { - throw new Error("DATABASE_URL is required for db:migrate"); -} + console.log(`Migrating database via ${resolved.source}`); -const before = await inspectMigrations(url); -if (before.status === "upToDate") { - console.log("No pending migrations"); -} else { - console.log(`Applying ${before.pendingMigrations.length} pending migration(s)...`); - await applyPendingMigrations(url); + try { + const before = await inspectMigrations(resolved.connectionString); + if (before.status === "upToDate") { + console.log("No pending migrations"); + return; + } - const after = await inspectMigrations(url); - if (after.status !== "upToDate") { - throw new Error(`Migrations incomplete: ${after.pendingMigrations.join(", ")}`); + console.log(`Applying ${before.pendingMigrations.length} pending migration(s)...`); + await applyPendingMigrations(resolved.connectionString); + + const after = await inspectMigrations(resolved.connectionString); + if (after.status !== "upToDate") { + throw new Error(`Migrations incomplete: ${after.pendingMigrations.join(", ")}`); + } + console.log("Migrations complete"); + } finally { + await resolved.stop(); } - console.log("Migrations complete"); } + +await main(); diff --git a/packages/db/src/migration-runtime.ts b/packages/db/src/migration-runtime.ts new file mode 100644 index 00000000..bc90b762 --- /dev/null +++ b/packages/db/src/migration-runtime.ts @@ -0,0 +1,134 @@ +import { existsSync, readFileSync, rmSync } from "node:fs"; +import { createRequire } from "node:module"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { ensurePostgresDatabase } from "./client.js"; +import { resolveDatabaseTarget } from "./runtime-config.js"; + +type EmbeddedPostgresInstance = { + initialise(): Promise; + start(): Promise; + stop(): Promise; +}; + +type EmbeddedPostgresCtor = new (opts: { + databaseDir: string; + user: string; + password: string; + port: number; + persistent: boolean; + onLog?: (message: unknown) => void; + onError?: (message: unknown) => void; +}) => EmbeddedPostgresInstance; + +export type MigrationConnection = { + connectionString: string; + source: string; + stop: () => Promise; +}; + +function readRunningPostmasterPid(postmasterPidFile: string): number | null { + if (!existsSync(postmasterPidFile)) return null; + try { + const pid = Number(readFileSync(postmasterPidFile, "utf8").split("\n")[0]?.trim()); + if (!Number.isInteger(pid) || pid <= 0) return null; + process.kill(pid, 0); + return pid; + } catch { + return null; + } +} + +function readPidFilePort(postmasterPidFile: string): number | null { + if (!existsSync(postmasterPidFile)) return null; + try { + const lines = readFileSync(postmasterPidFile, "utf8").split("\n"); + const port = Number(lines[3]?.trim()); + return Number.isInteger(port) && port > 0 ? port : null; + } catch { + return null; + } +} + +async function loadEmbeddedPostgresCtor(): Promise { + const require = createRequire(import.meta.url); + const resolveCandidates = [ + path.resolve(fileURLToPath(new URL("../..", import.meta.url))), + path.resolve(fileURLToPath(new URL("../../server", import.meta.url))), + path.resolve(fileURLToPath(new URL("../../cli", import.meta.url))), + process.cwd(), + ]; + + try { + const resolvedModulePath = require.resolve("embedded-postgres", { paths: resolveCandidates }); + const mod = await import(pathToFileURL(resolvedModulePath).href); + return mod.default as EmbeddedPostgresCtor; + } catch { + throw new Error( + "Embedded PostgreSQL support requires dependency `embedded-postgres`. Reinstall dependencies and try again.", + ); + } +} + +async function ensureEmbeddedPostgresConnection( + dataDir: string, + preferredPort: number, +): Promise { + const EmbeddedPostgres = await loadEmbeddedPostgresCtor(); + const postmasterPidFile = path.resolve(dataDir, "postmaster.pid"); + const runningPid = readRunningPostmasterPid(postmasterPidFile); + const runningPort = readPidFilePort(postmasterPidFile); + + if (runningPid) { + const port = runningPort ?? preferredPort; + const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/postgres`; + await ensurePostgresDatabase(adminConnectionString, "paperclip"); + return { + connectionString: `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`, + source: `embedded-postgres@${port}`, + stop: async () => {}, + }; + } + + const instance = new EmbeddedPostgres({ + databaseDir: dataDir, + user: "paperclip", + password: "paperclip", + port: preferredPort, + persistent: true, + onLog: () => {}, + onError: () => {}, + }); + + if (!existsSync(path.resolve(dataDir, "PG_VERSION"))) { + await instance.initialise(); + } + if (existsSync(postmasterPidFile)) { + rmSync(postmasterPidFile, { force: true }); + } + await instance.start(); + + const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${preferredPort}/postgres`; + await ensurePostgresDatabase(adminConnectionString, "paperclip"); + + return { + connectionString: `postgres://paperclip:paperclip@127.0.0.1:${preferredPort}/paperclip`, + source: `embedded-postgres@${preferredPort}`, + stop: async () => { + await instance.stop(); + }, + }; +} + +export async function resolveMigrationConnection(): Promise { + const target = resolveDatabaseTarget(); + if (target.mode === "postgres") { + return { + connectionString: target.connectionString, + source: target.source, + stop: async () => {}, + }; + } + + return ensureEmbeddedPostgresConnection(target.dataDir, target.port); +} diff --git a/packages/db/src/migration-status.ts b/packages/db/src/migration-status.ts new file mode 100644 index 00000000..3d0cc8f4 --- /dev/null +++ b/packages/db/src/migration-status.ts @@ -0,0 +1,45 @@ +import { inspectMigrations } from "./client.js"; +import { resolveMigrationConnection } from "./migration-runtime.js"; + +const jsonMode = process.argv.includes("--json"); + +async function main(): Promise { + const connection = await resolveMigrationConnection(); + + try { + const state = await inspectMigrations(connection.connectionString); + const payload = + state.status === "upToDate" + ? { + source: connection.source, + status: "upToDate" as const, + tableCount: state.tableCount, + pendingMigrations: [] as string[], + } + : { + source: connection.source, + status: "needsMigrations" as const, + tableCount: state.tableCount, + pendingMigrations: state.pendingMigrations, + reason: state.reason, + }; + + if (jsonMode) { + console.log(JSON.stringify(payload)); + return; + } + + if (payload.status === "upToDate") { + console.log(`Database is up to date via ${payload.source}`); + return; + } + + console.log( + `Pending migrations via ${payload.source}: ${payload.pendingMigrations.join(", ")}`, + ); + } finally { + await connection.stop(); + } +} + +await main(); diff --git a/packages/db/src/runtime-config.test.ts b/packages/db/src/runtime-config.test.ts new file mode 100644 index 00000000..55371e09 --- /dev/null +++ b/packages/db/src/runtime-config.test.ts @@ -0,0 +1,107 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { resolveDatabaseTarget } from "./runtime-config.js"; + +const ORIGINAL_CWD = process.cwd(); +const ORIGINAL_ENV = { ...process.env }; + +function writeJson(filePath: string, value: unknown) { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, JSON.stringify(value, null, 2)); +} + +function writeText(filePath: string, value: string) { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, value); +} + +afterEach(() => { + process.chdir(ORIGINAL_CWD); + for (const key of Object.keys(process.env)) { + if (!(key in ORIGINAL_ENV)) delete process.env[key]; + } + for (const [key, value] of Object.entries(ORIGINAL_ENV)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } +}); + +describe("resolveDatabaseTarget", () => { + it("uses DATABASE_URL from process env first", () => { + process.env.DATABASE_URL = "postgres://env-user:env-pass@db.example.com:5432/paperclip"; + + const target = resolveDatabaseTarget(); + + expect(target).toMatchObject({ + mode: "postgres", + connectionString: "postgres://env-user:env-pass@db.example.com:5432/paperclip", + source: "DATABASE_URL", + }); + }); + + it("uses DATABASE_URL from repo-local .paperclip/.env", () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-db-runtime-")); + const projectDir = path.join(tempDir, "repo"); + fs.mkdirSync(projectDir, { recursive: true }); + process.chdir(projectDir); + writeJson(path.join(projectDir, ".paperclip", "config.json"), { + database: { mode: "embedded-postgres", embeddedPostgresPort: 54329 }, + }); + writeText( + path.join(projectDir, ".paperclip", ".env"), + 'DATABASE_URL="postgres://file-user:file-pass@db.example.com:6543/paperclip"\n', + ); + + const target = resolveDatabaseTarget(); + + expect(target).toMatchObject({ + mode: "postgres", + connectionString: "postgres://file-user:file-pass@db.example.com:6543/paperclip", + source: "paperclip-env", + }); + }); + + it("uses config postgres connection string when configured", () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-db-runtime-")); + const configPath = path.join(tempDir, "instance", "config.json"); + process.env.PAPERCLIP_CONFIG = configPath; + writeJson(configPath, { + database: { + mode: "postgres", + connectionString: "postgres://cfg-user:cfg-pass@db.example.com:5432/paperclip", + }, + }); + + const target = resolveDatabaseTarget(); + + expect(target).toMatchObject({ + mode: "postgres", + connectionString: "postgres://cfg-user:cfg-pass@db.example.com:5432/paperclip", + source: "config.database.connectionString", + }); + }); + + it("falls back to embedded postgres settings from config", () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-db-runtime-")); + const configPath = path.join(tempDir, "instance", "config.json"); + process.env.PAPERCLIP_CONFIG = configPath; + writeJson(configPath, { + database: { + mode: "embedded-postgres", + embeddedPostgresDataDir: "~/paperclip-test-db", + embeddedPostgresPort: 55444, + }, + }); + + const target = resolveDatabaseTarget(); + + expect(target).toMatchObject({ + mode: "embedded-postgres", + dataDir: path.resolve(os.homedir(), "paperclip-test-db"), + port: 55444, + source: "embedded-postgres@55444", + }); + }); +}); diff --git a/packages/db/src/runtime-config.ts b/packages/db/src/runtime-config.ts new file mode 100644 index 00000000..c6c64a38 --- /dev/null +++ b/packages/db/src/runtime-config.ts @@ -0,0 +1,267 @@ +import { existsSync, readFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const DEFAULT_INSTANCE_ID = "default"; +const CONFIG_BASENAME = "config.json"; +const ENV_BASENAME = ".env"; +const INSTANCE_ID_RE = /^[a-zA-Z0-9_-]+$/; + +type PartialConfig = { + database?: { + mode?: "embedded-postgres" | "postgres"; + connectionString?: string; + embeddedPostgresDataDir?: string; + embeddedPostgresPort?: number; + pgliteDataDir?: string; + pglitePort?: number; + }; +}; + +export type ResolvedDatabaseTarget = + | { + mode: "postgres"; + connectionString: string; + source: "DATABASE_URL" | "paperclip-env" | "config.database.connectionString"; + configPath: string; + envPath: string; + } + | { + mode: "embedded-postgres"; + dataDir: string; + port: number; + source: `embedded-postgres@${number}`; + configPath: string; + envPath: string; + }; + +function expandHomePrefix(value: string): string { + if (value === "~") return os.homedir(); + if (value.startsWith("~/")) return path.resolve(os.homedir(), value.slice(2)); + return value; +} + +function resolvePaperclipHomeDir(): string { + const envHome = process.env.PAPERCLIP_HOME?.trim(); + if (envHome) return path.resolve(expandHomePrefix(envHome)); + return path.resolve(os.homedir(), ".paperclip"); +} + +function resolvePaperclipInstanceId(): string { + const raw = process.env.PAPERCLIP_INSTANCE_ID?.trim() || DEFAULT_INSTANCE_ID; + if (!INSTANCE_ID_RE.test(raw)) { + throw new Error(`Invalid PAPERCLIP_INSTANCE_ID '${raw}'.`); + } + return raw; +} + +function resolveDefaultConfigPath(): string { + return path.resolve( + resolvePaperclipHomeDir(), + "instances", + resolvePaperclipInstanceId(), + CONFIG_BASENAME, + ); +} + +function resolveDefaultEmbeddedPostgresDir(): string { + return path.resolve(resolvePaperclipHomeDir(), "instances", resolvePaperclipInstanceId(), "db"); +} + +function resolveHomeAwarePath(value: string): string { + return path.resolve(expandHomePrefix(value)); +} + +function findConfigFileFromAncestors(startDir: string): string | null { + let currentDir = path.resolve(startDir); + + while (true) { + const candidate = path.resolve(currentDir, ".paperclip", CONFIG_BASENAME); + if (existsSync(candidate)) return candidate; + + const nextDir = path.resolve(currentDir, ".."); + if (nextDir === currentDir) return null; + currentDir = nextDir; + } +} + +function resolvePaperclipConfigPath(): string { + if (process.env.PAPERCLIP_CONFIG?.trim()) { + return path.resolve(process.env.PAPERCLIP_CONFIG.trim()); + } + return findConfigFileFromAncestors(process.cwd()) ?? resolveDefaultConfigPath(); +} + +function resolvePaperclipEnvPath(configPath: string): string { + return path.resolve(path.dirname(configPath), ENV_BASENAME); +} + +function parseEnvFile(contents: string): Record { + const entries: Record = {}; + + for (const rawLine of contents.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line || line.startsWith("#")) continue; + + const match = rawLine.match(/^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)\s*$/); + if (!match) continue; + + const [, key, rawValue] = match; + const value = rawValue.trim(); + if (!value) { + entries[key] = ""; + continue; + } + + if ( + (value.startsWith("\"") && value.endsWith("\"")) || + (value.startsWith("'") && value.endsWith("'")) + ) { + entries[key] = value.slice(1, -1); + continue; + } + + entries[key] = value.replace(/\s+#.*$/, "").trim(); + } + + return entries; +} + +function readEnvEntries(envPath: string): Record { + if (!existsSync(envPath)) return {}; + return parseEnvFile(readFileSync(envPath, "utf8")); +} + +function migrateLegacyConfig(raw: unknown): PartialConfig | null { + if (typeof raw !== "object" || raw === null || Array.isArray(raw)) return null; + + const config = { ...(raw as Record) }; + const databaseRaw = config.database; + if (typeof databaseRaw !== "object" || databaseRaw === null || Array.isArray(databaseRaw)) { + return config; + } + + const database = { ...(databaseRaw as Record) }; + if (database.mode === "pglite") { + database.mode = "embedded-postgres"; + + if ( + typeof database.embeddedPostgresDataDir !== "string" && + typeof database.pgliteDataDir === "string" + ) { + database.embeddedPostgresDataDir = database.pgliteDataDir; + } + if ( + typeof database.embeddedPostgresPort !== "number" && + typeof database.pglitePort === "number" && + Number.isFinite(database.pglitePort) + ) { + database.embeddedPostgresPort = database.pglitePort; + } + } + + config.database = database; + return config as PartialConfig; +} + +function asPositiveInt(value: unknown): number | null { + if (typeof value !== "number" || !Number.isFinite(value)) return null; + const rounded = Math.trunc(value); + return rounded > 0 ? rounded : null; +} + +function readConfig(configPath: string): PartialConfig | null { + if (!existsSync(configPath)) return null; + + let parsed: unknown; + try { + parsed = JSON.parse(readFileSync(configPath, "utf8")); + } catch (err) { + throw new Error( + `Failed to parse config at ${configPath}: ${err instanceof Error ? err.message : String(err)}`, + ); + } + + const migrated = migrateLegacyConfig(parsed); + if (migrated === null || typeof migrated !== "object" || Array.isArray(migrated)) { + throw new Error(`Invalid config at ${configPath}: expected a JSON object`); + } + + const database = + typeof migrated.database === "object" && + migrated.database !== null && + !Array.isArray(migrated.database) + ? migrated.database + : undefined; + + return { + database: database + ? { + mode: database.mode === "postgres" ? "postgres" : "embedded-postgres", + connectionString: + typeof database.connectionString === "string" ? database.connectionString : undefined, + embeddedPostgresDataDir: + typeof database.embeddedPostgresDataDir === "string" + ? database.embeddedPostgresDataDir + : undefined, + embeddedPostgresPort: asPositiveInt(database.embeddedPostgresPort) ?? undefined, + pgliteDataDir: typeof database.pgliteDataDir === "string" ? database.pgliteDataDir : undefined, + pglitePort: asPositiveInt(database.pglitePort) ?? undefined, + } + : undefined, + }; +} + +export function resolveDatabaseTarget(): ResolvedDatabaseTarget { + const configPath = resolvePaperclipConfigPath(); + const envPath = resolvePaperclipEnvPath(configPath); + const envEntries = readEnvEntries(envPath); + + const envUrl = process.env.DATABASE_URL?.trim(); + if (envUrl) { + return { + mode: "postgres", + connectionString: envUrl, + source: "DATABASE_URL", + configPath, + envPath, + }; + } + + const fileEnvUrl = envEntries.DATABASE_URL?.trim(); + if (fileEnvUrl) { + return { + mode: "postgres", + connectionString: fileEnvUrl, + source: "paperclip-env", + configPath, + envPath, + }; + } + + const config = readConfig(configPath); + const connectionString = config?.database?.connectionString?.trim(); + if (config?.database?.mode === "postgres" && connectionString) { + return { + mode: "postgres", + connectionString, + source: "config.database.connectionString", + configPath, + envPath, + }; + } + + const port = config?.database?.embeddedPostgresPort ?? 54329; + const dataDir = resolveHomeAwarePath( + config?.database?.embeddedPostgresDataDir ?? resolveDefaultEmbeddedPostgresDir(), + ); + + return { + mode: "embedded-postgres", + dataDir, + port, + source: `embedded-postgres@${port}`, + configPath, + envPath, + }; +} diff --git a/scripts/dev-runner.mjs b/scripts/dev-runner.mjs index 43262fd6..d4e4c231 100644 --- a/scripts/dev-runner.mjs +++ b/scripts/dev-runner.mjs @@ -1,5 +1,7 @@ #!/usr/bin/env node import { spawn } from "node:child_process"; +import { createInterface } from "node:readline/promises"; +import { stdin, stdout } from "node:process"; const mode = process.argv[2] === "watch" ? "watch" : "dev"; const cliArgs = process.argv.slice(3); @@ -43,6 +45,121 @@ if (tailscaleAuth) { } const pnpmBin = process.platform === "win32" ? "pnpm.cmd" : "pnpm"; + +function formatPendingMigrationSummary(migrations) { + if (migrations.length === 0) return "none"; + return migrations.length > 3 + ? `${migrations.slice(0, 3).join(", ")} (+${migrations.length - 3} more)` + : migrations.join(", "); +} + +async function runPnpm(args, options = {}) { + return await new Promise((resolve, reject) => { + const child = spawn(pnpmBin, args, { + stdio: options.stdio ?? ["ignore", "pipe", "pipe"], + env: options.env ?? process.env, + shell: process.platform === "win32", + }); + + let stdoutBuffer = ""; + let stderrBuffer = ""; + + if (child.stdout) { + child.stdout.on("data", (chunk) => { + stdoutBuffer += String(chunk); + }); + } + if (child.stderr) { + child.stderr.on("data", (chunk) => { + stderrBuffer += String(chunk); + }); + } + + child.on("error", reject); + child.on("exit", (code, signal) => { + resolve({ + code: code ?? 0, + signal, + stdout: stdoutBuffer, + stderr: stderrBuffer, + }); + }); + }); +} + +async function maybePreflightMigrations() { + if (mode !== "watch") return; + if (process.env.PAPERCLIP_MIGRATION_PROMPT === "never") return; + + const status = await runPnpm( + ["--filter", "@paperclipai/db", "exec", "tsx", "src/migration-status.ts", "--json"], + { env }, + ); + if (status.code !== 0) { + process.stderr.write(status.stderr || status.stdout); + process.exit(status.code); + } + + let payload; + try { + payload = JSON.parse(status.stdout.trim()); + } catch (error) { + process.stderr.write(status.stderr || status.stdout); + throw error; + } + + if (payload.status !== "needsMigrations" || payload.pendingMigrations.length === 0) { + return; + } + + const autoApply = process.env.PAPERCLIP_MIGRATION_AUTO_APPLY === "true"; + let shouldApply = autoApply; + + if (!autoApply) { + if (!stdin.isTTY || !stdout.isTTY) { + shouldApply = true; + } else { + const prompt = createInterface({ input: stdin, output: stdout }); + try { + const answer = ( + await prompt.question( + `Apply pending migrations (${formatPendingMigrationSummary(payload.pendingMigrations)}) now? (y/N): `, + ) + ) + .trim() + .toLowerCase(); + shouldApply = answer === "y" || answer === "yes"; + } finally { + prompt.close(); + } + } + } + + if (!shouldApply) return; + + const migrate = spawn(pnpmBin, ["db:migrate"], { + stdio: "inherit", + env, + shell: process.platform === "win32", + }); + const exit = await new Promise((resolve) => { + migrate.on("exit", (code, signal) => resolve({ code: code ?? 0, signal })); + }); + if (exit.signal) { + process.kill(process.pid, exit.signal); + return; + } + if (exit.code !== 0) { + process.exit(exit.code); + } +} + +await maybePreflightMigrations(); + +if (mode === "watch") { + env.PAPERCLIP_MIGRATION_PROMPT = "never"; +} + const serverScript = mode === "watch" ? "dev:watch" : "dev"; const child = spawn( pnpmBin, @@ -57,4 +174,3 @@ child.on("exit", (code, signal) => { } process.exit(code ?? 0); }); -