diff --git a/cli/src/__tests__/allowed-hostname.test.ts b/cli/src/__tests__/allowed-hostname.test.ts index 2b07ba29..92dfbf42 100644 --- a/cli/src/__tests__/allowed-hostname.test.ts +++ b/cli/src/__tests__/allowed-hostname.test.ts @@ -21,6 +21,12 @@ function writeBaseConfig(configPath: string) { mode: "embedded-postgres", embeddedPostgresDataDir: "/tmp/paperclip-db", embeddedPostgresPort: 54329, + backup: { + enabled: true, + intervalMinutes: 60, + retentionDays: 30, + dir: "/tmp/paperclip-backups", + }, }, logging: { mode: "file", @@ -68,4 +74,3 @@ describe("allowed-hostname command", () => { expect(raw.server.allowedHostnames).toEqual(["dotta-macbook-pro"]); }); }); - diff --git a/cli/src/commands/configure.ts b/cli/src/commands/configure.ts index a9ac3602..d072fee9 100644 --- a/cli/src/commands/configure.ts +++ b/cli/src/commands/configure.ts @@ -10,6 +10,7 @@ import { defaultSecretsConfig, promptSecrets } from "../prompts/secrets.js"; import { defaultStorageConfig, promptStorage } from "../prompts/storage.js"; import { promptServer } from "../prompts/server.js"; import { + resolveDefaultBackupDir, resolveDefaultEmbeddedPostgresDir, resolveDefaultLogsDir, resolvePaperclipInstanceId, @@ -39,6 +40,12 @@ function defaultConfig(): PaperclipConfig { mode: "embedded-postgres", embeddedPostgresDataDir: resolveDefaultEmbeddedPostgresDir(instanceId), embeddedPostgresPort: 54329, + backup: { + enabled: true, + intervalMinutes: 60, + retentionDays: 30, + dir: resolveDefaultBackupDir(instanceId), + }, }, logging: { mode: "file", @@ -118,7 +125,7 @@ export async function configure(opts: { switch (section) { case "database": - config.database = await promptDatabase(); + config.database = await promptDatabase(config.database); break; case "llm": { const llm = await promptLlm(); diff --git a/cli/src/commands/onboard.ts b/cli/src/commands/onboard.ts index d9547ded..050925e4 100644 --- a/cli/src/commands/onboard.ts +++ b/cli/src/commands/onboard.ts @@ -12,6 +12,7 @@ import { defaultStorageConfig, promptStorage } from "../prompts/storage.js"; import { promptServer } from "../prompts/server.js"; import { describeLocalInstancePaths, + resolveDefaultBackupDir, resolveDefaultEmbeddedPostgresDir, resolveDefaultLogsDir, resolvePaperclipInstanceId, @@ -35,6 +36,12 @@ function quickstartDefaults(): Pick { if (setupMode === "advanced") { p.log.step(pc.bold("Database")); - database = await promptDatabase(); + database = await promptDatabase(database); if (database.mode === "postgres" && database.connectionString) { const s = p.spinner(); diff --git a/cli/src/config/home.ts b/cli/src/config/home.ts index 0cc2d102..b1fafd83 100644 --- a/cli/src/config/home.ts +++ b/cli/src/config/home.ts @@ -49,6 +49,10 @@ export function resolveDefaultStorageDir(instanceId?: string): string { return path.resolve(resolvePaperclipInstanceRoot(instanceId), "data", "storage"); } +export function resolveDefaultBackupDir(instanceId?: string): string { + return path.resolve(resolvePaperclipInstanceRoot(instanceId), "data", "backups"); +} + export function expandHomePrefix(value: string): string { if (value === "~") return os.homedir(); if (value.startsWith("~/")) return path.resolve(os.homedir(), value.slice(2)); @@ -64,6 +68,7 @@ export function describeLocalInstancePaths(instanceId?: string) { instanceRoot, configPath: resolveDefaultConfigPath(resolvedInstanceId), embeddedPostgresDataDir: resolveDefaultEmbeddedPostgresDir(resolvedInstanceId), + backupDir: resolveDefaultBackupDir(resolvedInstanceId), logDir: resolveDefaultLogsDir(resolvedInstanceId), secretsKeyFilePath: resolveDefaultSecretsKeyFilePath(resolvedInstanceId), storageDir: resolveDefaultStorageDir(resolvedInstanceId), diff --git a/cli/src/config/schema.ts b/cli/src/config/schema.ts index 76fb93ca..12316faa 100644 --- a/cli/src/config/schema.ts +++ b/cli/src/config/schema.ts @@ -2,6 +2,7 @@ export { paperclipConfigSchema, configMetaSchema, llmConfigSchema, + databaseBackupConfigSchema, databaseConfigSchema, loggingConfigSchema, serverConfigSchema, @@ -13,6 +14,7 @@ export { secretsLocalEncryptedConfigSchema, type PaperclipConfig, type LlmConfig, + type DatabaseBackupConfig, type DatabaseConfig, type LoggingConfig, type ServerConfig, diff --git a/cli/src/prompts/database.ts b/cli/src/prompts/database.ts index 64dbf46c..b4ba075f 100644 --- a/cli/src/prompts/database.ts +++ b/cli/src/prompts/database.ts @@ -1,9 +1,26 @@ import * as p from "@clack/prompts"; import type { DatabaseConfig } from "../config/schema.js"; -import { resolveDefaultEmbeddedPostgresDir, resolvePaperclipInstanceId } from "../config/home.js"; +import { + resolveDefaultBackupDir, + resolveDefaultEmbeddedPostgresDir, + resolvePaperclipInstanceId, +} from "../config/home.js"; -export async function promptDatabase(): Promise { - const defaultEmbeddedDir = resolveDefaultEmbeddedPostgresDir(resolvePaperclipInstanceId()); +export async function promptDatabase(current?: DatabaseConfig): Promise { + const instanceId = resolvePaperclipInstanceId(); + const defaultEmbeddedDir = resolveDefaultEmbeddedPostgresDir(instanceId); + const defaultBackupDir = resolveDefaultBackupDir(instanceId); + const base: DatabaseConfig = current ?? { + mode: "embedded-postgres", + embeddedPostgresDataDir: defaultEmbeddedDir, + embeddedPostgresPort: 54329, + backup: { + enabled: true, + intervalMinutes: 60, + retentionDays: 30, + dir: defaultBackupDir, + }, + }; const mode = await p.select({ message: "Database mode", @@ -11,6 +28,7 @@ export async function promptDatabase(): Promise { { value: "embedded-postgres" as const, label: "Embedded PostgreSQL (managed locally)", hint: "recommended" }, { value: "postgres" as const, label: "PostgreSQL (external server)" }, ], + initialValue: base.mode, }); if (p.isCancel(mode)) { @@ -18,9 +36,14 @@ export async function promptDatabase(): Promise { process.exit(0); } + let connectionString: string | undefined = base.connectionString; + let embeddedPostgresDataDir = base.embeddedPostgresDataDir || defaultEmbeddedDir; + let embeddedPostgresPort = base.embeddedPostgresPort || 54329; + if (mode === "postgres") { - const connectionString = await p.text({ + const value = await p.text({ message: "PostgreSQL connection string", + defaultValue: base.connectionString ?? "", placeholder: "postgres://user:pass@localhost:5432/paperclip", validate: (val) => { if (!val) return "Connection string is required for PostgreSQL mode"; @@ -28,48 +51,107 @@ export async function promptDatabase(): Promise { }, }); - if (p.isCancel(connectionString)) { + if (p.isCancel(value)) { p.cancel("Setup cancelled."); process.exit(0); } - return { - mode: "postgres", - connectionString, - embeddedPostgresDataDir: defaultEmbeddedDir, - embeddedPostgresPort: 54329, - }; + connectionString = value; + } else { + const dataDir = await p.text({ + message: "Embedded PostgreSQL data directory", + defaultValue: base.embeddedPostgresDataDir || defaultEmbeddedDir, + placeholder: defaultEmbeddedDir, + }); + + if (p.isCancel(dataDir)) { + p.cancel("Setup cancelled."); + process.exit(0); + } + + embeddedPostgresDataDir = dataDir || defaultEmbeddedDir; + + const portValue = await p.text({ + message: "Embedded PostgreSQL port", + defaultValue: String(base.embeddedPostgresPort || 54329), + placeholder: "54329", + validate: (val) => { + const n = Number(val); + if (!Number.isInteger(n) || n < 1 || n > 65535) return "Port must be an integer between 1 and 65535"; + }, + }); + + if (p.isCancel(portValue)) { + p.cancel("Setup cancelled."); + process.exit(0); + } + + embeddedPostgresPort = Number(portValue || "54329"); + connectionString = undefined; } - const embeddedPostgresDataDir = await p.text({ - message: "Embedded PostgreSQL data directory", - defaultValue: defaultEmbeddedDir, - placeholder: defaultEmbeddedDir, + const backupEnabled = await p.confirm({ + message: "Enable automatic database backups?", + initialValue: base.backup.enabled, }); - - if (p.isCancel(embeddedPostgresDataDir)) { + if (p.isCancel(backupEnabled)) { p.cancel("Setup cancelled."); process.exit(0); } - const embeddedPostgresPort = await p.text({ - message: "Embedded PostgreSQL port", - defaultValue: "54329", - placeholder: "54329", + const backupDirInput = await p.text({ + message: "Backup directory", + defaultValue: base.backup.dir || defaultBackupDir, + placeholder: defaultBackupDir, + validate: (val) => (!val || val.trim().length === 0 ? "Backup directory is required" : undefined), + }); + if (p.isCancel(backupDirInput)) { + p.cancel("Setup cancelled."); + process.exit(0); + } + + const backupIntervalInput = await p.text({ + message: "Backup interval (minutes)", + defaultValue: String(base.backup.intervalMinutes || 60), + placeholder: "60", validate: (val) => { const n = Number(val); - if (!Number.isInteger(n) || n < 1 || n > 65535) return "Port must be an integer between 1 and 65535"; + if (!Number.isInteger(n) || n < 1) return "Interval must be a positive integer"; + if (n > 10080) return "Interval must be 10080 minutes (7 days) or less"; + return undefined; }, }); + if (p.isCancel(backupIntervalInput)) { + p.cancel("Setup cancelled."); + process.exit(0); + } - if (p.isCancel(embeddedPostgresPort)) { + const backupRetentionInput = await p.text({ + message: "Backup retention (days)", + defaultValue: String(base.backup.retentionDays || 30), + placeholder: "30", + validate: (val) => { + const n = Number(val); + if (!Number.isInteger(n) || n < 1) return "Retention must be a positive integer"; + if (n > 3650) return "Retention must be 3650 days or less"; + return undefined; + }, + }); + if (p.isCancel(backupRetentionInput)) { p.cancel("Setup cancelled."); process.exit(0); } return { - mode: "embedded-postgres", - embeddedPostgresDataDir: embeddedPostgresDataDir || defaultEmbeddedDir, - embeddedPostgresPort: Number(embeddedPostgresPort || "54329"), + mode, + connectionString, + embeddedPostgresDataDir, + embeddedPostgresPort, + backup: { + enabled: backupEnabled, + intervalMinutes: Number(backupIntervalInput || "60"), + retentionDays: Number(backupRetentionInput || "30"), + dir: backupDirInput || defaultBackupDir, + }, }; } diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 831af15f..65774b5d 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -141,6 +141,28 @@ pnpm dev If you set `DATABASE_URL`, the server will use that instead of embedded PostgreSQL. +## Automatic DB Backups + +Paperclip can run automatic DB backups on a timer. Defaults: + +- enabled +- every 60 minutes +- retain 30 days +- backup dir: `~/.paperclip/instances/default/data/backups` + +Configure these in: + +```sh +pnpm paperclipai configure --section database +``` + +Environment overrides: + +- `PAPERCLIP_DB_BACKUP_ENABLED=true|false` +- `PAPERCLIP_DB_BACKUP_INTERVAL_MINUTES=` +- `PAPERCLIP_DB_BACKUP_RETENTION_DAYS=` +- `PAPERCLIP_DB_BACKUP_DIR=/absolute/or/~/path` + ## Secrets in Dev Agent env vars now support secret references. By default, secret values are stored with local encryption and only secret refs are persisted in agent config. diff --git a/packages/db/src/backup-lib.ts b/packages/db/src/backup-lib.ts new file mode 100644 index 00000000..951540b1 --- /dev/null +++ b/packages/db/src/backup-lib.ts @@ -0,0 +1,333 @@ +import { existsSync, mkdirSync, readdirSync, statSync, unlinkSync } from "node:fs"; +import { writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import postgres from "postgres"; + +export type RunDatabaseBackupOptions = { + connectionString: string; + backupDir: string; + retentionDays: number; + filenamePrefix?: string; + connectTimeoutSeconds?: number; +}; + +export type RunDatabaseBackupResult = { + backupFile: string; + sizeBytes: number; + prunedCount: number; +}; + +function timestamp(date: Date = new Date()): string { + const pad = (n: number) => String(n).padStart(2, "0"); + return `${date.getFullYear()}${pad(date.getMonth() + 1)}${pad(date.getDate())}-${pad(date.getHours())}${pad(date.getMinutes())}${pad(date.getSeconds())}`; +} + +function pruneOldBackups(backupDir: string, retentionDays: number, filenamePrefix: string): number { + if (!existsSync(backupDir)) return 0; + const safeRetention = Math.max(1, Math.trunc(retentionDays)); + const cutoff = Date.now() - safeRetention * 24 * 60 * 60 * 1000; + let pruned = 0; + + for (const name of readdirSync(backupDir)) { + if (!name.startsWith(`${filenamePrefix}-`) || !name.endsWith(".sql")) continue; + const fullPath = resolve(backupDir, name); + const stat = statSync(fullPath); + if (stat.mtimeMs < cutoff) { + unlinkSync(fullPath); + pruned++; + } + } + + return pruned; +} + +function formatBackupSize(sizeBytes: number): string { + if (sizeBytes < 1024) return `${sizeBytes}B`; + if (sizeBytes < 1024 * 1024) return `${(sizeBytes / 1024).toFixed(1)}K`; + return `${(sizeBytes / (1024 * 1024)).toFixed(1)}M`; +} + +export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise { + const filenamePrefix = opts.filenamePrefix ?? "paperclip"; + const retentionDays = Math.max(1, Math.trunc(opts.retentionDays)); + const connectTimeout = Math.max(1, Math.trunc(opts.connectTimeoutSeconds ?? 5)); + const sql = postgres(opts.connectionString, { max: 1, connect_timeout: connectTimeout }); + + try { + await sql`SELECT 1`; + + const lines: string[] = []; + const emit = (line: string) => lines.push(line); + + emit("-- Paperclip database backup"); + emit(`-- Created: ${new Date().toISOString()}`); + emit(""); + emit("BEGIN;"); + emit(""); + + // Get all enums + const enums = await sql<{ typname: string; labels: string[] }[]>` + SELECT t.typname, array_agg(e.enumlabel ORDER BY e.enumsortorder) AS labels + FROM pg_type t + JOIN pg_enum e ON t.oid = e.enumtypid + JOIN pg_namespace n ON t.typnamespace = n.oid + WHERE n.nspname = 'public' + GROUP BY t.typname + ORDER BY t.typname + `; + + for (const e of enums) { + const labels = e.labels.map((l) => `'${l.replace(/'/g, "''")}'`).join(", "); + emit(`CREATE TYPE "public"."${e.typname}" AS ENUM (${labels});`); + } + if (enums.length > 0) emit(""); + + // Get tables in dependency order (referenced tables first) + const tables = await sql<{ tablename: string }[]>` + SELECT c.relname AS tablename + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' + AND c.relkind = 'r' + AND c.relname != '__drizzle_migrations' + ORDER BY c.relname + `; + + // Get full CREATE TABLE DDL via column info + for (const { tablename } of tables) { + const columns = await sql<{ + column_name: string; + data_type: string; + udt_name: string; + is_nullable: string; + column_default: string | null; + character_maximum_length: number | null; + numeric_precision: number | null; + numeric_scale: number | null; + }[]>` + SELECT column_name, data_type, udt_name, is_nullable, column_default, + character_maximum_length, numeric_precision, numeric_scale + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = ${tablename} + ORDER BY ordinal_position + `; + + emit(`-- Table: ${tablename}`); + emit(`DROP TABLE IF EXISTS "${tablename}" CASCADE;`); + + const colDefs: string[] = []; + for (const col of columns) { + let typeStr: string; + if (col.data_type === "USER-DEFINED") { + typeStr = `"${col.udt_name}"`; + } else if (col.data_type === "ARRAY") { + typeStr = `${col.udt_name.replace(/^_/, "")}[]`; + } else if (col.data_type === "character varying") { + typeStr = col.character_maximum_length + ? `varchar(${col.character_maximum_length})` + : "varchar"; + } else if (col.data_type === "numeric" && col.numeric_precision != null) { + typeStr = + col.numeric_scale != null + ? `numeric(${col.numeric_precision}, ${col.numeric_scale})` + : `numeric(${col.numeric_precision})`; + } else { + typeStr = col.data_type; + } + + let def = ` "${col.column_name}" ${typeStr}`; + if (col.column_default != null) def += ` DEFAULT ${col.column_default}`; + if (col.is_nullable === "NO") def += " NOT NULL"; + colDefs.push(def); + } + + // Primary key + const pk = await sql<{ constraint_name: string; column_names: string[] }[]>` + SELECT c.conname AS constraint_name, + array_agg(a.attname ORDER BY array_position(c.conkey, a.attnum)) AS column_names + FROM pg_constraint c + JOIN pg_class t ON t.oid = c.conrelid + JOIN pg_namespace n ON n.oid = t.relnamespace + JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(c.conkey) + WHERE n.nspname = 'public' AND t.relname = ${tablename} AND c.contype = 'p' + GROUP BY c.conname + `; + for (const p of pk) { + const cols = p.column_names.map((c) => `"${c}"`).join(", "); + colDefs.push(` CONSTRAINT "${p.constraint_name}" PRIMARY KEY (${cols})`); + } + + emit(`CREATE TABLE "${tablename}" (`); + emit(colDefs.join(",\n")); + emit(");"); + emit(""); + } + + // Foreign keys (after all tables created) + const fks = await sql<{ + constraint_name: string; + source_table: string; + source_columns: string[]; + target_table: string; + target_columns: string[]; + update_rule: string; + delete_rule: string; + }[]>` + SELECT + c.conname AS constraint_name, + src.relname AS source_table, + array_agg(sa.attname ORDER BY array_position(c.conkey, sa.attnum)) AS source_columns, + tgt.relname AS target_table, + array_agg(ta.attname ORDER BY array_position(c.confkey, ta.attnum)) AS target_columns, + CASE c.confupdtype WHEN 'a' THEN 'NO ACTION' WHEN 'r' THEN 'RESTRICT' WHEN 'c' THEN 'CASCADE' WHEN 'n' THEN 'SET NULL' WHEN 'd' THEN 'SET DEFAULT' END AS update_rule, + CASE c.confdeltype WHEN 'a' THEN 'NO ACTION' WHEN 'r' THEN 'RESTRICT' WHEN 'c' THEN 'CASCADE' WHEN 'n' THEN 'SET NULL' WHEN 'd' THEN 'SET DEFAULT' END AS delete_rule + FROM pg_constraint c + JOIN pg_class src ON src.oid = c.conrelid + JOIN pg_class tgt ON tgt.oid = c.confrelid + JOIN pg_namespace n ON n.oid = src.relnamespace + JOIN pg_attribute sa ON sa.attrelid = src.oid AND sa.attnum = ANY(c.conkey) + JOIN pg_attribute ta ON ta.attrelid = tgt.oid AND ta.attnum = ANY(c.confkey) + WHERE c.contype = 'f' AND n.nspname = 'public' + GROUP BY c.conname, src.relname, tgt.relname, c.confupdtype, c.confdeltype + ORDER BY src.relname, c.conname + `; + + if (fks.length > 0) { + emit("-- Foreign keys"); + for (const fk of fks) { + const srcCols = fk.source_columns.map((c) => `"${c}"`).join(", "); + const tgtCols = fk.target_columns.map((c) => `"${c}"`).join(", "); + emit( + `ALTER TABLE "${fk.source_table}" ADD CONSTRAINT "${fk.constraint_name}" FOREIGN KEY (${srcCols}) REFERENCES "${fk.target_table}" (${tgtCols}) ON UPDATE ${fk.update_rule} ON DELETE ${fk.delete_rule};`, + ); + } + emit(""); + } + + // Unique constraints + const uniques = await sql<{ + constraint_name: string; + tablename: string; + column_names: string[]; + }[]>` + SELECT c.conname AS constraint_name, + t.relname AS tablename, + array_agg(a.attname ORDER BY array_position(c.conkey, a.attnum)) AS column_names + FROM pg_constraint c + JOIN pg_class t ON t.oid = c.conrelid + JOIN pg_namespace n ON n.oid = t.relnamespace + JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(c.conkey) + WHERE n.nspname = 'public' AND c.contype = 'u' + GROUP BY c.conname, t.relname + ORDER BY t.relname, c.conname + `; + + if (uniques.length > 0) { + emit("-- Unique constraints"); + for (const u of uniques) { + const cols = u.column_names.map((c) => `"${c}"`).join(", "); + emit(`ALTER TABLE "${u.tablename}" ADD CONSTRAINT "${u.constraint_name}" UNIQUE (${cols});`); + } + emit(""); + } + + // Indexes (non-primary, non-unique-constraint) + const indexes = await sql<{ indexdef: string }[]>` + SELECT indexdef + FROM pg_indexes + WHERE schemaname = 'public' + AND indexname NOT IN ( + SELECT conname FROM pg_constraint + WHERE connamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'public') + ) + ORDER BY tablename, indexname + `; + + if (indexes.length > 0) { + emit("-- Indexes"); + for (const idx of indexes) { + emit(`${idx.indexdef};`); + } + emit(""); + } + + // Dump data for each table + for (const { tablename } of tables) { + const count = await sql<{ n: number }[]>` + SELECT count(*)::int AS n FROM ${sql(tablename)} + `; + if ((count[0]?.n ?? 0) === 0) continue; + + // Get column info for this table + const cols = await sql<{ column_name: string; data_type: string }[]>` + SELECT column_name, data_type + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = ${tablename} + ORDER BY ordinal_position + `; + const colNames = cols.map((c) => `"${c.column_name}"`).join(", "); + + emit(`-- Data for: ${tablename} (${count[0]!.n} rows)`); + + const rows = await sql`SELECT * FROM ${sql(tablename)}`.values(); + for (const row of rows) { + const values = row.map((val: unknown) => { + if (val === null || val === undefined) return "NULL"; + if (typeof val === "boolean") return val ? "true" : "false"; + if (typeof val === "number") return String(val); + if (val instanceof Date) return `'${val.toISOString()}'`; + if (typeof val === "object") return `'${JSON.stringify(val).replace(/'/g, "''")}'`; + return `'${String(val).replace(/'/g, "''")}'`; + }); + emit(`INSERT INTO "${tablename}" (${colNames}) VALUES (${values.join(", ")});`); + } + emit(""); + } + + // Sequence values + const sequences = await sql<{ sequence_name: string }[]>` + SELECT sequence_name + FROM information_schema.sequences + WHERE sequence_schema = 'public' + ORDER BY sequence_name + `; + + if (sequences.length > 0) { + emit("-- Sequence values"); + for (const seq of sequences) { + const val = await sql<{ last_value: string }[]>` + SELECT last_value::text FROM ${sql(seq.sequence_name)} + `; + if (val[0]) { + emit(`SELECT setval('"${seq.sequence_name}"', ${val[0].last_value});`); + } + } + emit(""); + } + + emit("COMMIT;"); + emit(""); + + // Write the backup file + mkdirSync(opts.backupDir, { recursive: true }); + const backupFile = resolve(opts.backupDir, `${filenamePrefix}-${timestamp()}.sql`); + await writeFile(backupFile, lines.join("\n"), "utf8"); + + const sizeBytes = statSync(backupFile).size; + const prunedCount = pruneOldBackups(opts.backupDir, retentionDays, filenamePrefix); + + return { + backupFile, + sizeBytes, + prunedCount, + }; + } finally { + await sql.end(); + } +} + +export function formatDatabaseBackupResult(result: RunDatabaseBackupResult): string { + const size = formatBackupSize(result.sizeBytes); + const pruned = result.prunedCount > 0 ? `; pruned ${result.prunedCount} old backup(s)` : ""; + return `${result.backupFile} (${size}${pruned})`; +} diff --git a/packages/db/src/backup.ts b/packages/db/src/backup.ts index 0c7b44ae..f07dc646 100644 --- a/packages/db/src/backup.ts +++ b/packages/db/src/backup.ts @@ -1,338 +1,122 @@ -import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, unlinkSync } from "node:fs"; -import { writeFile } from "node:fs/promises"; -import { resolve } from "node:path"; -import postgres from "postgres"; +import { existsSync, readFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { formatDatabaseBackupResult, runDatabaseBackup } from "./backup-lib.js"; -const PROJECT_ROOT = resolve(import.meta.dirname, "../../.."); -const BACKUP_DIR = resolve(PROJECT_ROOT, "data/backups"); -const CONFIG_FILE = resolve(PROJECT_ROOT, ".paperclip/config.json"); -const MAX_AGE_DAYS = 30; +type PartialConfig = { + database?: { + mode?: "embedded-postgres" | "postgres"; + connectionString?: string; + embeddedPostgresPort?: number; + backup?: { + dir?: string; + retentionDays?: number; + }; + }; +}; -function loadPort(): number { - try { - const raw = readFileSync(CONFIG_FILE, "utf8"); - const config = JSON.parse(raw); - const port = config?.database?.embeddedPostgresPort; - if (typeof port === "number" && Number.isFinite(port)) return port; - } catch {} - return 54329; +function expandHomePrefix(value: string): string { + if (value === "~") return os.homedir(); + if (value.startsWith("~/")) return path.resolve(os.homedir(), value.slice(2)); + return value; } -function timestamp(): string { - const d = new Date(); - const pad = (n: number) => String(n).padStart(2, "0"); - return `${d.getFullYear()}${pad(d.getMonth() + 1)}${pad(d.getDate())}-${pad(d.getHours())}${pad(d.getMinutes())}${pad(d.getSeconds())}`; +function resolvePaperclipHomeDir(): string { + const envHome = process.env.PAPERCLIP_HOME?.trim(); + if (envHome) return path.resolve(expandHomePrefix(envHome)); + return path.resolve(os.homedir(), ".paperclip"); } -function pruneOldBackups() { - if (!existsSync(BACKUP_DIR)) return; - const cutoff = Date.now() - MAX_AGE_DAYS * 24 * 60 * 60 * 1000; - let pruned = 0; - for (const name of readdirSync(BACKUP_DIR)) { - if (!name.startsWith("paperclip-") || !name.endsWith(".sql")) continue; - const fullPath = resolve(BACKUP_DIR, name); - const stat = statSync(fullPath); - if (stat.mtimeMs < cutoff) { - unlinkSync(fullPath); - pruned++; - } +function resolvePaperclipInstanceId(): string { + const raw = process.env.PAPERCLIP_INSTANCE_ID?.trim() || "default"; + if (!/^[a-zA-Z0-9_-]+$/.test(raw)) { + throw new Error(`Invalid PAPERCLIP_INSTANCE_ID '${raw}'.`); } - if (pruned > 0) console.log(`Pruned ${pruned} backup(s) older than ${MAX_AGE_DAYS} days.`); + return raw; +} + +function resolveDefaultConfigPath(): string { + return path.resolve(resolvePaperclipHomeDir(), "instances", resolvePaperclipInstanceId(), "config.json"); +} + +function readConfig(configPath: string): PartialConfig | null { + if (!existsSync(configPath)) return null; + try { + const parsed = JSON.parse(readFileSync(configPath, "utf8")); + return typeof parsed === "object" && parsed ? (parsed as PartialConfig) : null; + } catch { + return null; + } +} + +function asPositiveInt(value: unknown): number | null { + if (typeof value !== "number" || !Number.isFinite(value)) return null; + const rounded = Math.trunc(value); + return rounded > 0 ? rounded : null; +} + +function resolveEmbeddedPort(config: PartialConfig | null): number { + return asPositiveInt(config?.database?.embeddedPostgresPort) ?? 54329; +} + +function resolveConnectionString(config: PartialConfig | null): string { + const envUrl = process.env.DATABASE_URL?.trim(); + if (envUrl) return envUrl; + + if (config?.database?.mode === "postgres" && typeof config.database.connectionString === "string") { + const trimmed = config.database.connectionString.trim(); + if (trimmed) return trimmed; + } + + const port = resolveEmbeddedPort(config); + return `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`; +} + +function resolveDefaultBackupDir(): string { + return path.resolve(resolvePaperclipHomeDir(), "instances", resolvePaperclipInstanceId(), "data", "backups"); +} + +function resolveBackupDir(config: PartialConfig | null): string { + const raw = config?.database?.backup?.dir; + if (typeof raw === "string" && raw.trim().length > 0) { + return path.resolve(expandHomePrefix(raw.trim())); + } + return resolveDefaultBackupDir(); +} + +function resolveRetentionDays(config: PartialConfig | null): number { + return asPositiveInt(config?.database?.backup?.retentionDays) ?? 30; } async function main() { - const port = loadPort(); - const connString = `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`; + const configPath = resolveDefaultConfigPath(); + const config = readConfig(configPath); + const connectionString = resolveConnectionString(config); + const backupDir = resolveBackupDir(config); + const retentionDays = resolveRetentionDays(config); - console.log(`Connecting to embedded PostgreSQL on port ${port}...`); - - const sql = postgres(connString, { max: 1, connect_timeout: 5 }); + console.log(`Config path: ${configPath}`); + console.log(`Backing up database to: ${backupDir}`); + console.log(`Retention window: ${retentionDays} day(s)`); try { - // Verify connection - await sql`SELECT 1`; - } catch (err: any) { - console.error(`Error: Cannot connect to embedded PostgreSQL on port ${port}.`); - console.error(" Make sure the server is running (pnpm dev)."); + const result = await runDatabaseBackup({ + connectionString, + backupDir, + retentionDays, + filenamePrefix: "paperclip", + }); + + console.log(`Backup saved: ${formatDatabaseBackupResult(result)}`); + } catch (err) { + console.error("Backup failed."); + if (err instanceof Error) { + console.error(err.message); + } else { + console.error(String(err)); + } process.exit(1); } - - try { - const lines: string[] = []; - const emit = (line: string) => lines.push(line); - - emit("-- Paperclip database backup"); - emit(`-- Created: ${new Date().toISOString()}`); - emit(`-- Server port: ${port}`); - emit(""); - emit("BEGIN;"); - emit(""); - - // Get all enums - const enums = await sql<{ typname: string; labels: string[] }[]>` - SELECT t.typname, array_agg(e.enumlabel ORDER BY e.enumsortorder) AS labels - FROM pg_type t - JOIN pg_enum e ON t.oid = e.enumtypid - JOIN pg_namespace n ON t.typnamespace = n.oid - WHERE n.nspname = 'public' - GROUP BY t.typname - ORDER BY t.typname - `; - - for (const e of enums) { - const labels = e.labels.map((l) => `'${l.replace(/'/g, "''")}'`).join(", "); - emit(`CREATE TYPE "public"."${e.typname}" AS ENUM (${labels});`); - } - if (enums.length > 0) emit(""); - - // Get tables in dependency order (referenced tables first) - const tables = await sql<{ tablename: string }[]>` - SELECT c.relname AS tablename - FROM pg_class c - JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE n.nspname = 'public' - AND c.relkind = 'r' - AND c.relname != '__drizzle_migrations' - ORDER BY c.relname - `; - - // Get full CREATE TABLE DDL via column info - for (const { tablename } of tables) { - const columns = await sql<{ - column_name: string; - data_type: string; - udt_name: string; - is_nullable: string; - column_default: string | null; - character_maximum_length: number | null; - numeric_precision: number | null; - numeric_scale: number | null; - }[]>` - SELECT column_name, data_type, udt_name, is_nullable, column_default, - character_maximum_length, numeric_precision, numeric_scale - FROM information_schema.columns - WHERE table_schema = 'public' AND table_name = ${tablename} - ORDER BY ordinal_position - `; - - emit(`-- Table: ${tablename}`); - emit(`DROP TABLE IF EXISTS "${tablename}" CASCADE;`); - - const colDefs: string[] = []; - for (const col of columns) { - let typeStr: string; - if (col.data_type === "USER-DEFINED") { - typeStr = `"${col.udt_name}"`; - } else if (col.data_type === "ARRAY") { - typeStr = `${col.udt_name.replace(/^_/, "")}[]`; - } else if (col.data_type === "character varying") { - typeStr = col.character_maximum_length - ? `varchar(${col.character_maximum_length})` - : "varchar"; - } else if (col.data_type === "numeric" && col.numeric_precision != null) { - typeStr = - col.numeric_scale != null - ? `numeric(${col.numeric_precision}, ${col.numeric_scale})` - : `numeric(${col.numeric_precision})`; - } else { - typeStr = col.data_type; - } - - let def = ` "${col.column_name}" ${typeStr}`; - if (col.column_default != null) def += ` DEFAULT ${col.column_default}`; - if (col.is_nullable === "NO") def += " NOT NULL"; - colDefs.push(def); - } - - // Primary key - const pk = await sql<{ constraint_name: string; column_names: string[] }[]>` - SELECT c.conname AS constraint_name, - array_agg(a.attname ORDER BY array_position(c.conkey, a.attnum)) AS column_names - FROM pg_constraint c - JOIN pg_class t ON t.oid = c.conrelid - JOIN pg_namespace n ON n.oid = t.relnamespace - JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(c.conkey) - WHERE n.nspname = 'public' AND t.relname = ${tablename} AND c.contype = 'p' - GROUP BY c.conname - `; - for (const p of pk) { - const cols = p.column_names.map((c) => `"${c}"`).join(", "); - colDefs.push(` CONSTRAINT "${p.constraint_name}" PRIMARY KEY (${cols})`); - } - - emit(`CREATE TABLE "${tablename}" (`); - emit(colDefs.join(",\n")); - emit(");"); - emit(""); - } - - // Foreign keys (after all tables created) - const fks = await sql<{ - constraint_name: string; - source_table: string; - source_columns: string[]; - target_table: string; - target_columns: string[]; - update_rule: string; - delete_rule: string; - }[]>` - SELECT - c.conname AS constraint_name, - src.relname AS source_table, - array_agg(sa.attname ORDER BY array_position(c.conkey, sa.attnum)) AS source_columns, - tgt.relname AS target_table, - array_agg(ta.attname ORDER BY array_position(c.confkey, ta.attnum)) AS target_columns, - CASE c.confupdtype WHEN 'a' THEN 'NO ACTION' WHEN 'r' THEN 'RESTRICT' WHEN 'c' THEN 'CASCADE' WHEN 'n' THEN 'SET NULL' WHEN 'd' THEN 'SET DEFAULT' END AS update_rule, - CASE c.confdeltype WHEN 'a' THEN 'NO ACTION' WHEN 'r' THEN 'RESTRICT' WHEN 'c' THEN 'CASCADE' WHEN 'n' THEN 'SET NULL' WHEN 'd' THEN 'SET DEFAULT' END AS delete_rule - FROM pg_constraint c - JOIN pg_class src ON src.oid = c.conrelid - JOIN pg_class tgt ON tgt.oid = c.confrelid - JOIN pg_namespace n ON n.oid = src.relnamespace - JOIN pg_attribute sa ON sa.attrelid = src.oid AND sa.attnum = ANY(c.conkey) - JOIN pg_attribute ta ON ta.attrelid = tgt.oid AND ta.attnum = ANY(c.confkey) - WHERE c.contype = 'f' AND n.nspname = 'public' - GROUP BY c.conname, src.relname, tgt.relname, c.confupdtype, c.confdeltype - ORDER BY src.relname, c.conname - `; - - if (fks.length > 0) { - emit("-- Foreign keys"); - for (const fk of fks) { - const srcCols = fk.source_columns.map((c) => `"${c}"`).join(", "); - const tgtCols = fk.target_columns.map((c) => `"${c}"`).join(", "); - emit( - `ALTER TABLE "${fk.source_table}" ADD CONSTRAINT "${fk.constraint_name}" FOREIGN KEY (${srcCols}) REFERENCES "${fk.target_table}" (${tgtCols}) ON UPDATE ${fk.update_rule} ON DELETE ${fk.delete_rule};`, - ); - } - emit(""); - } - - // Unique constraints - const uniques = await sql<{ - constraint_name: string; - tablename: string; - column_names: string[]; - }[]>` - SELECT c.conname AS constraint_name, - t.relname AS tablename, - array_agg(a.attname ORDER BY array_position(c.conkey, a.attnum)) AS column_names - FROM pg_constraint c - JOIN pg_class t ON t.oid = c.conrelid - JOIN pg_namespace n ON n.oid = t.relnamespace - JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(c.conkey) - WHERE n.nspname = 'public' AND c.contype = 'u' - GROUP BY c.conname, t.relname - ORDER BY t.relname, c.conname - `; - - if (uniques.length > 0) { - emit("-- Unique constraints"); - for (const u of uniques) { - const cols = u.column_names.map((c) => `"${c}"`).join(", "); - emit(`ALTER TABLE "${u.tablename}" ADD CONSTRAINT "${u.constraint_name}" UNIQUE (${cols});`); - } - emit(""); - } - - // Indexes (non-primary, non-unique-constraint) - const indexes = await sql<{ indexdef: string }[]>` - SELECT indexdef - FROM pg_indexes - WHERE schemaname = 'public' - AND indexname NOT IN ( - SELECT conname FROM pg_constraint - WHERE connamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'public') - ) - ORDER BY tablename, indexname - `; - - if (indexes.length > 0) { - emit("-- Indexes"); - for (const idx of indexes) { - emit(`${idx.indexdef};`); - } - emit(""); - } - - // Dump data for each table - for (const { tablename } of tables) { - const count = await sql<{ n: number }[]>` - SELECT count(*)::int AS n FROM ${sql(tablename)} - `; - if ((count[0]?.n ?? 0) === 0) continue; - - // Get column info for this table - const cols = await sql<{ column_name: string; data_type: string }[]>` - SELECT column_name, data_type - FROM information_schema.columns - WHERE table_schema = 'public' AND table_name = ${tablename} - ORDER BY ordinal_position - `; - const colNames = cols.map((c) => `"${c.column_name}"`).join(", "); - - emit(`-- Data for: ${tablename} (${count[0]!.n} rows)`); - - const rows = await sql`SELECT * FROM ${sql(tablename)}`.values(); - for (const row of rows) { - const values = row.map((val: any) => { - if (val === null || val === undefined) return "NULL"; - if (typeof val === "boolean") return val ? "true" : "false"; - if (typeof val === "number") return String(val); - if (val instanceof Date) return `'${val.toISOString()}'`; - if (typeof val === "object") return `'${JSON.stringify(val).replace(/'/g, "''")}'`; - return `'${String(val).replace(/'/g, "''")}'`; - }); - emit(`INSERT INTO "${tablename}" (${colNames}) VALUES (${values.join(", ")});`); - } - emit(""); - } - - // Sequence values - const sequences = await sql<{ sequence_name: string }[]>` - SELECT sequence_name - FROM information_schema.sequences - WHERE sequence_schema = 'public' - ORDER BY sequence_name - `; - - if (sequences.length > 0) { - emit("-- Sequence values"); - for (const seq of sequences) { - const val = await sql<{ last_value: string }[]>` - SELECT last_value::text FROM ${sql(seq.sequence_name)} - `; - if (val[0]) { - emit(`SELECT setval('"${seq.sequence_name}"', ${val[0].last_value});`); - } - } - emit(""); - } - - emit("COMMIT;"); - emit(""); - - // Write the backup file - mkdirSync(BACKUP_DIR, { recursive: true }); - const backupFile = resolve(BACKUP_DIR, `paperclip-${timestamp()}.sql`); - await writeFile(backupFile, lines.join("\n"), "utf8"); - - const sizeBytes = statSync(backupFile).size; - const sizeStr = - sizeBytes < 1024 - ? `${sizeBytes}B` - : sizeBytes < 1024 * 1024 - ? `${(sizeBytes / 1024).toFixed(1)}K` - : `${(sizeBytes / (1024 * 1024)).toFixed(1)}M`; - - console.log(`Backup saved: ${backupFile} (${sizeStr})`); - - pruneOldBackups(); - } finally { - await sql.end(); - } } -main().catch((err) => { - console.error(err); - process.exit(1); -}); +await main(); diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 0f3ee89f..3cafa7af 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -10,4 +10,10 @@ export { type MigrationBootstrapResult, type Db, } from "./client.js"; +export { + runDatabaseBackup, + formatDatabaseBackupResult, + type RunDatabaseBackupOptions, + type RunDatabaseBackupResult, +} from "./backup-lib.js"; export * from "./schema/index.js"; diff --git a/packages/shared/src/config-schema.ts b/packages/shared/src/config-schema.ts index 15e6d716..37903a90 100644 --- a/packages/shared/src/config-schema.ts +++ b/packages/shared/src/config-schema.ts @@ -18,11 +18,24 @@ export const llmConfigSchema = z.object({ apiKey: z.string().optional(), }); +export const databaseBackupConfigSchema = z.object({ + enabled: z.boolean().default(true), + intervalMinutes: z.number().int().min(1).max(7 * 24 * 60).default(60), + retentionDays: z.number().int().min(1).max(3650).default(30), + dir: z.string().default("~/.paperclip/instances/default/data/backups"), +}); + export const databaseConfigSchema = z.object({ mode: z.enum(["embedded-postgres", "postgres"]).default("embedded-postgres"), connectionString: z.string().optional(), embeddedPostgresDataDir: z.string().default("~/.paperclip/instances/default/db"), embeddedPostgresPort: z.number().int().min(1).max(65535).default(54329), + backup: databaseBackupConfigSchema.default({ + enabled: true, + intervalMinutes: 60, + retentionDays: 30, + dir: "~/.paperclip/instances/default/data/backups", + }), }); export const loggingConfigSchema = z.object({ @@ -160,3 +173,4 @@ export type SecretsConfig = z.infer; export type SecretsLocalEncryptedConfig = z.infer; export type AuthConfig = z.infer; export type ConfigMeta = z.infer; +export type DatabaseBackupConfig = z.infer; diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 3f8001d4..59ec9eb6 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -243,6 +243,7 @@ export { paperclipConfigSchema, configMetaSchema, llmConfigSchema, + databaseBackupConfigSchema, databaseConfigSchema, loggingConfigSchema, serverConfigSchema, @@ -254,6 +255,7 @@ export { secretsLocalEncryptedConfigSchema, type PaperclipConfig, type LlmConfig, + type DatabaseBackupConfig, type DatabaseConfig, type LoggingConfig, type ServerConfig, diff --git a/scripts/backup-db.sh b/scripts/backup-db.sh index d552f3a2..d77c922d 100755 --- a/scripts/backup-db.sh +++ b/scripts/backup-db.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash set -euo pipefail -# Backup the embedded PostgreSQL database to data/backups/ +# Backup the configured Paperclip database to the configured backup directory +# (default: ~/.paperclip/instances//data/backups) # # Usage: # ./scripts/backup-db.sh diff --git a/server/src/config.ts b/server/src/config.ts index ff46d34e..01a37588 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -15,6 +15,7 @@ import { type StorageProvider, } from "@paperclipai/shared"; import { + resolveDefaultBackupDir, resolveDefaultEmbeddedPostgresDir, resolveDefaultSecretsKeyFilePath, resolveDefaultStorageDir, @@ -40,6 +41,10 @@ export interface Config { databaseUrl: string | undefined; embeddedPostgresDataDir: string; embeddedPostgresPort: number; + databaseBackupEnabled: boolean; + databaseBackupIntervalMinutes: number; + databaseBackupRetentionDays: number; + databaseBackupDir: string; serveUi: boolean; uiDevMiddleware: boolean; secretsProvider: SecretProvider; @@ -66,6 +71,7 @@ export function loadConfig(): Config { fileDatabaseMode === "postgres" ? fileConfig?.database.connectionString : undefined; + const fileDatabaseBackup = fileConfig?.database.backup; const fileSecrets = fileConfig?.secrets; const fileStorage = fileConfig?.storage; const strictModeFromEnv = process.env.PAPERCLIP_SECRETS_STRICT_MODE; @@ -148,6 +154,27 @@ export function loadConfig(): Config { companyDeletionEnvRaw !== undefined ? companyDeletionEnvRaw === "true" : deploymentMode === "local_trusted"; + const databaseBackupEnabled = + process.env.PAPERCLIP_DB_BACKUP_ENABLED !== undefined + ? process.env.PAPERCLIP_DB_BACKUP_ENABLED === "true" + : (fileDatabaseBackup?.enabled ?? true); + const databaseBackupIntervalMinutes = Math.max( + 1, + Number(process.env.PAPERCLIP_DB_BACKUP_INTERVAL_MINUTES) || + fileDatabaseBackup?.intervalMinutes || + 60, + ); + const databaseBackupRetentionDays = Math.max( + 1, + Number(process.env.PAPERCLIP_DB_BACKUP_RETENTION_DAYS) || + fileDatabaseBackup?.retentionDays || + 30, + ); + const databaseBackupDir = resolveHomeAwarePath( + process.env.PAPERCLIP_DB_BACKUP_DIR ?? + fileDatabaseBackup?.dir ?? + resolveDefaultBackupDir(), + ); return { deploymentMode, @@ -163,6 +190,10 @@ export function loadConfig(): Config { fileConfig?.database.embeddedPostgresDataDir ?? resolveDefaultEmbeddedPostgresDir(), ), embeddedPostgresPort: fileConfig?.database.embeddedPostgresPort ?? 54329, + databaseBackupEnabled, + databaseBackupIntervalMinutes, + databaseBackupRetentionDays, + databaseBackupDir, serveUi: process.env.SERVE_UI !== undefined ? process.env.SERVE_UI === "true" diff --git a/server/src/home-paths.ts b/server/src/home-paths.ts index a1bb8f3f..d2b7e53a 100644 --- a/server/src/home-paths.ts +++ b/server/src/home-paths.ts @@ -49,6 +49,10 @@ export function resolveDefaultStorageDir(): string { return path.resolve(resolvePaperclipInstanceRoot(), "data", "storage"); } +export function resolveDefaultBackupDir(): string { + return path.resolve(resolvePaperclipInstanceRoot(), "data", "backups"); +} + export function resolveDefaultAgentWorkspaceDir(agentId: string): string { const trimmed = agentId.trim(); if (!PATH_SEGMENT_RE.test(trimmed)) { diff --git a/server/src/index.ts b/server/src/index.ts index e319e852..ada5743f 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -12,6 +12,8 @@ import { inspectMigrations, applyPendingMigrations, reconcilePendingMigrationHistory, + formatDatabaseBackupResult, + runDatabaseBackup, authUsers, companies, companyMemberships, @@ -220,6 +222,7 @@ let db; let embeddedPostgres: EmbeddedPostgresInstance | null = null; let embeddedPostgresStartedByThisProcess = false; let migrationSummary: MigrationSummary = "skipped"; +let activeDatabaseConnectionString: string; let startupDbInfo: | { mode: "external-postgres"; connectionString: string } | { mode: "embedded-postgres"; dataDir: string; port: number }; @@ -228,6 +231,7 @@ if (config.databaseUrl) { db = createDb(config.databaseUrl); logger.info("Using external PostgreSQL via DATABASE_URL/config"); + activeDatabaseConnectionString = config.databaseUrl; startupDbInfo = { mode: "external-postgres", connectionString: config.databaseUrl }; } else { const moduleName = "embedded-postgres"; @@ -364,6 +368,7 @@ if (config.databaseUrl) { db = createDb(embeddedConnectionString); logger.info("Embedded PostgreSQL ready"); + activeDatabaseConnectionString = embeddedConnectionString; startupDbInfo = { mode: "embedded-postgres", dataDir, port }; } @@ -489,6 +494,54 @@ if (config.heartbeatSchedulerEnabled) { }, config.heartbeatSchedulerIntervalMs); } +if (config.databaseBackupEnabled) { + const backupIntervalMs = config.databaseBackupIntervalMinutes * 60 * 1000; + let backupInFlight = false; + + const runScheduledBackup = async () => { + if (backupInFlight) { + logger.warn("Skipping scheduled database backup because a previous backup is still running"); + return; + } + + backupInFlight = true; + try { + const result = await runDatabaseBackup({ + connectionString: activeDatabaseConnectionString, + backupDir: config.databaseBackupDir, + retentionDays: config.databaseBackupRetentionDays, + filenamePrefix: "paperclip", + }); + logger.info( + { + backupFile: result.backupFile, + sizeBytes: result.sizeBytes, + prunedCount: result.prunedCount, + backupDir: config.databaseBackupDir, + retentionDays: config.databaseBackupRetentionDays, + }, + `Automatic database backup complete: ${formatDatabaseBackupResult(result)}`, + ); + } catch (err) { + logger.error({ err, backupDir: config.databaseBackupDir }, "Automatic database backup failed"); + } finally { + backupInFlight = false; + } + }; + + logger.info( + { + intervalMinutes: config.databaseBackupIntervalMinutes, + retentionDays: config.databaseBackupRetentionDays, + backupDir: config.databaseBackupDir, + }, + "Automatic database backups enabled", + ); + setInterval(() => { + void runScheduledBackup(); + }, backupIntervalMs); +} + server.listen(listenPort, config.host, () => { logger.info(`Server listening on ${config.host}:${listenPort}`); if (process.env.PAPERCLIP_OPEN_ON_LISTEN === "true") { @@ -515,6 +568,10 @@ server.listen(listenPort, config.host, () => { migrationSummary, heartbeatSchedulerEnabled: config.heartbeatSchedulerEnabled, heartbeatSchedulerIntervalMs: config.heartbeatSchedulerIntervalMs, + databaseBackupEnabled: config.databaseBackupEnabled, + databaseBackupIntervalMinutes: config.databaseBackupIntervalMinutes, + databaseBackupRetentionDays: config.databaseBackupRetentionDays, + databaseBackupDir: config.databaseBackupDir, }); const boardClaimUrl = getBoardClaimWarningUrl(config.host, listenPort); diff --git a/server/src/startup-banner.ts b/server/src/startup-banner.ts index fb22cb74..1a52731b 100644 --- a/server/src/startup-banner.ts +++ b/server/src/startup-banner.ts @@ -29,6 +29,10 @@ type StartupBannerOptions = { migrationSummary: string; heartbeatSchedulerEnabled: boolean; heartbeatSchedulerIntervalMs: number; + databaseBackupEnabled: boolean; + databaseBackupIntervalMinutes: number; + databaseBackupRetentionDays: number; + databaseBackupDir: string; }; const ansi = { @@ -125,6 +129,9 @@ export function printStartupBanner(opts: StartupBannerOptions): void { const heartbeat = opts.heartbeatSchedulerEnabled ? `enabled ${color(`(${opts.heartbeatSchedulerIntervalMs}ms)`, "dim")}` : color("disabled", "yellow"); + const dbBackup = opts.databaseBackupEnabled + ? `enabled ${color(`(every ${opts.databaseBackupIntervalMinutes}m, keep ${opts.databaseBackupRetentionDays}d)`, "dim")}` + : color("disabled", "yellow"); const art = [ color("██████╗ █████╗ ██████╗ ███████╗██████╗ ██████╗██╗ ██╗██████╗ ", "cyan"), @@ -154,6 +161,8 @@ export function printStartupBanner(opts: StartupBannerOptions): void { : color(agentJwtSecret.message, "yellow"), ), row("Heartbeat", heartbeat), + row("DB Backup", dbBackup), + row("Backup Dir", opts.databaseBackupDir), row("Config", configPath), agentJwtSecret.status === "warn" ? color(" ───────────────────────────────────────────────────────", "yellow")