Add configurable automatic database backup scheduling

This commit is contained in:
Dotta
2026-03-04 18:03:23 -06:00
parent f6a09bcbea
commit c145074daf
17 changed files with 722 additions and 351 deletions

View File

@@ -15,6 +15,7 @@ import {
type StorageProvider,
} from "@paperclipai/shared";
import {
resolveDefaultBackupDir,
resolveDefaultEmbeddedPostgresDir,
resolveDefaultSecretsKeyFilePath,
resolveDefaultStorageDir,
@@ -40,6 +41,10 @@ export interface Config {
databaseUrl: string | undefined;
embeddedPostgresDataDir: string;
embeddedPostgresPort: number;
databaseBackupEnabled: boolean;
databaseBackupIntervalMinutes: number;
databaseBackupRetentionDays: number;
databaseBackupDir: string;
serveUi: boolean;
uiDevMiddleware: boolean;
secretsProvider: SecretProvider;
@@ -66,6 +71,7 @@ export function loadConfig(): Config {
fileDatabaseMode === "postgres"
? fileConfig?.database.connectionString
: undefined;
const fileDatabaseBackup = fileConfig?.database.backup;
const fileSecrets = fileConfig?.secrets;
const fileStorage = fileConfig?.storage;
const strictModeFromEnv = process.env.PAPERCLIP_SECRETS_STRICT_MODE;
@@ -148,6 +154,27 @@ export function loadConfig(): Config {
companyDeletionEnvRaw !== undefined
? companyDeletionEnvRaw === "true"
: deploymentMode === "local_trusted";
const databaseBackupEnabled =
process.env.PAPERCLIP_DB_BACKUP_ENABLED !== undefined
? process.env.PAPERCLIP_DB_BACKUP_ENABLED === "true"
: (fileDatabaseBackup?.enabled ?? true);
const databaseBackupIntervalMinutes = Math.max(
1,
Number(process.env.PAPERCLIP_DB_BACKUP_INTERVAL_MINUTES) ||
fileDatabaseBackup?.intervalMinutes ||
60,
);
const databaseBackupRetentionDays = Math.max(
1,
Number(process.env.PAPERCLIP_DB_BACKUP_RETENTION_DAYS) ||
fileDatabaseBackup?.retentionDays ||
30,
);
const databaseBackupDir = resolveHomeAwarePath(
process.env.PAPERCLIP_DB_BACKUP_DIR ??
fileDatabaseBackup?.dir ??
resolveDefaultBackupDir(),
);
return {
deploymentMode,
@@ -163,6 +190,10 @@ export function loadConfig(): Config {
fileConfig?.database.embeddedPostgresDataDir ?? resolveDefaultEmbeddedPostgresDir(),
),
embeddedPostgresPort: fileConfig?.database.embeddedPostgresPort ?? 54329,
databaseBackupEnabled,
databaseBackupIntervalMinutes,
databaseBackupRetentionDays,
databaseBackupDir,
serveUi:
process.env.SERVE_UI !== undefined
? process.env.SERVE_UI === "true"

View File

@@ -49,6 +49,10 @@ export function resolveDefaultStorageDir(): string {
return path.resolve(resolvePaperclipInstanceRoot(), "data", "storage");
}
export function resolveDefaultBackupDir(): string {
return path.resolve(resolvePaperclipInstanceRoot(), "data", "backups");
}
export function resolveDefaultAgentWorkspaceDir(agentId: string): string {
const trimmed = agentId.trim();
if (!PATH_SEGMENT_RE.test(trimmed)) {

View File

@@ -12,6 +12,8 @@ import {
inspectMigrations,
applyPendingMigrations,
reconcilePendingMigrationHistory,
formatDatabaseBackupResult,
runDatabaseBackup,
authUsers,
companies,
companyMemberships,
@@ -220,6 +222,7 @@ let db;
let embeddedPostgres: EmbeddedPostgresInstance | null = null;
let embeddedPostgresStartedByThisProcess = false;
let migrationSummary: MigrationSummary = "skipped";
let activeDatabaseConnectionString: string;
let startupDbInfo:
| { mode: "external-postgres"; connectionString: string }
| { mode: "embedded-postgres"; dataDir: string; port: number };
@@ -228,6 +231,7 @@ if (config.databaseUrl) {
db = createDb(config.databaseUrl);
logger.info("Using external PostgreSQL via DATABASE_URL/config");
activeDatabaseConnectionString = config.databaseUrl;
startupDbInfo = { mode: "external-postgres", connectionString: config.databaseUrl };
} else {
const moduleName = "embedded-postgres";
@@ -364,6 +368,7 @@ if (config.databaseUrl) {
db = createDb(embeddedConnectionString);
logger.info("Embedded PostgreSQL ready");
activeDatabaseConnectionString = embeddedConnectionString;
startupDbInfo = { mode: "embedded-postgres", dataDir, port };
}
@@ -489,6 +494,54 @@ if (config.heartbeatSchedulerEnabled) {
}, config.heartbeatSchedulerIntervalMs);
}
if (config.databaseBackupEnabled) {
const backupIntervalMs = config.databaseBackupIntervalMinutes * 60 * 1000;
let backupInFlight = false;
const runScheduledBackup = async () => {
if (backupInFlight) {
logger.warn("Skipping scheduled database backup because a previous backup is still running");
return;
}
backupInFlight = true;
try {
const result = await runDatabaseBackup({
connectionString: activeDatabaseConnectionString,
backupDir: config.databaseBackupDir,
retentionDays: config.databaseBackupRetentionDays,
filenamePrefix: "paperclip",
});
logger.info(
{
backupFile: result.backupFile,
sizeBytes: result.sizeBytes,
prunedCount: result.prunedCount,
backupDir: config.databaseBackupDir,
retentionDays: config.databaseBackupRetentionDays,
},
`Automatic database backup complete: ${formatDatabaseBackupResult(result)}`,
);
} catch (err) {
logger.error({ err, backupDir: config.databaseBackupDir }, "Automatic database backup failed");
} finally {
backupInFlight = false;
}
};
logger.info(
{
intervalMinutes: config.databaseBackupIntervalMinutes,
retentionDays: config.databaseBackupRetentionDays,
backupDir: config.databaseBackupDir,
},
"Automatic database backups enabled",
);
setInterval(() => {
void runScheduledBackup();
}, backupIntervalMs);
}
server.listen(listenPort, config.host, () => {
logger.info(`Server listening on ${config.host}:${listenPort}`);
if (process.env.PAPERCLIP_OPEN_ON_LISTEN === "true") {
@@ -515,6 +568,10 @@ server.listen(listenPort, config.host, () => {
migrationSummary,
heartbeatSchedulerEnabled: config.heartbeatSchedulerEnabled,
heartbeatSchedulerIntervalMs: config.heartbeatSchedulerIntervalMs,
databaseBackupEnabled: config.databaseBackupEnabled,
databaseBackupIntervalMinutes: config.databaseBackupIntervalMinutes,
databaseBackupRetentionDays: config.databaseBackupRetentionDays,
databaseBackupDir: config.databaseBackupDir,
});
const boardClaimUrl = getBoardClaimWarningUrl(config.host, listenPort);

View File

@@ -29,6 +29,10 @@ type StartupBannerOptions = {
migrationSummary: string;
heartbeatSchedulerEnabled: boolean;
heartbeatSchedulerIntervalMs: number;
databaseBackupEnabled: boolean;
databaseBackupIntervalMinutes: number;
databaseBackupRetentionDays: number;
databaseBackupDir: string;
};
const ansi = {
@@ -125,6 +129,9 @@ export function printStartupBanner(opts: StartupBannerOptions): void {
const heartbeat = opts.heartbeatSchedulerEnabled
? `enabled ${color(`(${opts.heartbeatSchedulerIntervalMs}ms)`, "dim")}`
: color("disabled", "yellow");
const dbBackup = opts.databaseBackupEnabled
? `enabled ${color(`(every ${opts.databaseBackupIntervalMinutes}m, keep ${opts.databaseBackupRetentionDays}d)`, "dim")}`
: color("disabled", "yellow");
const art = [
color("██████╗ █████╗ ██████╗ ███████╗██████╗ ██████╗██╗ ██╗██████╗ ", "cyan"),
@@ -154,6 +161,8 @@ export function printStartupBanner(opts: StartupBannerOptions): void {
: color(agentJwtSecret.message, "yellow"),
),
row("Heartbeat", heartbeat),
row("DB Backup", dbBackup),
row("Backup Dir", opts.databaseBackupDir),
row("Config", configPath),
agentJwtSecret.status === "warn"
? color(" ───────────────────────────────────────────────────────", "yellow")