Add configurable automatic database backup scheduling
This commit is contained in:
@@ -15,6 +15,7 @@ import {
|
||||
type StorageProvider,
|
||||
} from "@paperclipai/shared";
|
||||
import {
|
||||
resolveDefaultBackupDir,
|
||||
resolveDefaultEmbeddedPostgresDir,
|
||||
resolveDefaultSecretsKeyFilePath,
|
||||
resolveDefaultStorageDir,
|
||||
@@ -40,6 +41,10 @@ export interface Config {
|
||||
databaseUrl: string | undefined;
|
||||
embeddedPostgresDataDir: string;
|
||||
embeddedPostgresPort: number;
|
||||
databaseBackupEnabled: boolean;
|
||||
databaseBackupIntervalMinutes: number;
|
||||
databaseBackupRetentionDays: number;
|
||||
databaseBackupDir: string;
|
||||
serveUi: boolean;
|
||||
uiDevMiddleware: boolean;
|
||||
secretsProvider: SecretProvider;
|
||||
@@ -66,6 +71,7 @@ export function loadConfig(): Config {
|
||||
fileDatabaseMode === "postgres"
|
||||
? fileConfig?.database.connectionString
|
||||
: undefined;
|
||||
const fileDatabaseBackup = fileConfig?.database.backup;
|
||||
const fileSecrets = fileConfig?.secrets;
|
||||
const fileStorage = fileConfig?.storage;
|
||||
const strictModeFromEnv = process.env.PAPERCLIP_SECRETS_STRICT_MODE;
|
||||
@@ -148,6 +154,27 @@ export function loadConfig(): Config {
|
||||
companyDeletionEnvRaw !== undefined
|
||||
? companyDeletionEnvRaw === "true"
|
||||
: deploymentMode === "local_trusted";
|
||||
const databaseBackupEnabled =
|
||||
process.env.PAPERCLIP_DB_BACKUP_ENABLED !== undefined
|
||||
? process.env.PAPERCLIP_DB_BACKUP_ENABLED === "true"
|
||||
: (fileDatabaseBackup?.enabled ?? true);
|
||||
const databaseBackupIntervalMinutes = Math.max(
|
||||
1,
|
||||
Number(process.env.PAPERCLIP_DB_BACKUP_INTERVAL_MINUTES) ||
|
||||
fileDatabaseBackup?.intervalMinutes ||
|
||||
60,
|
||||
);
|
||||
const databaseBackupRetentionDays = Math.max(
|
||||
1,
|
||||
Number(process.env.PAPERCLIP_DB_BACKUP_RETENTION_DAYS) ||
|
||||
fileDatabaseBackup?.retentionDays ||
|
||||
30,
|
||||
);
|
||||
const databaseBackupDir = resolveHomeAwarePath(
|
||||
process.env.PAPERCLIP_DB_BACKUP_DIR ??
|
||||
fileDatabaseBackup?.dir ??
|
||||
resolveDefaultBackupDir(),
|
||||
);
|
||||
|
||||
return {
|
||||
deploymentMode,
|
||||
@@ -163,6 +190,10 @@ export function loadConfig(): Config {
|
||||
fileConfig?.database.embeddedPostgresDataDir ?? resolveDefaultEmbeddedPostgresDir(),
|
||||
),
|
||||
embeddedPostgresPort: fileConfig?.database.embeddedPostgresPort ?? 54329,
|
||||
databaseBackupEnabled,
|
||||
databaseBackupIntervalMinutes,
|
||||
databaseBackupRetentionDays,
|
||||
databaseBackupDir,
|
||||
serveUi:
|
||||
process.env.SERVE_UI !== undefined
|
||||
? process.env.SERVE_UI === "true"
|
||||
|
||||
@@ -49,6 +49,10 @@ export function resolveDefaultStorageDir(): string {
|
||||
return path.resolve(resolvePaperclipInstanceRoot(), "data", "storage");
|
||||
}
|
||||
|
||||
export function resolveDefaultBackupDir(): string {
|
||||
return path.resolve(resolvePaperclipInstanceRoot(), "data", "backups");
|
||||
}
|
||||
|
||||
export function resolveDefaultAgentWorkspaceDir(agentId: string): string {
|
||||
const trimmed = agentId.trim();
|
||||
if (!PATH_SEGMENT_RE.test(trimmed)) {
|
||||
|
||||
@@ -12,6 +12,8 @@ import {
|
||||
inspectMigrations,
|
||||
applyPendingMigrations,
|
||||
reconcilePendingMigrationHistory,
|
||||
formatDatabaseBackupResult,
|
||||
runDatabaseBackup,
|
||||
authUsers,
|
||||
companies,
|
||||
companyMemberships,
|
||||
@@ -220,6 +222,7 @@ let db;
|
||||
let embeddedPostgres: EmbeddedPostgresInstance | null = null;
|
||||
let embeddedPostgresStartedByThisProcess = false;
|
||||
let migrationSummary: MigrationSummary = "skipped";
|
||||
let activeDatabaseConnectionString: string;
|
||||
let startupDbInfo:
|
||||
| { mode: "external-postgres"; connectionString: string }
|
||||
| { mode: "embedded-postgres"; dataDir: string; port: number };
|
||||
@@ -228,6 +231,7 @@ if (config.databaseUrl) {
|
||||
|
||||
db = createDb(config.databaseUrl);
|
||||
logger.info("Using external PostgreSQL via DATABASE_URL/config");
|
||||
activeDatabaseConnectionString = config.databaseUrl;
|
||||
startupDbInfo = { mode: "external-postgres", connectionString: config.databaseUrl };
|
||||
} else {
|
||||
const moduleName = "embedded-postgres";
|
||||
@@ -364,6 +368,7 @@ if (config.databaseUrl) {
|
||||
|
||||
db = createDb(embeddedConnectionString);
|
||||
logger.info("Embedded PostgreSQL ready");
|
||||
activeDatabaseConnectionString = embeddedConnectionString;
|
||||
startupDbInfo = { mode: "embedded-postgres", dataDir, port };
|
||||
}
|
||||
|
||||
@@ -489,6 +494,54 @@ if (config.heartbeatSchedulerEnabled) {
|
||||
}, config.heartbeatSchedulerIntervalMs);
|
||||
}
|
||||
|
||||
if (config.databaseBackupEnabled) {
|
||||
const backupIntervalMs = config.databaseBackupIntervalMinutes * 60 * 1000;
|
||||
let backupInFlight = false;
|
||||
|
||||
const runScheduledBackup = async () => {
|
||||
if (backupInFlight) {
|
||||
logger.warn("Skipping scheduled database backup because a previous backup is still running");
|
||||
return;
|
||||
}
|
||||
|
||||
backupInFlight = true;
|
||||
try {
|
||||
const result = await runDatabaseBackup({
|
||||
connectionString: activeDatabaseConnectionString,
|
||||
backupDir: config.databaseBackupDir,
|
||||
retentionDays: config.databaseBackupRetentionDays,
|
||||
filenamePrefix: "paperclip",
|
||||
});
|
||||
logger.info(
|
||||
{
|
||||
backupFile: result.backupFile,
|
||||
sizeBytes: result.sizeBytes,
|
||||
prunedCount: result.prunedCount,
|
||||
backupDir: config.databaseBackupDir,
|
||||
retentionDays: config.databaseBackupRetentionDays,
|
||||
},
|
||||
`Automatic database backup complete: ${formatDatabaseBackupResult(result)}`,
|
||||
);
|
||||
} catch (err) {
|
||||
logger.error({ err, backupDir: config.databaseBackupDir }, "Automatic database backup failed");
|
||||
} finally {
|
||||
backupInFlight = false;
|
||||
}
|
||||
};
|
||||
|
||||
logger.info(
|
||||
{
|
||||
intervalMinutes: config.databaseBackupIntervalMinutes,
|
||||
retentionDays: config.databaseBackupRetentionDays,
|
||||
backupDir: config.databaseBackupDir,
|
||||
},
|
||||
"Automatic database backups enabled",
|
||||
);
|
||||
setInterval(() => {
|
||||
void runScheduledBackup();
|
||||
}, backupIntervalMs);
|
||||
}
|
||||
|
||||
server.listen(listenPort, config.host, () => {
|
||||
logger.info(`Server listening on ${config.host}:${listenPort}`);
|
||||
if (process.env.PAPERCLIP_OPEN_ON_LISTEN === "true") {
|
||||
@@ -515,6 +568,10 @@ server.listen(listenPort, config.host, () => {
|
||||
migrationSummary,
|
||||
heartbeatSchedulerEnabled: config.heartbeatSchedulerEnabled,
|
||||
heartbeatSchedulerIntervalMs: config.heartbeatSchedulerIntervalMs,
|
||||
databaseBackupEnabled: config.databaseBackupEnabled,
|
||||
databaseBackupIntervalMinutes: config.databaseBackupIntervalMinutes,
|
||||
databaseBackupRetentionDays: config.databaseBackupRetentionDays,
|
||||
databaseBackupDir: config.databaseBackupDir,
|
||||
});
|
||||
|
||||
const boardClaimUrl = getBoardClaimWarningUrl(config.host, listenPort);
|
||||
|
||||
@@ -29,6 +29,10 @@ type StartupBannerOptions = {
|
||||
migrationSummary: string;
|
||||
heartbeatSchedulerEnabled: boolean;
|
||||
heartbeatSchedulerIntervalMs: number;
|
||||
databaseBackupEnabled: boolean;
|
||||
databaseBackupIntervalMinutes: number;
|
||||
databaseBackupRetentionDays: number;
|
||||
databaseBackupDir: string;
|
||||
};
|
||||
|
||||
const ansi = {
|
||||
@@ -125,6 +129,9 @@ export function printStartupBanner(opts: StartupBannerOptions): void {
|
||||
const heartbeat = opts.heartbeatSchedulerEnabled
|
||||
? `enabled ${color(`(${opts.heartbeatSchedulerIntervalMs}ms)`, "dim")}`
|
||||
: color("disabled", "yellow");
|
||||
const dbBackup = opts.databaseBackupEnabled
|
||||
? `enabled ${color(`(every ${opts.databaseBackupIntervalMinutes}m, keep ${opts.databaseBackupRetentionDays}d)`, "dim")}`
|
||||
: color("disabled", "yellow");
|
||||
|
||||
const art = [
|
||||
color("██████╗ █████╗ ██████╗ ███████╗██████╗ ██████╗██╗ ██╗██████╗ ", "cyan"),
|
||||
@@ -154,6 +161,8 @@ export function printStartupBanner(opts: StartupBannerOptions): void {
|
||||
: color(agentJwtSecret.message, "yellow"),
|
||||
),
|
||||
row("Heartbeat", heartbeat),
|
||||
row("DB Backup", dbBackup),
|
||||
row("Backup Dir", opts.databaseBackupDir),
|
||||
row("Config", configPath),
|
||||
agentJwtSecret.status === "warn"
|
||||
? color(" ───────────────────────────────────────────────────────", "yellow")
|
||||
|
||||
Reference in New Issue
Block a user