Files
paperclip/server/src/index.ts
2026-03-10 10:58:38 -05:00

694 lines
25 KiB
TypeScript

/// <reference path="./types/express.d.ts" />
import { existsSync, readFileSync, rmSync } from "node:fs";
import { createServer } from "node:http";
import { resolve } from "node:path";
import { createInterface } from "node:readline/promises";
import { stdin, stdout } from "node:process";
import { pathToFileURL } from "node:url";
import type { Request as ExpressRequest, RequestHandler } from "express";
import { and, eq } from "drizzle-orm";
import {
createDb,
ensurePostgresDatabase,
inspectMigrations,
applyPendingMigrations,
reconcilePendingMigrationHistory,
formatDatabaseBackupResult,
runDatabaseBackup,
authUsers,
companies,
companyMemberships,
instanceUserRoles,
} from "@paperclipai/db";
import detectPort from "detect-port";
import { createApp } from "./app.js";
import { loadConfig } from "./config.js";
import { logger } from "./middleware/logger.js";
import { setupLiveEventsWebSocketServer } from "./realtime/live-events-ws.js";
import { heartbeatService, reconcilePersistedRuntimeServicesOnStartup } from "./services/index.js";
import { createStorageServiceFromConfig } from "./storage/index.js";
import { printStartupBanner } from "./startup-banner.js";
import { getBoardClaimWarningUrl, initializeBoardClaimChallenge } from "./board-claim.js";
type BetterAuthSessionUser = {
id: string;
email?: string | null;
name?: string | null;
};
type BetterAuthSessionResult = {
session: { id: string; userId: string } | null;
user: BetterAuthSessionUser | null;
};
type EmbeddedPostgresInstance = {
initialise(): Promise<void>;
start(): Promise<void>;
stop(): Promise<void>;
};
type EmbeddedPostgresCtor = new (opts: {
databaseDir: string;
user: string;
password: string;
port: number;
persistent: boolean;
onLog?: (message: unknown) => void;
onError?: (message: unknown) => void;
}) => EmbeddedPostgresInstance;
export interface StartedServer {
server: ReturnType<typeof createServer>;
host: string;
listenPort: number;
apiUrl: string;
databaseUrl: string;
}
export async function startServer(): Promise<StartedServer> {
const config = loadConfig();
if (process.env.PAPERCLIP_SECRETS_PROVIDER === undefined) {
process.env.PAPERCLIP_SECRETS_PROVIDER = config.secretsProvider;
}
if (process.env.PAPERCLIP_SECRETS_STRICT_MODE === undefined) {
process.env.PAPERCLIP_SECRETS_STRICT_MODE = config.secretsStrictMode ? "true" : "false";
}
if (process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE === undefined) {
process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE = config.secretsMasterKeyFilePath;
}
type MigrationSummary =
| "skipped"
| "already applied"
| "applied (empty database)"
| "applied (pending migrations)"
| "pending migrations skipped";
function formatPendingMigrationSummary(migrations: string[]): string {
if (migrations.length === 0) return "none";
return migrations.length > 3
? `${migrations.slice(0, 3).join(", ")} (+${migrations.length - 3} more)`
: migrations.join(", ");
}
async function promptApplyMigrations(migrations: string[]): Promise<boolean> {
if (process.env.PAPERCLIP_MIGRATION_PROMPT === "never") return false;
if (process.env.PAPERCLIP_MIGRATION_AUTO_APPLY === "true") return true;
if (!stdin.isTTY || !stdout.isTTY) return true;
const prompt = createInterface({ input: stdin, output: stdout });
try {
const answer = (await prompt.question(
`Apply pending migrations (${formatPendingMigrationSummary(migrations)}) now? (y/N): `,
)).trim().toLowerCase();
return answer === "y" || answer === "yes";
} finally {
prompt.close();
}
}
type EnsureMigrationsOptions = {
autoApply?: boolean;
};
async function ensureMigrations(
connectionString: string,
label: string,
opts?: EnsureMigrationsOptions,
): Promise<MigrationSummary> {
const autoApply = opts?.autoApply === true;
let state = await inspectMigrations(connectionString);
if (state.status === "needsMigrations" && state.reason === "pending-migrations") {
const repair = await reconcilePendingMigrationHistory(connectionString);
if (repair.repairedMigrations.length > 0) {
logger.warn(
{ repairedMigrations: repair.repairedMigrations },
`${label} had drifted migration history; repaired migration journal entries from existing schema state.`,
);
state = await inspectMigrations(connectionString);
if (state.status === "upToDate") return "already applied";
}
}
if (state.status === "upToDate") return "already applied";
if (state.status === "needsMigrations" && state.reason === "no-migration-journal-non-empty-db") {
logger.warn(
{ tableCount: state.tableCount },
`${label} has existing tables but no migration journal. Run migrations manually to sync schema.`,
);
const apply = autoApply ? true : await promptApplyMigrations(state.pendingMigrations);
if (!apply) {
logger.warn(
{ pendingMigrations: state.pendingMigrations },
`${label} has pending migrations; continuing without applying. Run pnpm db:migrate to apply before startup.`,
);
return "pending migrations skipped";
}
logger.info({ pendingMigrations: state.pendingMigrations }, `Applying ${state.pendingMigrations.length} pending migrations for ${label}`);
await applyPendingMigrations(connectionString);
return "applied (pending migrations)";
}
const apply = autoApply ? true : await promptApplyMigrations(state.pendingMigrations);
if (!apply) {
logger.warn(
{ pendingMigrations: state.pendingMigrations },
`${label} has pending migrations; continuing without applying. Run pnpm db:migrate to apply before startup.`,
);
return "pending migrations skipped";
}
logger.info({ pendingMigrations: state.pendingMigrations }, `Applying ${state.pendingMigrations.length} pending migrations for ${label}`);
await applyPendingMigrations(connectionString);
return "applied (pending migrations)";
}
function isLoopbackHost(host: string): boolean {
const normalized = host.trim().toLowerCase();
return normalized === "127.0.0.1" || normalized === "localhost" || normalized === "::1";
}
const LOCAL_BOARD_USER_ID = "local-board";
const LOCAL_BOARD_USER_EMAIL = "local@paperclip.local";
const LOCAL_BOARD_USER_NAME = "Board";
async function ensureLocalTrustedBoardPrincipal(db: any): Promise<void> {
const now = new Date();
const existingUser = await db
.select({ id: authUsers.id })
.from(authUsers)
.where(eq(authUsers.id, LOCAL_BOARD_USER_ID))
.then((rows: Array<{ id: string }>) => rows[0] ?? null);
if (!existingUser) {
await db.insert(authUsers).values({
id: LOCAL_BOARD_USER_ID,
name: LOCAL_BOARD_USER_NAME,
email: LOCAL_BOARD_USER_EMAIL,
emailVerified: true,
image: null,
createdAt: now,
updatedAt: now,
});
}
const role = await db
.select({ id: instanceUserRoles.id })
.from(instanceUserRoles)
.where(and(eq(instanceUserRoles.userId, LOCAL_BOARD_USER_ID), eq(instanceUserRoles.role, "instance_admin")))
.then((rows: Array<{ id: string }>) => rows[0] ?? null);
if (!role) {
await db.insert(instanceUserRoles).values({
userId: LOCAL_BOARD_USER_ID,
role: "instance_admin",
});
}
const companyRows = await db.select({ id: companies.id }).from(companies);
for (const company of companyRows) {
const membership = await db
.select({ id: companyMemberships.id })
.from(companyMemberships)
.where(
and(
eq(companyMemberships.companyId, company.id),
eq(companyMemberships.principalType, "user"),
eq(companyMemberships.principalId, LOCAL_BOARD_USER_ID),
),
)
.then((rows: Array<{ id: string }>) => rows[0] ?? null);
if (membership) continue;
await db.insert(companyMemberships).values({
companyId: company.id,
principalType: "user",
principalId: LOCAL_BOARD_USER_ID,
status: "active",
membershipRole: "owner",
});
}
}
let db;
let embeddedPostgres: EmbeddedPostgresInstance | null = null;
let embeddedPostgresStartedByThisProcess = false;
let migrationSummary: MigrationSummary = "skipped";
let activeDatabaseConnectionString: string;
let startupDbInfo:
| { mode: "external-postgres"; connectionString: string }
| { mode: "embedded-postgres"; dataDir: string; port: number };
if (config.databaseUrl) {
migrationSummary = await ensureMigrations(config.databaseUrl, "PostgreSQL");
db = createDb(config.databaseUrl);
logger.info("Using external PostgreSQL via DATABASE_URL/config");
activeDatabaseConnectionString = config.databaseUrl;
startupDbInfo = { mode: "external-postgres", connectionString: config.databaseUrl };
} else {
const moduleName = "embedded-postgres";
let EmbeddedPostgres: EmbeddedPostgresCtor;
try {
const mod = await import(moduleName);
EmbeddedPostgres = mod.default as EmbeddedPostgresCtor;
} catch {
throw new Error(
"Embedded PostgreSQL mode requires dependency `embedded-postgres`. Reinstall dependencies (without omitting required packages), or set DATABASE_URL for external Postgres.",
);
}
const dataDir = resolve(config.embeddedPostgresDataDir);
const configuredPort = config.embeddedPostgresPort;
let port = configuredPort;
const embeddedPostgresLogBuffer: string[] = [];
const EMBEDDED_POSTGRES_LOG_BUFFER_LIMIT = 120;
const verboseEmbeddedPostgresLogs = process.env.PAPERCLIP_EMBEDDED_POSTGRES_VERBOSE === "true";
const appendEmbeddedPostgresLog = (message: unknown) => {
const text = typeof message === "string" ? message : message instanceof Error ? message.message : String(message ?? "");
for (const lineRaw of text.split(/\r?\n/)) {
const line = lineRaw.trim();
if (!line) continue;
embeddedPostgresLogBuffer.push(line);
if (embeddedPostgresLogBuffer.length > EMBEDDED_POSTGRES_LOG_BUFFER_LIMIT) {
embeddedPostgresLogBuffer.splice(0, embeddedPostgresLogBuffer.length - EMBEDDED_POSTGRES_LOG_BUFFER_LIMIT);
}
if (verboseEmbeddedPostgresLogs) {
logger.info({ embeddedPostgresLog: line }, "embedded-postgres");
}
}
};
const logEmbeddedPostgresFailure = (phase: "initialise" | "start", err: unknown) => {
if (embeddedPostgresLogBuffer.length > 0) {
logger.error(
{
phase,
recentLogs: embeddedPostgresLogBuffer,
err,
},
"Embedded PostgreSQL failed; showing buffered startup logs",
);
}
};
if (config.databaseMode === "postgres") {
logger.warn("Database mode is postgres but no connection string was set; falling back to embedded PostgreSQL");
}
const clusterVersionFile = resolve(dataDir, "PG_VERSION");
const clusterAlreadyInitialized = existsSync(clusterVersionFile);
const postmasterPidFile = resolve(dataDir, "postmaster.pid");
const isPidRunning = (pid: number): boolean => {
try {
process.kill(pid, 0);
return true;
} catch {
return false;
}
};
const getRunningPid = (): number | null => {
if (!existsSync(postmasterPidFile)) return null;
try {
const pidLine = readFileSync(postmasterPidFile, "utf8").split("\n")[0]?.trim();
const pid = Number(pidLine);
if (!Number.isInteger(pid) || pid <= 0) return null;
if (!isPidRunning(pid)) return null;
return pid;
} catch {
return null;
}
};
const runningPid = getRunningPid();
if (runningPid) {
logger.warn(`Embedded PostgreSQL already running; reusing existing process (pid=${runningPid}, port=${port})`);
} else {
const detectedPort = await detectPort(configuredPort);
if (detectedPort !== configuredPort) {
logger.warn(`Embedded PostgreSQL port is in use; using next free port (requestedPort=${configuredPort}, selectedPort=${detectedPort})`);
}
port = detectedPort;
logger.info(`Using embedded PostgreSQL because no DATABASE_URL set (dataDir=${dataDir}, port=${port})`);
embeddedPostgres = new EmbeddedPostgres({
databaseDir: dataDir,
user: "paperclip",
password: "paperclip",
port,
persistent: true,
onLog: appendEmbeddedPostgresLog,
onError: appendEmbeddedPostgresLog,
});
if (!clusterAlreadyInitialized) {
try {
await embeddedPostgres.initialise();
} catch (err) {
logEmbeddedPostgresFailure("initialise", err);
throw err;
}
} else {
logger.info(`Embedded PostgreSQL cluster already exists (${clusterVersionFile}); skipping init`);
}
if (existsSync(postmasterPidFile)) {
logger.warn("Removing stale embedded PostgreSQL lock file");
rmSync(postmasterPidFile, { force: true });
}
try {
await embeddedPostgres.start();
} catch (err) {
logEmbeddedPostgresFailure("start", err);
throw err;
}
embeddedPostgresStartedByThisProcess = true;
}
const embeddedAdminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/postgres`;
const dbStatus = await ensurePostgresDatabase(embeddedAdminConnectionString, "paperclip");
if (dbStatus === "created") {
logger.info("Created embedded PostgreSQL database: paperclip");
}
const embeddedConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`;
const shouldAutoApplyFirstRunMigrations = !clusterAlreadyInitialized || dbStatus === "created";
if (shouldAutoApplyFirstRunMigrations) {
logger.info("Detected first-run embedded PostgreSQL setup; applying pending migrations automatically");
}
migrationSummary = await ensureMigrations(embeddedConnectionString, "Embedded PostgreSQL", {
autoApply: shouldAutoApplyFirstRunMigrations,
});
db = createDb(embeddedConnectionString);
logger.info("Embedded PostgreSQL ready");
activeDatabaseConnectionString = embeddedConnectionString;
startupDbInfo = { mode: "embedded-postgres", dataDir, port };
}
if (config.deploymentMode === "local_trusted" && !isLoopbackHost(config.host)) {
throw new Error(
`local_trusted mode requires loopback host binding (received: ${config.host}). ` +
"Use authenticated mode for non-loopback deployments.",
);
}
if (config.deploymentMode === "local_trusted" && config.deploymentExposure !== "private") {
throw new Error("local_trusted mode only supports private exposure");
}
if (config.deploymentMode === "authenticated") {
if (config.authBaseUrlMode === "explicit" && !config.authPublicBaseUrl) {
throw new Error("auth.baseUrlMode=explicit requires auth.publicBaseUrl");
}
if (config.deploymentExposure === "public") {
if (config.authBaseUrlMode !== "explicit") {
throw new Error("authenticated public exposure requires auth.baseUrlMode=explicit");
}
if (!config.authPublicBaseUrl) {
throw new Error("authenticated public exposure requires auth.publicBaseUrl");
}
}
}
let authReady = config.deploymentMode === "local_trusted";
let betterAuthHandler: RequestHandler | undefined;
let resolveSession:
| ((req: ExpressRequest) => Promise<BetterAuthSessionResult | null>)
| undefined;
let resolveSessionFromHeaders:
| ((headers: Headers) => Promise<BetterAuthSessionResult | null>)
| undefined;
if (config.deploymentMode === "local_trusted") {
await ensureLocalTrustedBoardPrincipal(db as any);
}
if (config.deploymentMode === "authenticated") {
const {
createBetterAuthHandler,
createBetterAuthInstance,
deriveAuthTrustedOrigins,
resolveBetterAuthSession,
resolveBetterAuthSessionFromHeaders,
} = await import("./auth/better-auth.js");
const betterAuthSecret =
process.env.BETTER_AUTH_SECRET?.trim() ?? process.env.PAPERCLIP_AGENT_JWT_SECRET?.trim();
if (!betterAuthSecret) {
throw new Error(
"authenticated mode requires BETTER_AUTH_SECRET (or PAPERCLIP_AGENT_JWT_SECRET) to be set",
);
}
const derivedTrustedOrigins = deriveAuthTrustedOrigins(config);
const envTrustedOrigins = (process.env.BETTER_AUTH_TRUSTED_ORIGINS ?? "")
.split(",")
.map((value) => value.trim())
.filter((value) => value.length > 0);
const effectiveTrustedOrigins = Array.from(new Set([...derivedTrustedOrigins, ...envTrustedOrigins]));
logger.info(
{
authBaseUrlMode: config.authBaseUrlMode,
authPublicBaseUrl: config.authPublicBaseUrl ?? null,
trustedOrigins: effectiveTrustedOrigins,
trustedOriginsSource: {
derived: derivedTrustedOrigins.length,
env: envTrustedOrigins.length,
},
},
"Authenticated mode auth origin configuration",
);
const auth = createBetterAuthInstance(db as any, config, effectiveTrustedOrigins);
betterAuthHandler = createBetterAuthHandler(auth);
resolveSession = (req) => resolveBetterAuthSession(auth, req);
resolveSessionFromHeaders = (headers) => resolveBetterAuthSessionFromHeaders(auth, headers);
await initializeBoardClaimChallenge(db as any, { deploymentMode: config.deploymentMode });
authReady = true;
}
const listenPort = await detectPort(config.port);
const uiMode = config.uiDevMiddleware ? "vite-dev" : config.serveUi ? "static" : "none";
const storageService = createStorageServiceFromConfig(config);
const app = await createApp(db as any, {
uiMode,
serverPort: listenPort,
storageService,
deploymentMode: config.deploymentMode,
deploymentExposure: config.deploymentExposure,
allowedHostnames: config.allowedHostnames,
bindHost: config.host,
authReady,
companyDeletionEnabled: config.companyDeletionEnabled,
betterAuthHandler,
resolveSession,
});
const server = createServer(app as unknown as Parameters<typeof createServer>[0]);
if (listenPort !== config.port) {
logger.warn(`Requested port is busy; using next free port (requestedPort=${config.port}, selectedPort=${listenPort})`);
}
const runtimeListenHost = config.host;
const runtimeApiHost =
runtimeListenHost === "0.0.0.0" || runtimeListenHost === "::"
? "localhost"
: runtimeListenHost;
process.env.PAPERCLIP_LISTEN_HOST = runtimeListenHost;
process.env.PAPERCLIP_LISTEN_PORT = String(listenPort);
process.env.PAPERCLIP_API_URL = `http://${runtimeApiHost}:${listenPort}`;
setupLiveEventsWebSocketServer(server, db as any, {
deploymentMode: config.deploymentMode,
resolveSessionFromHeaders,
});
void reconcilePersistedRuntimeServicesOnStartup(db as any)
.then((result) => {
if (result.reconciled > 0) {
logger.warn(
{ reconciled: result.reconciled },
"reconciled persisted runtime services from a previous server process",
);
}
})
.catch((err) => {
logger.error({ err }, "startup reconciliation of persisted runtime services failed");
});
if (config.heartbeatSchedulerEnabled) {
const heartbeat = heartbeatService(db as any);
// Reap orphaned runs at startup (no threshold -- runningProcesses is empty)
void heartbeat.reapOrphanedRuns().catch((err) => {
logger.error({ err }, "startup reap of orphaned heartbeat runs failed");
});
setInterval(() => {
void heartbeat
.tickTimers(new Date())
.then((result) => {
if (result.enqueued > 0) {
logger.info({ ...result }, "heartbeat timer tick enqueued runs");
}
})
.catch((err) => {
logger.error({ err }, "heartbeat timer tick failed");
});
// Periodically reap orphaned runs (5-min staleness threshold)
void heartbeat
.reapOrphanedRuns({ staleThresholdMs: 5 * 60 * 1000 })
.catch((err) => {
logger.error({ err }, "periodic reap of orphaned heartbeat runs failed");
});
}, config.heartbeatSchedulerIntervalMs);
}
if (config.databaseBackupEnabled) {
const backupIntervalMs = config.databaseBackupIntervalMinutes * 60 * 1000;
let backupInFlight = false;
const runScheduledBackup = async () => {
if (backupInFlight) {
logger.warn("Skipping scheduled database backup because a previous backup is still running");
return;
}
backupInFlight = true;
try {
const result = await runDatabaseBackup({
connectionString: activeDatabaseConnectionString,
backupDir: config.databaseBackupDir,
retentionDays: config.databaseBackupRetentionDays,
filenamePrefix: "paperclip",
});
logger.info(
{
backupFile: result.backupFile,
sizeBytes: result.sizeBytes,
prunedCount: result.prunedCount,
backupDir: config.databaseBackupDir,
retentionDays: config.databaseBackupRetentionDays,
},
`Automatic database backup complete: ${formatDatabaseBackupResult(result)}`,
);
} catch (err) {
logger.error({ err, backupDir: config.databaseBackupDir }, "Automatic database backup failed");
} finally {
backupInFlight = false;
}
};
logger.info(
{
intervalMinutes: config.databaseBackupIntervalMinutes,
retentionDays: config.databaseBackupRetentionDays,
backupDir: config.databaseBackupDir,
},
"Automatic database backups enabled",
);
setInterval(() => {
void runScheduledBackup();
}, backupIntervalMs);
}
await new Promise<void>((resolveListen, rejectListen) => {
const onError = (err: Error) => {
server.off("error", onError);
rejectListen(err);
};
server.once("error", onError);
server.listen(listenPort, config.host, () => {
server.off("error", onError);
logger.info(`Server listening on ${config.host}:${listenPort}`);
if (process.env.PAPERCLIP_OPEN_ON_LISTEN === "true") {
const openHost = config.host === "0.0.0.0" || config.host === "::" ? "127.0.0.1" : config.host;
const url = `http://${openHost}:${listenPort}`;
void import("open")
.then((mod) => mod.default(url))
.then(() => {
logger.info(`Opened browser at ${url}`);
})
.catch((err) => {
logger.warn({ err, url }, "Failed to open browser on startup");
});
}
printStartupBanner({
host: config.host,
deploymentMode: config.deploymentMode,
deploymentExposure: config.deploymentExposure,
authReady,
requestedPort: config.port,
listenPort,
uiMode,
db: startupDbInfo,
migrationSummary,
heartbeatSchedulerEnabled: config.heartbeatSchedulerEnabled,
heartbeatSchedulerIntervalMs: config.heartbeatSchedulerIntervalMs,
databaseBackupEnabled: config.databaseBackupEnabled,
databaseBackupIntervalMinutes: config.databaseBackupIntervalMinutes,
databaseBackupRetentionDays: config.databaseBackupRetentionDays,
databaseBackupDir: config.databaseBackupDir,
});
const boardClaimUrl = getBoardClaimWarningUrl(config.host, listenPort);
if (boardClaimUrl) {
const red = "\x1b[41m\x1b[30m";
const yellow = "\x1b[33m";
const reset = "\x1b[0m";
console.log(
[
`${red} BOARD CLAIM REQUIRED ${reset}`,
`${yellow}This instance was previously local_trusted and still has local-board as the only admin.${reset}`,
`${yellow}Sign in with a real user and open this one-time URL to claim ownership:${reset}`,
`${yellow}${boardClaimUrl}${reset}`,
`${yellow}If you are connecting over Tailscale, replace the host in this URL with your Tailscale IP/MagicDNS name.${reset}`,
].join("\n"),
);
}
resolveListen();
});
});
if (embeddedPostgres && embeddedPostgresStartedByThisProcess) {
const shutdown = async (signal: "SIGINT" | "SIGTERM") => {
logger.info({ signal }, "Stopping embedded PostgreSQL");
try {
await embeddedPostgres?.stop();
} catch (err) {
logger.error({ err }, "Failed to stop embedded PostgreSQL cleanly");
} finally {
process.exit(0);
}
};
process.once("SIGINT", () => {
void shutdown("SIGINT");
});
process.once("SIGTERM", () => {
void shutdown("SIGTERM");
});
}
return {
server,
host: config.host,
listenPort,
apiUrl: process.env.PAPERCLIP_API_URL ?? `http://${runtimeApiHost}:${listenPort}`,
databaseUrl: activeDatabaseConnectionString,
};
}
function isMainModule(metaUrl: string): boolean {
const entry = process.argv[1];
if (!entry) return false;
try {
return pathToFileURL(resolve(entry)).href === metaUrl;
} catch {
return false;
}
}
if (isMainModule(import.meta.url)) {
void startServer().catch((err) => {
logger.error({ err }, "Paperclip server failed to start");
process.exit(1);
});
}