Replace PGlite with embedded-postgres and add startup banner

Switch from PGlite (WebAssembly) to embedded-postgres for zero-config local development — provides a real PostgreSQL server with full compatibility. Add startup banner with config summary on server boot. Improve server bootstrap with auto port detection, database creation, and migration on startup. Update DATABASE.md, DEVELOPING.md, and SPEC-implementation.md to reflect the change. Update CLI database check and prompts. Simplify OnboardingWizard database options. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-18 11:45:43 -06:00
parent 0d436911cd
commit cc24722090
18 changed files with 738 additions and 101 deletions
--- a/server/package.json
+++ b/server/package.json
@@ -4,7 +4,7 @@
  "private": true,
  "type": "module",
  "scripts": {
-    "dev": "tsx watch src/index.ts",
+    "dev": "tsx watch --ignore ../ui/node_modules --ignore ../ui/.vite --ignore ../ui/dist src/index.ts",
    "build": "tsc",
    "start": "node dist/index.js",
    "typecheck": "tsc --noEmit"
@@ -12,6 +12,7 @@
  "dependencies": {
    "@paperclip/db": "workspace:*",
    "@paperclip/shared": "workspace:*",
+    "detect-port": "^2.1.0",
    "drizzle-orm": "^0.38.4",
    "express": "^5.1.0",
    "pino": "^9.6.0",
@@ -19,6 +20,9 @@
    "ws": "^8.19.0",
    "zod": "^3.24.2"
  },
+  "optionalDependencies": {
+    "embedded-postgres": "^18.1.0-beta.16"
+  },
  "devDependencies": {
    "@types/express": "^5.0.0",
    "@types/express-serve-static-core": "^5.0.0",
@@ -26,6 +30,7 @@
    "supertest": "^7.0.0",
    "tsx": "^4.19.2",
    "typescript": "^5.7.3",
+    "vite": "^6.1.0",
    "vitest": "^3.0.5"
  }
 }
--- a/server/src/app.ts
+++ b/server/src/app.ts
@@ -1,5 +1,6 @@
 import express, { Router } from "express";
 import path from "node:path";
+import fs from "node:fs";
 import { fileURLToPath } from "node:url";
 import type { Db } from "@paperclip/db";
 import { httpLogger, errorHandler } from "./middleware/index.js";
@@ -15,7 +16,9 @@ import { costRoutes } from "./routes/costs.js";
 import { activityRoutes } from "./routes/activity.js";
 import { dashboardRoutes } from "./routes/dashboard.js";

-export function createApp(db: Db, opts: { serveUi: boolean }) {
+type UiMode = "none" | "static" | "vite-dev";
+
+export async function createApp(db: Db, opts: { uiMode: UiMode }) {
  const app = express();

  app.use(express.json());
@@ -36,16 +39,40 @@ export function createApp(db: Db, opts: { serveUi: boolean }) {
  api.use(dashboardRoutes(db));
  app.use("/api", api);

-  // SPA fallback for serving the UI build
-  if (opts.serveUi) {
-    const __dirname = path.dirname(fileURLToPath(import.meta.url));
+  const __dirname = path.dirname(fileURLToPath(import.meta.url));
+  if (opts.uiMode === "static") {
+    // Serve built UI from ui/dist in production.
    const uiDist = path.resolve(__dirname, "../../ui/dist");
    app.use(express.static(uiDist));
-    app.get("*", (_req, res) => {
+    app.get(/.*/, (_req, res) => {
      res.sendFile(path.join(uiDist, "index.html"));
    });
  }

+  if (opts.uiMode === "vite-dev") {
+    const uiRoot = path.resolve(__dirname, "../../ui");
+    const { createServer: createViteServer } = await import("vite");
+    const vite = await createViteServer({
+      root: uiRoot,
+      appType: "spa",
+      server: {
+        middlewareMode: true,
+      },
+    });
+
+    app.use(vite.middlewares);
+    app.get(/.*/, async (req, res, next) => {
+      try {
+        const templatePath = path.resolve(uiRoot, "index.html");
+        const template = fs.readFileSync(templatePath, "utf-8");
+        const html = await vite.transformIndexHtml(req.originalUrl, template);
+        res.status(200).set({ "Content-Type": "text/html" }).end(html);
+      } catch (err) {
+        next(err);
+      }
+    });
+  }
+
  app.use(errorHandler);

  return app;
--- a/server/src/config.ts
+++ b/server/src/config.ts
@@ -1,28 +1,40 @@
 import { readConfigFile } from "./config-file.js";

+type DatabaseMode = "embedded-postgres" | "postgres";
+
 export interface Config {
  port: number;
+  databaseMode: DatabaseMode;
  databaseUrl: string | undefined;
+  embeddedPostgresDataDir: string;
+  embeddedPostgresPort: number;
  serveUi: boolean;
+  uiDevMiddleware: boolean;
  heartbeatSchedulerEnabled: boolean;
  heartbeatSchedulerIntervalMs: number;
 }

 export function loadConfig(): Config {
  const fileConfig = readConfigFile();
+  const fileDatabaseMode =
+    (fileConfig?.database.mode === "postgres" ? "postgres" : "embedded-postgres") as DatabaseMode;

  const fileDbUrl =
-    fileConfig?.database.mode === "postgres"
-      ? fileConfig.database.connectionString
+    fileDatabaseMode === "postgres"
+      ? fileConfig?.database.connectionString
      : undefined;

  return {
    port: Number(process.env.PORT) || fileConfig?.server.port || 3100,
+    databaseMode: fileDatabaseMode,
    databaseUrl: process.env.DATABASE_URL ?? fileDbUrl,
+    embeddedPostgresDataDir: fileConfig?.database.embeddedPostgresDataDir ?? "./data/embedded-postgres",
+    embeddedPostgresPort: fileConfig?.database.embeddedPostgresPort ?? 54329,
    serveUi:
      process.env.SERVE_UI !== undefined
        ? process.env.SERVE_UI === "true"
        : fileConfig?.server.serveUi ?? false,
+    uiDevMiddleware: process.env.PAPERCLIP_UI_DEV_MIDDLEWARE === "true",
    heartbeatSchedulerEnabled: process.env.HEARTBEAT_SCHEDULER_ENABLED !== "false",
    heartbeatSchedulerIntervalMs: Math.max(10000, Number(process.env.HEARTBEAT_SCHEDULER_INTERVAL_MS) || 30000),
  };
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -1,26 +1,163 @@
+import { existsSync, readFileSync, rmSync } from "node:fs";
 import { createServer } from "node:http";
 import { resolve } from "node:path";
-import { createDb, createPgliteDb } from "@paperclip/db";
+import {
+  createDb,
+  ensurePostgresDatabase,
+  migratePostgresIfEmpty,
+} from "@paperclip/db";
+import detectPort from "detect-port";
 import { createApp } from "./app.js";
 import { loadConfig } from "./config.js";
 import { logger } from "./middleware/logger.js";
 import { setupLiveEventsWebSocketServer } from "./realtime/live-events-ws.js";
 import { heartbeatService } from "./services/index.js";
+import { printStartupBanner } from "./startup-banner.js";
+
+type EmbeddedPostgresInstance = {
+  initialise(): Promise<void>;
+  start(): Promise<void>;
+  stop(): Promise<void>;
+};
+
+type EmbeddedPostgresCtor = new (opts: {
+  databaseDir: string;
+  user: string;
+  password: string;
+  port: number;
+  persistent: boolean;
+}) => EmbeddedPostgresInstance;

 const config = loadConfig();

 let db;
+let embeddedPostgres: EmbeddedPostgresInstance | null = null;
+let embeddedPostgresStartedByThisProcess = false;
+let migrationSummary = "skipped";
+let startupDbInfo:
+  | { mode: "external-postgres"; connectionString: string }
+  | { mode: "embedded-postgres"; dataDir: string; port: number };
 if (config.databaseUrl) {
+  const migration = await migratePostgresIfEmpty(config.databaseUrl);
+  if (migration.migrated) {
+    logger.info("Empty PostgreSQL database detected; applied migrations");
+    migrationSummary = "applied (empty database)";
+  } else if (migration.reason === "not-empty-no-migration-journal") {
+    logger.warn(
+      { tableCount: migration.tableCount },
+      "PostgreSQL has existing tables but no migration journal; skipped auto-migrate",
+    );
+    migrationSummary = "skipped (existing schema, no migration journal)";
+  } else {
+    migrationSummary = "already applied";
+  }
+
  db = createDb(config.databaseUrl);
+  logger.info("Using external PostgreSQL via DATABASE_URL/config");
+  startupDbInfo = { mode: "external-postgres", connectionString: config.databaseUrl };
 } else {
-  const dataDir = resolve("./data/pglite");
-  logger.info(`No DATABASE_URL set — using embedded PGlite (${dataDir})`);
-  db = await createPgliteDb(dataDir);
-  logger.info("PGlite ready, schema pushed");
+  const moduleName = "embedded-postgres";
+  let EmbeddedPostgres: EmbeddedPostgresCtor;
+  try {
+    const mod = await import(moduleName);
+    EmbeddedPostgres = mod.default as EmbeddedPostgresCtor;
+  } catch {
+    throw new Error(
+      "Embedded PostgreSQL mode requires optional dependency `embedded-postgres`. Install optional dependencies or set DATABASE_URL for external Postgres.",
+    );
+  }
+
+  const dataDir = resolve(config.embeddedPostgresDataDir);
+  const port = config.embeddedPostgresPort;
+
+  if (config.databaseMode === "postgres") {
+    logger.warn("Database mode is postgres but no connection string was set; falling back to embedded PostgreSQL");
+  }
+
+  logger.info(`No DATABASE_URL set — using embedded PostgreSQL (${dataDir}) on port ${port}`);
+  embeddedPostgres = new EmbeddedPostgres({
+    databaseDir: dataDir,
+    user: "paperclip",
+    password: "paperclip",
+    port,
+    persistent: true,
+  });
+  const clusterVersionFile = resolve(dataDir, "PG_VERSION");
+  if (!existsSync(clusterVersionFile)) {
+    await embeddedPostgres.initialise();
+  } else {
+    logger.info(`Embedded PostgreSQL cluster already exists (${clusterVersionFile}); skipping init`);
+  }
+
+  const postmasterPidFile = resolve(dataDir, "postmaster.pid");
+  const isPidRunning = (pid: number): boolean => {
+    try {
+      process.kill(pid, 0);
+      return true;
+    } catch {
+      return false;
+    }
+  };
+
+  const getRunningPid = (): number | null => {
+    if (!existsSync(postmasterPidFile)) return null;
+    try {
+      const pidLine = readFileSync(postmasterPidFile, "utf8").split("\n")[0]?.trim();
+      const pid = Number(pidLine);
+      if (!Number.isInteger(pid) || pid <= 0) return null;
+      if (!isPidRunning(pid)) return null;
+      return pid;
+    } catch {
+      return null;
+    }
+  };
+
+  const runningPid = getRunningPid();
+  if (runningPid) {
+    logger.warn({ pid: runningPid }, "Embedded PostgreSQL already running; reusing existing process");
+  } else {
+    if (existsSync(postmasterPidFile)) {
+      logger.warn("Removing stale embedded PostgreSQL lock file");
+      rmSync(postmasterPidFile, { force: true });
+    }
+    await embeddedPostgres.start();
+    embeddedPostgresStartedByThisProcess = true;
+  }
+
+  const embeddedAdminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/postgres`;
+  const dbStatus = await ensurePostgresDatabase(embeddedAdminConnectionString, "paperclip");
+  if (dbStatus === "created") {
+    logger.info("Created embedded PostgreSQL database: paperclip");
+  }
+
+  const embeddedConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`;
+  const migration = await migratePostgresIfEmpty(embeddedConnectionString);
+  if (migration.migrated) {
+    logger.info("Empty embedded PostgreSQL database detected; applied migrations");
+    migrationSummary = "applied (empty database)";
+  } else if (migration.reason === "not-empty-no-migration-journal") {
+    logger.warn(
+      { tableCount: migration.tableCount },
+      "Embedded PostgreSQL has existing tables but no migration journal; skipped auto-migrate",
+    );
+    migrationSummary = "skipped (existing schema, no migration journal)";
+  } else {
+    migrationSummary = "already applied";
+  }
+
+  db = createDb(embeddedConnectionString);
+  logger.info("Embedded PostgreSQL ready");
+  startupDbInfo = { mode: "embedded-postgres", dataDir, port };
 }

-const app = createApp(db as any, { serveUi: config.serveUi });
+const uiMode = config.uiDevMiddleware ? "vite-dev" : config.serveUi ? "static" : "none";
+const app = await createApp(db as any, { uiMode });
 const server = createServer(app);
+const listenPort = await detectPort(config.port);
+
+if (listenPort !== config.port) {
+  logger.warn({ requestedPort: config.port, selectedPort: listenPort }, "Requested port is busy; using next free port");
+}

 setupLiveEventsWebSocketServer(server, db as any);

@@ -40,6 +177,35 @@ if (config.heartbeatSchedulerEnabled) {
  }, config.heartbeatSchedulerIntervalMs);
 }

-server.listen(config.port, () => {
-  logger.info(`Server listening on :${config.port}`);
+server.listen(listenPort, () => {
+  logger.info(`Server listening on :${listenPort}`);
+  printStartupBanner({
+    requestedPort: config.port,
+    listenPort,
+    uiMode,
+    db: startupDbInfo,
+    migrationSummary,
+    heartbeatSchedulerEnabled: config.heartbeatSchedulerEnabled,
+    heartbeatSchedulerIntervalMs: config.heartbeatSchedulerIntervalMs,
+  });
 });
+
+if (embeddedPostgres && embeddedPostgresStartedByThisProcess) {
+  const shutdown = async (signal: "SIGINT" | "SIGTERM") => {
+    logger.info({ signal }, "Stopping embedded PostgreSQL");
+    try {
+      await embeddedPostgres?.stop();
+    } catch (err) {
+      logger.error({ err }, "Failed to stop embedded PostgreSQL cleanly");
+    } finally {
+      process.exit(0);
+    }
+  };
+
+  process.once("SIGINT", () => {
+    void shutdown("SIGINT");
+  });
+  process.once("SIGTERM", () => {
+    void shutdown("SIGTERM");
+  });
+}
--- a/server/src/startup-banner.ts
+++ b/server/src/startup-banner.ts
@@ -0,0 +1,115 @@
+import { resolve } from "node:path";
+
+type UiMode = "none" | "static" | "vite-dev";
+
+type ExternalPostgresInfo = {
+  mode: "external-postgres";
+  connectionString: string;
+};
+
+type EmbeddedPostgresInfo = {
+  mode: "embedded-postgres";
+  dataDir: string;
+  port: number;
+};
+
+type StartupBannerOptions = {
+  requestedPort: number;
+  listenPort: number;
+  uiMode: UiMode;
+  db: ExternalPostgresInfo | EmbeddedPostgresInfo;
+  migrationSummary: string;
+  heartbeatSchedulerEnabled: boolean;
+  heartbeatSchedulerIntervalMs: number;
+};
+
+const ansi = {
+  reset: "\x1b[0m",
+  bold: "\x1b[1m",
+  dim: "\x1b[2m",
+  cyan: "\x1b[36m",
+  green: "\x1b[32m",
+  yellow: "\x1b[33m",
+  magenta: "\x1b[35m",
+  blue: "\x1b[34m",
+};
+
+function color(text: string, c: keyof typeof ansi): string {
+  return `${ansi[c]}${text}${ansi.reset}`;
+}
+
+function row(label: string, value: string): string {
+  return `${color(label.padEnd(16), "dim")} ${value}`;
+}
+
+function redactConnectionString(raw: string): string {
+  try {
+    const u = new URL(raw);
+    const user = u.username || "user";
+    const auth = `${user}:***@`;
+    return `${u.protocol}//${auth}${u.host}${u.pathname}`;
+  } catch {
+    return "<invalid DATABASE_URL>";
+  }
+}
+
+export function printStartupBanner(opts: StartupBannerOptions): void {
+  const baseUrl = `http://localhost:${opts.listenPort}`;
+  const apiUrl = `${baseUrl}/api`;
+  const uiUrl = opts.uiMode === "none" ? "disabled" : baseUrl;
+  const configPath = process.env.PAPERCLIP_CONFIG
+    ? resolve(process.env.PAPERCLIP_CONFIG)
+    : resolve(process.cwd(), ".paperclip/config.json");
+
+  const dbMode =
+    opts.db.mode === "embedded-postgres"
+      ? color("embedded-postgres", "green")
+      : color("external-postgres", "yellow");
+  const uiMode =
+    opts.uiMode === "vite-dev"
+      ? color("vite-dev-middleware", "cyan")
+      : opts.uiMode === "static"
+        ? color("static-ui", "magenta")
+        : color("headless-api", "yellow");
+
+  const portValue =
+    opts.requestedPort === opts.listenPort
+      ? `${opts.listenPort}`
+      : `${opts.listenPort} ${color(`(requested ${opts.requestedPort})`, "dim")}`;
+
+  const dbDetails =
+    opts.db.mode === "embedded-postgres"
+      ? `${opts.db.dataDir} ${color(`(pg:${opts.db.port})`, "dim")}`
+      : redactConnectionString(opts.db.connectionString);
+
+  const heartbeat = opts.heartbeatSchedulerEnabled
+    ? `enabled ${color(`(${opts.heartbeatSchedulerIntervalMs}ms)`, "dim")}`
+    : color("disabled", "yellow");
+
+  const art = [
+    color("██████╗  █████╗ ██████╗ ███████╗██████╗  ██████╗██╗     ██╗██████╗ ", "cyan"),
+    color("██╔══██╗██╔══██╗██╔══██╗██╔════╝██╔══██╗██╔════╝██║     ██║██╔══██╗", "cyan"),
+    color("██████╔╝███████║██████╔╝█████╗  ██████╔╝██║     ██║     ██║██████╔╝", "cyan"),
+    color("██╔═══╝ ██╔══██║██╔═══╝ ██╔══╝  ██╔══██╗██║     ██║     ██║██╔═══╝ ", "cyan"),
+    color("██║     ██║  ██║██║     ███████╗██║  ██║╚██████╗███████╗██║██║     ", "cyan"),
+    color("╚═╝     ╚═╝  ╚═╝╚═╝     ╚══════╝╚═╝  ╚═╝ ╚═════╝╚══════╝╚═╝╚═╝     ", "cyan"),
+  ];
+
+  const lines = [
+    "",
+    ...art,
+    color("  ───────────────────────────────────────────────────────", "blue"),
+    row("Mode", `${dbMode}  |  ${uiMode}`),
+    row("Server", portValue),
+    row("API", `${apiUrl} ${color(`(health: ${apiUrl}/health)`, "dim")}`),
+    row("UI", uiUrl),
+    row("Database", dbDetails),
+    row("Migrations", opts.migrationSummary),
+    row("Heartbeat", heartbeat),
+    row("Config", configPath),
+    color("  ───────────────────────────────────────────────────────", "blue"),
+    "",
+  ];
+
+  console.log(lines.join("\n"));
+}