From 09471314be6aa83773f5aaa5a396708b7a5c0b01 Mon Sep 17 00:00:00 2001 From: Forgotten Date: Mon, 16 Feb 2026 14:25:01 -0600 Subject: [PATCH] Add module system design plan Architecture for extending Paperclip with plugins: manifest-based modules with routes, UI pages, database tables, and hook subscriptions. Also covers company templates, the Company Store, and a phased implementation plan. Co-Authored-By: Claude Opus 4.6 --- plans/module-system.md | 685 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 685 insertions(+) create mode 100644 plans/module-system.md diff --git a/plans/module-system.md b/plans/module-system.md new file mode 100644 index 00000000..852c6a49 --- /dev/null +++ b/plans/module-system.md @@ -0,0 +1,685 @@ +# Paperclip Module System + +## Overview + +Paperclip's module system lets you extend the control plane with new capabilities — revenue tracking, observability, notifications, dashboards — without forking core. Modules are self-contained packages that register routes, UI pages, database tables, and lifecycle hooks. + +Separately, **Company Templates** are code-free data packages (agent teams, org charts, goal hierarchies) that you can import to bootstrap a new company. + +Both are discoverable through the **Company Store**. + +--- + +## Concepts + +| Concept | What it is | Contains code? | +|---------|-----------|----------------| +| **Module** | A package that extends Paperclip's API, UI, and data model | Yes | +| **Company Template** | A data snapshot — agents, projects, goals, org structure | No (JSON only) | +| **Company Store** | Registry for browsing/installing modules and templates | — | +| **Hook** | A named event in the core that modules can subscribe to | — | +| **Slot** | An exclusive category where only one module can be active (e.g., `observability`) | — | + +--- + +## Module Architecture + +### File Structure + +``` +modules/ + observability/ + paperclip.module.json # manifest (required) + src/ + index.ts # entry point — exports register function + routes.ts # Express router + hooks.ts # hook handlers + schema.ts # Drizzle table definitions + migrations/ # SQL migrations (generated by drizzle-kit) + ui/ # React components (lazy-loaded by the shell) + index.ts # exports page/widget definitions + TokenDashboard.tsx +``` + +Modules live in a top-level `modules/` directory. Each module is a pnpm workspace package. + +### Manifest (`paperclip.module.json`) + +```json +{ + "id": "observability", + "name": "Observability", + "description": "Token tracking, cost metrics, and agent performance instrumentation", + "version": "0.1.0", + "author": "paperclip", + + "slot": "observability", + + "hooks": [ + "agent:heartbeat", + "agent:created", + "issue:status_changed", + "budget:threshold_crossed" + ], + + "routes": { + "prefix": "/observability", + "entry": "./src/routes.ts" + }, + + "ui": { + "pages": [ + { + "path": "/observability", + "label": "Observability", + "entry": "./src/ui/index.ts" + } + ], + "widgets": [ + { + "id": "token-burn-rate", + "label": "Token Burn Rate", + "placement": "dashboard", + "entry": "./src/ui/index.ts" + } + ] + }, + + "schema": "./src/schema.ts", + + "configSchema": { + "type": "object", + "properties": { + "retentionDays": { "type": "number", "default": 30 }, + "enablePrometheus": { "type": "boolean", "default": false }, + "prometheusPort": { "type": "number", "default": 9090 } + } + }, + + "requires": { + "core": ">=0.1.0" + } +} +``` + +Key fields: + +- **`id`**: Unique identifier, used as the npm package name suffix (`@paperclip/mod-observability`) +- **`slot`**: Optional exclusive category. If set, only one module with this slot can be active. Omit for modules that can coexist freely. +- **`hooks`**: Which core events this module subscribes to. Declared upfront so the core knows what to emit. +- **`routes.prefix`**: Mounted under `/api/modules/`. The module owns this namespace. +- **`ui.pages`**: Adds entries to the sidebar. Lazy-loaded React components. +- **`ui.widgets`**: Injects components into existing pages (e.g., dashboard cards). +- **`schema`**: Drizzle table definitions for module-owned tables. Prefixed with `mod__` to avoid collisions. +- **`configSchema`**: JSON Schema for module configuration. Validated before the module loads. + +### Entry Point + +The module's `src/index.ts` exports a `register` function that receives the module API: + +```typescript +import type { ModuleAPI } from "@paperclip/core"; +import { createRouter } from "./routes.js"; +import { onHeartbeat, onBudgetThreshold } from "./hooks.js"; + +export default function register(api: ModuleAPI) { + // Register route handler + api.registerRoutes(createRouter(api.db, api.config)); + + // Subscribe to hooks + api.on("agent:heartbeat", onHeartbeat); + api.on("budget:threshold_crossed", onBudgetThreshold); + + // Register a background service (optional) + api.registerService({ + name: "metrics-aggregator", + interval: 60_000, // run every 60s + async run(ctx) { + await aggregateMetrics(ctx.db); + }, + }); +} +``` + +### Module API Surface + +```typescript +interface ModuleAPI { + // Identity + moduleId: string; + config: Record; // validated against configSchema + + // Database + db: Db; // shared Drizzle client + + // Routes + registerRoutes(router: Router): void; + + // Hooks + on(event: HookEvent, handler: HookHandler): void; + + // Background services + registerService(service: ServiceDef): void; + + // Logging (scoped to module) + logger: Logger; + + // Access core services (read-only helpers) + core: { + agents: AgentService; + issues: IssueService; + projects: ProjectService; + goals: GoalService; + activity: ActivityService; + }; +} +``` + +Modules get a scoped logger, access to the shared database, and read access to core services. They register their own routes and hook handlers. They do NOT monkey-patch core — they extend through defined interfaces. + +--- + +## Hook System + +### Core Hook Points + +Hooks are the primary integration point. The core emits events at well-defined moments. Modules subscribe in their `register` function. + +| Hook | Payload | When | +|------|---------|------| +| `server:started` | `{ port }` | After the Express server begins listening | +| `agent:created` | `{ agent }` | After a new agent is inserted | +| `agent:updated` | `{ agent, changes }` | After an agent record is modified | +| `agent:deleted` | `{ agent }` | After an agent is removed | +| `agent:heartbeat` | `{ agentId, timestamp, meta }` | When an agent checks in. `meta` carries tokens_used, cost, latency, etc. | +| `agent:status_changed` | `{ agent, from, to }` | When agent status transitions (idle→active, active→error, etc.) | +| `issue:created` | `{ issue }` | After a new issue is inserted | +| `issue:status_changed` | `{ issue, from, to }` | When issue moves between statuses | +| `issue:assigned` | `{ issue, agent }` | When an issue is assigned to an agent | +| `goal:created` | `{ goal }` | After a new goal is inserted | +| `goal:completed` | `{ goal }` | When a goal's status becomes complete | +| `budget:spend_recorded` | `{ agentId, amount, total }` | After spend is incremented | +| `budget:threshold_crossed` | `{ agentId, budget, spent, percent }` | When an agent crosses 80%, 90%, or 100% of budget | + +### Hook Execution Model + +```typescript +// In the core — hook emitter +class HookBus { + private handlers = new Map(); + + register(event: string, handler: HookHandler) { + const list = this.handlers.get(event) ?? []; + list.push(handler); + this.handlers.set(event, list); + } + + async emit(event: string, payload: unknown) { + const handlers = this.handlers.get(event) ?? []; + // Run all handlers concurrently. Failures are logged, never block core. + await Promise.allSettled( + handlers.map(h => h(payload)) + ); + } +} +``` + +Design rules: +- **Hooks are fire-and-forget.** A failing hook handler never crashes or blocks the core operation. +- **Hooks are concurrent.** All handlers for an event run in parallel via `Promise.allSettled`. +- **Hooks are post-commit.** They fire after the database write succeeds, not before. No vetoing. +- **Hooks receive immutable snapshots.** Handlers get a copy of the data, not a mutable reference. + +This keeps the core fast and resilient. If you need pre-commit validation (e.g., "reject this budget change"), that's a different mechanism (middleware/interceptor) we can add later if needed. + +### Observability Hook Example + +```typescript +// modules/observability/src/hooks.ts +import type { Db } from "@paperclip/db"; +import { tokenMetrics } from "./schema.js"; + +export function createHeartbeatHandler(db: Db) { + return async (payload: { + agentId: string; + timestamp: Date; + meta: { tokensUsed?: number; costCents?: number; model?: string }; + }) => { + const { agentId, timestamp, meta } = payload; + + if (meta.tokensUsed != null) { + await db.insert(tokenMetrics).values({ + agentId, + tokensUsed: meta.tokensUsed, + costCents: meta.costCents ?? 0, + model: meta.model ?? "unknown", + recordedAt: timestamp, + }); + } + }; +} +``` + +Every heartbeat, the observability module records token usage into its own `mod_observability_token_metrics` table. The core doesn't know or care about this table — it just emits the hook. + +--- + +## Database Strategy for Modules + +### Table Namespacing + +Module tables are prefixed with `mod__` to avoid collisions with core tables and other modules: + +```typescript +// modules/observability/src/schema.ts +import { pgTable, uuid, integer, text, timestamp } from "drizzle-orm/pg-core"; + +export const tokenMetrics = pgTable("mod_observability_token_metrics", { + id: uuid("id").primaryKey().defaultRandom(), + agentId: uuid("agent_id").notNull(), + tokensUsed: integer("tokens_used").notNull(), + costCents: integer("cost_cents").notNull().default(0), + model: text("model").notNull(), + recordedAt: timestamp("recorded_at", { withTimezone: true }).notNull().defaultNow(), +}); + +export const alertRules = pgTable("mod_observability_alert_rules", { + id: uuid("id").primaryKey().defaultRandom(), + agentId: uuid("agent_id"), + metricName: text("metric_name").notNull(), + threshold: integer("threshold").notNull(), + enabled: boolean("enabled").notNull().default(true), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), +}); +``` + +### Migration Strategy + +Each module manages its own migrations in `src/migrations/`. The core migration runner discovers and applies them: + +1. Core migrations run first (always) +2. Module migrations run in dependency order +3. Each module's migrations are tracked in a `mod_migrations` table with the module ID +4. `pnpm db:migrate` runs everything. `pnpm db:migrate --module observability` runs one. + +Modules can reference core tables via foreign keys (e.g., `agent_id → agents.id`) but core tables never reference module tables. This is a strict one-way dependency. + +--- + +## Module Loading & Lifecycle + +### Discovery + +On server startup: + +``` +1. Scan modules/ directory for paperclip.module.json manifests +2. Validate each manifest (JSON Schema check on configSchema, required fields) +3. Check slot conflicts (error if two active modules claim the same slot) +4. Topological sort by dependencies (if module A requires module B) +5. For each module in order: + a. Validate module config against configSchema + b. Run pending migrations + c. Import entry point and call register(api) + d. Mount routes at /api/modules/ + e. Start background services +6. Emit server:started hook +``` + +### Configuration + +Module config lives in the server's environment or a config file: + +```jsonc +// paperclip.config.json (or env vars) +{ + "modules": { + "enabled": ["observability", "revenue", "notifications"], + "config": { + "observability": { + "retentionDays": 90, + "enablePrometheus": true + }, + "revenue": { + "stripeSecretKey": "$STRIPE_SECRET_KEY" + } + } + } +} +``` + +`$ENV_VAR` references are resolved at load time. Secrets never go in the config file directly. + +### Disabling a Module + +Setting a module's enabled state to false: +1. Stops its background services +2. Unmounts its routes (returns 404) +3. Unsubscribes its hook handlers +4. Does NOT drop its database tables (data is preserved) + +--- + +## UI Integration + +### How Module UI Works + +The core UI shell provides: +- A sidebar with slots for module-contributed nav items +- A dashboard with widget mount points +- A module settings page + +Modules declare pages and widgets in the manifest. The shell lazy-loads them: + +```typescript +// ui/src/modules/loader.ts +// At build time or runtime, discover module UI entries and create lazy routes + +import { lazy } from "react"; + +// Generated from manifests +export const modulePages = [ + { + path: "/observability", + label: "Observability", + component: lazy(() => import("@paperclip/mod-observability/ui")), + }, +]; + +export const dashboardWidgets = [ + { + id: "token-burn-rate", + label: "Token Burn Rate", + placement: "dashboard", + component: lazy(() => import("@paperclip/mod-observability/ui").then(m => ({ default: m.TokenBurnRateWidget }))), + }, +]; +``` + +### Module UI Contract + +A module's UI entry exports named components: + +```typescript +// modules/observability/src/ui/index.ts +export { default } from "./ObservabilityPage"; +export { TokenBurnRateWidget } from "./TokenBurnRateWidget"; +``` + +Module UI components receive a standard props interface: + +```typescript +interface ModulePageProps { + moduleId: string; + config: Record; +} + +interface ModuleWidgetProps { + moduleId: string; + config: Record; + className?: string; +} +``` + +Module UI hits the module's own API routes (`/api/modules/observability/*`) for data. + +--- + +## Company Templates + +### Format + +A company template is a JSON file describing a full company structure: + +```json +{ + "id": "startup-in-a-box", + "name": "Startup in a Box", + "description": "A 5-agent startup team with engineering, product, and ops", + "version": "1.0.0", + "author": "paperclip", + + "agents": [ + { + "ref": "ceo", + "name": "CEO Agent", + "role": "pm", + "budgetCents": 100000, + "metadata": { "responsibilities": "Strategy, fundraising, hiring" } + }, + { + "ref": "eng-lead", + "name": "Engineering Lead", + "role": "engineer", + "reportsTo": "ceo", + "budgetCents": 50000 + }, + { + "ref": "eng-1", + "name": "Engineer", + "role": "engineer", + "reportsTo": "eng-lead", + "budgetCents": 30000 + }, + { + "ref": "designer", + "name": "Designer", + "role": "designer", + "reportsTo": "ceo", + "budgetCents": 20000 + }, + { + "ref": "ops", + "name": "Ops Agent", + "role": "devops", + "reportsTo": "ceo", + "budgetCents": 20000 + } + ], + + "goals": [ + { + "ref": "north-star", + "title": "Launch MVP", + "level": "company" + }, + { + "ref": "build-product", + "title": "Build the product", + "level": "team", + "parentRef": "north-star", + "ownerRef": "eng-lead" + }, + { + "ref": "design-brand", + "title": "Establish brand identity", + "level": "agent", + "parentRef": "north-star", + "ownerRef": "designer" + } + ], + + "projects": [ + { + "ref": "mvp", + "name": "MVP", + "description": "The first shippable version" + } + ], + + "issues": [ + { + "title": "Set up CI/CD pipeline", + "status": "todo", + "priority": "high", + "projectRef": "mvp", + "assigneeRef": "ops", + "goalRef": "build-product" + }, + { + "title": "Design landing page", + "status": "todo", + "priority": "medium", + "projectRef": "mvp", + "assigneeRef": "designer", + "goalRef": "design-brand" + } + ] +} +``` + +Templates use `ref` strings (not UUIDs) for internal cross-references. On import, the system maps refs to generated UUIDs. + +### Import Flow + +``` +1. Parse and validate the template JSON +2. Check for ref uniqueness and dangling references +3. Insert agents (topological sort by reportsTo) +4. Insert goals (topological sort by parentRef) +5. Insert projects +6. Insert issues (resolve projectRef, assigneeRef, goalRef to real IDs) +7. Log activity events for everything created +``` + +### Export Flow + +You can also export a running company as a template: + +``` +GET /api/templates/export → downloads the current company as a template JSON +``` + +This makes companies shareable and clonable. + +--- + +## Company Store + +The Company Store is a registry for discovering and installing modules and templates. For v1, it's a curated GitHub repo with a JSON index. Later it could become a hosted service. + +### Index Format + +```json +{ + "modules": [ + { + "id": "observability", + "name": "Observability", + "description": "Token tracking, cost metrics, and agent performance", + "repo": "github:paperclip/mod-observability", + "version": "0.1.0", + "tags": ["metrics", "monitoring", "tokens"] + } + ], + "templates": [ + { + "id": "startup-in-a-box", + "name": "Startup in a Box", + "description": "5-agent startup team", + "url": "https://store.paperclip.dev/templates/startup-in-a-box.json", + "tags": ["startup", "team"] + } + ] +} +``` + +### CLI Commands + +```bash +pnpm paperclip store list # browse available modules and templates +pnpm paperclip store install # install a module +pnpm paperclip store import # import a company template +pnpm paperclip store export # export current company as template +``` + +--- + +## Module Candidates + +### Tier 1 — Build first (core extensions) + +| Module | What it does | Key hooks | +|--------|-------------|-----------| +| **Observability** | Token usage tracking, cost metrics, agent performance dashboards, Prometheus export | `agent:heartbeat`, `budget:spend_recorded` | +| **Revenue Tracking** | Connect Stripe/crypto wallets, track income, show P&L against agent costs | `budget:spend_recorded` | +| **Notifications** | Slack/Discord/email alerts on configurable triggers | All hooks (configurable) | + +### Tier 2 — High value + +| Module | What it does | Key hooks | +|--------|-------------|-----------| +| **Analytics Dashboard** | Burn rate trends, agent utilization over time, goal velocity charts | `agent:heartbeat`, `issue:status_changed`, `goal:completed` | +| **Workflow Automation** | If/then rules: "when issue is done, create follow-up", "when budget at 90%, pause agent" | `issue:status_changed`, `budget:threshold_crossed` | +| **Knowledge Base** | Shared document store, vector search, agents read/write organizational knowledge | `agent:heartbeat` (for context injection) | + +### Tier 3 — Nice to have + +| Module | What it does | Key hooks | +|--------|-------------|-----------| +| **Audit & Compliance** | Immutable audit trail, approval workflows, spend authorization | All write hooks | +| **Agent Logs / Replay** | Full execution traces per agent, token-by-token replay | `agent:heartbeat` | +| **Multi-tenant** | Separate companies/orgs within one Paperclip instance | `server:started` | + +--- + +## Implementation Plan + +### Phase 1: Core infrastructure + +Add to `@paperclip/server`: + +1. **HookBus** — Event emitter with `register()` and `emit()`, using `Promise.allSettled` +2. **Module loader** — Scans `modules/`, validates manifests, calls `register(api)` +3. **Module API object** — `registerRoutes()`, `on()`, `registerService()`, logger, core service access +4. **Module config** — `paperclip.config.json` with per-module config, env var interpolation +5. **Module migration runner** — Extends `db:migrate` to discover and run module migrations +6. **Emit hooks from core services** — Add `hookBus.emit()` calls to existing CRUD operations + +Add to `@paperclip/ui`: + +7. **Module page loader** — Reads module manifests, generates lazy routes +8. **Dashboard widget slots** — Render module-contributed widgets on the Dashboard page +9. **Sidebar extension** — Dynamically add module nav items + +Add new package: + +10. **`@paperclip/module-sdk`** — TypeScript types for `ModuleAPI`, `HookEvent`, `HookHandler`, manifest schema + +### Phase 2: First module (observability) + +11. Build `modules/observability` as the reference implementation +12. Token metrics table + migration +13. Heartbeat hook handler recording token usage +14. Dashboard widget showing burn rate +15. API routes for querying metrics + +### Phase 3: Templates + +16. Template import endpoint (`POST /api/templates/import`) +17. Template export endpoint (`GET /api/templates/export`) +18. First template: "Startup in a Box" + +### Phase 4: Company Store + +19. GitHub-based store index +20. CLI commands for browse/install/import +21. UI page for browsing the store + +--- + +## Design Principles + +1. **Modules extend, never patch.** Modules add new routes, tables, and hook handlers. They never modify core tables or override core routes. + +2. **Hooks are post-commit, fire-and-forget.** Module failures never break core operations. + +3. **One-way dependency.** Modules depend on core. Core never depends on modules. Module tables can FK to core tables, not the reverse. + +4. **Declarative manifest, imperative registration.** Static metadata in JSON (validated without running code). Runtime behavior registered via the API. + +5. **Namespace isolation.** Module routes live under `/api/modules//`. Module tables are prefixed `mod__`. Module config is scoped to its ID. + +6. **Graceful degradation.** If a module fails to load, log the error and continue. The rest of the system works fine. + +7. **Data survives disable.** Disabling a module stops its code but preserves its data. Re-enabling picks up where it left off.