aevgarik/paperclip
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Simplify plugin runtime and cleanup lifecycle

Browse Source
This commit is contained in:
Dotta
2026-03-13 16:58:29 -05:00
parent 80cdbdbd47
commit 12ccfc2c9a
21 changed files with 120 additions and 838 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
server/src/services/plugin-event-bus.ts
View File

@@ -112,60 +112,6 @@ function passesFilter(event: PluginEvent, filter: EventFilter | null): boolean {
return true;
}
// ---------------------------------------------------------------------------
// Company availability checker
// ---------------------------------------------------------------------------
/**
* Callback that checks whether a plugin is enabled for a given company.
*
* The event bus calls this during `emit()` to enforce company-scoped delivery:
* events are only delivered to a plugin if the plugin is enabled for the
* company that owns the event.
*
* Implementations should be fast — the bus caches results internally with a
* short TTL so the checker is not invoked on every single event.
*
* @param pluginKey The plugin registry key — the string passed to `forPlugin()`
* (e.g. `"acme.linear"`). This is the same key used throughout the bus
* internally and should not be confused with a numeric or UUID plugin ID.
* @param companyId UUID of the company to check availability for.
*
* Return `true` if the plugin is enabled (or if no settings row exists, i.e.
* default-enabled), `false` if the company has explicitly disabled the plugin.
*/
export type CompanyAvailabilityChecker = (
pluginKey: string,
companyId: string,
) => Promise<boolean>;
/**
* Options for {@link createPluginEventBus}.
*/
export interface PluginEventBusOptions {
/**
* Optional checker that gates event delivery per company.
*
* When provided, the bus will skip delivery to a plugin if the checker
* returns `false` for the `(pluginKey, event.companyId)` pair, where
* `pluginKey` is the registry key supplied to `forPlugin()`. Results are
* cached with a short TTL (30 s) to avoid excessive lookups.
*
* When omitted, no company-scoping is applied (useful in tests).
*/
isPluginEnabledForCompany?: CompanyAvailabilityChecker;
}
// Default cache TTL in milliseconds (30 seconds).
const AVAILABILITY_CACHE_TTL_MS = 30_000;
// Maximum number of entries in the availability cache before it is cleared.
// Prevents unbounded memory growth in long-running processes with many unique
// (pluginKey, companyId) pairs. A full clear is intentionally simple — the
// cache is advisory (performance only) and a miss merely triggers one extra
// async lookup.
const MAX_AVAILABILITY_CACHE_SIZE = 10_000;
// ---------------------------------------------------------------------------
// Event bus factory
// ---------------------------------------------------------------------------
@@ -200,40 +146,10 @@ const MAX_AVAILABILITY_CACHE_SIZE = 10_000;
* });
* ```
*/
export function createPluginEventBus(options?: PluginEventBusOptions): PluginEventBus {
const checker = options?.isPluginEnabledForCompany ?? null;
export function createPluginEventBus(): PluginEventBus {
// Subscription registry: pluginKey → list of subscriptions
const registry = new Map<string, Subscription[]>();
// Short-TTL cache for company availability lookups: "pluginKey\0companyId" → { enabled, expiresAt }
const availabilityCache = new Map<string, { enabled: boolean; expiresAt: number }>();
function cacheKey(pluginKey: string, companyId: string): string {
return `${pluginKey}\0${companyId}`;
}
/**
* Check whether a plugin is enabled for a company, using the cached result
* when available and falling back to the injected checker.
*/
async function isEnabledForCompany(pluginKey: string, companyId: string): Promise<boolean> {
if (!checker) return true;
const key = cacheKey(pluginKey, companyId);
const cached = availabilityCache.get(key);
if (cached && cached.expiresAt > Date.now()) {
return cached.enabled;
}
const enabled = await checker(pluginKey, companyId);
if (availabilityCache.size >= MAX_AVAILABILITY_CACHE_SIZE) {
availabilityCache.clear();
}
availabilityCache.set(key, { enabled, expiresAt: Date.now() + AVAILABILITY_CACHE_TTL_MS });
return enabled;
}
/**
* Retrieve or create the subscription list for a plugin.
*/
@@ -257,26 +173,7 @@ export function createPluginEventBus(options?: PluginEventBusOptions): PluginEve
const errors: Array<{ pluginId: string; error: unknown }> = [];
const promises: Promise<void>[] = [];
// Pre-compute company availability for all registered plugins when the
// event carries a companyId and a checker is configured. This batches
// the (potentially async) lookups so we don't interleave them with
// handler dispatch.
let disabledPlugins: Set<string> | null = null;
if (checker && event.companyId) {
const pluginKeys = Array.from(registry.keys());
const checks = await Promise.all(
pluginKeys.map(async (pluginKey) => ({
pluginKey,
enabled: await isEnabledForCompany(pluginKey, event.companyId!),
})),
);
disabledPlugins = new Set(checks.filter((c) => !c.enabled).map((c) => c.pluginKey));
}
for (const [pluginId, subs] of registry) {
// Skip delivery to plugins that are disabled for this company.
if (disabledPlugins?.has(pluginId)) continue;
for (const sub of subs) {
if (!matchesPattern(event.eventType, sub.eventPattern)) continue;
if (!passesFilter(event, sub.filter)) continue;

24
server/src/services/plugin-host-services.ts
View File

@@ -466,18 +466,11 @@ export function buildHostServices(
};
/**
* Verify that this plugin is enabled for the given company.
* Throws if the plugin is disabled or unavailable, preventing
* worker-driven access to companies that have not opted in.
* Plugins are instance-wide in the current runtime. Company IDs are still
* required for company-scoped data access, but there is no per-company
* availability gate to enforce here.
*/
const ensurePluginAvailableForCompany = async (companyId: string) => {
const availability = await registry.getCompanyAvailability(companyId, pluginId);
if (!availability || !availability.available) {
throw new Error(
`Plugin "${pluginKey}" is not enabled for company "${companyId}"`,
);
}
};
const ensurePluginAvailableForCompany = async (_companyId: string) => {};
const inCompany = <T extends { companyId: string | null | undefined }>(
record: T | null | undefined,
@@ -656,14 +649,7 @@ export function buildHostServices(
companies: {
async list(_params) {
const allCompanies = (await companies.list()) as Company[];
if (allCompanies.length === 0) return [];
// Batch query: fetch all company settings for this plugin in one query
// instead of N+1 individual getCompanyAvailability() calls.
const companyIds = allCompanies.map((c) => c.id);
const disabledCompanyIds = await registry.getDisabledCompanyIds(companyIds, pluginId);
return allCompanies.filter((c) => !disabledCompanyIds.has(c.id));
return (await companies.list()) as Company[];
},
async get(params) {
await ensurePluginAvailableForCompany(params.companyId);

32
server/src/services/plugin-lifecycle.ts
View File

@@ -431,6 +431,22 @@ export function pluginLifecycleManager(
}
}
async function deactivatePluginRuntime(
pluginId: string,
pluginKey: string,
): Promise<void> {
const supportsRuntimeDeactivation =
typeof pluginLoaderInstance.hasRuntimeServices === "function"
&& typeof pluginLoaderInstance.unloadSingle === "function";
if (supportsRuntimeDeactivation && pluginLoaderInstance.hasRuntimeServices()) {
await pluginLoaderInstance.unloadSingle(pluginId, pluginKey);
return;
}
await stopWorkerIfRunning(pluginId, pluginKey);
}
// -----------------------------------------------------------------------
// Public API
// -----------------------------------------------------------------------
@@ -504,8 +520,7 @@ export function pluginLifecycleManager(
);
}
// Stop the worker before transitioning state
await stopWorkerIfRunning(pluginId, plugin.pluginKey);
await deactivatePluginRuntime(pluginId, plugin.pluginKey);
const result = await transition(pluginId, "disabled", reason ?? null, plugin);
emitDomain("plugin.disabled", {
@@ -526,6 +541,7 @@ export function pluginLifecycleManager(
// If already uninstalled and removeData, hard-delete
if (plugin.status === "uninstalled") {
if (removeData) {
await pluginLoaderInstance.cleanupInstallArtifacts(plugin);
const deleted = await registry.uninstall(pluginId, true);
log.info(
{ pluginId, pluginKey: plugin.pluginKey },
@@ -544,8 +560,8 @@ export function pluginLifecycleManager(
);
}
// Stop the worker before uninstalling
await stopWorkerIfRunning(pluginId, plugin.pluginKey);
await deactivatePluginRuntime(pluginId, plugin.pluginKey);
await pluginLoaderInstance.cleanupInstallArtifacts(plugin);
// Perform the uninstall via registry (handles soft/hard delete)
const result = await registry.uninstall(pluginId, removeData);
@@ -577,7 +593,7 @@ export function pluginLifecycleManager(
// continue running. The worker manager's auto-restart is disabled
// because we are intentionally taking the plugin offline.
const plugin = await requirePlugin(pluginId);
await stopWorkerIfRunning(pluginId, plugin.pluginKey);
await deactivatePluginRuntime(pluginId, plugin.pluginKey);
const result = await transition(pluginId, "error", error, plugin);
emitDomain("plugin.error", {
@@ -590,9 +606,8 @@ export function pluginLifecycleManager(
// -- markUpgradePending -----------------------------------------------
async markUpgradePending(pluginId: string): Promise<PluginRecord> {
// Stop the worker while waiting for operator approval of new capabilities
const plugin = await requirePlugin(pluginId);
await stopWorkerIfRunning(pluginId, plugin.pluginKey);
await deactivatePluginRuntime(pluginId, plugin.pluginKey);
const result = await transition(pluginId, "upgrade_pending", null, plugin);
emitDomain("plugin.upgrade_pending", {
@@ -637,8 +652,7 @@ export function pluginLifecycleManager(
"plugin lifecycle: upgrade requested",
);
// Stop the current worker before upgrading on disk
await stopWorkerIfRunning(pluginId, plugin.pluginKey);
await deactivatePluginRuntime(pluginId, plugin.pluginKey);
// 1. Download and validate new package via loader
const { oldManifest, newManifest, discovered } =

68
server/src/services/plugin-loader.ts
View File

@@ -25,7 +25,7 @@
* @see PLUGIN_SPEC.md §12 — Process Model
*/
import { existsSync } from "node:fs";
import { readdir, readFile, stat } from "node:fs/promises";
import { readdir, readFile, rm, stat } from "node:fs/promises";
import { execFile } from "node:child_process";
import os from "node:os";
import path from "node:path";
@@ -394,6 +394,14 @@ export interface PluginLoader {
*/
isSupportedApiVersion(apiVersion: number): boolean;
/**
* Remove runtime-managed on-disk install artifacts for a plugin.
*
* This only cleans files under the managed local plugin directory. Local-path
* source checkouts outside that directory are intentionally left alone.
*/
cleanupInstallArtifacts(plugin: PluginRecord): Promise<void>;
/**
* Get the local plugin directory this loader is configured to use.
*/
@@ -1334,6 +1342,50 @@ export function pluginLoader(
return manifestValidator.getSupportedVersions().includes(apiVersion);
},
// -----------------------------------------------------------------------
// cleanupInstallArtifacts
// -----------------------------------------------------------------------
async cleanupInstallArtifacts(plugin: PluginRecord): Promise<void> {
const managedTargets = new Set<string>();
const managedNodeModulesDir = resolveManagedInstallPackageDir(localPluginDir, plugin.packageName);
const directManagedDir = path.join(localPluginDir, plugin.packageName);
managedTargets.add(managedNodeModulesDir);
if (isPathInsideDir(directManagedDir, localPluginDir)) {
managedTargets.add(directManagedDir);
}
if (plugin.packagePath && isPathInsideDir(plugin.packagePath, localPluginDir)) {
managedTargets.add(path.resolve(plugin.packagePath));
}
const packageJsonPath = path.join(localPluginDir, "package.json");
if (existsSync(packageJsonPath)) {
try {
await execFileAsync(
"npm",
["uninstall", plugin.packageName, "--prefix", localPluginDir, "--ignore-scripts"],
{ timeout: 120_000 },
);
} catch (err) {
log.warn(
{
pluginId: plugin.id,
pluginKey: plugin.pluginKey,
packageName: plugin.packageName,
err: err instanceof Error ? err.message : String(err),
},
"plugin-loader: npm uninstall failed during cleanup, falling back to direct removal",
);
}
}
for (const target of managedTargets) {
if (!existsSync(target)) continue;
await rm(target, { recursive: true, force: true });
}
},
// -----------------------------------------------------------------------
// getLocalPluginDir
// -----------------------------------------------------------------------
@@ -1850,3 +1902,17 @@ function resolveWorkerEntrypoint(
`${path.resolve(directDir, workerRelPath)}`,
);
}
function resolveManagedInstallPackageDir(localPluginDir: string, packageName: string): string {
if (packageName.startsWith("@")) {
return path.join(localPluginDir, "node_modules", ...packageName.split("/"));
}
return path.join(localPluginDir, "node_modules", packageName);
}
function isPathInsideDir(candidatePath: string, parentDir: string): boolean {
const resolvedCandidate = path.resolve(candidatePath);
const resolvedParent = path.resolve(parentDir);
const relative = path.relative(resolvedParent, resolvedCandidate);
return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative));
}

283
server/src/services/plugin-registry.ts
View File

@@ -1,10 +1,8 @@
import { asc, eq, ne, sql, and, inArray } from "drizzle-orm";
import { asc, eq, ne, sql, and } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import {
plugins,
companies,
pluginConfig,
pluginCompanySettings,
pluginEntities,
pluginJobs,
pluginJobRuns,
@@ -17,10 +15,6 @@ import type {
UpdatePluginStatus,
UpsertPluginConfig,
PatchPluginConfig,
PluginCompanySettings,
CompanyPluginAvailability,
UpsertPluginCompanySettings,
UpdateCompanyPluginAvailability,
PluginEntityRecord,
PluginEntityQuery,
PluginJobRecord,
@@ -92,54 +86,6 @@ export function pluginRegistryService(db: Db) {
return (result[0]?.maxOrder ?? 0) + 1;
}
/**
* Load the persisted company override row for a plugin, if one exists.
*
* Missing rows are meaningful: the company inherits the default-enabled
* behavior and the caller should treat the plugin as available.
*/
async function getCompanySettingsRow(companyId: string, pluginId: string) {
return db
.select()
.from(pluginCompanySettings)
.where(and(
eq(pluginCompanySettings.companyId, companyId),
eq(pluginCompanySettings.pluginId, pluginId),
))
.then((rows) => rows[0] ?? null);
}
/**
* Normalize registry records into the API response returned by company
* plugin availability routes.
*
* The key business rule is captured here: plugins are enabled for a company
* unless an explicit `plugin_company_settings.enabled = false` override says
* otherwise.
*/
function toCompanyAvailability(
companyId: string,
plugin: Awaited<ReturnType<typeof getById>>,
settings: PluginCompanySettings | null,
): CompanyPluginAvailability {
if (!plugin) {
throw notFound("Plugin not found");
}
return {
companyId,
pluginId: plugin.id,
pluginKey: plugin.pluginKey,
pluginDisplayName: plugin.manifestJson.displayName,
pluginStatus: plugin.status,
available: settings?.enabled ?? true,
settingsJson: settings?.settingsJson ?? {},
lastError: settings?.lastError ?? null,
createdAt: settings?.createdAt ?? null,
updatedAt: settings?.updatedAt ?? null,
};
}
// -----------------------------------------------------------------------
// Public API
// -----------------------------------------------------------------------
@@ -416,233 +362,6 @@ export function pluginRegistryService(db: Db) {
.then((rows) => rows[0]);
},
// ----- Company-scoped settings ----------------------------------------
/** Retrieve a plugin's company-scoped settings row, if any. */
getCompanySettings: (companyId: string, pluginId: string) =>
getCompanySettingsRow(companyId, pluginId),
/** Create or replace the company-scoped settings row for a plugin. */
upsertCompanySettings: async (
companyId: string,
pluginId: string,
input: UpsertPluginCompanySettings,
) => {
const plugin = await getById(pluginId);
if (!plugin) throw notFound("Plugin not found");
const existing = await getCompanySettingsRow(companyId, pluginId);
if (existing) {
return db
.update(pluginCompanySettings)
.set({
enabled: true,
settingsJson: input.settingsJson ?? {},
lastError: input.lastError ?? null,
updatedAt: new Date(),
})
.where(eq(pluginCompanySettings.id, existing.id))
.returning()
.then((rows) => rows[0]);
}
return db
.insert(pluginCompanySettings)
.values({
companyId,
pluginId,
enabled: true,
settingsJson: input.settingsJson ?? {},
lastError: input.lastError ?? null,
})
.returning()
.then((rows) => rows[0]);
},
/** Delete the company-scoped settings row for a plugin if it exists. */
deleteCompanySettings: async (companyId: string, pluginId: string) => {
const plugin = await getById(pluginId);
if (!plugin) throw notFound("Plugin not found");
const existing = await getCompanySettingsRow(companyId, pluginId);
if (!existing) return null;
return db
.delete(pluginCompanySettings)
.where(eq(pluginCompanySettings.id, existing.id))
.returning()
.then((rows) => rows[0] ?? null);
},
/** List normalized company-plugin availability records across installed plugins. */
listCompanyAvailability: async (
companyId: string,
filter?: { available?: boolean },
) => {
const installed = await db
.select()
.from(plugins)
.where(ne(plugins.status, "uninstalled"))
.orderBy(asc(plugins.installOrder));
const settingsRows = await db
.select()
.from(pluginCompanySettings)
.where(eq(pluginCompanySettings.companyId, companyId));
const settingsByPluginId = new Map(settingsRows.map((row) => [row.pluginId, row]));
const availability = installed.map((plugin) => {
const row = settingsByPluginId.get(plugin.id) ?? null;
return {
...toCompanyAvailability(companyId, plugin, row),
};
});
if (filter?.available === undefined) return availability;
return availability.filter((item) => item.available === filter.available);
},
/**
* Batch-check which companies have this plugin explicitly disabled.
* Returns a Set of companyIds where `enabled = false`. Companies with
* no settings row default to enabled, so they are NOT in the result set.
*/
getDisabledCompanyIds: async (companyIds: string[], pluginId: string): Promise<Set<string>> => {
if (companyIds.length === 0) return new Set();
const rows = await db
.select({
companyId: pluginCompanySettings.companyId,
enabled: pluginCompanySettings.enabled,
})
.from(pluginCompanySettings)
.where(and(
inArray(pluginCompanySettings.companyId, companyIds),
eq(pluginCompanySettings.pluginId, pluginId),
));
const disabled = new Set<string>();
for (const row of rows) {
if (!row.enabled) disabled.add(row.companyId);
}
return disabled;
},
/** Get the normalized availability record for a single company/plugin pair. */
getCompanyAvailability: async (companyId: string, pluginId: string) => {
const plugin = await getById(pluginId);
if (!plugin || plugin.status === "uninstalled") return null;
const settings = await getCompanySettingsRow(companyId, pluginId);
return toCompanyAvailability(companyId, plugin, settings);
},
/** Update normalized company availability, persisting or deleting settings as needed. */
updateCompanyAvailability: async (
companyId: string,
pluginId: string,
input: UpdateCompanyPluginAvailability,
) => {
const plugin = await getById(pluginId);
if (!plugin || plugin.status === "uninstalled") throw notFound("Plugin not found");
const existing = await getCompanySettingsRow(companyId, pluginId);
if (!input.available) {
const row = await (existing
? db
.update(pluginCompanySettings)
.set({
enabled: false,
settingsJson: input.settingsJson ?? existing.settingsJson,
lastError: input.lastError ?? existing.lastError ?? null,
updatedAt: new Date(),
})
.where(eq(pluginCompanySettings.id, existing.id))
.returning()
.then((rows) => rows[0])
: db
.insert(pluginCompanySettings)
.values({
companyId,
pluginId,
enabled: false,
settingsJson: input.settingsJson ?? {},
lastError: input.lastError ?? null,
})
.returning()
.then((rows) => rows[0]));
return {
...toCompanyAvailability(companyId, plugin, row),
};
}
const row = await (existing
? db
.update(pluginCompanySettings)
.set({
enabled: true,
settingsJson: input.settingsJson ?? existing.settingsJson,
lastError: input.lastError ?? existing.lastError ?? null,
updatedAt: new Date(),
})
.where(eq(pluginCompanySettings.id, existing.id))
.returning()
.then((rows) => rows[0])
: db
.insert(pluginCompanySettings)
.values({
companyId,
pluginId,
enabled: true,
settingsJson: input.settingsJson ?? {},
lastError: input.lastError ?? null,
})
.returning()
.then((rows) => rows[0]));
return {
...toCompanyAvailability(companyId, plugin, row),
};
},
/**
* Ensure all companies have an explicit enabled row for this plugin.
*
* Company availability defaults to enabled when no row exists, but this
* helper persists explicit `enabled=true` rows so newly-installed plugins
* appear as enabled immediately and consistently in company-scoped views.
*/
seedEnabledForAllCompanies: async (pluginId: string) => {
const plugin = await getById(pluginId);
if (!plugin || plugin.status === "uninstalled") throw notFound("Plugin not found");
const companyRows = await db
.select({ id: companies.id })
.from(companies);
if (companyRows.length === 0) return 0;
const now = new Date();
await db
.insert(pluginCompanySettings)
.values(
companyRows.map((company) => ({
companyId: company.id,
pluginId,
enabled: true,
settingsJson: {},
lastError: null,
createdAt: now,
updatedAt: now,
})),
)
.onConflictDoNothing({
target: [pluginCompanySettings.companyId, pluginCompanySettings.pluginId],
});
return companyRows.length;
},
/**
* Record an error against a plugin's config (e.g. validation failure
* against the plugin's instanceConfigSchema).

13
server/src/services/plugin-secrets-handler.ts
View File

@@ -321,19 +321,6 @@ export function createPluginSecretsHandler(
throw secretNotFound(trimmedRef);
}
// ---------------------------------------------------------------
// 2b. Verify the plugin is available for the secret's company.
// This prevents cross-company secret access via UUID guessing.
// ---------------------------------------------------------------
const companyId = (secret as { companyId?: string }).companyId;
if (companyId) {
const availability = await registry.getCompanyAvailability(companyId, pluginId);
if (!availability || !availability.available) {
// Return the same error as "not found" to avoid leaking existence
throw secretNotFound(trimmedRef);
}
}
// ---------------------------------------------------------------
// 3. Fetch the latest version's material
// ---------------------------------------------------------------