From 271c2b9018da0930414bb99ce85dca34a3045c7d Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 13 Mar 2026 22:29:30 -0500 Subject: [PATCH] Implement markdown-first company package import export --- cli/src/commands/client/company.ts | 56 +- .../shared/src/types/company-portability.ts | 3 +- .../src/validators/company-portability.ts | 2 +- server/src/services/company-portability.ts | 606 ++++++++++++--- ui/src/pages/CompanySettings.tsx | 720 +++++++++++++++++- 5 files changed, 1230 insertions(+), 157 deletions(-) diff --git a/cli/src/commands/client/company.ts b/cli/src/commands/client/company.ts index b8ab3644..ae53864b 100644 --- a/cli/src/commands/client/company.ts +++ b/cli/src/commands/client/company.ts @@ -1,11 +1,10 @@ import { Command } from "commander"; -import { mkdir, readFile, stat, writeFile } from "node:fs/promises"; +import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises"; import path from "node:path"; import type { Company, CompanyPortabilityExportResult, CompanyPortabilityInclude, - CompanyPortabilityManifest, CompanyPortabilityPreviewResult, CompanyPortabilityImportResult, } from "@paperclipai/shared"; @@ -84,37 +83,39 @@ function isGithubUrl(input: string): boolean { return /^https?:\/\/github\.com\//i.test(input.trim()); } +async function collectPackageFiles(root: string, current: string, files: Record): Promise { + const entries = await readdir(current, { withFileTypes: true }); + for (const entry of entries) { + if (entry.name.startsWith(".git")) continue; + const absolutePath = path.join(current, entry.name); + if (entry.isDirectory()) { + await collectPackageFiles(root, absolutePath, files); + continue; + } + if (!entry.isFile() || !entry.name.endsWith(".md")) continue; + const relativePath = path.relative(root, absolutePath).replace(/\\/g, "/"); + files[relativePath] = await readFile(absolutePath, "utf8"); + } +} + async function resolveInlineSourceFromPath(inputPath: string): Promise<{ - manifest: CompanyPortabilityManifest; + rootPath: string; files: Record; }> { const resolved = path.resolve(inputPath); const resolvedStat = await stat(resolved); - const manifestPath = resolvedStat.isDirectory() - ? path.join(resolved, "paperclip.manifest.json") - : resolved; - const manifestBaseDir = path.dirname(manifestPath); - const manifestRaw = await readFile(manifestPath, "utf8"); - const manifest = JSON.parse(manifestRaw) as CompanyPortabilityManifest; + const rootDir = resolvedStat.isDirectory() ? resolved : path.dirname(resolved); const files: Record = {}; - - if (manifest.company?.path) { - const companyPath = manifest.company.path.replace(/\\/g, "/"); - files[companyPath] = await readFile(path.join(manifestBaseDir, companyPath), "utf8"); - } - for (const agent of manifest.agents ?? []) { - const agentPath = agent.path.replace(/\\/g, "/"); - files[agentPath] = await readFile(path.join(manifestBaseDir, agentPath), "utf8"); - } - - return { manifest, files }; + await collectPackageFiles(rootDir, rootDir, files); + return { + rootPath: path.basename(rootDir), + files, + }; } async function writeExportToFolder(outDir: string, exported: CompanyPortabilityExportResult): Promise { const root = path.resolve(outDir); await mkdir(root, { recursive: true }); - const manifestPath = path.join(root, "paperclip.manifest.json"); - await writeFile(manifestPath, JSON.stringify(exported.manifest, null, 2), "utf8"); for (const [relativePath, content] of Object.entries(exported.files)) { const normalized = relativePath.replace(/\\/g, "/"); const filePath = path.join(root, normalized); @@ -257,7 +258,7 @@ export function registerCompanyCommands(program: Command): void { addCommonClientOptions( company .command("export") - .description("Export a company into portable manifest + markdown files") + .description("Export a company into a portable markdown package") .argument("", "Company ID") .requiredOption("--out ", "Output directory") .option("--include ", "Comma-separated include set: company,agents", "company,agents") @@ -277,7 +278,8 @@ export function registerCompanyCommands(program: Command): void { { ok: true, out: path.resolve(opts.out!), - filesWritten: Object.keys(exported.files).length + 1, + rootPath: exported.rootPath, + filesWritten: Object.keys(exported.files).length, warningCount: exported.warnings.length, }, { json: ctx.json }, @@ -296,7 +298,7 @@ export function registerCompanyCommands(program: Command): void { addCommonClientOptions( company .command("import") - .description("Import a portable company package from local path, URL, or GitHub") + .description("Import a portable markdown company package from local path, URL, or GitHub") .requiredOption("--from ", "Source path or URL") .option("--include ", "Comma-separated include set: company,agents", "company,agents") .option("--target ", "Target mode: new | existing") @@ -343,7 +345,7 @@ export function registerCompanyCommands(program: Command): void { } let sourcePayload: - | { type: "inline"; manifest: CompanyPortabilityManifest; files: Record } + | { type: "inline"; rootPath?: string | null; files: Record } | { type: "url"; url: string } | { type: "github"; url: string }; @@ -355,7 +357,7 @@ export function registerCompanyCommands(program: Command): void { const inline = await resolveInlineSourceFromPath(from); sourcePayload = { type: "inline", - manifest: inline.manifest, + rootPath: inline.rootPath, files: inline.files, }; } diff --git a/packages/shared/src/types/company-portability.ts b/packages/shared/src/types/company-portability.ts index 389cd777..ce6be814 100644 --- a/packages/shared/src/types/company-portability.ts +++ b/packages/shared/src/types/company-portability.ts @@ -49,6 +49,7 @@ export interface CompanyPortabilityManifest { } export interface CompanyPortabilityExportResult { + rootPath: string; manifest: CompanyPortabilityManifest; files: Record; warnings: string[]; @@ -57,7 +58,7 @@ export interface CompanyPortabilityExportResult { export type CompanyPortabilitySource = | { type: "inline"; - manifest: CompanyPortabilityManifest; + rootPath?: string | null; files: Record; } | { diff --git a/packages/shared/src/validators/company-portability.ts b/packages/shared/src/validators/company-portability.ts index 4ba01a2c..7ce3e684 100644 --- a/packages/shared/src/validators/company-portability.ts +++ b/packages/shared/src/validators/company-portability.ts @@ -60,7 +60,7 @@ export const portabilityManifestSchema = z.object({ export const portabilitySourceSchema = z.discriminatedUnion("type", [ z.object({ type: z.literal("inline"), - manifest: portabilityManifestSchema, + rootPath: z.string().min(1).optional().nullable(), files: z.record(z.string()), }), z.object({ diff --git a/server/src/services/company-portability.ts b/server/src/services/company-portability.ts index f067e957..9927dffc 100644 --- a/server/src/services/company-portability.ts +++ b/server/src/services/company-portability.ts @@ -14,7 +14,7 @@ import type { CompanyPortabilityPreviewAgentPlan, CompanyPortabilityPreviewResult, } from "@paperclipai/shared"; -import { normalizeAgentUrlKey, portabilityManifestSchema } from "@paperclipai/shared"; +import { normalizeAgentUrlKey } from "@paperclipai/shared"; import { notFound, unprocessable } from "../errors.js"; import { accessService } from "./access.js"; import { agentService } from "./agents.js"; @@ -41,6 +41,10 @@ type MarkdownDoc = { body: string; }; +type CompanyPackageIncludeEntry = { + path: string; +}; + type ImportPlanInternal = { preview: CompanyPortabilityPreviewResult; source: ResolvedSource; @@ -146,6 +150,45 @@ function normalizeInclude(input?: Partial): CompanyPo }; } +function normalizePortablePath(input: string) { + const normalized = input.replace(/\\/g, "/").replace(/^\.\/+/, ""); + const parts: string[] = []; + for (const segment of normalized.split("/")) { + if (!segment || segment === ".") continue; + if (segment === "..") { + if (parts.length > 0) parts.pop(); + continue; + } + parts.push(segment); + } + return parts.join("/"); +} + +function resolvePortablePath(fromPath: string, targetPath: string) { + const baseDir = path.posix.dirname(fromPath.replace(/\\/g, "/")); + return normalizePortablePath(path.posix.join(baseDir, targetPath.replace(/\\/g, "/"))); +} + +function normalizeFileMap( + files: Record, + rootPath?: string | null, +): Record { + const normalizedRoot = rootPath ? normalizePortablePath(rootPath) : null; + const out: Record = {}; + for (const [rawPath, content] of Object.entries(files)) { + let nextPath = normalizePortablePath(rawPath); + if (normalizedRoot && nextPath === normalizedRoot) { + continue; + } + if (normalizedRoot && nextPath.startsWith(`${normalizedRoot}/`)) { + nextPath = nextPath.slice(normalizedRoot.length + 1); + } + if (!nextPath) continue; + out[nextPath] = content; + } + return out; +} + function ensureMarkdownPath(pathValue: string) { const normalized = pathValue.replace(/\\/g, "/"); if (!normalized.endsWith(".md")) { @@ -340,6 +383,129 @@ function renderCompanyAgentsSection(agentSummaries: Array<{ slug: string; name: return lines.join("\n"); } +function parseYamlScalar(rawValue: string): unknown { + const trimmed = rawValue.trim(); + if (trimmed === "") return ""; + if (trimmed === "null" || trimmed === "~") return null; + if (trimmed === "true") return true; + if (trimmed === "false") return false; + if (trimmed === "[]") return []; + if (trimmed === "{}") return {}; + if (/^-?\d+(\.\d+)?$/.test(trimmed)) return Number(trimmed); + if ( + trimmed.startsWith("\"") || + trimmed.startsWith("[") || + trimmed.startsWith("{") + ) { + try { + return JSON.parse(trimmed); + } catch { + return trimmed; + } + } + return trimmed; +} + +function prepareYamlLines(raw: string) { + return raw + .split("\n") + .map((line) => ({ + indent: line.match(/^ */)?.[0].length ?? 0, + content: line.trim(), + })) + .filter((line) => line.content.length > 0 && !line.content.startsWith("#")); +} + +function parseYamlBlock( + lines: Array<{ indent: number; content: string }>, + startIndex: number, + indentLevel: number, +): { value: unknown; nextIndex: number } { + let index = startIndex; + while (index < lines.length && lines[index]!.content.length === 0) { + index += 1; + } + if (index >= lines.length || lines[index]!.indent < indentLevel) { + return { value: {}, nextIndex: index }; + } + + const isArray = lines[index]!.indent === indentLevel && lines[index]!.content.startsWith("-"); + if (isArray) { + const values: unknown[] = []; + while (index < lines.length) { + const line = lines[index]!; + if (line.indent < indentLevel) break; + if (line.indent !== indentLevel || !line.content.startsWith("-")) break; + const remainder = line.content.slice(1).trim(); + index += 1; + if (!remainder) { + const nested = parseYamlBlock(lines, index, indentLevel + 2); + values.push(nested.value); + index = nested.nextIndex; + continue; + } + const inlineObjectSeparator = remainder.indexOf(":"); + if ( + inlineObjectSeparator > 0 && + !remainder.startsWith("\"") && + !remainder.startsWith("{") && + !remainder.startsWith("[") + ) { + const key = remainder.slice(0, inlineObjectSeparator).trim(); + const rawValue = remainder.slice(inlineObjectSeparator + 1).trim(); + const nextObject: Record = { + [key]: parseYamlScalar(rawValue), + }; + if (index < lines.length && lines[index]!.indent > indentLevel) { + const nested = parseYamlBlock(lines, index, indentLevel + 2); + if (isPlainRecord(nested.value)) { + Object.assign(nextObject, nested.value); + } + index = nested.nextIndex; + } + values.push(nextObject); + continue; + } + values.push(parseYamlScalar(remainder)); + } + return { value: values, nextIndex: index }; + } + + const record: Record = {}; + while (index < lines.length) { + const line = lines[index]!; + if (line.indent < indentLevel) break; + if (line.indent !== indentLevel) { + index += 1; + continue; + } + const separatorIndex = line.content.indexOf(":"); + if (separatorIndex <= 0) { + index += 1; + continue; + } + const key = line.content.slice(0, separatorIndex).trim(); + const remainder = line.content.slice(separatorIndex + 1).trim(); + index += 1; + if (!remainder) { + const nested = parseYamlBlock(lines, index, indentLevel + 2); + record[key] = nested.value; + index = nested.nextIndex; + continue; + } + record[key] = parseYamlScalar(remainder); + } + + return { value: record, nextIndex: index }; +} + +function parseYamlFrontmatter(raw: string): Record { + const prepared = prepareYamlLines(raw); + if (prepared.length === 0) return {}; + const parsed = parseYamlBlock(prepared, 0, prepared[0]!.indent); + return isPlainRecord(parsed.value) ? parsed.value : {}; +} + function parseFrontmatterMarkdown(raw: string): MarkdownDoc { const normalized = raw.replace(/\r\n/g, "\n"); if (!normalized.startsWith("---\n")) { @@ -351,41 +517,10 @@ function parseFrontmatterMarkdown(raw: string): MarkdownDoc { } const frontmatterRaw = normalized.slice(4, closing).trim(); const body = normalized.slice(closing + 5).trim(); - const frontmatter: Record = {}; - for (const line of frontmatterRaw.split("\n")) { - const idx = line.indexOf(":"); - if (idx <= 0) continue; - const key = line.slice(0, idx).trim(); - const rawValue = line.slice(idx + 1).trim(); - if (!key) continue; - if (rawValue === "null") { - frontmatter[key] = null; - continue; - } - if (rawValue === "true" || rawValue === "false") { - frontmatter[key] = rawValue === "true"; - continue; - } - if (/^-?\d+(\.\d+)?$/.test(rawValue)) { - frontmatter[key] = Number(rawValue); - continue; - } - try { - frontmatter[key] = JSON.parse(rawValue); - continue; - } catch { - frontmatter[key] = rawValue; - } - } - return { frontmatter, body }; -} - -async function fetchJson(url: string) { - const response = await fetch(url); - if (!response.ok) { - throw unprocessable(`Failed to fetch ${url}: ${response.status}`); - } - return response.json(); + return { + frontmatter: parseYamlFrontmatter(frontmatterRaw), + body, + }; } async function fetchText(url: string) { @@ -396,6 +531,15 @@ async function fetchText(url: string) { return response.text(); } +async function fetchOptionalText(url: string) { + const response = await fetch(url); + if (response.status === 404) return null; + if (!response.ok) { + throw unprocessable(`Failed to fetch ${url}: ${response.status}`); + } + return response.text(); +} + function dedupeRequiredSecrets(values: CompanyPortabilityManifest["requiredSecrets"]) { const seen = new Set(); const out: CompanyPortabilityManifest["requiredSecrets"] = []; @@ -408,7 +552,190 @@ function dedupeRequiredSecrets(values: CompanyPortabilityManifest["requiredSecre return out; } -function parseGitHubTreeUrl(rawUrl: string) { +function buildIncludes(paths: string[]): CompanyPackageIncludeEntry[] { + return paths.map((value) => ({ path: value })); +} + +function readCompanyApprovalDefault(frontmatter: Record) { + const topLevel = frontmatter.requireBoardApprovalForNewAgents; + if (typeof topLevel === "boolean") return topLevel; + const defaults = frontmatter.defaults; + if (isPlainRecord(defaults) && typeof defaults.requireBoardApprovalForNewAgents === "boolean") { + return defaults.requireBoardApprovalForNewAgents; + } + return true; +} + +function readIncludeEntries(frontmatter: Record): CompanyPackageIncludeEntry[] { + const includes = frontmatter.includes; + if (!Array.isArray(includes)) return []; + return includes.flatMap((entry) => { + if (typeof entry === "string") { + return [{ path: entry }]; + } + if (isPlainRecord(entry)) { + const pathValue = asString(entry.path); + return pathValue ? [{ path: pathValue }] : []; + } + return []; + }); +} + +function readAgentSecretRequirements( + frontmatter: Record, + agentSlug: string, +): CompanyPortabilityManifest["requiredSecrets"] { + const requirements = frontmatter.requirements; + const secretsFromRequirements = + isPlainRecord(requirements) && Array.isArray(requirements.secrets) + ? requirements.secrets + : []; + const legacyRequiredSecrets = Array.isArray(frontmatter.requiredSecrets) + ? frontmatter.requiredSecrets + : []; + const combined = [...secretsFromRequirements, ...legacyRequiredSecrets]; + + return combined.flatMap((entry) => { + if (typeof entry === "string" && entry.trim()) { + return [{ + key: entry.trim(), + description: `Set ${entry.trim()} for agent ${agentSlug}`, + agentSlug, + providerHint: null, + }]; + } + if (isPlainRecord(entry)) { + const key = asString(entry.key); + if (!key) return []; + return [{ + key, + description: asString(entry.description) ?? `Set ${key} for agent ${agentSlug}`, + agentSlug, + providerHint: asString(entry.providerHint), + }]; + } + return []; + }); +} + +function buildManifestFromPackageFiles( + files: Record, + opts?: { sourceLabel?: { companyId: string; companyName: string } | null }, +): ResolvedSource { + const normalizedFiles = normalizeFileMap(files); + const companyPath = + normalizedFiles["COMPANY.md"] + ?? undefined; + const resolvedCompanyPath = companyPath !== undefined + ? "COMPANY.md" + : Object.keys(normalizedFiles).find((entry) => entry.endsWith("/COMPANY.md") || entry === "COMPANY.md"); + if (!resolvedCompanyPath) { + throw unprocessable("Company package is missing COMPANY.md"); + } + + const companyDoc = parseFrontmatterMarkdown(normalizedFiles[resolvedCompanyPath]!); + const companyFrontmatter = companyDoc.frontmatter; + const companyName = + asString(companyFrontmatter.name) + ?? opts?.sourceLabel?.companyName + ?? "Imported Company"; + const companySlug = + asString(companyFrontmatter.slug) + ?? normalizeAgentUrlKey(companyName) + ?? "company"; + + const includeEntries = readIncludeEntries(companyFrontmatter); + const referencedAgentPaths = includeEntries + .map((entry) => resolvePortablePath(resolvedCompanyPath, entry.path)) + .filter((entry) => entry.endsWith("/AGENTS.md") || entry === "AGENTS.md"); + const discoveredAgentPaths = Object.keys(normalizedFiles).filter( + (entry) => entry.endsWith("/AGENTS.md") || entry === "AGENTS.md", + ); + const agentPaths = Array.from(new Set([...referencedAgentPaths, ...discoveredAgentPaths])).sort(); + + const manifest: CompanyPortabilityManifest = { + schemaVersion: 2, + generatedAt: new Date().toISOString(), + source: opts?.sourceLabel ?? null, + includes: { + company: true, + agents: true, + }, + company: { + path: resolvedCompanyPath, + name: companyName, + description: asString(companyFrontmatter.description), + brandColor: asString(companyFrontmatter.brandColor), + requireBoardApprovalForNewAgents: readCompanyApprovalDefault(companyFrontmatter), + }, + agents: [], + requiredSecrets: [], + }; + + const warnings: string[] = []; + for (const agentPath of agentPaths) { + const markdownRaw = normalizedFiles[agentPath]; + if (typeof markdownRaw !== "string") { + warnings.push(`Referenced agent file is missing from package: ${agentPath}`); + continue; + } + const agentDoc = parseFrontmatterMarkdown(markdownRaw); + const frontmatter = agentDoc.frontmatter; + const fallbackSlug = normalizeAgentUrlKey(path.posix.basename(path.posix.dirname(agentPath))) ?? "agent"; + const slug = asString(frontmatter.slug) ?? fallbackSlug; + const adapter = isPlainRecord(frontmatter.adapter) ? frontmatter.adapter : null; + const runtime = isPlainRecord(frontmatter.runtime) ? frontmatter.runtime : null; + const permissions = isPlainRecord(frontmatter.permissions) ? frontmatter.permissions : {}; + const metadata = isPlainRecord(frontmatter.metadata) ? frontmatter.metadata : null; + const adapterConfig = isPlainRecord(adapter?.config) + ? adapter.config + : isPlainRecord(frontmatter.adapterConfig) + ? frontmatter.adapterConfig + : {}; + const runtimeConfig = runtime ?? (isPlainRecord(frontmatter.runtimeConfig) ? frontmatter.runtimeConfig : {}); + const title = asString(frontmatter.title); + const capabilities = asString(frontmatter.capabilities); + + manifest.agents.push({ + slug, + name: asString(frontmatter.name) ?? title ?? slug, + path: agentPath, + role: asString(frontmatter.role) ?? "agent", + title, + icon: asString(frontmatter.icon), + capabilities, + reportsToSlug: asString(frontmatter.reportsTo), + adapterType: asString(adapter?.type) ?? asString(frontmatter.adapterType) ?? "process", + adapterConfig, + runtimeConfig, + permissions, + budgetMonthlyCents: + typeof frontmatter.budgetMonthlyCents === "number" && Number.isFinite(frontmatter.budgetMonthlyCents) + ? Math.max(0, Math.floor(frontmatter.budgetMonthlyCents)) + : 0, + metadata, + }); + + manifest.requiredSecrets.push(...readAgentSecretRequirements(frontmatter, slug)); + + if (frontmatter.kind !== "agent") { + warnings.push(`Agent markdown ${agentPath} does not declare kind: agent in frontmatter.`); + } + } + + manifest.requiredSecrets = dedupeRequiredSecrets(manifest.requiredSecrets); + return { + manifest, + files: normalizedFiles, + warnings, + }; +} + +function isGitCommitRef(value: string) { + return /^[0-9a-f]{40}$/i.test(value.trim()); +} + +function parseGitHubSourceUrl(rawUrl: string) { const url = new URL(rawUrl); if (url.hostname !== "github.com") { throw unprocessable("GitHub source must use github.com URL"); @@ -421,11 +748,21 @@ function parseGitHubTreeUrl(rawUrl: string) { const repo = parts[1]!.replace(/\.git$/i, ""); let ref = "main"; let basePath = ""; + let companyPath = "COMPANY.md"; if (parts[2] === "tree") { ref = parts[3] ?? "main"; basePath = parts.slice(4).join("/"); + } else if (parts[2] === "blob") { + ref = parts[3] ?? "main"; + const blobPath = parts.slice(4).join("/"); + if (!blobPath) { + throw unprocessable("Invalid GitHub blob URL"); + } + companyPath = blobPath; + basePath = path.posix.dirname(blobPath); + if (basePath === ".") basePath = ""; } - return { owner, repo, ref, basePath }; + return { owner, repo, ref, basePath, companyPath }; } function resolveRawGitHubUrl(owner: string, repo: string, ref: string, filePath: string) { @@ -485,65 +822,80 @@ export function companyPortabilityService(db: Db) { async function resolveSource(source: CompanyPortabilityPreview["source"]): Promise { if (source.type === "inline") { - return { - manifest: portabilityManifestSchema.parse(source.manifest), - files: source.files, - warnings: [], - }; + return buildManifestFromPackageFiles( + normalizeFileMap(source.files, source.rootPath), + ); } if (source.type === "url") { - const manifestJson = await fetchJson(source.url); - const manifest = portabilityManifestSchema.parse(manifestJson); - const base = new URL(".", source.url); - const files: Record = {}; - const warnings: string[] = []; + const normalizedUrl = source.url.trim(); + const companyUrl = normalizedUrl.endsWith(".md") + ? normalizedUrl + : new URL("COMPANY.md", normalizedUrl.endsWith("/") ? normalizedUrl : `${normalizedUrl}/`).toString(); + const companyMarkdown = await fetchText(companyUrl); + const files: Record = { + "COMPANY.md": companyMarkdown, + }; + const companyDoc = parseFrontmatterMarkdown(companyMarkdown); + const includeEntries = readIncludeEntries(companyDoc.frontmatter); - if (manifest.company?.path) { - const companyPath = ensureMarkdownPath(manifest.company.path); - files[companyPath] = await fetchText(new URL(companyPath, base).toString()); + for (const includeEntry of includeEntries) { + const includePath = normalizePortablePath(includeEntry.path); + if (!includePath.endsWith(".md")) continue; + const includeUrl = new URL(includeEntry.path, companyUrl).toString(); + files[includePath] = await fetchText(includeUrl); } - for (const agent of manifest.agents) { - const filePath = ensureMarkdownPath(agent.path); - files[filePath] = await fetchText(new URL(filePath, base).toString()); - } - - return { manifest, files, warnings }; + return buildManifestFromPackageFiles(files); } - const parsed = parseGitHubTreeUrl(source.url); + const parsed = parseGitHubSourceUrl(source.url); let ref = parsed.ref; - const manifestRelativePath = [parsed.basePath, "paperclip.manifest.json"].filter(Boolean).join("/"); - let manifest: CompanyPortabilityManifest | null = null; const warnings: string[] = []; + if (!isGitCommitRef(ref)) { + warnings.push("GitHub source is not pinned to a commit SHA; imports may drift if the ref changes."); + } + const companyRelativePath = parsed.companyPath === "COMPANY.md" + ? [parsed.basePath, "COMPANY.md"].filter(Boolean).join("/") + : parsed.companyPath; + let companyMarkdown: string | null = null; try { - manifest = portabilityManifestSchema.parse( - await fetchJson(resolveRawGitHubUrl(parsed.owner, parsed.repo, ref, manifestRelativePath)), + companyMarkdown = await fetchOptionalText( + resolveRawGitHubUrl(parsed.owner, parsed.repo, ref, companyRelativePath), ); } catch (err) { if (ref === "main") { ref = "master"; warnings.push("GitHub ref main not found; falling back to master."); - manifest = portabilityManifestSchema.parse( - await fetchJson(resolveRawGitHubUrl(parsed.owner, parsed.repo, ref, manifestRelativePath)), + companyMarkdown = await fetchOptionalText( + resolveRawGitHubUrl(parsed.owner, parsed.repo, ref, companyRelativePath), ); } else { throw err; } } + if (!companyMarkdown) { + throw unprocessable("GitHub company package is missing COMPANY.md"); + } - const files: Record = {}; - if (manifest.company?.path) { - files[manifest.company.path] = await fetchText( - resolveRawGitHubUrl(parsed.owner, parsed.repo, ref, [parsed.basePath, manifest.company.path].filter(Boolean).join("/")), + const companyPath = parsed.companyPath === "COMPANY.md" + ? "COMPANY.md" + : normalizePortablePath(path.posix.relative(parsed.basePath || ".", parsed.companyPath)); + const files: Record = { + [companyPath]: companyMarkdown, + }; + const companyDoc = parseFrontmatterMarkdown(companyMarkdown); + const includeEntries = readIncludeEntries(companyDoc.frontmatter); + for (const includeEntry of includeEntries) { + const repoPath = [parsed.basePath, includeEntry.path].filter(Boolean).join("/"); + if (!repoPath.endsWith(".md")) continue; + files[normalizePortablePath(includeEntry.path)] = await fetchText( + resolveRawGitHubUrl(parsed.owner, parsed.repo, ref, repoPath), ); } - for (const agent of manifest.agents) { - files[agent.path] = await fetchText( - resolveRawGitHubUrl(parsed.owner, parsed.repo, ref, [parsed.basePath, agent.path].filter(Boolean).join("/")), - ); - } - return { manifest, files, warnings }; + + const resolved = buildManifestFromPackageFiles(files); + resolved.warnings.unshift(...warnings); + return resolved; } async function exportBundle( @@ -557,20 +909,7 @@ export function companyPortabilityService(db: Db) { const files: Record = {}; const warnings: string[] = []; const requiredSecrets: CompanyPortabilityManifest["requiredSecrets"] = []; - const generatedAt = new Date().toISOString(); - - const manifest: CompanyPortabilityManifest = { - schemaVersion: 1, - generatedAt, - source: { - companyId: company.id, - companyName: company.name, - }, - includes: include, - company: null, - agents: [], - requiredSecrets: [], - }; + const rootPath = normalizeAgentUrlKey(company.name) ?? "company-package"; const allAgentRows = include.agents ? await agents.list(companyId, { includeTerminated: true }) : []; const agentRows = allAgentRows.filter((agent) => agent.status !== "terminated"); @@ -589,29 +928,32 @@ export function companyPortabilityService(db: Db) { idToSlug.set(agent.id, slug); } - if (include.company) { + { const companyPath = "COMPANY.md"; const companyAgentSummaries = agentRows.map((agent) => ({ slug: idToSlug.get(agent.id) ?? "agent", name: agent.name, })); + const includes = include.agents + ? buildIncludes( + companyAgentSummaries.map((agent) => `agents/${agent.slug}/AGENTS.md`), + ) + : []; files[companyPath] = buildMarkdown( { + schema: "company-packages/v0.1", kind: "company", + slug: rootPath, name: company.name, description: company.description ?? null, brandColor: company.brandColor ?? null, - requireBoardApprovalForNewAgents: company.requireBoardApprovalForNewAgents, + defaults: { + requireBoardApprovalForNewAgents: company.requireBoardApprovalForNewAgents, + }, + includes, }, renderCompanyAgentsSection(companyAgentSummaries), ); - manifest.company = { - path: companyPath, - name: company.name, - description: company.description ?? null, - brandColor: company.brandColor ?? null, - requireBoardApprovalForNewAgents: company.requireBoardApprovalForNewAgents, - }; } if (include.agents) { @@ -647,46 +989,52 @@ export function companyPortabilityService(db: Db) { files[agentPath] = buildMarkdown( { + schema: "company-packages/v0.1", name: agent.name, slug, - role: agent.role, - adapterType: agent.adapterType, kind: "agent", + role: agent.role, + title: agent.title ?? null, icon: agent.icon ?? null, capabilities: agent.capabilities ?? null, reportsTo: reportsToSlug, - runtimeConfig: portableRuntimeConfig, + adapter: { + type: agent.adapterType, + config: portableAdapterConfig, + }, + runtime: portableRuntimeConfig, permissions: portablePermissions, - adapterConfig: portableAdapterConfig, - requiredSecrets: agentRequiredSecrets, + budgetMonthlyCents: agent.budgetMonthlyCents ?? 0, + metadata: (agent.metadata as Record | null) ?? null, + requirements: agentRequiredSecrets.length > 0 + ? { + secrets: agentRequiredSecrets.map((secret) => ({ + key: secret.key, + description: secret.description, + providerHint: secret.providerHint, + })), + } + : {}, }, instructions.body, ); - - manifest.agents.push({ - slug, - name: agent.name, - path: agentPath, - role: agent.role, - title: agent.title ?? null, - icon: agent.icon ?? null, - capabilities: agent.capabilities ?? null, - reportsToSlug, - adapterType: agent.adapterType, - adapterConfig: portableAdapterConfig, - runtimeConfig: portableRuntimeConfig, - permissions: portablePermissions, - budgetMonthlyCents: agent.budgetMonthlyCents ?? 0, - metadata: (agent.metadata as Record | null) ?? null, - }); } } - manifest.requiredSecrets = dedupeRequiredSecrets(requiredSecrets); + const resolved = buildManifestFromPackageFiles(files, { + sourceLabel: { + companyId: company.id, + companyName: company.name, + }, + }); + resolved.manifest.includes = include; + resolved.manifest.requiredSecrets = dedupeRequiredSecrets(requiredSecrets); + resolved.warnings.unshift(...warnings); return { - manifest, + rootPath, + manifest: resolved.manifest, files, - warnings, + warnings: resolved.warnings, }; } @@ -702,11 +1050,17 @@ export function companyPortabilityService(db: Db) { errors.push("Manifest does not include company metadata."); } - const selectedSlugs = input.agents && input.agents !== "all" - ? Array.from(new Set(input.agents)) - : manifest.agents.map((agent) => agent.slug); + const selectedSlugs = include.agents + ? ( + input.agents && input.agents !== "all" + ? Array.from(new Set(input.agents)) + : manifest.agents.map((agent) => agent.slug) + ) + : []; - const selectedAgents = manifest.agents.filter((agent) => selectedSlugs.includes(agent.slug)); + const selectedAgents = include.agents + ? manifest.agents.filter((agent) => selectedSlugs.includes(agent.slug)) + : []; const selectedMissing = selectedSlugs.filter((slug) => !manifest.agents.some((agent) => agent.slug === slug)); for (const missing of selectedMissing) { errors.push(`Selected agent slug not found in manifest: ${missing}`); diff --git a/ui/src/pages/CompanySettings.tsx b/ui/src/pages/CompanySettings.tsx index 95ba1d75..13c55989 100644 --- a/ui/src/pages/CompanySettings.tsx +++ b/ui/src/pages/CompanySettings.tsx @@ -1,12 +1,20 @@ -import { useEffect, useState } from "react"; +import { useEffect, useMemo, useRef, useState, type ChangeEvent } from "react"; import { useMutation, useQueryClient } from "@tanstack/react-query"; +import type { + CompanyPortabilityCollisionStrategy, + CompanyPortabilityExportResult, + CompanyPortabilityPreviewRequest, + CompanyPortabilityPreviewResult, + CompanyPortabilitySource, +} from "@paperclipai/shared"; import { useCompany } from "../context/CompanyContext"; import { useBreadcrumbs } from "../context/BreadcrumbContext"; +import { useToast } from "../context/ToastContext"; import { companiesApi } from "../api/companies"; import { accessApi } from "../api/access"; import { queryKeys } from "../lib/queryKeys"; import { Button } from "@/components/ui/button"; -import { Settings, Check } from "lucide-react"; +import { Settings, Check, Download, Github, Link2, Upload } from "lucide-react"; import { CompanyPatternIcon } from "../components/CompanyPatternIcon"; import { Field, @@ -28,7 +36,9 @@ export function CompanySettings() { setSelectedCompanyId } = useCompany(); const { setBreadcrumbs } = useBreadcrumbs(); + const { pushToast } = useToast(); const queryClient = useQueryClient(); + const packageInputRef = useRef(null); // General settings local state const [companyName, setCompanyName] = useState(""); @@ -47,6 +57,18 @@ export function CompanySettings() { const [inviteSnippet, setInviteSnippet] = useState(null); const [snippetCopied, setSnippetCopied] = useState(false); const [snippetCopyDelightId, setSnippetCopyDelightId] = useState(0); + const [packageIncludeCompany, setPackageIncludeCompany] = useState(true); + const [packageIncludeAgents, setPackageIncludeAgents] = useState(true); + const [importSourceMode, setImportSourceMode] = useState<"github" | "url" | "local">("github"); + const [importUrl, setImportUrl] = useState(""); + const [importTargetMode, setImportTargetMode] = useState<"existing" | "new">("existing"); + const [newCompanyName, setNewCompanyName] = useState(""); + const [collisionStrategy, setCollisionStrategy] = useState("rename"); + const [localPackage, setLocalPackage] = useState<{ + rootPath: string | null; + files: Record; + } | null>(null); + const [importPreview, setImportPreview] = useState(null); const generalDirty = !!selectedCompany && @@ -54,6 +76,57 @@ export function CompanySettings() { description !== (selectedCompany.description ?? "") || brandColor !== (selectedCompany.brandColor ?? "")); + const packageInclude = useMemo( + () => ({ + company: packageIncludeCompany, + agents: packageIncludeAgents + }), + [packageIncludeAgents, packageIncludeCompany] + ); + + const importSource = useMemo(() => { + if (importSourceMode === "local") { + if (!localPackage || Object.keys(localPackage.files).length === 0) return null; + return { + type: "inline", + rootPath: localPackage.rootPath, + files: localPackage.files + }; + } + const trimmed = importUrl.trim(); + if (!trimmed) return null; + return importSourceMode === "github" + ? { type: "github", url: trimmed } + : { type: "url", url: trimmed }; + }, [importSourceMode, importUrl, localPackage]); + + const importPayload = useMemo(() => { + if (!importSource) return null; + return { + source: importSource, + include: packageInclude, + target: + importTargetMode === "new" + ? { + mode: "new_company", + newCompanyName: newCompanyName.trim() || null + } + : { + mode: "existing_company", + companyId: selectedCompanyId! + }, + agents: "all", + collisionStrategy + }; + }, [ + collisionStrategy, + importSource, + importTargetMode, + newCompanyName, + packageInclude, + selectedCompanyId + ]); + const generalMutation = useMutation({ mutationFn: (data: { name: string; @@ -75,6 +148,102 @@ export function CompanySettings() { } }); + const exportMutation = useMutation({ + mutationFn: () => + companiesApi.exportBundle(selectedCompanyId!, { + include: packageInclude + }), + onSuccess: async (exported) => { + await downloadCompanyPackage(exported); + pushToast({ + tone: "success", + title: "Company package exported", + body: `${exported.rootPath}.tar downloaded with ${Object.keys(exported.files).length} file${Object.keys(exported.files).length === 1 ? "" : "s"}.` + }); + if (exported.warnings.length > 0) { + pushToast({ + tone: "warn", + title: "Export completed with warnings", + body: exported.warnings[0] + }); + } + }, + onError: (err) => { + pushToast({ + tone: "error", + title: "Export failed", + body: err instanceof Error ? err.message : "Failed to export company package" + }); + } + }); + + const previewImportMutation = useMutation({ + mutationFn: (payload: CompanyPortabilityPreviewRequest) => + companiesApi.importPreview(payload), + onSuccess: (preview) => { + setImportPreview(preview); + if (preview.errors.length > 0) { + pushToast({ + tone: "warn", + title: "Import preview found issues", + body: preview.errors[0] + }); + return; + } + pushToast({ + tone: "success", + title: "Import preview ready", + body: `${preview.plan.agentPlans.length} agent action${preview.plan.agentPlans.length === 1 ? "" : "s"} planned.` + }); + }, + onError: (err) => { + setImportPreview(null); + pushToast({ + tone: "error", + title: "Import preview failed", + body: err instanceof Error ? err.message : "Failed to preview company package" + }); + } + }); + + const importPackageMutation = useMutation({ + mutationFn: (payload: CompanyPortabilityPreviewRequest) => + companiesApi.importBundle(payload), + onSuccess: async (result) => { + await Promise.all([ + queryClient.invalidateQueries({ queryKey: queryKeys.companies.all }), + queryClient.invalidateQueries({ queryKey: queryKeys.companies.stats }), + queryClient.invalidateQueries({ queryKey: queryKeys.agents.list(result.company.id) }), + queryClient.invalidateQueries({ queryKey: queryKeys.org(result.company.id) }) + ]); + if (importTargetMode === "new") { + setSelectedCompanyId(result.company.id); + } + pushToast({ + tone: "success", + title: "Company package imported", + body: `${result.agents.filter((agent) => agent.action !== "skipped").length} agent${result.agents.filter((agent) => agent.action !== "skipped").length === 1 ? "" : "s"} applied.` + }); + if (result.warnings.length > 0) { + pushToast({ + tone: "warn", + title: "Import completed with warnings", + body: result.warnings[0] + }); + } + setImportPreview(null); + setLocalPackage(null); + setImportUrl(""); + }, + onError: (err) => { + pushToast({ + tone: "error", + title: "Import failed", + body: err instanceof Error ? err.message : "Failed to import company package" + }); + } + }); + const inviteMutation = useMutation({ mutationFn: () => accessApi.createOpenClawInvitePrompt(selectedCompanyId!), @@ -134,6 +303,21 @@ export function CompanySettings() { setSnippetCopied(false); setSnippetCopyDelightId(0); }, [selectedCompanyId]); + + useEffect(() => { + setImportPreview(null); + }, [ + collisionStrategy, + importSourceMode, + importTargetMode, + importUrl, + localPackage, + newCompanyName, + packageIncludeAgents, + packageIncludeCompany, + selectedCompanyId + ]); + const archiveMutation = useMutation({ mutationFn: ({ companyId, @@ -178,6 +362,64 @@ export function CompanySettings() { }); } + async function handleChooseLocalPackage( + event: ChangeEvent + ) { + const selection = event.target.files; + if (!selection || selection.length === 0) { + setLocalPackage(null); + return; + } + try { + const parsed = await readLocalPackageSelection(selection); + setLocalPackage(parsed); + pushToast({ + tone: "success", + title: "Local package loaded", + body: `${Object.keys(parsed.files).length} markdown file${Object.keys(parsed.files).length === 1 ? "" : "s"} ready for preview.` + }); + } catch (err) { + setLocalPackage(null); + pushToast({ + tone: "error", + title: "Failed to read local package", + body: err instanceof Error ? err.message : "Could not read selected files" + }); + } finally { + event.target.value = ""; + } + } + + function handlePreviewImport() { + if (!importPayload) { + pushToast({ + tone: "warn", + title: "Source required", + body: + importSourceMode === "local" + ? "Choose a local folder with COMPANY.md before previewing." + : "Enter a company package URL before previewing." + }); + return; + } + previewImportMutation.mutate(importPayload); + } + + function handleApplyImport() { + if (!importPayload) { + pushToast({ + tone: "warn", + title: "Source required", + body: + importSourceMode === "local" + ? "Choose a local folder with COMPANY.md before importing." + : "Enter a company package URL before importing." + }); + return; + } + importPackageMutation.mutate(importPayload); + } + return (
@@ -379,6 +621,355 @@ export function CompanySettings() {
+ {/* Import / Export */} +
+
+ Company Packages +
+ +
+
+
+
Export markdown package
+

+ Download a markdown-first company package as a single tar file. +

+
+ +
+ +
+ + +
+ + {exportMutation.data && ( +
+
+ Last export +
+
+ {exportMutation.data.rootPath}.tar with{" "} + {Object.keys(exportMutation.data.files).length} file + {Object.keys(exportMutation.data.files).length === 1 ? "" : "s"}. +
+
+ {Object.keys(exportMutation.data.files).map((filePath) => ( + + {filePath} + + ))} +
+ {exportMutation.data.warnings.length > 0 && ( +
+ {exportMutation.data.warnings.map((warning) => ( +
{warning}
+ ))} +
+ )} +
+ )} +
+ +
+
+
Import company package
+

+ Preview a GitHub repo, direct COMPANY.md URL, or local folder before applying it. +

+
+ +
+ + + +
+ + {importSourceMode === "local" ? ( +
+ +
+ + {localPackage && ( + + {localPackage.rootPath ?? "package"} with{" "} + {Object.keys(localPackage.files).length} markdown file + {Object.keys(localPackage.files).length === 1 ? "" : "s"} + + )} +
+ {!localPackage && ( +

+ Select a folder that contains COMPANY.md and any referenced + AGENTS.md files. +

+ )} +
+ ) : ( + + setImportUrl(e.target.value)} + /> + + )} + +
+ + + + + + +
+ + {importTargetMode === "new" && ( + + setNewCompanyName(e.target.value)} + placeholder="Imported Company" + /> + + )} + +
+ + +
+ + {importPreview && ( +
+
+
+
+ Company action +
+
+ {importPreview.plan.companyAction} +
+
+
+
+ Agent actions +
+
+ {importPreview.plan.agentPlans.length} +
+
+
+ + {importPreview.plan.agentPlans.length > 0 && ( +
+ {importPreview.plan.agentPlans.map((agentPlan) => ( +
+
+ + {agentPlan.slug} {"->"} {agentPlan.plannedName} + + + {agentPlan.action} + +
+ {agentPlan.reason && ( +
+ {agentPlan.reason} +
+ )} +
+ ))} +
+ )} + + {importPreview.requiredSecrets.length > 0 && ( +
+
+ Required secrets +
+ {importPreview.requiredSecrets.map((secret) => ( +
+ {secret.key} + {secret.agentSlug ? ` for ${secret.agentSlug}` : ""} +
+ ))} +
+ )} + + {importPreview.warnings.length > 0 && ( +
+ {importPreview.warnings.map((warning) => ( +
{warning}
+ ))} +
+ )} + + {importPreview.errors.length > 0 && ( +
+ {importPreview.errors.map((error) => ( +
{error}
+ ))} +
+ )} +
+ )} +
+
+ {/* Danger Zone */}
@@ -435,6 +1026,131 @@ export function CompanySettings() { ); } +async function readLocalPackageSelection(fileList: FileList): Promise<{ + rootPath: string | null; + files: Record; +}> { + const files: Record = {}; + let rootPath: string | null = null; + + for (const file of Array.from(fileList)) { + const relativePath = + (file as File & { webkitRelativePath?: string }).webkitRelativePath?.replace( + /\\/g, + "/" + ) || file.name; + if (!relativePath.endsWith(".md")) continue; + const topLevel = relativePath.split("/")[0] ?? null; + if (!rootPath && topLevel) rootPath = topLevel; + files[relativePath] = await file.text(); + } + + if (Object.keys(files).length === 0) { + throw new Error("No markdown files were found in the selected folder."); + } + + return { rootPath, files }; +} + +async function downloadCompanyPackage( + exported: CompanyPortabilityExportResult +): Promise { + const tarBytes = createTarArchive(exported.files, exported.rootPath); + const tarBuffer = new ArrayBuffer(tarBytes.byteLength); + new Uint8Array(tarBuffer).set(tarBytes); + const blob = new Blob( + [tarBuffer], + { + type: "application/x-tar" + } + ); + const url = URL.createObjectURL(blob); + const anchor = document.createElement("a"); + anchor.href = url; + anchor.download = `${exported.rootPath}.tar`; + document.body.appendChild(anchor); + anchor.click(); + anchor.remove(); + window.setTimeout(() => URL.revokeObjectURL(url), 1000); +} + +function createTarArchive( + files: Record, + rootPath: string +): Uint8Array { + const encoder = new TextEncoder(); + const chunks: Uint8Array[] = []; + + for (const [relativePath, contents] of Object.entries(files)) { + const tarPath = `${rootPath}/${relativePath}`.replace(/\\/g, "/"); + const body = encoder.encode(contents); + chunks.push(buildTarHeader(tarPath, body.length)); + chunks.push(body); + const remainder = body.length % 512; + if (remainder > 0) { + chunks.push(new Uint8Array(512 - remainder)); + } + } + + chunks.push(new Uint8Array(1024)); + + const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0); + const archive = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + archive.set(chunk, offset); + offset += chunk.length; + } + return archive; +} + +function buildTarHeader(pathname: string, size: number): Uint8Array { + const header = new Uint8Array(512); + writeTarString(header, 0, 100, pathname); + writeTarOctal(header, 100, 8, 0o644); + writeTarOctal(header, 108, 8, 0); + writeTarOctal(header, 116, 8, 0); + writeTarOctal(header, 124, 12, size); + writeTarOctal(header, 136, 12, Math.floor(Date.now() / 1000)); + for (let i = 148; i < 156; i += 1) { + header[i] = 32; + } + header[156] = "0".charCodeAt(0); + writeTarString(header, 257, 6, "ustar"); + writeTarString(header, 263, 2, "00"); + const checksum = header.reduce((sum, byte) => sum + byte, 0); + writeTarChecksum(header, checksum); + return header; +} + +function writeTarString( + target: Uint8Array, + offset: number, + length: number, + value: string +) { + const encoded = new TextEncoder().encode(value); + target.set(encoded.slice(0, length), offset); +} + +function writeTarOctal( + target: Uint8Array, + offset: number, + length: number, + value: number +) { + const stringValue = value.toString(8).padStart(length - 1, "0"); + writeTarString(target, offset, length - 1, stringValue); + target[offset + length - 1] = 0; +} + +function writeTarChecksum(target: Uint8Array, checksum: number) { + const stringValue = checksum.toString(8).padStart(6, "0"); + writeTarString(target, 148, 6, stringValue); + target[154] = 0; + target[155] = 32; +} + function buildAgentSnippet(input: AgentSnippetInput) { const candidateUrls = buildCandidateOnboardingUrls(input); const resolutionTestUrl = buildResolutionTestUrl(input);