aevgarik/paperclip
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refine codex runtime skills and portability assets

Browse Source
This commit is contained in:
dotta
2026-03-19 07:15:36 -05:00
parent 01afa92424
commit b4e06c63e2
12 changed files with 277 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
server/src/__tests__/codex-local-execute.test.ts
View File

@@ -56,6 +56,7 @@ describe("codex execute", () => {
"company-1",
"codex-home",
);
const workspaceSkill = path.join(workspace, ".agents", "skills", "paperclip");
await fs.mkdir(workspace, { recursive: true });
await fs.mkdir(sharedCodexHome, { recursive: true });
await fs.writeFile(path.join(sharedCodexHome, "auth.json"), '{"token":"shared"}\n', "utf8");
@@ -124,13 +125,12 @@ describe("codex execute", () => {
const isolatedAuth = path.join(isolatedCodexHome, "auth.json");
const isolatedConfig = path.join(isolatedCodexHome, "config.toml");
const isolatedSkill = path.join(isolatedCodexHome, "skills", "paperclip");
expect((await fs.lstat(isolatedAuth)).isSymbolicLink()).toBe(true);
expect(await fs.realpath(isolatedAuth)).toBe(await fs.realpath(path.join(sharedCodexHome, "auth.json")));
expect((await fs.lstat(isolatedConfig)).isFile()).toBe(true);
expect(await fs.readFile(isolatedConfig, "utf8")).toBe('model = "codex-mini-latest"\n');
expect((await fs.lstat(isolatedSkill)).isSymbolicLink()).toBe(true);
expect((await fs.lstat(workspaceSkill)).isSymbolicLink()).toBe(true);
expect(logs).toContainEqual(
expect.objectContaining({
stream: "stdout",
@@ -217,6 +217,7 @@ describe("codex execute", () => {
const capture = JSON.parse(await fs.readFile(capturePath, "utf8")) as CapturePayload;
expect(capture.codexHome).toBe(explicitCodexHome);
expect((await fs.lstat(path.join(workspace, ".agents", "skills", "paperclip"))).isSymbolicLink()).toBe(true);
await expect(fs.lstat(path.join(paperclipHome, "instances", "worktree-1", "codex-home"))).rejects.toThrow();
} finally {
if (previousHome === undefined) delete process.env.HOME;

29
server/src/__tests__/codex-local-skill-injection.test.ts
View File

@@ -142,4 +142,33 @@ describe("codex local adapter skill injection", () => {
}),
);
});
it("preserves other live Paperclip skill symlinks in the shared workspace skill directory", async () => {
const currentRepo = await makeTempDir("paperclip-codex-current-");
const skillsHome = await makeTempDir("paperclip-codex-home-");
cleanupDirs.add(currentRepo);
cleanupDirs.add(skillsHome);
await createPaperclipRepoSkill(currentRepo, "paperclip");
await createPaperclipRepoSkill(currentRepo, "agent-browser");
await fs.symlink(
path.join(currentRepo, "skills", "agent-browser"),
path.join(skillsHome, "agent-browser"),
);
await ensureCodexSkillsInjected(async () => {}, {
skillsHome,
skillsEntries: [{
key: paperclipKey,
runtimeName: "paperclip",
source: path.join(currentRepo, "skills", "paperclip"),
}],
});
expect((await fs.lstat(path.join(skillsHome, "paperclip"))).isSymbolicLink()).toBe(true);
expect((await fs.lstat(path.join(skillsHome, "agent-browser"))).isSymbolicLink()).toBe(true);
expect(await fs.realpath(path.join(skillsHome, "agent-browser"))).toBe(
await fs.realpath(path.join(currentRepo, "skills", "agent-browser")),
);
});
});

126
server/src/__tests__/codex-local-skill-sync.test.ts
View File

@@ -11,13 +11,6 @@ async function makeTempDir(prefix: string): Promise<string> {
return fs.mkdtemp(path.join(os.tmpdir(), prefix));
}
async function createSkillDir(root: string, name: string) {
const skillDir = path.join(root, name);
await fs.mkdir(skillDir, { recursive: true });
await fs.writeFile(path.join(skillDir, "SKILL.md"), `---\nname: ${name}\n---\n`, "utf8");
return skillDir;
}
describe("codex local skill sync", () => {
const paperclipKey = "paperclipai/paperclip/paperclip";
const cleanupDirs = new Set<string>();
@@ -27,7 +20,7 @@ describe("codex local skill sync", () => {
cleanupDirs.clear();
});
it("reports configured Paperclip skills and installs them into the Codex skills home", async () => {
it("reports configured Paperclip skills for workspace injection on the next run", async () => {
const codexHome = await makeTempDir("paperclip-codex-skill-sync-");
cleanupDirs.add(codexHome);
@@ -46,65 +39,14 @@ describe("codex local skill sync", () => {
} as const;
const before = await listCodexSkills(ctx);
expect(before.mode).toBe("persistent");
expect(before.mode).toBe("ephemeral");
expect(before.desiredSkills).toContain(paperclipKey);
expect(before.entries.find((entry) => entry.key === paperclipKey)?.required).toBe(true);
expect(before.entries.find((entry) => entry.key === paperclipKey)?.state).toBe("missing");
const after = await syncCodexSkills(ctx, [paperclipKey]);
expect(after.entries.find((entry) => entry.key === paperclipKey)?.state).toBe("installed");
expect((await fs.lstat(path.join(codexHome, "skills", "paperclip"))).isSymbolicLink()).toBe(true);
expect(before.entries.find((entry) => entry.key === paperclipKey)?.state).toBe("configured");
expect(before.entries.find((entry) => entry.key === paperclipKey)?.detail).toContain(".agents/skills");
});
it("isolates default Codex skills by company when CODEX_HOME comes from process env", async () => {
const sharedCodexHome = await makeTempDir("paperclip-codex-skill-scope-");
cleanupDirs.add(sharedCodexHome);
const previousCodexHome = process.env.CODEX_HOME;
process.env.CODEX_HOME = sharedCodexHome;
try {
const companyAContext = {
agentId: "agent-a",
companyId: "company-a",
adapterType: "codex_local",
config: {
env: {},
paperclipSkillSync: {
desiredSkills: [paperclipKey],
},
},
} as const;
const companyBContext = {
agentId: "agent-b",
companyId: "company-b",
adapterType: "codex_local",
config: {
env: {},
paperclipSkillSync: {
desiredSkills: [paperclipKey],
},
},
} as const;
await syncCodexSkills(companyAContext, [paperclipKey]);
await syncCodexSkills(companyBContext, [paperclipKey]);
expect((await fs.lstat(path.join(sharedCodexHome, "companies", "company-a", "skills", "paperclip"))).isSymbolicLink()).toBe(true);
expect((await fs.lstat(path.join(sharedCodexHome, "companies", "company-b", "skills", "paperclip"))).isSymbolicLink()).toBe(true);
await expect(fs.lstat(path.join(sharedCodexHome, "skills", "paperclip"))).rejects.toMatchObject({
code: "ENOENT",
});
} finally {
if (previousCodexHome === undefined) {
delete process.env.CODEX_HOME;
} else {
process.env.CODEX_HOME = previousCodexHome;
}
}
});
it("keeps required bundled Paperclip skills installed even when the desired set is emptied", async () => {
it("does not persist Paperclip skills into CODEX_HOME during sync", async () => {
const codexHome = await makeTempDir("paperclip-codex-skill-prune-");
cleanupDirs.add(codexHome);
@@ -122,10 +64,22 @@ describe("codex local skill sync", () => {
},
} as const;
await syncCodexSkills(configuredCtx, [paperclipKey]);
const after = await syncCodexSkills(configuredCtx, [paperclipKey]);
expect(after.mode).toBe("ephemeral");
expect(after.entries.find((entry) => entry.key === paperclipKey)?.state).toBe("configured");
await expect(fs.lstat(path.join(codexHome, "skills", "paperclip"))).rejects.toMatchObject({
code: "ENOENT",
});
});
const clearedCtx = {
...configuredCtx,
it("keeps required bundled Paperclip skills configured even when the desired set is emptied", async () => {
const codexHome = await makeTempDir("paperclip-codex-skill-required-");
cleanupDirs.add(codexHome);
const configuredCtx = {
agentId: "agent-2",
companyId: "company-1",
adapterType: "codex_local",
config: {
env: {
CODEX_HOME: codexHome,
@@ -136,13 +90,12 @@ describe("codex local skill sync", () => {
},
} as const;
const after = await syncCodexSkills(clearedCtx, []);
const after = await syncCodexSkills(configuredCtx, []);
expect(after.desiredSkills).toContain(paperclipKey);
expect(after.entries.find((entry) => entry.key === paperclipKey)?.state).toBe("installed");
expect((await fs.lstat(path.join(codexHome, "skills", "paperclip"))).isSymbolicLink()).toBe(true);
expect(after.entries.find((entry) => entry.key === paperclipKey)?.state).toBe("configured");
});
it("normalizes legacy flat Paperclip skill refs before reporting persistent state", async () => {
it("normalizes legacy flat Paperclip skill refs before reporting configured state", async () => {
const codexHome = await makeTempDir("paperclip-codex-legacy-skill-sync-");
cleanupDirs.add(codexHome);
@@ -163,38 +116,7 @@ describe("codex local skill sync", () => {
expect(snapshot.warnings).toEqual([]);
expect(snapshot.desiredSkills).toContain(paperclipKey);
expect(snapshot.desiredSkills).not.toContain("paperclip");
expect(snapshot.entries.find((entry) => entry.key === paperclipKey)?.state).toBe("missing");
expect(snapshot.entries.find((entry) => entry.key === paperclipKey)?.state).toBe("configured");
expect(snapshot.entries.find((entry) => entry.key === "paperclip")).toBeUndefined();
});
it("reports unmanaged user-installed Codex skills with provenance metadata", async () => {
const codexHome = await makeTempDir("paperclip-codex-user-skills-");
cleanupDirs.add(codexHome);
const externalSkillDir = await createSkillDir(path.join(codexHome, "skills"), "crack-python");
expect(externalSkillDir).toContain(path.join(codexHome, "skills"));
const snapshot = await listCodexSkills({
agentId: "agent-4",
companyId: "company-1",
adapterType: "codex_local",
config: {
env: {
CODEX_HOME: codexHome,
},
},
});
expect(snapshot.entries).toContainEqual(expect.objectContaining({
key: "crack-python",
runtimeName: "crack-python",
state: "external",
managed: false,
origin: "user_installed",
originLabel: "User-installed",
locationLabel: "$CODEX_HOME/skills",
readOnly: true,
detail: "Installed outside Paperclip management.",
}));
});
});

156
server/src/services/company-portability.ts
View File

@@ -9,6 +9,7 @@ import type {
CompanyPortabilityCollisionStrategy,
CompanyPortabilityEnvInput,
CompanyPortabilityExport,
CompanyPortabilityFileEntry,
CompanyPortabilityExportPreviewResult,
CompanyPortabilityExportResult,
CompanyPortabilityImport,
@@ -35,9 +36,11 @@ import {
writePaperclipSkillSyncPreference,
} from "@paperclipai/adapter-utils/server-utils";
import { notFound, unprocessable } from "../errors.js";
import type { StorageService } from "../storage/types.js";
import { accessService } from "./access.js";
import { agentService } from "./agents.js";
import { agentInstructionsService } from "./agent-instructions.js";
import { assetService } from "./assets.js";
import { generateReadme } from "./company-export-readme.js";
import { companySkillService } from "./company-skills.js";
import { companyService } from "./companies.js";
@@ -323,7 +326,7 @@ function isSensitiveEnvKey(key: string) {
type ResolvedSource = {
manifest: CompanyPortabilityManifest;
files: Record<string, string>;
files: Record<string, CompanyPortabilityFileEntry>;
warnings: string[];
};
@@ -400,6 +403,16 @@ type EnvInputRecord = {
portability?: "portable" | "system_dependent";
};
const COMPANY_LOGO_CONTENT_TYPE_EXTENSIONS: Record<string, string> = {
"image/gif": ".gif",
"image/jpeg": ".jpg",
"image/png": ".png",
"image/svg+xml": ".svg",
"image/webp": ".webp",
};
const COMPANY_LOGO_FILE_NAME = "company-logo";
const RUNTIME_DEFAULT_RULES: Array<{ path: string[]; value: unknown }> = [
{ path: ["heartbeat", "cooldownSec"], value: 10 },
{ path: ["heartbeat", "intervalSec"], value: 3600 },
@@ -524,12 +537,83 @@ function resolvePortablePath(fromPath: string, targetPath: string) {
return normalizePortablePath(path.posix.join(baseDir, targetPath.replace(/\\/g, "/")));
}
function isPortableBinaryFile(
value: CompanyPortabilityFileEntry,
): value is Extract<CompanyPortabilityFileEntry, { encoding: "base64" }> {
return typeof value === "object" && value !== null && value.encoding === "base64" && typeof value.data === "string";
}
function readPortableTextFile(
files: Record<string, CompanyPortabilityFileEntry>,
filePath: string,
) {
const value = files[filePath];
return typeof value === "string" ? value : null;
}
function inferContentTypeFromPath(filePath: string) {
const extension = path.posix.extname(filePath).toLowerCase();
switch (extension) {
case ".gif":
return "image/gif";
case ".jpeg":
case ".jpg":
return "image/jpeg";
case ".png":
return "image/png";
case ".svg":
return "image/svg+xml";
case ".webp":
return "image/webp";
default:
return null;
}
}
function resolveCompanyLogoExtension(contentType: string | null | undefined, originalFilename: string | null | undefined) {
const fromContentType = contentType ? COMPANY_LOGO_CONTENT_TYPE_EXTENSIONS[contentType.toLowerCase()] : null;
if (fromContentType) return fromContentType;
const extension = originalFilename ? path.extname(originalFilename).toLowerCase() : "";
return extension || ".png";
}
function portableBinaryFileToBuffer(entry: Extract<CompanyPortabilityFileEntry, { encoding: "base64" }>) {
return Buffer.from(entry.data, "base64");
}
function portableFileToBuffer(entry: CompanyPortabilityFileEntry, filePath: string) {
if (typeof entry === "string") {
return Buffer.from(entry, "utf8");
}
if (isPortableBinaryFile(entry)) {
return portableBinaryFileToBuffer(entry);
}
throw unprocessable(`Unsupported file entry encoding for ${filePath}`);
}
function bufferToPortableBinaryFile(buffer: Buffer, contentType: string | null): CompanyPortabilityFileEntry {
return {
encoding: "base64",
data: buffer.toString("base64"),
contentType,
};
}
async function streamToBuffer(stream: NodeJS.ReadableStream) {
const chunks: Buffer[] = [];
for await (const chunk of stream) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
}
return Buffer.concat(chunks);
}
function normalizeFileMap(
files: Record<string, string>,
files: Record<string, CompanyPortabilityFileEntry>,
rootPath?: string | null,
): Record<string, string> {
): Record<string, CompanyPortabilityFileEntry> {
const normalizedRoot = rootPath ? normalizePortablePath(rootPath) : null;
const out: Record<string, string> = {};
const out: Record<string, CompanyPortabilityFileEntry> = {};
for (const [rawPath, content] of Object.entries(files)) {
let nextPath = normalizePortablePath(rawPath);
if (normalizedRoot && nextPath === normalizedRoot) {
@@ -627,7 +711,7 @@ function filterPortableExtensionYaml(yaml: string, selectedFiles: Set<string>) {
}
function filterExportFiles(
files: Record<string, string>,
files: Record<string, CompanyPortabilityFileEntry>,
selectedFilesInput: string[] | undefined,
paperclipExtensionPath: string,
) {
@@ -640,20 +724,21 @@ function filterExportFiles(
.map((entry) => normalizePortablePath(entry))
.filter((entry) => entry.length > 0),
);
const filtered: Record<string, string> = {};
const filtered: Record<string, CompanyPortabilityFileEntry> = {};
for (const [filePath, content] of Object.entries(files)) {
if (!selectedFiles.has(filePath)) continue;
filtered[filePath] = content;
}
if (selectedFiles.has(paperclipExtensionPath) && filtered[paperclipExtensionPath]) {
filtered[paperclipExtensionPath] = filterPortableExtensionYaml(filtered[paperclipExtensionPath]!, selectedFiles);
const extensionEntry = filtered[paperclipExtensionPath];
if (selectedFiles.has(paperclipExtensionPath) && typeof extensionEntry === "string") {
filtered[paperclipExtensionPath] = filterPortableExtensionYaml(extensionEntry, selectedFiles);
}
return filtered;
}
function findPaperclipExtensionPath(files: Record<string, string>) {
function findPaperclipExtensionPath(files: Record<string, CompanyPortabilityFileEntry>) {
if (typeof files[".paperclip.yaml"] === "string") return ".paperclip.yaml";
if (typeof files[".paperclip.yml"] === "string") return ".paperclip.yml";
return Object.keys(files).find((entry) => entry.endsWith("/.paperclip.yaml") || entry.endsWith("/.paperclip.yml")) ?? null;
@@ -1332,6 +1417,14 @@ async function fetchOptionalText(url: string) {
return response.text();
}
async function fetchBinary(url: string) {
const response = await fetch(url);
if (!response.ok) {
throw unprocessable(`Failed to fetch ${url}: ${response.status}`);
}
return Buffer.from(await response.arrayBuffer());
}
async function fetchJson<T>(url: string): Promise<T> {
const response = await fetch(url, {
headers: {
@@ -1425,13 +1518,13 @@ function readAgentSkillRefs(frontmatter: Record<string, unknown>) {
}
function buildManifestFromPackageFiles(
files: Record<string, string>,
files: Record<string, CompanyPortabilityFileEntry>,
opts?: { sourceLabel?: { companyId: string; companyName: string } | null },
): ResolvedSource {
const normalizedFiles = normalizeFileMap(files);
const companyPath =
normalizedFiles["COMPANY.md"]
?? undefined;
const companyPath = typeof normalizedFiles["COMPANY.md"] === "string"
? normalizedFiles["COMPANY.md"]
: undefined;
const resolvedCompanyPath = companyPath !== undefined
? "COMPANY.md"
: Object.keys(normalizedFiles).find((entry) => entry.endsWith("/COMPANY.md") || entry === "COMPANY.md");
@@ -1439,11 +1532,15 @@ function buildManifestFromPackageFiles(
throw unprocessable("Company package is missing COMPANY.md");
}
const companyDoc = parseFrontmatterMarkdown(normalizedFiles[resolvedCompanyPath]!);
const companyMarkdown = readPortableTextFile(normalizedFiles, resolvedCompanyPath);
if (typeof companyMarkdown !== "string") {
throw unprocessable(`Company package file is not readable as text: ${resolvedCompanyPath}`);
}
const companyDoc = parseFrontmatterMarkdown(companyMarkdown);
const companyFrontmatter = companyDoc.frontmatter;
const paperclipExtensionPath = findPaperclipExtensionPath(normalizedFiles);
const paperclipExtension = paperclipExtensionPath
? parseYamlFile(normalizedFiles[paperclipExtensionPath] ?? "")
? parseYamlFile(readPortableTextFile(normalizedFiles, paperclipExtensionPath) ?? "")
: {};
const paperclipCompany = isPlainRecord(paperclipExtension.company) ? paperclipExtension.company : {};
const paperclipAgents = isPlainRecord(paperclipExtension.agents) ? paperclipExtension.agents : {};
@@ -1503,6 +1600,7 @@ function buildManifestFromPackageFiles(
name: companyName,
description: asString(companyFrontmatter.description),
brandColor: asString(paperclipCompany.brandColor),
logoPath: asString(paperclipCompany.logoPath) ?? asString(paperclipCompany.logo),
requireBoardApprovalForNewAgents:
typeof paperclipCompany.requireBoardApprovalForNewAgents === "boolean"
? paperclipCompany.requireBoardApprovalForNewAgents
@@ -1516,8 +1614,11 @@ function buildManifestFromPackageFiles(
};
const warnings: string[] = [];
if (manifest.company?.logoPath && !normalizedFiles[manifest.company.logoPath]) {
warnings.push(`Referenced company logo file is missing from package: ${manifest.company.logoPath}`);
}
for (const agentPath of agentPaths) {
const markdownRaw = normalizedFiles[agentPath];
const markdownRaw = readPortableTextFile(normalizedFiles, agentPath);
if (typeof markdownRaw !== "string") {
warnings.push(`Referenced agent file is missing from package: ${agentPath}`);
continue;
@@ -1566,7 +1667,7 @@ function buildManifestFromPackageFiles(
}
for (const skillPath of skillPaths) {
const markdownRaw = normalizedFiles[skillPath];
const markdownRaw = readPortableTextFile(normalizedFiles, skillPath);
if (typeof markdownRaw !== "string") {
warnings.push(`Referenced skill file is missing from package: ${skillPath}`);
continue;
@@ -1651,7 +1752,7 @@ function buildManifestFromPackageFiles(
}
for (const projectPath of projectPaths) {
const markdownRaw = normalizedFiles[projectPath];
const markdownRaw = readPortableTextFile(normalizedFiles, projectPath);
if (typeof markdownRaw !== "string") {
warnings.push(`Referenced project file is missing from package: ${projectPath}`);
continue;
@@ -1685,7 +1786,7 @@ function buildManifestFromPackageFiles(
}
for (const taskPath of taskPaths) {
const markdownRaw = normalizedFiles[taskPath];
const markdownRaw = readPortableTextFile(normalizedFiles, taskPath);
if (typeof markdownRaw !== "string") {
warnings.push(`Referenced task file is missing from package: ${taskPath}`);
continue;
@@ -1773,9 +1874,10 @@ function resolveRawGitHubUrl(owner: string, repo: string, ref: string, filePath:
return `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${normalizedFilePath}`;
}
export function companyPortabilityService(db: Db) {
export function companyPortabilityService(db: Db, storage?: StorageService) {
const companies = companyService(db);
const agents = agentService(db);
const assetRecords = assetService(db);
const instructions = agentInstructionsService();
const access = accessService(db);
const projects = projectService(db);
@@ -1818,7 +1920,7 @@ export function companyPortabilityService(db: Db) {
const companyPath = parsed.companyPath === "COMPANY.md"
? "COMPANY.md"
: normalizePortablePath(path.posix.relative(parsed.basePath || ".", parsed.companyPath));
const files: Record<string, string> = {
const files: Record<string, CompanyPortabilityFileEntry> = {
[companyPath]: companyMarkdown,
};
const tree = await fetchJson<{ tree?: Array<{ path: string; type: string }> }>(
@@ -1859,6 +1961,18 @@ export function companyPortabilityService(db: Db) {
}
const resolved = buildManifestFromPackageFiles(files);
const companyLogoPath = resolved.manifest.company?.logoPath;
if (companyLogoPath && !resolved.files[companyLogoPath]) {
const repoPath = [parsed.basePath, companyLogoPath].filter(Boolean).join("/");
try {
const binary = await fetchBinary(
resolveRawGitHubUrl(parsed.owner, parsed.repo, ref, repoPath),
);
resolved.files[companyLogoPath] = bufferToPortableBinaryFile(binary, inferContentTypeFromPath(companyLogoPath));
} catch (err) {
warnings.push(`Failed to fetch company logo ${companyLogoPath} from GitHub: ${err instanceof Error ? err.message : String(err)}`);
}
}
resolved.warnings.unshift(...warnings);
return resolved;
}