aevgarik/paperclip
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add storage system with local disk and S3 providers

Browse Source 
Introduces a provider-agnostic storage subsystem for file attachments.
Includes local disk and S3 backends, asset/attachment DB schemas, issue
attachment CRUD routes with multer upload, CLI configure/doctor/env
integration, and enriched issue ancestors with project/goal resolution.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Forgotten
2026-02-20 10:31:56 -06:00
parent 32119f5c2f
commit fdd2ea6157
36 changed files with 1683 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
cli/src/checks/index.ts
View File

@@ -14,3 +14,4 @@ export { llmCheck } from "./llm-check.js";
export { logCheck } from "./log-check.js";
export { portCheck } from "./port-check.js";
export { secretsCheck } from "./secrets-check.js";
export { storageCheck } from "./storage-check.js";

60
cli/src/checks/storage-check.ts Normal file
View File

@@ -0,0 +1,60 @@
import fs from "node:fs";
import type { PaperclipConfig } from "../config/schema.js";
import type { CheckResult } from "./index.js";
import { resolveRuntimeLikePath } from "./path-resolver.js";
export function storageCheck(config: PaperclipConfig, configPath?: string): CheckResult {
if (config.storage.provider === "local_disk") {
const baseDir = resolveRuntimeLikePath(config.storage.localDisk.baseDir, configPath);
if (!fs.existsSync(baseDir)) {
return {
name: "Storage",
status: "warn",
message: `Local storage directory does not exist: ${baseDir}`,
canRepair: true,
repair: () => {
fs.mkdirSync(baseDir, { recursive: true });
},
repairHint: "Run with --repair to create local storage directory",
};
}
try {
fs.accessSync(baseDir, fs.constants.W_OK);
return {
name: "Storage",
status: "pass",
message: `Local disk storage is writable: ${baseDir}`,
};
} catch {
return {
name: "Storage",
status: "fail",
message: `Local storage directory is not writable: ${baseDir}`,
canRepair: false,
repairHint: "Check file permissions for storage.localDisk.baseDir",
};
}
}
const bucket = config.storage.s3.bucket.trim();
const region = config.storage.s3.region.trim();
if (!bucket || !region) {
return {
name: "Storage",
status: "fail",
message: "S3 storage requires non-empty bucket and region",
canRepair: false,
repairHint: "Run `paperclip configure --section storage`",
};
}
return {
name: "Storage",
status: "warn",
message: `S3 storage configured (bucket=${bucket}, region=${region}). Reachability check is skipped in doctor.`,
canRepair: false,
repairHint: "Verify credentials and endpoint in deployment environment",
};
}

8
cli/src/commands/configure.ts
View File

@@ -7,6 +7,7 @@ import { promptDatabase } from "../prompts/database.js";
import { promptLlm } from "../prompts/llm.js";
import { promptLogging } from "../prompts/logging.js";
import { defaultSecretsConfig, promptSecrets } from "../prompts/secrets.js";
import { defaultStorageConfig, promptStorage } from "../prompts/storage.js";
import { promptServer } from "../prompts/server.js";
import {
resolveDefaultEmbeddedPostgresDir,
@@ -14,13 +15,14 @@ import {
resolvePaperclipInstanceId,
} from "../config/home.js";
type Section = "llm" | "database" | "logging" | "server" | "secrets";
type Section = "llm" | "database" | "logging" | "server" | "storage" | "secrets";
const SECTION_LABELS: Record<Section, string> = {
llm: "LLM Provider",
database: "Database",
logging: "Logging",
server: "Server",
storage: "Storage",
secrets: "Secrets",
};
@@ -45,6 +47,7 @@ function defaultConfig(): PaperclipConfig {
port: 3100,
serveUi: true,
},
storage: defaultStorageConfig(),
secrets: defaultSecretsConfig(),
};
}
@@ -123,6 +126,9 @@ export async function configure(opts: {
case "server":
config.server = await promptServer();
break;
case "storage":
config.storage = await promptStorage(config.storage);
break;
case "secrets":
config.secrets = await promptSecrets(config.secrets);
{

15
cli/src/commands/doctor.ts
View File

@@ -10,6 +10,7 @@ import {
logCheck,
portCheck,
secretsCheck,
storageCheck,
type CheckResult,
} from "../checks/index.js";
@@ -66,24 +67,30 @@ export async function doctor(opts: {
printResult(secretsResult);
await maybeRepair(secretsResult, opts);
// 4. Database check
// 4. Storage check
const storageResult = storageCheck(config, configPath);
results.push(storageResult);
printResult(storageResult);
await maybeRepair(storageResult, opts);
// 5. Database check
const dbResult = await databaseCheck(config, configPath);
results.push(dbResult);
printResult(dbResult);
await maybeRepair(dbResult, opts);
// 5. LLM check
// 6. LLM check
const llmResult = await llmCheck(config);
results.push(llmResult);
printResult(llmResult);
// 6. Log directory check
// 7. Log directory check
const logResult = logCheck(config, configPath);
results.push(logResult);
printResult(logResult);
await maybeRepair(logResult, opts);
// 7. Port check
// 8. Port check
const portResult = await portCheck(config);
results.push(portResult);
printResult(portResult);

109
cli/src/commands/env.ts
View File

@@ -9,6 +9,7 @@ import {
} from "../config/env.js";
import {
resolveDefaultSecretsKeyFilePath,
resolveDefaultStorageDir,
resolvePaperclipInstanceId,
} from "../config/home.js";
@@ -27,9 +28,13 @@ const DEFAULT_AGENT_JWT_ISSUER = "paperclip";
const DEFAULT_AGENT_JWT_AUDIENCE = "paperclip-api";
const DEFAULT_HEARTBEAT_SCHEDULER_INTERVAL_MS = "30000";
const DEFAULT_SECRETS_PROVIDER = "local_encrypted";
const DEFAULT_STORAGE_PROVIDER = "local_disk";
function defaultSecretsKeyFilePath(): string {
return resolveDefaultSecretsKeyFilePath(resolvePaperclipInstanceId());
}
function defaultStorageBaseDir(): string {
return resolveDefaultStorageDir(resolvePaperclipInstanceId());
}
export async function envCommand(opts: { config?: string }): Promise<void> {
p.intro(pc.bgCyan(pc.black(" paperclip env ")));
@@ -127,6 +132,33 @@ function collectDeploymentEnvRows(config: PaperclipConfig | null, configPath: st
process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE ??
config?.secrets?.localEncrypted?.keyFilePath ??
defaultSecretsKeyFilePath();
const storageProvider =
process.env.PAPERCLIP_STORAGE_PROVIDER ??
config?.storage?.provider ??
DEFAULT_STORAGE_PROVIDER;
const storageLocalDir =
process.env.PAPERCLIP_STORAGE_LOCAL_DIR ??
config?.storage?.localDisk?.baseDir ??
defaultStorageBaseDir();
const storageS3Bucket =
process.env.PAPERCLIP_STORAGE_S3_BUCKET ??
config?.storage?.s3?.bucket ??
"paperclip";
const storageS3Region =
process.env.PAPERCLIP_STORAGE_S3_REGION ??
config?.storage?.s3?.region ??
"us-east-1";
const storageS3Endpoint =
process.env.PAPERCLIP_STORAGE_S3_ENDPOINT ??
config?.storage?.s3?.endpoint ??
"";
const storageS3Prefix =
process.env.PAPERCLIP_STORAGE_S3_PREFIX ??
config?.storage?.s3?.prefix ??
"";
const storageS3ForcePathStyle =
process.env.PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE ??
String(config?.storage?.s3?.forcePathStyle ?? false);
const rows: EnvVarRow[] = [
{
@@ -228,6 +260,83 @@ function collectDeploymentEnvRows(config: PaperclipConfig | null, configPath: st
required: false,
note: "Path to local encrypted secrets key file",
},
{
key: "PAPERCLIP_STORAGE_PROVIDER",
value: storageProvider,
source: process.env.PAPERCLIP_STORAGE_PROVIDER
? "env"
: config?.storage?.provider
? "config"
: "default",
required: false,
note: "Storage provider (local_disk or s3)",
},
{
key: "PAPERCLIP_STORAGE_LOCAL_DIR",
value: storageLocalDir,
source: process.env.PAPERCLIP_STORAGE_LOCAL_DIR
? "env"
: config?.storage?.localDisk?.baseDir
? "config"
: "default",
required: false,
note: "Local storage base directory for local_disk provider",
},
{
key: "PAPERCLIP_STORAGE_S3_BUCKET",
value: storageS3Bucket,
source: process.env.PAPERCLIP_STORAGE_S3_BUCKET
? "env"
: config?.storage?.s3?.bucket
? "config"
: "default",
required: false,
note: "S3 bucket name for s3 provider",
},
{
key: "PAPERCLIP_STORAGE_S3_REGION",
value: storageS3Region,
source: process.env.PAPERCLIP_STORAGE_S3_REGION
? "env"
: config?.storage?.s3?.region
? "config"
: "default",
required: false,
note: "S3 region for s3 provider",
},
{
key: "PAPERCLIP_STORAGE_S3_ENDPOINT",
value: storageS3Endpoint,
source: process.env.PAPERCLIP_STORAGE_S3_ENDPOINT
? "env"
: config?.storage?.s3?.endpoint
? "config"
: "default",
required: false,
note: "Optional custom endpoint for S3-compatible providers",
},
{
key: "PAPERCLIP_STORAGE_S3_PREFIX",
value: storageS3Prefix,
source: process.env.PAPERCLIP_STORAGE_S3_PREFIX
? "env"
: config?.storage?.s3?.prefix
? "config"
: "default",
required: false,
note: "Optional object key prefix",
},
{
key: "PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE",
value: storageS3ForcePathStyle,
source: process.env.PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE
? "env"
: config?.storage?.s3?.forcePathStyle !== undefined
? "config"
: "default",
required: false,
note: "Set true for path-style access on compatible providers",
},
];
const defaultConfigPath = resolveConfigPath();

7
cli/src/commands/onboard.ts
View File

@@ -8,6 +8,7 @@ import { promptDatabase } from "../prompts/database.js";
import { promptLlm } from "../prompts/llm.js";
import { promptLogging } from "../prompts/logging.js";
import { defaultSecretsConfig } from "../prompts/secrets.js";
import { defaultStorageConfig, promptStorage } from "../prompts/storage.js";
import { promptServer } from "../prompts/server.js";
import { describeLocalInstancePaths, resolvePaperclipInstanceId } from "../config/home.js";
@@ -107,6 +108,10 @@ export async function onboard(opts: { config?: string }): Promise<void> {
p.log.step(pc.bold("Server"));
const server = await promptServer();
// Storage
p.log.step(pc.bold("Storage"));
const storage = await promptStorage(defaultStorageConfig());
// Secrets
p.log.step(pc.bold("Secrets"));
const secrets = defaultSecretsConfig();
@@ -137,6 +142,7 @@ export async function onboard(opts: { config?: string }): Promise<void> {
database,
logging,
server,
storage,
secrets,
};
@@ -155,6 +161,7 @@ export async function onboard(opts: { config?: string }): Promise<void> {
llm ? `LLM: ${llm.provider}` : "LLM: not configured",
`Logging: ${logging.mode} → ${logging.logDir}`,
`Server: port ${server.port}`,
`Storage: ${storage.provider}`,
`Secrets: ${secrets.provider} (strict mode ${secrets.strictMode ? "on" : "off"})`,
`Agent auth: PAPERCLIP_AGENT_JWT_SECRET configured`,
].join("\n"),

5
cli/src/config/home.ts
View File

@@ -45,6 +45,10 @@ export function resolveDefaultSecretsKeyFilePath(instanceId?: string): string {
return path.resolve(resolvePaperclipInstanceRoot(instanceId), "secrets", "master.key");
}
export function resolveDefaultStorageDir(instanceId?: string): string {
return path.resolve(resolvePaperclipInstanceRoot(instanceId), "data", "storage");
}
export function expandHomePrefix(value: string): string {
if (value === "~") return os.homedir();
if (value.startsWith("~/")) return path.resolve(os.homedir(), value.slice(2));
@@ -62,5 +66,6 @@ export function describeLocalInstancePaths(instanceId?: string) {
embeddedPostgresDataDir: resolveDefaultEmbeddedPostgresDir(resolvedInstanceId),
logDir: resolveDefaultLogsDir(resolvedInstanceId),
secretsKeyFilePath: resolveDefaultSecretsKeyFilePath(resolvedInstanceId),
storageDir: resolveDefaultStorageDir(resolvedInstanceId),
};
}

6
cli/src/config/schema.ts
View File

@@ -5,6 +5,9 @@ export {
databaseConfigSchema,
loggingConfigSchema,
serverConfigSchema,
storageConfigSchema,
storageLocalDiskConfigSchema,
storageS3ConfigSchema,
secretsConfigSchema,
secretsLocalEncryptedConfigSchema,
type PaperclipConfig,
@@ -12,6 +15,9 @@ export {
type DatabaseConfig,
type LoggingConfig,
type ServerConfig,
type StorageConfig,
type StorageLocalDiskConfig,
type StorageS3Config,
type SecretsConfig,
type SecretsLocalEncryptedConfig,
type ConfigMeta,

2
cli/src/index.ts
View File

@@ -48,7 +48,7 @@ program
.command("configure")
.description("Update configuration sections")
.option("-c, --config <path>", "Path to config file")
.option("-s, --section <section>", "Section to configure (llm, database, logging, server, secrets)")
.option("-s, --section <section>", "Section to configure (llm, database, logging, server, storage, secrets)")
.action(configure);
program

146
cli/src/prompts/storage.ts Normal file
View File

@@ -0,0 +1,146 @@
import * as p from "@clack/prompts";
import type { StorageConfig } from "../config/schema.js";
import { resolveDefaultStorageDir, resolvePaperclipInstanceId } from "../config/home.js";
function defaultStorageBaseDir(): string {
return resolveDefaultStorageDir(resolvePaperclipInstanceId());
}
export function defaultStorageConfig(): StorageConfig {
return {
provider: "local_disk",
localDisk: {
baseDir: defaultStorageBaseDir(),
},
s3: {
bucket: "paperclip",
region: "us-east-1",
endpoint: undefined,
prefix: "",
forcePathStyle: false,
},
};
}
export async function promptStorage(current?: StorageConfig): Promise<StorageConfig> {
const base = current ?? defaultStorageConfig();
const provider = await p.select({
message: "Storage provider",
options: [
{
value: "local_disk" as const,
label: "Local disk (recommended)",
hint: "best for single-user local deployments",
},
{
value: "s3" as const,
label: "S3 compatible",
hint: "for cloud/object storage backends",
},
],
initialValue: base.provider,
});
if (p.isCancel(provider)) {
p.cancel("Setup cancelled.");
process.exit(0);
}
if (provider === "local_disk") {
const baseDir = await p.text({
message: "Local storage base directory",
defaultValue: base.localDisk.baseDir || defaultStorageBaseDir(),
placeholder: defaultStorageBaseDir(),
validate: (value) => {
if (!value || value.trim().length === 0) return "Storage base directory is required";
},
});
if (p.isCancel(baseDir)) {
p.cancel("Setup cancelled.");
process.exit(0);
}
return {
provider: "local_disk",
localDisk: {
baseDir: baseDir.trim(),
},
s3: base.s3,
};
}
const bucket = await p.text({
message: "S3 bucket",
defaultValue: base.s3.bucket || "paperclip",
placeholder: "paperclip",
validate: (value) => {
if (!value || value.trim().length === 0) return "Bucket is required";
},
});
if (p.isCancel(bucket)) {
p.cancel("Setup cancelled.");
process.exit(0);
}
const region = await p.text({
message: "S3 region",
defaultValue: base.s3.region || "us-east-1",
placeholder: "us-east-1",
validate: (value) => {
if (!value || value.trim().length === 0) return "Region is required";
},
});
if (p.isCancel(region)) {
p.cancel("Setup cancelled.");
process.exit(0);
}
const endpoint = await p.text({
message: "S3 endpoint (optional for compatible backends)",
defaultValue: base.s3.endpoint ?? "",
placeholder: "https://s3.amazonaws.com",
});
if (p.isCancel(endpoint)) {
p.cancel("Setup cancelled.");
process.exit(0);
}
const prefix = await p.text({
message: "Object key prefix (optional)",
defaultValue: base.s3.prefix ?? "",
placeholder: "paperclip/",
});
if (p.isCancel(prefix)) {
p.cancel("Setup cancelled.");
process.exit(0);
}
const forcePathStyle = await p.confirm({
message: "Use S3 path-style URLs?",
initialValue: base.s3.forcePathStyle ?? false,
});
if (p.isCancel(forcePathStyle)) {
p.cancel("Setup cancelled.");
process.exit(0);
}
return {
provider: "s3",
localDisk: base.localDisk,
s3: {
bucket: bucket.trim(),
region: region.trim(),
endpoint: endpoint.trim() || undefined,
prefix: prefix.trim(),
forcePathStyle,
},
};
}