diff --git a/.memory/worklog.json b/.memory/worklog.json index b4f4f62..d9ce37e 100644 --- a/.memory/worklog.json +++ b/.memory/worklog.json @@ -139,6 +139,13 @@ "message": "auto-save 2026-04-19 20:15 (~1)", "hash": "eea4230", "files_changed": 1 + }, + { + "ts": "2026-04-19T21:01:27+08:00", + "type": "commit", + "message": "auto-save 2026-04-19 21:01 (+3, ~2)", + "hash": "1b8a216", + "files_changed": 5 } ] } diff --git a/deploy/sandbox-orchestrator.service b/deploy/sandbox-orchestrator.service new file mode 100644 index 0000000..99a261e --- /dev/null +++ b/deploy/sandbox-orchestrator.service @@ -0,0 +1,25 @@ +[Unit] +Description=Lobe Sandbox Orchestrator +After=network-online.target incus.service +Wants=network-online.target +Requires=incus.service + +[Service] +Type=simple +User=root +Group=root +WorkingDirectory=/opt/lobe-sandbox/orchestrator +EnvironmentFile=/etc/lobe-sandbox/orchestrator.env +ExecStart=/usr/local/bin/bun run src/index.ts +Restart=on-failure +RestartSec=3 + +# 日志 +StandardOutput=append:/var/log/lobe-sandbox-orchestrator.log +StandardError=append:/var/log/lobe-sandbox-orchestrator.log + +# 资源限额 +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target diff --git a/images/base/build.sh b/images/base/build.sh old mode 100644 new mode 100755 diff --git a/images/base/patch-uv.sh b/images/base/patch-uv.sh old mode 100644 new mode 100755 diff --git a/orchestrator/src/export.ts b/orchestrator/src/export.ts new file mode 100644 index 0000000..71cd0d4 --- /dev/null +++ b/orchestrator/src/export.ts @@ -0,0 +1,83 @@ +import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { env } from './env.ts'; +import { incus, containerName } from './incus.ts'; +import { state } from './state.ts'; + +const s3 = new S3Client({ + endpoint: env.s3.endpoint, + region: env.s3.region, + forcePathStyle: env.s3.forcePathStyle, + credentials: { + accessKeyId: env.s3.accessKeyId, + secretAccessKey: env.s3.secretAccessKey, + }, +}); + +export interface ExportResult { + success: boolean; + key?: string; + size?: number; + mimeType?: string; + filename?: string; + error?: string; +} + +export const exportFile = async ( + userId: string, + path: string, + filename: string, +): Promise => { + const name = containerName(userId); + const s = await incus.state(name); + if (s === 'MISSING') return { success: false, error: `no sandbox for ${userId}` }; + if (s === 'STOPPED') await incus.start(name); + state.touch(userId); + + try { + const bytes = await incus.pull(name, path); + const today = new Date().toISOString().split('T')[0]; + const key = `exports/${today}/${userId}/${filename}`; + const mime = guessMime(filename); + + await s3.send( + new PutObjectCommand({ + Bucket: env.s3.bucket, + Key: key, + Body: bytes, + ContentType: mime, + }), + ); + + return { success: true, key, size: bytes.byteLength, mimeType: mime, filename }; + } catch (e) { + return { success: false, error: (e as Error).message }; + } +}; + +const MIME: Record = { + '.txt': 'text/plain', + '.md': 'text/markdown', + '.json': 'application/json', + '.csv': 'text/csv', + '.tsv': 'text/tab-separated-values', + '.pdf': 'application/pdf', + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.gif': 'image/gif', + '.webp': 'image/webp', + '.svg': 'image/svg+xml', + '.html': 'text/html', + '.xml': 'application/xml', + '.zip': 'application/zip', + '.gz': 'application/gzip', + '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', +}; + +const guessMime = (filename: string): string => { + const dot = filename.lastIndexOf('.'); + if (dot === -1) return 'application/octet-stream'; + return MIME[filename.slice(dot).toLowerCase()] ?? 'application/octet-stream'; +}; diff --git a/orchestrator/src/index.ts b/orchestrator/src/index.ts new file mode 100644 index 0000000..2940fbd --- /dev/null +++ b/orchestrator/src/index.ts @@ -0,0 +1,96 @@ +import { Hono } from 'hono'; +import { z } from 'zod'; +import { authMiddleware } from './auth.ts'; +import { env } from './env.ts'; +import { exportFile } from './export.ts'; +import { containerName, incus } from './incus.ts'; +import { startIdleReaper } from './reaper.ts'; +import { state } from './state.ts'; +import { handlers } from './tools.ts'; + +const app = new Hono(); + +app.get('/health', (c) => c.json({ ok: true, ts: Date.now() })); + +// 下面所有路由都要过 auth +app.use('/api/*', authMiddleware); + +// ---- User lifecycle ---- + +app.post('/api/v1/users', async (c) => { + const body = await c.req.json().catch(() => null); + const parsed = z.object({ userId: z.string().min(1).max(64) }).safeParse(body); + if (!parsed.success) return c.json({ error: 'invalid body' }, 400); + const { userId } = parsed.data; + + state.recordCreate(userId); + try { + await incus.provision(containerName(userId)); + state.markProvisioned(userId); + return c.json({ success: true, userId, containerName: containerName(userId) }); + } catch (e) { + return c.json({ success: false, error: (e as Error).message }, 500); + } +}); + +app.delete('/api/v1/users/:userId', async (c) => { + const userId = c.req.param('userId'); + const name = containerName(userId); + try { + if (await incus.exists(name)) { + await incus.stop(name); + // TODO: 备份 rootfs tar.gz 到 MinIO (下一轮实现) + await incus.remove(name); + } + state.markDeleted(userId); + return c.json({ success: true, userId }); + } catch (e) { + return c.json({ success: false, error: (e as Error).message }, 500); + } +}); + +// ---- Tools ---- + +const ToolBody = z.object({ + userId: z.string().min(1).max(64), + params: z.record(z.string(), z.unknown()).default({}), +}); + +app.post('/api/v1/tools/:toolName', async (c) => { + const toolName = c.req.param('toolName'); + const handler = handlers[toolName]; + if (!handler) return c.json({ success: false, error: { message: `unknown tool: ${toolName}` } }, 404); + const body = await c.req.json().catch(() => null); + const parsed = ToolBody.safeParse(body); + if (!parsed.success) return c.json({ success: false, error: { message: 'invalid body' } }, 400); + try { + const result = await handler(parsed.data.params, parsed.data.userId); + return c.json(result); + } catch (e) { + return c.json({ success: false, error: { message: (e as Error).message } }, 500); + } +}); + +// exportFile 特殊路由(因为要流 S3,不塞 tools 里) +app.post('/api/v1/export-file', async (c) => { + const body = await c.req.json().catch(() => null); + const parsed = z + .object({ userId: z.string(), path: z.string(), filename: z.string() }) + .safeParse(body); + if (!parsed.success) return c.json({ success: false, error: 'invalid body' }, 400); + const r = await exportFile(parsed.data.userId, parsed.data.path, parsed.data.filename); + return c.json(r); +}); + +// ---- Boot ---- + +startIdleReaper(); + +console.log(`[orchestrator] listening on ${env.host}:${env.port}`); +console.log(`[orchestrator] incus project=${env.incus.project} image=${env.incus.baseImage}`); + +export default { + port: env.port, + hostname: env.host, + fetch: app.fetch, +}; diff --git a/orchestrator/src/reaper.ts b/orchestrator/src/reaper.ts new file mode 100644 index 0000000..5a674c9 --- /dev/null +++ b/orchestrator/src/reaper.ts @@ -0,0 +1,33 @@ +import { env } from './env.ts'; +import { incus, containerName } from './incus.ts'; +import { state } from './state.ts'; + +let running = false; + +export const startIdleReaper = (): void => { + // 5 min 扫一次 + setInterval(() => { + if (running) return; + running = true; + Promise.resolve() + .then(async () => { + const idleUsers = state.findIdle(env.idleTimeoutMs); + for (const userId of idleUsers) { + const name = containerName(userId); + const s = await incus.state(name); + if (s === 'RUNNING') { + console.log(`[reaper] stopping idle sandbox for user ${userId}`); + try { + await incus.stop(name); + } catch (e) { + console.error(`[reaper] stop ${name} failed:`, (e as Error).message); + } + } + state.clearActivity(userId); + } + }) + .finally(() => { + running = false; + }); + }, 5 * 60 * 1000); +}; diff --git a/orchestrator/src/tools.ts b/orchestrator/src/tools.ts new file mode 100644 index 0000000..106bddb --- /dev/null +++ b/orchestrator/src/tools.ts @@ -0,0 +1,238 @@ +import { incus, containerName } from './incus.ts'; +import { state } from './state.ts'; + +export interface ToolResult { + success: boolean; + result?: unknown; + error?: { message: string; name?: string }; +} + +const ok = (result: unknown): ToolResult => ({ success: true, result }); +const fail = (message: string, name?: string): ToolResult => ({ + success: false, + error: { message, name }, +}); + +// 确保容器跑着 — 调任何工具前必须调 +const ensureRunning = async (userId: string): Promise => { + const name = containerName(userId); + const s = await incus.state(name); + if (s === 'MISSING') throw new Error(`Sandbox not provisioned for user ${userId}`); + if (s === 'STOPPED') await incus.start(name); + state.touch(userId); + return name; +}; + +// 临时路径生成(容器内) +const tmpPath = (ext: string) => `/tmp/sb-${Date.now()}-${Math.random().toString(36).slice(2, 8)}${ext}`; + +// ==================== executeCode ==================== + +const LANG_RUNNER: Record string[] }> = { + python: { ext: '.py', cmd: (p) => ['python3', p] }, + javascript: { ext: '.js', cmd: (p) => ['node', p] }, + typescript: { ext: '.ts', cmd: (p) => ['bun', 'run', p] }, +}; + +export const executeCode = async ( + params: { code: string; language: string; description?: string }, + userId: string, +): Promise => { + const runner = LANG_RUNNER[params.language]; + if (!runner) return fail(`unsupported language: ${params.language}`); + const name = await ensureRunning(userId); + const path = tmpPath(runner.ext); + try { + await incus.push(name, path, params.code, { mode: '0644' }); + await incus.exec(name, ['chown', 'sandbox:sandbox', path]); + const r = await incus.exec(name, runner.cmd(path), { + user: 1000, + cwd: '/workspace', + timeoutMs: 5 * 60 * 1000, + }); + return ok({ stdout: r.stdout, stderr: r.stderr, exitCode: r.exitCode }); + } finally { + await incus.exec(name, ['rm', '-f', path]).catch(() => {}); + } +}; + +// ==================== listLocalFiles ==================== + +export const listLocalFiles = async ( + params: { directoryPath: string }, + userId: string, +): Promise => { + const name = await ensureRunning(userId); + const r = await incus.exec( + name, + ['ls', '-la', '--time-style=long-iso', params.directoryPath], + { user: 1000 }, + ); + if (r.exitCode !== 0) return fail(r.stderr || 'ls failed'); + return ok({ output: r.stdout }); +}; + +// ==================== readLocalFile ==================== + +export const readLocalFile = async ( + params: { filePath: string; startLine?: number; endLine?: number }, + userId: string, +): Promise => { + const name = await ensureRunning(userId); + const cmd = + params.startLine || params.endLine + ? ['sed', '-n', `${params.startLine ?? 1},${params.endLine ?? '$'}p`, params.filePath] + : ['cat', params.filePath]; + const r = await incus.exec(name, cmd, { user: 1000 }); + if (r.exitCode !== 0) return fail(r.stderr || 'read failed'); + return ok({ content: r.stdout }); +}; + +// ==================== writeLocalFile ==================== + +export const writeLocalFile = async ( + params: { filePath: string; content: string }, + userId: string, +): Promise => { + const name = await ensureRunning(userId); + await incus.push(name, params.filePath, params.content, { mode: '0644' }); + await incus.exec(name, ['chown', 'sandbox:sandbox', params.filePath]); + return ok({ filePath: params.filePath, bytes: params.content.length }); +}; + +// ==================== editLocalFile ==================== + +export const editLocalFile = async ( + params: { filePath: string; oldContent: string; newContent: string }, + userId: string, +): Promise => { + const name = await ensureRunning(userId); + const r = await incus.exec(name, ['cat', params.filePath], { user: 1000 }); + if (r.exitCode !== 0) return fail(`read ${params.filePath}: ${r.stderr}`); + const idx = r.stdout.indexOf(params.oldContent); + if (idx === -1) return fail('oldContent not found'); + const nextIdx = r.stdout.indexOf(params.oldContent, idx + params.oldContent.length); + if (nextIdx !== -1) return fail('oldContent is ambiguous (multiple matches)'); + const updated = + r.stdout.slice(0, idx) + params.newContent + r.stdout.slice(idx + params.oldContent.length); + await incus.push(name, params.filePath, updated, { mode: '0644' }); + await incus.exec(name, ['chown', 'sandbox:sandbox', params.filePath]); + return ok({ filePath: params.filePath, replaced: 1 }); +}; + +// ==================== runCommand (sync, with timeout) ==================== + +export const runCommand = async ( + params: { command: string; cwd?: string; timeoutMs?: number }, + userId: string, +): Promise => { + const name = await ensureRunning(userId); + const r = await incus.exec(name, ['bash', '-lc', params.command], { + user: 1000, + cwd: params.cwd ?? '/workspace', + timeoutMs: params.timeoutMs ?? 2 * 60 * 1000, + }); + return ok({ stdout: r.stdout, stderr: r.stderr, exitCode: r.exitCode }); +}; + +// getCommandOutput + killCommand - MVP synchronous runCommand 已返回完整输出,长驻命令留给 v2 +export const getCommandOutput = async (): Promise => + fail('long-running commands not yet supported; runCommand is synchronous'); + +export const killCommand = async (): Promise => + fail('long-running commands not yet supported'); + +// ==================== searchLocalFiles (find by name) ==================== + +export const searchLocalFiles = async ( + params: { directoryPath: string; pattern: string }, + userId: string, +): Promise => { + const name = await ensureRunning(userId); + const r = await incus.exec( + name, + ['fd', '--no-ignore-vcs', params.pattern, params.directoryPath], + { user: 1000, timeoutMs: 30_000 }, + ); + return ok({ matches: r.stdout.split('\n').filter(Boolean) }); +}; + +// ==================== grepContent ==================== + +export const grepContent = async ( + params: { directoryPath: string; pattern: string; caseSensitive?: boolean }, + userId: string, +): Promise => { + const name = await ensureRunning(userId); + const args = ['rg', '-n', '--no-ignore-vcs']; + if (!params.caseSensitive) args.push('-i'); + args.push('--', params.pattern, params.directoryPath); + const r = await incus.exec(name, args, { user: 1000, timeoutMs: 30_000 }); + return ok({ matches: r.stdout }); +}; + +// ==================== globLocalFiles ==================== + +export const globLocalFiles = async ( + params: { pattern: string; cwd?: string }, + userId: string, +): Promise => { + const name = await ensureRunning(userId); + const r = await incus.exec( + name, + ['bash', '-lc', `shopt -s globstar nullglob dotglob; printf '%s\\n' ${JSON.stringify(params.pattern)}`], + { user: 1000, cwd: params.cwd ?? '/workspace' }, + ); + return ok({ matches: r.stdout.split('\n').filter(Boolean) }); +}; + +// ==================== moveLocalFiles ==================== + +export const moveLocalFiles = async ( + params: { sources: string[]; destination: string }, + userId: string, +): Promise => { + const name = await ensureRunning(userId); + const r = await incus.exec(name, ['mv', '--', ...params.sources, params.destination], { user: 1000 }); + if (r.exitCode !== 0) return fail(r.stderr); + return ok({ moved: params.sources.length }); +}; + +// ==================== renameLocalFile ==================== + +export const renameLocalFile = async ( + params: { oldPath: string; newPath: string }, + userId: string, +): Promise => { + const name = await ensureRunning(userId); + const r = await incus.exec(name, ['mv', '--', params.oldPath, params.newPath], { user: 1000 }); + if (r.exitCode !== 0) return fail(r.stderr); + return ok({ oldPath: params.oldPath, newPath: params.newPath }); +}; + +// ==================== exportFile ==================== + +// exportFile 在 routes 里直接处理(要流式 pull → S3 upload) +export const exportFileStub = async (): Promise => + fail('exportFile must be routed to /api/v1/export-file, not /api/v1/tools/exportFile'); + +// ==================== Dispatcher ==================== + +type ToolHandler = (params: any, userId: string) => Promise; + +export const handlers: Record = { + executeCode, + listLocalFiles, + readLocalFile, + writeLocalFile, + editLocalFile, + runCommand, + getCommandOutput, + killCommand, + searchLocalFiles, + grepContent, + globLocalFiles, + moveLocalFiles, + renameLocalFile, + exportFile: exportFileStub, +}; diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100755 index 0000000..b8f485e --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Deploy orchestrator to VPS. +# Idempotent: safe to run for updates (just re-syncs code and restarts). +set -euo pipefail + +VPS="${VPS:-root@2.24.28.41}" +REMOTE_DIR="/opt/lobe-sandbox" +ENV_FILE="/etc/lobe-sandbox/orchestrator.env" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +log() { echo "==> $*"; } + +log "1/6 Ensure Bun on VPS" +ssh "$VPS" ' + command -v bun >/dev/null || { + curl -fsSL https://bun.sh/install | env BUN_INSTALL=/usr/local bash + } + bun --version +' + +log "2/6 Ensure remote dirs" +ssh "$VPS" "mkdir -p $REMOTE_DIR /etc/lobe-sandbox /var/lib/lobe-sandbox" + +log "3/6 Sync orchestrator source" +rsync -az --delete \ + --exclude node_modules --exclude dist --exclude .env --exclude .env.local \ + "$PROJECT_ROOT/orchestrator/" \ + "$VPS:$REMOTE_DIR/orchestrator/" + +log "4/6 bun install on VPS" +ssh "$VPS" "cd $REMOTE_DIR/orchestrator && bun install --production --frozen-lockfile 2>/dev/null || bun install --production" + +log "5/6 Env file (only if missing — do not overwrite)" +ssh "$VPS" " + if [ ! -f $ENV_FILE ]; then + cat > $ENV_FILE <<'ENV' +PORT=8700 +HOST=127.0.0.1 +SANDBOX_ORCH_SECRET=CHANGEME_RUN_openssl_rand_base64_32 +INCUS_PROJECT=lobe-sandbox +INCUS_PROFILE=sandbox-default +INCUS_BASE_IMAGE=lobe-sandbox-base +INCUS_CONTAINER_PREFIX=sb- +IDLE_TIMEOUT_MS=1800000 +STATE_DB_PATH=/var/lib/lobe-sandbox/state.sqlite +S3_ENDPOINT=http://192.168.2.221:9000 +S3_REGION=us-east-1 +S3_BUCKET=lobe-sandbox-exports +S3_ACCESS_KEY_ID=admin +S3_SECRET_ACCESS_KEY=CHANGEME +S3_FORCE_PATH_STYLE=true +ENV + chmod 600 $ENV_FILE + echo '!!! First deploy. Edit $ENV_FILE before starting service !!!' + fi +" + +log "6/6 Install systemd unit and reload" +scp "$PROJECT_ROOT/deploy/sandbox-orchestrator.service" \ + "$VPS:/etc/systemd/system/sandbox-orchestrator.service" +ssh "$VPS" 'systemctl daemon-reload' + +log "DONE" +log "" +log "Next: ssh $VPS and run:" +log " 1. nano $ENV_FILE # set SANDBOX_ORCH_SECRET + S3_SECRET_ACCESS_KEY" +log " 2. systemctl enable --now sandbox-orchestrator" +log " 3. journalctl -u sandbox-orchestrator -f" +log " 4. curl http://127.0.0.1:8700/health" diff --git a/scripts/host-init.sh b/scripts/host-init.sh old mode 100644 new mode 100755