"""Trusted sandbox gateway. Every user operation executes in a restricted K8s Pod. No Docker socket, hostPath, privileged execution or host subprocess is used. The gateway is private infrastructure and requires a cluster service account. """ import base64 import hmac import json import logging import os import threading import time import uuid from pathlib import Path from contextlib import asynccontextmanager from fastapi import FastAPI, Header, HTTPException from kubernetes import client, config from kubernetes.client.rest import ApiException from kubernetes.stream import stream from pydantic import BaseModel, Field from urllib3.exceptions import HTTPError from websocket import WebSocketException TOKEN = os.environ.get("SANDBOX_TOKEN", "") NAMESPACE = os.environ.get("SANDBOX_NAMESPACE", "platform-sandboxes") REVIEW_NAMESPACE = os.environ.get("REVIEW_NAMESPACE", "platform-reviews") IMAGE = os.environ.get("SANDBOX_IMAGE", "") RUNTIME = os.environ.get("SANDBOX_RUNTIME_CLASS", "gvisor") LOCKS: dict[str, threading.Lock] = {} LOCK_GUARD = threading.Lock() MAX_OUTPUT = 100_000 LOG = logging.getLogger("uvicorn.error") TRANSIENT_KUBE_STATUSES = {0, 408, 429, 500, 502, 503, 504} @asynccontextmanager async def lifespan(app: FastAPI): if len(TOKEN) < 32 or not IMAGE or not RUNTIME: raise RuntimeError("Configure SANDBOX_TOKEN, SANDBOX_IMAGE and isolated runtime class") config.load_incluster_config() app.state.core = client.CoreV1Api() app.state.net = client.NetworkingV1Api() yield app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None) from runtime import router as runtime_router app.include_router(runtime_router) from domain_runtime import router as domain_router app.include_router(domain_router) class Tool(BaseModel): name: str = Field(max_length=40) arguments: dict trackChanges: bool = False def authorize(value: str | None): if len(TOKEN) < 32 or not hmac.compare_digest(value or "", "Bearer " + TOKEN): raise HTTPException(401, "Unauthorized") @app.get("/health/ready") def ready(authorization: str | None = Header(default=None)): authorize(authorization) try: app.state.core.list_namespaced_pod(NAMESPACE, limit=1, _request_timeout=(3, 5)) except Exception: raise HTTPException(503, "Sandbox Kubernetes API unavailable") from None return {"ready": True} def workspace_name(project: str) -> str: try: return "dev-" + uuid.UUID(project).hex except ValueError: raise HTTPException(400, "Invalid project ID") def is_quota_denial(exc): if not isinstance(exc, ApiException) or exc.status != 403: return False try: return "exceeded quota:" in json.loads(exc.body or "{}")["message"] except (KeyError, TypeError, ValueError): return False def kube_call(stage, operation): # Use only before sending tool input. A command or write may already have # happened once stdin is sent, so an interrupted result must not be retried. for attempt in range(3): try: return operation() except (ApiException, HTTPError) as exc: status = exc.status if isinstance(exc, ApiException) else 0 cause = exc.__context__ upstream = getattr(cause, "status_code", None) detail = "WORKSPACE_CAPACITY" if is_quota_denial(exc) else stage LOG.warning("Sandbox Kubernetes %s failed (attempt=%d, status=%s, cause=%s)", detail, attempt + 1, upstream or status, type(cause or exc).__name__) if attempt == 2 or (upstream or status) not in TRANSIENT_KUBE_STATUSES: raise HTTPException(503, detail) from None time.sleep(0.25 * 2 ** attempt) def ensure_resource(read, create, namespace, body): def get(): return read(body["metadata"]["name"], namespace, _request_timeout=(3, 5)) def find_or_create(): # Admission can charge quota for a duplicate POST before it returns 409. # Read first on every attempt, including after a lost create response. try: return get() except ApiException as exc: if exc.status != 404: raise try: return create(namespace=namespace, body=body, _request_timeout=(3, 5)) except ApiException as exc: if exc.status != 409 and not is_quota_denial(exc): raise # Another gateway may have created it between the read and create. try: return get() except ApiException as read_error: if read_error.status == 404: raise exc from None raise return kube_call("WORKSPACE_PROVISIONING", find_or_create) def ensure_workspace(project: str, review=False): if review and REVIEW_NAMESPACE == NAMESPACE: raise HTTPException(503,'Review namespace must be separate from development') name = workspace_name(project) namespace = REVIEW_NAMESPACE if review else NAMESPACE core, net = app.state.core, app.state.net # The namespace must enforce restricted Pod Security Admission. labels = {"platform/workspace": name} ensure_resource( net.read_namespaced_network_policy, net.create_namespaced_network_policy, namespace=namespace, body={ "apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", "metadata": {"name": name}, "spec": {"podSelector": {"matchLabels": labels}, "policyTypes": ["Ingress", "Egress"], "ingress": [], "egress": [ {"to": [{"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "kube-system"}}}], "ports": [{"port": 53, "protocol": "UDP"}, {"port": 53, "protocol": "TCP"}]}, {"to": [{"ipBlock": {"cidr": "0.0.0.0/0", "except": [ "0.0.0.0/8", "10.0.0.0/8", "100.64.0.0/10", "127.0.0.0/8", "169.254.0.0/16", "172.16.0.0/12", "192.0.0.0/24", "192.168.0.0/16", "198.18.0.0/15", "224.0.0.0/4", "240.0.0.0/4"]}}], "ports": [{"port": 443, "protocol": "TCP"}, {"port": 80, "protocol": "TCP"}]} ]} }) ensure_resource(core.read_namespaced_persistent_volume_claim, core.create_namespaced_persistent_volume_claim, namespace=namespace, body={ "apiVersion": "v1", "kind": "PersistentVolumeClaim", "metadata": {"name": name}, "spec": {"accessModes": ["ReadWriteOnce"], "resources": {"requests": {"storage": "5Gi"}}}}) pod = ensure_resource(core.read_namespaced_pod, core.create_namespaced_pod, namespace=namespace, body={ "apiVersion": "v1", "kind": "Pod", "metadata": {"name": name, "labels": labels}, "spec": {"runtimeClassName": RUNTIME, "automountServiceAccountToken": False, "restartPolicy": "Always", "enableServiceLinks": False, "securityContext": {"runAsNonRoot": True, "runAsUser": 10001, "runAsGroup": 10001, "fsGroup": 10001, "seccompProfile": {"type": "RuntimeDefault"}}, "containers": [{"name": "workspace", "image": IMAGE, "command": ["python3", "-c", "import time; time.sleep(2147483647)"], "workingDir": "/workspace", "securityContext": {"allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"]}}, "env": [{"name": "HOME", "value": "/workspace"}, {"name": "PIP_CACHE_DIR", "value": "/tmp/pip-cache"}], "resources": {"requests": {"cpu": "100m", "memory": "128Mi"}, "limits": {"cpu": "1", "memory": "1Gi", "ephemeral-storage": "2Gi"}}, "volumeMounts": [{"name": "work", "mountPath": "/workspace"}, {"name": "tmp", "mountPath": "/tmp"}]}], "volumes": [{"name": "work", "persistentVolumeClaim": {"claimName": name}}, {"name": "tmp", "emptyDir": {"sizeLimit": "1Gi"}}]}}) deadline = time.monotonic() + 90 while time.monotonic() < deadline: if pod.status.phase == "Running" and pod.status.container_statuses and all(s.ready for s in pod.status.container_statuses): return name if pod.status.phase in ("Failed", "Succeeded"): LOG.warning("Sandbox pod %s is %s", name, pod.status.phase) raise HTTPException(503, "WORKSPACE_TERMINATED") time.sleep(1) pod = kube_call("WORKSPACE_PROVISIONING", lambda: core.read_namespaced_pod(name, namespace, _request_timeout=(3, 5))) LOG.warning("Sandbox pod %s did not become ready (phase=%s)", name, pod.status.phase) raise HTTPException(503, "WORKSPACE_STARTING") # This code runs ONLY inside the workspace Pod via Kubernetes exec. SCRIPT = r''' import pathlib,os,json,base64,subprocess,signal,tempfile args=json.loads(base64.b64decode(PAYLOAD)) root=pathlib.Path('/workspace').resolve() name=args['name']; values=args['arguments'] track_changes = args.get('trackChanges') and name in {'write_file', 'delete_file', 'run_command'} temporary = None before_complete = False if track_changes: try: temporary = tempfile.TemporaryDirectory(prefix='webup-changes-', ignore_cleanup_errors=True) change_dir = pathlib.Path(temporary.name) snapshot_changes(root, change_dir / 'before') before_complete = True except (OSError, ValueError): pass def path(value): if not isinstance(value,str) or len(value)>1000: raise ValueError('Invalid path') target=(root/value).resolve() if not target.is_relative_to(root): raise ValueError('Path outside workspace') return target try: if name=='run_command': command=values.get('command','') if not isinstance(command,str) or not command or len(command)>10000: raise ValueError('Invalid command') # Output goes to bounded tmpfs, not an unbounded pipe in the gateway. with tempfile.TemporaryFile() as out: p=subprocess.Popen(['/bin/sh','-lc',command],cwd=root,stdout=out,stderr=subprocess.STDOUT,start_new_session=True) timed_out=False try: p.wait(timeout=120) except subprocess.TimeoutExpired: timed_out=True;os.killpg(p.pid,signal.SIGKILL);p.wait() out.seek(0);output=out.read(64001) result={'exitCode':p.returncode,'timedOut':timed_out,'output':output[:64000].decode('utf-8','replace'),'truncated':len(output)>64000} elif name=='review_export': result=review_export(root) elif name=='release_snapshot': result=release_snapshot(root) elif name=='review_import': result=review_import(root,values) elif name=='review_validate': result=review_validate(root) elif name=='workspace_inspect': source_extensions={'.html','.htm','.js','.jsx','.ts','.tsx','.vue','.svelte','.css','.scss','.less','.py','.java','.go','.rs','.php','.rb','.cs','.sql'} excluded={'.git','node_modules','.venv','__pycache__','.next','dist','build','coverage'} files=[];source_count=0;total=0;visited=0;complete=True for directory,dirs,names in os.walk(root,followlinks=False): dirs[:]=sorted(d for d in dirs if d not in excluded and not (pathlib.Path(directory)/d).is_symlink()) for filename in sorted(names): visited+=1 if visited>5000:complete=False;break file=pathlib.Path(directory)/filename if file.is_symlink() or not file.is_file() or filename=='.env' or filename.startswith('.env.'):continue try:size=file.stat().st_size except OSError:continue total+=size if size>0 and file.suffix.lower() in source_extensions:source_count+=1 if len(files)<100:files.append({'path':str(file.relative_to(root)),'bytes':size}) if not complete:break result={'sourceFiles':source_count,'files':files,'totalBytes':total,'complete':complete} missing_start = workspace_missing_start(root) if missing_start: result['missingStartFile'] = missing_start elif name=='archive_source': import io,zipfile output=io.BytesIO();total=0;count=0 with zipfile.ZipFile(output,'w',zipfile.ZIP_DEFLATED) as archive: for file in root.rglob('*'): rel=file.relative_to(root) if any(part in {'.git','node_modules','.venv','__pycache__'} for part in rel.parts): continue if file.is_symlink() or not file.is_file(): continue target=path(str(rel));size=target.stat().st_size;total+=size;count+=1 if total>10_000_000 or count>5000: raise ValueError('Source archive exceeds 10MB / 5000 files limit') archive.write(target,str(rel)) result={'base64':base64.b64encode(output.getvalue()).decode(),'files':count} elif name in ('version_save','version_list'): def git(*arguments): # Kubernetes owns the mount directory; Git should trust only this workspace. return subprocess.run( ['git', '-c', f'safe.directory={root.as_posix()}', '-c', 'core.hooksPath=/dev/null', *arguments], cwd=root, capture_output=True, text=True, timeout=25, ) if name=='version_save': message=values.get('message','') if not isinstance(message,str) or not message or len(message)>200: raise ValueError('Invalid version message') for arguments in [('init',), ('config', 'user.name', 'Workspace'), ('config', 'user.email', 'workspace@localhost')]: if git(*arguments).returncode: raise ValueError('Git initialization failed') ignore=root/'.git/info/exclude' if ignore.parent.is_symlink() or ignore.is_symlink(): raise ValueError('Invalid Git metadata path') with ignore.open('a') as f:f.write('\nnode_modules/\n.venv/\n.env\n.env.*\n!.env.example\n') added=git('add','-A') if added.returncode: raise ValueError('Git staging failed') committed=git('commit','-m',message) if committed.returncode and 'nothing to commit' not in committed.stdout: raise ValueError('Git commit failed') result={'versions': []} if (root / '.git').exists() and git('show-ref', '--head', '--quiet').returncode != 1: log=git('log','-30','--format=%H%x09%ct%x09%s') if log.returncode: raise ValueError('Git history unavailable') result['versions'] = [ dict(zip(['commit', 'timestamp', 'message'], line.split('\t', 2))) for line in log.stdout.splitlines() if '\t' in line ] elif name=='list_files': target=path(values.get('path','.')) result={'files':[{'name':p.name,'directory':p.is_dir(),'symlink':p.is_symlink()} for p in sorted(target.iterdir())][:1000]} elif name=='read_file': target=path(values['path']) if target.stat().st_size>64000: raise ValueError('File exceeds 64KB reading limit') result={'content':target.read_text(encoding='utf-8')} elif name=='write_file': target=path(values['path']);content=values.get('content','') if not isinstance(content,str) or len(content.encode('utf-8'))>256000: raise ValueError('Content exceeds limit') target.parent.mkdir(parents=True,exist_ok=True);target=path(values['path']) target.write_text(content,encoding='utf-8');result={'written':len(content.encode('utf-8'))} elif name=='delete_file': target=path(values['path']) if not target.is_file(): raise ValueError('Only individual files may be deleted') target.unlink();result={'deleted':True} else: raise ValueError('Unknown tool') except Exception as exc: result = {'error': str(exc)[:500]} if track_changes: result['fileChanges'] = {'complete': False, 'files': []} try: if before_complete: snapshot_changes(root, change_dir / 'after') result['fileChanges'] = summarize_changes(change_dir) except (OSError, ValueError, subprocess.SubprocessError): pass finally: if temporary is not None: temporary.cleanup() print(json.dumps(result, ensure_ascii=True if track_changes else False)) ''' SCRIPT = '\n'.join(Path(__file__).with_name(name).read_text(encoding='utf-8') for name in ('review_tools.py', 'workspace_changes.py')) + '\n' + SCRIPT def read_exec_output(connection, limit): try: return drain_exec_output(connection, limit) finally: connection.close() def drain_exec_output(connection, limit): # _preload_content=True lets ApiClient parse JSON and coerce dictionaries back to # Python repr strings. Read raw stdout explicitly and keep stderr separate. deadline = time.monotonic() + 135 chunks, size, error_size = [], 0, 0 while connection.is_open(): remaining = deadline - time.monotonic() if remaining <= 0: raise HTTPException(504, 'Sandbox execution timed out; result must be checked') connection.update(timeout=min(1, remaining)) if connection.peek_stdout(): chunk = connection.read_stdout() size += len(chunk.encode('utf-8')) if size > limit: raise HTTPException(502, 'Tool output exceeded limit') chunks.append(chunk) if connection.peek_stderr(): error_size += len(connection.read_stderr().encode('utf-8')) if error_size > 64000: raise HTTPException(502, 'Sandbox diagnostic output exceeded limit') try: code = connection.returncode except Exception: raise HTTPException(502, 'WORKSPACE_RESULT_UNKNOWN') from None if code != 0: raise HTTPException(502, 'Sandbox wrapper did not complete successfully') return ''.join(chunks) @app.post("/workspaces/{project}/tools") def tool(project: str, request: Tool, authorization: str | None = Header(default=None)): return execute_tool(project,request,authorization,False) @app.post("/reviews/{project}/tools") def review_tool(project: str, request: Tool, authorization: str | None = Header(default=None)): return execute_tool(project,request,authorization,True) @app.delete('/reviews/{project}') def dispose_review(project: str, authorization: str | None = Header(default=None)): authorize(authorization) if REVIEW_NAMESPACE == NAMESPACE:raise HTTPException(503,'Review namespace must be separate') name=workspace_name(project) for operation in (app.state.core.delete_namespaced_pod,app.state.core.delete_namespaced_persistent_volume_claim,app.state.net.delete_namespaced_network_policy): try:operation(name,REVIEW_NAMESPACE,_request_timeout=(3,10)) except ApiException as error: if error.status!=404:raise HTTPException(503,'Review cleanup failed') from None return {'cleanupRequested':True} def execute_tool(project,request,authorization,review): authorize(authorization) name = workspace_name(project) allowed = {'review_import','review_validate','workspace_inspect'} if review else {"list_files", "read_file", "write_file", "delete_file", "run_command", "version_save", "version_list", "archive_source", "workspace_inspect",'review_export','release_snapshot'} if request.name not in allowed: raise HTTPException(400, "Unknown tool") payload = json.dumps(request.model_dump(), ensure_ascii=False).encode() if len(payload) > (16_000_000 if request.name=='review_import' else 300_000): raise HTTPException(413, "Tool payload too large") with LOCK_GUARD: lock = LOCKS.setdefault(('review:' if review else '')+name, threading.Lock()) if not lock.acquire(timeout=1): raise HTTPException(409, "WORKSPACE_BUSY") try: ensure_workspace(project,review) encoded = base64.b64encode(payload).decode() code = "import sys\nPAYLOAD=sys.stdin.read(" + str(len(encoded)) + ")\n" + SCRIPT # Streaming temporarily overrides ApiClient.request. Never share that client # with concurrent health checks, workspace provisioning, or another project. with client.ApiClient() as api_client: core = client.CoreV1Api(api_client) connection = kube_call("WORKSPACE_CONNECTION", lambda: stream( core.connect_get_namespaced_pod_exec, name, REVIEW_NAMESPACE if review else NAMESPACE, container="workspace", command=["timeout", "130", "python3", "-I", "-c", code], stderr=True, stdin=True, stdout=True, tty=False, _request_timeout=135, _preload_content=False)) try: for offset in range(0,len(encoded),16384): connection.write_stdin(encoded[offset:offset+16384]) except Exception: connection.close() raise output_limit = 16_000_000 if request.name in {'archive_source','review_export','release_snapshot'} else MAX_OUTPUT if request.trackChanges: output_limit = 512_000 output = read_exec_output(connection, output_limit) try: return json.loads(output) except ValueError: raise HTTPException(502, "Sandbox returned invalid output") from None except (ApiException, HTTPError, OSError, WebSocketException) as exc: LOG.warning("Sandbox tool result unavailable (workspace=%s, tool=%s, cause=%s)", name, request.name, type(exc).__name__) raise HTTPException(502, "WORKSPACE_RESULT_UNKNOWN") from None finally: lock.release()