Files

444 lines
22 KiB
Python

"""Trusted sandbox gateway. Every user operation executes in a restricted K8s Pod.
No Docker socket, hostPath, privileged execution or host subprocess is used.
The gateway is private infrastructure and requires a cluster service account.
"""
import base64
import hmac
import json
import logging
import os
import threading
import time
import uuid
from pathlib import Path
from contextlib import asynccontextmanager
from fastapi import FastAPI, Header, HTTPException
from kubernetes import client, config
from kubernetes.client.rest import ApiException
from kubernetes.stream import stream
from pydantic import BaseModel, Field
from urllib3.exceptions import HTTPError
from websocket import WebSocketException
TOKEN = os.environ.get("SANDBOX_TOKEN", "")
NAMESPACE = os.environ.get("SANDBOX_NAMESPACE", "platform-sandboxes")
REVIEW_NAMESPACE = os.environ.get("REVIEW_NAMESPACE", "platform-reviews")
IMAGE = os.environ.get("SANDBOX_IMAGE", "")
RUNTIME = os.environ.get("SANDBOX_RUNTIME_CLASS", "gvisor")
LOCKS: dict[str, threading.Lock] = {}
LOCK_GUARD = threading.Lock()
MAX_OUTPUT = 100_000
LOG = logging.getLogger("uvicorn.error")
TRANSIENT_KUBE_STATUSES = {0, 408, 429, 500, 502, 503, 504}
@asynccontextmanager
async def lifespan(app: FastAPI):
if len(TOKEN) < 32 or not IMAGE or not RUNTIME:
raise RuntimeError("Configure SANDBOX_TOKEN, SANDBOX_IMAGE and isolated runtime class")
config.load_incluster_config()
app.state.core = client.CoreV1Api()
app.state.net = client.NetworkingV1Api()
yield
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None)
from runtime import router as runtime_router
app.include_router(runtime_router)
from domain_runtime import router as domain_router
app.include_router(domain_router)
class Tool(BaseModel):
name: str = Field(max_length=40)
arguments: dict
trackChanges: bool = False
def authorize(value: str | None):
if len(TOKEN) < 32 or not hmac.compare_digest(value or "", "Bearer " + TOKEN):
raise HTTPException(401, "Unauthorized")
@app.get("/health/ready")
def ready(authorization: str | None = Header(default=None)):
authorize(authorization)
try:
app.state.core.list_namespaced_pod(NAMESPACE, limit=1, _request_timeout=(3, 5))
except Exception:
raise HTTPException(503, "Sandbox Kubernetes API unavailable") from None
return {"ready": True}
def workspace_name(project: str) -> str:
try:
return "dev-" + uuid.UUID(project).hex
except ValueError:
raise HTTPException(400, "Invalid project ID")
def is_quota_denial(exc):
if not isinstance(exc, ApiException) or exc.status != 403:
return False
try:
return "exceeded quota:" in json.loads(exc.body or "{}")["message"]
except (KeyError, TypeError, ValueError):
return False
def kube_call(stage, operation):
# Use only before sending tool input. A command or write may already have
# happened once stdin is sent, so an interrupted result must not be retried.
for attempt in range(3):
try:
return operation()
except (ApiException, HTTPError) as exc:
status = exc.status if isinstance(exc, ApiException) else 0
cause = exc.__context__
upstream = getattr(cause, "status_code", None)
detail = "WORKSPACE_CAPACITY" if is_quota_denial(exc) else stage
LOG.warning("Sandbox Kubernetes %s failed (attempt=%d, status=%s, cause=%s)",
detail, attempt + 1, upstream or status, type(cause or exc).__name__)
if attempt == 2 or (upstream or status) not in TRANSIENT_KUBE_STATUSES:
raise HTTPException(503, detail) from None
time.sleep(0.25 * 2 ** attempt)
def ensure_resource(read, create, namespace, body):
def get():
return read(body["metadata"]["name"], namespace, _request_timeout=(3, 5))
def find_or_create():
# Admission can charge quota for a duplicate POST before it returns 409.
# Read first on every attempt, including after a lost create response.
try:
return get()
except ApiException as exc:
if exc.status != 404:
raise
try:
return create(namespace=namespace, body=body, _request_timeout=(3, 5))
except ApiException as exc:
if exc.status != 409 and not is_quota_denial(exc):
raise
# Another gateway may have created it between the read and create.
try:
return get()
except ApiException as read_error:
if read_error.status == 404:
raise exc from None
raise
return kube_call("WORKSPACE_PROVISIONING", find_or_create)
def ensure_workspace(project: str, review=False):
if review and REVIEW_NAMESPACE == NAMESPACE:
raise HTTPException(503,'Review namespace must be separate from development')
name = workspace_name(project)
namespace = REVIEW_NAMESPACE if review else NAMESPACE
core, net = app.state.core, app.state.net
# The namespace must enforce restricted Pod Security Admission.
labels = {"platform/workspace": name}
ensure_resource(
net.read_namespaced_network_policy,
net.create_namespaced_network_policy,
namespace=namespace,
body={
"apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy",
"metadata": {"name": name},
"spec": {"podSelector": {"matchLabels": labels}, "policyTypes": ["Ingress", "Egress"],
"ingress": [],
"egress": [
{"to": [{"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "kube-system"}}}],
"ports": [{"port": 53, "protocol": "UDP"}, {"port": 53, "protocol": "TCP"}]},
{"to": [{"ipBlock": {"cidr": "0.0.0.0/0", "except": [
"0.0.0.0/8", "10.0.0.0/8", "100.64.0.0/10", "127.0.0.0/8",
"169.254.0.0/16", "172.16.0.0/12", "192.0.0.0/24", "192.168.0.0/16",
"198.18.0.0/15", "224.0.0.0/4", "240.0.0.0/4"]}}],
"ports": [{"port": 443, "protocol": "TCP"}, {"port": 80, "protocol": "TCP"}]}
]}
})
ensure_resource(core.read_namespaced_persistent_volume_claim,
core.create_namespaced_persistent_volume_claim, namespace=namespace, body={
"apiVersion": "v1", "kind": "PersistentVolumeClaim", "metadata": {"name": name},
"spec": {"accessModes": ["ReadWriteOnce"], "resources": {"requests": {"storage": "5Gi"}}}})
pod = ensure_resource(core.read_namespaced_pod, core.create_namespaced_pod, namespace=namespace, body={
"apiVersion": "v1", "kind": "Pod", "metadata": {"name": name, "labels": labels},
"spec": {"runtimeClassName": RUNTIME, "automountServiceAccountToken": False,
"restartPolicy": "Always", "enableServiceLinks": False,
"securityContext": {"runAsNonRoot": True, "runAsUser": 10001, "runAsGroup": 10001,
"fsGroup": 10001, "seccompProfile": {"type": "RuntimeDefault"}},
"containers": [{"name": "workspace", "image": IMAGE,
"command": ["python3", "-c", "import time; time.sleep(2147483647)"],
"workingDir": "/workspace",
"securityContext": {"allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True,
"capabilities": {"drop": ["ALL"]}},
"env": [{"name": "HOME", "value": "/workspace"}, {"name": "PIP_CACHE_DIR", "value": "/tmp/pip-cache"}],
"resources": {"requests": {"cpu": "100m", "memory": "128Mi"},
"limits": {"cpu": "1", "memory": "1Gi", "ephemeral-storage": "2Gi"}},
"volumeMounts": [{"name": "work", "mountPath": "/workspace"}, {"name": "tmp", "mountPath": "/tmp"}]}],
"volumes": [{"name": "work", "persistentVolumeClaim": {"claimName": name}},
{"name": "tmp", "emptyDir": {"sizeLimit": "1Gi"}}]}})
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
if pod.status.phase == "Running" and pod.status.container_statuses and all(s.ready for s in pod.status.container_statuses):
return name
if pod.status.phase in ("Failed", "Succeeded"):
LOG.warning("Sandbox pod %s is %s", name, pod.status.phase)
raise HTTPException(503, "WORKSPACE_TERMINATED")
time.sleep(1)
pod = kube_call("WORKSPACE_PROVISIONING",
lambda: core.read_namespaced_pod(name, namespace, _request_timeout=(3, 5)))
LOG.warning("Sandbox pod %s did not become ready (phase=%s)", name, pod.status.phase)
raise HTTPException(503, "WORKSPACE_STARTING")
# This code runs ONLY inside the workspace Pod via Kubernetes exec.
SCRIPT = r'''
import pathlib,os,json,base64,subprocess,signal,tempfile
args=json.loads(base64.b64decode(PAYLOAD))
root=pathlib.Path('/workspace').resolve()
name=args['name']; values=args['arguments']
track_changes = args.get('trackChanges') and name in {'write_file', 'delete_file', 'run_command'}
temporary = None
before_complete = False
if track_changes:
try:
temporary = tempfile.TemporaryDirectory(prefix='webup-changes-', ignore_cleanup_errors=True)
change_dir = pathlib.Path(temporary.name)
snapshot_changes(root, change_dir / 'before')
before_complete = True
except (OSError, ValueError):
pass
def path(value):
if not isinstance(value,str) or len(value)>1000: raise ValueError('Invalid path')
target=(root/value).resolve()
if not target.is_relative_to(root): raise ValueError('Path outside workspace')
return target
try:
if name=='run_command':
command=values.get('command','')
if not isinstance(command,str) or not command or len(command)>10000: raise ValueError('Invalid command')
# Output goes to bounded tmpfs, not an unbounded pipe in the gateway.
with tempfile.TemporaryFile() as out:
p=subprocess.Popen(['/bin/sh','-lc',command],cwd=root,stdout=out,stderr=subprocess.STDOUT,start_new_session=True)
timed_out=False
try: p.wait(timeout=120)
except subprocess.TimeoutExpired:
timed_out=True;os.killpg(p.pid,signal.SIGKILL);p.wait()
out.seek(0);output=out.read(64001)
result={'exitCode':p.returncode,'timedOut':timed_out,'output':output[:64000].decode('utf-8','replace'),'truncated':len(output)>64000}
elif name=='review_export':
result=review_export(root)
elif name=='release_snapshot':
result=release_snapshot(root)
elif name=='review_import':
result=review_import(root,values)
elif name=='review_validate':
result=review_validate(root)
elif name=='workspace_inspect':
source_extensions={'.html','.htm','.js','.jsx','.ts','.tsx','.vue','.svelte','.css','.scss','.less','.py','.java','.go','.rs','.php','.rb','.cs','.sql'}
excluded={'.git','node_modules','.venv','__pycache__','.next','dist','build','coverage'}
files=[];source_count=0;total=0;visited=0;complete=True
for directory,dirs,names in os.walk(root,followlinks=False):
dirs[:]=sorted(d for d in dirs if d not in excluded and not (pathlib.Path(directory)/d).is_symlink())
for filename in sorted(names):
visited+=1
if visited>5000:complete=False;break
file=pathlib.Path(directory)/filename
if file.is_symlink() or not file.is_file() or filename=='.env' or filename.startswith('.env.'):continue
try:size=file.stat().st_size
except OSError:continue
total+=size
if size>0 and file.suffix.lower() in source_extensions:source_count+=1
if len(files)<100:files.append({'path':str(file.relative_to(root)),'bytes':size})
if not complete:break
result={'sourceFiles':source_count,'files':files,'totalBytes':total,'complete':complete}
missing_start = workspace_missing_start(root)
if missing_start:
result['missingStartFile'] = missing_start
elif name=='archive_source':
import io,zipfile
output=io.BytesIO();total=0;count=0
with zipfile.ZipFile(output,'w',zipfile.ZIP_DEFLATED) as archive:
for file in root.rglob('*'):
rel=file.relative_to(root)
if any(part in {'.git','node_modules','.venv','__pycache__'} for part in rel.parts): continue
if file.is_symlink() or not file.is_file(): continue
target=path(str(rel));size=target.stat().st_size;total+=size;count+=1
if total>10_000_000 or count>5000: raise ValueError('Source archive exceeds 10MB / 5000 files limit')
archive.write(target,str(rel))
result={'base64':base64.b64encode(output.getvalue()).decode(),'files':count}
elif name in ('version_save','version_list'):
def git(*arguments):
# Kubernetes owns the mount directory; Git should trust only this workspace.
return subprocess.run(
['git', '-c', f'safe.directory={root.as_posix()}', '-c', 'core.hooksPath=/dev/null', *arguments],
cwd=root, capture_output=True, text=True, timeout=25,
)
if name=='version_save':
message=values.get('message','')
if not isinstance(message,str) or not message or len(message)>200: raise ValueError('Invalid version message')
for arguments in [('init',), ('config', 'user.name', 'Workspace'), ('config', 'user.email', 'workspace@localhost')]:
if git(*arguments).returncode:
raise ValueError('Git initialization failed')
ignore=root/'.git/info/exclude'
if ignore.parent.is_symlink() or ignore.is_symlink(): raise ValueError('Invalid Git metadata path')
with ignore.open('a') as f:f.write('\nnode_modules/\n.venv/\n.env\n.env.*\n!.env.example\n')
added=git('add','-A')
if added.returncode: raise ValueError('Git staging failed')
committed=git('commit','-m',message)
if committed.returncode and 'nothing to commit' not in committed.stdout: raise ValueError('Git commit failed')
result={'versions': []}
if (root / '.git').exists() and git('show-ref', '--head', '--quiet').returncode != 1:
log=git('log','-30','--format=%H%x09%ct%x09%s')
if log.returncode:
raise ValueError('Git history unavailable')
result['versions'] = [
dict(zip(['commit', 'timestamp', 'message'], line.split('\t', 2)))
for line in log.stdout.splitlines() if '\t' in line
]
elif name=='list_files':
target=path(values.get('path','.'))
result={'files':[{'name':p.name,'directory':p.is_dir(),'symlink':p.is_symlink()} for p in sorted(target.iterdir())][:1000]}
elif name=='read_file':
target=path(values['path'])
if target.stat().st_size>64000: raise ValueError('File exceeds 64KB reading limit')
result={'content':target.read_text(encoding='utf-8')}
elif name=='write_file':
target=path(values['path']);content=values.get('content','')
if not isinstance(content,str) or len(content.encode('utf-8'))>256000: raise ValueError('Content exceeds limit')
target.parent.mkdir(parents=True,exist_ok=True);target=path(values['path'])
target.write_text(content,encoding='utf-8');result={'written':len(content.encode('utf-8'))}
elif name=='delete_file':
target=path(values['path'])
if not target.is_file(): raise ValueError('Only individual files may be deleted')
target.unlink();result={'deleted':True}
else: raise ValueError('Unknown tool')
except Exception as exc:
result = {'error': str(exc)[:500]}
if track_changes:
result['fileChanges'] = {'complete': False, 'files': []}
try:
if before_complete:
snapshot_changes(root, change_dir / 'after')
result['fileChanges'] = summarize_changes(change_dir)
except (OSError, ValueError, subprocess.SubprocessError):
pass
finally:
if temporary is not None:
temporary.cleanup()
print(json.dumps(result, ensure_ascii=True if track_changes else False))
'''
SCRIPT = '\n'.join(Path(__file__).with_name(name).read_text(encoding='utf-8')
for name in ('review_tools.py', 'workspace_changes.py')) + '\n' + SCRIPT
def read_exec_output(connection, limit):
try:
return drain_exec_output(connection, limit)
finally:
connection.close()
def drain_exec_output(connection, limit):
# _preload_content=True lets ApiClient parse JSON and coerce dictionaries back to
# Python repr strings. Read raw stdout explicitly and keep stderr separate.
deadline = time.monotonic() + 135
chunks, size, error_size = [], 0, 0
while connection.is_open():
remaining = deadline - time.monotonic()
if remaining <= 0:
raise HTTPException(504, 'Sandbox execution timed out; result must be checked')
connection.update(timeout=min(1, remaining))
if connection.peek_stdout():
chunk = connection.read_stdout()
size += len(chunk.encode('utf-8'))
if size > limit:
raise HTTPException(502, 'Tool output exceeded limit')
chunks.append(chunk)
if connection.peek_stderr():
error_size += len(connection.read_stderr().encode('utf-8'))
if error_size > 64000:
raise HTTPException(502, 'Sandbox diagnostic output exceeded limit')
try:
code = connection.returncode
except Exception:
raise HTTPException(502, 'WORKSPACE_RESULT_UNKNOWN') from None
if code != 0:
raise HTTPException(502, 'Sandbox wrapper did not complete successfully')
return ''.join(chunks)
@app.post("/workspaces/{project}/tools")
def tool(project: str, request: Tool, authorization: str | None = Header(default=None)):
return execute_tool(project,request,authorization,False)
@app.post("/reviews/{project}/tools")
def review_tool(project: str, request: Tool, authorization: str | None = Header(default=None)):
return execute_tool(project,request,authorization,True)
@app.delete('/reviews/{project}')
def dispose_review(project: str, authorization: str | None = Header(default=None)):
authorize(authorization)
if REVIEW_NAMESPACE == NAMESPACE:raise HTTPException(503,'Review namespace must be separate')
name=workspace_name(project)
for operation in (app.state.core.delete_namespaced_pod,app.state.core.delete_namespaced_persistent_volume_claim,app.state.net.delete_namespaced_network_policy):
try:operation(name,REVIEW_NAMESPACE,_request_timeout=(3,10))
except ApiException as error:
if error.status!=404:raise HTTPException(503,'Review cleanup failed') from None
return {'cleanupRequested':True}
def execute_tool(project,request,authorization,review):
authorize(authorization)
name = workspace_name(project)
allowed = {'review_import','review_validate','workspace_inspect'} if review else {"list_files", "read_file", "write_file", "delete_file", "run_command", "version_save", "version_list", "archive_source", "workspace_inspect",'review_export','release_snapshot'}
if request.name not in allowed:
raise HTTPException(400, "Unknown tool")
payload = json.dumps(request.model_dump(), ensure_ascii=False).encode()
if len(payload) > (16_000_000 if request.name=='review_import' else 300_000):
raise HTTPException(413, "Tool payload too large")
with LOCK_GUARD:
lock = LOCKS.setdefault(('review:' if review else '')+name, threading.Lock())
if not lock.acquire(timeout=1):
raise HTTPException(409, "WORKSPACE_BUSY")
try:
ensure_workspace(project,review)
encoded = base64.b64encode(payload).decode()
code = "import sys\nPAYLOAD=sys.stdin.read(" + str(len(encoded)) + ")\n" + SCRIPT
# Streaming temporarily overrides ApiClient.request. Never share that client
# with concurrent health checks, workspace provisioning, or another project.
with client.ApiClient() as api_client:
core = client.CoreV1Api(api_client)
connection = kube_call("WORKSPACE_CONNECTION", lambda: stream(
core.connect_get_namespaced_pod_exec, name, REVIEW_NAMESPACE if review else NAMESPACE,
container="workspace", command=["timeout", "130", "python3", "-I", "-c", code],
stderr=True, stdin=True, stdout=True, tty=False, _request_timeout=135,
_preload_content=False))
try:
for offset in range(0,len(encoded),16384):
connection.write_stdin(encoded[offset:offset+16384])
except Exception:
connection.close()
raise
output_limit = 16_000_000 if request.name in {'archive_source','review_export','release_snapshot'} else MAX_OUTPUT
if request.trackChanges:
output_limit = 512_000
output = read_exec_output(connection, output_limit)
try:
return json.loads(output)
except ValueError:
raise HTTPException(502, "Sandbox returned invalid output") from None
except (ApiException, HTTPError, OSError, WebSocketException) as exc:
LOG.warning("Sandbox tool result unavailable (workspace=%s, tool=%s, cause=%s)",
name, request.name, type(exc).__name__)
raise HTTPException(502, "WORKSPACE_RESULT_UNKNOWN") from None
finally:
lock.release()