426 lines
21 KiB
Python
426 lines
21 KiB
Python
"""Trusted sandbox gateway. Every user operation executes in a restricted K8s Pod.
|
|
|
|
No Docker socket, hostPath, privileged execution or host subprocess is used.
|
|
The gateway is private infrastructure and requires a cluster service account.
|
|
"""
|
|
import base64
|
|
import hmac
|
|
import json
|
|
import logging
|
|
import os
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
from contextlib import asynccontextmanager
|
|
|
|
from fastapi import FastAPI, Header, HTTPException
|
|
from kubernetes import client, config
|
|
from kubernetes.client.rest import ApiException
|
|
from kubernetes.stream import stream
|
|
from pydantic import BaseModel, Field
|
|
from urllib3.exceptions import HTTPError
|
|
from websocket import WebSocketException
|
|
|
|
TOKEN = os.environ.get("SANDBOX_TOKEN", "")
|
|
NAMESPACE = os.environ.get("SANDBOX_NAMESPACE", "platform-sandboxes")
|
|
REVIEW_NAMESPACE = os.environ.get("REVIEW_NAMESPACE", "platform-reviews")
|
|
IMAGE = os.environ.get("SANDBOX_IMAGE", "")
|
|
RUNTIME = os.environ.get("SANDBOX_RUNTIME_CLASS", "gvisor")
|
|
LOCKS: dict[str, threading.Lock] = {}
|
|
LOCK_GUARD = threading.Lock()
|
|
MAX_OUTPUT = 100_000
|
|
LOG = logging.getLogger("uvicorn.error")
|
|
TRANSIENT_KUBE_STATUSES = {0, 408, 429, 500, 502, 503, 504}
|
|
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(app: FastAPI):
|
|
if len(TOKEN) < 32 or not IMAGE or not RUNTIME:
|
|
raise RuntimeError("Configure SANDBOX_TOKEN, SANDBOX_IMAGE and isolated runtime class")
|
|
config.load_incluster_config()
|
|
app.state.core = client.CoreV1Api()
|
|
app.state.net = client.NetworkingV1Api()
|
|
yield
|
|
|
|
|
|
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None)
|
|
from runtime import router as runtime_router
|
|
app.include_router(runtime_router)
|
|
from domain_runtime import router as domain_router
|
|
app.include_router(domain_router)
|
|
|
|
|
|
class Tool(BaseModel):
|
|
name: str = Field(max_length=40)
|
|
arguments: dict
|
|
trackChanges: bool = False
|
|
|
|
|
|
def authorize(value: str | None):
|
|
if len(TOKEN) < 32 or not hmac.compare_digest(value or "", "Bearer " + TOKEN):
|
|
raise HTTPException(401, "Unauthorized")
|
|
|
|
|
|
@app.get("/health/ready")
|
|
def ready(authorization: str | None = Header(default=None)):
|
|
authorize(authorization)
|
|
try:
|
|
app.state.core.list_namespaced_pod(NAMESPACE, limit=1, _request_timeout=(3, 5))
|
|
except Exception:
|
|
raise HTTPException(503, "Sandbox Kubernetes API unavailable") from None
|
|
return {"ready": True}
|
|
|
|
|
|
def workspace_name(project: str) -> str:
|
|
try:
|
|
return "dev-" + uuid.UUID(project).hex
|
|
except ValueError:
|
|
raise HTTPException(400, "Invalid project ID")
|
|
|
|
|
|
def is_quota_denial(exc):
|
|
if not isinstance(exc, ApiException) or exc.status != 403:
|
|
return False
|
|
try:
|
|
return "exceeded quota:" in json.loads(exc.body or "{}")["message"]
|
|
except (KeyError, TypeError, ValueError):
|
|
return False
|
|
|
|
|
|
def kube_call(stage, operation):
|
|
# Use only before sending tool input. A command or write may already have
|
|
# happened once stdin is sent, so an interrupted result must not be retried.
|
|
for attempt in range(3):
|
|
try:
|
|
return operation()
|
|
except (ApiException, HTTPError) as exc:
|
|
status = exc.status if isinstance(exc, ApiException) else 0
|
|
cause = exc.__context__
|
|
upstream = getattr(cause, "status_code", None)
|
|
detail = "WORKSPACE_CAPACITY" if is_quota_denial(exc) else stage
|
|
LOG.warning("Sandbox Kubernetes %s failed (attempt=%d, status=%s, cause=%s)",
|
|
detail, attempt + 1, upstream or status, type(cause or exc).__name__)
|
|
if attempt == 2 or (upstream or status) not in TRANSIENT_KUBE_STATUSES:
|
|
raise HTTPException(503, detail) from None
|
|
time.sleep(0.25 * 2 ** attempt)
|
|
|
|
|
|
def ensure_resource(read, create, namespace, body):
|
|
def get():
|
|
return read(body["metadata"]["name"], namespace, _request_timeout=(3, 5))
|
|
|
|
def find_or_create():
|
|
# Admission can charge quota for a duplicate POST before it returns 409.
|
|
# Read first on every attempt, including after a lost create response.
|
|
try:
|
|
return get()
|
|
except ApiException as exc:
|
|
if exc.status != 404:
|
|
raise
|
|
try:
|
|
return create(namespace=namespace, body=body, _request_timeout=(3, 5))
|
|
except ApiException as exc:
|
|
if exc.status != 409 and not is_quota_denial(exc):
|
|
raise
|
|
# Another gateway may have created it between the read and create.
|
|
try:
|
|
return get()
|
|
except ApiException as read_error:
|
|
if read_error.status == 404:
|
|
raise exc from None
|
|
raise
|
|
|
|
return kube_call("WORKSPACE_PROVISIONING", find_or_create)
|
|
|
|
|
|
def ensure_workspace(project: str, review=False):
|
|
if review and REVIEW_NAMESPACE == NAMESPACE:
|
|
raise HTTPException(503,'Review namespace must be separate from development')
|
|
name = workspace_name(project)
|
|
namespace = REVIEW_NAMESPACE if review else NAMESPACE
|
|
core, net = app.state.core, app.state.net
|
|
# The namespace must enforce restricted Pod Security Admission.
|
|
labels = {"platform/workspace": name}
|
|
ensure_resource(
|
|
net.read_namespaced_network_policy,
|
|
net.create_namespaced_network_policy,
|
|
namespace=namespace,
|
|
body={
|
|
"apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy",
|
|
"metadata": {"name": name},
|
|
"spec": {"podSelector": {"matchLabels": labels}, "policyTypes": ["Ingress", "Egress"],
|
|
"ingress": [],
|
|
"egress": [
|
|
{"to": [{"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "kube-system"}}}],
|
|
"ports": [{"port": 53, "protocol": "UDP"}, {"port": 53, "protocol": "TCP"}]},
|
|
{"to": [{"ipBlock": {"cidr": "0.0.0.0/0", "except": [
|
|
"0.0.0.0/8", "10.0.0.0/8", "100.64.0.0/10", "127.0.0.0/8",
|
|
"169.254.0.0/16", "172.16.0.0/12", "192.0.0.0/24", "192.168.0.0/16",
|
|
"198.18.0.0/15", "224.0.0.0/4", "240.0.0.0/4"]}}],
|
|
"ports": [{"port": 443, "protocol": "TCP"}, {"port": 80, "protocol": "TCP"}]}
|
|
]}
|
|
})
|
|
ensure_resource(core.read_namespaced_persistent_volume_claim,
|
|
core.create_namespaced_persistent_volume_claim, namespace=namespace, body={
|
|
"apiVersion": "v1", "kind": "PersistentVolumeClaim", "metadata": {"name": name},
|
|
"spec": {"accessModes": ["ReadWriteOnce"], "resources": {"requests": {"storage": "5Gi"}}}})
|
|
pod = ensure_resource(core.read_namespaced_pod, core.create_namespaced_pod, namespace=namespace, body={
|
|
"apiVersion": "v1", "kind": "Pod", "metadata": {"name": name, "labels": labels},
|
|
"spec": {"runtimeClassName": RUNTIME, "automountServiceAccountToken": False,
|
|
"restartPolicy": "Always", "enableServiceLinks": False,
|
|
"securityContext": {"runAsNonRoot": True, "runAsUser": 10001, "runAsGroup": 10001,
|
|
"fsGroup": 10001, "seccompProfile": {"type": "RuntimeDefault"}},
|
|
"containers": [{"name": "workspace", "image": IMAGE,
|
|
"command": ["python3", "-c", "import time; time.sleep(2147483647)"],
|
|
"workingDir": "/workspace",
|
|
"securityContext": {"allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True,
|
|
"capabilities": {"drop": ["ALL"]}},
|
|
"env": [{"name": "HOME", "value": "/workspace"}, {"name": "PIP_CACHE_DIR", "value": "/tmp/pip-cache"}],
|
|
"resources": {"requests": {"cpu": "100m", "memory": "128Mi"},
|
|
"limits": {"cpu": "1", "memory": "1Gi", "ephemeral-storage": "2Gi"}},
|
|
"volumeMounts": [{"name": "work", "mountPath": "/workspace"}, {"name": "tmp", "mountPath": "/tmp"}]}],
|
|
"volumes": [{"name": "work", "persistentVolumeClaim": {"claimName": name}},
|
|
{"name": "tmp", "emptyDir": {"sizeLimit": "1Gi"}}]}})
|
|
deadline = time.monotonic() + 90
|
|
while time.monotonic() < deadline:
|
|
if pod.status.phase == "Running" and pod.status.container_statuses and all(s.ready for s in pod.status.container_statuses):
|
|
return name
|
|
if pod.status.phase in ("Failed", "Succeeded"):
|
|
LOG.warning("Sandbox pod %s is %s", name, pod.status.phase)
|
|
raise HTTPException(503, "WORKSPACE_TERMINATED")
|
|
time.sleep(1)
|
|
pod = kube_call("WORKSPACE_PROVISIONING",
|
|
lambda: core.read_namespaced_pod(name, namespace, _request_timeout=(3, 5)))
|
|
LOG.warning("Sandbox pod %s did not become ready (phase=%s)", name, pod.status.phase)
|
|
raise HTTPException(503, "WORKSPACE_STARTING")
|
|
|
|
|
|
# This code runs ONLY inside the workspace Pod via Kubernetes exec.
|
|
SCRIPT = r'''
|
|
import pathlib,os,json,base64,subprocess,signal,tempfile
|
|
args=json.loads(base64.b64decode(PAYLOAD))
|
|
root=pathlib.Path('/workspace').resolve()
|
|
name=args['name']; values=args['arguments']
|
|
track_changes = args.get('trackChanges') and name in {'write_file', 'delete_file', 'run_command'}
|
|
temporary = None
|
|
before_complete = False
|
|
if track_changes:
|
|
try:
|
|
temporary = tempfile.TemporaryDirectory(prefix='webup-changes-', ignore_cleanup_errors=True)
|
|
change_dir = pathlib.Path(temporary.name)
|
|
snapshot_changes(root, change_dir / 'before')
|
|
before_complete = True
|
|
except (OSError, ValueError):
|
|
pass
|
|
def path(value):
|
|
if not isinstance(value,str) or len(value)>1000: raise ValueError('Invalid path')
|
|
target=(root/value).resolve()
|
|
if not target.is_relative_to(root): raise ValueError('Path outside workspace')
|
|
return target
|
|
try:
|
|
if name=='run_command':
|
|
command=values.get('command','')
|
|
if not isinstance(command,str) or not command or len(command)>10000: raise ValueError('Invalid command')
|
|
# Output goes to bounded tmpfs, not an unbounded pipe in the gateway.
|
|
with tempfile.TemporaryFile() as out:
|
|
p=subprocess.Popen(['/bin/sh','-lc',command],cwd=root,stdout=out,stderr=subprocess.STDOUT,start_new_session=True)
|
|
timed_out=False
|
|
try: p.wait(timeout=120)
|
|
except subprocess.TimeoutExpired:
|
|
timed_out=True;os.killpg(p.pid,signal.SIGKILL);p.wait()
|
|
out.seek(0);output=out.read(64001)
|
|
result={'exitCode':p.returncode,'timedOut':timed_out,'output':output[:64000].decode('utf-8','replace'),'truncated':len(output)>64000}
|
|
elif name=='review_export':
|
|
result=review_export(root)
|
|
elif name=='review_import':
|
|
result=review_import(root,values)
|
|
elif name=='review_validate':
|
|
result=review_validate(root)
|
|
elif name=='workspace_inspect':
|
|
source_extensions={'.html','.htm','.js','.jsx','.ts','.tsx','.vue','.svelte','.css','.scss','.less','.py','.java','.go','.rs','.php','.rb','.cs','.sql'}
|
|
excluded={'.git','node_modules','.venv','__pycache__','.next','dist','build','coverage'}
|
|
files=[];source_count=0;total=0;visited=0;complete=True
|
|
for directory,dirs,names in os.walk(root,followlinks=False):
|
|
dirs[:]=sorted(d for d in dirs if d not in excluded and not (pathlib.Path(directory)/d).is_symlink())
|
|
for filename in sorted(names):
|
|
visited+=1
|
|
if visited>5000:complete=False;break
|
|
file=pathlib.Path(directory)/filename
|
|
if file.is_symlink() or not file.is_file() or filename=='.env' or filename.startswith('.env.'):continue
|
|
try:size=file.stat().st_size
|
|
except OSError:continue
|
|
total+=size
|
|
if size>0 and file.suffix.lower() in source_extensions:source_count+=1
|
|
if len(files)<100:files.append({'path':str(file.relative_to(root)),'bytes':size})
|
|
if not complete:break
|
|
result={'sourceFiles':source_count,'files':files,'totalBytes':total,'complete':complete}
|
|
elif name=='archive_source':
|
|
import io,zipfile
|
|
output=io.BytesIO();total=0;count=0
|
|
with zipfile.ZipFile(output,'w',zipfile.ZIP_DEFLATED) as archive:
|
|
for file in root.rglob('*'):
|
|
rel=file.relative_to(root)
|
|
if any(part in {'.git','node_modules','.venv','__pycache__'} for part in rel.parts): continue
|
|
if file.is_symlink() or not file.is_file(): continue
|
|
target=path(str(rel));size=target.stat().st_size;total+=size;count+=1
|
|
if total>10_000_000 or count>5000: raise ValueError('Source archive exceeds 10MB / 5000 files limit')
|
|
archive.write(target,str(rel))
|
|
result={'base64':base64.b64encode(output.getvalue()).decode(),'files':count}
|
|
elif name in ('version_save','version_list'):
|
|
def git(*arguments):
|
|
return subprocess.run(['git','-c','core.hooksPath=/dev/null',*arguments],cwd=root,capture_output=True,text=True,timeout=25)
|
|
if name=='version_save':
|
|
message=values.get('message','')
|
|
if not isinstance(message,str) or not message or len(message)>200: raise ValueError('Invalid version message')
|
|
git('init');git('config','user.name','Workspace');git('config','user.email','workspace@localhost')
|
|
ignore=root/'.git/info/exclude'
|
|
if ignore.parent.is_symlink() or ignore.is_symlink(): raise ValueError('Invalid Git metadata path')
|
|
with ignore.open('a') as f:f.write('\nnode_modules/\n.venv/\n.env\n.env.*\n!.env.example\n')
|
|
added=git('add','-A')
|
|
if added.returncode: raise ValueError('Git staging failed')
|
|
committed=git('commit','-m',message)
|
|
if committed.returncode and 'nothing to commit' not in committed.stdout: raise ValueError('Git commit failed')
|
|
log=git('log','-30','--format=%H%x09%ct%x09%s')
|
|
result={'versions':[dict(zip(['commit','timestamp','message'],line.split('\t',2))) for line in log.stdout.splitlines() if '\t' in line]}
|
|
elif name=='list_files':
|
|
target=path(values.get('path','.'))
|
|
result={'files':[{'name':p.name,'directory':p.is_dir(),'symlink':p.is_symlink()} for p in sorted(target.iterdir())][:1000]}
|
|
elif name=='read_file':
|
|
target=path(values['path'])
|
|
if target.stat().st_size>64000: raise ValueError('File exceeds 64KB reading limit')
|
|
result={'content':target.read_text(encoding='utf-8')}
|
|
elif name=='write_file':
|
|
target=path(values['path']);content=values.get('content','')
|
|
if not isinstance(content,str) or len(content.encode('utf-8'))>256000: raise ValueError('Content exceeds limit')
|
|
target.parent.mkdir(parents=True,exist_ok=True);target=path(values['path'])
|
|
target.write_text(content,encoding='utf-8');result={'written':len(content.encode('utf-8'))}
|
|
elif name=='delete_file':
|
|
target=path(values['path'])
|
|
if not target.is_file(): raise ValueError('Only individual files may be deleted')
|
|
target.unlink();result={'deleted':True}
|
|
else: raise ValueError('Unknown tool')
|
|
except Exception as exc:
|
|
result = {'error': str(exc)[:500]}
|
|
if track_changes:
|
|
result['fileChanges'] = {'complete': False, 'files': []}
|
|
try:
|
|
if before_complete:
|
|
snapshot_changes(root, change_dir / 'after')
|
|
result['fileChanges'] = summarize_changes(change_dir)
|
|
except (OSError, ValueError, subprocess.SubprocessError):
|
|
pass
|
|
finally:
|
|
if temporary is not None:
|
|
temporary.cleanup()
|
|
print(json.dumps(result, ensure_ascii=True if track_changes else False))
|
|
'''
|
|
SCRIPT = '\n'.join(Path(__file__).with_name(name).read_text(encoding='utf-8')
|
|
for name in ('review_tools.py', 'workspace_changes.py')) + '\n' + SCRIPT
|
|
|
|
|
|
def read_exec_output(connection, limit):
|
|
try:
|
|
return drain_exec_output(connection, limit)
|
|
finally:
|
|
connection.close()
|
|
|
|
|
|
def drain_exec_output(connection, limit):
|
|
# _preload_content=True lets ApiClient parse JSON and coerce dictionaries back to
|
|
# Python repr strings. Read raw stdout explicitly and keep stderr separate.
|
|
deadline = time.monotonic() + 135
|
|
chunks, size, error_size = [], 0, 0
|
|
while connection.is_open():
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
raise HTTPException(504, 'Sandbox execution timed out; result must be checked')
|
|
connection.update(timeout=min(1, remaining))
|
|
if connection.peek_stdout():
|
|
chunk = connection.read_stdout()
|
|
size += len(chunk.encode('utf-8'))
|
|
if size > limit:
|
|
raise HTTPException(502, 'Tool output exceeded limit')
|
|
chunks.append(chunk)
|
|
if connection.peek_stderr():
|
|
error_size += len(connection.read_stderr().encode('utf-8'))
|
|
if error_size > 64000:
|
|
raise HTTPException(502, 'Sandbox diagnostic output exceeded limit')
|
|
try:
|
|
code = connection.returncode
|
|
except Exception:
|
|
raise HTTPException(502, 'WORKSPACE_RESULT_UNKNOWN') from None
|
|
if code != 0:
|
|
raise HTTPException(502, 'Sandbox wrapper did not complete successfully')
|
|
return ''.join(chunks)
|
|
|
|
|
|
@app.post("/workspaces/{project}/tools")
|
|
def tool(project: str, request: Tool, authorization: str | None = Header(default=None)):
|
|
return execute_tool(project,request,authorization,False)
|
|
|
|
|
|
@app.post("/reviews/{project}/tools")
|
|
def review_tool(project: str, request: Tool, authorization: str | None = Header(default=None)):
|
|
return execute_tool(project,request,authorization,True)
|
|
|
|
|
|
@app.delete('/reviews/{project}')
|
|
def dispose_review(project: str, authorization: str | None = Header(default=None)):
|
|
authorize(authorization)
|
|
if REVIEW_NAMESPACE == NAMESPACE:raise HTTPException(503,'Review namespace must be separate')
|
|
name=workspace_name(project)
|
|
for operation in (app.state.core.delete_namespaced_pod,app.state.core.delete_namespaced_persistent_volume_claim,app.state.net.delete_namespaced_network_policy):
|
|
try:operation(name,REVIEW_NAMESPACE,_request_timeout=(3,10))
|
|
except ApiException as error:
|
|
if error.status!=404:raise HTTPException(503,'Review cleanup failed') from None
|
|
return {'cleanupRequested':True}
|
|
|
|
|
|
def execute_tool(project,request,authorization,review):
|
|
authorize(authorization)
|
|
name = workspace_name(project)
|
|
allowed = {'review_import','review_validate','workspace_inspect'} if review else {"list_files", "read_file", "write_file", "delete_file", "run_command", "version_save", "version_list", "archive_source", "workspace_inspect",'review_export'}
|
|
if request.name not in allowed:
|
|
raise HTTPException(400, "Unknown tool")
|
|
payload = json.dumps(request.model_dump(), ensure_ascii=False).encode()
|
|
if len(payload) > (16_000_000 if request.name=='review_import' else 300_000):
|
|
raise HTTPException(413, "Tool payload too large")
|
|
with LOCK_GUARD:
|
|
lock = LOCKS.setdefault(('review:' if review else '')+name, threading.Lock())
|
|
if not lock.acquire(timeout=1):
|
|
raise HTTPException(409, "WORKSPACE_BUSY")
|
|
try:
|
|
ensure_workspace(project,review)
|
|
encoded = base64.b64encode(payload).decode()
|
|
code = "import sys\nPAYLOAD=sys.stdin.read(" + str(len(encoded)) + ")\n" + SCRIPT
|
|
# Streaming temporarily overrides ApiClient.request. Never share that client
|
|
# with concurrent health checks, workspace provisioning, or another project.
|
|
with client.ApiClient() as api_client:
|
|
core = client.CoreV1Api(api_client)
|
|
connection = kube_call("WORKSPACE_CONNECTION", lambda: stream(
|
|
core.connect_get_namespaced_pod_exec, name, REVIEW_NAMESPACE if review else NAMESPACE,
|
|
container="workspace", command=["timeout", "130", "python3", "-I", "-c", code],
|
|
stderr=True, stdin=True, stdout=True, tty=False, _request_timeout=135,
|
|
_preload_content=False))
|
|
try:
|
|
for offset in range(0,len(encoded),16384):
|
|
connection.write_stdin(encoded[offset:offset+16384])
|
|
except Exception:
|
|
connection.close()
|
|
raise
|
|
output_limit = 16_000_000 if request.name in {'archive_source','review_export'} else MAX_OUTPUT
|
|
if request.trackChanges:
|
|
output_limit = 512_000
|
|
output = read_exec_output(connection, output_limit)
|
|
try:
|
|
return json.loads(output)
|
|
except ValueError:
|
|
raise HTTPException(502, "Sandbox returned invalid output") from None
|
|
except (ApiException, HTTPError, OSError, WebSocketException) as exc:
|
|
LOG.warning("Sandbox tool result unavailable (workspace=%s, tool=%s, cause=%s)",
|
|
name, request.name, type(exc).__name__)
|
|
raise HTTPException(502, "WORKSPACE_RESULT_UNKNOWN") from None
|
|
finally:
|
|
lock.release()
|