Files
webup-cloud/node-agent/runtime.py
T
WebUp Cloud d1d14f0f77
Verify control plane / verify (push) Canceled after 0s
Verify control plane / verify (pull_request) Canceled after 0s
Publish AI workspace source through the isolated runtime
2026-11-08 17:29:54 +08:00

354 lines
20 KiB
Python

"""Restricted OCI runtime controller. This module never builds or runs code on the gateway host."""
import hmac
import base64
import json
import os
import threading
import time
import uuid
import html
from datetime import datetime, timezone
from fastapi import APIRouter, Header, HTTPException
from kubernetes import client
from kubernetes.client.rest import ApiException
from pydantic import BaseModel, Field
from source_runtime import Source, attach_source, source_chunks, source_image
router = APIRouter()
NAMESPACE = os.environ.get("RUNTIME_NAMESPACE", "platform-runtimes")
TOKEN = os.environ.get("RUNTIME_TOKEN", "")
RUNTIME = os.environ.get("RUNTIME_CLASS", "gvisor")
locks = {}
guard = threading.Lock()
def auth(authorization):
if len(TOKEN) < 32 or not hmac.compare_digest(authorization or "", "Bearer " + TOKEN):
raise HTTPException(401, "Unauthorized runtime request")
def identifier(value):
try:
return uuid.UUID(str(value)).hex
except ValueError:
raise HTTPException(400, "Invalid resource ID")
class Service(BaseModel):
id: uuid.UUID
name: str = Field(pattern=r"^[a-z][a-z0-9-]{0,62}$")
image: str = Field(pattern=r"^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$")
command: list[str] = Field(max_length=30)
environment: dict[str, str] = Field(max_length=100)
port: int = Field(ge=1, le=65535)
cpu_millis: int = Field(ge=50, le=64000)
memory_mb: int = Field(ge=64, le=262144)
replicas: int = Field(ge=1, le=20)
health_path: str = Field(pattern=r"^/[^\s]{0,499}$|^$")
source_managed: bool = False
class Deployment(BaseModel):
projectId: uuid.UUID
deploymentId: uuid.UUID
services: list[Service] = Field(min_length=1, max_length=100)
branding: dict[str, str] = Field(default_factory=dict)
source: Source | None = None
class ProjectState(BaseModel):
action: str = Field(pattern=r'^(SUSPEND|RESUME)$')
operationId: uuid.UUID
def blocked(core,project):
try:
return core.read_namespaced_config_map('control-'+project,NAMESPACE).data.get('state')=='SUSPEND'
except ApiException as e:
if e.status==404:
return False
raise
def branding_config(port, branding):
if branding.get('hostedFooterEnabled') != 'true':
return None
text = html.escape(branding.get('hostedFooterText', '由 WebUp Cloud 提供服务'), quote=True)
url = branding.get('hostedFooterLink', 'https://webup.run')
if url and not url.startswith('https://'):
raise HTTPException(400, 'Footer link must use HTTPS')
label = ('<a href="' + html.escape(url, quote=True) + '" target="_blank" rel="noopener noreferrer" style="color:inherit!important;text-decoration:none!important">' + text + '</a>') if url else text
footer = '<div id="webup-powered-by" style="box-sizing:border-box!important;clear:both!important;text-align:center!important;padding:14px 10px!important;font:12px/1.6 system-ui,sans-serif!important;color:#8a96a8!important;background:transparent!important">' + label + '</div>'
footer = footer.replace('$','&#36;').replace('\\','&#92;').replace('\n',' ').replace('\r',' ')
return '''map $http_upgrade $connection_upgrade { default upgrade; '' close; }
server {
listen 8081;
server_tokens off;
client_max_body_size 32m;
location / {
proxy_pass http://127.0.0.1:PORT;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_set_header Accept-Encoding "";
proxy_read_timeout 3600s;
sub_filter_once on;
sub_filter '</body>' 'FOOTER</body>';
}
}
'''.replace('PORT',str(port)).replace('FOOTER',footer)
def ensure(method, **kwargs):
try:
return method(**kwargs)
except ApiException as exc:
if exc.status != 409:
raise
def network_policy(project):
return {"apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", "metadata": {"name": "project-" + project},
"spec": {"podSelector": {"matchLabels": {"platform/project": project}}, "policyTypes": ["Ingress", "Egress"],
"ingress": [{"from": [{"podSelector": {"matchLabels": {"platform/project": project}}},
{"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "ingress-nginx"}},
"podSelector": {"matchLabels": {"app.kubernetes.io/name": "ingress-nginx"}}}]}],
"egress": [{"to": [{"podSelector": {"matchLabels": {"platform/project": project}}}]},
{"to": [{"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "kube-system"}}}],
"ports": [{"port": 53, "protocol": "UDP"}, {"port": 53, "protocol": "TCP"}]},
{"to": [{"ipBlock": {"cidr": "0.0.0.0/0", "except": ["0.0.0.0/8", "10.0.0.0/8", "100.64.0.0/10", "127.0.0.0/8", "169.254.0.0/16", "172.16.0.0/12", "192.0.0.0/24", "192.168.0.0/16", "198.18.0.0/15", "224.0.0.0/4", "240.0.0.0/4"]}}],
"ports": [{"port": 443, "protocol": "TCP"}, {"port": 80, "protocol": "TCP"}]}]}}
def deployment_manifest(project, deployment, service, branding=None):
sid = identifier(service.id)
labels = {"platform/project": project, "platform/service": sid, "platform/revision": deployment}
name = "app-" + sid[:20] + "-" + deployment[:12]
security = {"allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"]}}
probe = {"httpGet": {"path": service.health_path, "port": service.port}} if service.health_path else {"tcpSocket": {"port": service.port}}
container = {"name": "app", "image": service.image, "ports": [{"containerPort": service.port}], "securityContext": security,
"envFrom": [{"secretRef": {"name": name}}], "volumeMounts": [{"name": "tmp", "mountPath": "/tmp"}],
"resources": {"requests": {"cpu": str(service.cpu_millis) + "m", "memory": str(service.memory_mb) + "Mi"},
"limits": {"cpu": str(service.cpu_millis) + "m", "memory": str(service.memory_mb) + "Mi", "ephemeral-storage": "1Gi"}},
"readinessProbe": {**probe, "initialDelaySeconds": 2, "periodSeconds": 3, "timeoutSeconds": 2},
"livenessProbe": {**probe, "initialDelaySeconds": 30, "periodSeconds": 10, "timeoutSeconds": 3},
"startupProbe": {**probe, "periodSeconds": 3, "failureThreshold": 25}}
if service.command:
container["command"] = service.command
manifest = {"apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"name": name, "labels": labels},
"spec": {"replicas": service.replicas, "selector": {"matchLabels": labels},
"template": {"metadata": {"labels": labels}, "spec": {"runtimeClassName": RUNTIME,
"automountServiceAccountToken": False, "enableServiceLinks": False,
"securityContext": {"runAsNonRoot": True, "runAsUser": 10001, "runAsGroup": 10001, "seccompProfile": {"type": "RuntimeDefault"}},
"containers": [container], "volumes": [{"name": "tmp", "emptyDir": {"sizeLimit": "512Mi"}}]}}}}
if branding_config(service.port, branding or {}) is not None:
pod = manifest['spec']['template']['spec']
pod['volumes'].extend([{'name':'branding-config','configMap':{'name':name+'-branding'}},{'name':'proxy-tmp','emptyDir':{'sizeLimit':'64Mi'}}])
pod['containers'].append({'name':'webup-branding','image':os.environ.get('BRANDING_PROXY_IMAGE','nginxinc/nginx-unprivileged:1.28-alpine'),
'ports':[{'containerPort':8081}],'securityContext':security,
'resources':{'requests':{'cpu':'10m','memory':'16Mi'},'limits':{'cpu':'100m','memory':'64Mi'}},
'volumeMounts':[{'name':'branding-config','mountPath':'/etc/nginx/conf.d','readOnly':True},{'name':'proxy-tmp','mountPath':'/tmp'}],
'readinessProbe':{'tcpSocket':{'port':8081},'periodSeconds':3},'livenessProbe':{'tcpSocket':{'port':8081},'periodSeconds':10}})
return manifest
@router.get("/runtime/source-ready")
def source_ready(authorization: str | None = Header(default=None)):
auth(authorization)
return {"ready": bool(source_image())}
@router.post("/runtime/deployments/{deployment}")
def deploy(deployment: str, request: Deployment, authorization: str | None = Header(default=None)):
auth(authorization)
did, project = identifier(deployment), identifier(request.projectId)
if did != identifier(request.deploymentId):
raise HTTPException(400, "Deployment ID mismatch")
managed = [service for service in request.services if service.source_managed]
if len(managed) != int(request.source is not None):
raise HTTPException(400, "Source snapshot must belong to exactly one managed service")
if request.source and not source_image():
raise HTTPException(503, "Source runtime image is not configured")
chunks = source_chunks(request.source) if request.source else []
with guard:
lock = locks.setdefault(project, threading.Lock())
if not lock.acquire(timeout=1):
raise HTTPException(409, "Project deployment in progress")
core, apps, networking = client.CoreV1Api(), client.AppsV1Api(), client.NetworkingV1Api()
receipt = "release-" + did
old = {}
try:
if blocked(core,project):
raise HTTPException(409,'Project runtime is suspended; deployment denied')
try:
record = core.read_namespaced_config_map(receipt, NAMESPACE)
if record.data.get("project") != project:
raise HTTPException(409, "Deployment receipt conflict")
if record.data.get("status") == "ACTIVE":
return {"ready": True, "deploymentId": deployment, "replayed": True}
old = json.loads(record.data["selectors"])
except ApiException as exc:
if exc.status != 404:
raise
for service in request.services:
stable = "svc-" + identifier(service.id)[:20]
try:
old[stable] = core.read_namespaced_service(stable, NAMESPACE).spec.selector
except ApiException as missing:
if missing.status != 404:
raise
old[stable] = None
ensure(core.create_namespaced_config_map, namespace=NAMESPACE, body={"metadata": {"name": receipt}, "data": {"project": project, "status": "PENDING", "selectors": json.dumps(old)}})
ensure(networking.create_namespaced_network_policy, namespace=NAMESPACE, body=network_policy(project))
names = []
for service in request.services:
manifest = deployment_manifest(project, did, service, request.branding)
name = manifest["metadata"]["name"]
if service.source_managed:
attach_source(manifest, request.source, len(chunks))
for index, chunk in enumerate(chunks):
ensure(core.create_namespaced_config_map, namespace=NAMESPACE, body={
"metadata": {"name": f"{name}-source-{index}", "labels": manifest["metadata"]["labels"]},
"immutable": True, "binaryData": {f"part-{index:03}": base64.b64encode(chunk).decode()},
})
if request.source.runtime == "NODE":
ensure(core.create_namespaced_persistent_volume_claim, namespace=NAMESPACE, body={
"metadata": {"name": "data-" + identifier(service.id), "labels": {"platform/project": project}},
"spec": {"accessModes": ["ReadWriteOnce"], "resources": {"requests": {"storage": "1Gi"}}},
})
ensure(core.create_namespaced_secret, namespace=NAMESPACE, body={"metadata": {"name": name}, "type": "Opaque", "stringData": service.environment})
proxy_config = branding_config(service.port,request.branding)
if proxy_config is not None:
ensure(core.create_namespaced_config_map,namespace=NAMESPACE,body={'metadata':{'name':name+'-branding'},'data':{'default.conf':proxy_config}})
ensure(apps.create_namespaced_deployment, namespace=NAMESPACE, body=manifest)
stable = "svc-" + identifier(service.id)[:20]
target_port = 8081 if proxy_config is not None else service.port
ensure(core.create_namespaced_service, namespace=NAMESPACE, body={"metadata": {"name": stable, "labels": {"platform/project": project}}, "spec": {"selector": manifest["metadata"]["labels"], "ports": [{"port": service.port, "targetPort": target_port}]}})
names.append((name, stable, manifest["metadata"]["labels"], service.port, service.replicas))
deadline = time.monotonic() + (200 if request.source else 90)
while time.monotonic() < deadline:
if all((apps.read_namespaced_deployment(n, NAMESPACE).status.ready_replicas or 0) >= replicas for n, _, _, _, replicas in names):
break
time.sleep(2)
else:
return {"ready": False, "error": "New revision did not become ready; previous traffic selectors retained"}
for _, stable, labels, port, _ in names:
if blocked(core,project):
raise HTTPException(409,'Project was suspended during deployment')
core.patch_namespaced_service(stable, NAMESPACE, {"spec": {"selector": labels, "ports": [{"port": port, "targetPort":8081 if request.branding.get('hostedFooterEnabled')=='true' else port}]}})
core.patch_namespaced_config_map(receipt, NAMESPACE, {"data": {"status": "ACTIVE"}})
cleanup_warnings = []
for previous in apps.list_namespaced_deployment(NAMESPACE, label_selector="platform/project=" + project).items:
if previous.metadata.labels.get("platform/revision") != did and previous.spec.replicas:
try:
apps.patch_namespaced_deployment(previous.metadata.name, NAMESPACE, {"spec": {"replicas": 0}})
except ApiException:
cleanup_warnings.append("Could not scale down retained revision " + previous.metadata.name)
return {"ready": True, "deploymentId": deployment, "cleanupWarnings": cleanup_warnings, "services": [{"internalAddress": stable + "." + NAMESPACE + ".svc.cluster.local:" + str(port)} for _, stable, _, port, _ in names]}
except ApiException:
# If switching a multi-service release fails halfway, restore the previously recorded selectors.
for stable, selector in old.items():
if selector:
try:
core.patch_namespaced_service(stable, NAMESPACE, {"spec": {"selector": selector}})
except ApiException:
pass # The control plane retains a failure event for explicit operator recovery.
raise HTTPException(503, "Runtime operation failed; check release receipt before retrying") from None
finally:
lock.release()
def active(project, service):
project, service = identifier(project), identifier(service)
core = client.CoreV1Api()
try:
selected = core.read_namespaced_service("svc-" + service[:20], NAMESPACE).spec.selector
if selected.get("platform/project") != project or selected.get("platform/service") != service:
raise HTTPException(404, "Service not found")
return core, selected
except ApiException:
raise HTTPException(404, "Service not found") from None
@router.get("/runtime/projects/{project}/services/{service}")
def status(project: str, service: str, authorization: str | None = Header(default=None)):
auth(authorization)
core, selected = active(project, service)
selector = ",".join(k + "=" + v for k, v in selected.items())
pods = core.list_namespaced_pod(NAMESPACE, label_selector=selector).items
return {"pods": [{"name": p.metadata.name, "phase": p.status.phase, "node": p.spec.node_name,
"ready": all(c.ready for c in (p.status.container_statuses or [])),
"restarts": sum(c.restart_count for c in (p.status.container_statuses or []))} for p in pods],
"internalAddress": "svc-" + identifier(service)[:20] + "." + NAMESPACE + ".svc.cluster.local"}
@router.get("/runtime/projects/{project}/services/{service}/logs")
def logs(project: str, service: str, authorization: str | None = Header(default=None)):
auth(authorization)
core, selected = active(project, service)
pods = core.list_namespaced_pod(NAMESPACE, label_selector=",".join(k + "=" + v for k, v in selected.items())).items
return {"logs": [{"pod": p.metadata.name, "output": core.read_namespaced_pod_log(p.metadata.name, NAMESPACE, container="app", tail_lines=200, limit_bytes=64000)} for p in pods[:20]]}
@router.post("/runtime/projects/{project}/services/{service}/restart")
def restart(project: str, service: str, authorization: str | None = Header(default=None)):
auth(authorization)
_, selected = active(project, service)
if blocked(client.CoreV1Api(),identifier(project)):
raise HTTPException(409,'Project runtime is suspended')
apps = client.AppsV1Api()
deployments = apps.list_namespaced_deployment(NAMESPACE, label_selector=",".join(k + "=" + v for k, v in selected.items())).items
for d in deployments:
apps.patch_namespaced_deployment(d.metadata.name, NAMESPACE, {"spec": {"template": {"metadata": {"annotations": {"platform/restarted-at": datetime.now(timezone.utc).isoformat()}}}}})
return {"message": "Restart requested", "deployments": len(deployments)}
@router.post('/runtime/projects/{project}/state')
def project_state(project: str,request: ProjectState,authorization: str | None = Header(default=None)):
auth(authorization)
project=identifier(project)
core,apps=client.CoreV1Api(),client.AppsV1Api()
name='control-'+project
ensure(core.create_namespaced_config_map,namespace=NAMESPACE,body={'metadata':{'name':name},'data':{'state':request.action}})
core.patch_namespaced_config_map(name,NAMESPACE,{'data':{'state':request.action,'operation':str(request.operationId)}})
changed=0
services=core.list_namespaced_service(NAMESPACE,label_selector='platform/project='+project).items
for service in services:
selector=service.spec.selector or {}
if not selector:
continue
for deployment in apps.list_namespaced_deployment(NAMESPACE,label_selector=','.join(k+'='+v for k,v in selector.items())).items:
annotations=deployment.metadata.annotations or {}
if request.action=='SUSPEND':
desired=deployment.spec.replicas or int(annotations.get('platform/desired-replicas','1'))
apps.patch_namespaced_deployment(deployment.metadata.name,NAMESPACE,{'metadata':{'annotations':{'platform/desired-replicas':str(desired)}},'spec':{'replicas':0}})
else:
apps.patch_namespaced_deployment(deployment.metadata.name,NAMESPACE,{'spec':{'replicas':int(annotations.get('platform/desired-replicas','1'))}})
changed+=1
return {'state':request.action,'changedDeployments':changed,'operationId':str(request.operationId)}
@router.get('/runtime/nodes')
def nodes(authorization: str | None = Header(default=None)):
auth(authorization)
core=client.CoreV1Api()
metrics={}
try:
for m in client.CustomObjectsApi().list_cluster_custom_object('metrics.k8s.io','v1beta1','nodes').get('items',[]):
metrics[m['metadata']['name']]=m.get('usage',{})
except ApiException:
pass
pods=core.list_namespaced_pod(NAMESPACE).items
result=[]
for n in core.list_node().items:
conditions=n.status.conditions or []
ready=any(c.type=='Ready' and c.status=='True' for c in conditions)
info=n.status.node_info
result.append({'name':n.metadata.name,'status':'READY' if ready else 'NOT_READY','cpu':n.status.allocatable.get('cpu',''),
'memory':n.status.allocatable.get('memory',''),'internalIp':next((a.address for a in n.status.addresses or [] if a.type=='InternalIP'),''),
'kubeletVersion':info.kubelet_version,'os':info.os_image,'runtime':info.container_runtime_version,
'unschedulable':bool(n.spec.unschedulable),'managedPods':sum(1 for p in pods if p.spec.node_name==n.metadata.name),'usage':metrics.get(n.metadata.name)})
return {'items':result,'metricsAvailable':bool(metrics)}