"""Restricted OCI runtime controller. This module never builds or runs code on the gateway host.""" import hmac import base64 import json import os import threading import time import uuid import html from datetime import datetime, timezone from fastapi import APIRouter, Header, HTTPException from kubernetes import client from kubernetes.client.rest import ApiException from pydantic import BaseModel, Field from source_runtime import Source, attach_source, source_chunks, source_image router = APIRouter() NAMESPACE = os.environ.get("RUNTIME_NAMESPACE", "platform-runtimes") TOKEN = os.environ.get("RUNTIME_TOKEN", "") RUNTIME = os.environ.get("RUNTIME_CLASS", "gvisor") locks = {} guard = threading.Lock() def auth(authorization): if len(TOKEN) < 32 or not hmac.compare_digest(authorization or "", "Bearer " + TOKEN): raise HTTPException(401, "Unauthorized runtime request") def identifier(value): try: return uuid.UUID(str(value)).hex except ValueError: raise HTTPException(400, "Invalid resource ID") class Service(BaseModel): id: uuid.UUID name: str = Field(pattern=r"^[a-z][a-z0-9-]{0,62}$") image: str = Field(pattern=r"^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$") command: list[str] = Field(max_length=30) environment: dict[str, str] = Field(max_length=100) port: int = Field(ge=1, le=65535) cpu_millis: int = Field(ge=50, le=64000) memory_mb: int = Field(ge=64, le=262144) replicas: int = Field(ge=1, le=20) health_path: str = Field(pattern=r"^/[^\s]{0,499}$|^$") source_managed: bool = False class Deployment(BaseModel): projectId: uuid.UUID deploymentId: uuid.UUID services: list[Service] = Field(min_length=1, max_length=100) branding: dict[str, str] = Field(default_factory=dict) source: Source | None = None class ProjectState(BaseModel): action: str = Field(pattern=r'^(SUSPEND|RESUME)$') operationId: uuid.UUID def blocked(core,project): try: return core.read_namespaced_config_map('control-'+project,NAMESPACE).data.get('state')=='SUSPEND' except ApiException as e: if e.status==404: return False raise def branding_config(port, branding): if branding.get('hostedFooterEnabled') != 'true': return None text = html.escape(branding.get('hostedFooterText', '由 WebUp Cloud 提供服务'), quote=True) url = branding.get('hostedFooterLink', 'https://webup.run') if url and not url.startswith('https://'): raise HTTPException(400, 'Footer link must use HTTPS') label = ('' + text + '') if url else text footer = '
' + label + '
' footer = footer.replace('$','$').replace('\\','\').replace('\n',' ').replace('\r',' ') return '''map $http_upgrade $connection_upgrade { default upgrade; '' close; } server { listen 8081; server_tokens off; client_max_body_size 32m; location / { proxy_pass http://127.0.0.1:PORT; proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; proxy_set_header Accept-Encoding ""; proxy_read_timeout 3600s; sub_filter_once on; sub_filter '' 'FOOTER'; } } '''.replace('PORT',str(port)).replace('FOOTER',footer) def ensure(method, **kwargs): try: return method(**kwargs) except ApiException as exc: if exc.status != 409: raise def network_policy(project): return {"apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", "metadata": {"name": "project-" + project}, "spec": {"podSelector": {"matchLabels": {"platform/project": project}}, "policyTypes": ["Ingress", "Egress"], "ingress": [{"from": [{"podSelector": {"matchLabels": {"platform/project": project}}}, {"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "ingress-nginx"}}, "podSelector": {"matchLabels": {"app.kubernetes.io/name": "ingress-nginx"}}}]}], "egress": [{"to": [{"podSelector": {"matchLabels": {"platform/project": project}}}]}, {"to": [{"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "kube-system"}}}], "ports": [{"port": 53, "protocol": "UDP"}, {"port": 53, "protocol": "TCP"}]}, {"to": [{"ipBlock": {"cidr": "0.0.0.0/0", "except": ["0.0.0.0/8", "10.0.0.0/8", "100.64.0.0/10", "127.0.0.0/8", "169.254.0.0/16", "172.16.0.0/12", "192.0.0.0/24", "192.168.0.0/16", "198.18.0.0/15", "224.0.0.0/4", "240.0.0.0/4"]}}], "ports": [{"port": 443, "protocol": "TCP"}, {"port": 80, "protocol": "TCP"}]}]}} def deployment_manifest(project, deployment, service, branding=None): sid = identifier(service.id) labels = {"platform/project": project, "platform/service": sid, "platform/revision": deployment} name = "app-" + sid[:20] + "-" + deployment[:12] security = {"allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"]}} probe = {"httpGet": {"path": service.health_path, "port": service.port}} if service.health_path else {"tcpSocket": {"port": service.port}} container = {"name": "app", "image": service.image, "ports": [{"containerPort": service.port}], "securityContext": security, "envFrom": [{"secretRef": {"name": name}}], "volumeMounts": [{"name": "tmp", "mountPath": "/tmp"}], "resources": {"requests": {"cpu": str(service.cpu_millis) + "m", "memory": str(service.memory_mb) + "Mi"}, "limits": {"cpu": str(service.cpu_millis) + "m", "memory": str(service.memory_mb) + "Mi", "ephemeral-storage": "1Gi"}}, "readinessProbe": {**probe, "initialDelaySeconds": 2, "periodSeconds": 3, "timeoutSeconds": 2}, "livenessProbe": {**probe, "initialDelaySeconds": 30, "periodSeconds": 10, "timeoutSeconds": 3}, "startupProbe": {**probe, "periodSeconds": 3, "failureThreshold": 25}} if service.command: container["command"] = service.command manifest = {"apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"name": name, "labels": labels}, "spec": {"replicas": service.replicas, "selector": {"matchLabels": labels}, "template": {"metadata": {"labels": labels}, "spec": {"runtimeClassName": RUNTIME, "automountServiceAccountToken": False, "enableServiceLinks": False, "securityContext": {"runAsNonRoot": True, "runAsUser": 10001, "runAsGroup": 10001, "seccompProfile": {"type": "RuntimeDefault"}}, "containers": [container], "volumes": [{"name": "tmp", "emptyDir": {"sizeLimit": "512Mi"}}]}}}} if branding_config(service.port, branding or {}) is not None: pod = manifest['spec']['template']['spec'] pod['volumes'].extend([{'name':'branding-config','configMap':{'name':name+'-branding'}},{'name':'proxy-tmp','emptyDir':{'sizeLimit':'64Mi'}}]) pod['containers'].append({'name':'webup-branding','image':os.environ.get('BRANDING_PROXY_IMAGE','nginxinc/nginx-unprivileged:1.28-alpine'), 'ports':[{'containerPort':8081}],'securityContext':security, 'resources':{'requests':{'cpu':'10m','memory':'16Mi'},'limits':{'cpu':'100m','memory':'64Mi'}}, 'volumeMounts':[{'name':'branding-config','mountPath':'/etc/nginx/conf.d','readOnly':True},{'name':'proxy-tmp','mountPath':'/tmp'}], 'readinessProbe':{'tcpSocket':{'port':8081},'periodSeconds':3},'livenessProbe':{'tcpSocket':{'port':8081},'periodSeconds':10}}) return manifest @router.get("/runtime/source-ready") def source_ready(authorization: str | None = Header(default=None)): auth(authorization) return {"ready": bool(source_image())} @router.post("/runtime/deployments/{deployment}") def deploy(deployment: str, request: Deployment, authorization: str | None = Header(default=None)): auth(authorization) did, project = identifier(deployment), identifier(request.projectId) if did != identifier(request.deploymentId): raise HTTPException(400, "Deployment ID mismatch") managed = [service for service in request.services if service.source_managed] if len(managed) != int(request.source is not None): raise HTTPException(400, "Source snapshot must belong to exactly one managed service") if request.source and not source_image(): raise HTTPException(503, "Source runtime image is not configured") chunks = source_chunks(request.source) if request.source else [] with guard: lock = locks.setdefault(project, threading.Lock()) if not lock.acquire(timeout=1): raise HTTPException(409, "Project deployment in progress") core, apps, networking = client.CoreV1Api(), client.AppsV1Api(), client.NetworkingV1Api() receipt = "release-" + did old = {} try: if blocked(core,project): raise HTTPException(409,'Project runtime is suspended; deployment denied') try: record = core.read_namespaced_config_map(receipt, NAMESPACE) if record.data.get("project") != project: raise HTTPException(409, "Deployment receipt conflict") if record.data.get("status") == "ACTIVE": return {"ready": True, "deploymentId": deployment, "replayed": True} old = json.loads(record.data["selectors"]) except ApiException as exc: if exc.status != 404: raise for service in request.services: stable = "svc-" + identifier(service.id)[:20] try: old[stable] = core.read_namespaced_service(stable, NAMESPACE).spec.selector except ApiException as missing: if missing.status != 404: raise old[stable] = None ensure(core.create_namespaced_config_map, namespace=NAMESPACE, body={"metadata": {"name": receipt}, "data": {"project": project, "status": "PENDING", "selectors": json.dumps(old)}}) ensure(networking.create_namespaced_network_policy, namespace=NAMESPACE, body=network_policy(project)) names = [] for service in request.services: manifest = deployment_manifest(project, did, service, request.branding) name = manifest["metadata"]["name"] if service.source_managed: attach_source(manifest, request.source, len(chunks)) for index, chunk in enumerate(chunks): ensure(core.create_namespaced_config_map, namespace=NAMESPACE, body={ "metadata": {"name": f"{name}-source-{index}", "labels": manifest["metadata"]["labels"]}, "immutable": True, "binaryData": {f"part-{index:03}": base64.b64encode(chunk).decode()}, }) if request.source.runtime == "NODE": ensure(core.create_namespaced_persistent_volume_claim, namespace=NAMESPACE, body={ "metadata": {"name": "data-" + identifier(service.id), "labels": {"platform/project": project}}, "spec": {"accessModes": ["ReadWriteOnce"], "resources": {"requests": {"storage": "1Gi"}}}, }) ensure(core.create_namespaced_secret, namespace=NAMESPACE, body={"metadata": {"name": name}, "type": "Opaque", "stringData": service.environment}) proxy_config = branding_config(service.port,request.branding) if proxy_config is not None: ensure(core.create_namespaced_config_map,namespace=NAMESPACE,body={'metadata':{'name':name+'-branding'},'data':{'default.conf':proxy_config}}) ensure(apps.create_namespaced_deployment, namespace=NAMESPACE, body=manifest) stable = "svc-" + identifier(service.id)[:20] target_port = 8081 if proxy_config is not None else service.port ensure(core.create_namespaced_service, namespace=NAMESPACE, body={"metadata": {"name": stable, "labels": {"platform/project": project}}, "spec": {"selector": manifest["metadata"]["labels"], "ports": [{"port": service.port, "targetPort": target_port}]}}) names.append((name, stable, manifest["metadata"]["labels"], service.port, service.replicas)) deadline = time.monotonic() + (200 if request.source else 90) while time.monotonic() < deadline: if all((apps.read_namespaced_deployment(n, NAMESPACE).status.ready_replicas or 0) >= replicas for n, _, _, _, replicas in names): break time.sleep(2) else: return {"ready": False, "error": "New revision did not become ready; previous traffic selectors retained"} for _, stable, labels, port, _ in names: if blocked(core,project): raise HTTPException(409,'Project was suspended during deployment') core.patch_namespaced_service(stable, NAMESPACE, {"spec": {"selector": labels, "ports": [{"port": port, "targetPort":8081 if request.branding.get('hostedFooterEnabled')=='true' else port}]}}) core.patch_namespaced_config_map(receipt, NAMESPACE, {"data": {"status": "ACTIVE"}}) cleanup_warnings = [] for previous in apps.list_namespaced_deployment(NAMESPACE, label_selector="platform/project=" + project).items: if previous.metadata.labels.get("platform/revision") != did and previous.spec.replicas: try: apps.patch_namespaced_deployment(previous.metadata.name, NAMESPACE, {"spec": {"replicas": 0}}) except ApiException: cleanup_warnings.append("Could not scale down retained revision " + previous.metadata.name) return {"ready": True, "deploymentId": deployment, "cleanupWarnings": cleanup_warnings, "services": [{"internalAddress": stable + "." + NAMESPACE + ".svc.cluster.local:" + str(port)} for _, stable, _, port, _ in names]} except ApiException: # If switching a multi-service release fails halfway, restore the previously recorded selectors. for stable, selector in old.items(): if selector: try: core.patch_namespaced_service(stable, NAMESPACE, {"spec": {"selector": selector}}) except ApiException: pass # The control plane retains a failure event for explicit operator recovery. raise HTTPException(503, "Runtime operation failed; check release receipt before retrying") from None finally: lock.release() def active(project, service): project, service = identifier(project), identifier(service) core = client.CoreV1Api() try: selected = core.read_namespaced_service("svc-" + service[:20], NAMESPACE).spec.selector if selected.get("platform/project") != project or selected.get("platform/service") != service: raise HTTPException(404, "Service not found") return core, selected except ApiException: raise HTTPException(404, "Service not found") from None @router.get("/runtime/projects/{project}/services/{service}") def status(project: str, service: str, authorization: str | None = Header(default=None)): auth(authorization) core, selected = active(project, service) selector = ",".join(k + "=" + v for k, v in selected.items()) pods = core.list_namespaced_pod(NAMESPACE, label_selector=selector).items return {"pods": [{"name": p.metadata.name, "phase": p.status.phase, "node": p.spec.node_name, "ready": all(c.ready for c in (p.status.container_statuses or [])), "restarts": sum(c.restart_count for c in (p.status.container_statuses or []))} for p in pods], "internalAddress": "svc-" + identifier(service)[:20] + "." + NAMESPACE + ".svc.cluster.local"} @router.get("/runtime/projects/{project}/services/{service}/logs") def logs(project: str, service: str, authorization: str | None = Header(default=None)): auth(authorization) core, selected = active(project, service) pods = core.list_namespaced_pod(NAMESPACE, label_selector=",".join(k + "=" + v for k, v in selected.items())).items return {"logs": [{"pod": p.metadata.name, "output": core.read_namespaced_pod_log(p.metadata.name, NAMESPACE, container="app", tail_lines=200, limit_bytes=64000)} for p in pods[:20]]} @router.post("/runtime/projects/{project}/services/{service}/restart") def restart(project: str, service: str, authorization: str | None = Header(default=None)): auth(authorization) _, selected = active(project, service) if blocked(client.CoreV1Api(),identifier(project)): raise HTTPException(409,'Project runtime is suspended') apps = client.AppsV1Api() deployments = apps.list_namespaced_deployment(NAMESPACE, label_selector=",".join(k + "=" + v for k, v in selected.items())).items for d in deployments: apps.patch_namespaced_deployment(d.metadata.name, NAMESPACE, {"spec": {"template": {"metadata": {"annotations": {"platform/restarted-at": datetime.now(timezone.utc).isoformat()}}}}}) return {"message": "Restart requested", "deployments": len(deployments)} @router.post('/runtime/projects/{project}/state') def project_state(project: str,request: ProjectState,authorization: str | None = Header(default=None)): auth(authorization) project=identifier(project) core,apps=client.CoreV1Api(),client.AppsV1Api() name='control-'+project ensure(core.create_namespaced_config_map,namespace=NAMESPACE,body={'metadata':{'name':name},'data':{'state':request.action}}) core.patch_namespaced_config_map(name,NAMESPACE,{'data':{'state':request.action,'operation':str(request.operationId)}}) changed=0 services=core.list_namespaced_service(NAMESPACE,label_selector='platform/project='+project).items for service in services: selector=service.spec.selector or {} if not selector: continue for deployment in apps.list_namespaced_deployment(NAMESPACE,label_selector=','.join(k+'='+v for k,v in selector.items())).items: annotations=deployment.metadata.annotations or {} if request.action=='SUSPEND': desired=deployment.spec.replicas or int(annotations.get('platform/desired-replicas','1')) apps.patch_namespaced_deployment(deployment.metadata.name,NAMESPACE,{'metadata':{'annotations':{'platform/desired-replicas':str(desired)}},'spec':{'replicas':0}}) else: apps.patch_namespaced_deployment(deployment.metadata.name,NAMESPACE,{'spec':{'replicas':int(annotations.get('platform/desired-replicas','1'))}}) changed+=1 return {'state':request.action,'changedDeployments':changed,'operationId':str(request.operationId)} @router.get('/runtime/nodes') def nodes(authorization: str | None = Header(default=None)): auth(authorization) core=client.CoreV1Api() metrics={} try: for m in client.CustomObjectsApi().list_cluster_custom_object('metrics.k8s.io','v1beta1','nodes').get('items',[]): metrics[m['metadata']['name']]=m.get('usage',{}) except ApiException: pass pods=core.list_namespaced_pod(NAMESPACE).items result=[] for n in core.list_node().items: conditions=n.status.conditions or [] ready=any(c.type=='Ready' and c.status=='True' for c in conditions) info=n.status.node_info result.append({'name':n.metadata.name,'status':'READY' if ready else 'NOT_READY','cpu':n.status.allocatable.get('cpu',''), 'memory':n.status.allocatable.get('memory',''),'internalIp':next((a.address for a in n.status.addresses or [] if a.type=='InternalIP'),''), 'kubeletVersion':info.kubelet_version,'os':info.os_image,'runtime':info.container_runtime_version, 'unschedulable':bool(n.spec.unschedulable),'managedPods':sum(1 for p in pods if p.spec.node_name==n.metadata.name),'usage':metrics.get(n.metadata.name)}) return {'items':result,'metricsAvailable':bool(metrics)}