forked from xhgit/webup-cloud
354 lines
20 KiB
Python
354 lines
20 KiB
Python
"""Restricted OCI runtime controller. This module never builds or runs code on the gateway host."""
|
|
import hmac
|
|
import base64
|
|
import json
|
|
import os
|
|
import threading
|
|
import time
|
|
import uuid
|
|
import html
|
|
from datetime import datetime, timezone
|
|
|
|
from fastapi import APIRouter, Header, HTTPException
|
|
from kubernetes import client
|
|
from kubernetes.client.rest import ApiException
|
|
from pydantic import BaseModel, Field
|
|
from source_runtime import Source, attach_source, source_chunks, source_image
|
|
|
|
router = APIRouter()
|
|
NAMESPACE = os.environ.get("RUNTIME_NAMESPACE", "platform-runtimes")
|
|
TOKEN = os.environ.get("RUNTIME_TOKEN", "")
|
|
RUNTIME = os.environ.get("RUNTIME_CLASS", "gvisor")
|
|
locks = {}
|
|
guard = threading.Lock()
|
|
|
|
|
|
def auth(authorization):
|
|
if len(TOKEN) < 32 or not hmac.compare_digest(authorization or "", "Bearer " + TOKEN):
|
|
raise HTTPException(401, "Unauthorized runtime request")
|
|
|
|
|
|
def identifier(value):
|
|
try:
|
|
return uuid.UUID(str(value)).hex
|
|
except ValueError:
|
|
raise HTTPException(400, "Invalid resource ID")
|
|
|
|
|
|
class Service(BaseModel):
|
|
id: uuid.UUID
|
|
name: str = Field(pattern=r"^[a-z][a-z0-9-]{0,62}$")
|
|
image: str = Field(pattern=r"^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$")
|
|
command: list[str] = Field(max_length=30)
|
|
environment: dict[str, str] = Field(max_length=100)
|
|
port: int = Field(ge=1, le=65535)
|
|
cpu_millis: int = Field(ge=50, le=64000)
|
|
memory_mb: int = Field(ge=64, le=262144)
|
|
replicas: int = Field(ge=1, le=20)
|
|
health_path: str = Field(pattern=r"^/[^\s]{0,499}$|^$")
|
|
source_managed: bool = False
|
|
|
|
|
|
class Deployment(BaseModel):
|
|
projectId: uuid.UUID
|
|
deploymentId: uuid.UUID
|
|
services: list[Service] = Field(min_length=1, max_length=100)
|
|
branding: dict[str, str] = Field(default_factory=dict)
|
|
source: Source | None = None
|
|
|
|
|
|
class ProjectState(BaseModel):
|
|
action: str = Field(pattern=r'^(SUSPEND|RESUME)$')
|
|
operationId: uuid.UUID
|
|
|
|
|
|
def blocked(core,project):
|
|
try:
|
|
return core.read_namespaced_config_map('control-'+project,NAMESPACE).data.get('state')=='SUSPEND'
|
|
except ApiException as e:
|
|
if e.status==404:
|
|
return False
|
|
raise
|
|
|
|
|
|
def branding_config(port, branding):
|
|
if branding.get('hostedFooterEnabled') != 'true':
|
|
return None
|
|
text = html.escape(branding.get('hostedFooterText', '由 WebUp Cloud 提供服务'), quote=True)
|
|
url = branding.get('hostedFooterLink', 'https://webup.run')
|
|
if url and not url.startswith('https://'):
|
|
raise HTTPException(400, 'Footer link must use HTTPS')
|
|
label = ('<a href="' + html.escape(url, quote=True) + '" target="_blank" rel="noopener noreferrer" style="color:inherit!important;text-decoration:none!important">' + text + '</a>') if url else text
|
|
footer = '<div id="webup-powered-by" style="box-sizing:border-box!important;clear:both!important;text-align:center!important;padding:14px 10px!important;font:12px/1.6 system-ui,sans-serif!important;color:#8a96a8!important;background:transparent!important">' + label + '</div>'
|
|
footer = footer.replace('$','$').replace('\\','\').replace('\n',' ').replace('\r',' ')
|
|
return '''map $http_upgrade $connection_upgrade { default upgrade; '' close; }
|
|
server {
|
|
listen 8081;
|
|
server_tokens off;
|
|
client_max_body_size 32m;
|
|
location / {
|
|
proxy_pass http://127.0.0.1:PORT;
|
|
proxy_http_version 1.1;
|
|
proxy_set_header Host $host;
|
|
proxy_set_header Upgrade $http_upgrade;
|
|
proxy_set_header Connection $connection_upgrade;
|
|
proxy_set_header Accept-Encoding "";
|
|
proxy_read_timeout 3600s;
|
|
sub_filter_once on;
|
|
sub_filter '</body>' 'FOOTER</body>';
|
|
}
|
|
}
|
|
'''.replace('PORT',str(port)).replace('FOOTER',footer)
|
|
|
|
|
|
def ensure(method, **kwargs):
|
|
try:
|
|
return method(**kwargs)
|
|
except ApiException as exc:
|
|
if exc.status != 409:
|
|
raise
|
|
|
|
|
|
def network_policy(project):
|
|
return {"apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", "metadata": {"name": "project-" + project},
|
|
"spec": {"podSelector": {"matchLabels": {"platform/project": project}}, "policyTypes": ["Ingress", "Egress"],
|
|
"ingress": [{"from": [{"podSelector": {"matchLabels": {"platform/project": project}}},
|
|
{"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "ingress-nginx"}},
|
|
"podSelector": {"matchLabels": {"app.kubernetes.io/name": "ingress-nginx"}}}]}],
|
|
"egress": [{"to": [{"podSelector": {"matchLabels": {"platform/project": project}}}]},
|
|
{"to": [{"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "kube-system"}}}],
|
|
"ports": [{"port": 53, "protocol": "UDP"}, {"port": 53, "protocol": "TCP"}]},
|
|
{"to": [{"ipBlock": {"cidr": "0.0.0.0/0", "except": ["0.0.0.0/8", "10.0.0.0/8", "100.64.0.0/10", "127.0.0.0/8", "169.254.0.0/16", "172.16.0.0/12", "192.0.0.0/24", "192.168.0.0/16", "198.18.0.0/15", "224.0.0.0/4", "240.0.0.0/4"]}}],
|
|
"ports": [{"port": 443, "protocol": "TCP"}, {"port": 80, "protocol": "TCP"}]}]}}
|
|
|
|
|
|
def deployment_manifest(project, deployment, service, branding=None):
|
|
sid = identifier(service.id)
|
|
labels = {"platform/project": project, "platform/service": sid, "platform/revision": deployment}
|
|
name = "app-" + sid[:20] + "-" + deployment[:12]
|
|
security = {"allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"]}}
|
|
probe = {"httpGet": {"path": service.health_path, "port": service.port}} if service.health_path else {"tcpSocket": {"port": service.port}}
|
|
container = {"name": "app", "image": service.image, "ports": [{"containerPort": service.port}], "securityContext": security,
|
|
"envFrom": [{"secretRef": {"name": name}}], "volumeMounts": [{"name": "tmp", "mountPath": "/tmp"}],
|
|
"resources": {"requests": {"cpu": str(service.cpu_millis) + "m", "memory": str(service.memory_mb) + "Mi"},
|
|
"limits": {"cpu": str(service.cpu_millis) + "m", "memory": str(service.memory_mb) + "Mi", "ephemeral-storage": "1Gi"}},
|
|
"readinessProbe": {**probe, "initialDelaySeconds": 2, "periodSeconds": 3, "timeoutSeconds": 2},
|
|
"livenessProbe": {**probe, "initialDelaySeconds": 30, "periodSeconds": 10, "timeoutSeconds": 3},
|
|
"startupProbe": {**probe, "periodSeconds": 3, "failureThreshold": 25}}
|
|
if service.command:
|
|
container["command"] = service.command
|
|
manifest = {"apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"name": name, "labels": labels},
|
|
"spec": {"replicas": service.replicas, "selector": {"matchLabels": labels},
|
|
"template": {"metadata": {"labels": labels}, "spec": {"runtimeClassName": RUNTIME,
|
|
"automountServiceAccountToken": False, "enableServiceLinks": False,
|
|
"securityContext": {"runAsNonRoot": True, "runAsUser": 10001, "runAsGroup": 10001, "seccompProfile": {"type": "RuntimeDefault"}},
|
|
"containers": [container], "volumes": [{"name": "tmp", "emptyDir": {"sizeLimit": "512Mi"}}]}}}}
|
|
if branding_config(service.port, branding or {}) is not None:
|
|
pod = manifest['spec']['template']['spec']
|
|
pod['volumes'].extend([{'name':'branding-config','configMap':{'name':name+'-branding'}},{'name':'proxy-tmp','emptyDir':{'sizeLimit':'64Mi'}}])
|
|
pod['containers'].append({'name':'webup-branding','image':os.environ.get('BRANDING_PROXY_IMAGE','nginxinc/nginx-unprivileged:1.28-alpine'),
|
|
'ports':[{'containerPort':8081}],'securityContext':security,
|
|
'resources':{'requests':{'cpu':'10m','memory':'16Mi'},'limits':{'cpu':'100m','memory':'64Mi'}},
|
|
'volumeMounts':[{'name':'branding-config','mountPath':'/etc/nginx/conf.d','readOnly':True},{'name':'proxy-tmp','mountPath':'/tmp'}],
|
|
'readinessProbe':{'tcpSocket':{'port':8081},'periodSeconds':3},'livenessProbe':{'tcpSocket':{'port':8081},'periodSeconds':10}})
|
|
return manifest
|
|
|
|
|
|
@router.get("/runtime/source-ready")
|
|
def source_ready(authorization: str | None = Header(default=None)):
|
|
auth(authorization)
|
|
return {"ready": bool(source_image())}
|
|
|
|
|
|
@router.post("/runtime/deployments/{deployment}")
|
|
def deploy(deployment: str, request: Deployment, authorization: str | None = Header(default=None)):
|
|
auth(authorization)
|
|
did, project = identifier(deployment), identifier(request.projectId)
|
|
if did != identifier(request.deploymentId):
|
|
raise HTTPException(400, "Deployment ID mismatch")
|
|
managed = [service for service in request.services if service.source_managed]
|
|
if len(managed) != int(request.source is not None):
|
|
raise HTTPException(400, "Source snapshot must belong to exactly one managed service")
|
|
if request.source and not source_image():
|
|
raise HTTPException(503, "Source runtime image is not configured")
|
|
chunks = source_chunks(request.source) if request.source else []
|
|
with guard:
|
|
lock = locks.setdefault(project, threading.Lock())
|
|
if not lock.acquire(timeout=1):
|
|
raise HTTPException(409, "Project deployment in progress")
|
|
core, apps, networking = client.CoreV1Api(), client.AppsV1Api(), client.NetworkingV1Api()
|
|
receipt = "release-" + did
|
|
old = {}
|
|
try:
|
|
if blocked(core,project):
|
|
raise HTTPException(409,'Project runtime is suspended; deployment denied')
|
|
try:
|
|
record = core.read_namespaced_config_map(receipt, NAMESPACE)
|
|
if record.data.get("project") != project:
|
|
raise HTTPException(409, "Deployment receipt conflict")
|
|
if record.data.get("status") == "ACTIVE":
|
|
return {"ready": True, "deploymentId": deployment, "replayed": True}
|
|
old = json.loads(record.data["selectors"])
|
|
except ApiException as exc:
|
|
if exc.status != 404:
|
|
raise
|
|
for service in request.services:
|
|
stable = "svc-" + identifier(service.id)[:20]
|
|
try:
|
|
old[stable] = core.read_namespaced_service(stable, NAMESPACE).spec.selector
|
|
except ApiException as missing:
|
|
if missing.status != 404:
|
|
raise
|
|
old[stable] = None
|
|
ensure(core.create_namespaced_config_map, namespace=NAMESPACE, body={"metadata": {"name": receipt}, "data": {"project": project, "status": "PENDING", "selectors": json.dumps(old)}})
|
|
ensure(networking.create_namespaced_network_policy, namespace=NAMESPACE, body=network_policy(project))
|
|
names = []
|
|
for service in request.services:
|
|
manifest = deployment_manifest(project, did, service, request.branding)
|
|
name = manifest["metadata"]["name"]
|
|
if service.source_managed:
|
|
attach_source(manifest, request.source, len(chunks))
|
|
for index, chunk in enumerate(chunks):
|
|
ensure(core.create_namespaced_config_map, namespace=NAMESPACE, body={
|
|
"metadata": {"name": f"{name}-source-{index}", "labels": manifest["metadata"]["labels"]},
|
|
"immutable": True, "binaryData": {f"part-{index:03}": base64.b64encode(chunk).decode()},
|
|
})
|
|
if request.source.runtime == "NODE":
|
|
ensure(core.create_namespaced_persistent_volume_claim, namespace=NAMESPACE, body={
|
|
"metadata": {"name": "data-" + identifier(service.id), "labels": {"platform/project": project}},
|
|
"spec": {"accessModes": ["ReadWriteOnce"], "resources": {"requests": {"storage": "1Gi"}}},
|
|
})
|
|
ensure(core.create_namespaced_secret, namespace=NAMESPACE, body={"metadata": {"name": name}, "type": "Opaque", "stringData": service.environment})
|
|
proxy_config = branding_config(service.port,request.branding)
|
|
if proxy_config is not None:
|
|
ensure(core.create_namespaced_config_map,namespace=NAMESPACE,body={'metadata':{'name':name+'-branding'},'data':{'default.conf':proxy_config}})
|
|
ensure(apps.create_namespaced_deployment, namespace=NAMESPACE, body=manifest)
|
|
stable = "svc-" + identifier(service.id)[:20]
|
|
target_port = 8081 if proxy_config is not None else service.port
|
|
ensure(core.create_namespaced_service, namespace=NAMESPACE, body={"metadata": {"name": stable, "labels": {"platform/project": project}}, "spec": {"selector": manifest["metadata"]["labels"], "ports": [{"port": service.port, "targetPort": target_port}]}})
|
|
names.append((name, stable, manifest["metadata"]["labels"], service.port, service.replicas))
|
|
deadline = time.monotonic() + (200 if request.source else 90)
|
|
while time.monotonic() < deadline:
|
|
if all((apps.read_namespaced_deployment(n, NAMESPACE).status.ready_replicas or 0) >= replicas for n, _, _, _, replicas in names):
|
|
break
|
|
time.sleep(2)
|
|
else:
|
|
return {"ready": False, "error": "New revision did not become ready; previous traffic selectors retained"}
|
|
for _, stable, labels, port, _ in names:
|
|
if blocked(core,project):
|
|
raise HTTPException(409,'Project was suspended during deployment')
|
|
core.patch_namespaced_service(stable, NAMESPACE, {"spec": {"selector": labels, "ports": [{"port": port, "targetPort":8081 if request.branding.get('hostedFooterEnabled')=='true' else port}]}})
|
|
core.patch_namespaced_config_map(receipt, NAMESPACE, {"data": {"status": "ACTIVE"}})
|
|
cleanup_warnings = []
|
|
for previous in apps.list_namespaced_deployment(NAMESPACE, label_selector="platform/project=" + project).items:
|
|
if previous.metadata.labels.get("platform/revision") != did and previous.spec.replicas:
|
|
try:
|
|
apps.patch_namespaced_deployment(previous.metadata.name, NAMESPACE, {"spec": {"replicas": 0}})
|
|
except ApiException:
|
|
cleanup_warnings.append("Could not scale down retained revision " + previous.metadata.name)
|
|
return {"ready": True, "deploymentId": deployment, "cleanupWarnings": cleanup_warnings, "services": [{"internalAddress": stable + "." + NAMESPACE + ".svc.cluster.local:" + str(port)} for _, stable, _, port, _ in names]}
|
|
except ApiException:
|
|
# If switching a multi-service release fails halfway, restore the previously recorded selectors.
|
|
for stable, selector in old.items():
|
|
if selector:
|
|
try:
|
|
core.patch_namespaced_service(stable, NAMESPACE, {"spec": {"selector": selector}})
|
|
except ApiException:
|
|
pass # The control plane retains a failure event for explicit operator recovery.
|
|
raise HTTPException(503, "Runtime operation failed; check release receipt before retrying") from None
|
|
finally:
|
|
lock.release()
|
|
|
|
|
|
def active(project, service):
|
|
project, service = identifier(project), identifier(service)
|
|
core = client.CoreV1Api()
|
|
try:
|
|
selected = core.read_namespaced_service("svc-" + service[:20], NAMESPACE).spec.selector
|
|
if selected.get("platform/project") != project or selected.get("platform/service") != service:
|
|
raise HTTPException(404, "Service not found")
|
|
return core, selected
|
|
except ApiException:
|
|
raise HTTPException(404, "Service not found") from None
|
|
|
|
|
|
@router.get("/runtime/projects/{project}/services/{service}")
|
|
def status(project: str, service: str, authorization: str | None = Header(default=None)):
|
|
auth(authorization)
|
|
core, selected = active(project, service)
|
|
selector = ",".join(k + "=" + v for k, v in selected.items())
|
|
pods = core.list_namespaced_pod(NAMESPACE, label_selector=selector).items
|
|
return {"pods": [{"name": p.metadata.name, "phase": p.status.phase, "node": p.spec.node_name,
|
|
"ready": all(c.ready for c in (p.status.container_statuses or [])),
|
|
"restarts": sum(c.restart_count for c in (p.status.container_statuses or []))} for p in pods],
|
|
"internalAddress": "svc-" + identifier(service)[:20] + "." + NAMESPACE + ".svc.cluster.local"}
|
|
|
|
|
|
@router.get("/runtime/projects/{project}/services/{service}/logs")
|
|
def logs(project: str, service: str, authorization: str | None = Header(default=None)):
|
|
auth(authorization)
|
|
core, selected = active(project, service)
|
|
pods = core.list_namespaced_pod(NAMESPACE, label_selector=",".join(k + "=" + v for k, v in selected.items())).items
|
|
return {"logs": [{"pod": p.metadata.name, "output": core.read_namespaced_pod_log(p.metadata.name, NAMESPACE, container="app", tail_lines=200, limit_bytes=64000)} for p in pods[:20]]}
|
|
|
|
|
|
@router.post("/runtime/projects/{project}/services/{service}/restart")
|
|
def restart(project: str, service: str, authorization: str | None = Header(default=None)):
|
|
auth(authorization)
|
|
_, selected = active(project, service)
|
|
if blocked(client.CoreV1Api(),identifier(project)):
|
|
raise HTTPException(409,'Project runtime is suspended')
|
|
apps = client.AppsV1Api()
|
|
deployments = apps.list_namespaced_deployment(NAMESPACE, label_selector=",".join(k + "=" + v for k, v in selected.items())).items
|
|
for d in deployments:
|
|
apps.patch_namespaced_deployment(d.metadata.name, NAMESPACE, {"spec": {"template": {"metadata": {"annotations": {"platform/restarted-at": datetime.now(timezone.utc).isoformat()}}}}})
|
|
return {"message": "Restart requested", "deployments": len(deployments)}
|
|
|
|
|
|
@router.post('/runtime/projects/{project}/state')
|
|
def project_state(project: str,request: ProjectState,authorization: str | None = Header(default=None)):
|
|
auth(authorization)
|
|
project=identifier(project)
|
|
core,apps=client.CoreV1Api(),client.AppsV1Api()
|
|
name='control-'+project
|
|
ensure(core.create_namespaced_config_map,namespace=NAMESPACE,body={'metadata':{'name':name},'data':{'state':request.action}})
|
|
core.patch_namespaced_config_map(name,NAMESPACE,{'data':{'state':request.action,'operation':str(request.operationId)}})
|
|
changed=0
|
|
services=core.list_namespaced_service(NAMESPACE,label_selector='platform/project='+project).items
|
|
for service in services:
|
|
selector=service.spec.selector or {}
|
|
if not selector:
|
|
continue
|
|
for deployment in apps.list_namespaced_deployment(NAMESPACE,label_selector=','.join(k+'='+v for k,v in selector.items())).items:
|
|
annotations=deployment.metadata.annotations or {}
|
|
if request.action=='SUSPEND':
|
|
desired=deployment.spec.replicas or int(annotations.get('platform/desired-replicas','1'))
|
|
apps.patch_namespaced_deployment(deployment.metadata.name,NAMESPACE,{'metadata':{'annotations':{'platform/desired-replicas':str(desired)}},'spec':{'replicas':0}})
|
|
else:
|
|
apps.patch_namespaced_deployment(deployment.metadata.name,NAMESPACE,{'spec':{'replicas':int(annotations.get('platform/desired-replicas','1'))}})
|
|
changed+=1
|
|
return {'state':request.action,'changedDeployments':changed,'operationId':str(request.operationId)}
|
|
|
|
|
|
@router.get('/runtime/nodes')
|
|
def nodes(authorization: str | None = Header(default=None)):
|
|
auth(authorization)
|
|
core=client.CoreV1Api()
|
|
metrics={}
|
|
try:
|
|
for m in client.CustomObjectsApi().list_cluster_custom_object('metrics.k8s.io','v1beta1','nodes').get('items',[]):
|
|
metrics[m['metadata']['name']]=m.get('usage',{})
|
|
except ApiException:
|
|
pass
|
|
pods=core.list_namespaced_pod(NAMESPACE).items
|
|
result=[]
|
|
for n in core.list_node().items:
|
|
conditions=n.status.conditions or []
|
|
ready=any(c.type=='Ready' and c.status=='True' for c in conditions)
|
|
info=n.status.node_info
|
|
result.append({'name':n.metadata.name,'status':'READY' if ready else 'NOT_READY','cpu':n.status.allocatable.get('cpu',''),
|
|
'memory':n.status.allocatable.get('memory',''),'internalIp':next((a.address for a in n.status.addresses or [] if a.type=='InternalIP'),''),
|
|
'kubeletVersion':info.kubelet_version,'os':info.os_image,'runtime':info.container_runtime_version,
|
|
'unschedulable':bool(n.spec.unschedulable),'managedPods':sum(1 for p in pods if p.spec.node_name==n.metadata.name),'usage':metrics.get(n.metadata.name)})
|
|
return {'items':result,'metricsAvailable':bool(metrics)}
|