forked from xhgit/webup-cloud
Initial import of WebUp Cloud
This commit is contained in:
commit
db82325ed7
216 files changed
+33718
No files matched your search
@@ -0,0 +1,325 @@
|
||||
"""Restricted OCI runtime controller. This module never builds or runs code on the gateway host."""
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
import html
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Header, HTTPException
|
||||
from kubernetes import client
|
||||
from kubernetes.client.rest import ApiException
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
router = APIRouter()
|
||||
NAMESPACE = os.environ.get("RUNTIME_NAMESPACE", "platform-runtimes")
|
||||
TOKEN = os.environ.get("RUNTIME_TOKEN", "")
|
||||
RUNTIME = os.environ.get("RUNTIME_CLASS", "gvisor")
|
||||
locks = {}
|
||||
guard = threading.Lock()
|
||||
|
||||
|
||||
def auth(authorization):
|
||||
if len(TOKEN) < 32 or not hmac.compare_digest(authorization or "", "Bearer " + TOKEN):
|
||||
raise HTTPException(401, "Unauthorized runtime request")
|
||||
|
||||
|
||||
def identifier(value):
|
||||
try:
|
||||
return uuid.UUID(str(value)).hex
|
||||
except ValueError:
|
||||
raise HTTPException(400, "Invalid resource ID")
|
||||
|
||||
|
||||
class Service(BaseModel):
|
||||
id: uuid.UUID
|
||||
name: str = Field(pattern=r"^[a-z][a-z0-9-]{0,62}$")
|
||||
image: str = Field(pattern=r"^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$")
|
||||
command: list[str] = Field(max_length=30)
|
||||
environment: dict[str, str] = Field(max_length=100)
|
||||
port: int = Field(ge=1, le=65535)
|
||||
cpu_millis: int = Field(ge=50, le=64000)
|
||||
memory_mb: int = Field(ge=64, le=262144)
|
||||
replicas: int = Field(ge=1, le=20)
|
||||
health_path: str = Field(pattern=r"^/[^\s]{0,499}$|^$")
|
||||
|
||||
|
||||
class Deployment(BaseModel):
|
||||
projectId: uuid.UUID
|
||||
deploymentId: uuid.UUID
|
||||
services: list[Service] = Field(min_length=1, max_length=100)
|
||||
branding: dict[str, str] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class ProjectState(BaseModel):
|
||||
action: str = Field(pattern=r'^(SUSPEND|RESUME)$')
|
||||
operationId: uuid.UUID
|
||||
|
||||
|
||||
def blocked(core,project):
|
||||
try:
|
||||
return core.read_namespaced_config_map('control-'+project,NAMESPACE).data.get('state')=='SUSPEND'
|
||||
except ApiException as e:
|
||||
if e.status==404:
|
||||
return False
|
||||
raise
|
||||
|
||||
|
||||
def branding_config(port, branding):
|
||||
if branding.get('hostedFooterEnabled') != 'true':
|
||||
return None
|
||||
text = html.escape(branding.get('hostedFooterText', '由 WebUp Cloud 提供服务'), quote=True)
|
||||
url = branding.get('hostedFooterLink', 'https://webup.run')
|
||||
if url and not url.startswith('https://'):
|
||||
raise HTTPException(400, 'Footer link must use HTTPS')
|
||||
label = ('<a href="' + html.escape(url, quote=True) + '" target="_blank" rel="noopener noreferrer" style="color:inherit!important;text-decoration:none!important">' + text + '</a>') if url else text
|
||||
footer = '<div id="webup-powered-by" style="box-sizing:border-box!important;clear:both!important;text-align:center!important;padding:14px 10px!important;font:12px/1.6 system-ui,sans-serif!important;color:#8a96a8!important;background:transparent!important">' + label + '</div>'
|
||||
footer = footer.replace('$','$').replace('\\','\').replace('\n',' ').replace('\r',' ')
|
||||
return '''map $http_upgrade $connection_upgrade { default upgrade; '' close; }
|
||||
server {
|
||||
listen 8081;
|
||||
server_tokens off;
|
||||
client_max_body_size 32m;
|
||||
location / {
|
||||
proxy_pass http://127.0.0.1:PORT;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection $connection_upgrade;
|
||||
proxy_set_header Accept-Encoding "";
|
||||
proxy_read_timeout 3600s;
|
||||
sub_filter_once on;
|
||||
sub_filter '</body>' 'FOOTER</body>';
|
||||
}
|
||||
}
|
||||
'''.replace('PORT',str(port)).replace('FOOTER',footer)
|
||||
|
||||
|
||||
def ensure(method, **kwargs):
|
||||
try:
|
||||
return method(**kwargs)
|
||||
except ApiException as exc:
|
||||
if exc.status != 409:
|
||||
raise
|
||||
|
||||
|
||||
def network_policy(project):
|
||||
return {"apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", "metadata": {"name": "project-" + project},
|
||||
"spec": {"podSelector": {"matchLabels": {"platform/project": project}}, "policyTypes": ["Ingress", "Egress"],
|
||||
"ingress": [{"from": [{"podSelector": {"matchLabels": {"platform/project": project}}},
|
||||
{"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "ingress-nginx"}},
|
||||
"podSelector": {"matchLabels": {"app.kubernetes.io/name": "ingress-nginx"}}}]}],
|
||||
"egress": [{"to": [{"podSelector": {"matchLabels": {"platform/project": project}}}]},
|
||||
{"to": [{"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "kube-system"}}}],
|
||||
"ports": [{"port": 53, "protocol": "UDP"}, {"port": 53, "protocol": "TCP"}]},
|
||||
{"to": [{"ipBlock": {"cidr": "0.0.0.0/0", "except": ["0.0.0.0/8", "10.0.0.0/8", "100.64.0.0/10", "127.0.0.0/8", "169.254.0.0/16", "172.16.0.0/12", "192.0.0.0/24", "192.168.0.0/16", "198.18.0.0/15", "224.0.0.0/4", "240.0.0.0/4"]}}],
|
||||
"ports": [{"port": 443, "protocol": "TCP"}, {"port": 80, "protocol": "TCP"}]}]}}
|
||||
|
||||
|
||||
def deployment_manifest(project, deployment, service, branding=None):
|
||||
sid = identifier(service.id)
|
||||
labels = {"platform/project": project, "platform/service": sid, "platform/revision": deployment}
|
||||
name = "app-" + sid[:20] + "-" + deployment[:12]
|
||||
security = {"allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"]}}
|
||||
probe = {"httpGet": {"path": service.health_path, "port": service.port}} if service.health_path else {"tcpSocket": {"port": service.port}}
|
||||
container = {"name": "app", "image": service.image, "ports": [{"containerPort": service.port}], "securityContext": security,
|
||||
"envFrom": [{"secretRef": {"name": name}}], "volumeMounts": [{"name": "tmp", "mountPath": "/tmp"}],
|
||||
"resources": {"requests": {"cpu": str(service.cpu_millis) + "m", "memory": str(service.memory_mb) + "Mi"},
|
||||
"limits": {"cpu": str(service.cpu_millis) + "m", "memory": str(service.memory_mb) + "Mi", "ephemeral-storage": "1Gi"}},
|
||||
"readinessProbe": {**probe, "initialDelaySeconds": 2, "periodSeconds": 3, "timeoutSeconds": 2},
|
||||
"livenessProbe": {**probe, "initialDelaySeconds": 30, "periodSeconds": 10, "timeoutSeconds": 3},
|
||||
"startupProbe": {**probe, "periodSeconds": 3, "failureThreshold": 25}}
|
||||
if service.command:
|
||||
container["command"] = service.command
|
||||
manifest = {"apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"name": name, "labels": labels},
|
||||
"spec": {"replicas": service.replicas, "selector": {"matchLabels": labels},
|
||||
"template": {"metadata": {"labels": labels}, "spec": {"runtimeClassName": RUNTIME,
|
||||
"automountServiceAccountToken": False, "enableServiceLinks": False,
|
||||
"securityContext": {"runAsNonRoot": True, "runAsUser": 10001, "runAsGroup": 10001, "seccompProfile": {"type": "RuntimeDefault"}},
|
||||
"containers": [container], "volumes": [{"name": "tmp", "emptyDir": {"sizeLimit": "512Mi"}}]}}}}
|
||||
if branding_config(service.port, branding or {}) is not None:
|
||||
pod = manifest['spec']['template']['spec']
|
||||
pod['volumes'].extend([{'name':'branding-config','configMap':{'name':name+'-branding'}},{'name':'proxy-tmp','emptyDir':{'sizeLimit':'64Mi'}}])
|
||||
pod['containers'].append({'name':'webup-branding','image':os.environ.get('BRANDING_PROXY_IMAGE','nginxinc/nginx-unprivileged:1.28-alpine'),
|
||||
'ports':[{'containerPort':8081}],'securityContext':security,
|
||||
'resources':{'requests':{'cpu':'10m','memory':'16Mi'},'limits':{'cpu':'100m','memory':'64Mi'}},
|
||||
'volumeMounts':[{'name':'branding-config','mountPath':'/etc/nginx/conf.d','readOnly':True},{'name':'proxy-tmp','mountPath':'/tmp'}],
|
||||
'readinessProbe':{'tcpSocket':{'port':8081},'periodSeconds':3},'livenessProbe':{'tcpSocket':{'port':8081},'periodSeconds':10}})
|
||||
return manifest
|
||||
|
||||
|
||||
@router.post("/runtime/deployments/{deployment}")
|
||||
def deploy(deployment: str, request: Deployment, authorization: str | None = Header(default=None)):
|
||||
auth(authorization)
|
||||
did, project = identifier(deployment), identifier(request.projectId)
|
||||
if did != identifier(request.deploymentId):
|
||||
raise HTTPException(400, "Deployment ID mismatch")
|
||||
with guard:
|
||||
lock = locks.setdefault(project, threading.Lock())
|
||||
if not lock.acquire(timeout=1):
|
||||
raise HTTPException(409, "Project deployment in progress")
|
||||
core, apps, networking = client.CoreV1Api(), client.AppsV1Api(), client.NetworkingV1Api()
|
||||
receipt = "release-" + did
|
||||
old = {}
|
||||
try:
|
||||
if blocked(core,project):
|
||||
raise HTTPException(409,'Project runtime is suspended; deployment denied')
|
||||
try:
|
||||
record = core.read_namespaced_config_map(receipt, NAMESPACE)
|
||||
if record.data.get("project") != project:
|
||||
raise HTTPException(409, "Deployment receipt conflict")
|
||||
if record.data.get("status") == "ACTIVE":
|
||||
return {"ready": True, "deploymentId": deployment, "replayed": True}
|
||||
old = json.loads(record.data["selectors"])
|
||||
except ApiException as exc:
|
||||
if exc.status != 404:
|
||||
raise
|
||||
for service in request.services:
|
||||
stable = "svc-" + identifier(service.id)[:20]
|
||||
try:
|
||||
old[stable] = core.read_namespaced_service(stable, NAMESPACE).spec.selector
|
||||
except ApiException as missing:
|
||||
if missing.status != 404:
|
||||
raise
|
||||
old[stable] = None
|
||||
ensure(core.create_namespaced_config_map, namespace=NAMESPACE, body={"metadata": {"name": receipt}, "data": {"project": project, "status": "PENDING", "selectors": json.dumps(old)}})
|
||||
ensure(networking.create_namespaced_network_policy, namespace=NAMESPACE, body=network_policy(project))
|
||||
names = []
|
||||
for service in request.services:
|
||||
manifest = deployment_manifest(project, did, service, request.branding)
|
||||
name = manifest["metadata"]["name"]
|
||||
ensure(core.create_namespaced_secret, namespace=NAMESPACE, body={"metadata": {"name": name}, "type": "Opaque", "stringData": service.environment})
|
||||
proxy_config = branding_config(service.port,request.branding)
|
||||
if proxy_config is not None:
|
||||
ensure(core.create_namespaced_config_map,namespace=NAMESPACE,body={'metadata':{'name':name+'-branding'},'data':{'default.conf':proxy_config}})
|
||||
ensure(apps.create_namespaced_deployment, namespace=NAMESPACE, body=manifest)
|
||||
stable = "svc-" + identifier(service.id)[:20]
|
||||
target_port = 8081 if proxy_config is not None else service.port
|
||||
ensure(core.create_namespaced_service, namespace=NAMESPACE, body={"metadata": {"name": stable, "labels": {"platform/project": project}}, "spec": {"selector": manifest["metadata"]["labels"], "ports": [{"port": service.port, "targetPort": target_port}]}})
|
||||
names.append((name, stable, manifest["metadata"]["labels"], service.port, service.replicas))
|
||||
deadline = time.monotonic() + 90
|
||||
while time.monotonic() < deadline:
|
||||
if all((apps.read_namespaced_deployment(n, NAMESPACE).status.ready_replicas or 0) >= replicas for n, _, _, _, replicas in names):
|
||||
break
|
||||
time.sleep(2)
|
||||
else:
|
||||
return {"ready": False, "error": "New revision did not become ready; previous traffic selectors retained"}
|
||||
for _, stable, labels, port, _ in names:
|
||||
if blocked(core,project):
|
||||
raise HTTPException(409,'Project was suspended during deployment')
|
||||
core.patch_namespaced_service(stable, NAMESPACE, {"spec": {"selector": labels, "ports": [{"port": port, "targetPort":8081 if request.branding.get('hostedFooterEnabled')=='true' else port}]}})
|
||||
core.patch_namespaced_config_map(receipt, NAMESPACE, {"data": {"status": "ACTIVE"}})
|
||||
cleanup_warnings = []
|
||||
for previous in apps.list_namespaced_deployment(NAMESPACE, label_selector="platform/project=" + project).items:
|
||||
if previous.metadata.labels.get("platform/revision") != did and previous.spec.replicas:
|
||||
try:
|
||||
apps.patch_namespaced_deployment(previous.metadata.name, NAMESPACE, {"spec": {"replicas": 0}})
|
||||
except ApiException:
|
||||
cleanup_warnings.append("Could not scale down retained revision " + previous.metadata.name)
|
||||
return {"ready": True, "deploymentId": deployment, "cleanupWarnings": cleanup_warnings, "services": [{"internalAddress": stable + "." + NAMESPACE + ".svc.cluster.local:" + str(port)} for _, stable, _, port, _ in names]}
|
||||
except ApiException:
|
||||
# If switching a multi-service release fails halfway, restore the previously recorded selectors.
|
||||
for stable, selector in old.items():
|
||||
if selector:
|
||||
try:
|
||||
core.patch_namespaced_service(stable, NAMESPACE, {"spec": {"selector": selector}})
|
||||
except ApiException:
|
||||
pass # The control plane retains a failure event for explicit operator recovery.
|
||||
raise HTTPException(503, "Runtime operation failed; check release receipt before retrying") from None
|
||||
finally:
|
||||
lock.release()
|
||||
|
||||
|
||||
def active(project, service):
|
||||
project, service = identifier(project), identifier(service)
|
||||
core = client.CoreV1Api()
|
||||
try:
|
||||
selected = core.read_namespaced_service("svc-" + service[:20], NAMESPACE).spec.selector
|
||||
if selected.get("platform/project") != project or selected.get("platform/service") != service:
|
||||
raise HTTPException(404, "Service not found")
|
||||
return core, selected
|
||||
except ApiException:
|
||||
raise HTTPException(404, "Service not found") from None
|
||||
|
||||
|
||||
@router.get("/runtime/projects/{project}/services/{service}")
|
||||
def status(project: str, service: str, authorization: str | None = Header(default=None)):
|
||||
auth(authorization)
|
||||
core, selected = active(project, service)
|
||||
selector = ",".join(k + "=" + v for k, v in selected.items())
|
||||
pods = core.list_namespaced_pod(NAMESPACE, label_selector=selector).items
|
||||
return {"pods": [{"name": p.metadata.name, "phase": p.status.phase, "node": p.spec.node_name,
|
||||
"ready": all(c.ready for c in (p.status.container_statuses or [])),
|
||||
"restarts": sum(c.restart_count for c in (p.status.container_statuses or []))} for p in pods],
|
||||
"internalAddress": "svc-" + identifier(service)[:20] + "." + NAMESPACE + ".svc.cluster.local"}
|
||||
|
||||
|
||||
@router.get("/runtime/projects/{project}/services/{service}/logs")
|
||||
def logs(project: str, service: str, authorization: str | None = Header(default=None)):
|
||||
auth(authorization)
|
||||
core, selected = active(project, service)
|
||||
pods = core.list_namespaced_pod(NAMESPACE, label_selector=",".join(k + "=" + v for k, v in selected.items())).items
|
||||
return {"logs": [{"pod": p.metadata.name, "output": core.read_namespaced_pod_log(p.metadata.name, NAMESPACE, container="app", tail_lines=200, limit_bytes=64000)} for p in pods[:20]]}
|
||||
|
||||
|
||||
@router.post("/runtime/projects/{project}/services/{service}/restart")
|
||||
def restart(project: str, service: str, authorization: str | None = Header(default=None)):
|
||||
auth(authorization)
|
||||
_, selected = active(project, service)
|
||||
if blocked(client.CoreV1Api(),identifier(project)):
|
||||
raise HTTPException(409,'Project runtime is suspended')
|
||||
apps = client.AppsV1Api()
|
||||
deployments = apps.list_namespaced_deployment(NAMESPACE, label_selector=",".join(k + "=" + v for k, v in selected.items())).items
|
||||
for d in deployments:
|
||||
apps.patch_namespaced_deployment(d.metadata.name, NAMESPACE, {"spec": {"template": {"metadata": {"annotations": {"platform/restarted-at": datetime.now(timezone.utc).isoformat()}}}}})
|
||||
return {"message": "Restart requested", "deployments": len(deployments)}
|
||||
|
||||
|
||||
@router.post('/runtime/projects/{project}/state')
|
||||
def project_state(project: str,request: ProjectState,authorization: str | None = Header(default=None)):
|
||||
auth(authorization)
|
||||
project=identifier(project)
|
||||
core,apps=client.CoreV1Api(),client.AppsV1Api()
|
||||
name='control-'+project
|
||||
ensure(core.create_namespaced_config_map,namespace=NAMESPACE,body={'metadata':{'name':name},'data':{'state':request.action}})
|
||||
core.patch_namespaced_config_map(name,NAMESPACE,{'data':{'state':request.action,'operation':str(request.operationId)}})
|
||||
changed=0
|
||||
services=core.list_namespaced_service(NAMESPACE,label_selector='platform/project='+project).items
|
||||
for service in services:
|
||||
selector=service.spec.selector or {}
|
||||
if not selector:
|
||||
continue
|
||||
for deployment in apps.list_namespaced_deployment(NAMESPACE,label_selector=','.join(k+'='+v for k,v in selector.items())).items:
|
||||
annotations=deployment.metadata.annotations or {}
|
||||
if request.action=='SUSPEND':
|
||||
desired=deployment.spec.replicas or int(annotations.get('platform/desired-replicas','1'))
|
||||
apps.patch_namespaced_deployment(deployment.metadata.name,NAMESPACE,{'metadata':{'annotations':{'platform/desired-replicas':str(desired)}},'spec':{'replicas':0}})
|
||||
else:
|
||||
apps.patch_namespaced_deployment(deployment.metadata.name,NAMESPACE,{'spec':{'replicas':int(annotations.get('platform/desired-replicas','1'))}})
|
||||
changed+=1
|
||||
return {'state':request.action,'changedDeployments':changed,'operationId':str(request.operationId)}
|
||||
|
||||
|
||||
@router.get('/runtime/nodes')
|
||||
def nodes(authorization: str | None = Header(default=None)):
|
||||
auth(authorization)
|
||||
core=client.CoreV1Api()
|
||||
metrics={}
|
||||
try:
|
||||
for m in client.CustomObjectsApi().list_cluster_custom_object('metrics.k8s.io','v1beta1','nodes').get('items',[]):
|
||||
metrics[m['metadata']['name']]=m.get('usage',{})
|
||||
except ApiException:
|
||||
pass
|
||||
pods=core.list_namespaced_pod(NAMESPACE).items
|
||||
result=[]
|
||||
for n in core.list_node().items:
|
||||
conditions=n.status.conditions or []
|
||||
ready=any(c.type=='Ready' and c.status=='True' for c in conditions)
|
||||
info=n.status.node_info
|
||||
result.append({'name':n.metadata.name,'status':'READY' if ready else 'NOT_READY','cpu':n.status.allocatable.get('cpu',''),
|
||||
'memory':n.status.allocatable.get('memory',''),'internalIp':next((a.address for a in n.status.addresses or [] if a.type=='InternalIP'),''),
|
||||
'kubeletVersion':info.kubelet_version,'os':info.os_image,'runtime':info.container_runtime_version,
|
||||
'unschedulable':bool(n.spec.unschedulable),'managedPods':sum(1 for p in pods if p.spec.node_name==n.metadata.name),'usage':metrics.get(n.metadata.name)})
|
||||
return {'items':result,'metricsAvailable':bool(metrics)}
|
||||
Reference in new issue
Block a user