Reuse sandbox resources instead of charging quota on every tool call

This commit is contained in:
WebUp Cloud committed 2026-11-07 23:26:34 +08:00
1 parent cb896bb0fa
commit 8f232c442b
3 files changed
+83 -14

No files matched your search

+46 -13
View File
@@ -78,7 +78,16 @@ def workspace_name(project: str) -> str:
raise HTTPException(400, "Invalid project ID")
def kube_call(stage, operation, *, already_exists_ok=False):
def is_quota_denial(exc):
if not isinstance(exc, ApiException) or exc.status != 403:
return False
try:
return "exceeded quota:" in json.loads(exc.body or "{}")["message"]
except (KeyError, TypeError, ValueError):
return False
def kube_call(stage, operation):
# Use only before sending tool input. A command or write may already have
# happened once stdin is sent, so an interrupted result must not be retried.
for attempt in range(3):
@@ -88,18 +97,40 @@ def kube_call(stage, operation, *, already_exists_ok=False):
status = exc.status if isinstance(exc, ApiException) else 0
cause = exc.__context__
upstream = getattr(cause, "status_code", None)
if already_exists_ok and status == 409:
return None
detail = "WORKSPACE_CAPACITY" if is_quota_denial(exc) else stage
LOG.warning("Sandbox Kubernetes %s failed (attempt=%d, status=%s, cause=%s)",
stage, attempt + 1, upstream or status, type(cause or exc).__name__)
detail, attempt + 1, upstream or status, type(cause or exc).__name__)
if attempt == 2 or (upstream or status) not in TRANSIENT_KUBE_STATUSES:
raise HTTPException(503, stage) from None
raise HTTPException(503, detail) from None
time.sleep(0.25 * 2 ** attempt)
def create_once(method, **kwargs):
kube_call("WORKSPACE_PROVISIONING",
lambda: method(**kwargs, _request_timeout=(3, 5)), already_exists_ok=True)
def ensure_resource(read, create, namespace, body):
def get():
return read(body["metadata"]["name"], namespace, _request_timeout=(3, 5))
def find_or_create():
# Admission can charge quota for a duplicate POST before it returns 409.
# Read first on every attempt, including after a lost create response.
try:
return get()
except ApiException as exc:
if exc.status != 404:
raise
try:
return create(namespace=namespace, body=body, _request_timeout=(3, 5))
except ApiException as exc:
if exc.status != 409 and not is_quota_denial(exc):
raise
# Another gateway may have created it between the read and create.
try:
return get()
except ApiException as read_error:
if read_error.status == 404:
raise exc from None
raise
return kube_call("WORKSPACE_PROVISIONING", find_or_create)
def ensure_workspace(project: str, review=False):
@@ -110,7 +141,8 @@ def ensure_workspace(project: str, review=False):
core, net = app.state.core, app.state.net
# The namespace MUST enforce restricted Pod Security Admission and ResourceQuota.
labels = {"platform/workspace": name}
create_once(
ensure_resource(
net.read_namespaced_network_policy,
net.create_namespaced_network_policy,
namespace=namespace,
body={
@@ -128,10 +160,11 @@ def ensure_workspace(project: str, review=False):
"ports": [{"port": 443, "protocol": "TCP"}, {"port": 80, "protocol": "TCP"}]}
]}
})
create_once(core.create_namespaced_persistent_volume_claim, namespace=namespace, body={
ensure_resource(core.read_namespaced_persistent_volume_claim,
core.create_namespaced_persistent_volume_claim, namespace=namespace, body={
"apiVersion": "v1", "kind": "PersistentVolumeClaim", "metadata": {"name": name},
"spec": {"accessModes": ["ReadWriteOnce"], "resources": {"requests": {"storage": "5Gi"}}}})
create_once(core.create_namespaced_pod, namespace=namespace, body={
pod = ensure_resource(core.read_namespaced_pod, core.create_namespaced_pod, namespace=namespace, body={
"apiVersion": "v1", "kind": "Pod", "metadata": {"name": name, "labels": labels},
"spec": {"runtimeClassName": RUNTIME, "automountServiceAccountToken": False,
"restartPolicy": "Always", "enableServiceLinks": False,
@@ -150,14 +183,14 @@ def ensure_workspace(project: str, review=False):
{"name": "tmp", "emptyDir": {"sizeLimit": "1Gi"}}]}})
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
pod = kube_call("WORKSPACE_PROVISIONING",
lambda: core.read_namespaced_pod(name, namespace, _request_timeout=(3, 5)))
if pod.status.phase == "Running" and pod.status.container_statuses and all(s.ready for s in pod.status.container_statuses):
return name
if pod.status.phase in ("Failed", "Succeeded"):
LOG.warning("Sandbox pod %s is %s", name, pod.status.phase)
raise HTTPException(503, "WORKSPACE_TERMINATED")
time.sleep(1)
pod = kube_call("WORKSPACE_PROVISIONING",
lambda: core.read_namespaced_pod(name, namespace, _request_timeout=(3, 5)))
LOG.warning("Sandbox pod %s did not become ready (phase=%s)", name, pod.status.phase)
raise HTTPException(503, "WORKSPACE_STARTING")