From 8f232c442bd5c678a1b00fa3470a592154c766ca Mon Sep 17 00:00:00 2001 From: WebUp Cloud Date: Sat, 7 Nov 2026 23:26:34 +0800 Subject: [PATCH] Reuse sandbox resources instead of charging quota on every tool call --- .../run/webup/platform/SandboxClient.java | 1 + node-agent/app.py | 59 +++++++++++++++---- node-agent/test_gateway.py | 37 +++++++++++- 3 files changed, 83 insertions(+), 14 deletions(-) diff --git a/backend/src/main/java/run/webup/platform/SandboxClient.java b/backend/src/main/java/run/webup/platform/SandboxClient.java index 21a44d7..4c976d1 100644 --- a/backend/src/main/java/run/webup/platform/SandboxClient.java +++ b/backend/src/main/java/run/webup/platform/SandboxClient.java @@ -235,6 +235,7 @@ public class SandboxClient { 503, switch (stage) { case "WORKSPACE_PROVISIONING" -> "隔离环境暂时无法准备,请稍后重试"; + case "WORKSPACE_CAPACITY" -> "隔离环境容量已满,请联系管理员检查集群配额"; case "WORKSPACE_STARTING" -> "隔离环境仍在启动,请稍后刷新"; case "WORKSPACE_TERMINATED" -> "隔离环境已退出,请联系管理员检查"; case "WORKSPACE_CONNECTION" -> "暂时无法连接项目的隔离环境,请稍后重试"; diff --git a/node-agent/app.py b/node-agent/app.py index dbccb0e..c856608 100644 --- a/node-agent/app.py +++ b/node-agent/app.py @@ -78,7 +78,16 @@ def workspace_name(project: str) -> str: raise HTTPException(400, "Invalid project ID") -def kube_call(stage, operation, *, already_exists_ok=False): +def is_quota_denial(exc): + if not isinstance(exc, ApiException) or exc.status != 403: + return False + try: + return "exceeded quota:" in json.loads(exc.body or "{}")["message"] + except (KeyError, TypeError, ValueError): + return False + + +def kube_call(stage, operation): # Use only before sending tool input. A command or write may already have # happened once stdin is sent, so an interrupted result must not be retried. for attempt in range(3): @@ -88,18 +97,40 @@ def kube_call(stage, operation, *, already_exists_ok=False): status = exc.status if isinstance(exc, ApiException) else 0 cause = exc.__context__ upstream = getattr(cause, "status_code", None) - if already_exists_ok and status == 409: - return None + detail = "WORKSPACE_CAPACITY" if is_quota_denial(exc) else stage LOG.warning("Sandbox Kubernetes %s failed (attempt=%d, status=%s, cause=%s)", - stage, attempt + 1, upstream or status, type(cause or exc).__name__) + detail, attempt + 1, upstream or status, type(cause or exc).__name__) if attempt == 2 or (upstream or status) not in TRANSIENT_KUBE_STATUSES: - raise HTTPException(503, stage) from None + raise HTTPException(503, detail) from None time.sleep(0.25 * 2 ** attempt) -def create_once(method, **kwargs): - kube_call("WORKSPACE_PROVISIONING", - lambda: method(**kwargs, _request_timeout=(3, 5)), already_exists_ok=True) +def ensure_resource(read, create, namespace, body): + def get(): + return read(body["metadata"]["name"], namespace, _request_timeout=(3, 5)) + + def find_or_create(): + # Admission can charge quota for a duplicate POST before it returns 409. + # Read first on every attempt, including after a lost create response. + try: + return get() + except ApiException as exc: + if exc.status != 404: + raise + try: + return create(namespace=namespace, body=body, _request_timeout=(3, 5)) + except ApiException as exc: + if exc.status != 409 and not is_quota_denial(exc): + raise + # Another gateway may have created it between the read and create. + try: + return get() + except ApiException as read_error: + if read_error.status == 404: + raise exc from None + raise + + return kube_call("WORKSPACE_PROVISIONING", find_or_create) def ensure_workspace(project: str, review=False): @@ -110,7 +141,8 @@ def ensure_workspace(project: str, review=False): core, net = app.state.core, app.state.net # The namespace MUST enforce restricted Pod Security Admission and ResourceQuota. labels = {"platform/workspace": name} - create_once( + ensure_resource( + net.read_namespaced_network_policy, net.create_namespaced_network_policy, namespace=namespace, body={ @@ -128,10 +160,11 @@ def ensure_workspace(project: str, review=False): "ports": [{"port": 443, "protocol": "TCP"}, {"port": 80, "protocol": "TCP"}]} ]} }) - create_once(core.create_namespaced_persistent_volume_claim, namespace=namespace, body={ + ensure_resource(core.read_namespaced_persistent_volume_claim, + core.create_namespaced_persistent_volume_claim, namespace=namespace, body={ "apiVersion": "v1", "kind": "PersistentVolumeClaim", "metadata": {"name": name}, "spec": {"accessModes": ["ReadWriteOnce"], "resources": {"requests": {"storage": "5Gi"}}}}) - create_once(core.create_namespaced_pod, namespace=namespace, body={ + pod = ensure_resource(core.read_namespaced_pod, core.create_namespaced_pod, namespace=namespace, body={ "apiVersion": "v1", "kind": "Pod", "metadata": {"name": name, "labels": labels}, "spec": {"runtimeClassName": RUNTIME, "automountServiceAccountToken": False, "restartPolicy": "Always", "enableServiceLinks": False, @@ -150,14 +183,14 @@ def ensure_workspace(project: str, review=False): {"name": "tmp", "emptyDir": {"sizeLimit": "1Gi"}}]}}) deadline = time.monotonic() + 90 while time.monotonic() < deadline: - pod = kube_call("WORKSPACE_PROVISIONING", - lambda: core.read_namespaced_pod(name, namespace, _request_timeout=(3, 5))) if pod.status.phase == "Running" and pod.status.container_statuses and all(s.ready for s in pod.status.container_statuses): return name if pod.status.phase in ("Failed", "Succeeded"): LOG.warning("Sandbox pod %s is %s", name, pod.status.phase) raise HTTPException(503, "WORKSPACE_TERMINATED") time.sleep(1) + pod = kube_call("WORKSPACE_PROVISIONING", + lambda: core.read_namespaced_pod(name, namespace, _request_timeout=(3, 5))) LOG.warning("Sandbox pod %s did not become ready (phase=%s)", name, pod.status.phase) raise HTTPException(503, "WORKSPACE_STARTING") diff --git a/node-agent/test_gateway.py b/node-agent/test_gateway.py index 7cbf9f8..5e7e9b4 100644 --- a/node-agent/test_gateway.py +++ b/node-agent/test_gateway.py @@ -73,7 +73,10 @@ class GatewayTests(unittest.TestCase): def test_created_workspace_has_isolation_and_persistent_data(self): core, net = MagicMock(), MagicMock() - core.read_namespaced_pod.return_value = types.SimpleNamespace(status=types.SimpleNamespace(phase="Running", container_statuses=[types.SimpleNamespace(ready=True)])) + net.read_namespaced_network_policy.side_effect = gateway.ApiException(status=404) + core.read_namespaced_persistent_volume_claim.side_effect = gateway.ApiException(status=404) + core.read_namespaced_pod.side_effect = gateway.ApiException(status=404) + core.create_namespaced_pod.return_value = types.SimpleNamespace(status=types.SimpleNamespace(phase="Running", container_statuses=[types.SimpleNamespace(ready=True)])) with patch.object(gateway.app.state, "core", core, create=True), patch.object(gateway.app.state, "net", net, create=True), patch.object(gateway, "IMAGE", "test/isolated-image:local"): gateway.ensure_workspace("00000000-0000-0000-0000-000000000001") pod = core.create_namespaced_pod.call_args.kwargs["body"]["spec"] @@ -86,6 +89,38 @@ class GatewayTests(unittest.TestCase): self.assertEqual([], policy["ingress"]) self.assertIn("169.254.0.0/16", policy["egress"][1]["to"][0]["ipBlock"]["except"]) + def test_existing_workspace_does_not_submit_creates_to_admission(self): + core, net = MagicMock(), MagicMock() + core.read_namespaced_pod.return_value = types.SimpleNamespace( + status=types.SimpleNamespace(phase="Running", container_statuses=[types.SimpleNamespace(ready=True)]) + ) + project = "00000000-0000-0000-0000-000000000004" + with (patch.object(gateway.app.state, "core", core, create=True), + patch.object(gateway.app.state, "net", net, create=True)): + for _ in range(3): + self.assertEqual("dev-00000000000000000000000000000004", gateway.ensure_workspace(project)) + net.create_namespaced_network_policy.assert_not_called() + core.create_namespaced_persistent_volume_claim.assert_not_called() + core.create_namespaced_pod.assert_not_called() + + def test_create_with_lost_response_checks_for_resource_before_retry(self): + resource = object() + read = MagicMock(side_effect=[gateway.ApiException(status=404), resource]) + create = MagicMock(side_effect=gateway.ApiException(status=503)) + with patch.object(gateway.time, "sleep"): + self.assertIs(resource, gateway.ensure_resource(read, create, "qa", {"metadata": {"name": "workspace"}})) + create.assert_called_once() + + def test_quota_denial_of_new_resource_reports_capacity(self): + read = MagicMock(side_effect=gateway.ApiException(status=404)) + denial = gateway.ApiException(status=403) + denial.body = json.dumps({"message": 'pods "workspace" is forbidden: exceeded quota: quota, requested: pods=1'}) + create = MagicMock(side_effect=denial) + with self.assertRaises(gateway.HTTPException) as error: + gateway.ensure_resource(read, create, "qa", {"metadata": {"name": "workspace"}}) + self.assertEqual((503, "WORKSPACE_CAPACITY"), (error.exception.status_code, error.exception.detail)) + create.assert_called_once() + def test_transient_exec_handshake_recovers_before_tool_input(self): project = "00000000-0000-0000-0000-000000000002" request = gateway.Tool(name="write_file", arguments={"path": "index.html", "content": "test"})