复用已有工作区,修复重复创建耗尽隔离配额 #5

Merged
xhgit merged 1 commits from codex/reuse-sandbox-resources into main 2026-11-08 00:37:33 +08:00
3 changed files with 83 additions and 14 deletions

No files matched your search

@@ -235,6 +235,7 @@ public class SandboxClient {
503,
switch (stage) {
case "WORKSPACE_PROVISIONING" -> "隔离环境暂时无法准备,请稍后重试";
case "WORKSPACE_CAPACITY" -> "隔离环境容量已满,请联系管理员检查集群配额";
case "WORKSPACE_STARTING" -> "隔离环境仍在启动,请稍后刷新";
case "WORKSPACE_TERMINATED" -> "隔离环境已退出,请联系管理员检查";
case "WORKSPACE_CONNECTION" -> "暂时无法连接项目的隔离环境,请稍后重试";
+46 -13
View File
@@ -78,7 +78,16 @@ def workspace_name(project: str) -> str:
raise HTTPException(400, "Invalid project ID")
def kube_call(stage, operation, *, already_exists_ok=False):
def is_quota_denial(exc):
if not isinstance(exc, ApiException) or exc.status != 403:
return False
try:
return "exceeded quota:" in json.loads(exc.body or "{}")["message"]
except (KeyError, TypeError, ValueError):
return False
def kube_call(stage, operation):
# Use only before sending tool input. A command or write may already have
# happened once stdin is sent, so an interrupted result must not be retried.
for attempt in range(3):
@@ -88,18 +97,40 @@ def kube_call(stage, operation, *, already_exists_ok=False):
status = exc.status if isinstance(exc, ApiException) else 0
cause = exc.__context__
upstream = getattr(cause, "status_code", None)
if already_exists_ok and status == 409:
return None
detail = "WORKSPACE_CAPACITY" if is_quota_denial(exc) else stage
LOG.warning("Sandbox Kubernetes %s failed (attempt=%d, status=%s, cause=%s)",
stage, attempt + 1, upstream or status, type(cause or exc).__name__)
detail, attempt + 1, upstream or status, type(cause or exc).__name__)
if attempt == 2 or (upstream or status) not in TRANSIENT_KUBE_STATUSES:
raise HTTPException(503, stage) from None
raise HTTPException(503, detail) from None
time.sleep(0.25 * 2 ** attempt)
def create_once(method, **kwargs):
kube_call("WORKSPACE_PROVISIONING",
lambda: method(**kwargs, _request_timeout=(3, 5)), already_exists_ok=True)
def ensure_resource(read, create, namespace, body):
def get():
return read(body["metadata"]["name"], namespace, _request_timeout=(3, 5))
def find_or_create():
# Admission can charge quota for a duplicate POST before it returns 409.
# Read first on every attempt, including after a lost create response.
try:
return get()
except ApiException as exc:
if exc.status != 404:
raise
try:
return create(namespace=namespace, body=body, _request_timeout=(3, 5))
except ApiException as exc:
if exc.status != 409 and not is_quota_denial(exc):
raise
# Another gateway may have created it between the read and create.
try:
return get()
except ApiException as read_error:
if read_error.status == 404:
raise exc from None
raise
return kube_call("WORKSPACE_PROVISIONING", find_or_create)
def ensure_workspace(project: str, review=False):
@@ -110,7 +141,8 @@ def ensure_workspace(project: str, review=False):
core, net = app.state.core, app.state.net
# The namespace MUST enforce restricted Pod Security Admission and ResourceQuota.
labels = {"platform/workspace": name}
create_once(
ensure_resource(
net.read_namespaced_network_policy,
net.create_namespaced_network_policy,
namespace=namespace,
body={
@@ -128,10 +160,11 @@ def ensure_workspace(project: str, review=False):
"ports": [{"port": 443, "protocol": "TCP"}, {"port": 80, "protocol": "TCP"}]}
]}
})
create_once(core.create_namespaced_persistent_volume_claim, namespace=namespace, body={
ensure_resource(core.read_namespaced_persistent_volume_claim,
core.create_namespaced_persistent_volume_claim, namespace=namespace, body={
"apiVersion": "v1", "kind": "PersistentVolumeClaim", "metadata": {"name": name},
"spec": {"accessModes": ["ReadWriteOnce"], "resources": {"requests": {"storage": "5Gi"}}}})
create_once(core.create_namespaced_pod, namespace=namespace, body={
pod = ensure_resource(core.read_namespaced_pod, core.create_namespaced_pod, namespace=namespace, body={
"apiVersion": "v1", "kind": "Pod", "metadata": {"name": name, "labels": labels},
"spec": {"runtimeClassName": RUNTIME, "automountServiceAccountToken": False,
"restartPolicy": "Always", "enableServiceLinks": False,
@@ -150,14 +183,14 @@ def ensure_workspace(project: str, review=False):
{"name": "tmp", "emptyDir": {"sizeLimit": "1Gi"}}]}})
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
pod = kube_call("WORKSPACE_PROVISIONING",
lambda: core.read_namespaced_pod(name, namespace, _request_timeout=(3, 5)))
if pod.status.phase == "Running" and pod.status.container_statuses and all(s.ready for s in pod.status.container_statuses):
return name
if pod.status.phase in ("Failed", "Succeeded"):
LOG.warning("Sandbox pod %s is %s", name, pod.status.phase)
raise HTTPException(503, "WORKSPACE_TERMINATED")
time.sleep(1)
pod = kube_call("WORKSPACE_PROVISIONING",
lambda: core.read_namespaced_pod(name, namespace, _request_timeout=(3, 5)))
LOG.warning("Sandbox pod %s did not become ready (phase=%s)", name, pod.status.phase)
raise HTTPException(503, "WORKSPACE_STARTING")
+36 -1
View File
@@ -73,7 +73,10 @@ class GatewayTests(unittest.TestCase):
def test_created_workspace_has_isolation_and_persistent_data(self):
core, net = MagicMock(), MagicMock()
core.read_namespaced_pod.return_value = types.SimpleNamespace(status=types.SimpleNamespace(phase="Running", container_statuses=[types.SimpleNamespace(ready=True)]))
net.read_namespaced_network_policy.side_effect = gateway.ApiException(status=404)
core.read_namespaced_persistent_volume_claim.side_effect = gateway.ApiException(status=404)
core.read_namespaced_pod.side_effect = gateway.ApiException(status=404)
core.create_namespaced_pod.return_value = types.SimpleNamespace(status=types.SimpleNamespace(phase="Running", container_statuses=[types.SimpleNamespace(ready=True)]))
with patch.object(gateway.app.state, "core", core, create=True), patch.object(gateway.app.state, "net", net, create=True), patch.object(gateway, "IMAGE", "test/isolated-image:local"):
gateway.ensure_workspace("00000000-0000-0000-0000-000000000001")
pod = core.create_namespaced_pod.call_args.kwargs["body"]["spec"]
@@ -86,6 +89,38 @@ class GatewayTests(unittest.TestCase):
self.assertEqual([], policy["ingress"])
self.assertIn("169.254.0.0/16", policy["egress"][1]["to"][0]["ipBlock"]["except"])
def test_existing_workspace_does_not_submit_creates_to_admission(self):
core, net = MagicMock(), MagicMock()
core.read_namespaced_pod.return_value = types.SimpleNamespace(
status=types.SimpleNamespace(phase="Running", container_statuses=[types.SimpleNamespace(ready=True)])
)
project = "00000000-0000-0000-0000-000000000004"
with (patch.object(gateway.app.state, "core", core, create=True),
patch.object(gateway.app.state, "net", net, create=True)):
for _ in range(3):
self.assertEqual("dev-00000000000000000000000000000004", gateway.ensure_workspace(project))
net.create_namespaced_network_policy.assert_not_called()
core.create_namespaced_persistent_volume_claim.assert_not_called()
core.create_namespaced_pod.assert_not_called()
def test_create_with_lost_response_checks_for_resource_before_retry(self):
resource = object()
read = MagicMock(side_effect=[gateway.ApiException(status=404), resource])
create = MagicMock(side_effect=gateway.ApiException(status=503))
with patch.object(gateway.time, "sleep"):
self.assertIs(resource, gateway.ensure_resource(read, create, "qa", {"metadata": {"name": "workspace"}}))
create.assert_called_once()
def test_quota_denial_of_new_resource_reports_capacity(self):
read = MagicMock(side_effect=gateway.ApiException(status=404))
denial = gateway.ApiException(status=403)
denial.body = json.dumps({"message": 'pods "workspace" is forbidden: exceeded quota: quota, requested: pods=1'})
create = MagicMock(side_effect=denial)
with self.assertRaises(gateway.HTTPException) as error:
gateway.ensure_resource(read, create, "qa", {"metadata": {"name": "workspace"}})
self.assertEqual((503, "WORKSPACE_CAPACITY"), (error.exception.status_code, error.exception.detail))
create.assert_called_once()
def test_transient_exec_handshake_recovers_before_tool_input(self):
project = "00000000-0000-0000-0000-000000000002"
request = gateway.Tool(name="write_file", arguments={"path": "index.html", "content": "test"})