forked from xhgit/webup-cloud
Merge pull request '复用已有工作区,修复重复创建耗尽隔离配额' (#5) from codex/reuse-sandbox-resources into main
Reviewed-on: http://git.xhmcl.com/xhgit/webup-cloud/pulls/5
This commit is contained in:
commit
ed6ddc007d
3 files changed
+83
-14
No files matched your search
@@ -235,6 +235,7 @@ public class SandboxClient {
|
||||
503,
|
||||
switch (stage) {
|
||||
case "WORKSPACE_PROVISIONING" -> "隔离环境暂时无法准备,请稍后重试";
|
||||
case "WORKSPACE_CAPACITY" -> "隔离环境容量已满,请联系管理员检查集群配额";
|
||||
case "WORKSPACE_STARTING" -> "隔离环境仍在启动,请稍后刷新";
|
||||
case "WORKSPACE_TERMINATED" -> "隔离环境已退出,请联系管理员检查";
|
||||
case "WORKSPACE_CONNECTION" -> "暂时无法连接项目的隔离环境,请稍后重试";
|
||||
|
||||
+46
-13
@@ -78,7 +78,16 @@ def workspace_name(project: str) -> str:
|
||||
raise HTTPException(400, "Invalid project ID")
|
||||
|
||||
|
||||
def kube_call(stage, operation, *, already_exists_ok=False):
|
||||
def is_quota_denial(exc):
|
||||
if not isinstance(exc, ApiException) or exc.status != 403:
|
||||
return False
|
||||
try:
|
||||
return "exceeded quota:" in json.loads(exc.body or "{}")["message"]
|
||||
except (KeyError, TypeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def kube_call(stage, operation):
|
||||
# Use only before sending tool input. A command or write may already have
|
||||
# happened once stdin is sent, so an interrupted result must not be retried.
|
||||
for attempt in range(3):
|
||||
@@ -88,18 +97,40 @@ def kube_call(stage, operation, *, already_exists_ok=False):
|
||||
status = exc.status if isinstance(exc, ApiException) else 0
|
||||
cause = exc.__context__
|
||||
upstream = getattr(cause, "status_code", None)
|
||||
if already_exists_ok and status == 409:
|
||||
return None
|
||||
detail = "WORKSPACE_CAPACITY" if is_quota_denial(exc) else stage
|
||||
LOG.warning("Sandbox Kubernetes %s failed (attempt=%d, status=%s, cause=%s)",
|
||||
stage, attempt + 1, upstream or status, type(cause or exc).__name__)
|
||||
detail, attempt + 1, upstream or status, type(cause or exc).__name__)
|
||||
if attempt == 2 or (upstream or status) not in TRANSIENT_KUBE_STATUSES:
|
||||
raise HTTPException(503, stage) from None
|
||||
raise HTTPException(503, detail) from None
|
||||
time.sleep(0.25 * 2 ** attempt)
|
||||
|
||||
|
||||
def create_once(method, **kwargs):
|
||||
kube_call("WORKSPACE_PROVISIONING",
|
||||
lambda: method(**kwargs, _request_timeout=(3, 5)), already_exists_ok=True)
|
||||
def ensure_resource(read, create, namespace, body):
|
||||
def get():
|
||||
return read(body["metadata"]["name"], namespace, _request_timeout=(3, 5))
|
||||
|
||||
def find_or_create():
|
||||
# Admission can charge quota for a duplicate POST before it returns 409.
|
||||
# Read first on every attempt, including after a lost create response.
|
||||
try:
|
||||
return get()
|
||||
except ApiException as exc:
|
||||
if exc.status != 404:
|
||||
raise
|
||||
try:
|
||||
return create(namespace=namespace, body=body, _request_timeout=(3, 5))
|
||||
except ApiException as exc:
|
||||
if exc.status != 409 and not is_quota_denial(exc):
|
||||
raise
|
||||
# Another gateway may have created it between the read and create.
|
||||
try:
|
||||
return get()
|
||||
except ApiException as read_error:
|
||||
if read_error.status == 404:
|
||||
raise exc from None
|
||||
raise
|
||||
|
||||
return kube_call("WORKSPACE_PROVISIONING", find_or_create)
|
||||
|
||||
|
||||
def ensure_workspace(project: str, review=False):
|
||||
@@ -110,7 +141,8 @@ def ensure_workspace(project: str, review=False):
|
||||
core, net = app.state.core, app.state.net
|
||||
# The namespace MUST enforce restricted Pod Security Admission and ResourceQuota.
|
||||
labels = {"platform/workspace": name}
|
||||
create_once(
|
||||
ensure_resource(
|
||||
net.read_namespaced_network_policy,
|
||||
net.create_namespaced_network_policy,
|
||||
namespace=namespace,
|
||||
body={
|
||||
@@ -128,10 +160,11 @@ def ensure_workspace(project: str, review=False):
|
||||
"ports": [{"port": 443, "protocol": "TCP"}, {"port": 80, "protocol": "TCP"}]}
|
||||
]}
|
||||
})
|
||||
create_once(core.create_namespaced_persistent_volume_claim, namespace=namespace, body={
|
||||
ensure_resource(core.read_namespaced_persistent_volume_claim,
|
||||
core.create_namespaced_persistent_volume_claim, namespace=namespace, body={
|
||||
"apiVersion": "v1", "kind": "PersistentVolumeClaim", "metadata": {"name": name},
|
||||
"spec": {"accessModes": ["ReadWriteOnce"], "resources": {"requests": {"storage": "5Gi"}}}})
|
||||
create_once(core.create_namespaced_pod, namespace=namespace, body={
|
||||
pod = ensure_resource(core.read_namespaced_pod, core.create_namespaced_pod, namespace=namespace, body={
|
||||
"apiVersion": "v1", "kind": "Pod", "metadata": {"name": name, "labels": labels},
|
||||
"spec": {"runtimeClassName": RUNTIME, "automountServiceAccountToken": False,
|
||||
"restartPolicy": "Always", "enableServiceLinks": False,
|
||||
@@ -150,14 +183,14 @@ def ensure_workspace(project: str, review=False):
|
||||
{"name": "tmp", "emptyDir": {"sizeLimit": "1Gi"}}]}})
|
||||
deadline = time.monotonic() + 90
|
||||
while time.monotonic() < deadline:
|
||||
pod = kube_call("WORKSPACE_PROVISIONING",
|
||||
lambda: core.read_namespaced_pod(name, namespace, _request_timeout=(3, 5)))
|
||||
if pod.status.phase == "Running" and pod.status.container_statuses and all(s.ready for s in pod.status.container_statuses):
|
||||
return name
|
||||
if pod.status.phase in ("Failed", "Succeeded"):
|
||||
LOG.warning("Sandbox pod %s is %s", name, pod.status.phase)
|
||||
raise HTTPException(503, "WORKSPACE_TERMINATED")
|
||||
time.sleep(1)
|
||||
pod = kube_call("WORKSPACE_PROVISIONING",
|
||||
lambda: core.read_namespaced_pod(name, namespace, _request_timeout=(3, 5)))
|
||||
LOG.warning("Sandbox pod %s did not become ready (phase=%s)", name, pod.status.phase)
|
||||
raise HTTPException(503, "WORKSPACE_STARTING")
|
||||
|
||||
|
||||
@@ -73,7 +73,10 @@ class GatewayTests(unittest.TestCase):
|
||||
|
||||
def test_created_workspace_has_isolation_and_persistent_data(self):
|
||||
core, net = MagicMock(), MagicMock()
|
||||
core.read_namespaced_pod.return_value = types.SimpleNamespace(status=types.SimpleNamespace(phase="Running", container_statuses=[types.SimpleNamespace(ready=True)]))
|
||||
net.read_namespaced_network_policy.side_effect = gateway.ApiException(status=404)
|
||||
core.read_namespaced_persistent_volume_claim.side_effect = gateway.ApiException(status=404)
|
||||
core.read_namespaced_pod.side_effect = gateway.ApiException(status=404)
|
||||
core.create_namespaced_pod.return_value = types.SimpleNamespace(status=types.SimpleNamespace(phase="Running", container_statuses=[types.SimpleNamespace(ready=True)]))
|
||||
with patch.object(gateway.app.state, "core", core, create=True), patch.object(gateway.app.state, "net", net, create=True), patch.object(gateway, "IMAGE", "test/isolated-image:local"):
|
||||
gateway.ensure_workspace("00000000-0000-0000-0000-000000000001")
|
||||
pod = core.create_namespaced_pod.call_args.kwargs["body"]["spec"]
|
||||
@@ -86,6 +89,38 @@ class GatewayTests(unittest.TestCase):
|
||||
self.assertEqual([], policy["ingress"])
|
||||
self.assertIn("169.254.0.0/16", policy["egress"][1]["to"][0]["ipBlock"]["except"])
|
||||
|
||||
def test_existing_workspace_does_not_submit_creates_to_admission(self):
|
||||
core, net = MagicMock(), MagicMock()
|
||||
core.read_namespaced_pod.return_value = types.SimpleNamespace(
|
||||
status=types.SimpleNamespace(phase="Running", container_statuses=[types.SimpleNamespace(ready=True)])
|
||||
)
|
||||
project = "00000000-0000-0000-0000-000000000004"
|
||||
with (patch.object(gateway.app.state, "core", core, create=True),
|
||||
patch.object(gateway.app.state, "net", net, create=True)):
|
||||
for _ in range(3):
|
||||
self.assertEqual("dev-00000000000000000000000000000004", gateway.ensure_workspace(project))
|
||||
net.create_namespaced_network_policy.assert_not_called()
|
||||
core.create_namespaced_persistent_volume_claim.assert_not_called()
|
||||
core.create_namespaced_pod.assert_not_called()
|
||||
|
||||
def test_create_with_lost_response_checks_for_resource_before_retry(self):
|
||||
resource = object()
|
||||
read = MagicMock(side_effect=[gateway.ApiException(status=404), resource])
|
||||
create = MagicMock(side_effect=gateway.ApiException(status=503))
|
||||
with patch.object(gateway.time, "sleep"):
|
||||
self.assertIs(resource, gateway.ensure_resource(read, create, "qa", {"metadata": {"name": "workspace"}}))
|
||||
create.assert_called_once()
|
||||
|
||||
def test_quota_denial_of_new_resource_reports_capacity(self):
|
||||
read = MagicMock(side_effect=gateway.ApiException(status=404))
|
||||
denial = gateway.ApiException(status=403)
|
||||
denial.body = json.dumps({"message": 'pods "workspace" is forbidden: exceeded quota: quota, requested: pods=1'})
|
||||
create = MagicMock(side_effect=denial)
|
||||
with self.assertRaises(gateway.HTTPException) as error:
|
||||
gateway.ensure_resource(read, create, "qa", {"metadata": {"name": "workspace"}})
|
||||
self.assertEqual((503, "WORKSPACE_CAPACITY"), (error.exception.status_code, error.exception.detail))
|
||||
create.assert_called_once()
|
||||
|
||||
def test_transient_exec_handshake_recovers_before_tool_input(self):
|
||||
project = "00000000-0000-0000-0000-000000000002"
|
||||
request = gateway.Tool(name="write_file", arguments={"path": "index.html", "content": "test"})
|
||||
|
||||
Reference in new issue
Block a user