From cb896bb0fa159d81d27c8949a477b05c5cf3e74d Mon Sep 17 00:00:00 2001 From: WebUp Cloud Date: Sat, 7 Nov 2026 19:07:16 +0800 Subject: [PATCH] Recover transient sandbox connections before executing tools --- .../run/webup/platform/SandboxClient.java | 33 ++++- frontend/src/views/Develop.vue | 30 +++-- node-agent/app.py | 116 ++++++++++++------ node-agent/test_gateway.py | 29 +++++ 4 files changed, 156 insertions(+), 52 deletions(-) diff --git a/backend/src/main/java/run/webup/platform/SandboxClient.java b/backend/src/main/java/run/webup/platform/SandboxClient.java index 85da7a7..21a44d7 100644 --- a/backend/src/main/java/run/webup/platform/SandboxClient.java +++ b/backend/src/main/java/run/webup/platform/SandboxClient.java @@ -14,6 +14,7 @@ public class SandboxClient { private final ObjectMapper json; private final HttpClient client = HttpClient.newBuilder() + .version(HttpClient.Version.HTTP_1_1) .connectTimeout(Duration.ofSeconds(5)) .followRedirects(HttpClient.Redirect.NEVER) .build(); @@ -209,8 +210,7 @@ public class SandboxClient { try (var input = response.body()) { byte[] bytes = input.readNBytes(limit + 1); if (bytes.length > limit) throw new ApiError.Failure(502, "工具输出过大"); - if (response.statusCode() != 200) - throw new ApiError.Failure(502, "隔离工具返回 HTTP " + response.statusCode()); + if (response.statusCode() != 200) throw toolFailure(response.statusCode(), bytes); return json.readTree(bytes).toString(); } } catch (ApiError.Failure f) { @@ -220,4 +220,33 @@ public class SandboxClient { throw new ApiError.Failure(502, "隔离工具调用失败,执行结果可能不确定,请核对后处理"); } } + + private ApiError.Failure toolFailure(int status, byte[] bytes) { + String stage = ""; + try { + stage = json.readTree(bytes).path("detail").asText(""); + } catch (Exception ignored) { + // The response may come from an intermediate proxy rather than the gateway. + } + return switch (status) { + case 409 -> new ApiError.Failure(409, "工作区正在执行其他操作,请稍后刷新"); + case 503 -> + new ApiError.Failure( + 503, + switch (stage) { + case "WORKSPACE_PROVISIONING" -> "隔离环境暂时无法准备,请稍后重试"; + case "WORKSPACE_STARTING" -> "隔离环境仍在启动,请稍后刷新"; + case "WORKSPACE_TERMINATED" -> "隔离环境已退出,请联系管理员检查"; + case "WORKSPACE_CONNECTION" -> "暂时无法连接项目的隔离环境,请稍后重试"; + default -> "隔离执行服务暂时不可用,请稍后重试"; + }); + case 504 -> new ApiError.Failure(504, "隔离工具执行超时,请先核对结果再重试"); + default -> + new ApiError.Failure( + 502, + stage.equals("WORKSPACE_RESULT_UNKNOWN") + ? "隔离工具连接中断,请先核对结果再重试" + : "隔离工具返回 HTTP " + status); + }; + } } diff --git a/frontend/src/views/Develop.vue b/frontend/src/views/Develop.vue index 7735e58..b0541a7 100644 --- a/frontend/src/views/Develop.vue +++ b/frontend/src/views/Develop.vue @@ -19,6 +19,8 @@ const fileBusy = ref(false), fileError = ref(""); let fileGeneration = 0; let fileTaskSignature = ""; +let fileRetryCount = 0; +let nextFileRefresh = 0; const logPanel = ref(null); const followLogs = ref(true); let logRequest = 0; @@ -103,11 +105,15 @@ async function load() { .map((t) => `${t.id}:${t.status}:${t.step_count}`) .join("|"); if ( - signature !== fileTaskSignature || - projectTasks.some((t) => t.status === "RUNNING") + execution.value.filesConfigured && + !fileBusy.value && + Date.now() >= nextFileRefresh && + (signature !== fileTaskSignature || + fileRetryCount > 0 || + projectTasks.some((t) => t.status === "RUNNING")) ) { fileTaskSignature = signature; - if (execution.value.filesConfigured && !fileBusy.value) void list(); + void list(); } const id = selected.value?.id; if (id) { @@ -118,13 +124,8 @@ async function load() { ]); if (selected.value?.id === id && request > appliedLogRequest) { appliedLogRequest = request; - const changed = - selected.value.status !== task.status || - events.value.at(-1)?.id !== log.at(-1)?.id; selected.value = task; events.value = log; - if (changed && execution.value.filesConfigured && !fileBusy.value) - void list(); } } } catch (e) { @@ -226,6 +227,8 @@ async function list() { ) { files.value = result.files || []; fileLoaded.value = true; + fileRetryCount = 0; + nextFileRefresh = Date.now() + 9000; if (!editingPath.value && !content.value) { const first = files.value.find( @@ -243,7 +246,11 @@ async function list() { } } } catch (e) { - if (generation === fileGeneration) fileError.value = (e as Error).message; + if (generation === fileGeneration && directory === filePath.value) { + fileError.value = (e as Error).message; + fileRetryCount = Math.min(4, fileRetryCount + 1); + nextFileRefresh = Date.now() + Math.min(30000, 3000 * 2 ** fileRetryCount); + } } finally { fileBusy.value = false; if ( @@ -335,6 +342,9 @@ watch( () => form.projectId, () => { fileGeneration++; + fileTaskSignature = ""; + fileRetryCount = 0; + nextFileRefresh = 0; files.value = []; content.value = ""; editingPath.value = ""; @@ -348,7 +358,7 @@ watch( watch( () => execution.value.filesConfigured, (configured) => { - if (configured) void list(); + if (configured && Date.now() >= nextFileRefresh) void list(); }, ); onMounted(async () => { diff --git a/node-agent/app.py b/node-agent/app.py index a21c69e..dbccb0e 100644 --- a/node-agent/app.py +++ b/node-agent/app.py @@ -6,6 +6,7 @@ The gateway is private infrastructure and requires a cluster service account. import base64 import hmac import json +import logging import os import threading import time @@ -18,6 +19,8 @@ from kubernetes import client, config from kubernetes.client.rest import ApiException from kubernetes.stream import stream from pydantic import BaseModel, Field +from urllib3.exceptions import HTTPError +from websocket import WebSocketException TOKEN = os.environ.get("SANDBOX_TOKEN", "") NAMESPACE = os.environ.get("SANDBOX_NAMESPACE", "platform-sandboxes") @@ -27,6 +30,8 @@ RUNTIME = os.environ.get("SANDBOX_RUNTIME_CLASS", "gvisor") LOCKS: dict[str, threading.Lock] = {} LOCK_GUARD = threading.Lock() MAX_OUTPUT = 100_000 +LOG = logging.getLogger("uvicorn.error") +TRANSIENT_KUBE_STATUSES = {0, 408, 429, 500, 502, 503, 504} @asynccontextmanager @@ -73,12 +78,28 @@ def workspace_name(project: str) -> str: raise HTTPException(400, "Invalid project ID") +def kube_call(stage, operation, *, already_exists_ok=False): + # Use only before sending tool input. A command or write may already have + # happened once stdin is sent, so an interrupted result must not be retried. + for attempt in range(3): + try: + return operation() + except (ApiException, HTTPError) as exc: + status = exc.status if isinstance(exc, ApiException) else 0 + cause = exc.__context__ + upstream = getattr(cause, "status_code", None) + if already_exists_ok and status == 409: + return None + LOG.warning("Sandbox Kubernetes %s failed (attempt=%d, status=%s, cause=%s)", + stage, attempt + 1, upstream or status, type(cause or exc).__name__) + if attempt == 2 or (upstream or status) not in TRANSIENT_KUBE_STATUSES: + raise HTTPException(503, stage) from None + time.sleep(0.25 * 2 ** attempt) + + def create_once(method, **kwargs): - try: - method(**kwargs) - except ApiException as exc: - if exc.status != 409: - raise HTTPException(503, "Kubernetes resource provisioning failed") from None + kube_call("WORKSPACE_PROVISIONING", + lambda: method(**kwargs, _request_timeout=(3, 5)), already_exists_ok=True) def ensure_workspace(project: str, review=False): @@ -129,13 +150,16 @@ def ensure_workspace(project: str, review=False): {"name": "tmp", "emptyDir": {"sizeLimit": "1Gi"}}]}}) deadline = time.monotonic() + 90 while time.monotonic() < deadline: - pod = core.read_namespaced_pod(name, namespace) + pod = kube_call("WORKSPACE_PROVISIONING", + lambda: core.read_namespaced_pod(name, namespace, _request_timeout=(3, 5))) if pod.status.phase == "Running" and pod.status.container_statuses and all(s.ready for s in pod.status.container_statuses): return name if pod.status.phase in ("Failed", "Succeeded"): - raise HTTPException(503, "Sandbox terminated; requires operator inspection") + LOG.warning("Sandbox pod %s is %s", name, pod.status.phase) + raise HTTPException(503, "WORKSPACE_TERMINATED") time.sleep(1) - raise HTTPException(503, "Sandbox not ready; check isolated runtime, image and storage provisioner") + LOG.warning("Sandbox pod %s did not become ready (phase=%s)", name, pod.status.phase) + raise HTTPException(503, "WORKSPACE_STARTING") # This code runs ONLY inside the workspace Pod via Kubernetes exec. @@ -239,35 +263,39 @@ SCRIPT = Path(__file__).with_name('review_tools.py').read_text(encoding='utf-8') def read_exec_output(connection, limit): + try: + return drain_exec_output(connection, limit) + finally: + connection.close() + + +def drain_exec_output(connection, limit): # _preload_content=True lets ApiClient parse JSON and coerce dictionaries back to # Python repr strings. Read raw stdout explicitly and keep stderr separate. deadline = time.monotonic() + 135 chunks, size, error_size = [], 0, 0 + while connection.is_open(): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise HTTPException(504, 'Sandbox execution timed out; result must be checked') + connection.update(timeout=min(1, remaining)) + if connection.peek_stdout(): + chunk = connection.read_stdout() + size += len(chunk.encode('utf-8')) + if size > limit: + raise HTTPException(502, 'Tool output exceeded limit') + chunks.append(chunk) + if connection.peek_stderr(): + error_size += len(connection.read_stderr().encode('utf-8')) + if error_size > 64000: + raise HTTPException(502, 'Sandbox diagnostic output exceeded limit') try: - while connection.is_open(): - remaining = deadline - time.monotonic() - if remaining <= 0: - raise HTTPException(504, 'Sandbox execution timed out; result must be checked') - connection.update(timeout=min(1, remaining)) - if connection.peek_stdout(): - chunk = connection.read_stdout() - size += len(chunk.encode('utf-8')) - if size > limit: - raise HTTPException(502, 'Tool output exceeded limit') - chunks.append(chunk) - if connection.peek_stderr(): - error_size += len(connection.read_stderr().encode('utf-8')) - if error_size > 64000: - raise HTTPException(502, 'Sandbox diagnostic output exceeded limit') - try: - code = connection.returncode - except Exception: - raise HTTPException(502, 'Sandbox completion status is unknown') from None - if code != 0: - raise HTTPException(502, 'Sandbox wrapper did not complete successfully') - return ''.join(chunks) - finally: - connection.close() + code = connection.returncode + except Exception: + raise HTTPException(502, 'WORKSPACE_RESULT_UNKNOWN') from None + if code != 0: + raise HTTPException(502, 'Sandbox wrapper did not complete successfully') + return ''.join(chunks) @app.post("/workspaces/{project}/tools") @@ -304,7 +332,7 @@ def execute_tool(project,request,authorization,review): with LOCK_GUARD: lock = LOCKS.setdefault(('review:' if review else '')+name, threading.Lock()) if not lock.acquire(timeout=1): - raise HTTPException(409, "Workspace is busy") + raise HTTPException(409, "WORKSPACE_BUSY") try: ensure_workspace(project,review) encoded = base64.b64encode(payload).decode() @@ -313,17 +341,25 @@ def execute_tool(project,request,authorization,review): # with concurrent health checks, workspace provisioning, or another project. with client.ApiClient() as api_client: core = client.CoreV1Api(api_client) - connection = stream(core.connect_get_namespaced_pod_exec, name, REVIEW_NAMESPACE if review else NAMESPACE, - container="workspace", command=["timeout", "130", "python3", "-I", "-c", code], - stderr=True, stdin=True, stdout=True, tty=False, _request_timeout=135, - _preload_content=False) - for offset in range(0,len(encoded),16384):connection.write_stdin(encoded[offset:offset+16384]) + connection = kube_call("WORKSPACE_CONNECTION", lambda: stream( + core.connect_get_namespaced_pod_exec, name, REVIEW_NAMESPACE if review else NAMESPACE, + container="workspace", command=["timeout", "130", "python3", "-I", "-c", code], + stderr=True, stdin=True, stdout=True, tty=False, _request_timeout=135, + _preload_content=False)) + try: + for offset in range(0,len(encoded),16384): + connection.write_stdin(encoded[offset:offset+16384]) + except Exception: + connection.close() + raise output = read_exec_output(connection, 16_000_000 if request.name in {'archive_source','review_export'} else MAX_OUTPUT) try: return json.loads(output) except ValueError: raise HTTPException(502, "Sandbox returned invalid output") from None - except ApiException: - raise HTTPException(503, "Kubernetes sandbox operation failed") from None + except (ApiException, HTTPError, OSError, WebSocketException) as exc: + LOG.warning("Sandbox tool result unavailable (workspace=%s, tool=%s, cause=%s)", + name, request.name, type(exc).__name__) + raise HTTPException(502, "WORKSPACE_RESULT_UNKNOWN") from None finally: lock.release() diff --git a/node-agent/test_gateway.py b/node-agent/test_gateway.py index c36fb5d..7cbf9f8 100644 --- a/node-agent/test_gateway.py +++ b/node-agent/test_gateway.py @@ -86,6 +86,35 @@ class GatewayTests(unittest.TestCase): self.assertEqual([], policy["ingress"]) self.assertIn("169.254.0.0/16", policy["egress"][1]["to"][0]["ipBlock"]["except"]) + def test_transient_exec_handshake_recovers_before_tool_input(self): + project = "00000000-0000-0000-0000-000000000002" + request = gateway.Tool(name="write_file", arguments={"path": "index.html", "content": "test"}) + connection = MagicMock() + with (patch.object(gateway, "TOKEN", "x" * 32), + patch.object(gateway, "ensure_workspace"), + patch.object(gateway, "stream", side_effect=[gateway.ApiException(status=503), connection]) as stream, + patch.object(gateway.time, "sleep"), + patch.object(gateway, "drain_exec_output", return_value='{"written":4}')): + self.assertEqual({"written": 4}, gateway.execute_tool(project, request, "Bearer " + "x" * 32, False)) + self.assertEqual(2, stream.call_count) + connection.write_stdin.assert_called_once() + connection.close.assert_called_once() + + def test_interrupted_tool_input_is_not_replayed(self): + project = "00000000-0000-0000-0000-000000000003" + request = gateway.Tool(name="run_command", arguments={"command": "npm install"}) + connection = MagicMock() + connection.write_stdin.side_effect = gateway.WebSocketException("disconnected") + with (patch.object(gateway, "TOKEN", "x" * 32), + patch.object(gateway, "ensure_workspace"), + patch.object(gateway, "stream", return_value=connection) as stream): + with self.assertRaises(gateway.HTTPException) as error: + gateway.execute_tool(project, request, "Bearer " + "x" * 32, False) + self.assertEqual((502, "WORKSPACE_RESULT_UNKNOWN"), (error.exception.status_code, error.exception.detail)) + stream.assert_called_once() + connection.write_stdin.assert_called_once() + connection.close.assert_called_once() + if __name__ == "__main__": unittest.main()