Recover transient sandbox connections before executing tools
Verify control plane / verify (push) Canceled after 0s
Verify control plane / verify (pull_request) Canceled after 0s

This commit is contained in:
WebUp Cloud committed 2026-11-07 19:07:16 +08:00
1 parent 4f2cf2fc87
commit cb896bb0fa
4 files changed
+156 -52

No files matched your search

@@ -14,6 +14,7 @@ public class SandboxClient {
private final ObjectMapper json; private final ObjectMapper json;
private final HttpClient client = private final HttpClient client =
HttpClient.newBuilder() HttpClient.newBuilder()
.version(HttpClient.Version.HTTP_1_1)
.connectTimeout(Duration.ofSeconds(5)) .connectTimeout(Duration.ofSeconds(5))
.followRedirects(HttpClient.Redirect.NEVER) .followRedirects(HttpClient.Redirect.NEVER)
.build(); .build();
@@ -209,8 +210,7 @@ public class SandboxClient {
try (var input = response.body()) { try (var input = response.body()) {
byte[] bytes = input.readNBytes(limit + 1); byte[] bytes = input.readNBytes(limit + 1);
if (bytes.length > limit) throw new ApiError.Failure(502, "工具输出过大"); if (bytes.length > limit) throw new ApiError.Failure(502, "工具输出过大");
if (response.statusCode() != 200) if (response.statusCode() != 200) throw toolFailure(response.statusCode(), bytes);
throw new ApiError.Failure(502, "隔离工具返回 HTTP " + response.statusCode());
return json.readTree(bytes).toString(); return json.readTree(bytes).toString();
} }
} catch (ApiError.Failure f) { } catch (ApiError.Failure f) {
@@ -220,4 +220,33 @@ public class SandboxClient {
throw new ApiError.Failure(502, "隔离工具调用失败,执行结果可能不确定,请核对后处理"); throw new ApiError.Failure(502, "隔离工具调用失败,执行结果可能不确定,请核对后处理");
} }
} }
private ApiError.Failure toolFailure(int status, byte[] bytes) {
String stage = "";
try {
stage = json.readTree(bytes).path("detail").asText("");
} catch (Exception ignored) {
// The response may come from an intermediate proxy rather than the gateway.
}
return switch (status) {
case 409 -> new ApiError.Failure(409, "工作区正在执行其他操作,请稍后刷新");
case 503 ->
new ApiError.Failure(
503,
switch (stage) {
case "WORKSPACE_PROVISIONING" -> "隔离环境暂时无法准备,请稍后重试";
case "WORKSPACE_STARTING" -> "隔离环境仍在启动,请稍后刷新";
case "WORKSPACE_TERMINATED" -> "隔离环境已退出,请联系管理员检查";
case "WORKSPACE_CONNECTION" -> "暂时无法连接项目的隔离环境,请稍后重试";
default -> "隔离执行服务暂时不可用,请稍后重试";
});
case 504 -> new ApiError.Failure(504, "隔离工具执行超时,请先核对结果再重试");
default ->
new ApiError.Failure(
502,
stage.equals("WORKSPACE_RESULT_UNKNOWN")
? "隔离工具连接中断,请先核对结果再重试"
: "隔离工具返回 HTTP " + status);
};
}
} }
+20 -10
View File
@@ -19,6 +19,8 @@ const fileBusy = ref(false),
fileError = ref(""); fileError = ref("");
let fileGeneration = 0; let fileGeneration = 0;
let fileTaskSignature = ""; let fileTaskSignature = "";
let fileRetryCount = 0;
let nextFileRefresh = 0;
const logPanel = ref<HTMLElement | null>(null); const logPanel = ref<HTMLElement | null>(null);
const followLogs = ref(true); const followLogs = ref(true);
let logRequest = 0; let logRequest = 0;
@@ -103,11 +105,15 @@ async function load() {
.map((t) => `${t.id}:${t.status}:${t.step_count}`) .map((t) => `${t.id}:${t.status}:${t.step_count}`)
.join("|"); .join("|");
if ( if (
signature !== fileTaskSignature || execution.value.filesConfigured &&
projectTasks.some((t) => t.status === "RUNNING") !fileBusy.value &&
Date.now() >= nextFileRefresh &&
(signature !== fileTaskSignature ||
fileRetryCount > 0 ||
projectTasks.some((t) => t.status === "RUNNING"))
) { ) {
fileTaskSignature = signature; fileTaskSignature = signature;
if (execution.value.filesConfigured && !fileBusy.value) void list(); void list();
} }
const id = selected.value?.id; const id = selected.value?.id;
if (id) { if (id) {
@@ -118,13 +124,8 @@ async function load() {
]); ]);
if (selected.value?.id === id && request > appliedLogRequest) { if (selected.value?.id === id && request > appliedLogRequest) {
appliedLogRequest = request; appliedLogRequest = request;
const changed =
selected.value.status !== task.status ||
events.value.at(-1)?.id !== log.at(-1)?.id;
selected.value = task; selected.value = task;
events.value = log; events.value = log;
if (changed && execution.value.filesConfigured && !fileBusy.value)
void list();
} }
} }
} catch (e) { } catch (e) {
@@ -226,6 +227,8 @@ async function list() {
) { ) {
files.value = result.files || []; files.value = result.files || [];
fileLoaded.value = true; fileLoaded.value = true;
fileRetryCount = 0;
nextFileRefresh = Date.now() + 9000;
if (!editingPath.value && !content.value) { if (!editingPath.value && !content.value) {
const first = const first =
files.value.find( files.value.find(
@@ -243,7 +246,11 @@ async function list() {
} }
} }
} catch (e) { } catch (e) {
if (generation === fileGeneration) fileError.value = (e as Error).message; if (generation === fileGeneration && directory === filePath.value) {
fileError.value = (e as Error).message;
fileRetryCount = Math.min(4, fileRetryCount + 1);
nextFileRefresh = Date.now() + Math.min(30000, 3000 * 2 ** fileRetryCount);
}
} finally { } finally {
fileBusy.value = false; fileBusy.value = false;
if ( if (
@@ -335,6 +342,9 @@ watch(
() => form.projectId, () => form.projectId,
() => { () => {
fileGeneration++; fileGeneration++;
fileTaskSignature = "";
fileRetryCount = 0;
nextFileRefresh = 0;
files.value = []; files.value = [];
content.value = ""; content.value = "";
editingPath.value = ""; editingPath.value = "";
@@ -348,7 +358,7 @@ watch(
watch( watch(
() => execution.value.filesConfigured, () => execution.value.filesConfigured,
(configured) => { (configured) => {
if (configured) void list(); if (configured && Date.now() >= nextFileRefresh) void list();
}, },
); );
onMounted(async () => { onMounted(async () => {
+76 -40
View File
@@ -6,6 +6,7 @@ The gateway is private infrastructure and requires a cluster service account.
import base64 import base64
import hmac import hmac
import json import json
import logging
import os import os
import threading import threading
import time import time
@@ -18,6 +19,8 @@ from kubernetes import client, config
from kubernetes.client.rest import ApiException from kubernetes.client.rest import ApiException
from kubernetes.stream import stream from kubernetes.stream import stream
from pydantic import BaseModel, Field from pydantic import BaseModel, Field
from urllib3.exceptions import HTTPError
from websocket import WebSocketException
TOKEN = os.environ.get("SANDBOX_TOKEN", "") TOKEN = os.environ.get("SANDBOX_TOKEN", "")
NAMESPACE = os.environ.get("SANDBOX_NAMESPACE", "platform-sandboxes") NAMESPACE = os.environ.get("SANDBOX_NAMESPACE", "platform-sandboxes")
@@ -27,6 +30,8 @@ RUNTIME = os.environ.get("SANDBOX_RUNTIME_CLASS", "gvisor")
LOCKS: dict[str, threading.Lock] = {} LOCKS: dict[str, threading.Lock] = {}
LOCK_GUARD = threading.Lock() LOCK_GUARD = threading.Lock()
MAX_OUTPUT = 100_000 MAX_OUTPUT = 100_000
LOG = logging.getLogger("uvicorn.error")
TRANSIENT_KUBE_STATUSES = {0, 408, 429, 500, 502, 503, 504}
@asynccontextmanager @asynccontextmanager
@@ -73,12 +78,28 @@ def workspace_name(project: str) -> str:
raise HTTPException(400, "Invalid project ID") raise HTTPException(400, "Invalid project ID")
def kube_call(stage, operation, *, already_exists_ok=False):
# Use only before sending tool input. A command or write may already have
# happened once stdin is sent, so an interrupted result must not be retried.
for attempt in range(3):
try:
return operation()
except (ApiException, HTTPError) as exc:
status = exc.status if isinstance(exc, ApiException) else 0
cause = exc.__context__
upstream = getattr(cause, "status_code", None)
if already_exists_ok and status == 409:
return None
LOG.warning("Sandbox Kubernetes %s failed (attempt=%d, status=%s, cause=%s)",
stage, attempt + 1, upstream or status, type(cause or exc).__name__)
if attempt == 2 or (upstream or status) not in TRANSIENT_KUBE_STATUSES:
raise HTTPException(503, stage) from None
time.sleep(0.25 * 2 ** attempt)
def create_once(method, **kwargs): def create_once(method, **kwargs):
try: kube_call("WORKSPACE_PROVISIONING",
method(**kwargs) lambda: method(**kwargs, _request_timeout=(3, 5)), already_exists_ok=True)
except ApiException as exc:
if exc.status != 409:
raise HTTPException(503, "Kubernetes resource provisioning failed") from None
def ensure_workspace(project: str, review=False): def ensure_workspace(project: str, review=False):
@@ -129,13 +150,16 @@ def ensure_workspace(project: str, review=False):
{"name": "tmp", "emptyDir": {"sizeLimit": "1Gi"}}]}}) {"name": "tmp", "emptyDir": {"sizeLimit": "1Gi"}}]}})
deadline = time.monotonic() + 90 deadline = time.monotonic() + 90
while time.monotonic() < deadline: while time.monotonic() < deadline:
pod = core.read_namespaced_pod(name, namespace) pod = kube_call("WORKSPACE_PROVISIONING",
lambda: core.read_namespaced_pod(name, namespace, _request_timeout=(3, 5)))
if pod.status.phase == "Running" and pod.status.container_statuses and all(s.ready for s in pod.status.container_statuses): if pod.status.phase == "Running" and pod.status.container_statuses and all(s.ready for s in pod.status.container_statuses):
return name return name
if pod.status.phase in ("Failed", "Succeeded"): if pod.status.phase in ("Failed", "Succeeded"):
raise HTTPException(503, "Sandbox terminated; requires operator inspection") LOG.warning("Sandbox pod %s is %s", name, pod.status.phase)
raise HTTPException(503, "WORKSPACE_TERMINATED")
time.sleep(1) time.sleep(1)
raise HTTPException(503, "Sandbox not ready; check isolated runtime, image and storage provisioner") LOG.warning("Sandbox pod %s did not become ready (phase=%s)", name, pod.status.phase)
raise HTTPException(503, "WORKSPACE_STARTING")
# This code runs ONLY inside the workspace Pod via Kubernetes exec. # This code runs ONLY inside the workspace Pod via Kubernetes exec.
@@ -239,35 +263,39 @@ SCRIPT = Path(__file__).with_name('review_tools.py').read_text(encoding='utf-8')
def read_exec_output(connection, limit): def read_exec_output(connection, limit):
try:
return drain_exec_output(connection, limit)
finally:
connection.close()
def drain_exec_output(connection, limit):
# _preload_content=True lets ApiClient parse JSON and coerce dictionaries back to # _preload_content=True lets ApiClient parse JSON and coerce dictionaries back to
# Python repr strings. Read raw stdout explicitly and keep stderr separate. # Python repr strings. Read raw stdout explicitly and keep stderr separate.
deadline = time.monotonic() + 135 deadline = time.monotonic() + 135
chunks, size, error_size = [], 0, 0 chunks, size, error_size = [], 0, 0
while connection.is_open():
remaining = deadline - time.monotonic()
if remaining <= 0:
raise HTTPException(504, 'Sandbox execution timed out; result must be checked')
connection.update(timeout=min(1, remaining))
if connection.peek_stdout():
chunk = connection.read_stdout()
size += len(chunk.encode('utf-8'))
if size > limit:
raise HTTPException(502, 'Tool output exceeded limit')
chunks.append(chunk)
if connection.peek_stderr():
error_size += len(connection.read_stderr().encode('utf-8'))
if error_size > 64000:
raise HTTPException(502, 'Sandbox diagnostic output exceeded limit')
try: try:
while connection.is_open(): code = connection.returncode
remaining = deadline - time.monotonic() except Exception:
if remaining <= 0: raise HTTPException(502, 'WORKSPACE_RESULT_UNKNOWN') from None
raise HTTPException(504, 'Sandbox execution timed out; result must be checked') if code != 0:
connection.update(timeout=min(1, remaining)) raise HTTPException(502, 'Sandbox wrapper did not complete successfully')
if connection.peek_stdout(): return ''.join(chunks)
chunk = connection.read_stdout()
size += len(chunk.encode('utf-8'))
if size > limit:
raise HTTPException(502, 'Tool output exceeded limit')
chunks.append(chunk)
if connection.peek_stderr():
error_size += len(connection.read_stderr().encode('utf-8'))
if error_size > 64000:
raise HTTPException(502, 'Sandbox diagnostic output exceeded limit')
try:
code = connection.returncode
except Exception:
raise HTTPException(502, 'Sandbox completion status is unknown') from None
if code != 0:
raise HTTPException(502, 'Sandbox wrapper did not complete successfully')
return ''.join(chunks)
finally:
connection.close()
@app.post("/workspaces/{project}/tools") @app.post("/workspaces/{project}/tools")
@@ -304,7 +332,7 @@ def execute_tool(project,request,authorization,review):
with LOCK_GUARD: with LOCK_GUARD:
lock = LOCKS.setdefault(('review:' if review else '')+name, threading.Lock()) lock = LOCKS.setdefault(('review:' if review else '')+name, threading.Lock())
if not lock.acquire(timeout=1): if not lock.acquire(timeout=1):
raise HTTPException(409, "Workspace is busy") raise HTTPException(409, "WORKSPACE_BUSY")
try: try:
ensure_workspace(project,review) ensure_workspace(project,review)
encoded = base64.b64encode(payload).decode() encoded = base64.b64encode(payload).decode()
@@ -313,17 +341,25 @@ def execute_tool(project,request,authorization,review):
# with concurrent health checks, workspace provisioning, or another project. # with concurrent health checks, workspace provisioning, or another project.
with client.ApiClient() as api_client: with client.ApiClient() as api_client:
core = client.CoreV1Api(api_client) core = client.CoreV1Api(api_client)
connection = stream(core.connect_get_namespaced_pod_exec, name, REVIEW_NAMESPACE if review else NAMESPACE, connection = kube_call("WORKSPACE_CONNECTION", lambda: stream(
container="workspace", command=["timeout", "130", "python3", "-I", "-c", code], core.connect_get_namespaced_pod_exec, name, REVIEW_NAMESPACE if review else NAMESPACE,
stderr=True, stdin=True, stdout=True, tty=False, _request_timeout=135, container="workspace", command=["timeout", "130", "python3", "-I", "-c", code],
_preload_content=False) stderr=True, stdin=True, stdout=True, tty=False, _request_timeout=135,
for offset in range(0,len(encoded),16384):connection.write_stdin(encoded[offset:offset+16384]) _preload_content=False))
try:
for offset in range(0,len(encoded),16384):
connection.write_stdin(encoded[offset:offset+16384])
except Exception:
connection.close()
raise
output = read_exec_output(connection, 16_000_000 if request.name in {'archive_source','review_export'} else MAX_OUTPUT) output = read_exec_output(connection, 16_000_000 if request.name in {'archive_source','review_export'} else MAX_OUTPUT)
try: try:
return json.loads(output) return json.loads(output)
except ValueError: except ValueError:
raise HTTPException(502, "Sandbox returned invalid output") from None raise HTTPException(502, "Sandbox returned invalid output") from None
except ApiException: except (ApiException, HTTPError, OSError, WebSocketException) as exc:
raise HTTPException(503, "Kubernetes sandbox operation failed") from None LOG.warning("Sandbox tool result unavailable (workspace=%s, tool=%s, cause=%s)",
name, request.name, type(exc).__name__)
raise HTTPException(502, "WORKSPACE_RESULT_UNKNOWN") from None
finally: finally:
lock.release() lock.release()
+29
View File
@@ -86,6 +86,35 @@ class GatewayTests(unittest.TestCase):
self.assertEqual([], policy["ingress"]) self.assertEqual([], policy["ingress"])
self.assertIn("169.254.0.0/16", policy["egress"][1]["to"][0]["ipBlock"]["except"]) self.assertIn("169.254.0.0/16", policy["egress"][1]["to"][0]["ipBlock"]["except"])
def test_transient_exec_handshake_recovers_before_tool_input(self):
project = "00000000-0000-0000-0000-000000000002"
request = gateway.Tool(name="write_file", arguments={"path": "index.html", "content": "test"})
connection = MagicMock()
with (patch.object(gateway, "TOKEN", "x" * 32),
patch.object(gateway, "ensure_workspace"),
patch.object(gateway, "stream", side_effect=[gateway.ApiException(status=503), connection]) as stream,
patch.object(gateway.time, "sleep"),
patch.object(gateway, "drain_exec_output", return_value='{"written":4}')):
self.assertEqual({"written": 4}, gateway.execute_tool(project, request, "Bearer " + "x" * 32, False))
self.assertEqual(2, stream.call_count)
connection.write_stdin.assert_called_once()
connection.close.assert_called_once()
def test_interrupted_tool_input_is_not_replayed(self):
project = "00000000-0000-0000-0000-000000000003"
request = gateway.Tool(name="run_command", arguments={"command": "npm install"})
connection = MagicMock()
connection.write_stdin.side_effect = gateway.WebSocketException("disconnected")
with (patch.object(gateway, "TOKEN", "x" * 32),
patch.object(gateway, "ensure_workspace"),
patch.object(gateway, "stream", return_value=connection) as stream):
with self.assertRaises(gateway.HTTPException) as error:
gateway.execute_tool(project, request, "Bearer " + "x" * 32, False)
self.assertEqual((502, "WORKSPACE_RESULT_UNKNOWN"), (error.exception.status_code, error.exception.detail))
stream.assert_called_once()
connection.write_stdin.assert_called_once()
connection.close.assert_called_once()
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()