Recover transient sandbox connections before executing tools

This commit is contained in:
WebUp Cloud committed 2026-11-07 19:07:16 +08:00
1 parent 4f2cf2fc87
commit cb896bb0fa
4 files changed
+156 -52

No files matched your search

@@ -14,6 +14,7 @@ public class SandboxClient {
private final ObjectMapper json;
private final HttpClient client =
HttpClient.newBuilder()
.version(HttpClient.Version.HTTP_1_1)
.connectTimeout(Duration.ofSeconds(5))
.followRedirects(HttpClient.Redirect.NEVER)
.build();
@@ -209,8 +210,7 @@ public class SandboxClient {
try (var input = response.body()) {
byte[] bytes = input.readNBytes(limit + 1);
if (bytes.length > limit) throw new ApiError.Failure(502, "工具输出过大");
if (response.statusCode() != 200)
throw new ApiError.Failure(502, "隔离工具返回 HTTP " + response.statusCode());
if (response.statusCode() != 200) throw toolFailure(response.statusCode(), bytes);
return json.readTree(bytes).toString();
}
} catch (ApiError.Failure f) {
@@ -220,4 +220,33 @@ public class SandboxClient {
throw new ApiError.Failure(502, "隔离工具调用失败,执行结果可能不确定,请核对后处理");
}
}
private ApiError.Failure toolFailure(int status, byte[] bytes) {
String stage = "";
try {
stage = json.readTree(bytes).path("detail").asText("");
} catch (Exception ignored) {
// The response may come from an intermediate proxy rather than the gateway.
}
return switch (status) {
case 409 -> new ApiError.Failure(409, "工作区正在执行其他操作,请稍后刷新");
case 503 ->
new ApiError.Failure(
503,
switch (stage) {
case "WORKSPACE_PROVISIONING" -> "隔离环境暂时无法准备,请稍后重试";
case "WORKSPACE_STARTING" -> "隔离环境仍在启动,请稍后刷新";
case "WORKSPACE_TERMINATED" -> "隔离环境已退出,请联系管理员检查";
case "WORKSPACE_CONNECTION" -> "暂时无法连接项目的隔离环境,请稍后重试";
default -> "隔离执行服务暂时不可用,请稍后重试";
});
case 504 -> new ApiError.Failure(504, "隔离工具执行超时,请先核对结果再重试");
default ->
new ApiError.Failure(
502,
stage.equals("WORKSPACE_RESULT_UNKNOWN")
? "隔离工具连接中断,请先核对结果再重试"
: "隔离工具返回 HTTP " + status);
};
}
}
+20 -10
View File
@@ -19,6 +19,8 @@ const fileBusy = ref(false),
fileError = ref("");
let fileGeneration = 0;
let fileTaskSignature = "";
let fileRetryCount = 0;
let nextFileRefresh = 0;
const logPanel = ref<HTMLElement | null>(null);
const followLogs = ref(true);
let logRequest = 0;
@@ -103,11 +105,15 @@ async function load() {
.map((t) => `${t.id}:${t.status}:${t.step_count}`)
.join("|");
if (
signature !== fileTaskSignature ||
projectTasks.some((t) => t.status === "RUNNING")
execution.value.filesConfigured &&
!fileBusy.value &&
Date.now() >= nextFileRefresh &&
(signature !== fileTaskSignature ||
fileRetryCount > 0 ||
projectTasks.some((t) => t.status === "RUNNING"))
) {
fileTaskSignature = signature;
if (execution.value.filesConfigured && !fileBusy.value) void list();
void list();
}
const id = selected.value?.id;
if (id) {
@@ -118,13 +124,8 @@ async function load() {
]);
if (selected.value?.id === id && request > appliedLogRequest) {
appliedLogRequest = request;
const changed =
selected.value.status !== task.status ||
events.value.at(-1)?.id !== log.at(-1)?.id;
selected.value = task;
events.value = log;
if (changed && execution.value.filesConfigured && !fileBusy.value)
void list();
}
}
} catch (e) {
@@ -226,6 +227,8 @@ async function list() {
) {
files.value = result.files || [];
fileLoaded.value = true;
fileRetryCount = 0;
nextFileRefresh = Date.now() + 9000;
if (!editingPath.value && !content.value) {
const first =
files.value.find(
@@ -243,7 +246,11 @@ async function list() {
}
}
} catch (e) {
if (generation === fileGeneration) fileError.value = (e as Error).message;
if (generation === fileGeneration && directory === filePath.value) {
fileError.value = (e as Error).message;
fileRetryCount = Math.min(4, fileRetryCount + 1);
nextFileRefresh = Date.now() + Math.min(30000, 3000 * 2 ** fileRetryCount);
}
} finally {
fileBusy.value = false;
if (
@@ -335,6 +342,9 @@ watch(
() => form.projectId,
() => {
fileGeneration++;
fileTaskSignature = "";
fileRetryCount = 0;
nextFileRefresh = 0;
files.value = [];
content.value = "";
editingPath.value = "";
@@ -348,7 +358,7 @@ watch(
watch(
() => execution.value.filesConfigured,
(configured) => {
if (configured) void list();
if (configured && Date.now() >= nextFileRefresh) void list();
},
);
onMounted(async () => {
+76 -40
View File
@@ -6,6 +6,7 @@ The gateway is private infrastructure and requires a cluster service account.
import base64
import hmac
import json
import logging
import os
import threading
import time
@@ -18,6 +19,8 @@ from kubernetes import client, config
from kubernetes.client.rest import ApiException
from kubernetes.stream import stream
from pydantic import BaseModel, Field
from urllib3.exceptions import HTTPError
from websocket import WebSocketException
TOKEN = os.environ.get("SANDBOX_TOKEN", "")
NAMESPACE = os.environ.get("SANDBOX_NAMESPACE", "platform-sandboxes")
@@ -27,6 +30,8 @@ RUNTIME = os.environ.get("SANDBOX_RUNTIME_CLASS", "gvisor")
LOCKS: dict[str, threading.Lock] = {}
LOCK_GUARD = threading.Lock()
MAX_OUTPUT = 100_000
LOG = logging.getLogger("uvicorn.error")
TRANSIENT_KUBE_STATUSES = {0, 408, 429, 500, 502, 503, 504}
@asynccontextmanager
@@ -73,12 +78,28 @@ def workspace_name(project: str) -> str:
raise HTTPException(400, "Invalid project ID")
def kube_call(stage, operation, *, already_exists_ok=False):
# Use only before sending tool input. A command or write may already have
# happened once stdin is sent, so an interrupted result must not be retried.
for attempt in range(3):
try:
return operation()
except (ApiException, HTTPError) as exc:
status = exc.status if isinstance(exc, ApiException) else 0
cause = exc.__context__
upstream = getattr(cause, "status_code", None)
if already_exists_ok and status == 409:
return None
LOG.warning("Sandbox Kubernetes %s failed (attempt=%d, status=%s, cause=%s)",
stage, attempt + 1, upstream or status, type(cause or exc).__name__)
if attempt == 2 or (upstream or status) not in TRANSIENT_KUBE_STATUSES:
raise HTTPException(503, stage) from None
time.sleep(0.25 * 2 ** attempt)
def create_once(method, **kwargs):
try:
method(**kwargs)
except ApiException as exc:
if exc.status != 409:
raise HTTPException(503, "Kubernetes resource provisioning failed") from None
kube_call("WORKSPACE_PROVISIONING",
lambda: method(**kwargs, _request_timeout=(3, 5)), already_exists_ok=True)
def ensure_workspace(project: str, review=False):
@@ -129,13 +150,16 @@ def ensure_workspace(project: str, review=False):
{"name": "tmp", "emptyDir": {"sizeLimit": "1Gi"}}]}})
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
pod = core.read_namespaced_pod(name, namespace)
pod = kube_call("WORKSPACE_PROVISIONING",
lambda: core.read_namespaced_pod(name, namespace, _request_timeout=(3, 5)))
if pod.status.phase == "Running" and pod.status.container_statuses and all(s.ready for s in pod.status.container_statuses):
return name
if pod.status.phase in ("Failed", "Succeeded"):
raise HTTPException(503, "Sandbox terminated; requires operator inspection")
LOG.warning("Sandbox pod %s is %s", name, pod.status.phase)
raise HTTPException(503, "WORKSPACE_TERMINATED")
time.sleep(1)
raise HTTPException(503, "Sandbox not ready; check isolated runtime, image and storage provisioner")
LOG.warning("Sandbox pod %s did not become ready (phase=%s)", name, pod.status.phase)
raise HTTPException(503, "WORKSPACE_STARTING")
# This code runs ONLY inside the workspace Pod via Kubernetes exec.
@@ -239,35 +263,39 @@ SCRIPT = Path(__file__).with_name('review_tools.py').read_text(encoding='utf-8')
def read_exec_output(connection, limit):
try:
return drain_exec_output(connection, limit)
finally:
connection.close()
def drain_exec_output(connection, limit):
# _preload_content=True lets ApiClient parse JSON and coerce dictionaries back to
# Python repr strings. Read raw stdout explicitly and keep stderr separate.
deadline = time.monotonic() + 135
chunks, size, error_size = [], 0, 0
while connection.is_open():
remaining = deadline - time.monotonic()
if remaining <= 0:
raise HTTPException(504, 'Sandbox execution timed out; result must be checked')
connection.update(timeout=min(1, remaining))
if connection.peek_stdout():
chunk = connection.read_stdout()
size += len(chunk.encode('utf-8'))
if size > limit:
raise HTTPException(502, 'Tool output exceeded limit')
chunks.append(chunk)
if connection.peek_stderr():
error_size += len(connection.read_stderr().encode('utf-8'))
if error_size > 64000:
raise HTTPException(502, 'Sandbox diagnostic output exceeded limit')
try:
while connection.is_open():
remaining = deadline - time.monotonic()
if remaining <= 0:
raise HTTPException(504, 'Sandbox execution timed out; result must be checked')
connection.update(timeout=min(1, remaining))
if connection.peek_stdout():
chunk = connection.read_stdout()
size += len(chunk.encode('utf-8'))
if size > limit:
raise HTTPException(502, 'Tool output exceeded limit')
chunks.append(chunk)
if connection.peek_stderr():
error_size += len(connection.read_stderr().encode('utf-8'))
if error_size > 64000:
raise HTTPException(502, 'Sandbox diagnostic output exceeded limit')
try:
code = connection.returncode
except Exception:
raise HTTPException(502, 'Sandbox completion status is unknown') from None
if code != 0:
raise HTTPException(502, 'Sandbox wrapper did not complete successfully')
return ''.join(chunks)
finally:
connection.close()
code = connection.returncode
except Exception:
raise HTTPException(502, 'WORKSPACE_RESULT_UNKNOWN') from None
if code != 0:
raise HTTPException(502, 'Sandbox wrapper did not complete successfully')
return ''.join(chunks)
@app.post("/workspaces/{project}/tools")
@@ -304,7 +332,7 @@ def execute_tool(project,request,authorization,review):
with LOCK_GUARD:
lock = LOCKS.setdefault(('review:' if review else '')+name, threading.Lock())
if not lock.acquire(timeout=1):
raise HTTPException(409, "Workspace is busy")
raise HTTPException(409, "WORKSPACE_BUSY")
try:
ensure_workspace(project,review)
encoded = base64.b64encode(payload).decode()
@@ -313,17 +341,25 @@ def execute_tool(project,request,authorization,review):
# with concurrent health checks, workspace provisioning, or another project.
with client.ApiClient() as api_client:
core = client.CoreV1Api(api_client)
connection = stream(core.connect_get_namespaced_pod_exec, name, REVIEW_NAMESPACE if review else NAMESPACE,
container="workspace", command=["timeout", "130", "python3", "-I", "-c", code],
stderr=True, stdin=True, stdout=True, tty=False, _request_timeout=135,
_preload_content=False)
for offset in range(0,len(encoded),16384):connection.write_stdin(encoded[offset:offset+16384])
connection = kube_call("WORKSPACE_CONNECTION", lambda: stream(
core.connect_get_namespaced_pod_exec, name, REVIEW_NAMESPACE if review else NAMESPACE,
container="workspace", command=["timeout", "130", "python3", "-I", "-c", code],
stderr=True, stdin=True, stdout=True, tty=False, _request_timeout=135,
_preload_content=False))
try:
for offset in range(0,len(encoded),16384):
connection.write_stdin(encoded[offset:offset+16384])
except Exception:
connection.close()
raise
output = read_exec_output(connection, 16_000_000 if request.name in {'archive_source','review_export'} else MAX_OUTPUT)
try:
return json.loads(output)
except ValueError:
raise HTTPException(502, "Sandbox returned invalid output") from None
except ApiException:
raise HTTPException(503, "Kubernetes sandbox operation failed") from None
except (ApiException, HTTPError, OSError, WebSocketException) as exc:
LOG.warning("Sandbox tool result unavailable (workspace=%s, tool=%s, cause=%s)",
name, request.name, type(exc).__name__)
raise HTTPException(502, "WORKSPACE_RESULT_UNKNOWN") from None
finally:
lock.release()
+29
View File
@@ -86,6 +86,35 @@ class GatewayTests(unittest.TestCase):
self.assertEqual([], policy["ingress"])
self.assertIn("169.254.0.0/16", policy["egress"][1]["to"][0]["ipBlock"]["except"])
def test_transient_exec_handshake_recovers_before_tool_input(self):
project = "00000000-0000-0000-0000-000000000002"
request = gateway.Tool(name="write_file", arguments={"path": "index.html", "content": "test"})
connection = MagicMock()
with (patch.object(gateway, "TOKEN", "x" * 32),
patch.object(gateway, "ensure_workspace"),
patch.object(gateway, "stream", side_effect=[gateway.ApiException(status=503), connection]) as stream,
patch.object(gateway.time, "sleep"),
patch.object(gateway, "drain_exec_output", return_value='{"written":4}')):
self.assertEqual({"written": 4}, gateway.execute_tool(project, request, "Bearer " + "x" * 32, False))
self.assertEqual(2, stream.call_count)
connection.write_stdin.assert_called_once()
connection.close.assert_called_once()
def test_interrupted_tool_input_is_not_replayed(self):
project = "00000000-0000-0000-0000-000000000003"
request = gateway.Tool(name="run_command", arguments={"command": "npm install"})
connection = MagicMock()
connection.write_stdin.side_effect = gateway.WebSocketException("disconnected")
with (patch.object(gateway, "TOKEN", "x" * 32),
patch.object(gateway, "ensure_workspace"),
patch.object(gateway, "stream", return_value=connection) as stream):
with self.assertRaises(gateway.HTTPException) as error:
gateway.execute_tool(project, request, "Bearer " + "x" * 32, False)
self.assertEqual((502, "WORKSPACE_RESULT_UNKNOWN"), (error.exception.status_code, error.exception.detail))
stream.assert_called_once()
connection.write_stdin.assert_called_once()
connection.close.assert_called_once()
if __name__ == "__main__":
unittest.main()