Merge pull request '恢复隔离工具的临时连接故障,减少重复 503' (#3) from codex/sandbox-tool-recovery into main
Verify control plane / verify (push) Canceled after 0s
Verify control plane / verify (push) Canceled after 0s
Reviewed-on: http://git.xhmcl.com/xhgit/webup-cloud/pulls/3
This commit was merged in pull request #3.
This commit is contained in:
commit
fbefbb8c3b
4 files changed
+156
-52
No files matched your search
@@ -14,6 +14,7 @@ public class SandboxClient {
|
||||
private final ObjectMapper json;
|
||||
private final HttpClient client =
|
||||
HttpClient.newBuilder()
|
||||
.version(HttpClient.Version.HTTP_1_1)
|
||||
.connectTimeout(Duration.ofSeconds(5))
|
||||
.followRedirects(HttpClient.Redirect.NEVER)
|
||||
.build();
|
||||
@@ -209,8 +210,7 @@ public class SandboxClient {
|
||||
try (var input = response.body()) {
|
||||
byte[] bytes = input.readNBytes(limit + 1);
|
||||
if (bytes.length > limit) throw new ApiError.Failure(502, "工具输出过大");
|
||||
if (response.statusCode() != 200)
|
||||
throw new ApiError.Failure(502, "隔离工具返回 HTTP " + response.statusCode());
|
||||
if (response.statusCode() != 200) throw toolFailure(response.statusCode(), bytes);
|
||||
return json.readTree(bytes).toString();
|
||||
}
|
||||
} catch (ApiError.Failure f) {
|
||||
@@ -220,4 +220,33 @@ public class SandboxClient {
|
||||
throw new ApiError.Failure(502, "隔离工具调用失败,执行结果可能不确定,请核对后处理");
|
||||
}
|
||||
}
|
||||
|
||||
private ApiError.Failure toolFailure(int status, byte[] bytes) {
|
||||
String stage = "";
|
||||
try {
|
||||
stage = json.readTree(bytes).path("detail").asText("");
|
||||
} catch (Exception ignored) {
|
||||
// The response may come from an intermediate proxy rather than the gateway.
|
||||
}
|
||||
return switch (status) {
|
||||
case 409 -> new ApiError.Failure(409, "工作区正在执行其他操作,请稍后刷新");
|
||||
case 503 ->
|
||||
new ApiError.Failure(
|
||||
503,
|
||||
switch (stage) {
|
||||
case "WORKSPACE_PROVISIONING" -> "隔离环境暂时无法准备,请稍后重试";
|
||||
case "WORKSPACE_STARTING" -> "隔离环境仍在启动,请稍后刷新";
|
||||
case "WORKSPACE_TERMINATED" -> "隔离环境已退出,请联系管理员检查";
|
||||
case "WORKSPACE_CONNECTION" -> "暂时无法连接项目的隔离环境,请稍后重试";
|
||||
default -> "隔离执行服务暂时不可用,请稍后重试";
|
||||
});
|
||||
case 504 -> new ApiError.Failure(504, "隔离工具执行超时,请先核对结果再重试");
|
||||
default ->
|
||||
new ApiError.Failure(
|
||||
502,
|
||||
stage.equals("WORKSPACE_RESULT_UNKNOWN")
|
||||
? "隔离工具连接中断,请先核对结果再重试"
|
||||
: "隔离工具返回 HTTP " + status);
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -19,6 +19,8 @@ const fileBusy = ref(false),
|
||||
fileError = ref("");
|
||||
let fileGeneration = 0;
|
||||
let fileTaskSignature = "";
|
||||
let fileRetryCount = 0;
|
||||
let nextFileRefresh = 0;
|
||||
const logPanel = ref<HTMLElement | null>(null);
|
||||
const followLogs = ref(true);
|
||||
let logRequest = 0;
|
||||
@@ -103,11 +105,15 @@ async function load() {
|
||||
.map((t) => `${t.id}:${t.status}:${t.step_count}`)
|
||||
.join("|");
|
||||
if (
|
||||
signature !== fileTaskSignature ||
|
||||
projectTasks.some((t) => t.status === "RUNNING")
|
||||
execution.value.filesConfigured &&
|
||||
!fileBusy.value &&
|
||||
Date.now() >= nextFileRefresh &&
|
||||
(signature !== fileTaskSignature ||
|
||||
fileRetryCount > 0 ||
|
||||
projectTasks.some((t) => t.status === "RUNNING"))
|
||||
) {
|
||||
fileTaskSignature = signature;
|
||||
if (execution.value.filesConfigured && !fileBusy.value) void list();
|
||||
void list();
|
||||
}
|
||||
const id = selected.value?.id;
|
||||
if (id) {
|
||||
@@ -118,13 +124,8 @@ async function load() {
|
||||
]);
|
||||
if (selected.value?.id === id && request > appliedLogRequest) {
|
||||
appliedLogRequest = request;
|
||||
const changed =
|
||||
selected.value.status !== task.status ||
|
||||
events.value.at(-1)?.id !== log.at(-1)?.id;
|
||||
selected.value = task;
|
||||
events.value = log;
|
||||
if (changed && execution.value.filesConfigured && !fileBusy.value)
|
||||
void list();
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
@@ -226,6 +227,8 @@ async function list() {
|
||||
) {
|
||||
files.value = result.files || [];
|
||||
fileLoaded.value = true;
|
||||
fileRetryCount = 0;
|
||||
nextFileRefresh = Date.now() + 9000;
|
||||
if (!editingPath.value && !content.value) {
|
||||
const first =
|
||||
files.value.find(
|
||||
@@ -243,7 +246,11 @@ async function list() {
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
if (generation === fileGeneration) fileError.value = (e as Error).message;
|
||||
if (generation === fileGeneration && directory === filePath.value) {
|
||||
fileError.value = (e as Error).message;
|
||||
fileRetryCount = Math.min(4, fileRetryCount + 1);
|
||||
nextFileRefresh = Date.now() + Math.min(30000, 3000 * 2 ** fileRetryCount);
|
||||
}
|
||||
} finally {
|
||||
fileBusy.value = false;
|
||||
if (
|
||||
@@ -335,6 +342,9 @@ watch(
|
||||
() => form.projectId,
|
||||
() => {
|
||||
fileGeneration++;
|
||||
fileTaskSignature = "";
|
||||
fileRetryCount = 0;
|
||||
nextFileRefresh = 0;
|
||||
files.value = [];
|
||||
content.value = "";
|
||||
editingPath.value = "";
|
||||
@@ -348,7 +358,7 @@ watch(
|
||||
watch(
|
||||
() => execution.value.filesConfigured,
|
||||
(configured) => {
|
||||
if (configured) void list();
|
||||
if (configured && Date.now() >= nextFileRefresh) void list();
|
||||
},
|
||||
);
|
||||
onMounted(async () => {
|
||||
|
||||
+76
-40
@@ -6,6 +6,7 @@ The gateway is private infrastructure and requires a cluster service account.
|
||||
import base64
|
||||
import hmac
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
@@ -18,6 +19,8 @@ from kubernetes import client, config
|
||||
from kubernetes.client.rest import ApiException
|
||||
from kubernetes.stream import stream
|
||||
from pydantic import BaseModel, Field
|
||||
from urllib3.exceptions import HTTPError
|
||||
from websocket import WebSocketException
|
||||
|
||||
TOKEN = os.environ.get("SANDBOX_TOKEN", "")
|
||||
NAMESPACE = os.environ.get("SANDBOX_NAMESPACE", "platform-sandboxes")
|
||||
@@ -27,6 +30,8 @@ RUNTIME = os.environ.get("SANDBOX_RUNTIME_CLASS", "gvisor")
|
||||
LOCKS: dict[str, threading.Lock] = {}
|
||||
LOCK_GUARD = threading.Lock()
|
||||
MAX_OUTPUT = 100_000
|
||||
LOG = logging.getLogger("uvicorn.error")
|
||||
TRANSIENT_KUBE_STATUSES = {0, 408, 429, 500, 502, 503, 504}
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
@@ -73,12 +78,28 @@ def workspace_name(project: str) -> str:
|
||||
raise HTTPException(400, "Invalid project ID")
|
||||
|
||||
|
||||
def kube_call(stage, operation, *, already_exists_ok=False):
|
||||
# Use only before sending tool input. A command or write may already have
|
||||
# happened once stdin is sent, so an interrupted result must not be retried.
|
||||
for attempt in range(3):
|
||||
try:
|
||||
return operation()
|
||||
except (ApiException, HTTPError) as exc:
|
||||
status = exc.status if isinstance(exc, ApiException) else 0
|
||||
cause = exc.__context__
|
||||
upstream = getattr(cause, "status_code", None)
|
||||
if already_exists_ok and status == 409:
|
||||
return None
|
||||
LOG.warning("Sandbox Kubernetes %s failed (attempt=%d, status=%s, cause=%s)",
|
||||
stage, attempt + 1, upstream or status, type(cause or exc).__name__)
|
||||
if attempt == 2 or (upstream or status) not in TRANSIENT_KUBE_STATUSES:
|
||||
raise HTTPException(503, stage) from None
|
||||
time.sleep(0.25 * 2 ** attempt)
|
||||
|
||||
|
||||
def create_once(method, **kwargs):
|
||||
try:
|
||||
method(**kwargs)
|
||||
except ApiException as exc:
|
||||
if exc.status != 409:
|
||||
raise HTTPException(503, "Kubernetes resource provisioning failed") from None
|
||||
kube_call("WORKSPACE_PROVISIONING",
|
||||
lambda: method(**kwargs, _request_timeout=(3, 5)), already_exists_ok=True)
|
||||
|
||||
|
||||
def ensure_workspace(project: str, review=False):
|
||||
@@ -129,13 +150,16 @@ def ensure_workspace(project: str, review=False):
|
||||
{"name": "tmp", "emptyDir": {"sizeLimit": "1Gi"}}]}})
|
||||
deadline = time.monotonic() + 90
|
||||
while time.monotonic() < deadline:
|
||||
pod = core.read_namespaced_pod(name, namespace)
|
||||
pod = kube_call("WORKSPACE_PROVISIONING",
|
||||
lambda: core.read_namespaced_pod(name, namespace, _request_timeout=(3, 5)))
|
||||
if pod.status.phase == "Running" and pod.status.container_statuses and all(s.ready for s in pod.status.container_statuses):
|
||||
return name
|
||||
if pod.status.phase in ("Failed", "Succeeded"):
|
||||
raise HTTPException(503, "Sandbox terminated; requires operator inspection")
|
||||
LOG.warning("Sandbox pod %s is %s", name, pod.status.phase)
|
||||
raise HTTPException(503, "WORKSPACE_TERMINATED")
|
||||
time.sleep(1)
|
||||
raise HTTPException(503, "Sandbox not ready; check isolated runtime, image and storage provisioner")
|
||||
LOG.warning("Sandbox pod %s did not become ready (phase=%s)", name, pod.status.phase)
|
||||
raise HTTPException(503, "WORKSPACE_STARTING")
|
||||
|
||||
|
||||
# This code runs ONLY inside the workspace Pod via Kubernetes exec.
|
||||
@@ -239,35 +263,39 @@ SCRIPT = Path(__file__).with_name('review_tools.py').read_text(encoding='utf-8')
|
||||
|
||||
|
||||
def read_exec_output(connection, limit):
|
||||
try:
|
||||
return drain_exec_output(connection, limit)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def drain_exec_output(connection, limit):
|
||||
# _preload_content=True lets ApiClient parse JSON and coerce dictionaries back to
|
||||
# Python repr strings. Read raw stdout explicitly and keep stderr separate.
|
||||
deadline = time.monotonic() + 135
|
||||
chunks, size, error_size = [], 0, 0
|
||||
while connection.is_open():
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise HTTPException(504, 'Sandbox execution timed out; result must be checked')
|
||||
connection.update(timeout=min(1, remaining))
|
||||
if connection.peek_stdout():
|
||||
chunk = connection.read_stdout()
|
||||
size += len(chunk.encode('utf-8'))
|
||||
if size > limit:
|
||||
raise HTTPException(502, 'Tool output exceeded limit')
|
||||
chunks.append(chunk)
|
||||
if connection.peek_stderr():
|
||||
error_size += len(connection.read_stderr().encode('utf-8'))
|
||||
if error_size > 64000:
|
||||
raise HTTPException(502, 'Sandbox diagnostic output exceeded limit')
|
||||
try:
|
||||
while connection.is_open():
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise HTTPException(504, 'Sandbox execution timed out; result must be checked')
|
||||
connection.update(timeout=min(1, remaining))
|
||||
if connection.peek_stdout():
|
||||
chunk = connection.read_stdout()
|
||||
size += len(chunk.encode('utf-8'))
|
||||
if size > limit:
|
||||
raise HTTPException(502, 'Tool output exceeded limit')
|
||||
chunks.append(chunk)
|
||||
if connection.peek_stderr():
|
||||
error_size += len(connection.read_stderr().encode('utf-8'))
|
||||
if error_size > 64000:
|
||||
raise HTTPException(502, 'Sandbox diagnostic output exceeded limit')
|
||||
try:
|
||||
code = connection.returncode
|
||||
except Exception:
|
||||
raise HTTPException(502, 'Sandbox completion status is unknown') from None
|
||||
if code != 0:
|
||||
raise HTTPException(502, 'Sandbox wrapper did not complete successfully')
|
||||
return ''.join(chunks)
|
||||
finally:
|
||||
connection.close()
|
||||
code = connection.returncode
|
||||
except Exception:
|
||||
raise HTTPException(502, 'WORKSPACE_RESULT_UNKNOWN') from None
|
||||
if code != 0:
|
||||
raise HTTPException(502, 'Sandbox wrapper did not complete successfully')
|
||||
return ''.join(chunks)
|
||||
|
||||
|
||||
@app.post("/workspaces/{project}/tools")
|
||||
@@ -304,7 +332,7 @@ def execute_tool(project,request,authorization,review):
|
||||
with LOCK_GUARD:
|
||||
lock = LOCKS.setdefault(('review:' if review else '')+name, threading.Lock())
|
||||
if not lock.acquire(timeout=1):
|
||||
raise HTTPException(409, "Workspace is busy")
|
||||
raise HTTPException(409, "WORKSPACE_BUSY")
|
||||
try:
|
||||
ensure_workspace(project,review)
|
||||
encoded = base64.b64encode(payload).decode()
|
||||
@@ -313,17 +341,25 @@ def execute_tool(project,request,authorization,review):
|
||||
# with concurrent health checks, workspace provisioning, or another project.
|
||||
with client.ApiClient() as api_client:
|
||||
core = client.CoreV1Api(api_client)
|
||||
connection = stream(core.connect_get_namespaced_pod_exec, name, REVIEW_NAMESPACE if review else NAMESPACE,
|
||||
container="workspace", command=["timeout", "130", "python3", "-I", "-c", code],
|
||||
stderr=True, stdin=True, stdout=True, tty=False, _request_timeout=135,
|
||||
_preload_content=False)
|
||||
for offset in range(0,len(encoded),16384):connection.write_stdin(encoded[offset:offset+16384])
|
||||
connection = kube_call("WORKSPACE_CONNECTION", lambda: stream(
|
||||
core.connect_get_namespaced_pod_exec, name, REVIEW_NAMESPACE if review else NAMESPACE,
|
||||
container="workspace", command=["timeout", "130", "python3", "-I", "-c", code],
|
||||
stderr=True, stdin=True, stdout=True, tty=False, _request_timeout=135,
|
||||
_preload_content=False))
|
||||
try:
|
||||
for offset in range(0,len(encoded),16384):
|
||||
connection.write_stdin(encoded[offset:offset+16384])
|
||||
except Exception:
|
||||
connection.close()
|
||||
raise
|
||||
output = read_exec_output(connection, 16_000_000 if request.name in {'archive_source','review_export'} else MAX_OUTPUT)
|
||||
try:
|
||||
return json.loads(output)
|
||||
except ValueError:
|
||||
raise HTTPException(502, "Sandbox returned invalid output") from None
|
||||
except ApiException:
|
||||
raise HTTPException(503, "Kubernetes sandbox operation failed") from None
|
||||
except (ApiException, HTTPError, OSError, WebSocketException) as exc:
|
||||
LOG.warning("Sandbox tool result unavailable (workspace=%s, tool=%s, cause=%s)",
|
||||
name, request.name, type(exc).__name__)
|
||||
raise HTTPException(502, "WORKSPACE_RESULT_UNKNOWN") from None
|
||||
finally:
|
||||
lock.release()
|
||||
@@ -86,6 +86,35 @@ class GatewayTests(unittest.TestCase):
|
||||
self.assertEqual([], policy["ingress"])
|
||||
self.assertIn("169.254.0.0/16", policy["egress"][1]["to"][0]["ipBlock"]["except"])
|
||||
|
||||
def test_transient_exec_handshake_recovers_before_tool_input(self):
|
||||
project = "00000000-0000-0000-0000-000000000002"
|
||||
request = gateway.Tool(name="write_file", arguments={"path": "index.html", "content": "test"})
|
||||
connection = MagicMock()
|
||||
with (patch.object(gateway, "TOKEN", "x" * 32),
|
||||
patch.object(gateway, "ensure_workspace"),
|
||||
patch.object(gateway, "stream", side_effect=[gateway.ApiException(status=503), connection]) as stream,
|
||||
patch.object(gateway.time, "sleep"),
|
||||
patch.object(gateway, "drain_exec_output", return_value='{"written":4}')):
|
||||
self.assertEqual({"written": 4}, gateway.execute_tool(project, request, "Bearer " + "x" * 32, False))
|
||||
self.assertEqual(2, stream.call_count)
|
||||
connection.write_stdin.assert_called_once()
|
||||
connection.close.assert_called_once()
|
||||
|
||||
def test_interrupted_tool_input_is_not_replayed(self):
|
||||
project = "00000000-0000-0000-0000-000000000003"
|
||||
request = gateway.Tool(name="run_command", arguments={"command": "npm install"})
|
||||
connection = MagicMock()
|
||||
connection.write_stdin.side_effect = gateway.WebSocketException("disconnected")
|
||||
with (patch.object(gateway, "TOKEN", "x" * 32),
|
||||
patch.object(gateway, "ensure_workspace"),
|
||||
patch.object(gateway, "stream", return_value=connection) as stream):
|
||||
with self.assertRaises(gateway.HTTPException) as error:
|
||||
gateway.execute_tool(project, request, "Bearer " + "x" * 32, False)
|
||||
self.assertEqual((502, "WORKSPACE_RESULT_UNKNOWN"), (error.exception.status_code, error.exception.detail))
|
||||
stream.assert_called_once()
|
||||
connection.write_stdin.assert_called_once()
|
||||
connection.close.assert_called_once()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user