forked from xhgit/webup-cloud
Merge pull request '恢复隔离工具的临时连接故障,减少重复 503' (#3) from codex/sandbox-tool-recovery into main
Reviewed-on: http://git.xhmcl.com/xhgit/webup-cloud/pulls/3
This commit is contained in:
commit
fbefbb8c3b
4 files changed
+156
-52
No files matched your search
@@ -14,6 +14,7 @@ public class SandboxClient {
|
|||||||
private final ObjectMapper json;
|
private final ObjectMapper json;
|
||||||
private final HttpClient client =
|
private final HttpClient client =
|
||||||
HttpClient.newBuilder()
|
HttpClient.newBuilder()
|
||||||
|
.version(HttpClient.Version.HTTP_1_1)
|
||||||
.connectTimeout(Duration.ofSeconds(5))
|
.connectTimeout(Duration.ofSeconds(5))
|
||||||
.followRedirects(HttpClient.Redirect.NEVER)
|
.followRedirects(HttpClient.Redirect.NEVER)
|
||||||
.build();
|
.build();
|
||||||
@@ -209,8 +210,7 @@ public class SandboxClient {
|
|||||||
try (var input = response.body()) {
|
try (var input = response.body()) {
|
||||||
byte[] bytes = input.readNBytes(limit + 1);
|
byte[] bytes = input.readNBytes(limit + 1);
|
||||||
if (bytes.length > limit) throw new ApiError.Failure(502, "工具输出过大");
|
if (bytes.length > limit) throw new ApiError.Failure(502, "工具输出过大");
|
||||||
if (response.statusCode() != 200)
|
if (response.statusCode() != 200) throw toolFailure(response.statusCode(), bytes);
|
||||||
throw new ApiError.Failure(502, "隔离工具返回 HTTP " + response.statusCode());
|
|
||||||
return json.readTree(bytes).toString();
|
return json.readTree(bytes).toString();
|
||||||
}
|
}
|
||||||
} catch (ApiError.Failure f) {
|
} catch (ApiError.Failure f) {
|
||||||
@@ -220,4 +220,33 @@ public class SandboxClient {
|
|||||||
throw new ApiError.Failure(502, "隔离工具调用失败,执行结果可能不确定,请核对后处理");
|
throw new ApiError.Failure(502, "隔离工具调用失败,执行结果可能不确定,请核对后处理");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private ApiError.Failure toolFailure(int status, byte[] bytes) {
|
||||||
|
String stage = "";
|
||||||
|
try {
|
||||||
|
stage = json.readTree(bytes).path("detail").asText("");
|
||||||
|
} catch (Exception ignored) {
|
||||||
|
// The response may come from an intermediate proxy rather than the gateway.
|
||||||
|
}
|
||||||
|
return switch (status) {
|
||||||
|
case 409 -> new ApiError.Failure(409, "工作区正在执行其他操作,请稍后刷新");
|
||||||
|
case 503 ->
|
||||||
|
new ApiError.Failure(
|
||||||
|
503,
|
||||||
|
switch (stage) {
|
||||||
|
case "WORKSPACE_PROVISIONING" -> "隔离环境暂时无法准备,请稍后重试";
|
||||||
|
case "WORKSPACE_STARTING" -> "隔离环境仍在启动,请稍后刷新";
|
||||||
|
case "WORKSPACE_TERMINATED" -> "隔离环境已退出,请联系管理员检查";
|
||||||
|
case "WORKSPACE_CONNECTION" -> "暂时无法连接项目的隔离环境,请稍后重试";
|
||||||
|
default -> "隔离执行服务暂时不可用,请稍后重试";
|
||||||
|
});
|
||||||
|
case 504 -> new ApiError.Failure(504, "隔离工具执行超时,请先核对结果再重试");
|
||||||
|
default ->
|
||||||
|
new ApiError.Failure(
|
||||||
|
502,
|
||||||
|
stage.equals("WORKSPACE_RESULT_UNKNOWN")
|
||||||
|
? "隔离工具连接中断,请先核对结果再重试"
|
||||||
|
: "隔离工具返回 HTTP " + status);
|
||||||
|
};
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -19,6 +19,8 @@ const fileBusy = ref(false),
|
|||||||
fileError = ref("");
|
fileError = ref("");
|
||||||
let fileGeneration = 0;
|
let fileGeneration = 0;
|
||||||
let fileTaskSignature = "";
|
let fileTaskSignature = "";
|
||||||
|
let fileRetryCount = 0;
|
||||||
|
let nextFileRefresh = 0;
|
||||||
const logPanel = ref<HTMLElement | null>(null);
|
const logPanel = ref<HTMLElement | null>(null);
|
||||||
const followLogs = ref(true);
|
const followLogs = ref(true);
|
||||||
let logRequest = 0;
|
let logRequest = 0;
|
||||||
@@ -103,11 +105,15 @@ async function load() {
|
|||||||
.map((t) => `${t.id}:${t.status}:${t.step_count}`)
|
.map((t) => `${t.id}:${t.status}:${t.step_count}`)
|
||||||
.join("|");
|
.join("|");
|
||||||
if (
|
if (
|
||||||
signature !== fileTaskSignature ||
|
execution.value.filesConfigured &&
|
||||||
projectTasks.some((t) => t.status === "RUNNING")
|
!fileBusy.value &&
|
||||||
|
Date.now() >= nextFileRefresh &&
|
||||||
|
(signature !== fileTaskSignature ||
|
||||||
|
fileRetryCount > 0 ||
|
||||||
|
projectTasks.some((t) => t.status === "RUNNING"))
|
||||||
) {
|
) {
|
||||||
fileTaskSignature = signature;
|
fileTaskSignature = signature;
|
||||||
if (execution.value.filesConfigured && !fileBusy.value) void list();
|
void list();
|
||||||
}
|
}
|
||||||
const id = selected.value?.id;
|
const id = selected.value?.id;
|
||||||
if (id) {
|
if (id) {
|
||||||
@@ -118,13 +124,8 @@ async function load() {
|
|||||||
]);
|
]);
|
||||||
if (selected.value?.id === id && request > appliedLogRequest) {
|
if (selected.value?.id === id && request > appliedLogRequest) {
|
||||||
appliedLogRequest = request;
|
appliedLogRequest = request;
|
||||||
const changed =
|
|
||||||
selected.value.status !== task.status ||
|
|
||||||
events.value.at(-1)?.id !== log.at(-1)?.id;
|
|
||||||
selected.value = task;
|
selected.value = task;
|
||||||
events.value = log;
|
events.value = log;
|
||||||
if (changed && execution.value.filesConfigured && !fileBusy.value)
|
|
||||||
void list();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
@@ -226,6 +227,8 @@ async function list() {
|
|||||||
) {
|
) {
|
||||||
files.value = result.files || [];
|
files.value = result.files || [];
|
||||||
fileLoaded.value = true;
|
fileLoaded.value = true;
|
||||||
|
fileRetryCount = 0;
|
||||||
|
nextFileRefresh = Date.now() + 9000;
|
||||||
if (!editingPath.value && !content.value) {
|
if (!editingPath.value && !content.value) {
|
||||||
const first =
|
const first =
|
||||||
files.value.find(
|
files.value.find(
|
||||||
@@ -243,7 +246,11 @@ async function list() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
if (generation === fileGeneration) fileError.value = (e as Error).message;
|
if (generation === fileGeneration && directory === filePath.value) {
|
||||||
|
fileError.value = (e as Error).message;
|
||||||
|
fileRetryCount = Math.min(4, fileRetryCount + 1);
|
||||||
|
nextFileRefresh = Date.now() + Math.min(30000, 3000 * 2 ** fileRetryCount);
|
||||||
|
}
|
||||||
} finally {
|
} finally {
|
||||||
fileBusy.value = false;
|
fileBusy.value = false;
|
||||||
if (
|
if (
|
||||||
@@ -335,6 +342,9 @@ watch(
|
|||||||
() => form.projectId,
|
() => form.projectId,
|
||||||
() => {
|
() => {
|
||||||
fileGeneration++;
|
fileGeneration++;
|
||||||
|
fileTaskSignature = "";
|
||||||
|
fileRetryCount = 0;
|
||||||
|
nextFileRefresh = 0;
|
||||||
files.value = [];
|
files.value = [];
|
||||||
content.value = "";
|
content.value = "";
|
||||||
editingPath.value = "";
|
editingPath.value = "";
|
||||||
@@ -348,7 +358,7 @@ watch(
|
|||||||
watch(
|
watch(
|
||||||
() => execution.value.filesConfigured,
|
() => execution.value.filesConfigured,
|
||||||
(configured) => {
|
(configured) => {
|
||||||
if (configured) void list();
|
if (configured && Date.now() >= nextFileRefresh) void list();
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
onMounted(async () => {
|
onMounted(async () => {
|
||||||
|
|||||||
+76
-40
@@ -6,6 +6,7 @@ The gateway is private infrastructure and requires a cluster service account.
|
|||||||
import base64
|
import base64
|
||||||
import hmac
|
import hmac
|
||||||
import json
|
import json
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
@@ -18,6 +19,8 @@ from kubernetes import client, config
|
|||||||
from kubernetes.client.rest import ApiException
|
from kubernetes.client.rest import ApiException
|
||||||
from kubernetes.stream import stream
|
from kubernetes.stream import stream
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
|
from urllib3.exceptions import HTTPError
|
||||||
|
from websocket import WebSocketException
|
||||||
|
|
||||||
TOKEN = os.environ.get("SANDBOX_TOKEN", "")
|
TOKEN = os.environ.get("SANDBOX_TOKEN", "")
|
||||||
NAMESPACE = os.environ.get("SANDBOX_NAMESPACE", "platform-sandboxes")
|
NAMESPACE = os.environ.get("SANDBOX_NAMESPACE", "platform-sandboxes")
|
||||||
@@ -27,6 +30,8 @@ RUNTIME = os.environ.get("SANDBOX_RUNTIME_CLASS", "gvisor")
|
|||||||
LOCKS: dict[str, threading.Lock] = {}
|
LOCKS: dict[str, threading.Lock] = {}
|
||||||
LOCK_GUARD = threading.Lock()
|
LOCK_GUARD = threading.Lock()
|
||||||
MAX_OUTPUT = 100_000
|
MAX_OUTPUT = 100_000
|
||||||
|
LOG = logging.getLogger("uvicorn.error")
|
||||||
|
TRANSIENT_KUBE_STATUSES = {0, 408, 429, 500, 502, 503, 504}
|
||||||
|
|
||||||
|
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
@@ -73,12 +78,28 @@ def workspace_name(project: str) -> str:
|
|||||||
raise HTTPException(400, "Invalid project ID")
|
raise HTTPException(400, "Invalid project ID")
|
||||||
|
|
||||||
|
|
||||||
|
def kube_call(stage, operation, *, already_exists_ok=False):
|
||||||
|
# Use only before sending tool input. A command or write may already have
|
||||||
|
# happened once stdin is sent, so an interrupted result must not be retried.
|
||||||
|
for attempt in range(3):
|
||||||
|
try:
|
||||||
|
return operation()
|
||||||
|
except (ApiException, HTTPError) as exc:
|
||||||
|
status = exc.status if isinstance(exc, ApiException) else 0
|
||||||
|
cause = exc.__context__
|
||||||
|
upstream = getattr(cause, "status_code", None)
|
||||||
|
if already_exists_ok and status == 409:
|
||||||
|
return None
|
||||||
|
LOG.warning("Sandbox Kubernetes %s failed (attempt=%d, status=%s, cause=%s)",
|
||||||
|
stage, attempt + 1, upstream or status, type(cause or exc).__name__)
|
||||||
|
if attempt == 2 or (upstream or status) not in TRANSIENT_KUBE_STATUSES:
|
||||||
|
raise HTTPException(503, stage) from None
|
||||||
|
time.sleep(0.25 * 2 ** attempt)
|
||||||
|
|
||||||
|
|
||||||
def create_once(method, **kwargs):
|
def create_once(method, **kwargs):
|
||||||
try:
|
kube_call("WORKSPACE_PROVISIONING",
|
||||||
method(**kwargs)
|
lambda: method(**kwargs, _request_timeout=(3, 5)), already_exists_ok=True)
|
||||||
except ApiException as exc:
|
|
||||||
if exc.status != 409:
|
|
||||||
raise HTTPException(503, "Kubernetes resource provisioning failed") from None
|
|
||||||
|
|
||||||
|
|
||||||
def ensure_workspace(project: str, review=False):
|
def ensure_workspace(project: str, review=False):
|
||||||
@@ -129,13 +150,16 @@ def ensure_workspace(project: str, review=False):
|
|||||||
{"name": "tmp", "emptyDir": {"sizeLimit": "1Gi"}}]}})
|
{"name": "tmp", "emptyDir": {"sizeLimit": "1Gi"}}]}})
|
||||||
deadline = time.monotonic() + 90
|
deadline = time.monotonic() + 90
|
||||||
while time.monotonic() < deadline:
|
while time.monotonic() < deadline:
|
||||||
pod = core.read_namespaced_pod(name, namespace)
|
pod = kube_call("WORKSPACE_PROVISIONING",
|
||||||
|
lambda: core.read_namespaced_pod(name, namespace, _request_timeout=(3, 5)))
|
||||||
if pod.status.phase == "Running" and pod.status.container_statuses and all(s.ready for s in pod.status.container_statuses):
|
if pod.status.phase == "Running" and pod.status.container_statuses and all(s.ready for s in pod.status.container_statuses):
|
||||||
return name
|
return name
|
||||||
if pod.status.phase in ("Failed", "Succeeded"):
|
if pod.status.phase in ("Failed", "Succeeded"):
|
||||||
raise HTTPException(503, "Sandbox terminated; requires operator inspection")
|
LOG.warning("Sandbox pod %s is %s", name, pod.status.phase)
|
||||||
|
raise HTTPException(503, "WORKSPACE_TERMINATED")
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
raise HTTPException(503, "Sandbox not ready; check isolated runtime, image and storage provisioner")
|
LOG.warning("Sandbox pod %s did not become ready (phase=%s)", name, pod.status.phase)
|
||||||
|
raise HTTPException(503, "WORKSPACE_STARTING")
|
||||||
|
|
||||||
|
|
||||||
# This code runs ONLY inside the workspace Pod via Kubernetes exec.
|
# This code runs ONLY inside the workspace Pod via Kubernetes exec.
|
||||||
@@ -239,35 +263,39 @@ SCRIPT = Path(__file__).with_name('review_tools.py').read_text(encoding='utf-8')
|
|||||||
|
|
||||||
|
|
||||||
def read_exec_output(connection, limit):
|
def read_exec_output(connection, limit):
|
||||||
|
try:
|
||||||
|
return drain_exec_output(connection, limit)
|
||||||
|
finally:
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
|
||||||
|
def drain_exec_output(connection, limit):
|
||||||
# _preload_content=True lets ApiClient parse JSON and coerce dictionaries back to
|
# _preload_content=True lets ApiClient parse JSON and coerce dictionaries back to
|
||||||
# Python repr strings. Read raw stdout explicitly and keep stderr separate.
|
# Python repr strings. Read raw stdout explicitly and keep stderr separate.
|
||||||
deadline = time.monotonic() + 135
|
deadline = time.monotonic() + 135
|
||||||
chunks, size, error_size = [], 0, 0
|
chunks, size, error_size = [], 0, 0
|
||||||
|
while connection.is_open():
|
||||||
|
remaining = deadline - time.monotonic()
|
||||||
|
if remaining <= 0:
|
||||||
|
raise HTTPException(504, 'Sandbox execution timed out; result must be checked')
|
||||||
|
connection.update(timeout=min(1, remaining))
|
||||||
|
if connection.peek_stdout():
|
||||||
|
chunk = connection.read_stdout()
|
||||||
|
size += len(chunk.encode('utf-8'))
|
||||||
|
if size > limit:
|
||||||
|
raise HTTPException(502, 'Tool output exceeded limit')
|
||||||
|
chunks.append(chunk)
|
||||||
|
if connection.peek_stderr():
|
||||||
|
error_size += len(connection.read_stderr().encode('utf-8'))
|
||||||
|
if error_size > 64000:
|
||||||
|
raise HTTPException(502, 'Sandbox diagnostic output exceeded limit')
|
||||||
try:
|
try:
|
||||||
while connection.is_open():
|
code = connection.returncode
|
||||||
remaining = deadline - time.monotonic()
|
except Exception:
|
||||||
if remaining <= 0:
|
raise HTTPException(502, 'WORKSPACE_RESULT_UNKNOWN') from None
|
||||||
raise HTTPException(504, 'Sandbox execution timed out; result must be checked')
|
if code != 0:
|
||||||
connection.update(timeout=min(1, remaining))
|
raise HTTPException(502, 'Sandbox wrapper did not complete successfully')
|
||||||
if connection.peek_stdout():
|
return ''.join(chunks)
|
||||||
chunk = connection.read_stdout()
|
|
||||||
size += len(chunk.encode('utf-8'))
|
|
||||||
if size > limit:
|
|
||||||
raise HTTPException(502, 'Tool output exceeded limit')
|
|
||||||
chunks.append(chunk)
|
|
||||||
if connection.peek_stderr():
|
|
||||||
error_size += len(connection.read_stderr().encode('utf-8'))
|
|
||||||
if error_size > 64000:
|
|
||||||
raise HTTPException(502, 'Sandbox diagnostic output exceeded limit')
|
|
||||||
try:
|
|
||||||
code = connection.returncode
|
|
||||||
except Exception:
|
|
||||||
raise HTTPException(502, 'Sandbox completion status is unknown') from None
|
|
||||||
if code != 0:
|
|
||||||
raise HTTPException(502, 'Sandbox wrapper did not complete successfully')
|
|
||||||
return ''.join(chunks)
|
|
||||||
finally:
|
|
||||||
connection.close()
|
|
||||||
|
|
||||||
|
|
||||||
@app.post("/workspaces/{project}/tools")
|
@app.post("/workspaces/{project}/tools")
|
||||||
@@ -304,7 +332,7 @@ def execute_tool(project,request,authorization,review):
|
|||||||
with LOCK_GUARD:
|
with LOCK_GUARD:
|
||||||
lock = LOCKS.setdefault(('review:' if review else '')+name, threading.Lock())
|
lock = LOCKS.setdefault(('review:' if review else '')+name, threading.Lock())
|
||||||
if not lock.acquire(timeout=1):
|
if not lock.acquire(timeout=1):
|
||||||
raise HTTPException(409, "Workspace is busy")
|
raise HTTPException(409, "WORKSPACE_BUSY")
|
||||||
try:
|
try:
|
||||||
ensure_workspace(project,review)
|
ensure_workspace(project,review)
|
||||||
encoded = base64.b64encode(payload).decode()
|
encoded = base64.b64encode(payload).decode()
|
||||||
@@ -313,17 +341,25 @@ def execute_tool(project,request,authorization,review):
|
|||||||
# with concurrent health checks, workspace provisioning, or another project.
|
# with concurrent health checks, workspace provisioning, or another project.
|
||||||
with client.ApiClient() as api_client:
|
with client.ApiClient() as api_client:
|
||||||
core = client.CoreV1Api(api_client)
|
core = client.CoreV1Api(api_client)
|
||||||
connection = stream(core.connect_get_namespaced_pod_exec, name, REVIEW_NAMESPACE if review else NAMESPACE,
|
connection = kube_call("WORKSPACE_CONNECTION", lambda: stream(
|
||||||
container="workspace", command=["timeout", "130", "python3", "-I", "-c", code],
|
core.connect_get_namespaced_pod_exec, name, REVIEW_NAMESPACE if review else NAMESPACE,
|
||||||
stderr=True, stdin=True, stdout=True, tty=False, _request_timeout=135,
|
container="workspace", command=["timeout", "130", "python3", "-I", "-c", code],
|
||||||
_preload_content=False)
|
stderr=True, stdin=True, stdout=True, tty=False, _request_timeout=135,
|
||||||
for offset in range(0,len(encoded),16384):connection.write_stdin(encoded[offset:offset+16384])
|
_preload_content=False))
|
||||||
|
try:
|
||||||
|
for offset in range(0,len(encoded),16384):
|
||||||
|
connection.write_stdin(encoded[offset:offset+16384])
|
||||||
|
except Exception:
|
||||||
|
connection.close()
|
||||||
|
raise
|
||||||
output = read_exec_output(connection, 16_000_000 if request.name in {'archive_source','review_export'} else MAX_OUTPUT)
|
output = read_exec_output(connection, 16_000_000 if request.name in {'archive_source','review_export'} else MAX_OUTPUT)
|
||||||
try:
|
try:
|
||||||
return json.loads(output)
|
return json.loads(output)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
raise HTTPException(502, "Sandbox returned invalid output") from None
|
raise HTTPException(502, "Sandbox returned invalid output") from None
|
||||||
except ApiException:
|
except (ApiException, HTTPError, OSError, WebSocketException) as exc:
|
||||||
raise HTTPException(503, "Kubernetes sandbox operation failed") from None
|
LOG.warning("Sandbox tool result unavailable (workspace=%s, tool=%s, cause=%s)",
|
||||||
|
name, request.name, type(exc).__name__)
|
||||||
|
raise HTTPException(502, "WORKSPACE_RESULT_UNKNOWN") from None
|
||||||
finally:
|
finally:
|
||||||
lock.release()
|
lock.release()
|
||||||
@@ -86,6 +86,35 @@ class GatewayTests(unittest.TestCase):
|
|||||||
self.assertEqual([], policy["ingress"])
|
self.assertEqual([], policy["ingress"])
|
||||||
self.assertIn("169.254.0.0/16", policy["egress"][1]["to"][0]["ipBlock"]["except"])
|
self.assertIn("169.254.0.0/16", policy["egress"][1]["to"][0]["ipBlock"]["except"])
|
||||||
|
|
||||||
|
def test_transient_exec_handshake_recovers_before_tool_input(self):
|
||||||
|
project = "00000000-0000-0000-0000-000000000002"
|
||||||
|
request = gateway.Tool(name="write_file", arguments={"path": "index.html", "content": "test"})
|
||||||
|
connection = MagicMock()
|
||||||
|
with (patch.object(gateway, "TOKEN", "x" * 32),
|
||||||
|
patch.object(gateway, "ensure_workspace"),
|
||||||
|
patch.object(gateway, "stream", side_effect=[gateway.ApiException(status=503), connection]) as stream,
|
||||||
|
patch.object(gateway.time, "sleep"),
|
||||||
|
patch.object(gateway, "drain_exec_output", return_value='{"written":4}')):
|
||||||
|
self.assertEqual({"written": 4}, gateway.execute_tool(project, request, "Bearer " + "x" * 32, False))
|
||||||
|
self.assertEqual(2, stream.call_count)
|
||||||
|
connection.write_stdin.assert_called_once()
|
||||||
|
connection.close.assert_called_once()
|
||||||
|
|
||||||
|
def test_interrupted_tool_input_is_not_replayed(self):
|
||||||
|
project = "00000000-0000-0000-0000-000000000003"
|
||||||
|
request = gateway.Tool(name="run_command", arguments={"command": "npm install"})
|
||||||
|
connection = MagicMock()
|
||||||
|
connection.write_stdin.side_effect = gateway.WebSocketException("disconnected")
|
||||||
|
with (patch.object(gateway, "TOKEN", "x" * 32),
|
||||||
|
patch.object(gateway, "ensure_workspace"),
|
||||||
|
patch.object(gateway, "stream", return_value=connection) as stream):
|
||||||
|
with self.assertRaises(gateway.HTTPException) as error:
|
||||||
|
gateway.execute_tool(project, request, "Bearer " + "x" * 32, False)
|
||||||
|
self.assertEqual((502, "WORKSPACE_RESULT_UNKNOWN"), (error.exception.status_code, error.exception.detail))
|
||||||
|
stream.assert_called_once()
|
||||||
|
connection.write_stdin.assert_called_once()
|
||||||
|
connection.close.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
Reference in new issue
Block a user