Files
ItxiaohuiContentGuard/scripts/run-stress.py
T

219 lines
12 KiB
Python

"""Run loopback Minecraft client load and save measured evidence for one core."""
import argparse
import html
import json
import platform
import re
import shutil
import sqlite3
import subprocess
import threading
import time
from contextlib import closing
from datetime import datetime
from pathlib import Path
import psutil
from compat_fixtures import fixtures
from importlib.util import module_from_spec, spec_from_file_location
spec = spec_from_file_location("compat_runner", Path(__file__).with_name("run-compat.py"))
shared = module_from_spec(spec)
spec.loader.exec_module(shared)
ROOT = shared.ROOT
def database_counts(directory):
file = directory / "plugins" / "ItxiaohuiContentGuard" / "database" / "audit.sqlite"
if not file.is_file():
return {"integrity": "missing"}
with closing(sqlite3.connect(file.as_uri() + "?mode=ro", uri=True, timeout=3)) as connection:
return {"integrity": connection.execute("PRAGMA quick_check").fetchone()[0],
"audit": dict(connection.execute("SELECT kind, count(*) FROM audit GROUP BY kind")),
"ai_failures": dict(connection.execute(
"SELECT reasoning, count(*) FROM audit WHERE kind='API_STATUS' GROUP BY reasoning")),
"mail_status": dict(connection.execute(
"SELECT action, count(*) FROM audit WHERE kind='MAIL' GROUP BY action")),
"word_hits": connection.execute("SELECT count(*) FROM word_hits").fetchone()[0]}
def sample(server, destination, period, run_finished):
process = psutil.Process(server.process.pid)
process.cpu_percent()
try:
with destination.open("w", encoding="utf-8") as evidence:
while not run_finished.is_set() and server.process.poll() is None:
memory = process.memory_info()
evidence.write(json.dumps({"time": shared.timestamp(), "cpu_percent": process.cpu_percent(),
"rss_bytes": memory.rss, "java_threads": process.num_threads()}) + "\n")
evidence.flush()
server.send("icgprobe metrics")
server.send("icg status")
if run_finished.wait(period):
break
except (OSError, psutil.NoSuchProcess):
return
def exercise_faults(server, smtp):
def probe(command, pattern):
start = len(server.lines)
server.send(command)
return server.await_line(pattern, 12, start)
time.sleep(2.5)
cases = {
"ai-http-503": probe("icg ai test ICG-error", r"HTTP_503"),
"ai-timeout": probe("icg ai test ICG-timeout", r"(?:TIMEOUT|DEADLINE)"),
"ai-disconnect": probe("icg ai test ICG-disconnect", r"NETWORK"),
"ai-circuit-open": probe("icg ai test circuit should be open", r"CIRCUIT_OPEN"),
}
time.sleep(2.3)
cases["ai-circuit-recovery"] = probe("icg ai test circuit should recover", r"safe=true category=SAFE")
smtp.fail_connections = True
try:
cases["smtp-unreachable-retry"] = probe("icg email test", r"SEND_FAILED attempts=3")
finally:
smtp.fail_connections = False
return cases
def main():
parser = argparse.ArgumentParser(description=__doc__)
for path in ("java", "plugin", "probe", "client"):
parser.add_argument(f"--{path}", type=lambda value: Path(value).resolve(), required=True)
parser.add_argument("--target", required=True, help="core/version")
parser.add_argument("--players", default="1,10,50,100")
parser.add_argument("--seconds", type=int, default=3)
parser.add_argument("--interval-ms", type=int, default=950)
parser.add_argument("--sample-seconds", type=int, default=3)
parser.add_argument("--name", default="stress")
parser.add_argument("--spread", action="store_true", help="Teleport Folia stress clients into four distant areas")
parser.add_argument("--faults", action="store_true")
parser.add_argument("--forced-restart", action="store_true")
parser.add_argument("--refill-books", action="store_true")
parser.add_argument("--rich-books", action="store_true")
parser.add_argument("--smtp-flood", action="store_true")
parser.add_argument("--healthy", action="store_true", help="Keep STRICT enabled and pace successful review requests")
args = parser.parse_args()
core, version = args.target.split("/", 1)
asset = ROOT / ".test-servers" / core / version
stamp = datetime.now(shared.TZ).strftime("%Y%m%d-%H%M%S")
directory = asset / "runs" / f"{args.name}-{stamp}"
output = ROOT / "test-results" / core / version / f"{args.name}-{stamp}"
output.mkdir(parents=True)
(directory / "plugins").mkdir(parents=True)
shutil.copy2(asset / "server.jar", directory / "server.jar")
shutil.copy2(args.plugin, directory / "plugins" / args.plugin.name)
shutil.copy2(args.probe, directory / "plugins" / args.probe.name)
port = shared.local_port()
(directory / "eula.txt").write_text("eula=true\n", encoding="ascii")
(directory / "server.properties").write_text(
f"server-ip=127.0.0.1\nserver-port={port}\nonline-mode=false\nenforce-secure-profile=false\n"
"max-players=120\nview-distance=3\nsimulation-distance=3\npause-when-empty-seconds=-1\n"
"level-name=world\nlevel-type=minecraft:flat\nwhite-list=false\nenable-rcon=false\ndifficulty=peaceful\n",
encoding="ascii")
command = [str(args.java), "-Dicg.compat.features=true", f"-Dicg.compat.spread={'true' if args.spread else 'false'}",
"-Dfile.encoding=UTF-8", "-Dterminal.jline=false", "-Dterminal.ansi=false",
"-XX:ActiveProcessorCount=8", "-Xms1G", "-Xmx4G", "-Xlog:gc:file=gc.log:time,uptime,level,tags",
"-jar", "server.jar", "nogui"]
began = shared.timestamp()
report = {"start": began, "core": core, "minecraft": version, "os": platform.platform(),
"jar_sha256": shared.digest(args.plugin), "server_sha256": shared.digest(asset / "server.jar"),
"local_directory": str(directory), "requested_players": args.players, "seconds_per_group": args.seconds,
"rich_books": args.rich_books, "refill_books": args.refill_books, "smtp_flood": args.smtp_flood,
"healthy_provider": args.healthy,
"cases": [], "errors": []}
manifest = output / "run.json"
manifest.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
with fixtures(args.plugin, directory) as (ai, smtp):
server = shared.ServerProcess(command, directory, output / "server.log")
try:
server.await_line(r"Done \([\d.]+s\)!", 240)
report["runtime"] = server.await_line(r"core=.* adapter=", 8)
completed = threading.Event()
tracker = threading.Thread(target=sample, args=(server, output / "samples.jsonl", args.sample_seconds, completed))
tracker.start()
try:
with (output / "client.log").open("w", encoding="utf-8") as client_log:
session = subprocess.run(["node", str(ROOT / "scripts" / "stress-client.cjs"), str(args.client),
str(port), version, args.players, str(args.seconds), str(args.interval_ms),
"true" if args.refill_books else "false", "true" if args.rich_books else "false",
"true" if args.smtp_flood else "false", "true" if args.healthy else "false"],
stdout=client_log, stderr=subprocess.STDOUT,
timeout=abs(args.seconds) * len(args.players.split(",")) + 400)
if session.returncode != 0:
report["errors"].append(f"Client exit: {session.returncode}")
for line in (output / "client.log").read_text(encoding="utf-8").splitlines():
if line.startswith("{"):
report["cases"].append(json.loads(line))
time.sleep(2)
finally:
completed.set()
tracker.join()
report["db_live"] = database_counts(directory)
if args.faults:
report["faults"] = exercise_faults(server, smtp)
if args.forced_restart:
server.process.kill()
report["forced_exit"] = server.process.wait(timeout=15)
except (RuntimeError, OSError, subprocess.SubprocessError, sqlite3.Error) as error:
report["errors"].append(str(error))
finally:
report["exit"] = server.stop()
if "forced_exit" in report:
restarted = shared.ServerProcess(command, directory, output / "restart.log")
try:
restarted.await_line(r"Done \([\d.]+s\)!", 240)
restarted.send("icg status")
try:
report["restarted"] = restarted.await_line(r"database=READY", 4)
except RuntimeError:
restarted.send("icg status")
report["restarted"] = restarted.await_line(r"database=READY", 10)
except RuntimeError as error:
report["errors"].append(str(error))
finally:
report["exit"] = restarted.stop()
report["ai_requests"] = ai.request_count
report["smtp_messages"] = smtp.message_count
report["end"] = shared.timestamp()
report["db_after_stop"] = database_counts(directory)
report["telemetry"] = [line for line in server.lines if "ICG_METRICS " in line or "database=READY" in line]
relevant = [line for line in server.lines if re.search(
r"IllegalStateException|NoClassDefFoundError|UnsupportedOperationException|Thread failed|SQLite .*failed|ICG_PROBE teleport error", line)]
report["exceptions"] = relevant
if (directory / "gc.log").exists():
shutil.copy2(directory / "gc.log", output / "gc.log")
passed = (len(report["cases"]) == len(args.players.split(",")) and
all(test["online"] == test["target"] and test["onlineAfter"] == test["target"] for test in report["cases"])
and report["exit"] == 0 and report["db_after_stop"].get("integrity") == "ok"
and not report["errors"] and not relevant)
if args.smtp_flood and not report["db_after_stop"].get("mail_status", {}).get("QUEUE_FULL", 0):
passed = False
report["errors"].append("SMTP queue did not reach its configured limit")
if args.healthy and any(case["firstClientObservedDeliveries"] != case["messagesSent"] for case in report["cases"]):
passed = False
report["errors"].append("Healthy provider run missed one or more expected public chat deliveries")
report["online_and_integrity_check"] = "PASS" if passed else "FAIL"
(output / "evidence.json").write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
markup = ('<!doctype html><html lang="zh-CN"><meta charset="utf-8"><title>实服并发</title>'
f'<h1>{html.escape(args.target)} 实服并发</h1><pre>'
+ html.escape(json.dumps(report, ensure_ascii=False, indent=2)) + "</pre></html>")
(output / "stress-report.html").write_text(markup, encoding="utf-8")
if args.name == "stress":
(output.parent / "stress-report.html").write_text(
'<!doctype html><meta charset="utf-8"><title>实服并发</title>'
f'<a href="{output.name}/stress-report.html">实服并发证据</a>', encoding="utf-8")
print(f"{args.target} {report['online_and_integrity_check']} {output}")
if not passed:
raise SystemExit(1)
if __name__ == "__main__":
main()