Publish AI workspace source through the isolated runtime

This commit is contained in:
WebUp Cloud committed 2026-11-08 17:29:54 +08:00
1 parent 03169bd0f6
commit d1d14f0f77
21 files changed
+487 -68

No files matched your search

+1 -1
View File
@@ -3,7 +3,7 @@ FROM ${BASE_IMAGE}
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY app.py runtime.py domain_runtime.py review_tools.py workspace_changes.py ./
COPY app.py runtime.py domain_runtime.py review_tools.py workspace_changes.py source_runtime.py ./
USER 10001:10001
EXPOSE 8090
CMD ["uvicorn","app:app","--host","0.0.0.0","--port","8090","--workers","1","--no-access-log"]
+4 -2
View File
@@ -233,6 +233,8 @@ try:
result={'exitCode':p.returncode,'timedOut':timed_out,'output':output[:64000].decode('utf-8','replace'),'truncated':len(output)>64000}
elif name=='review_export':
result=review_export(root)
elif name=='release_snapshot':
result=release_snapshot(root)
elif name=='review_import':
result=review_import(root,values)
elif name=='review_validate':
@@ -380,7 +382,7 @@ def dispose_review(project: str, authorization: str | None = Header(default=None
def execute_tool(project,request,authorization,review):
authorize(authorization)
name = workspace_name(project)
allowed = {'review_import','review_validate','workspace_inspect'} if review else {"list_files", "read_file", "write_file", "delete_file", "run_command", "version_save", "version_list", "archive_source", "workspace_inspect",'review_export'}
allowed = {'review_import','review_validate','workspace_inspect'} if review else {"list_files", "read_file", "write_file", "delete_file", "run_command", "version_save", "version_list", "archive_source", "workspace_inspect",'review_export','release_snapshot'}
if request.name not in allowed:
raise HTTPException(400, "Unknown tool")
payload = json.dumps(request.model_dump(), ensure_ascii=False).encode()
@@ -409,7 +411,7 @@ def execute_tool(project,request,authorization,review):
except Exception:
connection.close()
raise
output_limit = 16_000_000 if request.name in {'archive_source','review_export'} else MAX_OUTPUT
output_limit = 16_000_000 if request.name in {'archive_source','review_export','release_snapshot'} else MAX_OUTPUT
if request.trackChanges:
output_limit = 512_000
output = read_exec_output(connection, output_limit)
+45 -4
View File
@@ -32,16 +32,22 @@ def review_fingerprint(files):
return hashlib.sha256(json.dumps(files, sort_keys=True, separators=(',', ':'), ensure_ascii=True).encode()).hexdigest()
def review_export(root):
def review_export(root, release=False):
files, samples, total, sample_chars = [], [], 0, 0
excluded = {'.npm', '.cache', '.pytest_cache', '.turbo', '.output', '.nuxt'}
def excluded_path(path):
return review_secret(path) or (release and (
any(part in excluded for part in path.parts) or path.parts[0] == 'data'
or path.suffix.lower() in {'.db', '.sqlite', '.sqlite3'}
))
archive = io.BytesIO()
with zipfile.ZipFile(archive, 'w', zipfile.ZIP_DEFLATED) as output:
for directory, dirs, names in os.walk(root, followlinks=False):
dirs[:] = sorted(d for d in dirs if not review_secret(pathlib.Path(d)) and not (pathlib.Path(directory) / d).is_symlink())
dirs[:] = sorted(d for d in dirs if not excluded_path(pathlib.Path(d)) and not (pathlib.Path(directory) / d).is_symlink())
for name in sorted(names):
file = pathlib.Path(directory) / name
rel = file.relative_to(root)
if review_secret(rel) or file.is_symlink() or not file.is_file():
if excluded_path(rel) or file.is_symlink() or not file.is_file():
continue
size = file.stat().st_size
if size > 10_000_000 or total + size > 10_000_000 or len(files) >= 2000:
@@ -53,7 +59,7 @@ def review_export(root):
raise ValueError('Review snapshot exceeds size limit')
files.append({'path': rel.as_posix(), 'bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest()})
output.writestr(rel.as_posix(), data)
if file.suffix.lower() in REVIEW_TEXT and file.name not in {'package-lock.json', 'yarn.lock'} and sample_chars < 14000:
if not release and file.suffix.lower() in REVIEW_TEXT and file.name not in {'package-lock.json', 'yarn.lock'} and sample_chars < 14000:
text = data[:64000].decode('utf-8', 'replace')
shown = review_redact(text[:min(7000, 14000 - sample_chars)])
samples.append({'path': rel.as_posix(), 'content': shown, 'truncated': len(shown) < len(text)})
@@ -65,6 +71,41 @@ def review_export(root):
'complete': True, 'totalBytes': total}
def release_snapshot(root):
snapshot = review_export(root, release=True)
data = base64.b64decode(snapshot['archive'])
if len(data) > 4_000_000:
raise ValueError('Source ZIP exceeds 4 MB')
with zipfile.ZipFile(io.BytesIO(data)) as archive:
names = set(archive.namelist())
package = {}
if 'package.json' in names:
package = json.loads(archive.read('package.json'))
if not isinstance(package, dict) or not isinstance(package.get('scripts', {}), dict):
raise ValueError('package.json scripts must be an object')
scripts = package.get('scripts', {})
start = str(scripts.get('start', '')).strip()
build = isinstance(scripts.get('build'), str) and bool(scripts['build'].strip())
install = bool(package.get('dependencies') or package.get('devDependencies')
or package.get('optionalDependencies'))
if start:
runtime = 'NODE'
simple = re.fullmatch(r'node\s+(?:\./)?([\w./-]+\.(?:mjs|cjs|js))', start)
if simple and not build and simple.group(1) not in names:
raise ValueError('Start file missing: ' + simple.group(1))
elif build:
runtime = 'STATIC_BUILD'
elif 'index.html' in names:
runtime = 'STATIC'
install = False
else:
raise ValueError('Source needs package.json scripts.start, scripts.build, or index.html')
if install and not ({'package-lock.json', 'npm-shrinkwrap.json'} & names):
raise ValueError('Dependencies require package-lock.json or npm-shrinkwrap.json')
return {'archive': snapshot['archive'], 'sha256': snapshot['archiveSha256'],
'runtime': runtime, 'install': install, 'build': build}
def review_import(root, values):
if any(root.iterdir()):
raise ValueError('Review workspace must be empty; snapshots cannot overwrite existing data')
+29 -1
View File
@@ -1,5 +1,6 @@
"""Restricted OCI runtime controller. This module never builds or runs code on the gateway host."""
import hmac
import base64
import json
import os
import threading
@@ -12,6 +13,7 @@ from fastapi import APIRouter, Header, HTTPException
from kubernetes import client
from kubernetes.client.rest import ApiException
from pydantic import BaseModel, Field
from source_runtime import Source, attach_source, source_chunks, source_image
router = APIRouter()
NAMESPACE = os.environ.get("RUNTIME_NAMESPACE", "platform-runtimes")
@@ -44,6 +46,7 @@ class Service(BaseModel):
memory_mb: int = Field(ge=64, le=262144)
replicas: int = Field(ge=1, le=20)
health_path: str = Field(pattern=r"^/[^\s]{0,499}$|^$")
source_managed: bool = False
class Deployment(BaseModel):
@@ -51,6 +54,7 @@ class Deployment(BaseModel):
deploymentId: uuid.UUID
services: list[Service] = Field(min_length=1, max_length=100)
branding: dict[str, str] = Field(default_factory=dict)
source: Source | None = None
class ProjectState(BaseModel):
@@ -150,12 +154,24 @@ def deployment_manifest(project, deployment, service, branding=None):
return manifest
@router.get("/runtime/source-ready")
def source_ready(authorization: str | None = Header(default=None)):
auth(authorization)
return {"ready": bool(source_image())}
@router.post("/runtime/deployments/{deployment}")
def deploy(deployment: str, request: Deployment, authorization: str | None = Header(default=None)):
auth(authorization)
did, project = identifier(deployment), identifier(request.projectId)
if did != identifier(request.deploymentId):
raise HTTPException(400, "Deployment ID mismatch")
managed = [service for service in request.services if service.source_managed]
if len(managed) != int(request.source is not None):
raise HTTPException(400, "Source snapshot must belong to exactly one managed service")
if request.source and not source_image():
raise HTTPException(503, "Source runtime image is not configured")
chunks = source_chunks(request.source) if request.source else []
with guard:
lock = locks.setdefault(project, threading.Lock())
if not lock.acquire(timeout=1):
@@ -190,6 +206,18 @@ def deploy(deployment: str, request: Deployment, authorization: str | None = Hea
for service in request.services:
manifest = deployment_manifest(project, did, service, request.branding)
name = manifest["metadata"]["name"]
if service.source_managed:
attach_source(manifest, request.source, len(chunks))
for index, chunk in enumerate(chunks):
ensure(core.create_namespaced_config_map, namespace=NAMESPACE, body={
"metadata": {"name": f"{name}-source-{index}", "labels": manifest["metadata"]["labels"]},
"immutable": True, "binaryData": {f"part-{index:03}": base64.b64encode(chunk).decode()},
})
if request.source.runtime == "NODE":
ensure(core.create_namespaced_persistent_volume_claim, namespace=NAMESPACE, body={
"metadata": {"name": "data-" + identifier(service.id), "labels": {"platform/project": project}},
"spec": {"accessModes": ["ReadWriteOnce"], "resources": {"requests": {"storage": "1Gi"}}},
})
ensure(core.create_namespaced_secret, namespace=NAMESPACE, body={"metadata": {"name": name}, "type": "Opaque", "stringData": service.environment})
proxy_config = branding_config(service.port,request.branding)
if proxy_config is not None:
@@ -199,7 +227,7 @@ def deploy(deployment: str, request: Deployment, authorization: str | None = Hea
target_port = 8081 if proxy_config is not None else service.port
ensure(core.create_namespaced_service, namespace=NAMESPACE, body={"metadata": {"name": stable, "labels": {"platform/project": project}}, "spec": {"selector": manifest["metadata"]["labels"], "ports": [{"port": service.port, "targetPort": target_port}]}})
names.append((name, stable, manifest["metadata"]["labels"], service.port, service.replicas))
deadline = time.monotonic() + 90
deadline = time.monotonic() + (200 if request.source else 90)
while time.monotonic() < deadline:
if all((apps.read_namespaced_deployment(n, NAMESPACE).status.ready_replicas or 0) >= replicas for n, _, _, _, replicas in names):
break
+134
View File
@@ -0,0 +1,134 @@
"""Package immutable source data for the isolated runtime, without a container build daemon."""
import base64
import binascii
import hashlib
import os
from fastapi import HTTPException
from pydantic import BaseModel, Field
class Source(BaseModel):
archive: str = Field(min_length=1, max_length=5_333_336)
sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
runtime: str = Field(pattern=r"^(NODE|STATIC|STATIC_BUILD)$")
install: bool
build: bool
def source_image():
return os.environ.get("RUNTIME_SOURCE_IMAGE", "").strip()
def source_chunks(source):
try:
data = base64.b64decode(source.archive, validate=True)
except (ValueError, binascii.Error):
raise HTTPException(400, "Invalid source archive encoding") from None
if len(data) > 4_000_000 or hashlib.sha256(data).hexdigest() != source.sha256:
raise HTTPException(400, "Source archive checksum or size is invalid")
return [data[i:i + 480_000] for i in range(0, len(data), 480_000)]
PREPARE = r"""
import hashlib, io, os, pathlib, shutil, stat, subprocess, sys, zipfile
release = pathlib.Path('/release').resolve()
root = release
parts = pathlib.Path('/source-bundle')
archive = b''.join(path.read_bytes() for path in sorted(parts.glob('part-*')))
if hashlib.sha256(archive).hexdigest() != sys.argv[1]:
raise SystemExit('Source archive checksum mismatch')
with zipfile.ZipFile(io.BytesIO(archive)) as zip_file:
entries = zip_file.infolist()
if len(entries) > 2000 or sum(entry.file_size for entry in entries) > 10_000_000:
raise SystemExit('Source archive exceeds extraction limits')
for entry in entries:
path = pathlib.PurePosixPath(entry.filename)
if (not path.parts or path.is_absolute() or '..' in path.parts or '\\' in entry.filename
or stat.S_ISLNK(entry.external_attr >> 16) or entry.is_dir()):
raise SystemExit('Unsafe source archive member')
target = root / path
target.parent.mkdir(parents=True, exist_ok=True)
with zip_file.open(entry) as content, target.open('xb') as output:
shutil.copyfileobj(content, output)
environment = dict(os.environ, HOME='/tmp', npm_config_cache='/tmp/npm')
if sys.argv[2] == '1':
subprocess.run(['npm', 'ci', '--no-audit', '--no-fund'], cwd=root, env=environment, check=True, timeout=90)
if sys.argv[3] == '1':
subprocess.run(['npm', 'run', 'build'], cwd=root, env=environment, check=True, timeout=75)
if sys.argv[4] == 'STATIC_BUILD' and not any(
(root / folder / 'index.html').is_file() for folder in ('dist', 'build')
):
raise SystemExit('Static build must create dist/index.html or build/index.html')
"""
SERVE = r"""
import functools, http.server, os, pathlib, sys, urllib.parse
release = pathlib.Path('/release').resolve()
root = release
if sys.argv[1] == 'STATIC_BUILD':
root = next(path for path in (root / 'dist', root / 'build') if (path / 'index.html').is_file())
root = root.resolve()
if not root.is_relative_to(release):
raise SystemExit('Static output must stay inside the release directory')
class StaticHandler(http.server.SimpleHTTPRequestHandler):
def send_head(self):
path = pathlib.Path(self.translate_path(self.path))
if not path.resolve().is_relative_to(root) or any(
part.startswith('.') for part in urllib.parse.unquote(self.path.split('?', 1)[0]).split('/')
):
self.send_error(404)
return None
if path.is_dir() and any(
(path / name).is_file() and not (path / name).resolve().is_relative_to(root)
for name in ('index.html', 'index.htm')
):
self.send_error(404)
return None
return super().send_head()
def list_directory(self, path):
self.send_error(404)
return None
handler = functools.partial(StaticHandler, directory=root)
http.server.ThreadingHTTPServer(('0.0.0.0', int(os.environ['PORT'])), handler).serve_forever()
"""
def attach_source(manifest, source, chunk_count):
pod = manifest["spec"]["template"]["spec"]
container = pod["containers"][0]
name = manifest["metadata"]["name"]
container.update(
image=source_image(), workingDir="/release",
command=["npm", "start"] if source.runtime == "NODE"
else ["python3", "-I", "-c", SERVE, source.runtime],
env=[{"name": "HOST", "value": "0.0.0.0"}, {"name": "PORT", "value": str(container["ports"][0]["containerPort"])},
{"name": "HOME", "value": "/tmp"}, {"name": "DATA_DIR", "value": "/data"}],
)
container["volumeMounts"].append({"name": "release", "mountPath": "/release", "readOnly": True})
pod["securityContext"]["fsGroup"] = 10001
pod["volumes"].extend([
{"name": "release", "emptyDir": {"sizeLimit": "512Mi"}},
{"name": "source-bundle", "projected": {"sources": [
{"configMap": {"name": f"{name}-source-{index}"}} for index in range(chunk_count)
]}},
])
pod["initContainers"] = [{
"name": "prepare-source", "image": source_image(),
"command": ["python3", "-I", "-c", PREPARE, source.sha256,
str(int(source.install)), str(int(source.build)), source.runtime],
"securityContext": container["securityContext"], "resources": container["resources"],
"volumeMounts": [{"name": "release", "mountPath": "/release"},
{"name": "source-bundle", "mountPath": "/source-bundle", "readOnly": True},
{"name": "tmp", "mountPath": "/tmp"}],
}]
if source.runtime == "NODE":
service = manifest["metadata"]["labels"]["platform/service"]
pod["volumes"].append({"name": "app-data", "persistentVolumeClaim": {"claimName": "data-" + service}})
container["volumeMounts"].append({"name": "app-data", "mountPath": "/data"})
+15
View File
@@ -41,6 +41,21 @@ class GatewayTests(unittest.TestCase):
(root / "large.txt").write_bytes(b"x" * 64001)
self.assertIn("error", self.execute_file_tool(root, "read_file", {"path": "large.txt"}))
def test_source_snapshot_checks_start_and_excludes_credentials_and_local_data(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
(root / "package.json").write_text('{"scripts":{"start":"node server.js"}}')
result = self.execute_file_tool(root, "release_snapshot", {})
self.assertIn("server.js", result["error"])
(root / "server.js").write_text("console.log('ready')")
(root / ".env").write_text("secret")
(root / "data").mkdir()
(root / "data/students.db").write_text("private rows")
result = self.execute_file_tool(root, "release_snapshot", {})
self.assertEqual("NODE", result["runtime"])
with zipfile.ZipFile(io.BytesIO(base64.b64decode(result["archive"]))) as archive:
self.assertEqual({"package.json", "server.js"}, set(archive.namelist()))
def test_agent_file_change_counts_and_manual_edits_are_separate(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)