Files
webup-cloud/node-agent/source_runtime.py
T

135 lines
5.7 KiB
Python

"""Package immutable source data for the isolated runtime, without a container build daemon."""
import base64
import binascii
import hashlib
import os
from fastapi import HTTPException
from pydantic import BaseModel, Field
class Source(BaseModel):
archive: str = Field(min_length=1, max_length=5_333_336)
sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
runtime: str = Field(pattern=r"^(NODE|STATIC|STATIC_BUILD)$")
install: bool
build: bool
def source_image():
return os.environ.get("RUNTIME_SOURCE_IMAGE", "").strip()
def source_chunks(source):
try:
data = base64.b64decode(source.archive, validate=True)
except (ValueError, binascii.Error):
raise HTTPException(400, "Invalid source archive encoding") from None
if len(data) > 4_000_000 or hashlib.sha256(data).hexdigest() != source.sha256:
raise HTTPException(400, "Source archive checksum or size is invalid")
return [data[i:i + 480_000] for i in range(0, len(data), 480_000)]
PREPARE = r"""
import hashlib, io, os, pathlib, shutil, stat, subprocess, sys, zipfile
release = pathlib.Path('/release').resolve()
root = release
parts = pathlib.Path('/source-bundle')
archive = b''.join(path.read_bytes() for path in sorted(parts.glob('part-*')))
if hashlib.sha256(archive).hexdigest() != sys.argv[1]:
raise SystemExit('Source archive checksum mismatch')
with zipfile.ZipFile(io.BytesIO(archive)) as zip_file:
entries = zip_file.infolist()
if len(entries) > 2000 or sum(entry.file_size for entry in entries) > 10_000_000:
raise SystemExit('Source archive exceeds extraction limits')
for entry in entries:
path = pathlib.PurePosixPath(entry.filename)
if (not path.parts or path.is_absolute() or '..' in path.parts or '\\' in entry.filename
or stat.S_ISLNK(entry.external_attr >> 16) or entry.is_dir()):
raise SystemExit('Unsafe source archive member')
target = root / path
target.parent.mkdir(parents=True, exist_ok=True)
with zip_file.open(entry) as content, target.open('xb') as output:
shutil.copyfileobj(content, output)
environment = dict(os.environ, HOME='/tmp', npm_config_cache='/tmp/npm')
if sys.argv[2] == '1':
subprocess.run(['npm', 'ci', '--no-audit', '--no-fund'], cwd=root, env=environment, check=True, timeout=90)
if sys.argv[3] == '1':
subprocess.run(['npm', 'run', 'build'], cwd=root, env=environment, check=True, timeout=75)
if sys.argv[4] == 'STATIC_BUILD' and not any(
(root / folder / 'index.html').is_file() for folder in ('dist', 'build')
):
raise SystemExit('Static build must create dist/index.html or build/index.html')
"""
SERVE = r"""
import functools, http.server, os, pathlib, sys, urllib.parse
release = pathlib.Path('/release').resolve()
root = release
if sys.argv[1] == 'STATIC_BUILD':
root = next(path for path in (root / 'dist', root / 'build') if (path / 'index.html').is_file())
root = root.resolve()
if not root.is_relative_to(release):
raise SystemExit('Static output must stay inside the release directory')
class StaticHandler(http.server.SimpleHTTPRequestHandler):
def send_head(self):
path = pathlib.Path(self.translate_path(self.path))
if not path.resolve().is_relative_to(root) or any(
part.startswith('.') for part in urllib.parse.unquote(self.path.split('?', 1)[0]).split('/')
):
self.send_error(404)
return None
if path.is_dir() and any(
(path / name).is_file() and not (path / name).resolve().is_relative_to(root)
for name in ('index.html', 'index.htm')
):
self.send_error(404)
return None
return super().send_head()
def list_directory(self, path):
self.send_error(404)
return None
handler = functools.partial(StaticHandler, directory=root)
http.server.ThreadingHTTPServer(('0.0.0.0', int(os.environ['PORT'])), handler).serve_forever()
"""
def attach_source(manifest, source, chunk_count):
pod = manifest["spec"]["template"]["spec"]
container = pod["containers"][0]
name = manifest["metadata"]["name"]
container.update(
image=source_image(), workingDir="/release",
command=["npm", "start"] if source.runtime == "NODE"
else ["python3", "-I", "-c", SERVE, source.runtime],
env=[{"name": "HOST", "value": "0.0.0.0"}, {"name": "PORT", "value": str(container["ports"][0]["containerPort"])},
{"name": "HOME", "value": "/tmp"}, {"name": "DATA_DIR", "value": "/data"}],
)
container["volumeMounts"].append({"name": "release", "mountPath": "/release", "readOnly": True})
pod["securityContext"]["fsGroup"] = 10001
pod["volumes"].extend([
{"name": "release", "emptyDir": {"sizeLimit": "512Mi"}},
{"name": "source-bundle", "projected": {"sources": [
{"configMap": {"name": f"{name}-source-{index}"}} for index in range(chunk_count)
]}},
])
pod["initContainers"] = [{
"name": "prepare-source", "image": source_image(),
"command": ["python3", "-I", "-c", PREPARE, source.sha256,
str(int(source.install)), str(int(source.build)), source.runtime],
"securityContext": container["securityContext"], "resources": container["resources"],
"volumeMounts": [{"name": "release", "mountPath": "/release"},
{"name": "source-bundle", "mountPath": "/source-bundle", "readOnly": True},
{"name": "tmp", "mountPath": "/tmp"}],
}]
if source.runtime == "NODE":
service = manifest["metadata"]["labels"]["platform/service"]
pod["volumes"].append({"name": "app-data", "persistentVolumeClaim": {"claimName": "data-" + service}})
container["volumeMounts"].append({"name": "app-data", "mountPath": "/data"})