Files
WebUp Cloud 6d55806b4f
Verify control plane / verify (push) Canceled after 0s
Verify control plane / verify (pull_request) Canceled after 0s
Verify the declared Node entry before completing Agent tasks
2026-11-08 23:20:47 +08:00

265 lines
14 KiB
Python

"""Trusted review helpers, embedded into Python exec ONLY inside isolated workspace Pods."""
import base64
import hashlib
import io
import json
import os
import pathlib
import re
import signal
import subprocess
import tempfile
import time
import zipfile
from html.parser import HTMLParser
from urllib.parse import unquote, urlsplit
REVIEW_EXCLUDED = {'.git', 'node_modules', '.venv', '__pycache__', '.next', 'dist', 'build', 'coverage', '.ssh'}
REVIEW_TEXT = {'.html', '.css', '.js', '.jsx', '.ts', '.tsx', '.vue', '.svelte', '.py', '.java', '.go', '.rs', '.php', '.json'}
def review_secret(path):
return any(part in REVIEW_EXCLUDED or part.startswith('.env') or part.startswith('.webup-review')
for part in path.parts) or path.name in {'.npmrc', '.pypirc', '.netrc', 'credentials.json'} or path.suffix.lower() in {'.pem', '.key', '.p12', '.pfx'}
def review_redact(value):
value = re.sub(r'\bsk-[A-Za-z0-9_-]{16,}\b', '[REDACTED_KEY]', value)
return re.sub(r'(?i)((?:api[_-]?key|secret|password|access[_-]?token)\s*[=:]\s*[\"\x27]?)[^\s\"\x27;,}]{8,}', r'\1[REDACTED]', value)
def review_fingerprint(files):
return hashlib.sha256(json.dumps(files, sort_keys=True, separators=(',', ':'), ensure_ascii=True).encode()).hexdigest()
def review_export(root, release=False):
files, samples, total, sample_chars = [], [], 0, 0
excluded = {'.npm', '.cache', '.pytest_cache', '.turbo', '.output', '.nuxt'}
def excluded_path(path):
return review_secret(path) or (release and (
any(part in excluded for part in path.parts) or path.parts[0] == 'data'
or path.suffix.lower() in {'.db', '.sqlite', '.sqlite3'}
))
archive = io.BytesIO()
with zipfile.ZipFile(archive, 'w', zipfile.ZIP_DEFLATED) as output:
for directory, dirs, names in os.walk(root, followlinks=False):
dirs[:] = sorted(d for d in dirs if not excluded_path(pathlib.Path(d)) and not (pathlib.Path(directory) / d).is_symlink())
for name in sorted(names):
file = pathlib.Path(directory) / name
rel = file.relative_to(root)
if excluded_path(rel) or file.is_symlink() or not file.is_file():
continue
size = file.stat().st_size
if size > 10_000_000 or total + size > 10_000_000 or len(files) >= 2000:
raise ValueError('Review snapshot exceeds 10MB / 2000 files')
with file.open('rb') as stream:
data = stream.read(10_000_001)
total += len(data)
if total > 10_000_000 or len(data) > 10_000_000:
raise ValueError('Review snapshot exceeds size limit')
files.append({'path': rel.as_posix(), 'bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest()})
output.writestr(rel.as_posix(), data)
if not release and file.suffix.lower() in REVIEW_TEXT and file.name not in {'package-lock.json', 'yarn.lock'} and sample_chars < 14000:
text = data[:64000].decode('utf-8', 'replace')
shown = review_redact(text[:min(7000, 14000 - sample_chars)])
samples.append({'path': rel.as_posix(), 'content': shown, 'truncated': len(shown) < len(text)})
sample_chars += len(shown)
files.sort(key=lambda entry: entry['path'])
data = archive.getvalue()
return {'sourceHash': review_fingerprint(files), 'archiveSha256': hashlib.sha256(data).hexdigest(),
'archive': base64.b64encode(data).decode(), 'files': files, 'samples': samples,
'complete': True, 'totalBytes': total}
def node_start_entry(scripts):
start = scripts.get('start')
build = scripts.get('build')
if not isinstance(start, str) or (isinstance(build, str) and build.strip()):
return None
simple = re.fullmatch(r'node\s+(?:\./)?([\w./-]+\.(?:mjs|cjs|js))', start.strip())
if not simple:
return None
entry = pathlib.PurePosixPath(simple.group(1))
if entry.is_absolute() or '..' in entry.parts or simple.group(1).startswith('-'):
return None
return entry.as_posix()
def workspace_missing_start(root):
package_file = root / 'package.json'
try:
if package_file.is_symlink() or not package_file.is_file():
return None
with package_file.open('rb') as stream:
content = stream.read(256_001)
if len(content) > 256_000:
return None
package = json.loads(content)
except (OSError, ValueError):
return None
scripts = package.get('scripts') if isinstance(package, dict) else None
entry = node_start_entry(scripts) if isinstance(scripts, dict) else None
if entry:
target = root / entry
try:
if target.resolve() == target and target.is_file():
return None
except (OSError, RuntimeError):
pass
return entry
return None
def release_snapshot(root):
snapshot = review_export(root, release=True)
data = base64.b64decode(snapshot['archive'])
if len(data) > 4_000_000:
raise ValueError('Source ZIP exceeds 4 MB')
with zipfile.ZipFile(io.BytesIO(data)) as archive:
names = set(archive.namelist())
package = {}
if 'package.json' in names:
package = json.loads(archive.read('package.json'))
if not isinstance(package, dict) or not isinstance(package.get('scripts', {}), dict):
raise ValueError('package.json scripts must be an object')
scripts = package.get('scripts', {})
start = str(scripts.get('start', '')).strip()
build = isinstance(scripts.get('build'), str) and bool(scripts['build'].strip())
install = bool(package.get('dependencies') or package.get('devDependencies')
or package.get('optionalDependencies'))
if start:
runtime = 'NODE'
entry = node_start_entry(scripts)
if entry and entry not in names:
raise ValueError('Start file missing: ' + entry)
elif build:
runtime = 'STATIC_BUILD'
elif 'index.html' in names:
runtime = 'STATIC'
install = False
else:
raise ValueError('Source needs package.json scripts.start, scripts.build, or index.html')
if install and not ({'package-lock.json', 'npm-shrinkwrap.json'} & names):
raise ValueError('Dependencies require package-lock.json or npm-shrinkwrap.json')
return {'archive': snapshot['archive'], 'sha256': snapshot['archiveSha256'],
'runtime': runtime, 'install': install, 'build': build}
def review_import(root, values):
if any(root.iterdir()):
raise ValueError('Review workspace must be empty; snapshots cannot overwrite existing data')
data = base64.b64decode(values['archive'], validate=True)
if len(data) > 11_000_000 or hashlib.sha256(data).hexdigest() != values['archiveSha256']:
raise ValueError('Snapshot archive integrity mismatch')
files, entries, total = [], [], 0
with zipfile.ZipFile(io.BytesIO(data)) as archive:
for entry in archive.infolist():
rel = pathlib.PurePosixPath(entry.filename.replace('\\', '/'))
if rel.is_absolute() or '..' in rel.parts or review_secret(rel) or (entry.external_attr >> 16) & 0o170000 == 0o120000:
raise ValueError('Unsafe review snapshot entry')
if entry.is_dir():
continue
total += entry.file_size
if total > 10_000_000 or len(entries) >= 2000 or entry.filename in {e[0] for e in entries}:
raise ValueError('Invalid or oversized snapshot')
content = archive.read(entry)
if len(content) != entry.file_size:
raise ValueError('Snapshot entry size mismatch')
files.append({'path': str(rel), 'bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest()})
entries.append((str(rel), content))
files.sort(key=lambda entry: entry['path'])
if review_fingerprint(files) != values['sourceHash']:
raise ValueError('Source fingerprint mismatch')
for name, content in entries:
target = root / name
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(content)
(root / '.webup-review-manifest.json').write_text(json.dumps({'sourceHash': values['sourceHash'], 'files': files}))
return {'imported': len(files), 'sourceHash': values['sourceHash']}
def review_validate(root):
manifest = json.loads((root / '.webup-review-manifest.json').read_text())
deadline = time.monotonic() + 100
steps, issues, html_files = [], [], []
def run(label, command, seconds=35):
remaining = min(seconds, deadline - time.monotonic())
if remaining <= 0:
return {'name': label, 'status': 'NOT_RUN', 'reason': 'Review time budget exhausted'}
env = {'PATH': os.environ.get('PATH', ''), 'HOME': str(root), 'CI': 'true', 'LANG': 'C.UTF-8', 'npm_config_cache': '/tmp/npm-cache'}
with tempfile.TemporaryFile() as output:
try:
process = subprocess.Popen(command, cwd=root, env=env, stdout=output, stderr=subprocess.STDOUT, start_new_session=True)
try:
process.wait(timeout=remaining)
status = 'PASSED' if process.returncode == 0 else 'FAILED'
except subprocess.TimeoutExpired:
os.killpg(process.pid, signal.SIGKILL); process.wait(); status = 'TIMEOUT'
output.seek(0); captured = output.read(6001)
return {'name': label, 'command': command, 'status': status, 'exitCode': process.returncode,
'output': review_redact(captured[:6000].decode('utf-8', 'replace')), 'truncated': len(captured) > 6000}
except OSError:
return {'name': label, 'status': 'UNSUPPORTED', 'reason': 'Required tool is unavailable in review image'}
class Document(HTMLParser):
def __init__(self):
super().__init__(convert_charrefs=True); self.assets = []; self.title = False; self.ids=set(); self.anchors=[]
def handle_starttag(self, tag, attrs):
attrs = dict(attrs)
if attrs.get('id'): self.ids.add(attrs['id'])
if tag=='a' and attrs.get('href','').startswith('#') and len(attrs['href'])>1: self.anchors.append(unquote(attrs['href'][1:]))
if tag == 'title': self.title = True
if tag in {'script', 'img', 'source'} and attrs.get('src'): self.assets.append(attrs['src'])
if tag == 'link' and attrs.get('rel') == 'stylesheet': self.assets.append(attrs.get('href', ''))
for entry in manifest['files']:
file = root / entry['path']
if file.suffix.lower() in {'.html', '.htm'}:
html_files.append(entry['path'])
try:
document = Document(); document.feed(file.read_text()); document.close()
if not document.title: issues.append({'file': entry['path'], 'message': 'HTML document has no title'})
for anchor in document.anchors:
if anchor not in document.ids: issues.append({'file': entry['path'], 'message': 'Missing local anchor target: #'+anchor})
for asset in document.assets:
url = urlsplit(asset)
if url.scheme or url.netloc: continue
target = ((root if url.path.startswith('/') else file.parent) / unquote(url.path).lstrip('/')).resolve()
if not target.is_relative_to(root.resolve()) or not target.is_file():
issues.append({'file': entry['path'], 'message': 'Missing or unsafe local asset: ' + asset})
except (ValueError, OSError, UnicodeError):
issues.append({'file': entry['path'], 'message': 'HTML could not be parsed as UTF-8'})
if html_files:
steps.append({'name': 'html-assets', 'status': 'FAILED' if issues else 'PASSED', 'files': html_files, 'issues': issues})
if (root / 'package.json').is_file():
try:
package = json.loads((root / 'package.json').read_text())
if any((root / lock).exists() for lock in ('pnpm-lock.yaml', 'yarn.lock', 'bun.lockb', 'bun.lock')):
steps.append({'name': 'dependencies', 'status': 'UNSUPPORTED', 'reason': 'This runner currently requires npm; another lockfile was detected'})
else:
install = run('dependencies', ['npm', 'ci' if (root / 'package-lock.json').is_file() else 'install', '--no-audit', '--no-fund'], 45)
steps.append(install)
if install['status'] == 'PASSED':
scripts = package.get('scripts', {})
for name in ('build', 'test'):
steps.append(run(name, ['npm', 'run', name], 35) if scripts.get(name) else {'name': name, 'status': 'NOT_CONFIGURED'})
except (ValueError, OSError):
steps.append({'name': 'package-json', 'status': 'FAILED', 'reason': 'Invalid package.json'})
elif any((root / f['path']).suffix == '.py' for f in manifest['files']):
steps.append(run('python-syntax', ['python3', '-I', '-m', 'compileall', '-q', '.']))
steps.append({'name': 'python-tests', 'status': 'NOT_RUN', 'reason': 'Dependency installation and Python integration tests are not configured in this runner'})
elif html_files:
steps.append({'name': 'build', 'status': 'NOT_APPLICABLE', 'reason': 'Plain static HTML has no build step'})
else:
steps.append({'name': 'build', 'status': 'UNSUPPORTED', 'reason': 'Project build system is not supported by this runner'})
changed = []
for entry in manifest['files']:
file = root / entry['path']
try:
if file.is_symlink() or hashlib.sha256(file.read_bytes()).hexdigest() != entry['sha256']: changed.append(entry['path'])
except OSError: changed.append(entry['path'])
return {'sourceHash': manifest['sourceHash'], 'steps': steps, 'sourceChangedByBuild': changed,
'browser': {'status': 'NOT_RUN', 'reason': 'No browser or visual checks executed'},
'hasUi': bool(html_files), 'executionScope': 'Separate gVisor review Pod; development workspace was not modified'}