forked from xhgit/webup-cloud
226 lines
13 KiB
Python
226 lines
13 KiB
Python
"""Trusted review helpers, embedded into Python exec ONLY inside isolated workspace Pods."""
|
|
import base64
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import signal
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
import zipfile
|
|
from html.parser import HTMLParser
|
|
from urllib.parse import unquote, urlsplit
|
|
|
|
REVIEW_EXCLUDED = {'.git', 'node_modules', '.venv', '__pycache__', '.next', 'dist', 'build', 'coverage', '.ssh'}
|
|
REVIEW_TEXT = {'.html', '.css', '.js', '.jsx', '.ts', '.tsx', '.vue', '.svelte', '.py', '.java', '.go', '.rs', '.php', '.json'}
|
|
|
|
|
|
def review_secret(path):
|
|
return any(part in REVIEW_EXCLUDED or part.startswith('.env') or part.startswith('.webup-review')
|
|
for part in path.parts) or path.name in {'.npmrc', '.pypirc', '.netrc', 'credentials.json'} or path.suffix.lower() in {'.pem', '.key', '.p12', '.pfx'}
|
|
|
|
|
|
def review_redact(value):
|
|
value = re.sub(r'\bsk-[A-Za-z0-9_-]{16,}\b', '[REDACTED_KEY]', value)
|
|
return re.sub(r'(?i)((?:api[_-]?key|secret|password|access[_-]?token)\s*[=:]\s*[\"\x27]?)[^\s\"\x27;,}]{8,}', r'\1[REDACTED]', value)
|
|
|
|
|
|
def review_fingerprint(files):
|
|
return hashlib.sha256(json.dumps(files, sort_keys=True, separators=(',', ':'), ensure_ascii=True).encode()).hexdigest()
|
|
|
|
|
|
def review_export(root, release=False):
|
|
files, samples, total, sample_chars = [], [], 0, 0
|
|
excluded = {'.npm', '.cache', '.pytest_cache', '.turbo', '.output', '.nuxt'}
|
|
def excluded_path(path):
|
|
return review_secret(path) or (release and (
|
|
any(part in excluded for part in path.parts) or path.parts[0] == 'data'
|
|
or path.suffix.lower() in {'.db', '.sqlite', '.sqlite3'}
|
|
))
|
|
archive = io.BytesIO()
|
|
with zipfile.ZipFile(archive, 'w', zipfile.ZIP_DEFLATED) as output:
|
|
for directory, dirs, names in os.walk(root, followlinks=False):
|
|
dirs[:] = sorted(d for d in dirs if not excluded_path(pathlib.Path(d)) and not (pathlib.Path(directory) / d).is_symlink())
|
|
for name in sorted(names):
|
|
file = pathlib.Path(directory) / name
|
|
rel = file.relative_to(root)
|
|
if excluded_path(rel) or file.is_symlink() or not file.is_file():
|
|
continue
|
|
size = file.stat().st_size
|
|
if size > 10_000_000 or total + size > 10_000_000 or len(files) >= 2000:
|
|
raise ValueError('Review snapshot exceeds 10MB / 2000 files')
|
|
with file.open('rb') as stream:
|
|
data = stream.read(10_000_001)
|
|
total += len(data)
|
|
if total > 10_000_000 or len(data) > 10_000_000:
|
|
raise ValueError('Review snapshot exceeds size limit')
|
|
files.append({'path': rel.as_posix(), 'bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest()})
|
|
output.writestr(rel.as_posix(), data)
|
|
if not release and file.suffix.lower() in REVIEW_TEXT and file.name not in {'package-lock.json', 'yarn.lock'} and sample_chars < 14000:
|
|
text = data[:64000].decode('utf-8', 'replace')
|
|
shown = review_redact(text[:min(7000, 14000 - sample_chars)])
|
|
samples.append({'path': rel.as_posix(), 'content': shown, 'truncated': len(shown) < len(text)})
|
|
sample_chars += len(shown)
|
|
files.sort(key=lambda entry: entry['path'])
|
|
data = archive.getvalue()
|
|
return {'sourceHash': review_fingerprint(files), 'archiveSha256': hashlib.sha256(data).hexdigest(),
|
|
'archive': base64.b64encode(data).decode(), 'files': files, 'samples': samples,
|
|
'complete': True, 'totalBytes': total}
|
|
|
|
|
|
def release_snapshot(root):
|
|
snapshot = review_export(root, release=True)
|
|
data = base64.b64decode(snapshot['archive'])
|
|
if len(data) > 4_000_000:
|
|
raise ValueError('Source ZIP exceeds 4 MB')
|
|
with zipfile.ZipFile(io.BytesIO(data)) as archive:
|
|
names = set(archive.namelist())
|
|
package = {}
|
|
if 'package.json' in names:
|
|
package = json.loads(archive.read('package.json'))
|
|
if not isinstance(package, dict) or not isinstance(package.get('scripts', {}), dict):
|
|
raise ValueError('package.json scripts must be an object')
|
|
scripts = package.get('scripts', {})
|
|
start = str(scripts.get('start', '')).strip()
|
|
build = isinstance(scripts.get('build'), str) and bool(scripts['build'].strip())
|
|
install = bool(package.get('dependencies') or package.get('devDependencies')
|
|
or package.get('optionalDependencies'))
|
|
if start:
|
|
runtime = 'NODE'
|
|
simple = re.fullmatch(r'node\s+(?:\./)?([\w./-]+\.(?:mjs|cjs|js))', start)
|
|
if simple and not build and simple.group(1) not in names:
|
|
raise ValueError('Start file missing: ' + simple.group(1))
|
|
elif build:
|
|
runtime = 'STATIC_BUILD'
|
|
elif 'index.html' in names:
|
|
runtime = 'STATIC'
|
|
install = False
|
|
else:
|
|
raise ValueError('Source needs package.json scripts.start, scripts.build, or index.html')
|
|
if install and not ({'package-lock.json', 'npm-shrinkwrap.json'} & names):
|
|
raise ValueError('Dependencies require package-lock.json or npm-shrinkwrap.json')
|
|
return {'archive': snapshot['archive'], 'sha256': snapshot['archiveSha256'],
|
|
'runtime': runtime, 'install': install, 'build': build}
|
|
|
|
|
|
def review_import(root, values):
|
|
if any(root.iterdir()):
|
|
raise ValueError('Review workspace must be empty; snapshots cannot overwrite existing data')
|
|
data = base64.b64decode(values['archive'], validate=True)
|
|
if len(data) > 11_000_000 or hashlib.sha256(data).hexdigest() != values['archiveSha256']:
|
|
raise ValueError('Snapshot archive integrity mismatch')
|
|
files, entries, total = [], [], 0
|
|
with zipfile.ZipFile(io.BytesIO(data)) as archive:
|
|
for entry in archive.infolist():
|
|
rel = pathlib.PurePosixPath(entry.filename.replace('\\', '/'))
|
|
if rel.is_absolute() or '..' in rel.parts or review_secret(rel) or (entry.external_attr >> 16) & 0o170000 == 0o120000:
|
|
raise ValueError('Unsafe review snapshot entry')
|
|
if entry.is_dir():
|
|
continue
|
|
total += entry.file_size
|
|
if total > 10_000_000 or len(entries) >= 2000 or entry.filename in {e[0] for e in entries}:
|
|
raise ValueError('Invalid or oversized snapshot')
|
|
content = archive.read(entry)
|
|
if len(content) != entry.file_size:
|
|
raise ValueError('Snapshot entry size mismatch')
|
|
files.append({'path': str(rel), 'bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest()})
|
|
entries.append((str(rel), content))
|
|
files.sort(key=lambda entry: entry['path'])
|
|
if review_fingerprint(files) != values['sourceHash']:
|
|
raise ValueError('Source fingerprint mismatch')
|
|
for name, content in entries:
|
|
target = root / name
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_bytes(content)
|
|
(root / '.webup-review-manifest.json').write_text(json.dumps({'sourceHash': values['sourceHash'], 'files': files}))
|
|
return {'imported': len(files), 'sourceHash': values['sourceHash']}
|
|
|
|
|
|
def review_validate(root):
|
|
manifest = json.loads((root / '.webup-review-manifest.json').read_text())
|
|
deadline = time.monotonic() + 100
|
|
steps, issues, html_files = [], [], []
|
|
|
|
def run(label, command, seconds=35):
|
|
remaining = min(seconds, deadline - time.monotonic())
|
|
if remaining <= 0:
|
|
return {'name': label, 'status': 'NOT_RUN', 'reason': 'Review time budget exhausted'}
|
|
env = {'PATH': os.environ.get('PATH', ''), 'HOME': str(root), 'CI': 'true', 'LANG': 'C.UTF-8', 'npm_config_cache': '/tmp/npm-cache'}
|
|
with tempfile.TemporaryFile() as output:
|
|
try:
|
|
process = subprocess.Popen(command, cwd=root, env=env, stdout=output, stderr=subprocess.STDOUT, start_new_session=True)
|
|
try:
|
|
process.wait(timeout=remaining)
|
|
status = 'PASSED' if process.returncode == 0 else 'FAILED'
|
|
except subprocess.TimeoutExpired:
|
|
os.killpg(process.pid, signal.SIGKILL); process.wait(); status = 'TIMEOUT'
|
|
output.seek(0); captured = output.read(6001)
|
|
return {'name': label, 'command': command, 'status': status, 'exitCode': process.returncode,
|
|
'output': review_redact(captured[:6000].decode('utf-8', 'replace')), 'truncated': len(captured) > 6000}
|
|
except OSError:
|
|
return {'name': label, 'status': 'UNSUPPORTED', 'reason': 'Required tool is unavailable in review image'}
|
|
|
|
class Document(HTMLParser):
|
|
def __init__(self):
|
|
super().__init__(convert_charrefs=True); self.assets = []; self.title = False; self.ids=set(); self.anchors=[]
|
|
def handle_starttag(self, tag, attrs):
|
|
attrs = dict(attrs)
|
|
if attrs.get('id'): self.ids.add(attrs['id'])
|
|
if tag=='a' and attrs.get('href','').startswith('#') and len(attrs['href'])>1: self.anchors.append(unquote(attrs['href'][1:]))
|
|
if tag == 'title': self.title = True
|
|
if tag in {'script', 'img', 'source'} and attrs.get('src'): self.assets.append(attrs['src'])
|
|
if tag == 'link' and attrs.get('rel') == 'stylesheet': self.assets.append(attrs.get('href', ''))
|
|
|
|
for entry in manifest['files']:
|
|
file = root / entry['path']
|
|
if file.suffix.lower() in {'.html', '.htm'}:
|
|
html_files.append(entry['path'])
|
|
try:
|
|
document = Document(); document.feed(file.read_text()); document.close()
|
|
if not document.title: issues.append({'file': entry['path'], 'message': 'HTML document has no title'})
|
|
for anchor in document.anchors:
|
|
if anchor not in document.ids: issues.append({'file': entry['path'], 'message': 'Missing local anchor target: #'+anchor})
|
|
for asset in document.assets:
|
|
url = urlsplit(asset)
|
|
if url.scheme or url.netloc: continue
|
|
target = ((root if url.path.startswith('/') else file.parent) / unquote(url.path).lstrip('/')).resolve()
|
|
if not target.is_relative_to(root.resolve()) or not target.is_file():
|
|
issues.append({'file': entry['path'], 'message': 'Missing or unsafe local asset: ' + asset})
|
|
except (ValueError, OSError, UnicodeError):
|
|
issues.append({'file': entry['path'], 'message': 'HTML could not be parsed as UTF-8'})
|
|
if html_files:
|
|
steps.append({'name': 'html-assets', 'status': 'FAILED' if issues else 'PASSED', 'files': html_files, 'issues': issues})
|
|
if (root / 'package.json').is_file():
|
|
try:
|
|
package = json.loads((root / 'package.json').read_text())
|
|
if any((root / lock).exists() for lock in ('pnpm-lock.yaml', 'yarn.lock', 'bun.lockb', 'bun.lock')):
|
|
steps.append({'name': 'dependencies', 'status': 'UNSUPPORTED', 'reason': 'This runner currently requires npm; another lockfile was detected'})
|
|
else:
|
|
install = run('dependencies', ['npm', 'ci' if (root / 'package-lock.json').is_file() else 'install', '--no-audit', '--no-fund'], 45)
|
|
steps.append(install)
|
|
if install['status'] == 'PASSED':
|
|
scripts = package.get('scripts', {})
|
|
for name in ('build', 'test'):
|
|
steps.append(run(name, ['npm', 'run', name], 35) if scripts.get(name) else {'name': name, 'status': 'NOT_CONFIGURED'})
|
|
except (ValueError, OSError):
|
|
steps.append({'name': 'package-json', 'status': 'FAILED', 'reason': 'Invalid package.json'})
|
|
elif any((root / f['path']).suffix == '.py' for f in manifest['files']):
|
|
steps.append(run('python-syntax', ['python3', '-I', '-m', 'compileall', '-q', '.']))
|
|
steps.append({'name': 'python-tests', 'status': 'NOT_RUN', 'reason': 'Dependency installation and Python integration tests are not configured in this runner'})
|
|
elif html_files:
|
|
steps.append({'name': 'build', 'status': 'NOT_APPLICABLE', 'reason': 'Plain static HTML has no build step'})
|
|
else:
|
|
steps.append({'name': 'build', 'status': 'UNSUPPORTED', 'reason': 'Project build system is not supported by this runner'})
|
|
changed = []
|
|
for entry in manifest['files']:
|
|
file = root / entry['path']
|
|
try:
|
|
if file.is_symlink() or hashlib.sha256(file.read_bytes()).hexdigest() != entry['sha256']: changed.append(entry['path'])
|
|
except OSError: changed.append(entry['path'])
|
|
return {'sourceHash': manifest['sourceHash'], 'steps': steps, 'sourceChangedByBuild': changed,
|
|
'browser': {'status': 'NOT_RUN', 'reason': 'No browser or visual checks executed'},
|
|
'hasUi': bool(html_files), 'executionScope': 'Separate gVisor review Pod; development workspace was not modified'}
|