This commit is contained in:
commit
db82325ed7
216 files changed
+33718
No files matched your search
@@ -0,0 +1,13 @@
|
||||
param([string]$OutputDirectory = './data/backups')
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Set-Location -LiteralPath (Split-Path -Parent $PSScriptRoot)
|
||||
New-Item -ItemType Directory -Force -Path $OutputDirectory | Out-Null
|
||||
$fileName = 'platform-' + (Get-Date -Format 'yyyyMMdd-HHmmss') + '.dump'
|
||||
$backupTarget = Join-Path (Resolve-Path -LiteralPath $OutputDirectory).Path $fileName
|
||||
# Docker cp preserves binary format on Windows PowerShell 5.1.
|
||||
docker compose exec -T postgres sh -c 'pg_dump -U "$POSTGRES_USER" -Fc -f /tmp/platform-backup.dump platform'
|
||||
if ($LASTEXITCODE -ne 0) { throw 'pg_dump failed; backup not completed.' }
|
||||
$containerId = docker compose ps -q postgres
|
||||
docker cp "${containerId}:/tmp/platform-backup.dump" $backupTarget
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Backup copy failed.' }
|
||||
Write-Host "Backup created: $backupTarget. This covers platform PostgreSQL only; storage requires a separate S3 backup."
|
||||
Executable
+81
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
if [[ ${WSL_DISTRO_NAME:-} != WebUp-Local || $EUID != 0 ]]; then
|
||||
echo 'Run as root inside the dedicated WebUp-Local WSL distribution.' >&2
|
||||
exit 1
|
||||
fi
|
||||
project_root=$(realpath "$1")
|
||||
cache="$project_root/.tools/local-cluster"
|
||||
lock="$project_root/infra/local/versions.json"
|
||||
if [[ $(uname -m) != x86_64 ]]; then echo 'Pinned artifacts require x86_64.' >&2; exit 1; fi
|
||||
python3 - "$lock" "$cache" <<'PY'
|
||||
import hashlib,json,pathlib,sys
|
||||
config=json.load(open(sys.argv[1]));root=pathlib.Path(sys.argv[2])
|
||||
for name,metadata in config['files'].items():
|
||||
digest=hashlib.file_digest(open(root/name,'rb'),'sha256').hexdigest()
|
||||
if digest!=metadata['sha256']:raise SystemExit('Checksum mismatch: '+name)
|
||||
print('All official artifact checksums verified.')
|
||||
PY
|
||||
if [[ -e /etc/rancher/k3s/config.yaml && ! -e /etc/webup-local/environment ]]; then
|
||||
echo 'An unowned K3s installation already exists; refusing to overwrite it.' >&2
|
||||
exit 1
|
||||
fi
|
||||
install -d -m 0755 /etc/webup-local /opt/webup-local /etc/rancher/k3s
|
||||
printf '%s\n' WebUp-Local > /etc/webup-local/environment
|
||||
cp "$lock" /etc/webup-local/artifact-lock.json
|
||||
gvisor_release=$(python3 -c 'import json,sys;print(json.load(open(sys.argv[1]))["gvisorRelease"])' "$lock")
|
||||
gvisor_dir="/opt/webup-local/$gvisor_release"
|
||||
install -d -m 0755 "$gvisor_dir"
|
||||
if [[ ! -e "$gvisor_dir/.complete" ]]; then
|
||||
python3 - "$cache/gvisor-x86_64.tar.bz2" "$gvisor_dir" <<'PY'
|
||||
import tarfile,sys
|
||||
with tarfile.open(sys.argv[1]) as archive:archive.extractall(sys.argv[2],filter='data')
|
||||
PY
|
||||
chmod 0755 "$gvisor_dir/runsc" "$gvisor_dir/containerd-shim-runsc-v1"
|
||||
touch "$gvisor_dir/.complete"
|
||||
fi
|
||||
ln -sfn "$gvisor_dir/runsc" /usr/local/bin/runsc
|
||||
ln -sfn "$gvisor_dir/containerd-shim-runsc-v1" /usr/local/bin/containerd-shim-runsc-v1
|
||||
install -m 0755 "$cache/k3s" /usr/local/bin/k3s
|
||||
install -d -m 0755 /var/lib/rancher/k3s/agent/etc/containerd /var/lib/rancher/k3s/agent/etc/kubelet.conf.d /var/lib/rancher/k3s/agent/images
|
||||
cat > /var/lib/rancher/k3s/agent/etc/containerd/config-v3.toml.tmpl <<'TOML'
|
||||
{{ template "base" . }}
|
||||
[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runsc]
|
||||
runtime_type = "io.containerd.runsc.v1"
|
||||
TOML
|
||||
cat > /var/lib/rancher/k3s/agent/etc/kubelet.conf.d/20-webup-pids.conf <<'YAML'
|
||||
apiVersion: kubelet.config.k8s.io/v1beta1
|
||||
kind: KubeletConfiguration
|
||||
podPidsLimit: 512
|
||||
YAML
|
||||
cat > /etc/rancher/k3s/config.yaml <<'YAML'
|
||||
node-name: webup-local
|
||||
write-kubeconfig-mode: "0600"
|
||||
disable:
|
||||
- traefik
|
||||
- servicelb
|
||||
secrets-encryption: true
|
||||
YAML
|
||||
if [[ ! -e /var/lib/rancher/k3s/agent/images/k3s-airgap-images-amd64.tar.zst ]]; then
|
||||
cp "$cache/k3s-airgap-images-amd64.tar.zst" /var/lib/rancher/k3s/agent/images/
|
||||
fi
|
||||
version=$(python3 -c 'import json,sys;print(json.load(open(sys.argv[1]))["k3sVersion"])' "$lock")
|
||||
INSTALL_K3S_VERSION="$version" INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_SKIP_START=true sh "$cache/install-k3s.sh" server
|
||||
systemctl start k3s
|
||||
for attempt in $(seq 1 60); do
|
||||
if k3s kubectl get node webup-local >/dev/null 2>&1; then break; fi
|
||||
sleep 2
|
||||
done
|
||||
k3s kubectl wait --for=condition=Ready node/webup-local --timeout=120s
|
||||
cat <<'YAML' | k3s kubectl apply -f -
|
||||
apiVersion: node.k8s.io/v1
|
||||
kind: RuntimeClass
|
||||
metadata:
|
||||
name: gvisor
|
||||
handler: runsc
|
||||
YAML
|
||||
k3s kubectl apply -f "$project_root/infra/k8s/sandbox-rbac.yaml"
|
||||
k3s kubectl apply -f "$project_root/infra/k8s/runtime-rbac.yaml"
|
||||
k3s kubectl apply -f "$project_root/infra/k8s/review-rbac.yaml"
|
||||
k3s kubectl get nodes
|
||||
echo 'Local K3s installed. Isolation smoke tests are required before enabling AI execution.'
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
[[ ${WSL_DISTRO_NAME:-} == WebUp-Local && $EUID == 0 && -e /etc/webup-local/environment ]] || { echo 'Dedicated local environment required.' >&2; exit 1; }
|
||||
project_root=$(realpath "$1")
|
||||
docker build --build-arg BASE_IMAGE=public.ecr.aws/docker/library/python:3.12-slim \
|
||||
-t webup-sandbox-gateway:local-v1 "$project_root/node-agent"
|
||||
docker build --build-arg BASE_IMAGE=public.ecr.aws/docker/library/node:24-bookworm-slim \
|
||||
--build-arg DEBIAN_MIRROR=mirrors.ustc.edu.cn \
|
||||
-t webup-development-sandbox:local-v1 "$project_root/infra/sandbox"
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Restart only the dedicated test PostgreSQL and compare persisted business data."""
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
if os.environ.get('WSL_DISTRO_NAME') != 'WebUp-Local' or os.geteuid() != 0:
|
||||
raise SystemExit('Dedicated local environment required.')
|
||||
name = 'webup-test-postgres'
|
||||
|
||||
|
||||
def docker(*args):
|
||||
return subprocess.check_output(['docker', *args], text=True, timeout=60)
|
||||
|
||||
|
||||
metadata = json.loads(docker('container', 'inspect', name))[0]
|
||||
if metadata['Config'].get('Labels', {}).get('webup.local-test') != 'true':
|
||||
raise SystemExit('Refusing to restart an unowned container.')
|
||||
tables = ('users', 'wallets', 'wallet_ledger', 'ai_tasks', 'ai_usage_records')
|
||||
|
||||
|
||||
def fingerprint():
|
||||
result = {}
|
||||
for table in tables:
|
||||
key = 'user_id' if table == 'wallets' else 'id'
|
||||
query = "SELECT count(*) || ':' || md5(COALESCE(jsonb_agg(to_jsonb(t) ORDER BY " + key + ")::text,'[]')) FROM " + table + ' t'
|
||||
result[table] = docker('exec', name, 'psql', '-XAt', '-v', 'ON_ERROR_STOP=1', '-U', 'webup_test', '-d', 'webup_integration', '-c', query).strip()
|
||||
result['migrations'] = docker('exec', name, 'psql', '-XAt', '-v', 'ON_ERROR_STOP=1', '-U', 'webup_test', '-d', 'webup_integration', '-c', 'SELECT count(*) FROM flyway_schema_history WHERE success').strip()
|
||||
return result
|
||||
|
||||
|
||||
before = fingerprint()
|
||||
if int(before['users'].split(':')[0]) == 0 or int(before['wallet_ledger'].split(':')[0]) == 0:
|
||||
raise SystemExit('Run the integration tests first; an empty database is not a persistence check.')
|
||||
docker('restart', '--time', '20', name)
|
||||
for attempt in range(30):
|
||||
ready = subprocess.run(['docker', 'exec', name, 'pg_isready', '-U', 'webup_test', '-d', 'webup_integration'], capture_output=True, timeout=5)
|
||||
if ready.returncode == 0:
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise SystemExit('PostgreSQL failed to recover after restart.')
|
||||
after = fingerprint()
|
||||
if before != after:
|
||||
raise SystemExit('Persisted database fingerprints changed after restart.')
|
||||
report = {'passed': True, 'checkedAtUnix': time.time(), 'database': 'webup_integration', 'fingerprints': after}
|
||||
target = pathlib.Path(__file__).resolve().parents[2] / '.runtime/local-cluster/postgres-persistence.json'
|
||||
target.write_text(json.dumps(report, indent=2))
|
||||
print('PASS PostgreSQL restart preserved users, wallets, ledger, task / usage records and migration history.')
|
||||
Executable
+59
@@ -0,0 +1,59 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
[[ ${WSL_DISTRO_NAME:-} == WebUp-Local && $EUID == 0 && -e /etc/webup-local/environment ]] || { echo 'Dedicated local environment required.' >&2; exit 1; }
|
||||
project_root=$(realpath "$1")
|
||||
k3s kubectl apply -f "$project_root/infra/k8s/review-rbac.yaml"
|
||||
install -d -m 0700 /etc/webup-local
|
||||
if [[ ! -e /etc/webup-local/gateway.env ]]; then
|
||||
python3 - <<'PY'
|
||||
import os,secrets
|
||||
fd=os.open('/etc/webup-local/gateway.env',os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600)
|
||||
with os.fdopen(fd,'w') as stream:
|
||||
stream.write('SANDBOX_TOKEN='+secrets.token_urlsafe(32)+'\nRUNTIME_TOKEN='+secrets.token_urlsafe(32)+'\n')
|
||||
PY
|
||||
fi
|
||||
docker image inspect webup-sandbox-gateway:local-v1 webup-development-sandbox:local-v1 >/dev/null
|
||||
docker save webup-sandbox-gateway:local-v1 webup-development-sandbox:local-v1 | k3s ctr --namespace k8s.io images import -
|
||||
k3s kubectl --namespace platform-sandboxes create secret generic sandbox-gateway --from-env-file=/etc/webup-local/gateway.env --dry-run=client -o yaml | k3s kubectl apply -f -
|
||||
python3 - "$project_root/infra/k8s/sandbox-gateway.yaml" <<'PY' | k3s kubectl apply -f -
|
||||
import pathlib,sys
|
||||
text=pathlib.Path(sys.argv[1]).read_text()
|
||||
text=text.replace('registry.example.invalid/platform/sandbox-gateway:0.2.0','docker.io/library/webup-sandbox-gateway:local-v1')
|
||||
text=text.replace('registry.example.invalid/platform/development-sandbox:0.2.0','docker.io/library/webup-development-sandbox:local-v1')
|
||||
print(text)
|
||||
PY
|
||||
k3s kubectl --namespace platform-sandboxes rollout restart deployment/sandbox-gateway
|
||||
k3s kubectl --namespace platform-sandboxes rollout status deployment/sandbox-gateway --timeout=120s
|
||||
cat > /etc/systemd/system/webup-gateway-forward.service <<'UNIT'
|
||||
[Unit]
|
||||
Description=WebUp private local sandbox gateway forwarding
|
||||
After=k3s.service
|
||||
Requires=k3s.service
|
||||
[Service]
|
||||
Type=simple
|
||||
Environment=KUBECONFIG=/etc/rancher/k3s/k3s.yaml
|
||||
ExecStart=/usr/local/bin/k3s kubectl --namespace platform-sandboxes port-forward --address 127.0.0.1 service/sandbox-gateway 8090:8090
|
||||
Restart=always
|
||||
RestartSec=2
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
UNIT
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now webup-gateway-forward
|
||||
install -d -m 0700 "$project_root/.runtime/local-cluster"
|
||||
cp /etc/webup-local/gateway.env "$project_root/.runtime/local-cluster/gateway.env"
|
||||
python3 - <<'PY'
|
||||
import json,time,urllib.request
|
||||
values=dict(line.strip().split('=',1) for line in open('/etc/webup-local/gateway.env') if '=' in line)
|
||||
for attempt in range(30):
|
||||
try:
|
||||
request=urllib.request.Request('http://127.0.0.1:8090/health/ready',headers={'Authorization':'Bearer '+values['SANDBOX_TOKEN']})
|
||||
with urllib.request.urlopen(request,timeout=5) as response:
|
||||
if json.load(response).get('ready'):
|
||||
print('Private gateway authentication and Kubernetes health check passed.')
|
||||
break
|
||||
except Exception:
|
||||
time.sleep(2)
|
||||
else:
|
||||
raise SystemExit('Gateway readiness did not pass; AI execution remains disabled.')
|
||||
PY
|
||||
@@ -0,0 +1,38 @@
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$projectRoot = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '../..'))
|
||||
$localDirectory = Join-Path $projectRoot '.runtime/local-cluster'
|
||||
$gatewayFile = Join-Path $localDirectory 'gateway.env'
|
||||
$reportFile = Join-Path $localDirectory 'smoke-report.json'
|
||||
$triggerFile = Join-Path $projectRoot 'backend/target/classes/.reloadtrigger'
|
||||
if (-not (Test-Path -LiteralPath $reportFile)) { throw 'Run the actual local isolation smoke checks first.' }
|
||||
$report = Get-Content -LiteralPath $reportFile -Raw | ConvertFrom-Json
|
||||
if (-not $report.passed -or [DateTimeOffset]::UtcNow.ToUnixTimeSeconds() - $report.checkedAtUnix -gt 86400) {
|
||||
throw 'The local isolation checks must have passed within the last 24 hours.'
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $triggerFile)) { throw 'Compile and start the local preview backend first.' }
|
||||
# Restrict the directory before reading or writing local infrastructure credentials.
|
||||
$acl = [Security.AccessControl.DirectorySecurity]::new()
|
||||
$acl.SetAccessRuleProtection($true, $false)
|
||||
foreach ($identity in @([Security.Principal.WindowsIdentity]::GetCurrent().User, [Security.Principal.SecurityIdentifier]::new('S-1-5-18'))) {
|
||||
$acl.AddAccessRule([Security.AccessControl.FileSystemAccessRule]::new($identity, 'FullControl', 'ContainerInherit,ObjectInherit', 'None', 'Allow'))
|
||||
}
|
||||
Set-Acl -LiteralPath $localDirectory -AclObject $acl
|
||||
$gatewayValues = @{}
|
||||
Get-Content -LiteralPath $gatewayFile | ForEach-Object {
|
||||
$pair = $_ -split '=', 2
|
||||
if ($pair.Count -eq 2) { $gatewayValues[$pair[0]] = $pair[1] }
|
||||
}
|
||||
foreach ($key in @('SANDBOX_TOKEN', 'RUNTIME_TOKEN')) {
|
||||
if ($gatewayValues[$key].Length -lt 32) { throw 'Local gateway credentials are missing or invalid.' }
|
||||
}
|
||||
$health = Invoke-RestMethod -Uri http://127.0.0.1:8090/health/ready -Headers @{Authorization = ('Bearer ' + $gatewayValues['SANDBOX_TOKEN'])} -TimeoutSec 10
|
||||
if (-not $health.ready) { throw 'Gateway Kubernetes readiness check failed.' }
|
||||
$infra = @{app = @{
|
||||
'worker-enabled' = $true; 'worker-capacity' = 4
|
||||
'sandbox-url' = 'http://127.0.0.1:8090'; 'sandbox-token' = $gatewayValues['SANDBOX_TOKEN']
|
||||
'runtime-url' = 'http://127.0.0.1:8090'; 'runtime-token' = $gatewayValues['RUNTIME_TOKEN']
|
||||
'runtime-worker-enabled' = $false
|
||||
}}
|
||||
$infra | ConvertTo-Json -Depth 4 | Set-Content -LiteralPath (Join-Path $localDirectory 'preview-infrastructure.yml') -Encoding utf8
|
||||
Set-Content -LiteralPath $triggerFile -Value (Get-Date -Format o)
|
||||
Write-Output 'Local preview AI execution enabled. Check READY in the administrator server page.'
|
||||
@@ -0,0 +1,31 @@
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$projectRoot = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '../..'))
|
||||
$configuration = @{}
|
||||
Get-Content -LiteralPath (Join-Path $projectRoot '.runtime/local-cluster/services.env') | ForEach-Object {
|
||||
$pair = $_ -split '=', 2
|
||||
if ($pair.Count -eq 2 -and ($pair[0] -like 'TEST_*' -or $pair[0] -eq 'RUN_REDIS_TESTS')) {
|
||||
$configuration[$pair[0]] = $pair[1]
|
||||
}
|
||||
}
|
||||
# Tests intentionally clear their database. Only this dedicated local target is allowed.
|
||||
if ($configuration['TEST_DATABASE_URL'] -ne 'jdbc:postgresql://127.0.0.1:15432/webup_integration' -or
|
||||
$configuration['TEST_DATABASE_USER'] -ne 'webup_test' -or
|
||||
$configuration['TEST_REDIS_HOST'] -ne '127.0.0.1' -or $configuration['TEST_REDIS_PORT'] -ne '16379' -or
|
||||
$configuration['RUN_REDIS_TESTS'] -ne 'true') {
|
||||
throw 'Only the dedicated disposable local PostgreSQL / Redis targets are accepted.'
|
||||
}
|
||||
$previous = @{}
|
||||
foreach ($key in $configuration.Keys) { $previous[$key] = [Environment]::GetEnvironmentVariable($key, 'Process') }
|
||||
$originalJava = $env:JAVA_HOME
|
||||
Push-Location (Join-Path $projectRoot 'backend')
|
||||
try {
|
||||
foreach ($key in $configuration.Keys) { [Environment]::SetEnvironmentVariable($key, $configuration[$key], 'Process') }
|
||||
$bundledJava = Join-Path $projectRoot '.tools/jdk-21.0.13.0.2+1'
|
||||
if (Test-Path -LiteralPath $bundledJava) { $env:JAVA_HOME = $bundledJava }
|
||||
& .\mvnw.cmd verify
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Real PostgreSQL / Redis verification failed.' }
|
||||
} finally {
|
||||
foreach ($key in $previous.Keys) { [Environment]::SetEnvironmentVariable($key, $previous[$key], 'Process') }
|
||||
$env:JAVA_HOME = $originalJava
|
||||
Pop-Location
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Real local-only gateway, gVisor, persistence and network isolation checks."""
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
if os.environ.get('WSL_DISTRO_NAME') != 'WebUp-Local':
|
||||
raise SystemExit('Run inside WebUp-Local.')
|
||||
values = dict(line.strip().split('=', 1) for line in open('/etc/webup-local/gateway.env') if '=' in line)
|
||||
report = {'checks': [], 'passed': False}
|
||||
|
||||
|
||||
def check(condition, name):
|
||||
if not condition:
|
||||
raise RuntimeError('Local isolation check failed: ' + name)
|
||||
report['checks'].append(name)
|
||||
print('PASS ' + name, flush=True)
|
||||
|
||||
|
||||
def api(path, payload=None, authenticated=True):
|
||||
headers = {'Content-Type': 'application/json'}
|
||||
if authenticated:
|
||||
headers['Authorization'] = 'Bearer ' + values['SANDBOX_TOKEN']
|
||||
request = urllib.request.Request('http://127.0.0.1:8090' + path, data=None if payload is None else json.dumps(payload).encode(), headers=headers)
|
||||
with urllib.request.urlopen(request, timeout=145) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def kubectl(*args):
|
||||
return subprocess.check_output(['k3s', 'kubectl', *args], text=True, timeout=60)
|
||||
|
||||
|
||||
project = str(uuid.uuid4())
|
||||
pod_name = 'dev-' + uuid.UUID(project).hex
|
||||
try:
|
||||
try:
|
||||
api('/health/ready', authenticated=False)
|
||||
raise RuntimeError('Unauthenticated gateway request was accepted')
|
||||
except urllib.error.HTTPError as error:
|
||||
check(error.code == 401, 'gateway rejects unauthenticated requests')
|
||||
check(api('/health/ready').get('ready'), 'gateway performs actual Kubernetes health query')
|
||||
path = '/workspaces/' + project + '/tools'
|
||||
def tool(name, arguments):
|
||||
return api(path, {'name': name, 'arguments': arguments})
|
||||
check(tool('write_file', {'path': 'smoke/hello.txt', 'content': 'WebUp local isolated workspace'}).get('written') is not None, 'workspace provisions real PVC and writes file')
|
||||
check(tool('read_file', {'path': 'smoke/hello.txt'}).get('content') == 'WebUp local isolated workspace', 'workspace file roundtrip')
|
||||
check('error' in tool('write_file', {'path': '../outside.txt', 'content': 'denied'}), 'file tool rejects parent traversal')
|
||||
command = '''python3 - <<'PY'
|
||||
import os,pathlib,socket,json
|
||||
assert os.getuid()==10001
|
||||
assert not pathlib.Path('/var/run/secrets/kubernetes.io/serviceaccount/token').exists()
|
||||
assert not any(k in os.environ for k in ('SANDBOX_TOKEN','RUNTIME_TOKEN','APP_ENCRYPTION_KEY'))
|
||||
try:
|
||||
pathlib.Path('/webup-forbidden-write').write_text('denied')
|
||||
except OSError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError('Root filesystem was writable')
|
||||
for host in ('169.254.169.254','10.43.0.1'):
|
||||
try:
|
||||
socket.create_connection((host,443),timeout=3).close()
|
||||
except OSError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError('Private network reachable: '+host)
|
||||
socket.create_connection(('example.com',443),timeout=10).close()
|
||||
print(json.dumps({'uid':os.getuid(),'privateBlocked':True,'publicHttpsReachable':True}))
|
||||
PY'''
|
||||
result = tool('run_command', {'command': command})
|
||||
check(result.get('exitCode') == 0 and not result.get('timedOut'), 'non-root execution, no credentials, read-only root, blocked private network and working public HTTPS')
|
||||
pod = json.loads(kubectl('-n', 'platform-sandboxes', 'get', 'pod', pod_name, '-o', 'json'))
|
||||
check(pod['spec']['runtimeClassName'] == 'gvisor', 'workspace requests isolated RuntimeClass')
|
||||
container = pod['status']['containerStatuses'][0]['containerID'].split('://', 1)[1]
|
||||
runtime = json.loads(subprocess.check_output(['k3s', 'ctr', '--namespace', 'k8s.io', 'containers', 'info', container], text=True, timeout=20))
|
||||
actual = runtime.get('Runtime', runtime.get('runtime', {}))
|
||||
check(actual.get('Name', actual.get('name')) == 'io.containerd.runsc.v1', 'running container actually uses gVisor shim rather than runc')
|
||||
configuration = json.loads(kubectl('get', '--raw', '/api/v1/nodes/webup-local/proxy/configz'))
|
||||
check(configuration['kubeletconfig']['podPidsLimit'] == 512, 'kubelet enforces a finite per-pod process limit')
|
||||
kubectl('-n', 'platform-sandboxes', 'delete', 'pod', pod_name, '--wait=true', '--timeout=45s')
|
||||
check(tool('read_file', {'path': 'smoke/hello.txt'}).get('content') == 'WebUp local isolated workspace', 'PVC survives workspace pod recreation')
|
||||
report['passed'] = True
|
||||
finally:
|
||||
report['projectId'] = project
|
||||
report['pod'] = pod_name
|
||||
report['checkedAtUnix'] = time.time()
|
||||
target = pathlib.Path('/etc/webup-local/smoke-report.json')
|
||||
target.write_text(json.dumps(report, indent=2))
|
||||
host_report = pathlib.Path(__file__).resolve().parents[2] / '.runtime/local-cluster/smoke-report.json'
|
||||
host_report.write_text(json.dumps(report, indent=2))
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Actual isolated review copy / validation / cleanup. Only disposable random fixture IDs."""
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
if os.environ.get('WSL_DISTRO_NAME') != 'WebUp-Local' or os.geteuid() != 0:
|
||||
raise SystemExit('Dedicated local environment required.')
|
||||
tokens = dict(line.strip().split('=', 1) for line in open('/etc/webup-local/gateway.env'))
|
||||
original, review = str(uuid.uuid4()), str(uuid.uuid4())
|
||||
report = {'passed': False, 'checks': [], 'fixtureProject': original, 'reviewWorkspace': review}
|
||||
|
||||
|
||||
def api(path, payload=None, method=None):
|
||||
request = urllib.request.Request('http://127.0.0.1:8090' + path, headers={'Authorization': 'Bearer ' + tokens['SANDBOX_TOKEN'], 'Content-Type': 'application/json'},
|
||||
data=None if payload is None else json.dumps(payload).encode(), method=method)
|
||||
with urllib.request.urlopen(request, timeout=150) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def tool(identity, name, arguments, reviewing=False):
|
||||
return api(('/reviews/' if reviewing else '/workspaces/') + identity + '/tools', {'name': name, 'arguments': arguments})
|
||||
|
||||
|
||||
def check(value, label):
|
||||
if not value:
|
||||
raise RuntimeError('FAIL ' + label)
|
||||
report['checks'].append(label)
|
||||
print('PASS ' + label, flush=True)
|
||||
|
||||
|
||||
try:
|
||||
check(tool(original, 'write_file', {'path': 'index.html', 'content': '<title>Review fixture</title><link rel="stylesheet" href="styles.css">'}).get('written'), 'real development fixture writes through stdin')
|
||||
tool(original, 'write_file', {'path': 'styles.css', 'content': 'body { color: navy; }'})
|
||||
tool(original, 'write_file', {'path': '.env', 'content': 'PRIVATE_TEST_VALUE=must-not-copy'})
|
||||
check(tool(original, 'write_file', {'path': 'large.txt', 'content': '中' * 64000}).get('written') == 192000, 'large UTF-8 payload is not passed through process arguments')
|
||||
snapshot = tool(original, 'review_export', {})
|
||||
check(not any(entry['path'] == '.env' for entry in snapshot['files']), 'snapshot excludes credential files')
|
||||
imported = tool(review, 'review_import', snapshot, True)
|
||||
check(imported.get('sourceHash') == snapshot['sourceHash'], 'isolated copy matches exported source fingerprint')
|
||||
checked = tool(review, 'review_validate', {}, True)
|
||||
check(checked.get('sourceHash') == snapshot['sourceHash'] and checked['steps'][0]['status'] == 'PASSED', 'HTML and stylesheet verified in actual review Pod')
|
||||
check(checked['browser']['status'] == 'NOT_RUN', 'unexecuted browser checks are explicitly reported')
|
||||
check(tool(original, 'review_export', {})['sourceHash'] == snapshot['sourceHash'], 'development source unchanged by review execution')
|
||||
pod_name = 'dev-' + uuid.UUID(review).hex
|
||||
pod = json.loads(subprocess.check_output(['k3s', 'kubectl', '-n', 'platform-reviews', 'get', 'pod', pod_name, '-o', 'json'], text=True))
|
||||
check(pod['spec']['runtimeClassName'] == 'gvisor' and not pod['spec']['automountServiceAccountToken'], 'separate review namespace uses gVisor without service account token')
|
||||
container = pod['status']['containerStatuses'][0]['containerID'].split('://', 1)[1]
|
||||
runtime = json.loads(subprocess.check_output(['k3s', 'ctr', '-n', 'k8s.io', 'containers', 'info', container], text=True))
|
||||
check(runtime.get('Runtime', runtime.get('runtime', {})).get('Name') == 'io.containerd.runsc.v1', 'review container actually runs through runsc')
|
||||
try:
|
||||
tool(review, 'run_command', {'command': 'echo not allowed'}, True)
|
||||
raise RuntimeError('Review API accepted arbitrary command')
|
||||
except urllib.error.HTTPError as error:
|
||||
check(error.code == 400, 'private review API rejects arbitrary shell tools')
|
||||
api('/reviews/' + review, method='DELETE')
|
||||
review = str(uuid.uuid4())
|
||||
package = {'name':'webup-review-fixture','version':'1.0.0','private':True,
|
||||
'scripts':{'build':'node build.cjs','test':'node --test'}}
|
||||
tool(original,'write_file',{'path':'package.json','content':json.dumps(package)})
|
||||
tool(original,'write_file',{'path':'build.cjs','content':"const fs=require('node:fs');fs.mkdirSync('dist',{recursive:true});fs.copyFileSync('index.html','dist/index.html');console.log('REAL_BUILD_OK');"})
|
||||
tool(original,'write_file',{'path':'site.test.cjs','content':"const test=require('node:test'),assert=require('node:assert/strict'),fs=require('node:fs');test('actual HTML artifact exists',()=>{assert.ok(fs.readFileSync('dist/index.html','utf8').includes('Review fixture'));});"})
|
||||
node_snapshot=tool(original,'review_export',{})
|
||||
tool(review,'review_import',node_snapshot,True)
|
||||
node_checked=tool(review,'review_validate',{},True)
|
||||
statuses={step['name']:step['status'] for step in node_checked['steps']}
|
||||
check(statuses.get('dependencies')=='PASSED' and statuses.get('build')=='PASSED' and statuses.get('test')=='PASSED','actual npm installation, build and Node test execute successfully inside copied review Pod')
|
||||
check(tool(original,'review_export',{})['sourceHash']==node_snapshot['sourceHash'],'npm artifacts and dependency files do not modify development workspace')
|
||||
api('/reviews/'+review,method='DELETE')
|
||||
review=str(uuid.uuid4())
|
||||
tool(original,'write_file',{'path':'build.cjs','content':'process.exit(7);'})
|
||||
failed_snapshot=tool(original,'review_export',{})
|
||||
tool(review,'review_import',failed_snapshot,True)
|
||||
failed=tool(review,'review_validate',{},True)
|
||||
check(any(step['name']=='build' and step['status']=='FAILED' and step['exitCode']!=0 for step in failed['steps']),'real failing build is recorded as failure')
|
||||
report['passed'] = True
|
||||
finally:
|
||||
try:
|
||||
api('/reviews/' + review, method='DELETE')
|
||||
except Exception:
|
||||
report['cleanupError'] = True
|
||||
# These random IDs were generated above, never read from user data.
|
||||
own_name = 'dev-' + uuid.UUID(original).hex
|
||||
subprocess.run(['k3s', 'kubectl', '-n', 'platform-sandboxes', 'delete', 'pod,pvc,networkpolicy', own_name, '--ignore-not-found', '--wait=false'], stdout=subprocess.DEVNULL, timeout=20)
|
||||
report['checkedAtUnix'] = time.time()
|
||||
target = pathlib.Path(__file__).resolve().parents[2] / '.runtime/local-cluster/code-review-isolation.json'
|
||||
target.write_text(json.dumps(report, indent=2))
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Dedicated disposable PostgreSQL / Redis; never modifies the preview database."""
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import secrets
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
if os.environ.get('WSL_DISTRO_NAME') != 'WebUp-Local' or os.geteuid() != 0:
|
||||
raise SystemExit('Dedicated local environment required.')
|
||||
root = pathlib.Path('/etc/webup-local/test-services')
|
||||
root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
credentials = root / 'services.env'
|
||||
if not credentials.exists():
|
||||
password, redis_password = secrets.token_hex(32), secrets.token_hex(32)
|
||||
values = {
|
||||
'POSTGRES_DB': 'webup_integration', 'POSTGRES_USER': 'webup_test', 'POSTGRES_PASSWORD': password,
|
||||
'TEST_DATABASE_URL': 'jdbc:postgresql://127.0.0.1:15432/webup_integration',
|
||||
'TEST_DATABASE_USER': 'webup_test', 'TEST_DATABASE_PASSWORD': password,
|
||||
'TEST_REDIS_HOST': '127.0.0.1', 'TEST_REDIS_PORT': '16379',
|
||||
'TEST_REDIS_PASSWORD': redis_password, 'RUN_REDIS_TESTS': 'true',
|
||||
}
|
||||
credentials.write_text(''.join(k + '=' + v + '\n' for k, v in values.items()))
|
||||
credentials.chmod(0o600)
|
||||
values = dict(line.split('=', 1) for line in credentials.read_text().splitlines())
|
||||
postgres_env = root / 'postgres.env'
|
||||
postgres_env.write_text(''.join(k + '=' + values[k] + '\n' for k in ('POSTGRES_DB', 'POSTGRES_USER', 'POSTGRES_PASSWORD')))
|
||||
postgres_env.chmod(0o600)
|
||||
redis_config = root / 'redis.conf'
|
||||
redis_config.write_text('bind 0.0.0.0\nprotected-mode yes\nrequirepass ' + values['TEST_REDIS_PASSWORD'] + '\nsave ""\nappendonly no\n')
|
||||
# Readable by the container's unprivileged Redis user; parent directory stays root-only.
|
||||
redis_config.chmod(0o444)
|
||||
|
||||
|
||||
def docker(*args, **kwargs):
|
||||
return subprocess.run(['docker', *args], text=True, timeout=120, **kwargs)
|
||||
|
||||
|
||||
for name, args in (
|
||||
('webup-test-postgres', ['--publish', '127.0.0.1:15432:5432', '--env-file', str(postgres_env),
|
||||
'--volume', 'webup-test-postgres-data:/var/lib/postgresql/data',
|
||||
'public.ecr.aws/docker/library/postgres:17']),
|
||||
('webup-test-redis', ['--publish', '127.0.0.1:16379:6379', '--mount', 'type=bind,source=' + str(redis_config) + ',target=/etc/redis/redis.conf,readonly',
|
||||
'public.ecr.aws/docker/library/redis:7.4', 'redis-server', '/etc/redis/redis.conf']),
|
||||
):
|
||||
existing = docker('container', 'inspect', name, capture_output=True)
|
||||
if existing.returncode == 0:
|
||||
metadata = json.loads(existing.stdout)[0]
|
||||
if metadata['Config'].get('Labels', {}).get('webup.local-test') != 'true':
|
||||
raise SystemExit('Refusing to modify an unowned container: ' + name)
|
||||
docker('start', name, check=True, stdout=subprocess.DEVNULL)
|
||||
else:
|
||||
docker('run', '--detach', '--name', name, '--restart', 'unless-stopped', '--label', 'webup.local-test=true', *args,
|
||||
check=True, stdout=subprocess.DEVNULL)
|
||||
for attempt in range(30):
|
||||
ready = docker('exec', 'webup-test-postgres', 'pg_isready', '-U', 'webup_test', '-d', 'webup_integration', capture_output=True)
|
||||
if ready.returncode == 0:
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise SystemExit('PostgreSQL did not become ready')
|
||||
host_credentials = pathlib.Path(__file__).resolve().parents[2] / '.runtime/local-cluster/services.env'
|
||||
host_credentials.write_text(credentials.read_text())
|
||||
print('Disposable PostgreSQL 17 and password-protected Redis 7.4 started on loopback ports 15432 / 16379.')
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Download checksum-verified, pinned official Linux artifacts for the local WSL cluster."""
|
||||
import concurrent.futures
|
||||
import hashlib
|
||||
import json
|
||||
import pathlib
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
CACHE = ROOT / '.tools' / 'local-cluster'
|
||||
LOCK = ROOT / 'infra' / 'local' / 'versions.json'
|
||||
|
||||
|
||||
def read(url):
|
||||
request = urllib.request.Request(url, headers={'User-Agent': 'WebUp-Local-Development'})
|
||||
with urllib.request.urlopen(request, timeout=40) as response:
|
||||
return response.read()
|
||||
|
||||
|
||||
def checksums(data):
|
||||
return {parts[-1].lstrip('*').removeprefix('./'): parts[0] for line in data.decode().splitlines() if len(parts := line.split()) >= 2}
|
||||
|
||||
|
||||
def manifest():
|
||||
if LOCK.exists():
|
||||
return json.loads(LOCK.read_text(encoding='utf-8'))
|
||||
channels = json.loads(read('https://update.k3s.io/v1-release/channels'))
|
||||
version = next(item['latest'] for item in channels['data'] if item['name'] == 'stable')
|
||||
release = json.loads(read('https://api.github.com/repos/k3s-io/k3s/releases/tags/' + urllib.parse.quote(version, safe='')))
|
||||
assets = {item['name']: item['browser_download_url'] for item in release['assets']}
|
||||
hashes = checksums(read(assets['sha256sum-amd64.txt']))
|
||||
files = {name: {'url': assets[name], 'sha256': hashes[name]} for name in ['k3s', 'k3s-airgap-images-amd64.tar.zst']}
|
||||
installer = 'https://raw.githubusercontent.com/k3s-io/k3s/' + urllib.parse.quote(version, safe='') + '/install.sh'
|
||||
files['install-k3s.sh'] = {'url': installer, 'sha256': hashlib.sha256(read(installer)).hexdigest()}
|
||||
gvisor = json.loads(read('https://api.github.com/repos/google/gvisor/releases/latest'))
|
||||
assets = {item['name']: item['browser_download_url'] for item in gvisor['assets']}
|
||||
hashes = checksums(read(assets['SHA256SUMS']))
|
||||
name = 'gvisor-x86_64.tar.bz2'
|
||||
files[name] = {'url': assets[name], 'sha256': hashes[name]}
|
||||
result = {'k3sVersion': version, 'gvisorRelease': gvisor['tag_name'], 'architecture': 'x86_64', 'files': files}
|
||||
LOCK.parent.mkdir(parents=True, exist_ok=True)
|
||||
LOCK.write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8')
|
||||
return result
|
||||
|
||||
|
||||
def digest(path):
|
||||
sha = hashlib.sha256()
|
||||
with path.open('rb') as stream:
|
||||
while chunk := stream.read(1024 * 1024):
|
||||
sha.update(chunk)
|
||||
return sha.hexdigest()
|
||||
|
||||
|
||||
def download(item):
|
||||
name, metadata = item
|
||||
path = CACHE / name
|
||||
if path.exists() and digest(path) == metadata['sha256']:
|
||||
print(name + ': checksum verified (cached)', flush=True)
|
||||
return
|
||||
partial = path.with_name(path.name + '.part')
|
||||
request = urllib.request.Request(metadata['url'], headers={'User-Agent': 'WebUp-Local-Development'})
|
||||
print(name + ': downloading official release', flush=True)
|
||||
with urllib.request.urlopen(request, timeout=60) as response, partial.open('wb') as output:
|
||||
total = 0
|
||||
while chunk := response.read(1024 * 1024):
|
||||
total += len(chunk)
|
||||
if total > 512 * 1024 * 1024:
|
||||
raise RuntimeError('Artifact exceeds the configured size bound: ' + name)
|
||||
output.write(chunk)
|
||||
if digest(partial) != metadata['sha256']:
|
||||
raise RuntimeError('Checksum mismatch; artifact was not installed: ' + name)
|
||||
partial.replace(path)
|
||||
print(name + ': checksum verified', flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
CACHE.mkdir(parents=True, exist_ok=True)
|
||||
configuration = manifest()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
|
||||
list(pool.map(download, configuration['files'].items()))
|
||||
print('Pinned artifacts ready: ' + configuration['k3sVersion'] + ', ' + configuration['gvisorRelease'])
|
||||
@@ -0,0 +1,8 @@
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$projectRoot = Split-Path -Parent $PSScriptRoot
|
||||
Set-Location -LiteralPath $projectRoot
|
||||
if (!(Test-Path -LiteralPath '.env')) { throw 'Copy .env.example to .env and configure independent random passwords first.' }
|
||||
if ((Get-Content -LiteralPath '.env' -Raw) -match 'CHANGE_ME') { throw 'Replace every CHANGE_ME placeholder in .env before starting.' }
|
||||
docker compose --env-file .env up --build -d
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Docker Compose failed. Check Docker engine and logs.' }
|
||||
Write-Host 'Console: http://localhost:8088 (or configured HTTP_PORT). Inspect startup: docker compose logs -f backend'
|
||||
Reference in new issue
Block a user