"""Download checksum-verified, pinned official Linux artifacts for the local WSL cluster.""" import concurrent.futures import hashlib import json import pathlib import urllib.parse import urllib.request ROOT = pathlib.Path(__file__).resolve().parents[1] CACHE = ROOT / '.tools' / 'local-cluster' LOCK = ROOT / 'infra' / 'local' / 'versions.json' def read(url): request = urllib.request.Request(url, headers={'User-Agent': 'WebUp-Local-Development'}) with urllib.request.urlopen(request, timeout=40) as response: return response.read() def checksums(data): return {parts[-1].lstrip('*').removeprefix('./'): parts[0] for line in data.decode().splitlines() if len(parts := line.split()) >= 2} def manifest(): if LOCK.exists(): return json.loads(LOCK.read_text(encoding='utf-8')) channels = json.loads(read('https://update.k3s.io/v1-release/channels')) version = next(item['latest'] for item in channels['data'] if item['name'] == 'stable') release = json.loads(read('https://api.github.com/repos/k3s-io/k3s/releases/tags/' + urllib.parse.quote(version, safe=''))) assets = {item['name']: item['browser_download_url'] for item in release['assets']} hashes = checksums(read(assets['sha256sum-amd64.txt'])) files = {name: {'url': assets[name], 'sha256': hashes[name]} for name in ['k3s', 'k3s-airgap-images-amd64.tar.zst']} installer = 'https://raw.githubusercontent.com/k3s-io/k3s/' + urllib.parse.quote(version, safe='') + '/install.sh' files['install-k3s.sh'] = {'url': installer, 'sha256': hashlib.sha256(read(installer)).hexdigest()} gvisor = json.loads(read('https://api.github.com/repos/google/gvisor/releases/latest')) assets = {item['name']: item['browser_download_url'] for item in gvisor['assets']} hashes = checksums(read(assets['SHA256SUMS'])) name = 'gvisor-x86_64.tar.bz2' files[name] = {'url': assets[name], 'sha256': hashes[name]} result = {'k3sVersion': version, 'gvisorRelease': gvisor['tag_name'], 'architecture': 'x86_64', 'files': files} LOCK.parent.mkdir(parents=True, exist_ok=True) LOCK.write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8') return result def digest(path): sha = hashlib.sha256() with path.open('rb') as stream: while chunk := stream.read(1024 * 1024): sha.update(chunk) return sha.hexdigest() def download(item): name, metadata = item path = CACHE / name if path.exists() and digest(path) == metadata['sha256']: print(name + ': checksum verified (cached)', flush=True) return partial = path.with_name(path.name + '.part') request = urllib.request.Request(metadata['url'], headers={'User-Agent': 'WebUp-Local-Development'}) print(name + ': downloading official release', flush=True) with urllib.request.urlopen(request, timeout=60) as response, partial.open('wb') as output: total = 0 while chunk := response.read(1024 * 1024): total += len(chunk) if total > 512 * 1024 * 1024: raise RuntimeError('Artifact exceeds the configured size bound: ' + name) output.write(chunk) if digest(partial) != metadata['sha256']: raise RuntimeError('Checksum mismatch; artifact was not installed: ' + name) partial.replace(path) print(name + ': checksum verified', flush=True) if __name__ == '__main__': CACHE.mkdir(parents=True, exist_ok=True) configuration = manifest() with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: list(pool.map(download, configuration['files'].items())) print('Pinned artifacts ready: ' + configuration['k3sVersion'] + ', ' + configuration['gvisorRelease'])