Files
webup-cloud/scripts/prepare-local-artifacts.py
T

82 lines
3.6 KiB
Python

"""Download checksum-verified, pinned official Linux artifacts for the local WSL cluster."""
import concurrent.futures
import hashlib
import json
import pathlib
import urllib.parse
import urllib.request
ROOT = pathlib.Path(__file__).resolve().parents[1]
CACHE = ROOT / '.tools' / 'local-cluster'
LOCK = ROOT / 'infra' / 'local' / 'versions.json'
def read(url):
request = urllib.request.Request(url, headers={'User-Agent': 'WebUp-Local-Development'})
with urllib.request.urlopen(request, timeout=40) as response:
return response.read()
def checksums(data):
return {parts[-1].lstrip('*').removeprefix('./'): parts[0] for line in data.decode().splitlines() if len(parts := line.split()) >= 2}
def manifest():
if LOCK.exists():
return json.loads(LOCK.read_text(encoding='utf-8'))
channels = json.loads(read('https://update.k3s.io/v1-release/channels'))
version = next(item['latest'] for item in channels['data'] if item['name'] == 'stable')
release = json.loads(read('https://api.github.com/repos/k3s-io/k3s/releases/tags/' + urllib.parse.quote(version, safe='')))
assets = {item['name']: item['browser_download_url'] for item in release['assets']}
hashes = checksums(read(assets['sha256sum-amd64.txt']))
files = {name: {'url': assets[name], 'sha256': hashes[name]} for name in ['k3s', 'k3s-airgap-images-amd64.tar.zst']}
installer = 'https://raw.githubusercontent.com/k3s-io/k3s/' + urllib.parse.quote(version, safe='') + '/install.sh'
files['install-k3s.sh'] = {'url': installer, 'sha256': hashlib.sha256(read(installer)).hexdigest()}
gvisor = json.loads(read('https://api.github.com/repos/google/gvisor/releases/latest'))
assets = {item['name']: item['browser_download_url'] for item in gvisor['assets']}
hashes = checksums(read(assets['SHA256SUMS']))
name = 'gvisor-x86_64.tar.bz2'
files[name] = {'url': assets[name], 'sha256': hashes[name]}
result = {'k3sVersion': version, 'gvisorRelease': gvisor['tag_name'], 'architecture': 'x86_64', 'files': files}
LOCK.parent.mkdir(parents=True, exist_ok=True)
LOCK.write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8')
return result
def digest(path):
sha = hashlib.sha256()
with path.open('rb') as stream:
while chunk := stream.read(1024 * 1024):
sha.update(chunk)
return sha.hexdigest()
def download(item):
name, metadata = item
path = CACHE / name
if path.exists() and digest(path) == metadata['sha256']:
print(name + ': checksum verified (cached)', flush=True)
return
partial = path.with_name(path.name + '.part')
request = urllib.request.Request(metadata['url'], headers={'User-Agent': 'WebUp-Local-Development'})
print(name + ': downloading official release', flush=True)
with urllib.request.urlopen(request, timeout=60) as response, partial.open('wb') as output:
total = 0
while chunk := response.read(1024 * 1024):
total += len(chunk)
if total > 512 * 1024 * 1024:
raise RuntimeError('Artifact exceeds the configured size bound: ' + name)
output.write(chunk)
if digest(partial) != metadata['sha256']:
raise RuntimeError('Checksum mismatch; artifact was not installed: ' + name)
partial.replace(path)
print(name + ': checksum verified', flush=True)
if __name__ == '__main__':
CACHE.mkdir(parents=True, exist_ok=True)
configuration = manifest()
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
list(pool.map(download, configuration['files'].items()))
print('Pinned artifacts ready: ' + configuration['k3sVersion'] + ', ' + configuration['gvisorRelease'])