forked from xhgit/webup-cloud
82 lines
3.6 KiB
Python
82 lines
3.6 KiB
Python
"""Download checksum-verified, pinned official Linux artifacts for the local WSL cluster."""
|
|
import concurrent.futures
|
|
import hashlib
|
|
import json
|
|
import pathlib
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
|
CACHE = ROOT / '.tools' / 'local-cluster'
|
|
LOCK = ROOT / 'infra' / 'local' / 'versions.json'
|
|
|
|
|
|
def read(url):
|
|
request = urllib.request.Request(url, headers={'User-Agent': 'WebUp-Local-Development'})
|
|
with urllib.request.urlopen(request, timeout=40) as response:
|
|
return response.read()
|
|
|
|
|
|
def checksums(data):
|
|
return {parts[-1].lstrip('*').removeprefix('./'): parts[0] for line in data.decode().splitlines() if len(parts := line.split()) >= 2}
|
|
|
|
|
|
def manifest():
|
|
if LOCK.exists():
|
|
return json.loads(LOCK.read_text(encoding='utf-8'))
|
|
channels = json.loads(read('https://update.k3s.io/v1-release/channels'))
|
|
version = next(item['latest'] for item in channels['data'] if item['name'] == 'stable')
|
|
release = json.loads(read('https://api.github.com/repos/k3s-io/k3s/releases/tags/' + urllib.parse.quote(version, safe='')))
|
|
assets = {item['name']: item['browser_download_url'] for item in release['assets']}
|
|
hashes = checksums(read(assets['sha256sum-amd64.txt']))
|
|
files = {name: {'url': assets[name], 'sha256': hashes[name]} for name in ['k3s', 'k3s-airgap-images-amd64.tar.zst']}
|
|
installer = 'https://raw.githubusercontent.com/k3s-io/k3s/' + urllib.parse.quote(version, safe='') + '/install.sh'
|
|
files['install-k3s.sh'] = {'url': installer, 'sha256': hashlib.sha256(read(installer)).hexdigest()}
|
|
gvisor = json.loads(read('https://api.github.com/repos/google/gvisor/releases/latest'))
|
|
assets = {item['name']: item['browser_download_url'] for item in gvisor['assets']}
|
|
hashes = checksums(read(assets['SHA256SUMS']))
|
|
name = 'gvisor-x86_64.tar.bz2'
|
|
files[name] = {'url': assets[name], 'sha256': hashes[name]}
|
|
result = {'k3sVersion': version, 'gvisorRelease': gvisor['tag_name'], 'architecture': 'x86_64', 'files': files}
|
|
LOCK.parent.mkdir(parents=True, exist_ok=True)
|
|
LOCK.write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8')
|
|
return result
|
|
|
|
|
|
def digest(path):
|
|
sha = hashlib.sha256()
|
|
with path.open('rb') as stream:
|
|
while chunk := stream.read(1024 * 1024):
|
|
sha.update(chunk)
|
|
return sha.hexdigest()
|
|
|
|
|
|
def download(item):
|
|
name, metadata = item
|
|
path = CACHE / name
|
|
if path.exists() and digest(path) == metadata['sha256']:
|
|
print(name + ': checksum verified (cached)', flush=True)
|
|
return
|
|
partial = path.with_name(path.name + '.part')
|
|
request = urllib.request.Request(metadata['url'], headers={'User-Agent': 'WebUp-Local-Development'})
|
|
print(name + ': downloading official release', flush=True)
|
|
with urllib.request.urlopen(request, timeout=60) as response, partial.open('wb') as output:
|
|
total = 0
|
|
while chunk := response.read(1024 * 1024):
|
|
total += len(chunk)
|
|
if total > 512 * 1024 * 1024:
|
|
raise RuntimeError('Artifact exceeds the configured size bound: ' + name)
|
|
output.write(chunk)
|
|
if digest(partial) != metadata['sha256']:
|
|
raise RuntimeError('Checksum mismatch; artifact was not installed: ' + name)
|
|
partial.replace(path)
|
|
print(name + ': checksum verified', flush=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
CACHE.mkdir(parents=True, exist_ok=True)
|
|
configuration = manifest()
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
|
|
list(pool.map(download, configuration['files'].items()))
|
|
print('Pinned artifacts ready: ' + configuration['k3sVersion'] + ', ' + configuration['gvisorRelease'])
|