forked from xhgit/webup-cloud
Initial import of WebUp Cloud
This commit is contained in:
commit
db82325ed7
216 files changed
+33718
No files matched your search
@@ -0,0 +1,81 @@
|
||||
"""Download checksum-verified, pinned official Linux artifacts for the local WSL cluster."""
|
||||
import concurrent.futures
|
||||
import hashlib
|
||||
import json
|
||||
import pathlib
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
CACHE = ROOT / '.tools' / 'local-cluster'
|
||||
LOCK = ROOT / 'infra' / 'local' / 'versions.json'
|
||||
|
||||
|
||||
def read(url):
|
||||
request = urllib.request.Request(url, headers={'User-Agent': 'WebUp-Local-Development'})
|
||||
with urllib.request.urlopen(request, timeout=40) as response:
|
||||
return response.read()
|
||||
|
||||
|
||||
def checksums(data):
|
||||
return {parts[-1].lstrip('*').removeprefix('./'): parts[0] for line in data.decode().splitlines() if len(parts := line.split()) >= 2}
|
||||
|
||||
|
||||
def manifest():
|
||||
if LOCK.exists():
|
||||
return json.loads(LOCK.read_text(encoding='utf-8'))
|
||||
channels = json.loads(read('https://update.k3s.io/v1-release/channels'))
|
||||
version = next(item['latest'] for item in channels['data'] if item['name'] == 'stable')
|
||||
release = json.loads(read('https://api.github.com/repos/k3s-io/k3s/releases/tags/' + urllib.parse.quote(version, safe='')))
|
||||
assets = {item['name']: item['browser_download_url'] for item in release['assets']}
|
||||
hashes = checksums(read(assets['sha256sum-amd64.txt']))
|
||||
files = {name: {'url': assets[name], 'sha256': hashes[name]} for name in ['k3s', 'k3s-airgap-images-amd64.tar.zst']}
|
||||
installer = 'https://raw.githubusercontent.com/k3s-io/k3s/' + urllib.parse.quote(version, safe='') + '/install.sh'
|
||||
files['install-k3s.sh'] = {'url': installer, 'sha256': hashlib.sha256(read(installer)).hexdigest()}
|
||||
gvisor = json.loads(read('https://api.github.com/repos/google/gvisor/releases/latest'))
|
||||
assets = {item['name']: item['browser_download_url'] for item in gvisor['assets']}
|
||||
hashes = checksums(read(assets['SHA256SUMS']))
|
||||
name = 'gvisor-x86_64.tar.bz2'
|
||||
files[name] = {'url': assets[name], 'sha256': hashes[name]}
|
||||
result = {'k3sVersion': version, 'gvisorRelease': gvisor['tag_name'], 'architecture': 'x86_64', 'files': files}
|
||||
LOCK.parent.mkdir(parents=True, exist_ok=True)
|
||||
LOCK.write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8')
|
||||
return result
|
||||
|
||||
|
||||
def digest(path):
|
||||
sha = hashlib.sha256()
|
||||
with path.open('rb') as stream:
|
||||
while chunk := stream.read(1024 * 1024):
|
||||
sha.update(chunk)
|
||||
return sha.hexdigest()
|
||||
|
||||
|
||||
def download(item):
|
||||
name, metadata = item
|
||||
path = CACHE / name
|
||||
if path.exists() and digest(path) == metadata['sha256']:
|
||||
print(name + ': checksum verified (cached)', flush=True)
|
||||
return
|
||||
partial = path.with_name(path.name + '.part')
|
||||
request = urllib.request.Request(metadata['url'], headers={'User-Agent': 'WebUp-Local-Development'})
|
||||
print(name + ': downloading official release', flush=True)
|
||||
with urllib.request.urlopen(request, timeout=60) as response, partial.open('wb') as output:
|
||||
total = 0
|
||||
while chunk := response.read(1024 * 1024):
|
||||
total += len(chunk)
|
||||
if total > 512 * 1024 * 1024:
|
||||
raise RuntimeError('Artifact exceeds the configured size bound: ' + name)
|
||||
output.write(chunk)
|
||||
if digest(partial) != metadata['sha256']:
|
||||
raise RuntimeError('Checksum mismatch; artifact was not installed: ' + name)
|
||||
partial.replace(path)
|
||||
print(name + ': checksum verified', flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
CACHE.mkdir(parents=True, exist_ok=True)
|
||||
configuration = manifest()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
|
||||
list(pool.map(download, configuration['files'].items()))
|
||||
print('Pinned artifacts ready: ' + configuration['k3sVersion'] + ', ' + configuration['gvisorRelease'])
|
||||
Reference in new issue
Block a user