Preparing a Compatible Replication Package

# ==============================================================
import csv
import math
from pathlib import Path

import requests
import yaml
from IPython.display import HTML

DATA = Path("_data")

# Everything is GiB (powers of 1024), as `df` on a worker reports it.
nodes = list(csv.DictReader((DATA / "jetstream2-nodes.csv").open()))
node = next(n for n in nodes if n["default"] == "true")
DISK_GIB = float(node["disk_fs_gib"])
OVERHEAD_GIB = float(node["overhead_gib"])
FOOTPRINT_FACTOR = float(node["footprint_factor"])  # on-disk / compressed, see _data/README.md
BASE_FREE_GIB = DISK_GIB - OVERHEAD_GIB

images = list(csv.DictReader((DATA / "container-images.csv").open()))

repos_url = "https://raw.githubusercontent.com/SIVACOR/sivacor-repo-choice/main/allowed_repos.yaml"
allowed = yaml.safe_load(requests.get(repos_url).text)


def hub_compressed_gib(image, tag):
    info = requests.get(f"https://hub.docker.com/v2/repositories/{image}/tags/{tag}/").json()
    size = next((i["size"] for i in info["images"] if i["architecture"] == "amd64"), info["full_size"])
    return size / 1024**3


rows = []
for img in images:
    name = img["image"]
    if not allowed.get(name):  # not (or no longer) on the allow-list: skip, don't fail the build
        continue
    tag = str(allowed[name][0])
    compressed = float(img["compressed_gib"]) if img["compressed_gib"] else hub_compressed_gib(name, tag)
    on_disk = compressed * FOOTPRINT_FACTOR
    rows.append((img["software"], f"{name}:{tag}", compressed, on_disk, BASE_FREE_GIB - on_disk))

free = [r[4] for r in rows]
dynare = next((r for r in rows if r[1].startswith("dynare/")), None)

# Placeholders for the prose ({eval} shows a bare string).
disk_gb = node["disk_gb"]
disk_gib = f"{DISK_GIB:.0f}"
overhead_gib = f"{OVERHEAD_GIB:.1f}"
base_free_gib = f"{BASE_FREE_GIB:.0f}"
footprint_factor = f"{FOOTPRINT_FACTOR:g}"
free_min, free_max = f"{math.floor(min(free))}", f"{math.floor(max(free))}"
dynare_on_disk = f"{math.floor(dynare[3])}" if dynare else "n/a"  # "over ..."
dynare_free = f"{math.ceil(dynare[4])}" if dynare else "n/a"  # "less than ..."

# MyST does not render `text/markdown` outputs, so build HTML.
L, R = 'style="text-align:left"', 'style="text-align:right"'
table = (
    "<table>\n<thead><tr>"
    f"<th {L}>Software</th><th {L}>Image</th><th {R}>Download size</th>"
    f"<th {R}>Space it occupies*</th><th {R}>Free space for your package*</th>"
    "</tr></thead>\n<tbody>\n"
)
for software, image_tag, compressed, on_disk, free_gib in rows:
    table += (
        f"<tr><td {L}>{software}</td><td {L}><code>{image_tag}</code></td>"
        f"<td {R}>{compressed:.1f} GiB</td><td {R}>{on_disk:.1f} GiB</td>"
        f"<td {R}>{free_gib:.1f} GiB</td></tr>\n"
    )
table += "</tbody>\n</table>"
size_table = HTML(table)
# ==============================================================