feat: [NODE-1048] Share podman cache across image builds

ic-os/bootloader/build-bootloader-tree.sh

@@ -7,7 +7,8 @@
 set -exo pipefail
 
 cleanup() {
-    podman rm -f "${CONTAINER}"
+    podman container stop --all
+    podman container cleanup --all --rm
     rm -rf "${TMP_DIR}"
 }
 trap cleanup EXIT
@@ -28,7 +29,7 @@ TMP_DIR=$(mktemp -d -t build-image-XXXXXXXXXXXX)
 
 BASE_IMAGE="ghcr.io/dfinity/library/ubuntu@sha256:6015f66923d7afbc53558d7ccffd325d43b4e249f41a6e93eef074c9505d2233"
 
-podman build --no-cache --iidfile "${TMP_DIR}/iidfile" - <<<"
+podman build --iidfile "${TMP_DIR}/iidfile" - <<<"
     FROM $BASE_IMAGE
     USER root:root
     RUN apt-get -y update && apt-get -y --no-install-recommends install grub-efi faketime

toolchains/sysimage/build_container_base_image.py

@@ -4,10 +4,8 @@
 #
 from __future__ import annotations
 
-import atexit
 import os
 import shutil
-import signal
 import tempfile
 from dataclasses import dataclass
 from pathlib import Path
@@ -48,7 +46,7 @@ def build_image(image_tag: str, dockerfile: str, context_dir: str, build_args: L
     build_arg_strings_joined = " ".join(build_arg_strings)
 
     log.info("Building image...")
-    cmd = f"podman build --squash-all --no-cache --tag {image_tag} {build_arg_strings_joined} --file {dockerfile} {context_dir}"
+    cmd = f"podman build --squash-all --tag {image_tag} {build_arg_strings_joined} --file {dockerfile} {context_dir}"
     invoke.run(cmd)
     log.info("Image built successfully")
 
@@ -57,7 +55,6 @@ def save_image(image_tag: str, output_file: str):
     log.info("Saving image to tar file")
     cmd = f"podman image save --output {output_file} {image_tag}"
     invoke.run(cmd)
-    invoke.run("sync")  # For determinism (?)
 
     output_path = Path(output_file)
     assert output_path.exists()
@@ -88,14 +85,6 @@ def main():
     # manually, for now.
     os.environ["PATH"] = ":".join([x for x in [os.environ.get("PATH"), "/usr/bin"] if x is not None])
 
-    def cleanup():
-        invoke.run(f"podman rm -f {image_tag}")
-        invoke.run(f"podman rm -f {image_tag}_container")
-
-    atexit.register(lambda: cleanup())
-    signal.signal(signal.SIGTERM, lambda: cleanup())
-    signal.signal(signal.SIGINT, lambda: cleanup())
-
     build_args = list(build_args or [])
     context_dir = tempfile.mkdtemp()
 

toolchains/sysimage/build_container_filesystem_tar.py

@@ -65,7 +65,6 @@ def build_container(
     cmd += "build "
     cmd += f"-t {image_tag} "
     cmd += f"{build_arg_strings_joined} "
-    cmd += "--no-cache "
 
     if base_image_override:
         load_base_image_tar_file(base_image_override.image_file)
@@ -104,7 +103,6 @@ def export_container_filesystem(image_tag: str, destination_tar_filename: str):
     invoke.run(
         f"fakeroot -i {fakeroot_statefile} tar cf {destination_tar_filename} --numeric-owner --sort=name --exclude='run/*' -C {tar_dir} $(ls -A {tar_dir})"
     )
-    invoke.run("sync")
 
 
 def resolve_file_args(context_dir: str, file_build_args: List[str]) -> List[str]:
@@ -213,8 +211,8 @@ def main():
     component_files = args.component_files
 
     def cleanup():
-        invoke.run(f"podman rm -f {image_tag}_container")
-        invoke.run(f"podman rm -f {image_tag}")
+        invoke.run("podman container stop --all")
+        invoke.run("podman container cleanup --all --rm")
 
     atexit.register(lambda: cleanup())
     signal.signal(signal.SIGTERM, lambda: cleanup())

toolchains/sysimage/proc_wrapper.sh

@@ -6,12 +6,6 @@
 
 set -euo pipefail
 
-# Temporary shim to create tmpfs on demand, until we have userspace overlayfs,
-# or tmpfs natively available on CI.
-tmpfs_tmpdir=$(mktemp -d --tmpdir "icosbuildXXXX")
-sudo mount -t tmpfs none "${tmpfs_tmpdir}"
-
 tmpdir=$(mktemp -d --tmpdir "icosbuildXXXX")
-# NOTE: Ignore failure to cleanup the directory for now. This should not be a problem after NODE-1048.
-trap 'sudo umount "${tmpfs_tmpdir}"; sudo rm -rf "$tmpdir" "${tmpfs_tmpdir}" || true' INT TERM EXIT
-TMPDIR="$tmpdir" TMPFS_TMPDIR="${tmpfs_tmpdir}" "$@"
+trap 'sudo rm -rf "$tmpdir"' INT TERM EXIT
+TMPDIR="$tmpdir" "$@"