From 7105b984de14bff658c4fa4aead79375ae3b13d8 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Tue, 9 Dec 2025 15:00:01 -0600 Subject: [PATCH 1/2] ospool-image-puller: Use argparse; add the --keep-days argument for choosing how many days of images to keep (Copilot, GPT-5 mini) --- opensciencegrid/ospool-image-puller/update.py | 40 +++++++++++++------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/opensciencegrid/ospool-image-puller/update.py b/opensciencegrid/ospool-image-puller/update.py index 813b4e7a..554dc455 100755 --- a/opensciencegrid/ospool-image-puller/update.py +++ b/opensciencegrid/ospool-image-puller/update.py @@ -1,22 +1,37 @@ #!/usr/bin/env python3 import os -import sys import time import yaml import datetime import subprocess +import argparse start_dir = os.getcwd() def main(): - if len(sys.argv) != 2: - print("Usage: update.py ") - sys.exit(1) - - target_dir = sys.argv[1] - - with open("images.yaml") as file: + parser = argparse.ArgumentParser( + description='Pull and update container images from Docker to Apptainer/Singularity format' + ) + parser.add_argument( + 'target_dir', + help='Target directory where images will be stored' + ) + parser.add_argument( + '--config', + default='images.yaml', + help='Path to the configuration YAML file (default: images.yaml)' + ) + parser.add_argument( + '--keep-days', + type=int, + default=10, + help='Number of days to keep old images (default: 10)' + ) + + args = parser.parse_args() + + with open(args.config) as file: conf = yaml.safe_load(file) now = datetime.datetime.now(datetime.timezone.utc) @@ -36,13 +51,13 @@ def main(): sing_arch = "arm64" if arch == "aarch64" else "amd64" os.chdir(start_dir) - os.makedirs(os.path.join(target_dir, arch, img["name"]), exist_ok=True) - os.chdir(os.path.join(target_dir, arch, img["name"])) + os.makedirs(os.path.join(args.target_dir, arch, img["name"]), exist_ok=True) + os.chdir(os.path.join(args.target_dir, arch, img["name"])) print(f"Working in {os.getcwd()}") # log the build to a file in the same structure under logs/ - log_dir = os.path.join("../../..", target_dir, "logs", arch) + log_dir = os.path.join("../../..", args.target_dir, "logs", arch) os.makedirs(log_dir, exist_ok=True) log_file = os.path.join(log_dir, f"{img['name']}.txt") @@ -65,7 +80,7 @@ def main(): subprocess.run("ls *.sif | sort | tail -n 1 > latest.txt", shell=True) # cleanup old images, keep only latest N - subprocess.run("find . -maxdepth 1 -name \\*.sif -mtime +10 -exec rm -f {} \\;", shell=True) + subprocess.run(f"find . -maxdepth 1 -name \\*.sif -mtime +{args.keep_days} -exec rm -f {{}} \\;", shell=True) now2 = datetime.datetime.now(datetime.timezone.utc) now2_human = now2.strftime("%Y-%m-%d %H:%M:%S %Z") @@ -76,4 +91,3 @@ def main(): if __name__ == "__main__": main() - From e759ea6000cda63e44916662da2decbe87d4de16 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Tue, 9 Dec 2025 15:07:24 -0600 Subject: [PATCH 2/2] ospool-image-puller: Add KEEP_DAYS parameter to Dockerfile and cron job --- opensciencegrid/ospool-image-puller/Dockerfile | 2 +- opensciencegrid/ospool-image-puller/startup.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/opensciencegrid/ospool-image-puller/Dockerfile b/opensciencegrid/ospool-image-puller/Dockerfile index fe8a41b5..893a2e48 100644 --- a/opensciencegrid/ospool-image-puller/Dockerfile +++ b/opensciencegrid/ospool-image-puller/Dockerfile @@ -18,8 +18,8 @@ COPY --chmod=0755 startup.sh /bin/startup.sh ENV CRON_EXPR="30 5 * * *" ENV TIMEOUT="23h" ENV IMAGES_UID="1000" +ENV KEEP_DAYS=10 VOLUME ["/ospool/images"] CMD ["/bin/startup.sh"] - diff --git a/opensciencegrid/ospool-image-puller/startup.sh b/opensciencegrid/ospool-image-puller/startup.sh index 9828b35f..9f3e891d 100755 --- a/opensciencegrid/ospool-image-puller/startup.sh +++ b/opensciencegrid/ospool-image-puller/startup.sh @@ -16,7 +16,7 @@ install -o images -d "${TARGET_DIR}" # the container's stdout/err, but then drop privileges to run the # actual script. -echo "${CRON_EXPR?} root cd /ospool/images-scripts && /usr/sbin/runuser -u images -- timeout -k 60s ${TIMEOUT?} ./update.py ${TARGET_DIR}" '> /proc/1/fd/1 2>&1' > /etc/cron.d/update.cron +echo "${CRON_EXPR?} root cd /ospool/images-scripts && /usr/sbin/runuser -u images -- timeout -k 60s ${TIMEOUT?} ./update.py --keep-days=${KEEP_DAYS:-10} ${TARGET_DIR}" '> /proc/1/fd/1 2>&1' > /etc/cron.d/update.cron # Start cron in non-daemon (foreground) mode echo >&2 "Starting cron"