#!/usr/bin/env python3 # ex: set sts=4 ts=4 sw=4 et: # # A helper to freeze (and possibly downgrade) versions of specified containers, # or copy definition to be contained within the super-dataset. # # Example invocations # # 1. Just freeze given containers within this dataset, so when upgrading, # if .datalad/config changes due to new versions etc, conflicts would # come up and require manual conscious decision on how to upgrade. # # scripts/freeze_versions bids-mriqc=0.15.0 bids-fmriprep=1.4.1 bids-aa # # and it can take partial version specification if there is only one # image with that version prefix is available, e.g. # # scripts/freeze_versions neurodesk-fmriprep=20.1 # # would freeze to the 20.1.3 version as that is the only one available for 20.1. # # 2. Copy container definitions, and this way freeze within super-dataset # where this one was included as code/containers subdataset # # code/containers/scripts/freeze_versions --save-dataset=. bids-mriqc=0.15.0 bids-fmriprep # # COPYRIGHT: Yaroslav Halchenko 2019-2024 # # LICENSE: MIT # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. # import argparse from glob import escape import os.path from pathlib import Path import re import subprocess import sys import tempfile import warnings def error(message: str) -> None: print("ERROR:", message, file=sys.stderr) def debug(message: str) -> None: # print("DEBUG:", message, file=sys.stderr) pass topd = Path(__file__).parent.parent # shorten if possible if topd.resolve() == Path.cwd().resolve(): topd = Path(".") save_ds = None target_ds = topd # config of which to tune topd_rel = None # relative path to topd from target_ds frozen = "" debug(f"topd_rel: {topd_rel}") parser = argparse.ArgumentParser() parser.add_argument("--save-dataset", type=Path) parser.add_argument("images", nargs="*") args = parser.parse_args() if args.save_dataset: target_ds = save_ds = args.save_dataset debug(f"Target DS: {target_ds} == {target_ds.resolve()}; Save DS: {save_ds}") if save_ds is not None: # if we are asking to save into another dataset if topd.resolve() != save_ds.resolve(): # save_ds should be a parent of topd topd_rel = os.path.relpath(topd, save_ds) if topd_rel.startswith(".."): error( f"{topd} is not subdirectory of {save_ds}, cannot freeze/copy that way" ) sys.exit(2) elif topd_rel == ".": # the same dataset, no copying, just in place freezing topd_rel = None # empty is better else: print(f"I: We will be copying/freezing versions in {save_ds}") topd_rel = Path(topd_rel) if not (save_ds / ".datalad" / "config").exists(): error( f"{save_ds} folder has no .datalad/config. Please ensure that you are pointing to parent superdataset top directory" ) sys.exit(4) for arg in args.images: frozen = f"{frozen} {arg}" # just for commit message img, _, ver = arg.partition("=") if ver: # we had version specified print(f"I: {img} -> {ver}") imgprefix = topd / "images" / img.partition("-")[0] / f"{img}--{ver}" debug(f"Checking {imgprefix}") if imgprefix.is_symlink() or imgprefix.exists(): # we were specified precisely with extension etc imgpath = imgprefix else: imgpaths = list(imgprefix.parent.glob(f"{escape(imgprefix.name)}.*")) if len(imgpaths) == 0: error( f"There is no {imgprefix}.* files. Available images for the app are:" ) for p in imgprefix.parent.glob(f"{escape(img)}--*"): print(f" {p}", file=sys.stderr) sys.exit(1) elif len(imgpaths) == 1: imgpath = os.path.relpath( imgpaths[0], save_ds or "." ) # already would include topd else: error( "There are multiple images available. Include extension into your version specification. Available images are:" ) for p in imgpaths: print(f" {p}") sys.exit(1) else: # freeze to current r = subprocess.run( [ "git", "-C", topd, "config", "-f", ".datalad/config", f"datalad.containers.{img}.image", ], check=True, stdout=subprocess.PIPE, text=True, ) imgpath = (topd_rel or Path()) / r.stdout.rstrip("\n") # Point to specific image -- might be the same if topd=target_d and there # were no ver specified, but we do it here uniformly for consistency subprocess.run( [ "git", "config", "-f", target_ds / ".datalad" / "config", "--replace-all", f"datalad.containers.{img}.image", str(imgpath), ], check=True, ) # if it was a copy into some other super-dataset, we should copy some other # fields if topd_rel is not None: # if copying to some other dataset, procedure is different, since we # need to copy all r = subprocess.run( [ "git", "config", "-f", topd / ".datalad" / "config", "--get-regexp", rf"containers.{img}\.", ], check=True, stdout=subprocess.PIPE, text=True, ) for line in r.stdout.splitlines(): var, value = line.split(maxsplit=1) if var.endswith(".image"): continue # already done above, skip elif var.endswith(".cmdexec"): if value.startswith("{img_dspath}/"): # Not an f-less f-string value = value.replace("{img_dspath}/", f"{{img_dspath}}/{topd_rel}/") else: value = (topd_rel or Path()) / value subprocess.run( [ "git", "config", "-f", target_ds / ".datalad" / "config", "--replace-all", var, value, ], check=True, ) else: # if in current dataset, then # we would add the comment so that upon upgrade there for sure would be # a conflict needed to be consciously resolved (or -S ours used) fd, tmppath = tempfile.mkstemp(dir=topd / ".datalad") with os.fdopen(fd, "w", encoding="utf-8") as outfp: frozen_entries = 0 cfg_file = topd / ".datalad" / "config" with (cfg_file).open(encoding="utf-8") as infp: for line in infp: line = line.rstrip("\n") new_line = re.sub( rf"{re.escape(str(imgpath))}(?:[ \t].*)*$", f"{imgpath} # frozen", line, ) print(new_line, file=outfp) if new_line != line: frozen_entries += 1 if not frozen_entries: raise RuntimeError(f"Did not freeze any entry for {imgpath}") elif frozen_entries > 1: warnings.warn(f"Froze multiple ({frozen_entries}) entries for {imgpath}. Check {cfg_file} for consistency/desired effect.") os.replace(tmppath, topd / ".datalad" / "config") if save_ds is not None: subprocess.run( [ "datalad", "save", f"-d{save_ds}", "-m", f"Freeze container versions {frozen}", Path(save_ds) / ".datalad" / "config", ], check=True, )