diff --git a/src/vip_client/classes/VipCI.py b/src/vip_client/classes/VipCI.py index 7b8cb7a..456bd7b 100644 --- a/src/vip_client/classes/VipCI.py +++ b/src/vip_client/classes/VipCI.py @@ -51,13 +51,12 @@ class VipCI(VipLauncher): # Default backup location # (set to None to avoid saving and loading backup files) _BACKUP_LOCATION = "girder" - - # --- New Attributes --- - + # Prefix that defines a Girder ID _GIRDER_ID_PREFIX = "pilotGirder" # Grider portal _GIRDER_PORTAL = 'https://pilot-warehouse.creatis.insa-lyon.fr/api/v1' + ################# ################ Main Properties ################## @@ -137,7 +136,15 @@ def __init__( # Login to VIP and Girder @classmethod - def init(cls, vip_key="VIP_API_KEY", girder_key="GIRDER_API_KEY", verbose=True, **kwargs) -> VipCI: + def init( + cls, + vip_key="VIP_API_KEY", + girder_key="GIRDER_API_KEY", + verbose=True, + girder_api_url=None, + girder_id_prefix=None, + **kwargs + ) -> VipCI: """ Handshakes with VIP using your own API key. Returns a class instance which properties can be provided as keyword arguments. @@ -161,8 +168,11 @@ def init(cls, vip_key="VIP_API_KEY", girder_key="GIRDER_API_KEY", verbose=True, super().init(api_key=vip_key, verbose=False) # Restore the verbose state cls._VERBOSE = verbose + # Set the Girder ID prefix + cls._GIRDER_ID_PREFIX = girder_id_prefix if girder_id_prefix is not None else cls._GIRDER_ID_PREFIX + cls._GIRDER_PORTAL = girder_api_url if girder_api_url is not None else cls._GIRDER_PORTAL # Instantiate a Girder client - cls._girder_client = girder_client.GirderClient(apiUrl=cls._GIRDER_PORTAL) + cls._girder_client = girder_client.GirderClient(apiUrl=girder_api_url) # Check if `girder_key` is in a local file or environment variable true_key = cls._get_api_key(girder_key) # Authenticate with Girder API key diff --git a/src/vip_client/classes/VipLauncher.py b/src/vip_client/classes/VipLauncher.py index dc63315..b24804c 100644 --- a/src/vip_client/classes/VipLauncher.py +++ b/src/vip_client/classes/VipLauncher.py @@ -371,7 +371,8 @@ def __init__( # ($A.1) Login to VIP @classmethod - def init(cls, api_key="VIP_API_KEY", verbose=True, **kwargs) -> VipLauncher: + def init(cls, api_key="VIP_API_KEY", verbose=True, vip_portal_url=None, + **kwargs) -> VipLauncher: """ Handshakes with VIP using your own API key. Returns a class instance which properties can be provided as keyword arguments. @@ -391,11 +392,14 @@ def init(cls, api_key="VIP_API_KEY", verbose=True, **kwargs) -> VipLauncher: """ # Set the default verbose mode for all sessions cls._VERBOSE = verbose + # Set the VIP portal URL + cls._VIP_PORTAL = vip_portal_url if vip_portal_url else cls._VIP_PORTAL # Check if `api_key` is in a local file or environment variable true_key = cls._get_api_key(api_key) # Set User API key try: # setApiKey() may return False + vip.set_vip_url(cls._VIP_PORTAL) assert vip.setApiKey(true_key), \ f"(!) Unable to set the VIP API key: {true_key}.\nPlease check the key or retry later." except RuntimeError as vip_error: @@ -1641,6 +1645,7 @@ def _check_input_values(self, input_settings: dict, location: str) -> None: self._check_invalid_input(input_settings, self._pipeline_def['parameters']) wrong_type_inputs = [] + missing_files = [] for param in self._pipeline_def['parameters']: name = param['name'] # Check only defined inputs @@ -1650,11 +1655,10 @@ def _check_input_values(self, input_settings: dict, location: str) -> None: # If input is a File, check file(s) existence if param["type"] == "File": # Ensure every file exists at `location` - missing_file = self._first_missing_file(value, location) - if missing_file: - raise FileNotFoundError( - f"Parameter '{name}': The following file is missing in the {location.upper()} file system: {missing_file}" - ) + missing_files_found = self._missing_files(value, location) + if missing_files_found: + missing_files.extend(missing_files_found) + continue if param["type"] == "Boolean": if value not in ["true", "false"]: wrong_type_inputs.append(name) @@ -1666,6 +1670,10 @@ def _check_input_values(self, input_settings: dict, location: str) -> None: raise ValueError( f"Wrong type(s) for parameter(s): {', '.join(sorted(wrong_type_inputs))}" ) + if missing_files: + raise FileNotFoundError( + f"Missing file(s) for parameter(s): {', '.join(sorted(missing_files))}" + ) # ------------------------------------------------ @@ -1713,21 +1721,27 @@ def _invalid_chars_for_vip(cls, value) -> list: # Function to assert file existence in the input settings @classmethod - def _first_missing_file(cls, value, location: str) -> str: + def _missing_files(cls, value, location: str) -> list[str]: """ - Returns the path the first non-existent file in `value` (None by default). - - `value` can contain a single file path or a list of paths. - - `location` refers to the storage infrastructure (e.g., "vip") to feed in cls._exists(). + Returns a list of missing files for `value` at `location`. + + - `value` can be either a single file path or a list of paths. + - `location` refers to the storage infrastructure (e.g., "vip") used by cls._exists(). """ - # Case : list of files + missing_files = [] + + # Case: list of files if isinstance(value, list): - for file in value : - if cls._first_missing_file(value=file, location=location) is not None: - return file - return None + for file in value: + if not cls._exists(file, location=location): + missing_files.append(file) # Case: single file else: - return value if not cls._exists(path=value, location=location) else None + if not cls._exists(value, location=location): + missing_files.append(value) + + return missing_files + # ------------------------------------------------ ######################################## diff --git a/src/vip_client/classes/VipSession.py b/src/vip_client/classes/VipSession.py index 9047345..00ad805 100644 --- a/src/vip_client/classes/VipSession.py +++ b/src/vip_client/classes/VipSession.py @@ -350,8 +350,8 @@ def upload_inputs(self, input_dir=None, update_files=True) -> VipSession: elif not self._is_defined("_local_input_dir"): raise TypeError(f"Session '{self._session_name}': Please provide an input directory.") # Check local input directory - if not self._exists(self._local_input_dir, location="local"): - raise FileNotFoundError(f"Session '{self._session_name}': Input directory does not exist.") + if not self._exists(self._local_input_dir, location="local"): + raise FileNotFoundError(f"Session '{self._session_name}': Input directory '{self._local_input_dir}' does not exist.") # Check the local values of `input_settings` before uploading if self._is_defined("_input_settings"): self._print("Checking references to the dataset within Input Settings ... ", end="", flush=True) @@ -684,11 +684,12 @@ def _path_to_delete(self) -> dict: def _exists(cls, path: PurePath, location="local") -> bool: """ Checks existence of a distant (`location`="vip") or local (`location`="local") resource. - `path` can be a string or path-like object. + `path` can be a string or path-like object. If `location` is "local", empty folders are + considered as non-existent. """ # Check path existence in `location` if location=="local": - return os.path.exists(path) + return os.path.exists(path) and os.path.isdir(path) and os.listdir(path) else: return super()._exists(path=path, location=location) # ------------------------------------------------ @@ -831,9 +832,10 @@ def _upload_dir(self, local_path: Path, vip_path: PurePosixPath) -> list: failures = [] for local_file in files_to_upload : nFile+=1 - # Get the file size (if possible) - try: size = f"{local_file.stat().st_size/(1<<20):,.1f}MB" - except: size = "unknown size" + # Check the file size + size = local_file.stat().st_size + if size == 0: raise ValueError(f"{local_file} is an empty file. Empty file are not supported on VIP") + size = f"{size/(1<<20):,.1f}MB" # Display the current file self._print(f"\t[{nFile}/{len(files_to_upload)}] Uploading file: {local_file.name} ({size}) ...", end=" ") # Upload the file on VIP diff --git a/src/vip_client/utils/vip.py b/src/vip_client/utils/vip.py index 3e87153..29ffdd7 100644 --- a/src/vip_client/utils/vip.py +++ b/src/vip_client/utils/vip.py @@ -20,6 +20,11 @@ # API URL __PREFIX = "https://vip.creatis.insa-lyon.fr/rest/" +def set_vip_url(vip_portal_url: str) -> None: + """Change the API prefix""" + global __PREFIX + __PREFIX = vip_portal_url + "/rest/" + # API key __apikey = None __headers = {'apikey': __apikey} @@ -80,7 +85,7 @@ def setApiKey(value) -> bool: Return True is correct apikey, False otherwise. Raise an error if an other problems occured """ - url = __PREFIX + 'plateform' + url = __PREFIX + 'platform' head_test = { 'apikey': value, }