From 02ca314c89c786c4a94044f4dcdf3bdc2582d278 Mon Sep 17 00:00:00 2001 From: AymericGaudin Date: Wed, 3 Jun 2026 15:22:29 +0200 Subject: [PATCH 1/3] tools/scaling/scaling.py: Use one data folder for every workflow --- tools/scaling/scaling_cli.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/scaling/scaling_cli.py b/tools/scaling/scaling_cli.py index 519288ab..023da373 100644 --- a/tools/scaling/scaling_cli.py +++ b/tools/scaling/scaling_cli.py @@ -310,7 +310,7 @@ def organize_longitudinal( """ banner = r""" +----------------------------------+ - | Organize longitudianl data... | + | Organize longitudinal data... | +----------------------------------+ """ print(banner) @@ -482,7 +482,7 @@ def collect_config( else None ) workflow_parameters = workflow_parameters.format_map( - SafeDict({"outdir": output_dir / "data"}) + SafeDict({"outdir": working_dir / "data"}) ) print(f"- edited parameters: {workflow_parameters}") @@ -497,7 +497,7 @@ def collect_config( if infra == "slurm": image_parameters = ( f"--cleanenv --home {home_dir} --bind {bind_dir} " - f"--bind {output_dir} " + f"--bind {working_dir} " ) else: image_parameters = "" @@ -632,7 +632,7 @@ def scan_configs( print("No cache files found. Parsing data.") selected = None else: - print("Multiple cache files found:") + print("Cache file(s) found:") for idx, path in enumerate(cache_files, 1): print(f"{idx}. {path.name}") choice = input( From 72215a0a99fe35bb75a8ab63677b50a3afa4e21e Mon Sep 17 00:00:00 2001 From: AymericGaudin Date: Thu, 4 Jun 2026 10:53:06 +0200 Subject: [PATCH 2/3] tools/scaling/scaling.py: Change config.toml for group level (avoid data.tsv dependency) --- tools/scaling/scaling_cli.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/scaling/scaling_cli.py b/tools/scaling/scaling_cli.py index 023da373..0665a257 100644 --- a/tools/scaling/scaling_cli.py +++ b/tools/scaling/scaling_cli.py @@ -520,12 +520,16 @@ def collect_config( index=False, ) + commands = f'"brainprep {workflow_id} {workflow_parameters}"' + if "group" in workflow_id: + commands = f"[[{commands}]]" + config_template = config_file.read_text() config_str = config_template.format( name=workflow_name, operator="TO UPDATE", date=str(datetime.now().date()), - commands=f'"brainprep {workflow_id} {workflow_parameters}"', + commands=commands, parameters=image_parameters, cluster=infra, partition=partition, From a8d2be5d24a6198899f2fbb03e32d8f375d11e08 Mon Sep 17 00:00:00 2001 From: AymericGaudin Date: Wed, 26 Aug 2026 11:41:19 +0200 Subject: [PATCH 3/3] tools/scaling/scaling_cli: Skip longitudinal parsing when not needed --- tools/scaling/scaling_cli.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/tools/scaling/scaling_cli.py b/tools/scaling/scaling_cli.py index 0665a257..96e5d60d 100644 --- a/tools/scaling/scaling_cli.py +++ b/tools/scaling/scaling_cli.py @@ -654,10 +654,8 @@ def scan_configs( else: dfs = organize_bids_tab(tab_file=selected, with_hash=with_hash) htype = "md5" - long_dfs = organize_longitudinal(dfs, htype=htype) - print(long_dfs) - # Scan workflows + # get workflows workflows = workflow_resource["brainprep"]["workflow"] workflow_mapping = workflow_resource["brainprep"]["mapping"] known_workflows = [ @@ -676,6 +674,23 @@ def scan_configs( f"Unknown workflow IDs: {', '.join(sorted(unknown))}. " f"Valid workflows are: {', '.join(sorted(known_workflows))}" ) + + # parse longitudinal if needed + parse_long = any(["longitudinal" in workflow + for workflow in allowed_workflows]) + if parse_long: + long_dfs = organize_longitudinal(dfs, htype=htype) + print(long_dfs) + else: + banner = r""" + +----------------------------------+ + | Skip longitudinal parsing | + +----------------------------------+ + """ + print(banner) + long_dfs = {} + + # Scan workflows for workflow_id, workflow_parameters in workflows.items(): if workflow_id not in allowed_workflows: print(f"\n-- skip: {workflow_id} --")