Skip to content

Improvements of scaling #63

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AymericGaudin wants to merge 4 commits into
base: dev
Choose a base branch
from
Open

Improvements of scaling #63

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
02ca314
tools/scaling/scaling.py: Use one data folder for every workflow
AymericGaudin Jun 3, 2026
72215a0
tools/scaling/scaling.py: Change config.toml for group level (avoid d…
AymericGaudin Jun 4, 2026
37e79a3
Merge branch 'dev' of https://github.com/brainprepdesk/brainprep into…
AymericGaudin Aug 26, 2026
a8d2be5
tools/scaling/scaling_cli: Skip longitudinal parsing when not needed
AymericGaudin Aug 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 27 additions & 8 deletions tools/scaling/scaling_cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,7 @@ def organize_longitudinal(
"""
banner = r"""
+----------------------------------+
| Organize longitudianl data... |
| Organize longitudinal data... |
+----------------------------------+
"""
print(banner)
Expand Down Expand Up @@ -482,7 +482,7 @@ def collect_config(
else None
)
workflow_parameters = workflow_parameters.format_map(
SafeDict({"outdir": output_dir / "data"})
SafeDict({"outdir": working_dir / "data"})
)
print(f"- edited parameters: {workflow_parameters}")

Expand All @@ -497,7 +497,7 @@ def collect_config(
if infra == "slurm":
image_parameters = (
f"--cleanenv --home {home_dir} --bind {bind_dir} "
f"--bind {output_dir} "
f"--bind {working_dir} "
)
else:
image_parameters = ""
Expand All @@ -520,12 +520,16 @@ def collect_config(
index=False,
)

commands = f'"brainprep {workflow_id} {workflow_parameters}"'
if "group" in workflow_id:
commands = f"[[{commands}]]"

config_template = config_file.read_text()
config_str = config_template.format(
name=workflow_name,
operator="TO UPDATE",
date=str(datetime.now().date()),
commands=f'"brainprep {workflow_id} {workflow_parameters}"',
commands=commands,
parameters=image_parameters,
cluster=infra,
partition=partition,
Expand Down Expand Up @@ -632,7 +636,7 @@ def scan_configs(
print("No cache files found. Parsing data.")
selected = None
else:
print("Multiple cache files found:")
print("Cache file(s) found:")
for idx, path in enumerate(cache_files, 1):
print(f"{idx}. {path.name}")
choice = input(
Expand All @@ -650,10 +654,8 @@ def scan_configs(
else:
dfs = organize_bids_tab(tab_file=selected, with_hash=with_hash)
htype = "md5"
long_dfs = organize_longitudinal(dfs, htype=htype)
print(long_dfs)

# Scan workflows
# get workflows
workflows = workflow_resource["brainprep"]["workflow"]
workflow_mapping = workflow_resource["brainprep"]["mapping"]
known_workflows = [
Expand All @@ -672,6 +674,23 @@ def scan_configs(
f"Unknown workflow IDs: {', '.join(sorted(unknown))}. "
f"Valid workflows are: {', '.join(sorted(known_workflows))}"
)

# parse longitudinal if needed
parse_long = any(["longitudinal" in workflow
for workflow in allowed_workflows])
if parse_long:
long_dfs = organize_longitudinal(dfs, htype=htype)
print(long_dfs)
else:
banner = r"""
+----------------------------------+
| Skip longitudinal parsing |
+----------------------------------+
"""
print(banner)
long_dfs = {}

# Scan workflows
for workflow_id, workflow_parameters in workflows.items():
if workflow_id not in allowed_workflows:
print(f"\n-- skip: {workflow_id} --")
Expand Down
Loading