From bf955c5bd1a510fd23035bc6e677e00c811df412 Mon Sep 17 00:00:00 2001 From: 36000 Date: Sat, 29 Aug 2026 16:27:04 -0700 Subject: [PATCH 1/3] first draft of new cli --- .gitignore | 1 - AFQ/definitions/utils.py | 6 +- AFQ/tests/test_definitions.py | 2 +- AFQ/utils/bin.py | 286 +++++++--------------- AFQ/utils/docstring_parser.py | 16 +- bin/pyAFQ | 165 +++++++------ docs/Makefile | 1 - docs/source/conf.py | 1 - docs/source/developing/definitions.rst | 5 +- docs/source/howto/cleaning_params.rst | 5 +- docs/source/howto/converter.rst | 2 +- docs/source/howto/docker.rst | 8 +- docs/source/howto/image.rst | 12 +- docs/source/howto/index.rst | 1 - docs/source/howto/installation_guide.rst | 4 +- docs/source/howto/scalars.rst | 3 +- docs/source/howto/segmentation_params.rst | 4 +- docs/source/howto/singularity.rst | 6 +- docs/source/howto/tractography_params.rst | 4 +- docs/source/reference/cli.rst | 63 +++++ docs/source/reference/index.rst | 2 +- docs/source/reference/mapping.rst | 2 +- docs/source/sphinxext/updatedocs.py | 38 --- docs/source/tutorials/index.rst | 5 +- 24 files changed, 283 insertions(+), 359 deletions(-) create mode 100644 docs/source/reference/cli.rst delete mode 100644 docs/source/sphinxext/updatedocs.py diff --git a/.gitignore b/.gitignore index 066e1d58f..45c871a98 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ AFQ/version.py docs/_build docs/build docs/source/auto_examples/ -docs/source/reference/config.rst examples/**/*.nii.gz examples/**/*.trk examples/**/*.npy diff --git a/AFQ/definitions/utils.py b/AFQ/definitions/utils.py index cd34c0cbc..b64547109 100644 --- a/AFQ/definitions/utils.py +++ b/AFQ/definitions/utils.py @@ -27,9 +27,9 @@ def __init__(self): def find_path(self, bids_layout, from_path, subject, session, required=True): pass - def str_for_toml(self): + def str_formal(self): """ - Uses __init__ in str_for_toml to make string that will instantiate + Uses __init__ in str_formal to make string that will instantiate itself. Assumes object will have attributes of same name as __init__ args. This is important for reading/writing definitions as arguments to config files. @@ -56,7 +56,7 @@ def _arglist_to_string(args, get_attr=None): if get_attr is not None: arg = getattr(get_attr, arg) if isinstance(arg, Definition): - arg = arg.str_for_toml() + arg = arg.str_formal() elif isinstance(arg, str): arg = f'"{arg}"' elif isinstance(arg, list): diff --git a/AFQ/tests/test_definitions.py b/AFQ/tests/test_definitions.py index f0eb33ba7..2c6c04840 100644 --- a/AFQ/tests/test_definitions.py +++ b/AFQ/tests/test_definitions.py @@ -15,7 +15,7 @@ def test_str_instantiates_mixin(): thresh_image = ThresholdedScalarImage("dti_fa", lower_bound=0.2) - thresh_image_str = thresh_image.str_for_toml() + thresh_image_str = thresh_image.str_formal() thresh_image_from_str = eval(thresh_image_str) npt.assert_(thresh_image.combine == thresh_image_from_str.combine) diff --git a/AFQ/utils/bin.py b/AFQ/utils/bin.py index 7bba6bd4a..9581ea83b 100644 --- a/AFQ/utils/bin.py +++ b/AFQ/utils/bin.py @@ -1,122 +1,16 @@ import datetime import os.path as op import platform -from argparse import ArgumentParser -import toml - -from AFQ.api.bundle_dict import * # interprets bundle_dicts loaded from toml # noqa F403 +from AFQ.api.bundle_dict import * # interprets bundle_dicts loaded from command line # noqa F403 from AFQ.api.bundle_dict import BundleDict from AFQ.api.utils import kwargs_descriptors -from AFQ.definitions.image import * # interprets masks loaded from toml # noqa F403 -from AFQ.definitions.mapping import * # interprets mappings loaded from toml # noqa F403 +from AFQ.definitions.image import * # interprets masks loaded from command line # noqa F403 +from AFQ.definitions.mapping import * # interprets mappings loaded from command line # noqa F403 from AFQ.definitions.utils import Definition from AFQ.utils.docstring_parser import parse_numpy_docstring -def model_input_parser(usage): - parser = ArgumentParser(usage) - - parser.add_argument( - "-d", "--dwi", dest="dwi", action="append", help="DWI files (enter one or more)" - ) - - parser.add_argument( - "-l", - "--bval", - dest="bval", - action="append", - help="B-value files (enter one or more)", - ) - - parser.add_argument( - "-c", - "--bvec", - dest="bvec", - action="append", - help="B-vector files (enter one or more)", - ) - - parser.add_argument( - "-o", - "--out_dir", - dest="out_dir", - action="store", - help="""Full path to directory for files to be saved - (will be created if it doesn't exist)")""", - ) - - parser.add_argument( - "-m", "--mask", dest="mask", action="store", default=None, help="Mask file" - ) - - parser.add_argument( - "-b", - "--b0_threshold", - dest="b0_threshold", - action="store", - help="b0 threshold", - default=0, - ) - - return parser - - -def model_predict_input_parser(usage): - parser = ArgumentParser(usage) - - parser.add_argument( - "-p", - "--params", - dest="params", - action="store", - help="A file containing model params", - ) - - parser.add_argument( - "-l", - "--bval", - dest="bval", - action="append", - help="B-value files (enter one or more)", - ) - - parser.add_argument( - "-c", - "--bvec", - dest="bvec", - action="append", - help="B-vector files (enter one or more)", - ) - - parser.add_argument( - "-o", - "--out_dir", - dest="out_dir", - action="store", - help="""Full path to directory for files to be saved - (will be created if it doesn't exist)")""", - ) - - parser.add_argument( - "-s", - "--S0_file", - dest="S0_file", - action="store", - help="File containing S0 measurements to use in prediction", - ) - - parser.add_argument( - "-b", - "--b0_threshold", - dest="b0_threshold", - help="b0 threshold (default: 0)", - action="store", - default=0, - ) - return parser - - def pyafq_str_to_val(t): if isinstance(t, str) and len(t) < 1: return None @@ -146,11 +40,11 @@ def pyafq_str_to_val(t): return t -def val_to_toml(v): +def val_to_formal(v): if v is None: return '""' elif isinstance(v, Definition): - return f'"{v.str_for_toml()}"' + return f'"{v.str_formal()}"' elif isinstance(v, str): return f'"{v}"' elif isinstance(v, bool): @@ -168,24 +62,24 @@ def val_to_toml(v): return f"{v}" -def dict_to_toml(dictionary): - toml = "# Use '' to indicate None\n# Wrap dictionaries in quotes\n" - toml = toml + "# Wrap definition object instantiations in quotes\n\n" +def arg_dict_formatted(dictionary): + desc = "# Use '' to indicate None\n# Wrap dictionaries in quotes\n" + desc = desc + "# Wrap definition object instantiations in quotes\n\n" for section, args in dictionary.items(): if section == "AFQ_desc": - toml = "# " + dictionary["AFQ_desc"].replace("\n", "\n# ") + "\n\n" + toml + desc = "# " + dictionary["AFQ_desc"].replace("\n", "\n# ") + "\n\n" + desc continue - toml = toml + f"[{section}]\n" + desc = desc + f"[{section}]\n" for arg, arg_info in args.items(): - toml = toml + "\n" - if isinstance(arg_info, dict): + desc = desc + "\n" + if isinstance(arg_info, dict) and "default" in arg_info: if "desc" in arg_info: - toml = toml + arg_info["desc"] - toml = toml + f"{arg} = {val_to_toml(arg_info['default'])}\n" + desc = desc + arg_info["desc"] + desc = desc + f"{arg} = {val_to_formal(arg_info['default'])}\n" else: - toml = toml + f"{arg} = {val_to_toml(arg_info)}\n" - toml = toml + "\n" - return toml + "\n" + desc = desc + f"{arg} = {val_to_formal(arg_info)}\n" + desc = desc + "\n" + return desc + "\n" # these params are handled internally in the qsiprep pipeline, @@ -210,9 +104,9 @@ def dict_to_json(dictionary): continue local_ignore.append(arg) if isinstance(arg_info, dict): - json = json + f'"{arg}": {val_to_toml(arg_info["default"])}' + json = json + f'"{arg}": {val_to_formal(arg_info["default"])}' else: - json = json + f'"{arg}": {val_to_toml(arg_info)}' + json = json + f'"{arg}": {val_to_formal(arg_info)}' json = json + ",\n " return json[:-18] # remove trailing ,\n and indent @@ -221,14 +115,12 @@ def func_dict_to_arg_dict(func_dict=None, logger=None): if func_dict is None: import AFQ.tractography.tractography as aft from AFQ.api.group import GroupAFQ - from AFQ.recognition.cleaning import clean_bundle from AFQ.recognition.recognize import recognize func_dict = { "BIDS": GroupAFQ.__init__, "Tractography": aft.track, "Segmentation": recognize, - "Cleaning": clean_bundle, } arg_dict = {} @@ -283,55 +175,87 @@ def func_dict_to_arg_dict(func_dict=None, logger=None): def parse_config_run_afq( - toml_file, + dwi, + bval, + bvec, + t1, + o_folder, default_arg_dict, + cli_args, to_call="export_all", - overwrite=False, logger=None, verbose=False, dry_run=False, - special_args=None, ): from AFQ import __version__ - from AFQ.api.group import GroupAFQ + from AFQ.api.participant import ParticipantAFQ - # load configuration file - if special_args is None: - special_args = { - "SEGMENTATION_PARAMS": "segmentation_params", - "TRACTOGRAPHY_PARAMS": "tracking_params", - } - if not op.exists(toml_file): - raise FileExistsError( - "Config file does not exist. " - + "If you want to generate this file," - + " add the argument --generate-config-only" - ) - f_arg_dict = toml.load(toml_file) + f_arg_dict = vars(cli_args) + + special_args = { + "SEGMENTATION_PARAMS": "segmentation_params", + "TRACTOGRAPHY_PARAMS": "tracking_params", + } + + special_args_assignment = {} + for section_name, new_section_name in special_args.items(): + if section_name in default_arg_dict: + for arg in default_arg_dict[section_name].keys(): + special_args_assignment[arg] = new_section_name + + if bval is False: + bval = dwi.replace(".nii.gz", ".bval") + if not op.exists(bval): + bval = dwi.replace(".nii.gz", ".bvals") + if not op.exists(bval): + bval = dwi.replace(".nii", ".bval") + if not op.exists(bval): + bval = dwi.replace(".nii", ".bvals") + if not op.exists(bval): + raise FileNotFoundError( + "Could not find bval file. Please specify the path to the bval file." + ) + if bvec is False: + bvec = dwi.replace(".nii.gz", ".bvec") + if not op.exists(bvec): + bvec = dwi.replace(".nii.gz", ".bvecs") + if not op.exists(bvec): + bvec = dwi.replace(".nii", ".bvec") + if not op.exists(bvec): + bvec = dwi.replace(".nii", ".bvecs") + if not op.exists(bvec): + raise FileNotFoundError( + "Could not find bvec file. Please specify the path to the bvec file." + ) # extract arguments from file kwargs = {} - bids_path = "" - for section, args in f_arg_dict.items(): - for arg, default in args.items(): - if section not in default_arg_dict: - default_arg_dict[section] = {} - if arg == "bids_path": - bids_path = default - else: - val = pyafq_str_to_val(default) - is_special = False - for toml_key, doc_arg in special_args.items(): - if section == toml_key: - if doc_arg not in kwargs: - kwargs[doc_arg] = {} - kwargs[doc_arg][arg] = val - is_special = True - if not is_special: - kwargs[arg] = val - if arg not in default_arg_dict[section]: - default_arg_dict[section][arg] = {} - default_arg_dict[section][arg]["default"] = default + + for arg, default in f_arg_dict.items(): + if arg in [ + "dwi", + "bvec", + "bval", + "t1", + "o_folder", + "verbose", + "dry_run", + "to_call", + ]: + continue + val = pyafq_str_to_val(default) + if val is None: + continue + if arg in special_args_assignment: + section_name = special_args_assignment[arg] + if section_name not in kwargs: + kwargs[section_name] = {} + kwargs[section_name][arg] = val + else: + kwargs[arg] = val + if arg not in default_arg_dict: + default_arg_dict[arg] = {} + default_arg_dict[arg]["default"] = default if logger is not None and (verbose or dry_run): logger.info("The following arguments are recognized: " + str(kwargs)) @@ -339,16 +263,6 @@ def parse_config_run_afq( if dry_run: return - # if overwrite, write new file with updated docs / args - if overwrite: - if logger is not None: - logger.info("Updating configuration file.") - with open(toml_file, "w") as ff: - ff.write(dict_to_toml(default_arg_dict)) - - if bids_path == "": - raise RuntimeError("Config file must provide bids_path") - # generate metadata file for this run default_arg_dict["pyAFQ"] = {} default_arg_dict["pyAFQ"]["utc_time_started"] = datetime.datetime.now().isoformat( @@ -357,11 +271,11 @@ def parse_config_run_afq( default_arg_dict["pyAFQ"]["version"] = __version__ default_arg_dict["pyAFQ"]["platform"] = platform.system() - myafq = GroupAFQ(bids_path, **kwargs) + myafq = ParticipantAFQ(dwi, bval, bvec, t1, o_folder, **kwargs) - afq_metadata_file = op.join(myafq.afq_path, "afq_metadata.toml") + afq_metadata_file = op.join(o_folder, "afq_metadata.toml") with open(afq_metadata_file, "w") as ff: - ff.write(dict_to_toml(default_arg_dict)) + ff.write(arg_dict_formatted(default_arg_dict)) # call user specified function: if to_call == "all": @@ -372,21 +286,7 @@ def parse_config_run_afq( # If you got this far, you can report on time ended and record that: default_arg_dict["pyAFQ"]["utc_time_ended"] = datetime.datetime.now().isoformat("T") with open(afq_metadata_file, "w") as ff: - ff.write(dict_to_toml(default_arg_dict)) - - -def generate_config(toml_file, default_arg_dict, overwrite=False, logger=None): - if not overwrite and op.exists(toml_file): - raise FileExistsError( - "Config file already exists. " - + "If you want to overwrite this file," - + " add the argument --overwrite-config" - ) - if logger is not None: - logger.info("Generating default configuration file.") - toml_file = open(toml_file, "w") - toml_file.write(dict_to_toml(default_arg_dict)) - toml_file.close() + ff.write(arg_dict_formatted(default_arg_dict)) def generate_json(json_folder, overwrite=False, logger=None): diff --git a/AFQ/utils/docstring_parser.py b/AFQ/utils/docstring_parser.py index 18df5bd1c..70d89e2d6 100644 --- a/AFQ/utils/docstring_parser.py +++ b/AFQ/utils/docstring_parser.py @@ -102,16 +102,20 @@ def parse_numpy_docstring(docstring): default = None default_match = re.search(r"[Dd]efault:\s*([^\n]+)", desc) if default_match: + kwarg = True default = default_match.group(1).strip(" .") try: default = eval(default) except Exception: default = _white_space(default) - - params[name] = { - "help": _white_space(desc), - "metavar": _white_space(type_info), - "default": default, - } + else: + kwarg = False + + if kwarg: + params[name] = { + "help": _white_space(desc), + "metavar": _white_space(type_info), + "default": default, + } return {"description": _white_space(description), "arguments": params} diff --git a/bin/pyAFQ b/bin/pyAFQ index 5c649b50d..d6ee2fe3d 100755 --- a/bin/pyAFQ +++ b/bin/pyAFQ @@ -1,6 +1,7 @@ #!/usr/bin/env python -import os.path as op +import os +import sys import warnings from argparse import ArgumentParser @@ -16,58 +17,84 @@ with warnings.catch_warnings(): logger = logging.getLogger("AFQ") logger.setLevel(level=logging.INFO) -usage = """pyAFQ /path/to/afq_config.toml +usage = """ +pyAFQ [OPTIONS] dwi t1 o_folder +pyAFQ download +pyAFQ qsiprep -Runs full AFQ processing as specified in the configuration file. +The first form runs the pyAFQ tractometry pipeline on a single subject. -For details about configuration, see instructions in: -https://tractometry.org/pyAFQ/reference/config.html +Two subcommands are also available, and take no further arguments: -The default configuration file looks like: + download Fetch every template, atlas, and model pyAFQ may need and + cache them locally, then exit. Run this once if you intend + to use pyAFQ on a machine without internet access, or to + build a Docker image. + qsiprep Write default JSON configuration files into the current + working directory and exit. These can be used to define a + pyAFQ recon workflow in qsiprep; no pipeline is run. + +General options: + + -h, --help Show this help message and arguments. + -v, --verbose Verbose when reading the TOML file. + -d, --dry-run Print the recognized arguments without running pyAFQ. + -c, --call AFQ.api attribute to get. Defaults to 'all', which + runs the entire tractometry pipeline. """ def parse_cli(arg_dict): - cli_parser = ArgumentParser(usage=usage + afb.dict_to_toml(arg_dict)) + if len(sys.argv) == 2 and sys.argv[1] == "download": + logger.info("Downloading templates...") + download_templates() + exit() + elif len(sys.argv) >= 2 and sys.argv[1] == "qsiprep": + logger.info("Generating JSON config for qsiprep...") + afb.generate_json(os.getcwd(), logger=logger) + exit() + + cli_parser = ArgumentParser(usage=usage) + + cli_parser.add_argument(dest="dwi", action="store", help="Path to DWI data file") cli_parser.add_argument( - dest="config", + dest="t1", action="store", - help="Path to config file or folder. " - + "For example, /path/to/afq_config.toml", + help=( + "Path to T1-weighted image file. " + "Must already be registered to the " + "DWI data, though not resampled." + ), ) cli_parser.add_argument( - "-g", - "--generate-config-only", - dest="generate_toml", - action="store_true", - default=False, - help="Generate a default config file at the path" - + " specified without running pyAFQ.", + dest="o_folder", action="store", help="Path to output folder" ) cli_parser.add_argument( - "-q", - "--generate-qsiprep-json-only", - dest="generate_json", - action="store_true", + "--bval", + dest="bval", + action="store", default=False, - help="Generate two default json files at the path" - + " (which should be a folder) specified without running pyAFQ;" - + " this json can be used to define a recon workflow in qsiprep.", + help=( + "Path to bval file. " + "If none, the dwi data " + "file path will be used to find it." + ), ) cli_parser.add_argument( - "-o", - "--overwrite-config", - dest="overwrite", - action="store_true", + "--bvec", + dest="bvec", + action="store", default=False, - help="Overwrite config file at the path" - + " with current arguments and comments," - + " preserving previous defaults when applicable.", + help=( + "Path to bvec file. " + "If none, the dwi data " + "file path will be used to find it." + ), ) cli_parser.add_argument( @@ -93,31 +120,26 @@ def parse_cli(arg_dict): "-c", "--call", dest="to_call", + action="store", default="all", help="AFQ.api attribute to get using the specified config file." + " Defaults to 'all', which will perform the entire" + " tractometry pipeline.", ) - cli_parser.add_argument( - "-t", - "--notrack", - action="store_true", - default=False, - help="Disable the use of pyAFQ being recorded by Google Analytics. ", - ) + for key_, args in arg_dict.items(): + if key_ == "AFQ_desc" or key_ == "BIDS_PARAMS": + continue + for arg, arg_info in args.items(): + cli_parser.add_argument( + f"--{arg}", + dest=arg, + action="store", + default=arg_info["default"], + help=arg_info["desc"], + ) - opts = cli_parser.parse_args() - - return ( - opts.config, - opts.generate_toml, - opts.overwrite, - opts.verbose, - opts.dry_run, - opts.to_call, - opts.generate_json, - ) + return cli_parser.parse_args() def download_templates(): @@ -140,35 +162,18 @@ def download_templates(): if __name__ == "__main__": arg_dict = afb.func_dict_to_arg_dict(logger=logger) - config_file, generate_only, overwrite, verbose, dry_run, to_call, generate_json = ( - parse_cli(arg_dict) + args = parse_cli(arg_dict) + + afb.parse_config_run_afq( + args.dwi, + args.bval, + args.bvec, + args.t1, + args.o_folder, + arg_dict, + args, + to_call=args.to_call, + verbose=args.verbose, + dry_run=args.dry_run, + logger=logger, ) - - if config_file == "download": - logger.info("Downloading templates...") - download_templates() - exit() - - if generate_only and generate_json: - raise ValueError("Can only generate .toml or .json; not both") - if generate_json: - if not op.isdir(config_file): - raise ValueError("Config must be a folder when generating a .json") - else: - if config_file[-5:] != ".toml": - raise ValueError("Config file must be .toml unless generating a .json") - - if generate_only: - afb.generate_config(config_file, arg_dict, overwrite, logger=logger) - elif generate_json: - afb.generate_json(config_file, overwrite, logger=logger) - else: - afb.parse_config_run_afq( - config_file, - arg_dict, - to_call=to_call, - overwrite=overwrite, - verbose=verbose, - dry_run=dry_run, - logger=logger, - ) diff --git a/docs/Makefile b/docs/Makefile index 1feacc09f..c1868fc4f 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -45,7 +45,6 @@ ARTIFACTS := $(strip \ distclean: clean @echo Removing files created by sphinx-build rm -rf $(BUILDDIR) - rm -f $(SOURCEDIR)/reference/config.rst rm -rf $(SOURCEDIR)/auto_examples/ $(if $(ARTIFACTS),rm -f $(ARTIFACTS)) diff --git a/docs/source/conf.py b/docs/source/conf.py index 8c8da81f1..28b2f7bbb 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -56,7 +56,6 @@ 'sphinxcontrib.bibtex', 'autoapi.extension', 'numpydoc', - 'updatedocs', 'kwargsdocs', 'methodsdocs', ] diff --git a/docs/source/developing/definitions.rst b/docs/source/developing/definitions.rst index 734e07820..5f8a4d4f2 100644 --- a/docs/source/developing/definitions.rst +++ b/docs/source/developing/definitions.rst @@ -17,8 +17,7 @@ methods are described below: the :class:`AFQ.api.group.GroupAFQ` object. These `__init__` methods must be thoroughly documented as they are what the user interacts with. The class must have attributes of same name as the `__init__` - args. This is important for reading/writing `Definition`-inherited - classes as arguments to config files. + args. - The api calls `find_path` during the :class:`AFQ.api.group.GroupAFQ` object initialization to let the definition find relevant files for @@ -50,7 +49,7 @@ Image definitions require `get_name`, `get_image_getter`, - `get_image_getter` returns a method which can be called as task in the task workflow specified by its one input, `task_name`. This - method can have any valid inputs for its task module and ouputs an + method can have any valid inputs for its task module and outputs an image. - `get_image_direct` returns the image. It is similar to diff --git a/docs/source/howto/cleaning_params.rst b/docs/source/howto/cleaning_params.rst index 3aed36684..bd007f417 100644 --- a/docs/source/howto/cleaning_params.rst +++ b/docs/source/howto/cleaning_params.rst @@ -3,9 +3,8 @@ Cleaning Parameters ========================== This page documents the configuration options for controlling -bundle cleaning in pyAFQ. These parameters can be set in your -configuration file or passed as arguments when using the API. -Note that this goes inside of segmentation_params. +bundle cleaning in pyAFQ. Note that this goes inside of +segmentation_params. Example Usage ============= diff --git a/docs/source/howto/converter.rst b/docs/source/howto/converter.rst index a99a4d558..4566508b5 100644 --- a/docs/source/howto/converter.rst +++ b/docs/source/howto/converter.rst @@ -2,7 +2,7 @@ Tractography from other pipelines ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ pyAFQ can use tractography from other pipelines. To tell pyAFQ to use tractography from another pipeline, use the import_tract -argument in the AFQ.api objects or in the configuration file. This argument expects +argument in the AFQ.api objects or from the command line. This argument expects a dictionary of BIDS filters. pyAFQ will use these BIDS filters to find the tractography for each subject in each session. Here is an example import_tract:: diff --git a/docs/source/howto/docker.rst b/docs/source/howto/docker.rst index 7b152637e..6f4eb2fd8 100644 --- a/docs/source/howto/docker.rst +++ b/docs/source/howto/docker.rst @@ -1,15 +1,15 @@ The pyAFQ docker image ~~~~~~~~~~~~~~~~~~~~~~ -Everytime a new commit is made to master the +Every time a new commit is made to master the `pyAFQ github `_, a new image is pushed to the `NRDG github `_. This image contains an installation of the latest version of pyAFQ with fslpy. This image also contains an entrypoint, and can be run with:: - docker run -v bids_dir:/bids_dir:rw ghcr.io/nrdg/pyafq bids_dir/config.toml + docker run -v bids_dir:/bids_dir:rw ghcr.io/nrdg/pyafq path/to/dwi path/to/t1 path/to/output_folder -This is using the config file which you can -read about in `The pyAFQ configuration file <../reference/config.html>`_. +This is using the CLI which you can +read about in `The pyAFQ CLI <../reference/cli.html>`_. You can also launch python inside the container and use the normal pyAFQ. diff --git a/docs/source/howto/image.rst b/docs/source/howto/image.rst index 6ae08b932..6c86e1266 100644 --- a/docs/source/howto/image.rst +++ b/docs/source/howto/image.rst @@ -14,23 +14,21 @@ Currently, there are three different images that pyAFQ uses for tractometry: models. By default, it is calculated using :class:`AFQ.definitions.image.B0Image`. #. The tractography seed image. This image determines where tractography is - seeded. If it is floating point, the image is thresholded interally after + seeded. If it is floating point, the image is thresholded internally after interpolation using the seed_threshold parameter. This is recommended. - However, the seed image can aslo be a binary image. By default, the + However, the seed image can also be a binary image. By default, the seed image is :class:`AFQ.definitions.image.ScalarImage` (best_scalar) where best_scalar is chosen by the API based on valid scalars (typically "dti_fa"). #. The tractography stop image. This image determines where tractography stops. - If it is floating point, the image is thresholded interally after + If it is floating point, the image is thresholded internally after interpolation using the stop_threshold parameter. This is recommended. - However, the stop image can aslo be a binary image. By default, the + However, the stop image can also be a binary image. By default, the stop image is :class:`AFQ.definitions.image.ScalarImage` (best_scalar) where best_scalar is chosen by the API based on valid scalars (typically "dti_fa"). In AFQ/definitions/image.py, there are several image classes one can use to specify images. -As a user, one should initialize image classes and pass them to the AFQ.api objects, -or write out the initialization as a string inside of one's configuration file -for use with the CLI. +As a user, one should initialize image classes and pass them to the AFQ.api objects. - :class:`AFQ.definitions.image.ImageFile`: The simplest image class is :class:`AFQ.definitions.image.ImageFile`. If the image you want to use is already generated, use this class. It is initialized using BIDS filters, diff --git a/docs/source/howto/index.rst b/docs/source/howto/index.rst index e00992c1d..52b0966cb 100644 --- a/docs/source/howto/index.rst +++ b/docs/source/howto/index.rst @@ -14,7 +14,6 @@ software. data kwargs rerun - config mask scalars converter diff --git a/docs/source/howto/installation_guide.rst b/docs/source/howto/installation_guide.rst index 0a57ee2a2..86f67c711 100644 --- a/docs/source/howto/installation_guide.rst +++ b/docs/source/howto/installation_guide.rst @@ -60,7 +60,7 @@ On some platforms, you may need to add quotes around the ``.[]`` part: .. note:: It is also recommended that you utilize python virtual environment and - package mangagement tools (e.g., conda) and begin with a clean environment. + package management tools (e.g., conda) and begin with a clean environment. .. note:: @@ -106,7 +106,7 @@ specific commit or tag as well: *********************************************** If the user intends to execute pyAFQ as a program from the command line -(``$pyAFQ /path/to/config.toml``) in an administered environment where root +in an administered environment where root access is not available (e.g., High Performance Computing cluster) then one solution is to build an Apptainer (also known as Singularity) image from a local pull of the pyAFQ docker container. diff --git a/docs/source/howto/scalars.rst b/docs/source/howto/scalars.rst index 90a4ca217..5f508f30e 100644 --- a/docs/source/howto/scalars.rst +++ b/docs/source/howto/scalars.rst @@ -10,8 +10,7 @@ numeric value per voxel). In AFQ/definitions/image.py, there are many classes one can use to define custom images. Two of these classes are particularly useful to specify custom scalars. As a user, one should initialize one of these -classes and pass them to the AFQ.api objects, or write out the initialization as -a string inside of one's configuration file for use with the CLI. To do this, +classes and pass them to the AFQ.api objects. To do this, give an image object as an element of the scalars array passed to :class:`AFQ.api.group.GroupAFQ`. Then your custom image will be automatically used during tract profile extraction. diff --git a/docs/source/howto/segmentation_params.rst b/docs/source/howto/segmentation_params.rst index 6e8e6d783..ed50bbed6 100644 --- a/docs/source/howto/segmentation_params.rst +++ b/docs/source/howto/segmentation_params.rst @@ -3,8 +3,8 @@ Segmentation Parameters ========================== This page documents the configuration options for controlling -tractography in pyAFQ. These parameters can be set in your -configuration file or passed as arguments when using the API. +tractography in pyAFQ. These parameters can be passed directly +when using the CLI or passed as arguments when using the API. Example Usage ============= diff --git a/docs/source/howto/singularity.rst b/docs/source/howto/singularity.rst index cefd98983..f93923945 100644 --- a/docs/source/howto/singularity.rst +++ b/docs/source/howto/singularity.rst @@ -13,10 +13,10 @@ entrypoint ``pyafq`` to run the workflow with:: apptainer run \ --bind bids_dir:bids_dir \ - pyafq_latest.sif bids_dir/config.toml + pyafq_latest.sif /path/to/dwi /path/to/t1 /path/to/output_folder -This is using the config file which you can read about -in `The pyAFQ configuration file <../reference/config.html>`_. +This is using the CLI which you can read about +in `The pyAFQ CLI <../reference/cli.html>`_. You can also launch python inside the container and use the normal pyAFQ. .. note:: diff --git a/docs/source/howto/tractography_params.rst b/docs/source/howto/tractography_params.rst index 7e2a45bb7..8d93cfcc6 100644 --- a/docs/source/howto/tractography_params.rst +++ b/docs/source/howto/tractography_params.rst @@ -3,8 +3,8 @@ Tractography Parameters ========================== This page documents the configuration options for controlling -tractography in pyAFQ. These parameters can be set in your configuration file -or passed as arguments when using the API. +tractography in pyAFQ. These parameters can be passed directly +when using the CLI or passed as arguments when using the API. Example Usage ============= diff --git a/docs/source/reference/cli.rst b/docs/source/reference/cli.rst new file mode 100644 index 000000000..7f7e24bef --- /dev/null +++ b/docs/source/reference/cli.rst @@ -0,0 +1,63 @@ +.. _cli-label: + +The pyAFQ CLI +~~~~~~~~~~~~~ + +pyAFQ can be called from the command line. The following usage is available: + +.. code-block:: none + + pyAFQ [OPTIONS] dwi t1 o_folder + pyAFQ download + pyAFQ qsiprep + +The first form runs the pyAFQ tractometry pipeline on a single subject. + +Two subcommands are also available, and take no further arguments: + +``download`` + Fetch every template, atlas, and model pyAFQ may need and cache them + locally, then exit. Run this once if you intend to use pyAFQ on a + machine without internet access, or to build a Docker image. + +``qsiprep`` + Write default JSON configuration files into the current working + directory and exit. These can be used to define a pyAFQ recon workflow + in qsiprep; no pipeline is run. + +Positional arguments +-------------------- + +``dwi`` + Path to DWI data file. + +``t1`` + Path to T1-weighted image file. Must already be registered to the DWI + data, though not resampled. + +``o_folder`` + Path to output folder. + +Options +------- + +-h, --help Show this help message and exit. +--bval BVAL Path to bval file. If none, the DWI data file path + will be used to find it. +--bvec BVEC Path to bvec file. If none, the DWI data file path + will be used to find it. +-v, --verbose Verbose when reading the TOML file. +-d, --dry-run Perform a dry run — print the recognized arguments + without running pyAFQ. +-c TO_CALL, --call TO_CALL + AFQ.api attribute to get using the specified config + file. Defaults to ``all``, which performs the entire + tractometry pipeline. + +Note that all other pyAFQ optional parameters can also be passed in. To see them, +use the ``--help`` option, i.e., ``pyAFQ --help``. Here is a full example call to +the pyAFQ CLI: + +.. code-block:: none + + pyAFQ /home/john/AFQ_data/HBN/derivatives/qsiprep/sub-NDARAA948VFH/ses-HBNsiteRU/dwi/sub-NDARAA948VFH_ses-HBNsiteRU_acq-64dir_space-T1w_desc-preproc_dwi.nii.gz /home/john/AFQ_data/HBN/derivatives/qsiprep/sub-NDARAA948VFH/anat/sub-NDARAA948VFH_desc-preproc_T1w.nii.gz /home/john/AFQ_data/HBN/derivatives/afq/sub-NDARAA948VFH/ses-HBNsiteRU/dwi --rng_seed=2026 --return_idx=True --pve="multiaxial+brainchop+synthseg" diff --git a/docs/source/reference/index.rst b/docs/source/reference/index.rst index 9cc5686bb..11ca74ea3 100644 --- a/docs/source/reference/index.rst +++ b/docs/source/reference/index.rst @@ -13,4 +13,4 @@ kwargs viz_backend mapping - config + cli diff --git a/docs/source/reference/mapping.rst b/docs/source/reference/mapping.rst index 32704349f..0442da32d 100644 --- a/docs/source/reference/mapping.rst +++ b/docs/source/reference/mapping.rst @@ -9,7 +9,7 @@ our Mask API. In the :mod:`AFQ.definitions.mapping` module, there are four mapping classes one can use to specify the mapping. As a user, one should initialize mapping classes and pass them to the AFQ.api objects, or write out the initialization as a -string inside of one's configuration file for use with the CLI. +string inside to pass into the CLI. - :class:`AFQ.definitions.mapping.SynMap`: The default mapping class is to use Symmetric Diffeomorphic Image Registration (SyN). This is done with an diff --git a/docs/source/sphinxext/updatedocs.py b/docs/source/sphinxext/updatedocs.py deleted file mode 100644 index 148f288fd..000000000 --- a/docs/source/sphinxext/updatedocs.py +++ /dev/null @@ -1,38 +0,0 @@ -# This updates usage/config.rst to the latest cli -# Developers can run this after modifying any arguments the user can see - -from AFQ.utils.bin import dict_to_toml, func_dict_to_arg_dict - -prologue = """ -The pyAFQ configuration file ----------------------------- - -This file should be a `toml `_ file. At -minimum, the file should contain the BIDS path:: - - [files] - bids_path = "path/to/study" - - -But additional configuration options can be provided. -See an example configuration file below:: - - title = "My AFQ analysis" - -""" - -epilogue = """ -pyAFQ will store a copy of the configuration file alongside the computed -results. Note that the `title` variable and `[metadata]` section are both for -users to enter any title/metadata they would like and pyAFQ will generally -ignore them. -""" - - -def setup(app): - arg_dict = func_dict_to_arg_dict() - example_config = dict_to_toml(arg_dict) - example_config = " " + example_config.replace("\n", "\n ") - - with open("./source/reference/config.rst", "w") as ff: - ff.write(prologue + example_config + epilogue) diff --git a/docs/source/tutorials/index.rst b/docs/source/tutorials/index.rst index 64f4058b6..1ef17b40d 100644 --- a/docs/source/tutorials/index.rst +++ b/docs/source/tutorials/index.rst @@ -13,9 +13,8 @@ Then, you are ready to run pyAFQ in one of the following ways: Detailed tutorials for this are provided in the link at the bottom of the page. 1. The second is as a program run in the command line. After installing the software - and organizing the data, run: `pyAFQ /path/to/config.toml`, - pointing the program to the location of a configuration file (see - :doc:`configuration file specification ` for an + and organizing the data, run: `pyAFQ /path/to/dwi /path/to/t1 /path/to/output_folder`, + (see :doc:`The pyAFQ CLI ` for an explanation of this file). This will run whole-brain tractography, segment the tracts, and extract tract-profiles for each tract, generating a CSV file under that contains the tract profiles for all From 1eed74c4be4788374e1ef1c8860f2ef23e9b3fbf Mon Sep 17 00:00:00 2001 From: 36000 Date: Sat, 29 Aug 2026 19:51:31 -0700 Subject: [PATCH 2/3] copilot catches --- AFQ/recognition/recognize.py | 3 ++- AFQ/tractography/tractography.py | 11 ++++++----- AFQ/utils/bin.py | 9 ++++++--- bin/pyAFQ | 2 +- docs/source/howto/segmentation_params.rst | 2 +- docs/source/reference/cli.rst | 2 +- docs/source/tutorials/index.rst | 12 ++++++------ 7 files changed, 23 insertions(+), 18 deletions(-) diff --git a/AFQ/recognition/recognize.py b/AFQ/recognition/recognize.py index 12b9012b4..7e7bfeccb 100644 --- a/AFQ/recognition/recognize.py +++ b/AFQ/recognition/recognize.py @@ -114,7 +114,8 @@ def recognize( Default: 4 save_intermediates : str, optional The full path to a folder into which intermediate products - are saved. Default: None, means no saving of intermediates. + are saved. If None, means no saving of intermediates. + Default: None. cleaning_params : dict, optional Cleaning params to pass to seg.clean_bundle. This will override the default parameters of that method. However, this diff --git a/AFQ/tractography/tractography.py b/AFQ/tractography/tractography.py index 33eb97e4b..425ed0cba 100644 --- a/AFQ/tractography/tractography.py +++ b/AFQ/tractography/tractography.py @@ -69,18 +69,19 @@ def track( pft refers to Particle Filtering Tracking ([Girard2014]_). Default: "prob" max_angle : float, optional. - The maximum turning angle in each step. Default: 30 + The maximum turning angle in each step. + Default: 30 sphere : str or DIPY Sphere The discretization of the ODF. Can be a DIPY Sphere or a string name of a DIPY Sphere. Default: "repulsion724" seed_mask : array, optional. Float or binary mask describing the ROI within which we seed for - tracking. - Default to the entire volume (all ones). + tracking. If None, use the entire volume (all ones). + Default: None seed_threshold : float, optional. A value of the seed_mask above which tracking is seeded. - Default to 0. + Default: 0 gm_threshold : float, optional. A value of the pve_gm_data above which we consider a voxel to be GM for the purposes of ACT stopping criterion. Default: 0.4. @@ -114,7 +115,7 @@ def track( {"DTI", "CSD", "DKI", "GQ", "RUMBA", "MSMT_AODF", "CSD_AODF", "MSMTCSD"}. If a Definition, we assume it is a definition of a file containing Spherical Harmonics coefficients. - Defaults to use "CSD_AODF" + Default: "CSD_AODF" basis_type : str, optional The spherical harmonic basis type used to represent the coefficients. One of {"descoteaux07", "tournier07"}. Default: "descoteaux07" diff --git a/AFQ/utils/bin.py b/AFQ/utils/bin.py index 9581ea83b..733c679ce 100644 --- a/AFQ/utils/bin.py +++ b/AFQ/utils/bin.py @@ -253,9 +253,12 @@ def parse_config_run_afq( kwargs[section_name][arg] = val else: kwargs[arg] = val - if arg not in default_arg_dict: - default_arg_dict[arg] = {} - default_arg_dict[arg]["default"] = default + for section, args in default_arg_dict.items(): + if section == "AFQ_desc" or not isinstance(args, dict): + continue + if arg in args and isinstance(args[arg], dict): + args[arg]["default"] = default + break if logger is not None and (verbose or dry_run): logger.info("The following arguments are recognized: " + str(kwargs)) diff --git a/bin/pyAFQ b/bin/pyAFQ index d6ee2fe3d..833e2bfeb 100755 --- a/bin/pyAFQ +++ b/bin/pyAFQ @@ -38,7 +38,7 @@ Two subcommands are also available, and take no further arguments: General options: -h, --help Show this help message and arguments. - -v, --verbose Verbose when reading the TOML file. + -v, --verbose Verbose logging. -d, --dry-run Print the recognized arguments without running pyAFQ. -c, --call AFQ.api attribute to get. Defaults to 'all', which runs the entire tractometry pipeline. diff --git a/docs/source/howto/segmentation_params.rst b/docs/source/howto/segmentation_params.rst index ed50bbed6..982ab3cfa 100644 --- a/docs/source/howto/segmentation_params.rst +++ b/docs/source/howto/segmentation_params.rst @@ -3,7 +3,7 @@ Segmentation Parameters ========================== This page documents the configuration options for controlling -tractography in pyAFQ. These parameters can be passed directly +bundle recognition in pyAFQ. These parameters can be passed directly when using the CLI or passed as arguments when using the API. Example Usage diff --git a/docs/source/reference/cli.rst b/docs/source/reference/cli.rst index 7f7e24bef..14730a336 100644 --- a/docs/source/reference/cli.rst +++ b/docs/source/reference/cli.rst @@ -46,7 +46,7 @@ Options will be used to find it. --bvec BVEC Path to bvec file. If none, the DWI data file path will be used to find it. --v, --verbose Verbose when reading the TOML file. +-v, --verbose Verbose logging. -d, --dry-run Perform a dry run — print the recognized arguments without running pyAFQ. -c TO_CALL, --call TO_CALL diff --git a/docs/source/tutorials/index.rst b/docs/source/tutorials/index.rst index 1ef17b40d..d526dd732 100644 --- a/docs/source/tutorials/index.rst +++ b/docs/source/tutorials/index.rst @@ -13,12 +13,12 @@ Then, you are ready to run pyAFQ in one of the following ways: Detailed tutorials for this are provided in the link at the bottom of the page. 1. The second is as a program run in the command line. After installing the software - and organizing the data, run: `pyAFQ /path/to/dwi /path/to/t1 /path/to/output_folder`, - (see :doc:`The pyAFQ CLI ` for an - explanation of this file). This will run whole-brain tractography, segment - the tracts, and extract tract-profiles for each tract, generating a CSV - file under that contains the tract profiles for all - participants/tracts/statistics. + and organizing the data, run: `pyAFQ /path/to/dwi /path/to/t1 /path/to/output_folder`, + (see :doc:`The pyAFQ CLI `). + This will run whole-brain tractography, segment + the tracts, and extract tract-profiles for each tract, generating a CSV + file under that contains the tract profiles for all + participants/tracts/statistics. .. toctree:: :maxdepth: 2 From 1910750165522a7c10fd580d8b127c7c658a7152 Mon Sep 17 00:00:00 2001 From: 36000 Date: Sat, 29 Aug 2026 20:44:32 -0700 Subject: [PATCH 3/3] BFs --- AFQ/api/utils.py | 3 +++ AFQ/tests/test_api.py | 51 ++++++++++++++++--------------------------- AFQ/utils/bin.py | 48 +++++++++++++++++++++------------------- 3 files changed, 48 insertions(+), 54 deletions(-) diff --git a/AFQ/api/utils.py b/AFQ/api/utils.py index 4dd839307..8936b21c3 100644 --- a/AFQ/api/utils.py +++ b/AFQ/api/utils.py @@ -135,6 +135,9 @@ def check_attribute(attr_name): if attr_name[:-5] in task_modules: return None + if attr_name in ["tracking_params", "segmentation_params"]: + return "segmentation_imap" + if attr_name in methods_sections: return f"{methods_sections[attr_name]}_imap" diff --git a/AFQ/tests/test_api.py b/AFQ/tests/test_api.py index ccc15c820..603298a95 100644 --- a/AFQ/tests/test_api.py +++ b/AFQ/tests/test_api.py @@ -14,7 +14,6 @@ import numpy.testing as npt import pandas as pd import pytest -import toml from dipy.io.streamline import load_tractogram from dipy.segment.metric import mdf from pandas.testing import assert_series_equal @@ -991,41 +990,29 @@ def test_AFQ_data_waypoint(): "inclusive_labels=[1, 2]))" ) bm_def_as_str = ( - "LabelledImageFile(suffix='seg', " - "filters={'scope': 'freesurfer'}, " + 'LabelledImageFile(suffix="seg", ' + 'filters={"scope": "freesurfer"}, ' "exclusive_labels=[0])" ) - config = dict( - BIDS_PARAMS=dict( - bids_path=bids_path, - dwi_preproc_pipeline="vistasoft", - t1_preproc_pipeline="freesurfer", - ), - STRUCTURAL=dict( - brain_mask_definition=bm_def_as_str, - ), - DATA=dict(bundle_info=bundle_dict_as_str), - TISSUE=dict(pve=pve_as_str), - SEGMENTATION=dict( - n_points_profile=50, - scalars=[ - "dti_fa", - "dti_md", - "dti_ga", - "t1w_over_b0", - f"ImageFile('{t1_path_other}')", - f"TemplateImage('{t1_path}')", - ], - ), - TRACTOGRAPHY_PARAMS=tracking_params, - SEGMENTATION_PARAMS=segmentation_params, - ) - config_file = op.join(tmpdir, "afq_config.toml") - with open(config_file, "w") as ff: - toml.dump(config, ff) + cmd = ( + f"pyAFQ -v {op.join(vista_folder, 'sub-01_ses-01_dwi.nii.gz')}" + f" {op.join(freesurfer_folder, 'sub-01_ses-01_T1w.nii.gz')}" + f" {afq_folder}" + f" --brain_mask_definition='{bm_def_as_str}'" + f" --bundle_info='{bundle_dict_as_str}'" + f" --pve='{pve_as_str}'" + f" --n_points_profile=50" + f' --scalars=\'["dti_fa", "dti_md", "dti_ga", "t1w_over_b0", ' + f'ImageFile("{t1_path_other}"), TemplateImage("{t1_path}")]\'' + f" --odf_model=csd" + f" --n_seeds=2000" + f" --directions=prob" + f" --random_seeds=True" + f" --rng_seed=42" + f" --return_idx=True" + ) - cmd = f"pyAFQ -v {config_file}" completed_process = subprocess.run(cmd, shell=True, capture_output=True) if completed_process.returncode != 0: print(completed_process.stdout) diff --git a/AFQ/utils/bin.py b/AFQ/utils/bin.py index 733c679ce..70b6999da 100644 --- a/AFQ/utils/bin.py +++ b/AFQ/utils/bin.py @@ -1,3 +1,4 @@ +import ast import datetime import os.path as op import platform @@ -12,31 +13,34 @@ def pyafq_str_to_val(t): - if isinstance(t, str) and len(t) < 1: - return None - elif isinstance(t, list): - ls = [] - for e in t: - ls.append(pyafq_str_to_val(e)) - return ls - elif isinstance(t, str) and t[0] == "[": + if isinstance(t, list): + return [pyafq_str_to_val(e) for e in t] + + if not isinstance(t, str): + return t # already an int, float, bool, etc. + + if isinstance(t, str) and t[0] == "[": return eval(t) - elif isinstance(t, str) and t[0] == "{": - return eval(t) # interpret as dictionary - elif isinstance(t, str) and ( - "Image" in t or "Map" in t or "Dict" in t or "_bd(" in t - ): + + if isinstance(t, str) and t[0] == "{": + return eval(t) + + t = t.strip() + if not t: + return None + + # Strings that construct pyAFQ objects still need real eval. + if any(k in t for k in ("Image", "Map", "Dict", "_bd(")): try: - definition_or_dict = eval(t) - except NameError: - return t - if isinstance(definition_or_dict, Definition): - return definition_or_dict - elif isinstance(definition_or_dict, BundleDict): - return definition_or_dict - else: + val = eval(t) + except (NameError, SyntaxError, TypeError): return t - else: + return val if isinstance(val, (Definition, BundleDict)) else t + + # Default to literal_eval for other strings + try: + return ast.literal_eval(t) + except (ValueError, SyntaxError): return t