diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index ccc89f3..3545647 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,14 +1,18 @@ name: Deploy website -on: [push, pull_request] +on: + push: + branches: [main] + pull_request: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true permissions: - contents: write - + contents: read + pages: write + id-token: write jobs: build: runs-on: ubuntu-latest @@ -31,11 +35,19 @@ jobs: run: | make html - name: Upload site - uses: actions/upload-artifact@v4 - with: - name: site - path: build/html - - name: Publish docs to Github Pages - uses: JamesIves/github-pages-deploy-action@releases/v4 + uses: actions/upload-pages-artifact@v4 with: - folder: build/html + path: _build/html + + + deploy: + if: github.event_name != 'pull_request' + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v5 diff --git a/.gitignore b/.gitignore index 62b3d8f..c3b01b3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ .DS_Store -build/* \ No newline at end of file +_build/* \ No newline at end of file diff --git a/Makefile b/Makefile index d0c3cbf..92dd33a 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SOURCEDIR = source -BUILDDIR = build +BUILDDIR = _build # Put it first so that "make" without argument is like "make help". help: diff --git a/README.md b/README.md index b81b314..e198150 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,56 @@ # tee-ar-ex.github.io -A website for the TRX file format. \ No newline at end of file +A website for the TRX file format, built with Sphinx and the MyST parser. + +## Building the documentation + +### Prerequisites + +- Python 3.9+ + +### Local build + +1. Clone the repository: + + ```bash + git clone https://github.com/tee-ar-ex/tee-ar-ex.github.io.git + cd tee-ar-ex.github.io + ``` + +2. Create and activate a virtual environment (optional but recommended): + + ```bash + python -m venv venv + source venv/bin/activate # Windows: venv\Scripts\activate + ``` + +3. Install dependencies: + + ```bash + pip install -r requirements.txt + ``` + +4. Build the documentation: + + ```bash + make html + ``` + +5. Open `_build/html/index.html` in your browser. + +### Development workflow + +To rebuild automatically while editing: + +```bash +sphinx-autobuild source _build/html +``` + +Note: `sphinx-autobuild` needs to be installed separately +(`pip install sphinx-autobuild`). + +### Deployment + +The site is automatically deployed to GitHub Pages via GitHub Actions on +pushes to `main`. Pull requests trigger a build to validate changes without +deploying. diff --git a/requirements.txt b/requirements.txt index ba872ef..a62d34c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ sphinx myst-parser -pydata_sphinx_theme \ No newline at end of file +pydata_sphinx_theme +sphinxcontrib-bibtex +sphinx-design diff --git a/source/_templates/implementation-links.html b/source/_templates/implementation-links.html new file mode 100644 index 0000000..2b843ac --- /dev/null +++ b/source/_templates/implementation-links.html @@ -0,0 +1,11 @@ +
+

TRX specification and implementations

+ +
diff --git a/source/bib.md b/source/bib.md new file mode 100644 index 0000000..3df293c --- /dev/null +++ b/source/bib.md @@ -0,0 +1,9 @@ +# Bibliography + + +```{eval-rst} + +.. bibliography:: + :all: + +``` \ No newline at end of file diff --git a/source/capsule.md b/source/capsule.md deleted file mode 100644 index ef05e87..0000000 --- a/source/capsule.md +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/source/concepts.md b/source/concepts.md new file mode 100644 index 0000000..7ccc685 --- /dev/null +++ b/source/concepts.md @@ -0,0 +1,98 @@ +# Core Concepts + +This page explains how the TRX format represents tractography data on disk and +in memory. + +## The TRX format + +A TRX file is a ZIP archive (or on-disk directory) whose layout directly +encodes its data model: + +- `header.json` — spatial metadata +- `positions..` — all streamline vertices in a single flat array +- `offsets.` — prefix-sum index from streamlines into positions +- `dpv/.` — per-vertex metadata arrays +- `dps/.` — per-streamline metadata arrays +- `groups/.uint32` — named index sets of streamlines +- `dpg//.` — per-group metadata arrays + +Coordinates are stored in **RAS+ world space** (millimeters), matching the +convention used by MRtrix3 `.tck` and NIfTI qform outputs. + +## Positions array + +All streamline vertices are stored in a single flat matrix of shape +`(NB_VERTICES, 3)`. Keeping all vertices contiguous enables efficient memory +mapping and avoids per-streamline allocations. + +## Offsets and the sentinel + +`offsets` is a prefix-sum index of length `NB_STREAMLINES + 1`. Element *i* is +the offset in `positions` of the first vertex of streamline *i*. The final +element is a **sentinel** equal to `NB_VERTICES`, which makes length +computation trivial without special-casing the last streamline: + +``` +length_i = offsets[i + 1] - offsets[i] +``` + +This design avoids per-streamline allocations and makes slicing the global +positions array fast and uniform. + +## Data per vertex (DPV) + +A DPV array stores one value per vertex in `positions`. It has shape +`(NB_VERTICES, 1)` for scalar fields or `(NB_VERTICES, N)` for vector-valued +fields. Typical uses: + +- FA values along the tract +- Per-point RGB colors +- Confidence or weight measures per vertex + +DPV arrays live under `dpv/` and are memory-mapped in the same way as +`positions`. + +## Data per streamline (DPS) + +A DPS array stores one value per streamline. It has shape +`(NB_STREAMLINES, 1)` or `(NB_STREAMLINES, N)`. Typical uses: + +- Mean FA or average curvature per tract +- Per-streamline cluster labels +- Tractography algorithm weights + +DPS arrays live under `dps/` and are loaded as typed matrix fields. If the +on-disk dtype differs from the requested typed reader dtype, values are +converted during load. + +## Groups + +A group is a named list of streamline indices stored as a `uint32` array under +`groups/`. Groups enable sparse, overlapping labeling: a streamline can belong +to multiple groups, and groups can have different sizes. Typical uses: + +- Bundle labels (`CST_L`, `CC`, `SLF_R`, ...) +- Cluster assignments from QuickBundles or similar algorithms +- Connectivity subsets (streamlines connecting two ROIs) + +## Data per group (DPG) + +DPG attaches metadata to a group. Each group folder `dpg//` can contain +any number of scalar or vector arrays. Typical uses: + +- Mean FA across the bundle +- Per-bundle display color +- Volume or surface-area estimates + +## Header + +`header.json` stores: + +- `VOXEL_TO_RASMM` — 4x4 affine mapping voxel indices to RAS+ world + coordinates (mm) +- `DIMENSIONS` — reference image grid dimensions as three `uint16` values +- `NB_STREAMLINES` — number of streamlines (`uint32`) +- `NB_VERTICES` — total number of vertices across all streamlines (`uint64`) + +The header is primarily for human readability and downstream compatibility. The +authoritative sizes come from the array dimensions themselves. diff --git a/source/conf.py b/source/conf.py index 55361e7..0786e5a 100644 --- a/source/conf.py +++ b/source/conf.py @@ -6,17 +6,35 @@ # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information -project = 'tee-ar-ex' -copyright = '2026, TRX developers' -author = 'TRX developers' +from datetime import datetime as dt + +project = 'TRX' +copyright = f'2021-{dt.now().year}, The TRX developers' +author = 'The TRX developers' # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration extensions = [ "myst_parser", + "sphinxcontrib.bibtex", + "sphinx_design", ] +bibtex_bibfiles = ['references.bib'] +bibtex_default_style = 'plain' +bibtex_reference_style = 'author_year' + + +myst_enable_extensions = [ + "colon_fence", + "deflist", + "fieldlist", + "html_image", + "tasklist", +] +myst_heading_anchors = 3 + templates_path = ['_templates'] exclude_patterns = ['.DS_Store'] source_suffix = ['.rst', '.md'] @@ -32,9 +50,15 @@ html_logo = "_static/logo.png" html_theme_options = { - "secondary_sidebar_items": [], + "secondary_sidebar_items": ["page-toc", "edit-this-page", "sourcelink"], "use_edit_page_button": True, "icon_links": [ + { + "name": "Home", + "url": "index.html", + "icon": "fa-solid fa-home", + "attributes": {"target": "_self"}, + }, { "name": "GitHub", "url": "https://github.com/tee-ar-ex/", @@ -49,4 +73,8 @@ "github_repo": "tee-ar-ex.github.io", "github_version": "main", "doc_path": "source", +} + +html_sidebars = { + "**": ["sidebar-nav-bs.html", "implementation-links.html"], } \ No newline at end of file diff --git a/source/design.md b/source/design.md new file mode 100644 index 0000000..3a957df --- /dev/null +++ b/source/design.md @@ -0,0 +1,140 @@ +# Design Rationale + +This page explains the motivation and design decisions behind the TRX file +format. The TRX emerged from a +[community discussion initiated in 2020](https://github.com/nipy/nibabel/issues/942) +to address long-standing limitations in existing tractography file formats. + +## Motivation + +File formats that store the results of computational tractography were +traditionally developed within specific software packages. While this approach +facilitated many applications, it also generated insularity and limited +standardization. As tractography datasets grew in size and complexity, the need +for a community-driven standard became clear. + +## Limitations of existing formats + +The existing tractography file formats have several limitations: + +- **Lack of community-based development.** Formats were tied to specific + toolboxes rather than developed through community consensus. +- **Lack of efficiency in I/O and storage.** Many formats do not support + efficient memory mapping or compression. +- **Limited partial loading.** Most formats require loading all streamlines + at once, making it difficult to work with subsets. +- **Non-standard spatial transformations.** For example, the TRK format uses a + complex and non-standard spatial mapping based on voxel corners rather than + world coordinates. + +## Design goals + +The community identified several key features for a new tractography format: + +- **Community-based development.** The format should be developed by the + community, not tied to any single toolbox. +- **Cross-platform compatibility.** The format should work across Python, C++, + JavaScript, and other programming languages. +- **Simplicity.** The format should be easy to implement in any language + without relying on external tooling. As few elements as possible should be + required, and conventions should be explicit. +- **I/O and storage efficiency.** The format should support memory mapping and + allow direct access to data without conversion. IEEE floating-point + little-endian storage is preferred since it is the native format on all + commonly used CPUs. +- **Independence.** The format should be self-contained and not require external + references. Storing streamlines in world coordinates (RAS+ millimeters) + eliminates the need for a user-supplied reference image. +- **Extensibility.** The format should allow additional metadata to be stored + and for new fields to become part of the standard as their use becomes + commonplace. +- **Seamless spatial transformation.** Intuitive metadata handling at any scale + — vertices, streamlines, and bundles. + +## Streamline storage trade-offs + +A key design decision is how to store streamlines, which are "jagged arrays", +i.e., each streamline has a different number of vertices. + +**Approaches considered:** + +- **VTK-style** (explicit vertex indexing) — allows vertex sharing, efficient + for triangulated meshes, but leads to much larger files for streamlines where + each vertex is used once. +- **Single array with separators** (TCK, TRK, PDB style) — stores vertex + positions and end-of-streamline markers in one array. Good for writing during + streamline creation, but less efficient for random access. +- **Separate arrays for positions and offsets** (TRX approach) — stores vertex + positions in one array and a prefix-sum index in another. This enables + efficient memory mapping, random access, and block reads. Each data type + (float positions, integer offsets) can be read with a single block read. + +The TRX format adopts the **positions + offsets** approach because it provides: + +- Memory-mapped access to arbitrary streamline subsets +- Efficient random access without reading the entire file +- Clean separation of data types for SIMD and GPU compatibility + +**Vertex storage layout:** + +- **Array of Structures (AoS):** `x0, y0, z0, x1, y1, z1, ...` — intuitive, + matches how vertices are sent to GPUs. +- **Structure of Arrays (SoA):** `x0, x1, ..., y0, y1, ..., z0, z1, ...` — + can benefit from CPU-based SIMD programming. + +TRX uses AoS layout for positions, matching the convention used by MRtrix3 +`.tck` and OpenGL polyline rendering. + +## Metadata hierarchy + +TRX organizes metadata into four levels: + +- **Per-vertex (DPV):** scalar or vector values at each streamline vertex + (e.g., FA along a tract, RGB colors). +- **Per-streamline (DPS):** one value per streamline (e.g., mean FA, cluster + labels, algorithm weights). +- **Groups:** named index sets of streamlines enabling sparse, overlapping + labeling (e.g., bundle assignments, connectivity subsets). +- **Per-group (DPG):** metadata attached to groups (e.g., mean FA across a + bundle, display colors, volume estimates). + +This hierarchy covers the metadata needs of existing formats (TRK's +per-vertex/per-streamline values, TCK's bundle labels) while adding the +flexibility of group-level properties that no predecessor format provided. + +## Spatial coordinates + +TRX stores streamline vertices in **RAS+ world coordinates** (millimeters) +rather than voxel space. This decision was motivated by: + +- **Independence from external data.** The file is self-contained — no + reference image is needed to interpret vertex positions. +- **Consistency with MRtrix3.** The `.tck` format already uses world + coordinates, and TRX maintains compatibility. +- **Reduced errors.** Requiring a user-supplied reference image to convert + between voxel and world coordinates creates opportunities for mistakes. + +The `header.json` includes a `VOXEL_TO_RASMM` affine for tools that need to +convert to voxel space, but this field is optional to use — the data on disk +is already in world coordinates. + +## Compression + +TRX supports optional ZIP compression: + +- **Whole-archive compression** (`ZIP_DEFLATE`) is simple and widely supported. +- **Per-array compression** is possible within the ZIP structure, allowing + random access to uncompressed arrays while compressing others. +- **Byte shuffling** before compression can improve ratios for structured data + like offset arrays, where neighboring values share most-significant bytes. + +The format does not mandate a specific compression strategy, leaving this as +an implementation choice. + +In addition, a generic compression approach can be combined with +tractography-specific compression approaches, such as linearization +{cite:p}`Rheault2017`. + +Taken together, TRX provides multiple options to save storage space, that can +also be combined: down-casting, generic compression algorithms, and +tractography-specific compression. \ No newline at end of file diff --git a/source/history.md b/source/history.md new file mode 100644 index 0000000..31d9b85 --- /dev/null +++ b/source/history.md @@ -0,0 +1,66 @@ +# Timeline of TRX + +## Origins + +File formats that store the results of computational tractography were +traditionally developed within specific software packages. This approach has +facilitated a myriad of applications, but this development approach has also +generated insularity within software packages, and has limited standardization. +Moreover, because tractography file formats were developed to solve immediate +challenges, only a limited breadth of applications within a single software +package was envisioned, sometimes also neglecting computational performance. + +Given the growing interest in tractography methods and applications, and the +increasing size and complexity of datasets, a community-driven standardization +of tractography have become a priority +{cite:p}`Legarreta2026Gigascience, Descoteaux2025Millenium`. + +The TRX format emerged in 2020 +following [this discussion](https://github.com/nipy/nibabel/issues/942) on the nibabel +repository, where the tractography community began conversations about a shared +data format. + +## Community adoption and dissemination + +TRX has been developed and advocated by researchers across many institutions +and geographies. The format has been presented at conferences, workshops, and +through technical developments. + +### Conferences and workshops + +- **OHBM 2021** (online) — initial presentation of the format +- **OHBM 2022** (Glasgow) — in-person poster presentation +- **OHBM 2023** — progress presented at the Open Science Room +- **Tract Anat Retreat, Corsica, 2024** — presented and discussed at length +- **DIPY online workshops, 2025 and 2026** — presented to trainees +- **BRAIN CONNECTS gathering, Austin TX, 2025** — presented to the project + +### Institutional adoption + +- The NIH-funded **BIDS Connectivity project** incorporated TRX into its + proposal for extensions of BIDS to represent brain tractography. +- The **International Society for Tractography Standardization Unit** has been + carrying this work forward. + +### Community note + +For a community service project that is not currently funded by any agency, the +community around TRX has been exceptionally active in disseminating knowledge +about the format. Technical developments to ease the burden of TRX adoption +have also been remarkable, with the format now supported by many platforms and +languages, as described in the [Software](software.md) page. + +### Adoption of TRX in BIDS + +The [Brain Imaging Data Structure (BIDS)](https://bids.neuroimaging.io/) is a +widely-used standard for organizing and describing data collected during +neuroimaging experiments. As BIDS has evolved, it has shifted from a standard +that covers only experimental data to one that also covers the outputs of +processing pipelines. + +The proposal for standardization of tractography outputs (or, in BIDS jargon, +"tractography derivatives") currently being discussed as +[BIDS Enhancement Proposal \#46](https://github.com/bids-standard/bids-specification/pull/2333) +focuses on making TRX the only file format that will be accepted by the +standard. This means that standards-compliant experiments and projects will +eventually need to adopt TRX. diff --git a/source/index.md b/source/index.md index 83f2fe7..fa3b24a 100644 --- a/source/index.md +++ b/source/index.md @@ -1,53 +1,49 @@ -# TRX: A community-oriented file format for tractography - -### Rationale - -File formats that store the results of computational tractography were typically -developed within specific software packages. This approach has facilitated a -myriad of applications, but this development approach has also generated -insularity within software packages, and has limited standardization. Moreover, -because tractography file formats were developed to solve immediate challenges, -only a limited breadth of applications within a single software package was -envisioned, sometimes also neglecting computational performance. Given the -growing interest in tractography methods and applications, and the increasing -size and complexity of datasets, a community-driven standardization of -tractography have become a priority. To address these challenges, our community -initiated a discussion to design a new file format and agreed to participate in -its conception, development, and, if successful, its adoption. - -The goal of TRX is to become the standard amongst tractography file formats. As -with other file formats like NIfTI, we believe that TRX will serve the -community well and the growing computational needs of our field. We encourage -community members to consider early contributions to our proposal so as to -ensure the new standard will cover the needs of the wider audience of software -developers, toolboxes, and scientists. Our long-term plan is to integrate TRX -within the [Brain Imaging Data Structure (BIDS)](https://bids.neuroimaging.io/) -ecosystem. - -## Implementations +--- +bibliography: + - references.bib +--- -[*Python*](https://github.com/tee-ar-ex/trx-python) - -[*C++*](https://github.com/tee-ar-ex/trx-cpp) +# TRX: A community-oriented file format for tractography -[*Rust*](https://github.com/tee-ar-ex/trx-rs) +Welcome to the documentation of the TRX file format (pronounced "tee-ar-ex" or +"tracks"). -[*Javascript*](https://github.com/tee-ar-ex/trx-javascript) +The goal of TRX is to become the standard file format. As with other file +formats like NIfTI, we believe that TRX will serve the community well and the +growing computational needs of our field. We encourage community members to +consider early contributions to our proposal so as to ensure the new standard +will cover the needs of the wider audience of software developers, toolboxes, +and scientists. Our long-term plan is to integrate TRX within the [Brain +Imaging Data Structure (BIDS)](https://bids.neuroimaging.io/) ecosystem +{cite:p}`Gorgolewski2016BIDS`. -[*Pascal*](https://github.com/tee-ar-ex/trx-pascal) +````{dropdown} Concepts & Specification -```{note} +```{toctree} +concepts +specification +``` +```` -The TRX file-format started following [this discussion](https://github.com/nipy/nibabel/issues/942) +````{dropdown} Design & History +```{toctree} +design +history ``` +```` +````{dropdown} Software -```{eval-rst} +```{toctree} +software +``` +```` -.. toctree:: - :maxdepth: 2 - :caption: Contents: +````{dropdown} Bibliography -``` \ No newline at end of file +```{toctree} +bib +``` +```` \ No newline at end of file diff --git a/source/references.bib b/source/references.bib new file mode 100644 index 0000000..d83b158 --- /dev/null +++ b/source/references.bib @@ -0,0 +1,509 @@ +@ARTICLE{Rheault2017, + title = "Visualization, interaction and tractometry: Dealing with millions + of streamlines from diffusion {MRI} tractography", + author = "Rheault, Francois and Houde, Jean-Christophe and Descoteaux, + Maxime", + journal = "Front. Neuroinform.", + publisher = "Frontiers Media SA", + volume = 11, + pages = 42, + abstract = "Recently proposed tractography and connectomics approaches often + require a very large number of streamlines, in the order of + millions. Generating, storing and interacting with these datasets + is currently quite difficult, since they require a lot of space + in memory and processing time. Compression is a common approach + to reduce data size. Recently such an approach has been proposed + consisting in removing collinear points in the streamlines. + Removing points from streamlines results in files that cannot be + robustly post-processed and interacted with existing tools, which + are for the most part point-based. The aim of this work is to + improve visualization, interaction and tractometry algorithms to + robustly handle compressed tractography datasets. Our proposed + improvements are threefold: (i) An efficient loading procedure to + improve visualization (reduce memory usage up to 95\% for a 0.2 + mm step size); (ii) interaction techniques robust to compressed + tractograms; (iii) tractometry techniques robust to compressed + tractograms to eliminate biased in tract-based statistics. The + present work demonstrates the need of correctly handling + compressed streamlines to avoid biases in future tractometry and + connectomics studies.", + month = jun, + year = 2017, + keywords = "MI-Brain; compression; connectomics; diffusion MRI; + linearization; streamlines; tractography; tractometry" +} + + +@ARTICLE{Gorgolewski2016BIDS, + title = "The brain imaging data structure, a format for organizing and + describing outputs of neuroimaging experiments", + author = "Gorgolewski, Krzysztof J and Auer, Tibor and Calhoun, Vince D and + Cameron Craddock, R and Das, Samir and Duff, Eugene P and + Flandin, Guillaume and Ghosh, Satrajit S and Glatard, Tristan and + Halchenko, Yaroslav O and Handwerker, Daniel A and Hanke, Michael + and Keator, David and Li, Xiangrui and Michael, Zachary and + Maumet, Camille and Nolan Nichols, B and Nichols, Thomas E and + Pellman, John and Poline, Jean-Baptiste and Rokem, Ariel and + Schaefer, Gunnar and Sochat, Vanessa and Triplett, William and + Turner, Jessica A and Varoquaux, Gaël and Poldrack, Russell A", + journal = "Scientific data", + publisher = "Nature Publishing Group", + volume = 3, + number = 1, + pages = "1--9", + abstract = "The development of magnetic resonance imaging (MRI) techniques + has defined modern neuroimaging. Since its inception, tens of + thousands of studies using techniques such as functional MRI and + diffusion weighted imaging have allowed for the non-invasive + study of the brain. Despite the fact that MRI is routinely used + to obtain data for neuroscience research, there has been no + widely adopted standard for organizing and describing the data + collected in an imaging experiment. This renders sharing and + reusing data (within or between labs) difficult if not impossible + and unnecessarily complicates the application of automatic + pipelines and quality assurance protocols. To solve this problem, + we have developed the Brain Imaging Data Structure (BIDS), a + standard for organizing and describing MRI datasets. The BIDS + standard uses file formats compatible with existing software, + unifies the majority of practices already …", + month = jun, + year = 2016 +} + + +@ARTICLE{Descoteaux2025Millenium, + title = "Millennium Pathways for Tractography: 40 grand challenges to + shape the future of tractography", + author = "Descoteaux, Maxime and Schilling, Kurt G and Aydogan, Dogu + Baran and Beaulieu, Christian and Borra, Elena and + Chamberland, Maxime and Daducci, Alessandro and Luca, Alberto + De and Dell'Acqua, Flavio and Dubois, Jessica and Dyrby, Tim + B and Farquharson, Shawna and Forkel, Stephanie and Froeling, + Martijn and Griffa, Alessandra and Grotheer, Mareike and + Guevara, Pamela and Haber, Suzanne N and Jangir, Vinod Kumar + and Leemans, Alexander and Lefebvre, Joël and Lin, Ching-Po + and Little, Graham and Zac Lo, Chun-Yi and Maffei, Chiara and + Mayberg, Helen S and McNab, Jennifer A and Mukherjee, Pratik + and O'Donnell, Lauren J and Parent, Martin and Pierpaoli, + Carlo and Rheault, Francois and Rockland, Kathleen S and + Roebroeck, Alard and Rokem, Ariel and Rushmore, R Jarrett and + Sarubbo, Silvio and Schiavi, Simona and Sotiropoulos, + Stamatios N and Szczupak, Diego and de Schotten, Michel + Thiebaut and Tournier, J-Donald and Vergani, Francesco and + Yuan-Mou Yang, Joseph and Zhang, Fan and Jones, Derek and + Petit, Laurent", + journal = "ArXiv", + abstract = "In the spirit of the historic Millennium Prize Problems that + heralded a new era for mathematics, the newly formed + International Society for Tractography (IST) has launched the + Millennium Pathways for Tractography, a community-driven + roadmap designed to shape the future of the field. Conceived + during the inaugural Tract-Anat Retreat, this initiative + reflects a collective vision for advancing tractography over + the coming decade and beyond. The roadmap consists of 40 + grand challenges, developed by international experts and + organized into seven categories spanning three overarching + themes: neuroanatomy, tractography methods, and clinical + applications. By defining shared short-, medium-, and + long-term goals, these pathways provide a structured + framework to confront fundamental limitations, promote + rigorous validation, and accelerate the translation of + tractography into a robust tool for neuroscience and + medicine. Ultimately, the Millennium Pathways aim to guide + and inspire future research and collaboration, ensuring the + continued scientific and clinical relevance of tractography + well into the future.", + month = sep, + year = 2025, + archivePrefix = "arXiv", + primaryClass = "physics.med-ph", + language = "en" +} + + +@ARTICLE{Norton2017-yz, + title = "{SlicerDMRI}: Open source diffusion {MRI} software for brain + cancer research", + author = "Norton, Isaiah and Essayed, Walid Ibn and Zhang, Fan and Pujol, + Sonia and Yarmarkovich, Alex and Golby, Alexandra J and + Kindlmann, Gordon and Wassermann, Demian and Estepar, Raul San + Jose and Rathi, Yogesh and Pieper, Steve and Kikinis, Ron and + Johnson, Hans J and Westin, Carl-Fredrik and O'Donnell, Lauren J", + journal = "Cancer Res.", + publisher = "American Association for Cancer Research (AACR)", + volume = 77, + number = 21, + pages = "e101--e103", + abstract = "Diffusion MRI (dMRI) is the only noninvasive method for mapping + white matter connections in the brain. We describe SlicerDMRI, a + software suite that enables visualization and analysis of dMRI + for neuroscientific studies and patient-specific anatomic + assessment. SlicerDMRI has been successfully applied in multiple + studies of the human brain in health and disease, and here, we + especially focus on its cancer research applications. As an + extension module of the 3D Slicer medical image computing + platform, the SlicerDMRI suite enables dMRI analysis in a + clinically relevant multimodal imaging workflow. Core SlicerDMRI + functionality includes diffusion tensor estimation, white matter + tractography with single and multi-fiber models, and dMRI + quantification. SlicerDMRI supports clinical DICOM and research + file formats, is open-source and cross-platform, and can be + installed as an extension to 3D Slicer (www.slicer.org). Cancer + Res; 77(21); e101-3. ©2017 AACR.", + month = nov, + year = 2017, + language = "en", + DOI = "https://doi.org/10.1158/0008-5472.CAN-17-0332", +} + + +@ARTICLE{Rorden2025-zq, + title = "Surfice: visualizing neuroimaging meshes, tractography + streamlines and connectomes", + author = "Rorden, Christopher", + journal = "Nat. Methods", + publisher = "Springer Science and Business Media LLC", + volume = 22, + number = 8, + pages = "1615--1616", + month = aug, + year = 2025, + language = "en", + DOI = "https://doi.org/10.1038/s41592-025-02764-6" +} + + +@ARTICLE{Aydogan2021-ug, + title = "Parallel transport tractography", + author = "Aydogan, Dogu Baran and Shi, Yonggang", + journal = "IEEE Trans. Med. Imaging", + publisher = "Institute of Electrical and Electronics Engineers (IEEE)", + volume = 40, + number = 2, + pages = "635--647", + abstract = "Tractography is an important technique that allows the in vivo + reconstruction of structural connections in the brain using + diffusion MRI. Although tracking algorithms have improved during + the last two decades, results of validation studies and + international challenges warn about the reliability of + tractography and point out the need for improved algorithms. In + propagation-based tracking, connections have traditionally been + modeled as piece-wise linear segments. In this work, we propose a + novel propagation-based tracker that is capable of generating + geometrically smooth ( C1 ) curves using parallel transport + frames. Notably, our approach does not increase the complexity of + the propagation problem that remains two-dimensional. Moreover, + our tracker has a novel mechanism to reduce noise related + propagation errors by incorporating topographic regularity of + connections, a neuroanatomic property of many brain pathways. We + ran extensive experiments and compared our approach against + deterministic and other probabilistic algorithms. Our experiments + on FiberCup and ISMRM 2015 challenge datasets as well as on 56 + subjects of the Human Connectome Project show highly promising + results both visually and quantitatively. Open-source + implementations of the algorithm are shared publicly.", + month = feb, + year = 2021, + language = "en", + DOI = "https://doi.org/10.1109/tmi.2020.3034038" +} + + +@ARTICLE{Hayashi2024-zm, + title = "brainlife.io: a decentralized and open-source cloud platform to + support neuroscience research", + author = "Hayashi, Soichi and Caron, Bradley A and Heinsfeld, Anibal Sólon + and Vinci-Booher, Sophia and McPherson, Brent and Bullock, Daniel + N and Bertò, Giulia and Niso, Guiomar and Hanekamp, Sandra and + Levitas, Daniel and Ray, Kimberly and MacKenzie, Anne and Avesani, + Paolo and Kitchell, Lindsey and Leong, Josiah K and + Nascimento-Silva, Filipi and Koudoro, Serge and Willis, Hanna and + Jolly, Jasleen K and Pisner, Derek and Zuidema, Taylor R and + Kurzawski, Jan W and Mikellidou, Kyriaki and Bussalb, Aurore and + Chaumon, Maximilien and George, Nathalie and Rorden, Christopher + and Victory, Conner and Bhatia, Dheeraj and Aydogan, Dogu Baran + and Yeh, Fang-Cheng F and Delogu, Franco and Guaje, Javier and + Veraart, Jelle and Fischer, Jeremy and Faskowitz, Joshua and + Fabrega, Ricardo and Hunt, David and McKee, Shawn and Brown, Shawn + T and Heyman, Stephanie and Iacovella, Vittorio and Mejia, Amanda + F and Marinazzo, Daniele and Craddock, R Cameron and Olivetti, + Emanuale and Hanson, Jamie L and Garyfallidis, Eleftherios and + Stanzione, Dan and Carson, James and Henschel, Robert and Hancock, + David Y and Stewart, Craig A and Schnyer, David and Eke, Damian O + and Poldrack, Russell A and Bollmann, Steffen and Stewart, Ashley + and Bridge, Holly and Sani, Ilaria and Freiwald, Winrich A and + Puce, Aina and Port, Nicholas L and Pestilli, Franco", + journal = "Nat. Methods", + volume = 21, + number = 5, + pages = "809--813", + abstract = "Neuroscience is advancing standardization and tool development to + support rigor and transparency. Consequently, data pipeline + complexity has increased, hindering FAIR (findable, accessible, + interoperable and reusable) access. brainlife.io was developed to + democratize neuroimaging research. The platform provides data + standardization, management, visualization and processing and + automatically tracks the provenance history of thousands of data + objects. Here, brainlife.io is described and evaluated for + validity, reliability, reproducibility, replicability and + scientific utility using four data modalities and 3,200 + participants.", + month = may, + year = 2024, + language = "en", + DOI = "https://doi.org/10.1038/s41592-024-02237-2" +} + + +@ARTICLE{Cieslak2021-ic, + title = "{QSIPrep}: an integrative platform for preprocessing and + reconstructing diffusion {MRI} data", + author = "Cieslak, Matthew and Cook, Philip A and He, Xiaosong and Yeh, + Fang-Cheng and Dhollander, Thijs and Adebimpe, Azeez and Aguirre, + Geoffrey K and Bassett, Danielle S and Betzel, Richard F and + Bourque, Josiane and Cabral, Laura M and Davatzikos, Christos and + Detre, John A and Earl, Eric and Elliott, Mark A and Fadnavis, + Shreyas and Fair, Damien A and Foran, Will and Fotiadis, + Panagiotis and Garyfallidis, Eleftherios and Giesbrecht, Barry + and Gur, Ruben C and Gur, Raquel E and Kelz, Max B and Keshavan, + Anisha and Larsen, Bart S and Luna, Beatriz and Mackey, Allyson P + and Milham, Michael P and Oathes, Desmond J and Perrone, Anders + and Pines, Adam R and Roalf, David R and Richie-Halford, Adam and + Rokem, Ariel and Sydnor, Valerie J and Tapera, Tinashe M and + Tooley, Ursula A and Vettel, Jean M and Yeatman, Jason D and + Grafton, Scott T and Satterthwaite, Theodore D", + journal = "Nat. Methods", + publisher = "Nature Publishing Group", + pages = "1--4", + abstract = "Diffusion-weighted magnetic resonance imaging (dMRI) is the + primary method for noninvasively studying the organization of + white matter in the human brain. Here we introduce QSIPrep, an + integrative software platform for the processing of diffusion + images that is compatible with nearly all dMRI sampling schemes. + Drawing on a diverse set of software suites to capitalize on + their complementary strengths, QSIPrep facilitates the + implementation of best practices for processing of diffusion + images. QSIPrep is a software platform for processing of most + diffusion MRI datasets and ensures that adequate workflows are + used.", + month = jun, + year = 2021, + language = "en", + DOI = "https://doi.org/10.1038/s41592-021-01185-5" +} + + +@ARTICLE{Tournier2019-cq, + title = "{MRtrix3}: A fast, flexible and open software framework for + medical image processing and visualisation", + author = "Tournier, J-Donald and Smith, Robert and Raffelt, David and + Tabbara, Rami and Dhollander, Thijs and Pietsch, Maximilian and + Christiaens, Daan and Jeurissen, Ben and Yeh, Chun-Hung and + Connelly, Alan", + journal = "Neuroimage", + volume = 202, + pages = 116137, + abstract = "MRtrix3 is an open-source, cross-platform software package for + medical image processing, analysis and visualisation, with a + particular emphasis on the investigation of the brain using + diffusion MRI. It is implemented using a fast, modular and + flexible general-purpose code framework for image data access and + manipulation, enabling efficient development of new applications, + whilst retaining high computational performance and a consistent + command-line interface between applications. In this article, we + provide a high-level overview of the features of the MRtrix3 + framework and general-purpose image processing applications + provided with the software.", + month = nov, + year = 2019, + keywords = "Image; MRI; Processing; Software; Visualisation", + language = "en", + DOI = "https://doi.org/10.1016/j.neuroimage.2019.116137" +} + + +@ARTICLE{Avants2014-ci, + title = "The Insight {ToolKit} image registration framework", + author = "Avants, Brian B and Tustison, Nicholas J and Stauffer, Michael and + Song, Gang and Wu, Baohua and Gee, James C", + journal = "Front. Neuroinform.", + volume = 8, + pages = 44, + abstract = "Publicly available scientific resources help establish evaluation + standards, provide a platform for teaching and improve + reproducibility. Version 4 of the Insight ToolKit (ITK(4)) seeks + to establish new standards in publicly available image + registration methodology. ITK(4) makes several advances in + comparison to previous versions of ITK. ITK(4) supports both + multivariate images and objective functions; it also unifies + high-dimensional (deformation field) and low-dimensional (affine) + transformations with metrics that are reusable across transform + types and with composite transforms that allow arbitrary series of + geometric mappings to be chained together seamlessly. Metrics and + optimizers take advantage of multi-core resources, when available. + Furthermore, ITK(4) reduces the parameter optimization burden via + principled heuristics that automatically set scaling across + disparate parameter types (rotations vs. translations). A related + approach also constrains steps sizes for gradient-based + optimizers. The result is that tuning for different metrics and/or + image pairs is rarely necessary allowing the researcher to more + easily focus on design/comparison of registration strategies. In + total, the ITK(4) contribution is intended as a structure to + support reproducible research practices, will provide a more + extensive foundation against which to evaluate new work in image + registration and also enable application level programmers a broad + suite of tools on which to build. Finally, we contextualize this + work with a reference registration evaluation study with + application to pediatric brain labeling.", + month = apr, + year = 2014, + keywords = "MRI; brain; death; open-source; registration", + language = "en", + DOI = "https://doi.org/10.3389/fninf.2014.00044" +} + + +@ARTICLE{Garyfallidis2014-el, + title = "Dipy, a library for the analysis of diffusion {MRI} data", + author = "Garyfallidis, Eleftherios and Brett, Matthew and Amirbekian, + Bagrat and Rokem, Ariel and van der Walt, Stefan and Descoteaux, + Maxime and Nimmo-Smith, Ian and {Dipy Contributors}", + journal = "Front. Neuroinform.", + volume = 8, + pages = 8, + abstract = "Diffusion Imaging in Python (Dipy) is a free and open source + software project for the analysis of data from diffusion magnetic + resonance imaging (dMRI) experiments. dMRI is an application of + MRI that can be used to measure structural features of brain white + matter. Many methods have been developed to use dMRI data to model + the local configuration of white matter nerve fiber bundles and + infer the trajectory of bundles connecting different parts of the + brain. Dipy gathers implementations of many different methods in + dMRI, including: diffusion signal pre-processing; reconstruction + of diffusion distributions in individual voxels; fiber + tractography and fiber track post-processing, analysis and + visualization. Dipy aims to provide transparent implementations + for all the different steps of dMRI analysis with a uniform + programming interface. We have implemented classical signal + reconstruction techniques, such as the diffusion tensor model and + deterministic fiber tractography. In addition, cutting edge novel + reconstruction techniques are implemented, such as constrained + spherical deconvolution and diffusion spectrum imaging (DSI) with + deconvolution, as well as methods for probabilistic tracking and + original methods for tractography clustering. Many additional + utility functions are provided to calculate various statistics, + informative visualizations, as well as file-handling routines to + assist in the development and use of novel techniques. In contrast + to many other scientific software projects, Dipy is not being + developed by a single research group. Rather, it is an open + project that encourages contributions from any scientist/developer + through GitHub and open discussions on the project mailing list. + Consequently, Dipy today has an international team of + contributors, spanning seven different academic institutions in + five countries and three continents, which is still growing.", + month = feb, + year = 2014, + keywords = "DSI; DTI; HARDI; Python; dMRI; diffusion MRI; free open source + software; tractography", + DOI = "https://doi.org/10.3389/fninf.2014.00008" +} + + +@ARTICLE{Legarreta2026Gigascience, + title = "What needs to be standardized for reliable, reproducible, and + robust tractography?", + author = "Legarreta, Jon Haitz and Schiavi, Simona and Tang, Wei and Banks, + Garrett and Cieslak, Matthew and Schilling, Kurt and De Luca, + Alberto and Tournier, Jacques-Donald and Kruper, John and + Rheault, Francois and Sotiropoulos, Stamatios N and Pestilli, + Franco and Veraart, Jelle and Yang, Joseph Yuan-Mou and + Descoteaux, Maxime and Heilbronner, Sarah and Rokem, Ariel", + journal = "Gigascience", + publisher = "Oxford University Press (OUP)", + volume = 15, + number = "giag034", + pages = "giag034", + abstract = "Tractography is a key component of efforts to map brain + connectivity. As a rapidly evolving field of neuroscience, + current tractography methods are diverse, often varying across + research laboratories and different software pipelines. + Therefore, it suffers from a lack of standardization, leading to + inconsistencies in results, which can limit reproducibility and + affect the robustness needed for research and clinical + applications of these methods. Variability in data acquisition + procedures, inconsistencies in spatial referencing schemes and + implementations, and anatomical heterogeneity-at the individual + level, across the lifespan, and across species-hinder comparative + analyses. Additionally, the lack of consensus on best practices + complicates the development of robust automated quality control + pipelines and limits the clinical translation of + tractography-based procedures. Establishing standardized + protocols for acquisition, preprocessing, and tractography + reconstruction is critical toward enabling reliable + tract-specific analyses, facilitating cross-study harmonization, + and supporting replicable large-scale population studies. The + present article provides an overview of the current challenges in + tractography standardization and identifies the key aspects that + require standardization for reliable, reproducible, and robust + tractography.", + month = jan, + year = 2026, + keywords = "brain connectivity; computational neuroimaging; neuroanatomy; + standardization; tractography; white matter", + language = "en" +} + + +@ARTICLE{Kruper2024HCP, + title = "Tractometry of the Human Connectome Project: resources and + insights", + author = "Kruper, John and Hagen, Mckenzie P and Rheault, François and + Crane, Isaac and Gilmore, Asa and Narayan, Manjari and Motwani, + Keshav and Lila, Eardi and Rorden, Chris and Yeatman, Jason D and + Rokem, Ariel", + journal = "Front. Neurosci.", + publisher = "Frontiers Media SA", + volume = 18, + pages = 1389680, + abstract = "Introduction: The Human Connectome Project (HCP) has become a + keystone dataset in human neuroscience, with a plethora of + important applications in advancing brain imaging methods and an + understanding of the human brain. We focused on tractometry of + HCP diffusion-weighted MRI (dMRI) data. Methods: We used an + open-source software library (pyAFQ; + https://yeatmanlab.github.io/pyAFQ) to perform probabilistic + tractography and delineate the major white matter pathways in the + HCP subjects that have a complete dMRI acquisition (n = 1,041). + We used diffusion kurtosis imaging (DKI) to model white matter + microstructure in each voxel of the white matter, and extracted + tract profiles of DKI-derived tissue properties along the length + of the tracts. We explored the empirical properties of the data: + first, we assessed the heritability of DKI tissue properties + using the known genetic linkage of the large number of twin pairs + sampled in HCP. Second, we tested the ability of tractometry to + serve as the basis for predictive models of individual + characteristics (e.g., age, crystallized/fluid intelligence, + reading ability, etc.), compared to local connectome features. To + facilitate the exploration of the dataset we created a new + web-based visualization tool and use this tool to visualize the + data in the HCP tractometry dataset. Finally, we used the HCP + dataset as a test-bed for a new technological innovation: the TRX + file-format for representation of dMRI-based streamlines. + Results: We released the processing outputs and tract profiles as + a publicly available data resource through the AWS Open Data + program's Open Neurodata repository. We found heritability as + high as 0.9 for DKI-based metrics in some brain pathways. We also + found that tractometry extracts as much useful information about + individual differences as the local connectome method. We + released a new web-based visualization tool for + tractometry-``Tractoscope'' (https://nrdg.github.io/tractoscope). + We found that the TRX files require considerably less disk + space-a crucial attribute for large datasets like HCP. In + addition, TRX incorporates a specification for grouping + streamlines, further simplifying tractometry analysis.", + month = jun, + year = 2024, + keywords = "MRI; Open Data; brain; data visualization; diffusion MRI; + heritability; predictive modeling; tractometry", + language = "en" +} diff --git a/source/software.md b/source/software.md new file mode 100644 index 0000000..c31a8e7 --- /dev/null +++ b/source/software.md @@ -0,0 +1,95 @@ +# Software + +TRX is integrated into several software libraries: + +## TRXViz + +[TRXViz](https://github.com/tee-ar-ex/TRXViz) is a tool for visualization and +processing of diffusion-MRI based tractography. Implemented in Rust, it +supports fast visualization and basic processing of TRX-based tractography. + +## pyAFQ + +[pyAFQ](https://tractometry.org/pyAFQ) is a Python library for automated +white matter tractography and tractometry. It uses TRX to parallelize and +accelerate tractography, as well as to perform batched operations over +groups of streamlines in ways that allow handling very large tractograms +(>100M streamlines) on standard hardware. See [this PR](https://github.com/tractometry/pyAFQ/pull/208). + +## Trekker + +Trekker implements parallel transport tractography for generating geometrically +smooth streamlines {cite:p}`Aydogan2021-ug`. It uses the TRX C++ implementation +for I/O. + +## DIPY + +[DIPY](https://dipy.org/) is a library for the analysis of diffusion MRI +data {cite:p}`Garyfallidis2014-el`. DIPY supports I/O in many different +file formats, including TRX. See [this PR](https://github.com/dipy/dipy/pull/3587) + +## QSIPrep & QSIRecon + +QSIPrep is an integrative platform for preprocessing and reconstructing +diffusion MRI data {cite:p}`Cieslak2021-ic`. QSIPrep and its post-processing +pipelines, implemented in QSIRecon advance the use of TRX as an interchange +format, because they knit together functionality from many different software +libraries. + +## Surfice + +Surfice is a tool for visualizing neuroimaging meshes, tractography +streamlines, and connectomes {cite:p}`Rorden2025-zq`. + +## SlicerDMRI + +SlicerDMRI is an open source diffusion MRI software, implemented as an +extension of 3D Slicer {cite:p}`Norton2017-yz`. It now supports use of TRX, +through its C++ implementation. See [this PR](https://github.com/SlicerDMRI/SlicerDMRI/pull/250). + +## FSL + +[FSL](https://fsl.fmrib.ox.ac.uk/fsl/) is a comprehensive library of tools +for functional MRI, structural MRI, and diffusion MRI analysis. The FSLeyes +viewer includes support for TRX as of its 1.20 version release. See [this PR](https://git.fmrib.ox.ac.uk/fsl/fsleyes/fsleyes/-/merge_requests/504). + +## COMMIT + +COMMIT (Convex Optimization Modeling for Microstructure Informed Tractography) +is a framework for microstructure-informed tractography. TRX support in +COMMIT using its C++ implementation is currently work-in-progress (see [this PR](https://github.com/daducci/COMMIT/pull/163)) + +## brainlife.io + +[brainlife.io](https://brainlife.io) is a decentralized and open-source cloud +platform to support neuroscience research {cite:p}`Hayashi2024-zm`. It uses +TRX for streamline I/O. + +## Niivue + +[Niivue](https://niivue.com/) is a WebGL-based visualization library for +neuroimaging data, including tractography streamlines. It uses features of TRX +to display different groups (e.g., tracts) and their distinct properties. See +[this PR](https://github.com/niivue/niivue/pull/841) + +## DWI2TRX + +The [DWI2TRX](https://tee-ar-ex.github.io/dwi2trx/) web application uses +WASM-based dcm2niix and niimath as well as WebGPU-based NiiVue, mindgrab and +GPUstreamlines to perform a minimal processing pipeline that generates a TRX +tractography from NIfTI/bval/bvec files or from a directory of DICOMs. All +computations are conducted client-side on the user's computer, and no data is +uploaded to any remote server. It works on any web-enabled device that supports +the full WebGPU standard. + +## Insight ToolKit (ITK) + +ITK is an open-source framework for image analysis. TRX has been integrated as +an external module, making it available for users of the ANTS registration +framework {cite:p}`Avants2014-ci`. See [this PR](https://github.com/InsightSoftwareConsortium/ITK/pull/6687). + +## MRtrix + +[MRtrix3](https://www.mrtrix.org/) is a fast, flexible and open software +framework for medical image processing and visualisation {cite:p}`Tournier2019-cq`. +Integration of TRX support is currently in progress (in [this PR](https://github.com/MRtrix3/mrtrix3/pull/3415)). diff --git a/source/specification.md b/source/specification.md new file mode 100644 index 0000000..4bd6d9d --- /dev/null +++ b/source/specification.md @@ -0,0 +1,175 @@ +# TRX Specification + +This page defines the TRX file format specification. The authoritative +specification is maintained in the +[trx-spec](https://github.com/tee-ar-ex/trx-spec) repository. + +## General + +- Uncompressed ZIP file or simple folder architecture +- File architecture describes the data +- Each file basename is the metadata's name +- Each file extension is the metadata's dtype +- Each file dimension is in the value between basename and metadata +- 1-dimensional arrays do not have to follow this convention for readability +- All arrays have a C-style memory layout (row-major) +- All arrays have a little-endian byte order +- Compression is optional + - Use `ZIP_STORE` if no compression is desired + - Use `ZIP_DEFLATE` if compression is desired + - Compressed TRX files will have to be decompressed before being loaded + +## Header + +The header is primarily for human readability, read-time checks, and broader +compatibility. + +- Dictionary in JSON + - `VOXEL_TO_RASMM` — 4 lists of 4 floats (4x4 transformation matrix) + - `DIMENSIONS` — list of 3 uint16 + - `NB_STREAMLINES` — uint32 + - `NB_VERTICES` — uint64 + +## Arrays + +### positions + +- Written in world space (RASMM), like TCK files +- Always float16, float32, or float64 (default: float16) +- Stored as contiguous 3D array `(NB_VERTICES, 3)` + +### offsets + +- Always uint32 or uint64 +- Contains the index where each streamline starts in the positions array, + beginning at 0 +- Two ways of knowing how many vertices there are: + - Check the header + - Positions array size / dtype / 3 +- To get streamline lengths: append the total number of vertices to the end of + offsets and take the differences between consecutive elements + (`numpy.ediff1d`) + +### dpv (data per vertex) + +- Always of size `(NB_VERTICES, 1)` or `(NB_VERTICES, N)` + +### dps (data per streamline) + +- Always of size `(NB_STREAMLINES, 1)` or `(NB_STREAMLINES, N)` + +### groups + +Groups are tables of indices that allow sparse and overlapping representation +(clusters, connectomics, bundles). + +- All indices must satisfy `0 <= id < NB_STREAMLINES` +- Datatype should be uint32 +- Allow efficient extraction of a predefined streamline subset from memmaps +- Variable sizes across groups + +### dpg (data per group) + +- Each subfolder is named after its corresponding group +- Not all metadata need be present in all groups +- Always of size `(1,)` or `(N,)` + +## Accepted datatypes + +- int8, int16, int32, int64 +- uint8, uint16, uint32, uint64 +- float16, float32, float64 +- bit (for binary/boolean arrays) + +## Example structure + +``` +OHBM_demo.trx +├── dpg +│ ├── AF_L +│ │ ├── mean_fa.float16 +│ │ ├── shuffle_colors.3.uint8 +│ │ └── volume.uint32 +│ ├── AF_R +│ │ ├── mean_fa.float16 +│ │ ├── shuffle_colors.3.uint8 +│ │ └── volume.uint32 +│ ├── CC +│ │ ├── mean_fa.float16 +│ │ ├── shuffle_colors.3.uint8 +│ │ └── volume.uint32 +│ ├── CST_L +│ │ └── shuffle_colors.3.uint8 +│ ├── CST_R +│ │ └── shuffle_colors.3.uint8 +│ ├── SLF_L +│ │ ├── mean_fa.float16 +│ │ ├── shuffle_colors.3.uint8 +│ │ └── volume.uint32 +│ └── SLF_R +│ ├── mean_fa.float16 +│ ├── shuffle_colors.3.uint8 +│ └── volume.uint32 +├── dpv +│ ├── color_x.uint8 +│ ├── color_y.uint8 +│ ├── color_z.uint8 +│ └── fa.float16 +├── dps +│ ├── algo.uint8 +│ ├── algo.json +│ ├── clusters_QB.uint16 +│ ├── commit_colors.3.uint8 +│ └── commit_weights.float32 +├── groups +│ ├── AF_L.uint32 +│ ├── AF_R.uint32 +│ ├── CC.uint32 +│ ├── CST_L.uint32 +│ ├── CST_R.uint32 +│ ├── SLF_L.uint32 +│ └── SLF_R.uint32 +├── header.json +├── offsets.uint64 +└── positions.3.float16 +``` + +## Example code + +```python +from trx_file_memmap import TrxFile, load, save +import numpy as np + +trx = load('complete_big_v5.trx') + +# Access the header (dict) / streamlines (ArraySequences) +trx.header +trx.streamlines + +# Access the dpv (dict) / dps (dict) +trx.data_per_vertex +trx.data_per_streamline + +# Access the groups (dict) / dpg (dict) +trx.groups +trx.data_per_group + +# Get a random subset of 10000 streamlines +indices = np.arange(len(trx.streamlines._lengths)) +np.random.shuffle(indices) +sub_trx = trx.select(indices[0:10000]) +save(sub_trx, 'random_1000.trx') + +# Get sub-groups only, from the random subset +for key in sub_trx.groups.keys(): + group_trx = sub_trx.get_group(key) + save(group_trx, '{}.trx'.format(key)) + +# Pre-allocate memmaps and append 100x the random subset +alloc_trx = TrxFile(nb_streamlines=1500000, nb_vertices=500000000, init_as=trx) +for i in range(100): + alloc_trx.append(sub_trx) + +# Resize to remove the unused portion of the memmap +alloc_trx.resize() +```