Add Omni-Fuse Tutorial

tutorials/multimodal/omni-fuse-data-curation/.env.example

@@ -0,0 +1,14 @@
+# Copy this file to .env and fill in real values. Do not commit .env.
+
+# Required for the API-first hybrid tutorial path.
+NV_BUILD_API_KEY=
+
+# Defaults to NVIDIA Build's OpenAI-compatible API base URL.
+NVIDIA_API_BASE_URL=https://integrate.api.nvidia.com/v1
+
+# Optional if you use Hugging Face gated/local model paths outside the API backend.
+HF_TOKEN=
+
+# Required for the LanguageBind fusion expert unless LanguageBind is cloned into
+# ./third_party/LanguageBind.
+LANGUAGEBIND_ROOT=

tutorials/multimodal/omni-fuse-data-curation/.python-version

@@ -0,0 +1 @@
+3.12.11

tutorials/multimodal/omni-fuse-data-curation/.rayignore

@@ -0,0 +1,12 @@
+.git/
+.venv/
+__pycache__/
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+datasets/
+model_files/
+outputs/
+tutorials/multimodal/omni-fuse-data-curation/outputs/
+tutorials/multimodal/omni-fuse-data-curation/tmp/
+*.egg-info/

tutorials/multimodal/omni-fuse-data-curation/0_validate_inputs.py

@@ -0,0 +1,31 @@
+# Copyright (c) 2026, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Validate config, data manifests, API keys, and local model assets."""
+
+from __future__ import annotations
+
+from utils import config_parser, load_tutorial_config, print_outputs, validate_inputs
+
+
+def main() -> int:
+    parser = config_parser(__doc__ or "")
+    args = parser.parse_args()
+    config = load_tutorial_config(args.config)
+    print_outputs(validate_inputs(config))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tutorials/multimodal/omni-fuse-data-curation/1_sns.py

@@ -0,0 +1,39 @@
+# Copyright (c) 2026, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Run Symmetric Nucleus Subsampling over paired multimodal records."""
+
+from __future__ import annotations
+
+from utils import config_parser, load_tutorial_config, print_outputs, run_sns
+
+
+def main() -> int:
+    parser = config_parser(__doc__ or "")
+    args = parser.parse_args()
+    config = load_tutorial_config(args.config)
+    task = run_sns(config)
+    metadata = dict(getattr(task, "_metadata", {}) or {})
+    print_outputs(
+        {
+            "run_dir": str(config.run_dir),
+            "sns_manifest_path": metadata.get("sns_manifest_path"),
+            "sns_records_path": metadata.get("sns_records_path"),
+        }
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tutorials/multimodal/omni-fuse-data-curation/2_embed.py

@@ -0,0 +1,39 @@
+# Copyright (c) 2026, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Run the Expert Embedding Engine for SNS-refined records."""
+
+from __future__ import annotations
+
+from utils import config_parser, load_tutorial_config, print_outputs, run_eee
+
+
+def main() -> int:
+    parser = config_parser(__doc__ or "")
+    args = parser.parse_args()
+    config = load_tutorial_config(args.config)
+    task = run_eee(config)
+    metadata = dict(getattr(task, "_metadata", {}) or {})
+    print_outputs(
+        {
+            "run_dir": str(config.run_dir),
+            "embedding_metadata_path": metadata.get("embedding_metadata_path"),
+            "embedding_records_path": metadata.get("embedding_records_path"),
+        }
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tutorials/multimodal/omni-fuse-data-curation/3_project.py

@@ -0,0 +1,41 @@
+# Copyright (c) 2026, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Train the Omni-Fuse projection network and project raw embeddings."""
+
+from __future__ import annotations
+
+from utils import config_parser, load_tutorial_config, print_outputs, run_projection
+
+
+def main() -> int:
+    parser = config_parser(__doc__ or "")
+    args = parser.parse_args()
+    config = load_tutorial_config(args.config)
+    task = run_projection(config)
+    metadata = dict(getattr(task, "_metadata", {}) or {})
+    print_outputs(
+        {
+            "run_dir": str(config.run_dir),
+            "projection_model_path": metadata.get("projection_model_path"),
+            "projection_metrics_path": metadata.get("projection_metrics_path"),
+            "projected_embeddings_path": metadata.get("projected_embeddings_path"),
+            "projection_records_path": metadata.get("projection_records_path"),
+        }
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tutorials/multimodal/omni-fuse-data-curation/4_datablend.py

@@ -0,0 +1,40 @@
+# Copyright (c) 2026, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Rank SNS-refined records into a query-conditioned datablend."""
+
+from __future__ import annotations
+
+from utils import config_parser, load_tutorial_config, print_outputs, run_datablend
+
+
+def main() -> int:
+    parser = config_parser(__doc__ or "")
+    args = parser.parse_args()
+    config = load_tutorial_config(args.config)
+    task = run_datablend(config)
+    metadata = dict(getattr(task, "_metadata", {}) or {})
+    print_outputs(
+        {
+            "run_dir": str(config.run_dir),
+            "datablend_ranked_path": metadata.get("datablend_ranked_path"),
+            "datablend_topk_path": metadata.get("datablend_topk_path"),
+            "datablend_size": metadata.get("datablend_size"),
+        }
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())