From 4eb247e60cf39e198ff9ca94c1fcfe5c6d9df6f1 Mon Sep 17 00:00:00 2001 From: Chana Orlinsky Date: Mon, 10 Nov 2025 15:26:47 +0200 Subject: [PATCH 1/4] Refactor dataset_preparation directory using git mv to preserve file history - Moved dataset_preparation/ to dataset/preparation/ using git mv - Moved src/optimal_class_mapping.py to dataset/ using git mv - Moved utilities/ to dataset/utilities/ using git mv - Updated imports in evaluation scripts to use new paths - All file history preserved as requested by @lyuzinmaxim --- .gitignore | 14 ++++---- {src => dataset}/optimal_class_mapping.py | 0 .../preparation}/compress_dataset.py | 4 +-- .../preparation}/dataset_prep.py | 2 +- .../preparation}/decompress_dataset.py | 4 +-- .../preparation}/prepare_crops_dataset.py | 6 ++-- .../utilities/datasets.py | 0 docs/dataset_preparation.md | 32 +++++++++---------- src/evaluation/run_evaluation.py | 2 +- src/evaluation/run_hierarchical_evaluation.py | 4 +-- 10 files changed, 35 insertions(+), 33 deletions(-) rename {src => dataset}/optimal_class_mapping.py (100%) rename {dataset_preparation => dataset/preparation}/compress_dataset.py (93%) rename {dataset_preparation => dataset/preparation}/dataset_prep.py (99%) rename {dataset_preparation => dataset/preparation}/decompress_dataset.py (93%) rename {dataset_preparation => dataset/preparation}/prepare_crops_dataset.py (97%) rename {dataset_preparation => dataset}/utilities/datasets.py (100%) diff --git a/.gitignore b/.gitignore index 31387fa3..d81945a0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -# Python +# Python __pycache__/ *.pyc *.pyo @@ -7,10 +7,8 @@ __pycache__/ # Outputs output/ -outputs/ -dataset/ -assets/classification_dataset -assets/output_images/ +data/classification_dataset +data/output_images/ database/data/ # Virtual environments @@ -29,4 +27,8 @@ deepstream_engines/ .ssh/ .vscode-server/ .bash_history -.gitconfig \ No newline at end of file +.gitconfig +# Dataset - exclude data files but keep structure +dataset/data/ +dataset/archives/*.zip +!dataset/archives/.gitkeep \ No newline at end of file diff --git a/src/optimal_class_mapping.py b/dataset/optimal_class_mapping.py similarity index 100% rename from src/optimal_class_mapping.py rename to dataset/optimal_class_mapping.py diff --git a/dataset_preparation/compress_dataset.py b/dataset/preparation/compress_dataset.py similarity index 93% rename from dataset_preparation/compress_dataset.py rename to dataset/preparation/compress_dataset.py index 3825b6df..3eb4fc69 100644 --- a/dataset_preparation/compress_dataset.py +++ b/dataset/preparation/compress_dataset.py @@ -3,7 +3,7 @@ from pathlib import Path -def compress_dataset(dataset_path="assets/classification_dataset", output_path="classification_dataset.zip"): +def compress_dataset(dataset_path="data/classification_dataset", output_path="dataset/archives/classification_dataset.zip"): """ Compress the classification dataset into a ZIP archive. @@ -55,4 +55,4 @@ def compress_dataset(dataset_path="assets/classification_dataset", output_path=" if __name__ == "__main__": - compress_dataset() + compress_dataset() \ No newline at end of file diff --git a/dataset_preparation/dataset_prep.py b/dataset/preparation/dataset_prep.py similarity index 99% rename from dataset_preparation/dataset_prep.py rename to dataset/preparation/dataset_prep.py index 3d7b09b7..0513e635 100644 --- a/dataset_preparation/dataset_prep.py +++ b/dataset/preparation/dataset_prep.py @@ -93,4 +93,4 @@ def main(): if __name__ == "__main__": - main() + main() \ No newline at end of file diff --git a/dataset_preparation/decompress_dataset.py b/dataset/preparation/decompress_dataset.py similarity index 93% rename from dataset_preparation/decompress_dataset.py rename to dataset/preparation/decompress_dataset.py index da2afd8f..6c2fe4e5 100644 --- a/dataset_preparation/decompress_dataset.py +++ b/dataset/preparation/decompress_dataset.py @@ -2,7 +2,7 @@ from pathlib import Path -def decompress_dataset(archive_path="classification_dataset.zip", output_dir="assets"): +def decompress_dataset(archive_path="dataset/archives/classification_dataset.zip", output_dir="data"): """ Decompress the classification dataset from a ZIP archive. @@ -53,4 +53,4 @@ def decompress_dataset(archive_path="classification_dataset.zip", output_dir="as if __name__ == "__main__": - decompress_dataset() + decompress_dataset() \ No newline at end of file diff --git a/dataset_preparation/prepare_crops_dataset.py b/dataset/preparation/prepare_crops_dataset.py similarity index 97% rename from dataset_preparation/prepare_crops_dataset.py rename to dataset/preparation/prepare_crops_dataset.py index f8012506..a7b974f7 100644 --- a/dataset_preparation/prepare_crops_dataset.py +++ b/dataset/preparation/prepare_crops_dataset.py @@ -170,7 +170,7 @@ def create_largest_k_classification_dataset( """ images_dir = Path(data_root) / "images" bboxes_dir = Path(data_root) / "bboxes" / "CropAndWeed" - output_dir = Path("assets") / "classification_dataset" / "images" + output_dir = Path("data") / "classification_dataset" / "images" output_dir.mkdir(parents=True, exist_ok=True) image_files = {img_file.stem: img_file for img_file in images_dir.glob("*.jpg")} @@ -188,7 +188,7 @@ def create_largest_k_classification_dataset( print("\n✂️ Creating dataset with LARGEST objects (Largest-K strategy)...") stats, total_objects = save_largest_k_crops(eligible_classes, output_dir, max_objects_per_class) - stats_file = Path("assets") / "classification_dataset" / "dataset_stats_largest_k.json" + stats_file = Path("data") / "classification_dataset" / "dataset_stats_largest_k.json" with open(stats_file, "w") as f: json.dump(stats, f, indent=2) @@ -202,4 +202,4 @@ def create_largest_k_classification_dataset( if __name__ == "__main__": - create_largest_k_classification_dataset("./dataset", 12, 12) + create_largest_k_classification_dataset("./dataset", 12, 12) \ No newline at end of file diff --git a/dataset_preparation/utilities/datasets.py b/dataset/utilities/datasets.py similarity index 100% rename from dataset_preparation/utilities/datasets.py rename to dataset/utilities/datasets.py diff --git a/docs/dataset_preparation.md b/docs/dataset_preparation.md index 2b247ed4..d2c66f55 100644 --- a/docs/dataset_preparation.md +++ b/docs/dataset_preparation.md @@ -16,29 +16,29 @@ This project uses a **two-stage dataset preparation process**: Download the original CropAndWeed dataset (5 tar archives, ~10,000 images) from the official source: ```bash -python dataset_preparation/dataset_prep.py --data_root ./dataset +python dataset/preparation/dataset_prep.py --data_root ./dataset/data ``` - **What gets downloaded:** Five large tar files containing images and annotations. - **Expected download time:** Each archive is several GB; total time depends on your internet speed (typically 10–30 minutes). - **Resulting structure:** ``` - dataset/ - ├── images/ - ├── bboxes/ - ├── labelIds/ - └── params/ +dataset/data/ +├── images/ +├── bboxes/ +├── labelIds/ +└── params/ ``` ### Stage 2: Create Balanced Classification Dataset Generate a balanced classification dataset (76 classes, 12 objects per class, 912 total) from the original data: ```bash -python dataset_preparation/prepare_crops_dataset.py --data_root ./dataset +python dataset/preparation/prepare_crops_dataset.py --data_root ./dataset/data ``` - **What happens:** The script filters and crops the original images using **PIL** to create a balanced classification dataset. - **Output:** ``` - classification_dataset/ + data/classification_dataset/ ├── images/ │ ├── 1/ │ ├── 2/ @@ -50,7 +50,7 @@ python dataset_preparation/prepare_crops_dataset.py --data_root ./dataset If you only need the ready-to-use classification dataset, extract it directly: ```bash -python dataset_preparation/decompress_dataset.py +python dataset/preparation/decompress_dataset.py ``` - **What happens:** Extracts a pre-built balanced dataset (912 images, 76 classes) for immediate use. @@ -75,7 +75,7 @@ The **CropAndWeed** dataset contains agricultural images with multi-level annota ### Native Format (Original) ``` -dataset/ +dataset/data/ ├── images/ # RGB images (JPG format) │ ├── ave-0000-0001.jpg │ └── ... @@ -163,10 +163,10 @@ To access the ready-to-use classification dataset: ```bash # Extract classification dataset archive (updated with Largest-K) -python dataset_preparation/decompress_dataset.py +python dataset/preparation/decompress_dataset.py # Or extract from custom location -python dataset_preparation/decompress_dataset.py classification_dataset.zip extracted_data/ +python dataset/preparation/decompress_dataset.py dataset/archives/classification_dataset.zip extracted_data/ ``` **Verification:** Script automatically verifies 912 images and 2 JSON files. @@ -185,12 +185,12 @@ The following issues are known and documented as part of the Largest-K approach: ```bash # Generate dataset with Largest-K strategy -python dataset_preparation/prepare_crops_dataset.py --data_root ./dataset +python dataset/preparation/prepare_crops_dataset.py --data_root ./dataset/data ``` ### Output files: -- `assets/classification_dataset/images/` - Cropped objects organized by class -- `assets/classification_dataset/dataset_stats_largest_k.json` - Size distribution statistics +- `data/classification_dataset/images/` - Cropped objects organized by class +- `data/classification_dataset/dataset_stats_largest_k.json` - Size distribution statistics --- @@ -205,4 +205,4 @@ python dataset_preparation/prepare_crops_dataset.py --data_root ./dataset - **Full dataset** (~10k images, 85 classes) - **Single-class images** (~1.5k images, ~60 classes) - **Single-object images** (~500 images, ~40 classes) -- **Largest-K dataset** (912 images, 76 classes) - **Current approach** +- **Largest-K dataset** (912 images, 76 classes) - **Current approach** \ No newline at end of file diff --git a/src/evaluation/run_evaluation.py b/src/evaluation/run_evaluation.py index 10748038..e0aee9fc 100644 --- a/src/evaluation/run_evaluation.py +++ b/src/evaluation/run_evaluation.py @@ -7,7 +7,7 @@ from src.evaluation.evaluator import EvaluationConfig, ModelEvaluator from src.inference import MobileNetInference, ResNetInference from src.onnx_model import OnnxClassifierInferenceBase as OnnxModel -from src.optimal_class_mapping import map_prediction +from dataset.optimal_class_mapping import map_prediction from src.path_utils import ensure_clean_directory MODELS_DIR_PATH = Path("models") diff --git a/src/evaluation/run_hierarchical_evaluation.py b/src/evaluation/run_hierarchical_evaluation.py index 5dee74e6..c1d6a7fc 100644 --- a/src/evaluation/run_hierarchical_evaluation.py +++ b/src/evaluation/run_hierarchical_evaluation.py @@ -4,11 +4,11 @@ from PIL import Image -from dataset_preparation.utilities.datasets import DATASETS +from dataset.utilities.datasets import DATASETS from metrics.metrics_api import compute_metrics from src.evaluation.evaluator import EvaluationConfig, ModelEvaluator from src.inference import MobileNetInference, ResNetInference -from src.optimal_class_mapping import map_prediction +from dataset.optimal_class_mapping import map_prediction class MappedModelWrapper: From 67e37df4ea14169882d190362a4b8fa19459ebfd Mon Sep 17 00:00:00 2001 From: chani0343 Date: Mon, 10 Nov 2025 16:11:22 +0200 Subject: [PATCH 2/4] Update compress_dataset.py --- dataset/preparation/compress_dataset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dataset/preparation/compress_dataset.py b/dataset/preparation/compress_dataset.py index 3eb4fc69..7edfc54f 100644 --- a/dataset/preparation/compress_dataset.py +++ b/dataset/preparation/compress_dataset.py @@ -55,4 +55,5 @@ def compress_dataset(dataset_path="data/classification_dataset", output_path="da if __name__ == "__main__": - compress_dataset() \ No newline at end of file + compress_dataset() + From 0158a5ffd259725445b5eb6114b6399e9122cc06 Mon Sep 17 00:00:00 2001 From: Chana Orlinsky Date: Mon, 10 Nov 2025 16:59:46 +0200 Subject: [PATCH 3/4] fix: apply targeted linting to dataset preparation files - Applied black formatting with line-length 119 - Applied isort with black profile - Only modified files related to dataset preparation task Addresses reviewer feedback --- dataset/optimal_class_mapping.py | 4 +--- dataset/preparation/compress_dataset.py | 5 +++-- dataset/preparation/dataset_prep.py | 2 +- dataset/preparation/decompress_dataset.py | 2 +- dataset/preparation/prepare_crops_dataset.py | 2 +- 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/dataset/optimal_class_mapping.py b/dataset/optimal_class_mapping.py index 669b464a..5597ca10 100644 --- a/dataset/optimal_class_mapping.py +++ b/dataset/optimal_class_mapping.py @@ -170,9 +170,7 @@ } # Create reverse lookup for dataset names -_DATASET_NAME_TO_ID: Dict[str, int] = { - name: class_id for class_id, name in DATASET_NAMES.items() -} +_DATASET_NAME_TO_ID: Dict[str, int] = {name: class_id for class_id, name in DATASET_NAMES.items()} def map_prediction(model_class_id: int) -> int: diff --git a/dataset/preparation/compress_dataset.py b/dataset/preparation/compress_dataset.py index 7edfc54f..1d32ed54 100644 --- a/dataset/preparation/compress_dataset.py +++ b/dataset/preparation/compress_dataset.py @@ -3,7 +3,9 @@ from pathlib import Path -def compress_dataset(dataset_path="data/classification_dataset", output_path="dataset/archives/classification_dataset.zip"): +def compress_dataset( + dataset_path="data/classification_dataset", output_path="dataset/archives/classification_dataset.zip" +): """ Compress the classification dataset into a ZIP archive. @@ -56,4 +58,3 @@ def compress_dataset(dataset_path="data/classification_dataset", output_path="da if __name__ == "__main__": compress_dataset() - diff --git a/dataset/preparation/dataset_prep.py b/dataset/preparation/dataset_prep.py index 0513e635..3d7b09b7 100644 --- a/dataset/preparation/dataset_prep.py +++ b/dataset/preparation/dataset_prep.py @@ -93,4 +93,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/dataset/preparation/decompress_dataset.py b/dataset/preparation/decompress_dataset.py index 6c2fe4e5..cdba65c9 100644 --- a/dataset/preparation/decompress_dataset.py +++ b/dataset/preparation/decompress_dataset.py @@ -53,4 +53,4 @@ def decompress_dataset(archive_path="dataset/archives/classification_dataset.zip if __name__ == "__main__": - decompress_dataset() \ No newline at end of file + decompress_dataset() diff --git a/dataset/preparation/prepare_crops_dataset.py b/dataset/preparation/prepare_crops_dataset.py index a7b974f7..21352dbb 100644 --- a/dataset/preparation/prepare_crops_dataset.py +++ b/dataset/preparation/prepare_crops_dataset.py @@ -202,4 +202,4 @@ def create_largest_k_classification_dataset( if __name__ == "__main__": - create_largest_k_classification_dataset("./dataset", 12, 12) \ No newline at end of file + create_largest_k_classification_dataset("./dataset", 12, 12) From b3ddc36836f5ac68d6a31c2b4f88a556e9817399 Mon Sep 17 00:00:00 2001 From: Chana Orlinsky Date: Mon, 10 Nov 2025 18:25:18 +0200 Subject: [PATCH 4/4] fix: simplify gitignore dataset exclusions Remove .gitkeep complexity and make dataset exclusions consistent with other entries --- .gitignore | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index d81945a0..ff1fd384 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -# Python +# Python __pycache__/ *.pyc *.pyo @@ -28,7 +28,7 @@ deepstream_engines/ .vscode-server/ .bash_history .gitconfig -# Dataset - exclude data files but keep structure + +# Dataset - exclude data files dataset/data/ dataset/archives/*.zip -!dataset/archives/.gitkeep \ No newline at end of file