From e07fd998adc744b430c3f73722ce444b8b464ca5 Mon Sep 17 00:00:00 2001 From: r83575 Date: Tue, 11 Nov 2025 15:27:56 +0200 Subject: [PATCH 1/3] Refactor script filenames and update references across the project --- docs/deepstream_pipeline.md | 2 +- docs/deepstream_validation.md | 4 ++-- docs/general/setup_guide.md | 2 +- docs/mongodb_setup.md | 14 +++++++------- docs/nvmsgbroker_integration.md | 12 ++++++------ {services/database => src/scripts}/cleanup.sh | 0 {scripts => src/scripts}/cleanup_frames.sh | 0 .../down.sh => src/scripts/database_down.sh | 0 .../database/up.sh => src/scripts/database_up.sh | 0 run_dev_jetson.sh => src/scripts/run_dev_jetson.sh | 0 run_dev.sh => src/scripts/run_dev_ml_cpu.sh | 0 .../scripts/start_mqtt_consumer.sh | 0 12 files changed, 17 insertions(+), 17 deletions(-) rename {services/database => src/scripts}/cleanup.sh (100%) rename {scripts => src/scripts}/cleanup_frames.sh (100%) rename services/database/down.sh => src/scripts/database_down.sh (100%) rename services/database/up.sh => src/scripts/database_up.sh (100%) rename run_dev_jetson.sh => src/scripts/run_dev_jetson.sh (100%) rename run_dev.sh => src/scripts/run_dev_ml_cpu.sh (100%) rename services/consumer/start_consumer.sh => src/scripts/start_mqtt_consumer.sh (100%) diff --git a/docs/deepstream_pipeline.md b/docs/deepstream_pipeline.md index 376a0c9e..b48f1389 100644 --- a/docs/deepstream_pipeline.md +++ b/docs/deepstream_pipeline.md @@ -25,7 +25,7 @@ ls -la output/frames/ ### 4. Cleanup Frames ```bash -bash ./scripts/cleanup_frames.sh +bash ./src/scripts/cleanup_frames.sh ``` ## Output diff --git a/docs/deepstream_validation.md b/docs/deepstream_validation.md index f795b62b..ee5d06e2 100644 --- a/docs/deepstream_validation.md +++ b/docs/deepstream_validation.md @@ -71,7 +71,7 @@ The models are **validated and ready for performance benchmarking**. - **Engines:** `/workspace/models/deepstream/` - **Script used:** `src/validate_deepstream_vs_onnx.py` - **Execution environments:** - - `run_dev.sh` (x86_64) → ONNX inference - - `run_dev_jetson.sh` (ARM Jetson) → DeepStream inference + - `src/scripts/run_dev_ml_cpu.sh` (x86_64) → ONNX inference + - `src/scripts/run_dev_jetson.sh` (ARM Jetson) → DeepStream inference --- diff --git a/docs/general/setup_guide.md b/docs/general/setup_guide.md index 76309b7f..70d0e661 100755 --- a/docs/general/setup_guide.md +++ b/docs/general/setup_guide.md @@ -23,7 +23,7 @@ We use **Docker** to make sure everyone has the same environment. You can think Then run: ```bash -bash run_dev.sh +bash src/scripts/run_dev_ml_cpu.sh ``` This will: diff --git a/docs/mongodb_setup.md b/docs/mongodb_setup.md index 7bf115e3..48b633b5 100644 --- a/docs/mongodb_setup.md +++ b/docs/mongodb_setup.md @@ -6,17 +6,17 @@ Deploy a simple MongoDB setup with scripts for up/down/cleanup, and provide usag ## Scripts -* `up.sh` - starts MongoDB container -* `down.sh` - stops MongoDB container -* `cleanup.sh` - removes container, network, and data +* `database_up.sh` - starts MongoDB container +* `database_down.sh` - stops MongoDB container +* `database_cleanup.sh` - removes container, network, and data ## Usage ```bash -cd ~/AgStream/database/scripts -./up.sh # Start MongoDB -./down.sh # Stop MongoDB -./cleanup.sh # Cleanup all data and containers +cd ~/AgStream/src/scripts +./database_up.sh # Start MongoDB +./database_down.sh # Stop MongoDB +./database_cleanup.sh # Cleanup all data and containers ``` ## Connection diff --git a/docs/nvmsgbroker_integration.md b/docs/nvmsgbroker_integration.md index 56afc0db..8f62c501 100644 --- a/docs/nvmsgbroker_integration.md +++ b/docs/nvmsgbroker_integration.md @@ -29,20 +29,20 @@ It processes **RTSP video streams**, runs **classification inference**, and stor ```bash # 1. Start database services (MongoDB + MQTT broker) -./services/database/up.sh +./src/scripts/database_up.sh # 2. Clear MongoDB (for fresh start) docker exec -it agstream_mongo mongosh agstream --eval "db.predictions.deleteMany({})" # 3. Start Consumer (builds image automatically on first run) -./services/consumer/start_consumer.sh +./src/scripts/start_mqtt_consumer.sh # 4. Verify Consumer is running docker logs mqtt_consumer -f # 5. Start DeepStream -./run_dev_arm.sh +./src/scripts/run_dev_jetson.sh # 6. Inside DeepStream container python3 src/rtsp/rtsp_server.py & @@ -64,7 +64,7 @@ python3 src/nvmsgbroker_pipeline.py * **Purpose:** Transfers MQTT messages to MongoDB * **Technology:** Python script using `paho-mqtt` and `pymongo` * **Auto-build:** First run creates Docker image with dependencies pre-installed -* **Restart:** `./start_consumer.sh` stops old container and starts a new one +* **Restart:** `./src/scripts/start_mqtt_consumer.sh` stops old container and starts a new one ### 🧩 Data Flow @@ -120,7 +120,7 @@ docker ps | grep agstream # Restart if needed docker compose up -d -./start_consumer.sh +./src/scripts/start_mqtt_consumer.sh ``` ### No data in MongoDB @@ -141,7 +141,7 @@ docker logs agstream_mosquitto **DeepStream → nvmsgbroker → MQTT → Consumer → MongoDB** 📝 **Note:** The Consumer is **REQUIRED** to transfer data from MQTT to MongoDB. -The `start_consumer.sh` script automatically builds the Docker image on first run. +The `start_mqtt_consumer.sh` script automatically builds the Docker image on first run. --- diff --git a/services/database/cleanup.sh b/src/scripts/cleanup.sh similarity index 100% rename from services/database/cleanup.sh rename to src/scripts/cleanup.sh diff --git a/scripts/cleanup_frames.sh b/src/scripts/cleanup_frames.sh similarity index 100% rename from scripts/cleanup_frames.sh rename to src/scripts/cleanup_frames.sh diff --git a/services/database/down.sh b/src/scripts/database_down.sh similarity index 100% rename from services/database/down.sh rename to src/scripts/database_down.sh diff --git a/services/database/up.sh b/src/scripts/database_up.sh similarity index 100% rename from services/database/up.sh rename to src/scripts/database_up.sh diff --git a/run_dev_jetson.sh b/src/scripts/run_dev_jetson.sh similarity index 100% rename from run_dev_jetson.sh rename to src/scripts/run_dev_jetson.sh diff --git a/run_dev.sh b/src/scripts/run_dev_ml_cpu.sh similarity index 100% rename from run_dev.sh rename to src/scripts/run_dev_ml_cpu.sh diff --git a/services/consumer/start_consumer.sh b/src/scripts/start_mqtt_consumer.sh similarity index 100% rename from services/consumer/start_consumer.sh rename to src/scripts/start_mqtt_consumer.sh From 1e25add6451ccb84f75827c1717517c330e03e07 Mon Sep 17 00:00:00 2001 From: r83575 Date: Tue, 11 Nov 2025 15:44:05 +0200 Subject: [PATCH 2/3] update env_dir path after scripts relocation --- src/scripts/run_dev_jetson.sh | 2 +- src/scripts/run_dev_ml_cpu.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scripts/run_dev_jetson.sh b/src/scripts/run_dev_jetson.sh index cbb41cd9..c2de2b5c 100755 --- a/src/scripts/run_dev_jetson.sh +++ b/src/scripts/run_dev_jetson.sh @@ -9,7 +9,7 @@ USER_ID=$(id -u) # Base directories script_dir=$(dirname "$(realpath "$0")") workspace_dir=${script_dir} -env_dir=${workspace_dir}/env/jetson +env_dir=${workspace_dir}/../../env/jetson # Container and image names IMAGE_NAME="inference-opt-jetson" diff --git a/src/scripts/run_dev_ml_cpu.sh b/src/scripts/run_dev_ml_cpu.sh index a219c46c..087db55c 100755 --- a/src/scripts/run_dev_ml_cpu.sh +++ b/src/scripts/run_dev_ml_cpu.sh @@ -6,7 +6,7 @@ set -o xtrace # print each command before executing # Base directories script_dir=$(dirname "$(realpath "$0")") workspace_dir=${script_dir} -env_dir=${workspace_dir}/env/ml_cpu +env_dir=${workspace_dir}/../../env/ml_cpu # Container and image names IMAGE_NAME="inference-opt" From 30ebbb5839ef9d4e558581c5642f50d8e50651b8 Mon Sep 17 00:00:00 2001 From: r83575 Date: Tue, 11 Nov 2025 23:21:21 +0200 Subject: [PATCH 3/3] Fixes incorrect script location from previous commit --- docs/deepstream_pipeline.md | 2 +- docs/deepstream_validation.md | 4 ++-- docs/general/setup_guide.md | 2 +- docs/mongodb_setup.md | 2 +- docs/nvmsgbroker_integration.md | 10 +++++----- {src/scripts => scripts}/cleanup_frames.sh | 0 src/scripts/cleanup.sh => scripts/database_cleanup.sh | 2 +- {src/scripts => scripts}/database_down.sh | 2 +- {src/scripts => scripts}/database_up.sh | 2 +- {src/scripts => scripts}/run_dev_jetson.sh | 2 +- {src/scripts => scripts}/run_dev_ml_cpu.sh | 2 +- {src/scripts => scripts}/start_mqtt_consumer.sh | 0 12 files changed, 15 insertions(+), 15 deletions(-) rename {src/scripts => scripts}/cleanup_frames.sh (100%) rename src/scripts/cleanup.sh => scripts/database_cleanup.sh (77%) rename {src/scripts => scripts}/database_down.sh (67%) rename {src/scripts => scripts}/database_up.sh (84%) rename {src/scripts => scripts}/run_dev_jetson.sh (96%) rename {src/scripts => scripts}/run_dev_ml_cpu.sh (94%) rename {src/scripts => scripts}/start_mqtt_consumer.sh (100%) diff --git a/docs/deepstream_pipeline.md b/docs/deepstream_pipeline.md index b48f1389..376a0c9e 100644 --- a/docs/deepstream_pipeline.md +++ b/docs/deepstream_pipeline.md @@ -25,7 +25,7 @@ ls -la output/frames/ ### 4. Cleanup Frames ```bash -bash ./src/scripts/cleanup_frames.sh +bash ./scripts/cleanup_frames.sh ``` ## Output diff --git a/docs/deepstream_validation.md b/docs/deepstream_validation.md index ee5d06e2..524472cc 100644 --- a/docs/deepstream_validation.md +++ b/docs/deepstream_validation.md @@ -71,7 +71,7 @@ The models are **validated and ready for performance benchmarking**. - **Engines:** `/workspace/models/deepstream/` - **Script used:** `src/validate_deepstream_vs_onnx.py` - **Execution environments:** - - `src/scripts/run_dev_ml_cpu.sh` (x86_64) → ONNX inference - - `src/scripts/run_dev_jetson.sh` (ARM Jetson) → DeepStream inference + - `scripts/run_dev_ml_cpu.sh` (x86_64) → ONNX inference + - `scripts/run_dev_jetson.sh` (ARM Jetson) → DeepStream inference --- diff --git a/docs/general/setup_guide.md b/docs/general/setup_guide.md index 70d0e661..c86f57fc 100755 --- a/docs/general/setup_guide.md +++ b/docs/general/setup_guide.md @@ -23,7 +23,7 @@ We use **Docker** to make sure everyone has the same environment. You can think Then run: ```bash -bash src/scripts/run_dev_ml_cpu.sh +bash scripts/run_dev_ml_cpu.sh ``` This will: diff --git a/docs/mongodb_setup.md b/docs/mongodb_setup.md index 48b633b5..31c12667 100644 --- a/docs/mongodb_setup.md +++ b/docs/mongodb_setup.md @@ -13,7 +13,7 @@ Deploy a simple MongoDB setup with scripts for up/down/cleanup, and provide usag ## Usage ```bash -cd ~/AgStream/src/scripts +cd ~/AgStream/scripts ./database_up.sh # Start MongoDB ./database_down.sh # Stop MongoDB ./database_cleanup.sh # Cleanup all data and containers diff --git a/docs/nvmsgbroker_integration.md b/docs/nvmsgbroker_integration.md index 8f62c501..549ec2fe 100644 --- a/docs/nvmsgbroker_integration.md +++ b/docs/nvmsgbroker_integration.md @@ -29,20 +29,20 @@ It processes **RTSP video streams**, runs **classification inference**, and stor ```bash # 1. Start database services (MongoDB + MQTT broker) -./src/scripts/database_up.sh +./scripts/database_up.sh # 2. Clear MongoDB (for fresh start) docker exec -it agstream_mongo mongosh agstream --eval "db.predictions.deleteMany({})" # 3. Start Consumer (builds image automatically on first run) -./src/scripts/start_mqtt_consumer.sh +./scripts/start_mqtt_consumer.sh # 4. Verify Consumer is running docker logs mqtt_consumer -f # 5. Start DeepStream -./src/scripts/run_dev_jetson.sh +./scripts/run_dev_jetson.sh # 6. Inside DeepStream container python3 src/rtsp/rtsp_server.py & @@ -64,7 +64,7 @@ python3 src/nvmsgbroker_pipeline.py * **Purpose:** Transfers MQTT messages to MongoDB * **Technology:** Python script using `paho-mqtt` and `pymongo` * **Auto-build:** First run creates Docker image with dependencies pre-installed -* **Restart:** `./src/scripts/start_mqtt_consumer.sh` stops old container and starts a new one +* **Restart:** `./scripts/start_mqtt_consumer.sh` stops old container and starts a new one ### 🧩 Data Flow @@ -120,7 +120,7 @@ docker ps | grep agstream # Restart if needed docker compose up -d -./src/scripts/start_mqtt_consumer.sh +./scripts/start_mqtt_consumer.sh ``` ### No data in MongoDB diff --git a/src/scripts/cleanup_frames.sh b/scripts/cleanup_frames.sh similarity index 100% rename from src/scripts/cleanup_frames.sh rename to scripts/cleanup_frames.sh diff --git a/src/scripts/cleanup.sh b/scripts/database_cleanup.sh similarity index 77% rename from src/scripts/cleanup.sh rename to scripts/database_cleanup.sh index 8663c1f4..150b0c09 100755 --- a/src/scripts/cleanup.sh +++ b/scripts/database_cleanup.sh @@ -4,7 +4,7 @@ set -e echo "Cleaning MongoDB data and containers..." # Move to MongoDB environment directory -docker compose -f "$(dirname "$0")/../../env/mongodb/docker-compose.yml" down -v +docker compose -f "$(dirname "$0")/../env/mongodb/docker-compose.yml" down -v # Remove any persisted data directory (if it exists) rm -rf "$(dirname "$0")/../../env/mongodb/data/" diff --git a/src/scripts/database_down.sh b/scripts/database_down.sh similarity index 67% rename from src/scripts/database_down.sh rename to scripts/database_down.sh index 47329e99..4f3bf4a0 100755 --- a/src/scripts/database_down.sh +++ b/scripts/database_down.sh @@ -4,6 +4,6 @@ set -e echo "Stopping MongoDB container..." # Move to MongoDB directory (relative to project root) -docker compose -f "$(dirname "$0")/../../env/mongodb/docker-compose.yml" down +docker compose -f "$(dirname "$0")/../env/mongodb/docker-compose.yml" down echo "MongoDB stopped and removed successfully." diff --git a/src/scripts/database_up.sh b/scripts/database_up.sh similarity index 84% rename from src/scripts/database_up.sh rename to scripts/database_up.sh index fd285679..00953f4b 100755 --- a/src/scripts/database_up.sh +++ b/scripts/database_up.sh @@ -4,7 +4,7 @@ set -e echo "Starting MongoDB container..." # Define the compose file path -COMPOSE_FILE="$(dirname "$0")/../../env/mongodb/docker-compose.yml" +COMPOSE_FILE="$(dirname "$0")/../env/mongodb/docker-compose.yml" # Ensure old containers are cleaned up if needed docker compose -f "$COMPOSE_FILE" down --remove-orphans >/dev/null 2>&1 || true diff --git a/src/scripts/run_dev_jetson.sh b/scripts/run_dev_jetson.sh similarity index 96% rename from src/scripts/run_dev_jetson.sh rename to scripts/run_dev_jetson.sh index c2de2b5c..057be4ae 100755 --- a/src/scripts/run_dev_jetson.sh +++ b/scripts/run_dev_jetson.sh @@ -9,7 +9,7 @@ USER_ID=$(id -u) # Base directories script_dir=$(dirname "$(realpath "$0")") workspace_dir=${script_dir} -env_dir=${workspace_dir}/../../env/jetson +env_dir=${workspace_dir}/../env/jetson # Container and image names IMAGE_NAME="inference-opt-jetson" diff --git a/src/scripts/run_dev_ml_cpu.sh b/scripts/run_dev_ml_cpu.sh similarity index 94% rename from src/scripts/run_dev_ml_cpu.sh rename to scripts/run_dev_ml_cpu.sh index 087db55c..2d463b8e 100755 --- a/src/scripts/run_dev_ml_cpu.sh +++ b/scripts/run_dev_ml_cpu.sh @@ -6,7 +6,7 @@ set -o xtrace # print each command before executing # Base directories script_dir=$(dirname "$(realpath "$0")") workspace_dir=${script_dir} -env_dir=${workspace_dir}/../../env/ml_cpu +env_dir=${workspace_dir}/../env/ml_cpu # Container and image names IMAGE_NAME="inference-opt" diff --git a/src/scripts/start_mqtt_consumer.sh b/scripts/start_mqtt_consumer.sh similarity index 100% rename from src/scripts/start_mqtt_consumer.sh rename to scripts/start_mqtt_consumer.sh