From 7e6d9f08c3ea399b2707f8d5fbf5b535a1ec2cc4 Mon Sep 17 00:00:00 2001 From: Aaron Ray Date: Tue, 28 Oct 2025 22:32:54 -0400 Subject: [PATCH 01/14] initial examples --- docker/chatdsg_demo.Dockerfile | 9 +++++ docker/chatdsg_demo_script.sh | 5 +++ docker/docker-compose.yaml | 64 ++++++++++++++++++++++++++++++++ docker/heracles_demo.sh | 4 ++ docker/minimal_db_cli.Dockerfile | 6 +++ 5 files changed, 88 insertions(+) create mode 100644 docker/chatdsg_demo.Dockerfile create mode 100755 docker/chatdsg_demo_script.sh create mode 100644 docker/docker-compose.yaml create mode 100755 docker/heracles_demo.sh create mode 100644 docker/minimal_db_cli.Dockerfile diff --git a/docker/chatdsg_demo.Dockerfile b/docker/chatdsg_demo.Dockerfile new file mode 100644 index 0000000..8111b3a --- /dev/null +++ b/docker/chatdsg_demo.Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.12-slim + +RUN apt-get update +RUN apt-get update +RUN apt-get install -y git +#COPY heracles heracles +RUN git clone https://github.com/GoldenZephyr/heracles_agents.git +RUN pip install ./heracles_agents[all] +WORKDIR /heracles_agents/examples/chatdsg diff --git a/docker/chatdsg_demo_script.sh b/docker/chatdsg_demo_script.sh new file mode 100755 index 0000000..eb40a30 --- /dev/null +++ b/docker/chatdsg_demo_script.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -e + +docker compose up -d neo4j +docker compose run --rm chatdsg diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml new file mode 100644 index 0000000..6b782da --- /dev/null +++ b/docker/docker-compose.yaml @@ -0,0 +1,64 @@ +services: + neo4j: + image: neo4j:5.25.1 + environment: + NEO4J_server_bolt_listen__address: :7683 + NEO4J_server_http_advertised__address: :7469 + NEO4J_server_http_listen__address: :7469 + NEO4J_AUTH: neo4j/neo4j_pw + network_mode: "host" + healthcheck: + test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:7474 || exit 1"] + interval: 1s + timeout: 10s + retries: 10 + start_period: 2s + + + #backend: + # build: ./backend + # depends_on: + # db: + # condition: service_healthy + # environment: + # DATABASE_URL: postgres://demo:demo@db:5432/demo + # ports: + # - "8080:8080" + + cli: + build: + context: .. + dockerfile: docker/minimal_db_cli.Dockerfile + depends_on: + neo4j: + condition: service_healthy + environment: + HERACLES_NEO4J_USERNAME: neo4j + HERACLES_NEO4J_PASSWORD: neo4j_pw + HERACLES_NEO4J_URI: neo4j://127.0.0.1:7683 + network_mode: "host" + stdin_open: true + tty: true + entrypoint: ["ipython", "-i", "/heracles/examples/load_scene_graph.py", "--", "--scene_graph", "/heracles/examples/scene_graphs/example_dsg.json"] + + chatdsg: + build: + context: . + dockerfile: chatdsg_demo.Dockerfile + depends_on: + neo4j: + condition: service_healthy + environment: + HERACLES_NEO4J_USERNAME: neo4j + HERACLES_NEO4J_PASSWORD: neo4j_pw + HERACLES_NEO4J_URI: neo4j://127.0.0.1:7683 + HERACLES_OPENAI_API_KEY: $HERACLES_OPENAI_API_KEY + HERACLES_AGENTS_PATH: /heracles_agents + ADT4_HERACLES_IP: 127.0.0.1 + ADT4_HERACLES_PORT: 7683 + network_mode: "host" + stdin_open: true + tty: true + entrypoint: ["python3", "chatdsg.py", "/heracles/examples/scene_graphs/example_dsg.json"] + #entrypoint: ["python3", "/heracles_agents/examples/chatdsg/chatdsg.py"] + #entrypoint: ["/bin/bash"] diff --git a/docker/heracles_demo.sh b/docker/heracles_demo.sh new file mode 100755 index 0000000..c844bff --- /dev/null +++ b/docker/heracles_demo.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -e +docker compose up -d neo4j +docker compose run --rm cli diff --git a/docker/minimal_db_cli.Dockerfile b/docker/minimal_db_cli.Dockerfile new file mode 100644 index 0000000..754fbd4 --- /dev/null +++ b/docker/minimal_db_cli.Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.12-slim + +RUN apt update +COPY heracles heracles +RUN pip install ./heracles +RUN pip install ipython From dc69d3839fd0e6316c184609c8c3cb07ed20cccb Mon Sep 17 00:00:00 2001 From: Aaron Ray Date: Tue, 28 Oct 2025 23:54:01 -0400 Subject: [PATCH 02/14] examples almost working --- docker/chatdsg_demo.Dockerfile | 1 - docker/chatdsg_demo_script.sh | 2 +- docker/docker-compose.yaml | 14 ++++++++++++++ docker/hydra.Dockerfile | 21 +++++++++++++++++++++ docker/run_hydra_visualization.sh | 5 +++++ 5 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 docker/hydra.Dockerfile create mode 100755 docker/run_hydra_visualization.sh diff --git a/docker/chatdsg_demo.Dockerfile b/docker/chatdsg_demo.Dockerfile index 8111b3a..51c4e03 100644 --- a/docker/chatdsg_demo.Dockerfile +++ b/docker/chatdsg_demo.Dockerfile @@ -1,6 +1,5 @@ FROM python:3.12-slim -RUN apt-get update RUN apt-get update RUN apt-get install -y git #COPY heracles heracles diff --git a/docker/chatdsg_demo_script.sh b/docker/chatdsg_demo_script.sh index eb40a30..6722d2d 100755 --- a/docker/chatdsg_demo_script.sh +++ b/docker/chatdsg_demo_script.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash set -e -docker compose up -d neo4j +docker compose up -d neo4j hydra_visualization docker compose run --rm chatdsg diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 6b782da..b4c0d7d 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -62,3 +62,17 @@ services: entrypoint: ["python3", "chatdsg.py", "/heracles/examples/scene_graphs/example_dsg.json"] #entrypoint: ["python3", "/heracles_agents/examples/chatdsg/chatdsg.py"] #entrypoint: ["/bin/bash"] + + + hydra_visualization: + build: + dockerfile: hydra.Dockerfile + depends_on: + neo4j: + condition: service_healthy + environment: + HERACLES_NEO4J_USERNAME: neo4j + HERACLES_NEO4J_PASSWORD: neo4j_pw + HERACLES_NEO4J_URI: neo4j://127.0.0.1:7683 + network_mode: "host" + entrypoint: ["/bin/bash", "/run_hydra_visualization.sh"] diff --git a/docker/hydra.Dockerfile b/docker/hydra.Dockerfile new file mode 100644 index 0000000..22b3169 --- /dev/null +++ b/docker/hydra.Dockerfile @@ -0,0 +1,21 @@ +FROM ros:jazzy-ros-core + +RUN apt-get update +RUN apt-get install -y git ros-dev-tools python3.12-venv python3-pip +RUN rosdep init && rosdep update + +RUN mkdir -p /hydra_ws/src +WORKDIR /hydra_ws +RUN echo "build: {cmake-args: [-DCMAKE_BUILD_TYPE=Release]}" > colcon_defaults.yaml + +WORKDIR /hydra_ws/src +RUN git clone https://github.com/MIT-SPARK/Hydra-ROS.git hydra_ros +RUN git clone https://github.com/GoldenZephyr/heracles_agents.git + +RUN python3 -m venv /venv --system-site-packages +RUN vcs import . < hydra_ros/install/ros2_docker.yaml +RUN rosdep install --from-paths . --ignore-src -r -y +WORKDIR /hydra_ws +RUN . /opt/ros/jazzy/setup.sh && colcon build +RUN . /venv/bin/activate && pip install ./src/heracles_agents[all] +COPY run_hydra_visualization.sh /run_hydra_visualization.sh diff --git a/docker/run_hydra_visualization.sh b/docker/run_hydra_visualization.sh new file mode 100755 index 0000000..9ba40dc --- /dev/null +++ b/docker/run_hydra_visualization.sh @@ -0,0 +1,5 @@ +#!/bin/sh +. /opt/ros/jazzy/setup.sh +. /hydra_ws/install/setup.sh +. /venv/bin/activate +ros2 launch heracles_ros demo.launch.yaml launch_heracles_publisher:=true launch_hydra_visualizer:=true launch_rviz:=true From 5f5813471c1d466ddfed1e08c0bb10152b2dce6a Mon Sep 17 00:00:00 2001 From: Aaron Ray Date: Wed, 29 Oct 2025 10:06:01 -0400 Subject: [PATCH 03/14] ros docker demo works --- docker/chatdsg_demo_script.sh | 3 ++- docker/chatdsg_demo_viser.sh | 7 ++++++ docker/docker-compose.yaml | 40 +++++++++++++++++++++---------- docker/hydra.Dockerfile | 2 ++ docker/spark_dsg_viser.Dockerfile | 8 +++++++ 5 files changed, 46 insertions(+), 14 deletions(-) create mode 100755 docker/chatdsg_demo_viser.sh create mode 100644 docker/spark_dsg_viser.Dockerfile diff --git a/docker/chatdsg_demo_script.sh b/docker/chatdsg_demo_script.sh index 6722d2d..13aeafd 100755 --- a/docker/chatdsg_demo_script.sh +++ b/docker/chatdsg_demo_script.sh @@ -2,4 +2,5 @@ set -e docker compose up -d neo4j hydra_visualization -docker compose run --rm chatdsg +docker compose run --rm chatdsg # Blocks until terminal exits +docker compose down hydra_visualization diff --git a/docker/chatdsg_demo_viser.sh b/docker/chatdsg_demo_viser.sh new file mode 100755 index 0000000..1a140cf --- /dev/null +++ b/docker/chatdsg_demo_viser.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -e + +#docker compose up -d neo4j viser_visualization +#docker compose run --rm chatdsg + +docker compose run --rm viser_visualization diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index b4c0d7d..48c4d89 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,4 +1,6 @@ services: + + # Graph database storing scene graph neo4j: image: neo4j:5.25.1 environment: @@ -14,17 +16,7 @@ services: retries: 10 start_period: 2s - - #backend: - # build: ./backend - # depends_on: - # db: - # condition: service_healthy - # environment: - # DATABASE_URL: postgres://demo:demo@db:5432/demo - # ports: - # - "8080:8080" - + # Standalone python interpreter example for running queries against database cli: build: context: .. @@ -41,6 +33,7 @@ services: tty: true entrypoint: ["ipython", "-i", "/heracles/examples/load_scene_graph.py", "--", "--scene_graph", "/heracles/examples/scene_graphs/example_dsg.json"] + # Interactive LLM-based agent interaction with database chatdsg: build: context: . @@ -63,7 +56,7 @@ services: #entrypoint: ["python3", "/heracles_agents/examples/chatdsg/chatdsg.py"] #entrypoint: ["/bin/bash"] - + # Visualize scene graph in RVIZ using Hydra ROS visualizer hydra_visualization: build: dockerfile: hydra.Dockerfile @@ -73,6 +66,27 @@ services: environment: HERACLES_NEO4J_USERNAME: neo4j HERACLES_NEO4J_PASSWORD: neo4j_pw - HERACLES_NEO4J_URI: neo4j://127.0.0.1:7683 + #HERACLES_NEO4J_URI: neo4j://127.0.0.1:7683 + HERACLES_IP: 127.0.0.1 + HERACLES_PORT: 7683 + HERACLES_VENV: "" + DISPLAY: $DISPLAY + QT_X11_NO_MITSHM: 1 network_mode: "host" + volumes: + - /tmp/.X11-unix:/tmp/.X11-unix:rw entrypoint: ["/bin/bash", "/run_hydra_visualization.sh"] + + # Visualize scene graph in Viser using spark_dsg Viser visualization + viser_visualization: + build: + context: .. + dockerfile: docker/spark_dsg_viser.Dockerfile + environment: + HERACLES_NEO4J_USERNAME: neo4j + HERACLES_NEO4J_PASSWORD: neo4j_pw + HERACLES_NEO4J_URI: neo4j://127.0.0.1:7683 + SPARK_DSG_VISER_IP: 127.0.0.1 + SPARK_DSG_VISER_PORT: 8081 + network_mode: "host" + entrypoint: ["python", "/heracles-viser/heracles_viser_publisher.py"] diff --git a/docker/hydra.Dockerfile b/docker/hydra.Dockerfile index 22b3169..faa0f0b 100644 --- a/docker/hydra.Dockerfile +++ b/docker/hydra.Dockerfile @@ -11,10 +11,12 @@ RUN echo "build: {cmake-args: [-DCMAKE_BUILD_TYPE=Release]}" > colcon_defaults.y WORKDIR /hydra_ws/src RUN git clone https://github.com/MIT-SPARK/Hydra-ROS.git hydra_ros RUN git clone https://github.com/GoldenZephyr/heracles_agents.git +RUN git clone https://github.com/GoldenZephyr/heracles.git RUN python3 -m venv /venv --system-site-packages RUN vcs import . < hydra_ros/install/ros2_docker.yaml RUN rosdep install --from-paths . --ignore-src -r -y +RUN cd ianvs && git checkout feature/allow_empty_pyenv WORKDIR /hydra_ws RUN . /opt/ros/jazzy/setup.sh && colcon build RUN . /venv/bin/activate && pip install ./src/heracles_agents[all] diff --git a/docker/spark_dsg_viser.Dockerfile b/docker/spark_dsg_viser.Dockerfile new file mode 100644 index 0000000..1dbefb5 --- /dev/null +++ b/docker/spark_dsg_viser.Dockerfile @@ -0,0 +1,8 @@ +FROM python:3.12-slim + +RUN apt update +RUN apt install -y git libzmq3-dev nlohmann-json3-dev build-essential +COPY heracles heracles +RUN git clone https://github.com/MIT-SPARK/Spark-DSG.git spark_dsg +RUN pip install ./heracles ipython ./spark_dsg +#RUN pip install ipython From 0d555a5d48875937954db07541c80c021fa7cbee Mon Sep 17 00:00:00 2001 From: Aaron Ray Date: Wed, 29 Oct 2025 10:07:00 -0400 Subject: [PATCH 04/14] formatting --- docker/docker-compose.yaml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 48c4d89..b957f0c 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,3 +1,4 @@ +--- services: # Graph database storing scene graph @@ -8,9 +9,9 @@ services: NEO4J_server_http_advertised__address: :7469 NEO4J_server_http_listen__address: :7469 NEO4J_AUTH: neo4j/neo4j_pw - network_mode: "host" + network_mode: host healthcheck: - test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:7474 || exit 1"] + test: [CMD-SHELL, wget --no-verbose --tries=1 --spider http://localhost:7474 || exit 1] interval: 1s timeout: 10s retries: 10 @@ -28,10 +29,10 @@ services: HERACLES_NEO4J_USERNAME: neo4j HERACLES_NEO4J_PASSWORD: neo4j_pw HERACLES_NEO4J_URI: neo4j://127.0.0.1:7683 - network_mode: "host" + network_mode: host stdin_open: true tty: true - entrypoint: ["ipython", "-i", "/heracles/examples/load_scene_graph.py", "--", "--scene_graph", "/heracles/examples/scene_graphs/example_dsg.json"] + entrypoint: [ipython, -i, /heracles/examples/load_scene_graph.py, --, --scene_graph, /heracles/examples/scene_graphs/example_dsg.json] # Interactive LLM-based agent interaction with database chatdsg: @@ -49,12 +50,12 @@ services: HERACLES_AGENTS_PATH: /heracles_agents ADT4_HERACLES_IP: 127.0.0.1 ADT4_HERACLES_PORT: 7683 - network_mode: "host" + network_mode: host stdin_open: true tty: true - entrypoint: ["python3", "chatdsg.py", "/heracles/examples/scene_graphs/example_dsg.json"] - #entrypoint: ["python3", "/heracles_agents/examples/chatdsg/chatdsg.py"] - #entrypoint: ["/bin/bash"] + entrypoint: [python3, chatdsg.py, /heracles/examples/scene_graphs/example_dsg.json] + # entrypoint: ["python3", "/heracles_agents/examples/chatdsg/chatdsg.py"] + # entrypoint: ["/bin/bash"] # Visualize scene graph in RVIZ using Hydra ROS visualizer hydra_visualization: @@ -66,16 +67,15 @@ services: environment: HERACLES_NEO4J_USERNAME: neo4j HERACLES_NEO4J_PASSWORD: neo4j_pw - #HERACLES_NEO4J_URI: neo4j://127.0.0.1:7683 + # HERACLES_NEO4J_URI: neo4j://127.0.0.1:7683 HERACLES_IP: 127.0.0.1 HERACLES_PORT: 7683 - HERACLES_VENV: "" + HERACLES_VENV: '' DISPLAY: $DISPLAY QT_X11_NO_MITSHM: 1 - network_mode: "host" - volumes: - - /tmp/.X11-unix:/tmp/.X11-unix:rw - entrypoint: ["/bin/bash", "/run_hydra_visualization.sh"] + network_mode: host + volumes: [/tmp/.X11-unix:/tmp/.X11-unix:rw] + entrypoint: [/bin/bash, /run_hydra_visualization.sh] # Visualize scene graph in Viser using spark_dsg Viser visualization viser_visualization: @@ -88,5 +88,5 @@ services: HERACLES_NEO4J_URI: neo4j://127.0.0.1:7683 SPARK_DSG_VISER_IP: 127.0.0.1 SPARK_DSG_VISER_PORT: 8081 - network_mode: "host" - entrypoint: ["python", "/heracles-viser/heracles_viser_publisher.py"] + network_mode: host + entrypoint: [python, /heracles-viser/heracles_viser_publisher.py] From 52d59a7ffba69f73730b829dfda12e35ecc506ef Mon Sep 17 00:00:00 2001 From: Aaron Ray Date: Wed, 29 Oct 2025 10:33:34 -0400 Subject: [PATCH 05/14] viser and ROS demos both work in docker --- docker/chatdsg_demo_viser.sh | 7 +++---- docker/docker-compose.yaml | 4 +--- docker/hydra.Dockerfile | 7 +++++-- docker/spark_dsg_viser.Dockerfile | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docker/chatdsg_demo_viser.sh b/docker/chatdsg_demo_viser.sh index 1a140cf..abe9724 100755 --- a/docker/chatdsg_demo_viser.sh +++ b/docker/chatdsg_demo_viser.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash set -e -#docker compose up -d neo4j viser_visualization -#docker compose run --rm chatdsg - -docker compose run --rm viser_visualization +docker compose up -d neo4j viser_visualization +docker compose run --rm chatdsg +docker compose down viser_visualization diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index b957f0c..4291227 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -86,7 +86,5 @@ services: HERACLES_NEO4J_USERNAME: neo4j HERACLES_NEO4J_PASSWORD: neo4j_pw HERACLES_NEO4J_URI: neo4j://127.0.0.1:7683 - SPARK_DSG_VISER_IP: 127.0.0.1 - SPARK_DSG_VISER_PORT: 8081 network_mode: host - entrypoint: [python, /heracles-viser/heracles_viser_publisher.py] + entrypoint: [python, /heracles/src/heracles/heracles_viser_publisher.py, --ip, 127.0.0.1, --port, "8081"] diff --git a/docker/hydra.Dockerfile b/docker/hydra.Dockerfile index faa0f0b..51a0186 100644 --- a/docker/hydra.Dockerfile +++ b/docker/hydra.Dockerfile @@ -14,10 +14,13 @@ RUN git clone https://github.com/GoldenZephyr/heracles_agents.git RUN git clone https://github.com/GoldenZephyr/heracles.git RUN python3 -m venv /venv --system-site-packages -RUN vcs import . < hydra_ros/install/ros2_docker.yaml +RUN git clone https://github.com/MIT-SPARK/config_utilities.git +RUN git clone https://github.com/MIT-SPARK/Kimera-PGMO.git +RUN git clone https://github.com/MIT-SPARK/Spark-DSG.git + RUN rosdep install --from-paths . --ignore-src -r -y RUN cd ianvs && git checkout feature/allow_empty_pyenv WORKDIR /hydra_ws -RUN . /opt/ros/jazzy/setup.sh && colcon build +RUN . /opt/ros/jazzy/setup.sh && colcon build --packages-up-to hydra_visualizer RUN . /venv/bin/activate && pip install ./src/heracles_agents[all] COPY run_hydra_visualization.sh /run_hydra_visualization.sh diff --git a/docker/spark_dsg_viser.Dockerfile b/docker/spark_dsg_viser.Dockerfile index 1dbefb5..35c0718 100644 --- a/docker/spark_dsg_viser.Dockerfile +++ b/docker/spark_dsg_viser.Dockerfile @@ -3,6 +3,6 @@ FROM python:3.12-slim RUN apt update RUN apt install -y git libzmq3-dev nlohmann-json3-dev build-essential COPY heracles heracles -RUN git clone https://github.com/MIT-SPARK/Spark-DSG.git spark_dsg +RUN git clone -b feature/viser --single-branch https://github.com/MIT-SPARK/Spark-DSG.git spark_dsg RUN pip install ./heracles ipython ./spark_dsg #RUN pip install ipython From d597b76c7f90de297c802088ed8cba12ab5ffecf Mon Sep 17 00:00:00 2001 From: Aaron Ray Date: Wed, 29 Oct 2025 10:33:45 -0400 Subject: [PATCH 06/14] formatting --- docker/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 4291227..0786893 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -87,4 +87,4 @@ services: HERACLES_NEO4J_PASSWORD: neo4j_pw HERACLES_NEO4J_URI: neo4j://127.0.0.1:7683 network_mode: host - entrypoint: [python, /heracles/src/heracles/heracles_viser_publisher.py, --ip, 127.0.0.1, --port, "8081"] + entrypoint: [python, /heracles/src/heracles/heracles_viser_publisher.py, --ip, 127.0.0.1, --port, '8081'] From 1a8875dd7b4007d8ad653ad1b734395d38f31598 Mon Sep 17 00:00:00 2001 From: Aaron Ray Date: Wed, 29 Oct 2025 10:39:17 -0400 Subject: [PATCH 07/14] move demo scripts --- chatdsg_ros_demo.sh | 6 ++++++ chatdsg_viser_demo.sh | 6 ++++++ docker/chatdsg_demo_script.sh | 6 ------ docker/chatdsg_demo_viser.sh | 6 ------ docker/heracles_demo.sh | 4 ---- heracles_demo.sh | 4 ++++ 6 files changed, 16 insertions(+), 16 deletions(-) create mode 100755 chatdsg_ros_demo.sh create mode 100755 chatdsg_viser_demo.sh delete mode 100755 docker/chatdsg_demo_script.sh delete mode 100755 docker/chatdsg_demo_viser.sh delete mode 100755 docker/heracles_demo.sh create mode 100755 heracles_demo.sh diff --git a/chatdsg_ros_demo.sh b/chatdsg_ros_demo.sh new file mode 100755 index 0000000..69626c4 --- /dev/null +++ b/chatdsg_ros_demo.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -e + +docker compose -f docker/docker-compose.yaml up -d neo4j hydra_visualization +docker compose -f docker/docker-compose.yaml run --rm chatdsg # Blocks until terminal exits +docker compose -f docker/docker-compose.yaml down hydra_visualization diff --git a/chatdsg_viser_demo.sh b/chatdsg_viser_demo.sh new file mode 100755 index 0000000..0c192a5 --- /dev/null +++ b/chatdsg_viser_demo.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -e + +docker compose -f docker/docker-compose.yaml up -d neo4j viser_visualization +docker compose -f docker/docker-compose.yaml run --rm chatdsg +docker compose -f docker/docker-compose.yaml down viser_visualization diff --git a/docker/chatdsg_demo_script.sh b/docker/chatdsg_demo_script.sh deleted file mode 100755 index 13aeafd..0000000 --- a/docker/chatdsg_demo_script.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -set -e - -docker compose up -d neo4j hydra_visualization -docker compose run --rm chatdsg # Blocks until terminal exits -docker compose down hydra_visualization diff --git a/docker/chatdsg_demo_viser.sh b/docker/chatdsg_demo_viser.sh deleted file mode 100755 index abe9724..0000000 --- a/docker/chatdsg_demo_viser.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -set -e - -docker compose up -d neo4j viser_visualization -docker compose run --rm chatdsg -docker compose down viser_visualization diff --git a/docker/heracles_demo.sh b/docker/heracles_demo.sh deleted file mode 100755 index c844bff..0000000 --- a/docker/heracles_demo.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash -set -e -docker compose up -d neo4j -docker compose run --rm cli diff --git a/heracles_demo.sh b/heracles_demo.sh new file mode 100755 index 0000000..b366ff4 --- /dev/null +++ b/heracles_demo.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -e +docker compose -f docker/docker-compose.yaml up -d neo4j +docker compose -f docker/docker-compose.yaml run --rm cli From 61003726f704a862a35ed6ade492e04398741e1a Mon Sep 17 00:00:00 2001 From: Aaron Ray Date: Wed, 29 Oct 2025 12:56:19 -0400 Subject: [PATCH 08/14] docker fixup --- docker/docker-compose.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 0786893..3f60206 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -11,7 +11,7 @@ services: NEO4J_AUTH: neo4j/neo4j_pw network_mode: host healthcheck: - test: [CMD-SHELL, wget --no-verbose --tries=1 --spider http://localhost:7474 || exit 1] + test: [CMD-SHELL, wget --no-verbose --tries=1 --spider http://localhost:7469 || exit 1] interval: 1s timeout: 10s retries: 10 @@ -54,8 +54,6 @@ services: stdin_open: true tty: true entrypoint: [python3, chatdsg.py, /heracles/examples/scene_graphs/example_dsg.json] - # entrypoint: ["python3", "/heracles_agents/examples/chatdsg/chatdsg.py"] - # entrypoint: ["/bin/bash"] # Visualize scene graph in RVIZ using Hydra ROS visualizer hydra_visualization: From 5e9f3ecee9c7799f7d887514851968ed7d1513df Mon Sep 17 00:00:00 2001 From: Aaron Ray Date: Wed, 29 Oct 2025 16:34:50 -0400 Subject: [PATCH 09/14] more fixes --- docker/chatdsg_demo.Dockerfile | 4 ++-- docker/docker-compose.yaml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/chatdsg_demo.Dockerfile b/docker/chatdsg_demo.Dockerfile index 51c4e03..5f5ba35 100644 --- a/docker/chatdsg_demo.Dockerfile +++ b/docker/chatdsg_demo.Dockerfile @@ -2,7 +2,7 @@ FROM python:3.12-slim RUN apt-get update RUN apt-get install -y git -#COPY heracles heracles -RUN git clone https://github.com/GoldenZephyr/heracles_agents.git +COPY heracles/examples/scene_graphs/example_dsg.json example_dsg.json +RUN git clone --branch feature/chatdsg_optional_dsg_reset https://github.com/GoldenZephyr/heracles_agents.git RUN pip install ./heracles_agents[all] WORKDIR /heracles_agents/examples/chatdsg diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 3f60206..b35d122 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -37,8 +37,8 @@ services: # Interactive LLM-based agent interaction with database chatdsg: build: - context: . - dockerfile: chatdsg_demo.Dockerfile + context: .. + dockerfile: docker/chatdsg_demo.Dockerfile depends_on: neo4j: condition: service_healthy @@ -53,7 +53,7 @@ services: network_mode: host stdin_open: true tty: true - entrypoint: [python3, chatdsg.py, /heracles/examples/scene_graphs/example_dsg.json] + entrypoint: [python3, chatdsg.py, --scene-graph, /example_dsg.json] # Visualize scene graph in RVIZ using Hydra ROS visualizer hydra_visualization: From 9c86d2a948384d917da1ef44318780550be53aeb Mon Sep 17 00:00:00 2001 From: Aaron Ray Date: Wed, 29 Oct 2025 20:24:02 -0400 Subject: [PATCH 10/14] docker fixups --- docker/hydra.Dockerfile | 7 ++++--- docker/spark_dsg_viser.Dockerfile | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docker/hydra.Dockerfile b/docker/hydra.Dockerfile index 51a0186..7e065fb 100644 --- a/docker/hydra.Dockerfile +++ b/docker/hydra.Dockerfile @@ -1,7 +1,7 @@ FROM ros:jazzy-ros-core RUN apt-get update -RUN apt-get install -y git ros-dev-tools python3.12-venv python3-pip +RUN apt-get install -y git ros-dev-tools python3.12-venv python3-pip libgoogle-glog-dev RUN rosdep init && rosdep update RUN mkdir -p /hydra_ws/src @@ -15,12 +15,13 @@ RUN git clone https://github.com/GoldenZephyr/heracles.git RUN python3 -m venv /venv --system-site-packages RUN git clone https://github.com/MIT-SPARK/config_utilities.git -RUN git clone https://github.com/MIT-SPARK/Kimera-PGMO.git +RUN git clone --branch ros2 https://github.com/MIT-SPARK/Kimera-PGMO.git RUN git clone https://github.com/MIT-SPARK/Spark-DSG.git +RUN git clone https://github.com/MIT-SPARK/ianvs.git RUN rosdep install --from-paths . --ignore-src -r -y RUN cd ianvs && git checkout feature/allow_empty_pyenv WORKDIR /hydra_ws -RUN . /opt/ros/jazzy/setup.sh && colcon build --packages-up-to hydra_visualizer +RUN . /opt/ros/jazzy/setup.sh && colcon build --packages-up-to hydra_visualizer && colcon build --packages-up-to heracles heracles_ros heracles_agents RUN . /venv/bin/activate && pip install ./src/heracles_agents[all] COPY run_hydra_visualization.sh /run_hydra_visualization.sh diff --git a/docker/spark_dsg_viser.Dockerfile b/docker/spark_dsg_viser.Dockerfile index 35c0718..4c7c53a 100644 --- a/docker/spark_dsg_viser.Dockerfile +++ b/docker/spark_dsg_viser.Dockerfile @@ -3,6 +3,6 @@ FROM python:3.12-slim RUN apt update RUN apt install -y git libzmq3-dev nlohmann-json3-dev build-essential COPY heracles heracles -RUN git clone -b feature/viser --single-branch https://github.com/MIT-SPARK/Spark-DSG.git spark_dsg +RUN git clone -b feature/viser_aaron --single-branch https://github.com/MIT-SPARK/Spark-DSG.git spark_dsg RUN pip install ./heracles ipython ./spark_dsg #RUN pip install ipython From 8789baa2f35e0a69fde0e58bd4272c6f258f7a3b Mon Sep 17 00:00:00 2001 From: Aaron Ray Date: Wed, 29 Oct 2025 20:57:39 -0400 Subject: [PATCH 11/14] update README --- README.md | 70 ++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 61 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 1f042d7..9c6caf0 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,63 @@

# What is Heracles? -This repository provides a library for encoding a [Hydra](https://github.com/MIT-SPARK/Hydra) 3D scene graph into a Neo4j database. It expects a scene graph to initially be loaded from file via [spark\_dsg](https://github.com/MIT-SPARK/Spark-DSG) and provides utilities for translating back and forth between spark\_dsg and the Neo4j database. -## Setup +This repository provides a library for encoding a +[Hydra](https://github.com/MIT-SPARK/Hydra) 3D scene graph into a Neo4j +database. It expects a scene graph to initially be loaded from file via +[spark\_dsg](https://github.com/MIT-SPARK/Spark-DSG) and provides utilities for +translating back and forth between spark\_dsg and the Neo4j database. + +We provide three examples that should be pretty easy to run. + +## Minimal Example + +To run the first example, run +```bash +./heracles_demo.sh +``` + +This script will load an example scene graph into the graph database and put +you in a python terminal where you can run example queries. For example, try +running the following query to print the location of all trashcans: +```python +db.query("MATCH (n: Object {class: 'trash'}) RETURN n.center as center""") +``` + +## Interactive ROS Example + +The second example will render the current state of the scene graph in the +database in Rviz. You do _not_ need ROS installed to run this demo. However, +you will need to specify an OpenAI api key `HERACLES_OPENAI_API_KEY` if you +want to use the LLM interaction functionality. + +```bash +export HERACLES_OPENAI_API_KEY= +xhost +local:docker && ./chatdsg_ros_demo.sh +``` + +Try asking the LLM how many objects are in the scene graph (press `ctrl-b` to +send your message to the LLM). Then try telling it to update some of the +objects (e.g., turn `sign`s into `light`s). + +## Interactive Viser Example + +Finally, we provide an example of visualizing the scene graph using Viser, so +that no ROS is necessary. Again, you will need an OpenAI API key if you want to +use the LLM functionality. Make sure to go to http://127.0.0.1:8081 in your +browser after running the script! + +```bash +export HERACLES_OPENAI_API_KEY= +./chatdsg_viser_demo.sh # Then navigate to http://127.0.0.1:8081 in your browser +``` + + +# Development Setup + +If you want functionality beyond these basic demos, you will want to install +`heracles` locally. This requires 1) spinning up a database, and 2) installing +`heracles. Pull the Neo4j database image: ```bash @@ -35,15 +89,13 @@ for an example, as that successfully installs and runs the library for CI. If you don't have a scene graph to test with, you can use a large example scene graph [here](https://drive.google.com/file/d/1aktyS792PUrj2ACRu1DoxMGse55GWloB/view?usp=drive_link). This scene graph has 2D places and objects, and 3D places (that don't make much -sense), but no regions or rooms. - -The script `examples/dsg_test.py` is a small example of loading an existing -scene graph file into a graph database and running some simple queries. You can -run it in interaction mode (`ipython3 -i heracles/examples/dsg_test.py`) and -then try executing some of the other example queries below. +sense), but no regions or rooms. This repo contains an example scene graph as well. ## Demo - Loading a Scene Graph -We provide a demo program that loads a scene graph from file, encodes it into a database, and performs a simple query for validation. The demo program is implemented with interactive mode in mind to make it easy to try other kinds of queries. +We provide a demo program that loads a scene graph from file, encodes it into a +database, and performs a simple query for validation. The demo program is +implemented with interactive mode in mind to make it easy to try other kinds of +queries. ### Environment Variables We expect certain environment variables to be set, listed below. `HERACLES\_NEO4J\_URI` can alternatively be provided via the command line. An example of `HERACLES\_NEO4J\_URI` is `neo4j://127.0.0.1:7688`. From d326d0fc892049ebe9621277b1645962eb1d901a Mon Sep 17 00:00:00 2001 From: Aaron Ray Date: Wed, 29 Oct 2025 21:01:13 -0400 Subject: [PATCH 12/14] README fixup --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9c6caf0..e285378 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,8 @@ database. It expects a scene graph to initially be loaded from file via [spark\_dsg](https://github.com/MIT-SPARK/Spark-DSG) and provides utilities for translating back and forth between spark\_dsg and the Neo4j database. -We provide three examples that should be pretty easy to run. +We provide three examples that should be pretty easy to run. They require that +Docker is installed, but they should have no other dependencies. ## Minimal Example From 490cb85d6b1dfbe5993b8207d5058a8e622568b2 Mon Sep 17 00:00:00 2001 From: Aaron Ray Date: Wed, 29 Oct 2025 21:03:47 -0400 Subject: [PATCH 13/14] README fixup --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index e285378..2bdf6d7 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,9 @@ export HERACLES_OPENAI_API_KEY= ./chatdsg_viser_demo.sh # Then navigate to http://127.0.0.1:8081 in your browser ``` +You may have to pan the camera around if the scene graph is rendered out of the +initial camera frame. + # Development Setup From 6f424911721c84c2d0d89307cbc0c9f4e73957ad Mon Sep 17 00:00:00 2001 From: Aaron Ray Date: Wed, 29 Oct 2025 21:20:34 -0400 Subject: [PATCH 14/14] docker cleanup --- docker/chatdsg_demo.Dockerfile | 2 +- docker/docker-compose.yaml | 1 - docker/hydra.Dockerfile | 1 - docker/minimal_db_cli.Dockerfile | 3 +-- docker/spark_dsg_viser.Dockerfile | 1 - 5 files changed, 2 insertions(+), 6 deletions(-) diff --git a/docker/chatdsg_demo.Dockerfile b/docker/chatdsg_demo.Dockerfile index 5f5ba35..d969133 100644 --- a/docker/chatdsg_demo.Dockerfile +++ b/docker/chatdsg_demo.Dockerfile @@ -3,6 +3,6 @@ FROM python:3.12-slim RUN apt-get update RUN apt-get install -y git COPY heracles/examples/scene_graphs/example_dsg.json example_dsg.json -RUN git clone --branch feature/chatdsg_optional_dsg_reset https://github.com/GoldenZephyr/heracles_agents.git +RUN git clone https://github.com/GoldenZephyr/heracles_agents.git RUN pip install ./heracles_agents[all] WORKDIR /heracles_agents/examples/chatdsg diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index b35d122..dddfd9f 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -65,7 +65,6 @@ services: environment: HERACLES_NEO4J_USERNAME: neo4j HERACLES_NEO4J_PASSWORD: neo4j_pw - # HERACLES_NEO4J_URI: neo4j://127.0.0.1:7683 HERACLES_IP: 127.0.0.1 HERACLES_PORT: 7683 HERACLES_VENV: '' diff --git a/docker/hydra.Dockerfile b/docker/hydra.Dockerfile index 7e065fb..1cbae96 100644 --- a/docker/hydra.Dockerfile +++ b/docker/hydra.Dockerfile @@ -20,7 +20,6 @@ RUN git clone https://github.com/MIT-SPARK/Spark-DSG.git RUN git clone https://github.com/MIT-SPARK/ianvs.git RUN rosdep install --from-paths . --ignore-src -r -y -RUN cd ianvs && git checkout feature/allow_empty_pyenv WORKDIR /hydra_ws RUN . /opt/ros/jazzy/setup.sh && colcon build --packages-up-to hydra_visualizer && colcon build --packages-up-to heracles heracles_ros heracles_agents RUN . /venv/bin/activate && pip install ./src/heracles_agents[all] diff --git a/docker/minimal_db_cli.Dockerfile b/docker/minimal_db_cli.Dockerfile index 754fbd4..1f2102e 100644 --- a/docker/minimal_db_cli.Dockerfile +++ b/docker/minimal_db_cli.Dockerfile @@ -2,5 +2,4 @@ FROM python:3.12-slim RUN apt update COPY heracles heracles -RUN pip install ./heracles -RUN pip install ipython +RUN pip install ./heracles ipython diff --git a/docker/spark_dsg_viser.Dockerfile b/docker/spark_dsg_viser.Dockerfile index 4c7c53a..4216e63 100644 --- a/docker/spark_dsg_viser.Dockerfile +++ b/docker/spark_dsg_viser.Dockerfile @@ -5,4 +5,3 @@ RUN apt install -y git libzmq3-dev nlohmann-json3-dev build-essential COPY heracles heracles RUN git clone -b feature/viser_aaron --single-branch https://github.com/MIT-SPARK/Spark-DSG.git spark_dsg RUN pip install ./heracles ipython ./spark_dsg -#RUN pip install ipython