interuss · mickmis · Apr 15, 2026
diff --git a/build/dev/docker-compose.yaml b/build/dev/docker-compose.yaml
@@ -1,22 +1,19 @@
-# Brings up a local interoperability ecosystem consisting of a DSS instance and dummy OAuth server.
-
-# To bring up this system, run ./run_locally.sh up -d and wait for all containers to succeed or become healthy.
-# If needed, set environment variable `COMPOSE_PROFILES` to `with-yugabyte` to use Yugabyte instead of CockroachDB
+# This Docker Compose file must be used through `./run_locally.sh`.
 
 services:
 
   crdb:
-    hostname: crdb.uss1.localutm
     image: cockroachdb/cockroach:v24.1.3
-    command: start-single-node --insecure
-    expose:
-      - 26257
-    ports:
-      - "8080:8080"
-      - "26257:26257"
+    command: start --insecure --join=db1.uss1.localutm
+    profiles: [crdb]
     restart: always
+    hostname: db${USS_NODE_IDX:?}.uss${USS_IDX:?}.localutm
     networks:
-      - dss_internal_network
+      dss_internal_network:
+        ipv4_address: 172.27.${USS_IDX:?}.${USS_NODE_IDX:?}
+    ports:
+      - "81${PADDED_NODE_IDX:?}:8080"
+      - "262${PADDED_NODE_IDX:?}:26257"
     healthcheck:
       test: curl -f 'http://localhost:8080/health?ready=1' || exit 1
       interval: 3m
@@ -26,133 +23,153 @@ services:
   ybdb:
     image: interuss/yugabyte:2025.1.2.1-interuss
     # ysql_output_buffer_size needs to be increased to allow ysql to retry read restart errors. https://docs.yugabyte.com/preview/reference/configuration/yb-tserver/#ysql-output-buffer-size
-    command: bin/yugabyted start --background=false --tserver_flags="ysql_output_buffer_size=1048576"
-    ports:
-      - "7000:7000"
-      - "9000:9000"
-      - "15433:15433"
-      - "5433:5433"
-      - "9042:9042"
+    command: sh -c "bin/yugabyted start --background=false --tserver_flags=ysql_output_buffer_size=1048576 --advertise_address=$$YBDB_HOST --join=$$([ "$$YBDB_HOST" = db1.uss1.localutm ] || echo db1.uss1.localutm)"
+    profiles: [ybdb]
     restart: always
+    hostname: db${USS_NODE_IDX:?}.uss${USS_IDX:?}.localutm
     networks:
-      - dss_internal_network
-    profiles: ["with-yugabyte"]
+      dss_internal_network:
+        ipv4_address: 172.27.${USS_IDX:?}.${USS_NODE_IDX:?}
+    ports:
+      - "70${PADDED_NODE_IDX:?}:7000"
+      - "90${PADDED_NODE_IDX:?}:9000"
+      - "154${PADDED_NODE_IDX:?}:15433"
+      - "54${PADDED_NODE_IDX:?}:5433"
+      - "91${PADDED_NODE_IDX:?}:9042"
+    environment:
+      - YBDB_HOST=db${USS_NODE_IDX:?}.uss${USS_IDX:?}.localutm
     healthcheck:
-      test: ["CMD", "/home/yugabyte/postgres/bin/pg_isready", "-h", "ybdb"]
+      test: /home/yugabyte/postgres/bin/pg_isready -h $$YBDB_HOST || exit 1
       interval: 5s
       timeout: 5s
       retries: 10
 
+  crdb-init:
+    image: cockroachdb/cockroach:v24.1.3
+    profiles: [bootstrap-crdb]
+    depends_on: [crdb]
+    entrypoint: >
+      sh -c "
+        out=$$(/cockroach/cockroach.sh init --insecure --host=db1.uss1.localutm 2>&1);
+        if [ $$? -eq 0 ]; then
+          echo "$$out";
+          exit 0;
+        fi;
+        echo "$$out";
+        if echo \"$$out\" | grep -q \"already been initialized\"; then
+          exit 0;
+        fi;
+        exit 1;
+      "
+    networks: [dss_internal_network]
+
   rid_bootstrapper:
     image: interuss/dss:v0.21.0-rc1
-    command: /usr/bin/db-manager migrate --schemas_dir=/db-schemas/rid --db_version "latest" --cockroach_host crdb
+    profiles: [bootstrap-crdb]
+    entrypoint: sh -c "/usr/bin/db-manager migrate --schemas_dir=/db-schemas/$$BOOTSTRAP_DB --db_version latest --cockroach_host db1.uss1.localutm"
+    environment:
+      - BOOTSTRAP_DB=rid
     depends_on:
-      crdb:
-        condition: service_healthy
-    networks:
-      - dss_internal_network
+      crdb-init:
+        condition: service_completed_successfully
+    networks: [dss_internal_network]
+    healthcheck:
+      disable: true
 
   scd_bootstrapper:
-    image: interuss/dss:v0.21.0-rc1
-    command: /usr/bin/db-manager migrate --schemas_dir=/db-schemas/scd --db_version "latest" --cockroach_host crdb
-    depends_on:
-      crdb:
-        condition: service_healthy
-    networks:
-      - dss_internal_network
+    extends:
+      service: rid_bootstrapper
+    environment:
+      - BOOTSTRAP_DB=scd
 
   aux_bootstrapper:
-    image: interuss/dss:v0.21.0-rc1
-    command: /usr/bin/db-manager migrate --schemas_dir=/db-schemas/aux_ --db_version "latest" --cockroach_host crdb
-    depends_on:
-      crdb:
-        condition: service_healthy
-    networks:
-      - dss_internal_network
+    extends:
+      service: rid_bootstrapper
+    environment:
+      - BOOTSTRAP_DB=aux_
 
   rid_bootstrapper-ybdb:
     image: interuss/dss:v0.21.0-rc1
-    entrypoint: /usr/bin/db-manager migrate --schemas_dir=/db-schemas/yugabyte/rid --db_version "latest" --datastore_host ybdb --datastore_user yugabyte --datastore_port 5433
-    depends_on:
-      ybdb:
-        condition: service_healthy
-    networks:
-      - dss_internal_network
-    profiles: ["with-yugabyte"]
+    profiles: [bootstrap-ybdb]
+    entrypoint: sh -c "/usr/bin/db-manager migrate --schemas_dir=/db-schemas/yugabyte/$$BOOTSTRAP_DB --db_version latest --datastore_host db1.uss1.localutm --datastore_user yugabyte --datastore_port 5433"
+    environment:
+      - BOOTSTRAP_DB=rid
+    restart: on-failure
+    networks: [dss_internal_network]
+    healthcheck:
+      disable: true
 
   scd_bootstrapper-ybdb:
-    image: interuss/dss:v0.21.0-rc1
-    entrypoint: /usr/bin/db-manager migrate --schemas_dir=/db-schemas/yugabyte/scd --db_version "latest" --datastore_host ybdb --datastore_user yugabyte --datastore_port 5433
-    depends_on:
-      ybdb:
-        condition: service_healthy
-    networks:
-      - dss_internal_network
-    profiles: ["with-yugabyte"]
+    extends:
+      service: rid_bootstrapper-ybdb
+    environment:
+      - BOOTSTRAP_DB=scd
 
   aux_bootstrapper-ybdb:
-    image: interuss/dss:v0.21.0-rc1
-    entrypoint: /usr/bin/db-manager migrate --schemas_dir=/db-schemas/yugabyte/aux_ --db_version "latest" --datastore_host ybdb --datastore_user yugabyte --datastore_port 5433
-    depends_on:
-      ybdb:
-        condition: service_healthy
-    networks:
-      - dss_internal_network
-    profiles: ["with-yugabyte"]
+    extends:
+      service: rid_bootstrapper-ybdb
+    environment:
+      - BOOTSTRAP_DB=aux_
 
   dss:
-    hostname: dss.uss1.localutm
     image: interuss/dss:v0.21.0-rc1
+    command: /startup/core_service.sh ${DEBUG_ON:-0}
+    profiles: [ crdb, ybdb ]
+    restart: always
     volumes:
       - $PWD/../test-certs:/var/test-certs:ro
       - $PWD/startup/core_service.sh:/startup/core_service.sh:ro
-    command: /startup/core_service.sh ${DEBUG_ON:-0}
-    environment:
-      COMPOSE_PROFILES: ${COMPOSE_PROFILES}
-    expose:
-      - 8082
-    ports:
-      - "4000:4000"
-      - "8082:80"
     depends_on:
       rid_bootstrapper:
         condition: service_completed_successfully
+        required: false
       scd_bootstrapper:
         condition: service_completed_successfully
+        required: false
       aux_bootstrapper:
         condition: service_completed_successfully
+        required: false
       rid_bootstrapper-ybdb:
         condition: service_completed_successfully
         required: false
       scd_bootstrapper-ybdb:
-        required: false
         condition: service_completed_successfully
+        required: false
       aux_bootstrapper-ybdb:
         condition: service_completed_successfully
         required: false
+    hostname: dss${USS_NODE_IDX:?}.uss${USS_IDX:?}.localutm
     networks:
       dss_internal_network: {}
       interop_ecosystem_network:
         aliases:
-          - dss.uss2.localutm
+          - dss.uss${USS_IDX:?}.localutm
+    ports:
+      - "40${PADDED_NODE_IDX:?}:4000"
+      - "80${PADDED_NODE_IDX:?}:80"
+    environment:
+      COMPOSE_PROFILES: ${COMPOSE_PROFILES}
+      JWT_AUDIENCES: dss.uss${USS_IDX:?}.localutm,dss${USS_NODE_IDX:?}.uss${USS_IDX:?}.localutm
+      DATASTORE_HOST: db${USS_NODE_IDX:?}.uss${USS_IDX:?}.localutm
     healthcheck:
       test: wget -O - 'http://localhost/healthy' || exit 1
       interval: 3m
       start_period: 30s
       start_interval: 5s
 
   oauth:
-    hostname: oauth.authority.localutm
     image: interuss/dummy-oauth
     command: -private_key_file /var/test-certs/auth2.key
     volumes:
       - $PWD/../test-certs:/var/test-certs:ro
+    profiles: [oauth]
+    hostname: oauth.authority.localutm
+    networks:
+      - interop_ecosystem_network
     expose:
       - 8085
     ports:
       - "8085:8085"
-    networks:
-      - interop_ecosystem_network
     healthcheck:
       test: wget -O - 'http://localhost:8085/token?intended_audience=-&scope=-' || exit 1
       interval: 3m
@@ -162,8 +179,10 @@ services:
 networks:
   dss_internal_network:
     name: dss_internal_network
+    external: true
   interop_ecosystem_network:
     name: interop_ecosystem_network
+    external: true
 
 volumes:
   dss_component_coordination:
diff --git a/build/dev/run_locally.sh b/build/dev/run_locally.sh
@@ -2,8 +2,13 @@
 
 set -eo pipefail
 
-# This script will deploy an interoperability ecosystem consisting of a standalone DSS instance and dummy OAuth server
-# (both accessible on the interop_ecosystem_network) with docker compose using the DSS image from Docker Hub.
+# This script will deploy an interoperability ecosystem consisting of a chosen number of DSS instances and a dummy OAuth
+# server (all accessible on the interop_ecosystem_network) with docker compose using the DSS image from Docker Hub.
+# Run `./run_locally.sh up -d` to start a single DSS instance using CockroachDB.
+# The following environment variables may be used:
+# NUM_USS: number of USSs
+# NUM_NODES: number of nodes per USS
+# DB_TYPE: crdb or ybdb
 
 if [[ -z $(command -v docker) ]]; then
   echo "docker is required but not installed.  Visit https://docs.docker.com/install/ to install."
@@ -20,6 +25,10 @@ fi
 
 cd "${BASEDIR}" || exit 1
 
+NUM_USS=${NUM_USS:-2}
+NUM_NODES=${NUM_NODES:-1}
+DB_TYPE=${DB_TYPE:-crdb}
+
 DC_COMMAND=$*
 
 if [[ ! "$DC_COMMAND" ]]; then
@@ -32,5 +41,39 @@ elif [[ "$DC_COMMAND" == "debug" ]]; then
   export DEBUG_ON=1
 fi
 
-# shellcheck disable=SC2086
-docker compose -f docker-compose.yaml -p local_infra $DC_COMMAND $DC_OPTIONS
+if [[ "$DC_COMMAND" == up* ]]; then
+  echo "Creating networks..."
+  docker network create --subnet=172.27.0.0/16 \
+                        --ip-range=172.27.0.0/24 \
+                        --gateway=172.27.0.1 \
+                        dss_internal_network || true
+  docker network create interop_ecosystem_network || true
+  echo "Starting containers..."
+fi
+
+for ((i=1; i<=NUM_USS; i++)); do
+  for ((j=1; j<=NUM_NODES; j++)); do
+    export USS_IDX=$i
+    export USS_NODE_IDX=$j
+    PADDED_NODE_IDX=$(printf "%02d" $(( (i-1) * NUM_NODES + j)))
+    export PADDED_NODE_IDX
+
+    export COMPOSE_PROFILES=${DB_TYPE}
+    if [ "$i" -eq 1 ] && [ "$j" -eq 1 ]; then
+      export COMPOSE_PROFILES=${COMPOSE_PROFILES},oauth
+    fi
+    if [ "$i" -eq "$NUM_USS" ] && [ "$j" -eq "$NUM_NODES" ]; then
+      export COMPOSE_PROFILES=${COMPOSE_PROFILES},bootstrap-${DB_TYPE}
+    fi
+
+    # shellcheck disable=SC2086
+    docker compose -f docker-compose.yaml -p "local_infra_${USS_IDX}-${USS_NODE_IDX}" $DC_COMMAND $DC_OPTIONS &
+  done
+done
+wait
+
+if [[ "$DC_COMMAND" == "down" ]]; then
+  echo "Removing networks..."
+  docker network rm dss_internal_network || true
+  docker network rm interop_ecosystem_network || true
+fi
diff --git a/build/dev/startup/core_service.sh b/build/dev/startup/core_service.sh
@@ -6,14 +6,15 @@ set -e
 # started by docker-compose.yaml, not on a local system.
 
 DEBUG_ON=${1:-0}
+JWT_AUDIENCES="localhost,host.docker.internal,${JWT_AUDIENCES}"
 
-# POSIX compliant test to check if with-yugabyte profile is enabled.
-if [ "${COMPOSE_PROFILES#*"with-yugabyte"}" != "${COMPOSE_PROFILES}" ]; then
+# POSIX compliant test to check if ybdb profile is enabled.
+if [ "${COMPOSE_PROFILES#*"ybdb"}" != "${COMPOSE_PROFILES}" ]; then
   echo "Using Yugabyte"
-  DATASTORE_CONNECTION="-datastore_host ybdb -datastore_user yugabyte --datastore_port 5433"
+  DATASTORE_CONNECTION="-datastore_host ${DATASTORE_HOST} -datastore_user yugabyte --datastore_port 5433"
 else
   echo "Using CockroachDB"
-  DATASTORE_CONNECTION="-datastore_host crdb.uss1.localutm"
+  DATASTORE_CONNECTION="-datastore_host ${DATASTORE_HOST}"
 fi
 
 if [ "$DEBUG_ON" = "1" ]; then
@@ -27,7 +28,7 @@ if [ "$DEBUG_ON" = "1" ]; then
   -log_format console \
   -dump_requests \
   -addr :80 \
-  -accepted_jwt_audiences localhost,host.docker.internal,dss.uss1.localutm,dss.uss2.localutm \
+  -accepted_jwt_audiences ${JWT_AUDIENCES} \
   -enable_scd \
   -allow_http_base_urls \
   -locality local_dev \
@@ -43,7 +44,7 @@ else
   -log_format console \
   -dump_requests \
   -addr :80 \
-  -accepted_jwt_audiences localhost,host.docker.internal,dss.uss1.localutm,dss.uss2.localutm \
+  -accepted_jwt_audiences ${JWT_AUDIENCES} \
   -enable_scd \
   -allow_http_base_urls \
   -locality local_dev \

diff --git a/monitoring/prober/run_locally.sh b/monitoring/prober/run_locally.sh
@@ -18,8 +18,8 @@ cd monitoring || exit 1
 make image
 )
 
-CORE_SERVICE_CONTAINER="local_infra-dss-1"
-OAUTH_CONTAINER="local_infra-oauth-1"
+CORE_SERVICE_CONTAINER="local_infra_1-1-dss-1"
+OAUTH_CONTAINER="local_infra_1-1-oauth-1"
 declare -a localhost_containers=("$CORE_SERVICE_CONTAINER" "$OAUTH_CONTAINER")
 
 for container_name in "${localhost_containers[@]}"; do