diff --git a/build/dev/docker-compose.yaml b/build/dev/docker-compose.yaml index 9d492948fe..5f89ae47c5 100644 --- a/build/dev/docker-compose.yaml +++ b/build/dev/docker-compose.yaml @@ -1,22 +1,19 @@ -# Brings up a local interoperability ecosystem consisting of a DSS instance and dummy OAuth server. - -# To bring up this system, run ./run_locally.sh up -d and wait for all containers to succeed or become healthy. -# If needed, set environment variable `COMPOSE_PROFILES` to `with-yugabyte` to use Yugabyte instead of CockroachDB +# This Docker Compose file must be used through `./run_locally.sh`. services: crdb: - hostname: crdb.uss1.localutm image: cockroachdb/cockroach:v24.1.3 - command: start-single-node --insecure - expose: - - 26257 - ports: - - "8080:8080" - - "26257:26257" + command: start --insecure --join=db1.uss1.localutm + profiles: [crdb] restart: always + hostname: db${USS_NODE_IDX:?}.uss${USS_IDX:?}.localutm networks: - - dss_internal_network + dss_internal_network: + ipv4_address: 172.27.${USS_IDX:?}.${USS_NODE_IDX:?} + ports: + - "81${PADDED_NODE_IDX:?}:8080" + - "262${PADDED_NODE_IDX:?}:26257" healthcheck: test: curl -f 'http://localhost:8080/health?ready=1' || exit 1 interval: 3m @@ -26,115 +23,134 @@ services: ybdb: image: interuss/yugabyte:2025.1.2.1-interuss # ysql_output_buffer_size needs to be increased to allow ysql to retry read restart errors. https://docs.yugabyte.com/preview/reference/configuration/yb-tserver/#ysql-output-buffer-size - command: bin/yugabyted start --background=false --tserver_flags="ysql_output_buffer_size=1048576" - ports: - - "7000:7000" - - "9000:9000" - - "15433:15433" - - "5433:5433" - - "9042:9042" + command: sh -c "bin/yugabyted start --background=false --tserver_flags=ysql_output_buffer_size=1048576 --advertise_address=$$YBDB_HOST --join=$$([ "$$YBDB_HOST" = db1.uss1.localutm ] || echo db1.uss1.localutm)" + profiles: [ybdb] restart: always + hostname: db${USS_NODE_IDX:?}.uss${USS_IDX:?}.localutm networks: - - dss_internal_network - profiles: ["with-yugabyte"] + dss_internal_network: + ipv4_address: 172.27.${USS_IDX:?}.${USS_NODE_IDX:?} + ports: + - "70${PADDED_NODE_IDX:?}:7000" + - "90${PADDED_NODE_IDX:?}:9000" + - "154${PADDED_NODE_IDX:?}:15433" + - "54${PADDED_NODE_IDX:?}:5433" + - "91${PADDED_NODE_IDX:?}:9042" + environment: + - YBDB_HOST=db${USS_NODE_IDX:?}.uss${USS_IDX:?}.localutm healthcheck: - test: ["CMD", "/home/yugabyte/postgres/bin/pg_isready", "-h", "ybdb"] + test: /home/yugabyte/postgres/bin/pg_isready -h $$YBDB_HOST || exit 1 interval: 5s timeout: 5s retries: 10 + crdb-init: + image: cockroachdb/cockroach:v24.1.3 + profiles: [bootstrap-crdb] + depends_on: [crdb] + entrypoint: > + sh -c " + out=$$(/cockroach/cockroach.sh init --insecure --host=db1.uss1.localutm 2>&1); + if [ $$? -eq 0 ]; then + echo "$$out"; + exit 0; + fi; + echo "$$out"; + if echo \"$$out\" | grep -q \"already been initialized\"; then + exit 0; + fi; + exit 1; + " + networks: [dss_internal_network] + rid_bootstrapper: image: interuss/dss:v0.21.0-rc1 - command: /usr/bin/db-manager migrate --schemas_dir=/db-schemas/rid --db_version "latest" --cockroach_host crdb + profiles: [bootstrap-crdb] + entrypoint: sh -c "/usr/bin/db-manager migrate --schemas_dir=/db-schemas/$$BOOTSTRAP_DB --db_version latest --cockroach_host db1.uss1.localutm" + environment: + - BOOTSTRAP_DB=rid depends_on: - crdb: - condition: service_healthy - networks: - - dss_internal_network + crdb-init: + condition: service_completed_successfully + networks: [dss_internal_network] + healthcheck: + disable: true scd_bootstrapper: - image: interuss/dss:v0.21.0-rc1 - command: /usr/bin/db-manager migrate --schemas_dir=/db-schemas/scd --db_version "latest" --cockroach_host crdb - depends_on: - crdb: - condition: service_healthy - networks: - - dss_internal_network + extends: + service: rid_bootstrapper + environment: + - BOOTSTRAP_DB=scd aux_bootstrapper: - image: interuss/dss:v0.21.0-rc1 - command: /usr/bin/db-manager migrate --schemas_dir=/db-schemas/aux_ --db_version "latest" --cockroach_host crdb - depends_on: - crdb: - condition: service_healthy - networks: - - dss_internal_network + extends: + service: rid_bootstrapper + environment: + - BOOTSTRAP_DB=aux_ rid_bootstrapper-ybdb: image: interuss/dss:v0.21.0-rc1 - entrypoint: /usr/bin/db-manager migrate --schemas_dir=/db-schemas/yugabyte/rid --db_version "latest" --datastore_host ybdb --datastore_user yugabyte --datastore_port 5433 - depends_on: - ybdb: - condition: service_healthy - networks: - - dss_internal_network - profiles: ["with-yugabyte"] + profiles: [bootstrap-ybdb] + entrypoint: sh -c "/usr/bin/db-manager migrate --schemas_dir=/db-schemas/yugabyte/$$BOOTSTRAP_DB --db_version latest --datastore_host db1.uss1.localutm --datastore_user yugabyte --datastore_port 5433" + environment: + - BOOTSTRAP_DB=rid + restart: on-failure + networks: [dss_internal_network] + healthcheck: + disable: true scd_bootstrapper-ybdb: - image: interuss/dss:v0.21.0-rc1 - entrypoint: /usr/bin/db-manager migrate --schemas_dir=/db-schemas/yugabyte/scd --db_version "latest" --datastore_host ybdb --datastore_user yugabyte --datastore_port 5433 - depends_on: - ybdb: - condition: service_healthy - networks: - - dss_internal_network - profiles: ["with-yugabyte"] + extends: + service: rid_bootstrapper-ybdb + environment: + - BOOTSTRAP_DB=scd aux_bootstrapper-ybdb: - image: interuss/dss:v0.21.0-rc1 - entrypoint: /usr/bin/db-manager migrate --schemas_dir=/db-schemas/yugabyte/aux_ --db_version "latest" --datastore_host ybdb --datastore_user yugabyte --datastore_port 5433 - depends_on: - ybdb: - condition: service_healthy - networks: - - dss_internal_network - profiles: ["with-yugabyte"] + extends: + service: rid_bootstrapper-ybdb + environment: + - BOOTSTRAP_DB=aux_ dss: - hostname: dss.uss1.localutm image: interuss/dss:v0.21.0-rc1 + command: /startup/core_service.sh ${DEBUG_ON:-0} + profiles: [ crdb, ybdb ] + restart: always volumes: - $PWD/../test-certs:/var/test-certs:ro - $PWD/startup/core_service.sh:/startup/core_service.sh:ro - command: /startup/core_service.sh ${DEBUG_ON:-0} - environment: - COMPOSE_PROFILES: ${COMPOSE_PROFILES} - expose: - - 8082 - ports: - - "4000:4000" - - "8082:80" depends_on: rid_bootstrapper: condition: service_completed_successfully + required: false scd_bootstrapper: condition: service_completed_successfully + required: false aux_bootstrapper: condition: service_completed_successfully + required: false rid_bootstrapper-ybdb: condition: service_completed_successfully required: false scd_bootstrapper-ybdb: - required: false condition: service_completed_successfully + required: false aux_bootstrapper-ybdb: condition: service_completed_successfully required: false + hostname: dss${USS_NODE_IDX:?}.uss${USS_IDX:?}.localutm networks: dss_internal_network: {} interop_ecosystem_network: aliases: - - dss.uss2.localutm + - dss.uss${USS_IDX:?}.localutm + ports: + - "40${PADDED_NODE_IDX:?}:4000" + - "80${PADDED_NODE_IDX:?}:80" + environment: + COMPOSE_PROFILES: ${COMPOSE_PROFILES} + JWT_AUDIENCES: dss.uss${USS_IDX:?}.localutm,dss${USS_NODE_IDX:?}.uss${USS_IDX:?}.localutm + DATASTORE_HOST: db${USS_NODE_IDX:?}.uss${USS_IDX:?}.localutm healthcheck: test: wget -O - 'http://localhost/healthy' || exit 1 interval: 3m @@ -142,17 +158,18 @@ services: start_interval: 5s oauth: - hostname: oauth.authority.localutm image: interuss/dummy-oauth command: -private_key_file /var/test-certs/auth2.key volumes: - $PWD/../test-certs:/var/test-certs:ro + profiles: [oauth] + hostname: oauth.authority.localutm + networks: + - interop_ecosystem_network expose: - 8085 ports: - "8085:8085" - networks: - - interop_ecosystem_network healthcheck: test: wget -O - 'http://localhost:8085/token?intended_audience=-&scope=-' || exit 1 interval: 3m @@ -162,8 +179,10 @@ services: networks: dss_internal_network: name: dss_internal_network + external: true interop_ecosystem_network: name: interop_ecosystem_network + external: true volumes: dss_component_coordination: diff --git a/build/dev/run_locally.sh b/build/dev/run_locally.sh index a4c842a174..716f1e01e3 100755 --- a/build/dev/run_locally.sh +++ b/build/dev/run_locally.sh @@ -2,8 +2,13 @@ set -eo pipefail -# This script will deploy an interoperability ecosystem consisting of a standalone DSS instance and dummy OAuth server -# (both accessible on the interop_ecosystem_network) with docker compose using the DSS image from Docker Hub. +# This script will deploy an interoperability ecosystem consisting of a chosen number of DSS instances and a dummy OAuth +# server (all accessible on the interop_ecosystem_network) with docker compose using the DSS image from Docker Hub. +# Run `./run_locally.sh up -d` to start a single DSS instance using CockroachDB. +# The following environment variables may be used: +# NUM_USS: number of USSs +# NUM_NODES: number of nodes per USS +# DB_TYPE: crdb or ybdb if [[ -z $(command -v docker) ]]; then echo "docker is required but not installed. Visit https://docs.docker.com/install/ to install." @@ -20,6 +25,10 @@ fi cd "${BASEDIR}" || exit 1 +NUM_USS=${NUM_USS:-2} +NUM_NODES=${NUM_NODES:-1} +DB_TYPE=${DB_TYPE:-crdb} + DC_COMMAND=$* if [[ ! "$DC_COMMAND" ]]; then @@ -32,5 +41,39 @@ elif [[ "$DC_COMMAND" == "debug" ]]; then export DEBUG_ON=1 fi -# shellcheck disable=SC2086 -docker compose -f docker-compose.yaml -p local_infra $DC_COMMAND $DC_OPTIONS +if [[ "$DC_COMMAND" == up* ]]; then + echo "Creating networks..." + docker network create --subnet=172.27.0.0/16 \ + --ip-range=172.27.0.0/24 \ + --gateway=172.27.0.1 \ + dss_internal_network || true + docker network create interop_ecosystem_network || true + echo "Starting containers..." +fi + +for ((i=1; i<=NUM_USS; i++)); do + for ((j=1; j<=NUM_NODES; j++)); do + export USS_IDX=$i + export USS_NODE_IDX=$j + PADDED_NODE_IDX=$(printf "%02d" $(( (i-1) * NUM_NODES + j))) + export PADDED_NODE_IDX + + export COMPOSE_PROFILES=${DB_TYPE} + if [ "$i" -eq 1 ] && [ "$j" -eq 1 ]; then + export COMPOSE_PROFILES=${COMPOSE_PROFILES},oauth + fi + if [ "$i" -eq "$NUM_USS" ] && [ "$j" -eq "$NUM_NODES" ]; then + export COMPOSE_PROFILES=${COMPOSE_PROFILES},bootstrap-${DB_TYPE} + fi + + # shellcheck disable=SC2086 + docker compose -f docker-compose.yaml -p "local_infra_${USS_IDX}-${USS_NODE_IDX}" $DC_COMMAND $DC_OPTIONS & + done +done +wait + +if [[ "$DC_COMMAND" == "down" ]]; then + echo "Removing networks..." + docker network rm dss_internal_network || true + docker network rm interop_ecosystem_network || true +fi diff --git a/build/dev/startup/core_service.sh b/build/dev/startup/core_service.sh index 6c1da4c36a..085ff8317e 100755 --- a/build/dev/startup/core_service.sh +++ b/build/dev/startup/core_service.sh @@ -6,14 +6,15 @@ set -e # started by docker-compose.yaml, not on a local system. DEBUG_ON=${1:-0} +JWT_AUDIENCES="localhost,host.docker.internal,${JWT_AUDIENCES}" -# POSIX compliant test to check if with-yugabyte profile is enabled. -if [ "${COMPOSE_PROFILES#*"with-yugabyte"}" != "${COMPOSE_PROFILES}" ]; then +# POSIX compliant test to check if ybdb profile is enabled. +if [ "${COMPOSE_PROFILES#*"ybdb"}" != "${COMPOSE_PROFILES}" ]; then echo "Using Yugabyte" - DATASTORE_CONNECTION="-datastore_host ybdb -datastore_user yugabyte --datastore_port 5433" + DATASTORE_CONNECTION="-datastore_host ${DATASTORE_HOST} -datastore_user yugabyte --datastore_port 5433" else echo "Using CockroachDB" - DATASTORE_CONNECTION="-datastore_host crdb.uss1.localutm" + DATASTORE_CONNECTION="-datastore_host ${DATASTORE_HOST}" fi if [ "$DEBUG_ON" = "1" ]; then @@ -27,7 +28,7 @@ if [ "$DEBUG_ON" = "1" ]; then -log_format console \ -dump_requests \ -addr :80 \ - -accepted_jwt_audiences localhost,host.docker.internal,dss.uss1.localutm,dss.uss2.localutm \ + -accepted_jwt_audiences ${JWT_AUDIENCES} \ -enable_scd \ -allow_http_base_urls \ -locality local_dev \ @@ -43,7 +44,7 @@ else -log_format console \ -dump_requests \ -addr :80 \ - -accepted_jwt_audiences localhost,host.docker.internal,dss.uss1.localutm,dss.uss2.localutm \ + -accepted_jwt_audiences ${JWT_AUDIENCES} \ -enable_scd \ -allow_http_base_urls \ -locality local_dev \ diff --git a/monitoring/prober/run_locally.sh b/monitoring/prober/run_locally.sh index f756e8e1d9..efd3e3c9a5 100755 --- a/monitoring/prober/run_locally.sh +++ b/monitoring/prober/run_locally.sh @@ -18,8 +18,8 @@ cd monitoring || exit 1 make image ) -CORE_SERVICE_CONTAINER="local_infra-dss-1" -OAUTH_CONTAINER="local_infra-oauth-1" +CORE_SERVICE_CONTAINER="local_infra_1-1-dss-1" +OAUTH_CONTAINER="local_infra_1-1-oauth-1" declare -a localhost_containers=("$CORE_SERVICE_CONTAINER" "$OAUTH_CONTAINER") for container_name in "${localhost_containers[@]}"; do