From dd0a3f41f2f3d362e68bc9e0924bf718b8f6bff5 Mon Sep 17 00:00:00 2001 From: Jake Barnes Date: Mon, 11 Nov 2019 14:52:19 +1100 Subject: [PATCH 001/253] Remove restriction on build and run --- hooks/command | 8 -------- 1 file changed, 8 deletions(-) diff --git a/hooks/command b/hooks/command index 8e0692b1..41c857ca 100755 --- a/hooks/command +++ b/hooks/command @@ -12,14 +12,6 @@ commands=() [[ -n "$(plugin_read_list BUILD)" ]] && commands+=("BUILD") [[ -n "$(plugin_read_list RUN)" ]] && commands+=("RUN") - -# Check we've only got one of BUILD or RUN -if [[ ${#commands[@]} -gt 1 ]] ; then - echo "+++ Docker Compose plugin error" - echo "Only one of build or run is supported. More than one was used." - exit 1 -fi - [[ -n "$(plugin_read_list PUSH)" ]] && commands+=("PUSH") # Don't convert paths on gitbash on windows From 1c01b5c859085d2b3e5ac9f37ed6a03bc5210ebe Mon Sep 17 00:00:00 2001 From: Jonathan Chu Date: Tue, 11 Aug 2020 07:20:49 -0700 Subject: [PATCH 002/253] add support for mount-buildkite-agent (copied from docker-buildkite-plugin) --- commands/run.sh | 22 ++++++++++++++++++++++ plugin.yml | 3 +++ 2 files changed, 25 insertions(+) diff --git a/commands/run.sh b/commands/run.sh index f3dfc5ff..1cb820cc 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -155,6 +155,28 @@ if [[ -n "$(plugin_read_config ENTRYPOINT)" ]] ; then run_params+=("--entrypoint \"$(plugin_read_config ENTRYPOINT)\"") fi +# Optionally handle the mount-buildkite-agent option +if [[ "$(plugin_read_config MOUNT_BUILDKITE_AGENT "false")" == "true" ]]; then + if [[ -z "${BUILDKITE_AGENT_BINARY_PATH:-}" ]] ; then + if ! command -v buildkite-agent >/dev/null 2>&1 ; then + echo -n "+++ 🚨 Failed to find buildkite-agent in PATH to mount into container, " + echo "you can disable this behaviour with 'mount-buildkite-agent:false'" + else + BUILDKITE_AGENT_BINARY_PATH=$(command -v buildkite-agent) + fi + fi +fi + +# Mount buildkite-agent if we have a path for it +if [[ -n "${BUILDKITE_AGENT_BINARY_PATH:-}" ]] ; then + run_params+=( + "--env" "BUILDKITE_JOB_ID" + "--env" "BUILDKITE_BUILD_ID" + "--env" "BUILDKITE_AGENT_ACCESS_TOKEN" + "--volume" "$BUILDKITE_AGENT_BINARY_PATH:/usr/bin/buildkite-agent" + ) +fi + run_params+=("$run_service") build_params=(--pull) diff --git a/plugin.yml b/plugin.yml index 39db0041..b6ed39b0 100644 --- a/plugin.yml +++ b/plugin.yml @@ -72,6 +72,8 @@ configuration: type: string propagate-uid-gid: type: boolean + mount-buildkite-agent: + type: boolean oneOf: - required: - run @@ -97,3 +99,4 @@ configuration: workdir: [ run ] user: [ run ] propagate-uid-gid: [ run ] + mount-buildkite-agent: [ run ] From 76a5f4dc4193f49c3fab298072ccc9da151f05ea Mon Sep 17 00:00:00 2001 From: Jonathan Chu Date: Tue, 11 Aug 2020 07:34:14 -0700 Subject: [PATCH 003/253] log full command --- commands/run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/commands/run.sh b/commands/run.sh index 1cb820cc..371561c2 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -320,7 +320,8 @@ set -e if [[ $exitcode -ne 0 ]] ; then echo "^^^ +++" - echo "+++ :warning: Failed to run command, exited with $exitcode" + echo "+++ :warning: Failed to run command, exited with $exitcode, run params:" + echo "${run_params[@]}" fi if [[ -n "${BUILDKITE_AGENT_ACCESS_TOKEN:-}" ]] ; then From a47a27bbb13c5752707e6525cc82dc615726c3cf Mon Sep 17 00:00:00 2001 From: Jonathan Chu Date: Tue, 11 Aug 2020 08:00:28 -0700 Subject: [PATCH 004/253] corrected params from docker to docker-compose --- commands/run.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/commands/run.sh b/commands/run.sh index 371561c2..026c6a72 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -170,10 +170,10 @@ fi # Mount buildkite-agent if we have a path for it if [[ -n "${BUILDKITE_AGENT_BINARY_PATH:-}" ]] ; then run_params+=( - "--env" "BUILDKITE_JOB_ID" - "--env" "BUILDKITE_BUILD_ID" - "--env" "BUILDKITE_AGENT_ACCESS_TOKEN" - "--volume" "$BUILDKITE_AGENT_BINARY_PATH:/usr/bin/buildkite-agent" + "-e" "BUILDKITE_JOB_ID" + "-e" "BUILDKITE_BUILD_ID" + "-e" "BUILDKITE_AGENT_ACCESS_TOKEN" + "-v" "$BUILDKITE_AGENT_BINARY_PATH:/usr/bin/buildkite-agent" ) fi From c83de3aef4fde773720ba058ebf0cb8b6344e53d Mon Sep 17 00:00:00 2001 From: Jonathan Chu Date: Tue, 25 Aug 2020 11:18:54 -0700 Subject: [PATCH 005/253] readme --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index bee93445..a645632b 100644 --- a/README.md +++ b/README.md @@ -359,6 +359,12 @@ Whether to match the user ID and group ID for the container user to the user ID Using this option ensures that any files created on shared mounts from within the container will be accessible to the host user. It is otherwise common to accidentally create root-owned files that Buildkite will be unable to remove, since containers by default run as the root user. +### `mount-buildkite-agent` (optional, run-only, boolean) + +Whether to automatically mount the `buildkite-agent` binary and associated environment variables from the host agent machine into the container. + +Default: `false` + ### `pull-retries` (optional) A number of times to retry failed docker pull. Defaults to 0. From 4fbfc3bf9a7e3b0a34e10284d58364478f720407 Mon Sep 17 00:00:00 2001 From: Jonathan Chu Date: Wed, 16 Sep 2020 09:51:41 -0700 Subject: [PATCH 006/253] cherry picked tests from @toolmantim --- tests/run.bats | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/run.bats b/tests/run.bats index 9541358c..32cb5767 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -803,6 +803,33 @@ export BUILDKITE_JOB_ID=1111 unstub buildkite-agent } +@test "Run with mount-buildkite-agent enabled" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_MOUNT_BUILDKITE_AGENT=true + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm -e BUILDKITE_JOB_ID -e BUILDKITE_BUILD_ID -e BUILDKITE_AGENT_ACCESS_TOKEN -v $BATS_MOCK_TMPDIR/bin/buildkite-agent:/usr/bin/buildkite-agent myservice : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "ran myservice" + unstub docker-compose + unstub buildkite-agent +} + @test "Run with various build arguments" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice From fd806202fc864341fabfc338b5c2aca5533f8b9d Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Mon, 28 Sep 2020 15:28:12 +1000 Subject: [PATCH 007/253] Support buildkite-agent's git-mirrors; mount BUILDKITE_REPO_MIRROR When BUILDKITE_REPO_MIRROR is present, that path is automatically mounted to the same location within the container. This makes repos created with `git clone --reference $BUILDKITE_REPO_MIRROR` accessible within the docker container. --- commands/run.sh | 5 +++++ tests/run.bats | 29 ++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/commands/run.sh b/commands/run.sh index 0a8c3a27..a9bb70e4 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -103,6 +103,11 @@ for vol in "${default_volumes[@]:-}" ; do [[ -n "$trimmed_vol" ]] && run_params+=("-v" "$(expand_relative_volume_path "$trimmed_vol")") done +# If there's a git mirror, mount it so that git references can be followed. +if [[ -n "${BUILDKITE_REPO_MIRROR:-}" ]]; then + run_params+=("-v" "$BUILDKITE_REPO_MIRROR:$BUILDKITE_REPO_MIRROR:ro") +fi + tty_default='true' # Set operating system specific defaults diff --git a/tests/run.bats b/tests/run.bats index dde52423..d03bf1e2 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -856,4 +856,31 @@ export BUILDKITE_JOB_ID=1111 assert_output --partial "ran myservice" unstub docker-compose unstub buildkite-agent -} \ No newline at end of file +} + +@test "Run with git-mirrors" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="echo hello world" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_REPO_MIRROR=/tmp/sample-mirror + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 -v /tmp/sample-mirror:/tmp/sample-mirror:ro --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "ran myservice" + unstub docker-compose + unstub buildkite-agent +} From 2f44a9c309336df0558d21b67770f5f89f4002fa Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Mon, 28 Sep 2020 17:18:00 +1000 Subject: [PATCH 008/253] README examples bumped to upcoming v3.7.0 --- README.md | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index a645632b..117951d2 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v3.5.0: + - docker-compose#v3.7.0: run: app ``` @@ -26,7 +26,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v3.5.0: + - docker-compose#v3.7.0: run: app config: docker-compose.tests.yml env: @@ -39,7 +39,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v3.5.0: + - docker-compose#v3.7.0: run: app config: - docker-compose.yml @@ -53,7 +53,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.5.0: + - docker-compose#v3.7.0: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -61,7 +61,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.5.0: + - docker-compose#v3.7.0: run: app ``` @@ -70,7 +70,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v3.5.0: + - docker-compose#v3.7.0: run: app command: ["custom", "command", "values"] ``` @@ -86,7 +86,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.5.0: + - docker-compose#v3.7.0: run: app ``` @@ -104,7 +104,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.5.0: + - docker-compose#v3.7.0: run: app volumes: - "./dist:/app/dist" @@ -125,7 +125,7 @@ this plugin offers a `environment` block of it's own: steps: - command: generate-dist.sh plugins: - - docker-compose#v3.5.0: + - docker-compose#v3.7.0: run: app env: - BUILDKITE_BUILD_NUMBER @@ -145,7 +145,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v3.5.0: + - docker-compose#v3.7.0: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -162,7 +162,7 @@ To speed up run steps that use the same service/image (such as steps that run in steps: - label: ":docker: Build" plugins: - - docker-compose#v3.5.0: + - docker-compose#v3.7.0: build: app image-repository: index.docker.io/myorg/myrepo @@ -172,7 +172,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.5.0: + - docker-compose#v3.7.0: run: app ``` @@ -188,7 +188,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v3.5.0: + - docker-compose#v3.7.0: build: - app - tests @@ -200,7 +200,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.5.0: + - docker-compose#v3.7.0: run: tests ``` @@ -212,7 +212,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v3.5.0: + - docker-compose#v3.7.0: push: app ``` @@ -224,7 +224,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.5.0: + - docker-compose#v3.7.0: push: app ``` @@ -236,7 +236,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.5.0: + - docker-compose#v3.7.0: push: - first-service - second-service @@ -250,7 +250,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.5.0: + - docker-compose#v3.7.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -264,14 +264,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker Build an image" plugins: - - docker-compose#v3.5.0: + - docker-compose#v3.7.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v3.5.0: + - docker-compose#v3.7.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest From bcb700d69be28a8044e025f49c78ab6dec7edc9e Mon Sep 17 00:00:00 2001 From: "boomper-bot[bot]" <40539397+boomper-bot[bot]@users.noreply.github.com> Date: Tue, 29 Sep 2020 06:45:48 +0000 Subject: [PATCH 009/253] Bump README.md for v3.6.0 release --- README.md | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 117951d2..3a137b18 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.6.0: run: app ``` @@ -26,7 +26,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.6.0: run: app config: docker-compose.tests.yml env: @@ -39,7 +39,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.6.0: run: app config: - docker-compose.yml @@ -53,7 +53,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.7.0: + - docker-compose#v3.6.0: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -61,7 +61,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.7.0: + - docker-compose#v3.6.0: run: app ``` @@ -70,7 +70,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.6.0: run: app command: ["custom", "command", "values"] ``` @@ -86,7 +86,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.6.0: run: app ``` @@ -104,7 +104,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.6.0: run: app volumes: - "./dist:/app/dist" @@ -125,7 +125,7 @@ this plugin offers a `environment` block of it's own: steps: - command: generate-dist.sh plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.6.0: run: app env: - BUILDKITE_BUILD_NUMBER @@ -145,7 +145,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.6.0: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -162,7 +162,7 @@ To speed up run steps that use the same service/image (such as steps that run in steps: - label: ":docker: Build" plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.6.0: build: app image-repository: index.docker.io/myorg/myrepo @@ -172,7 +172,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.6.0: run: app ``` @@ -188,7 +188,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.6.0: build: - app - tests @@ -200,7 +200,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.6.0: run: tests ``` @@ -212,7 +212,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.6.0: push: app ``` @@ -224,7 +224,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.7.0: + - docker-compose#v3.6.0: push: app ``` @@ -236,7 +236,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.7.0: + - docker-compose#v3.6.0: push: - first-service - second-service @@ -250,7 +250,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.7.0: + - docker-compose#v3.6.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -264,14 +264,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker Build an image" plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.6.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.6.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest From 07607ad023752fc409b0a2dd5af7feafb265271a Mon Sep 17 00:00:00 2001 From: "boomper-bot[bot]" <40539397+boomper-bot[bot]@users.noreply.github.com> Date: Tue, 29 Sep 2020 06:50:35 +0000 Subject: [PATCH 010/253] Bump README.md for v3.7.0 release --- README.md | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 3a137b18..117951d2 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v3.6.0: + - docker-compose#v3.7.0: run: app ``` @@ -26,7 +26,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v3.6.0: + - docker-compose#v3.7.0: run: app config: docker-compose.tests.yml env: @@ -39,7 +39,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v3.6.0: + - docker-compose#v3.7.0: run: app config: - docker-compose.yml @@ -53,7 +53,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.6.0: + - docker-compose#v3.7.0: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -61,7 +61,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.6.0: + - docker-compose#v3.7.0: run: app ``` @@ -70,7 +70,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v3.6.0: + - docker-compose#v3.7.0: run: app command: ["custom", "command", "values"] ``` @@ -86,7 +86,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.6.0: + - docker-compose#v3.7.0: run: app ``` @@ -104,7 +104,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.6.0: + - docker-compose#v3.7.0: run: app volumes: - "./dist:/app/dist" @@ -125,7 +125,7 @@ this plugin offers a `environment` block of it's own: steps: - command: generate-dist.sh plugins: - - docker-compose#v3.6.0: + - docker-compose#v3.7.0: run: app env: - BUILDKITE_BUILD_NUMBER @@ -145,7 +145,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v3.6.0: + - docker-compose#v3.7.0: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -162,7 +162,7 @@ To speed up run steps that use the same service/image (such as steps that run in steps: - label: ":docker: Build" plugins: - - docker-compose#v3.6.0: + - docker-compose#v3.7.0: build: app image-repository: index.docker.io/myorg/myrepo @@ -172,7 +172,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.6.0: + - docker-compose#v3.7.0: run: app ``` @@ -188,7 +188,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v3.6.0: + - docker-compose#v3.7.0: build: - app - tests @@ -200,7 +200,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.6.0: + - docker-compose#v3.7.0: run: tests ``` @@ -212,7 +212,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v3.6.0: + - docker-compose#v3.7.0: push: app ``` @@ -224,7 +224,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.6.0: + - docker-compose#v3.7.0: push: app ``` @@ -236,7 +236,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.6.0: + - docker-compose#v3.7.0: push: - first-service - second-service @@ -250,7 +250,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.6.0: + - docker-compose#v3.7.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -264,14 +264,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker Build an image" plugins: - - docker-compose#v3.6.0: + - docker-compose#v3.7.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v3.6.0: + - docker-compose#v3.7.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest From 9fd69c4090b1256107007e0d7cec14971a4cf5f9 Mon Sep 17 00:00:00 2001 From: Mark Allen Date: Tue, 13 Oct 2020 17:54:13 -0400 Subject: [PATCH 011/253] Fix missing colon typo in :docker: --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 117951d2..60938c02 100644 --- a/README.md +++ b/README.md @@ -262,7 +262,7 @@ A newly spawned agent won't contain any of the docker caches for the first run w ```yaml steps: - - label: ":docker Build an image" + - label: ":docker: Build an image" plugins: - docker-compose#v3.7.0: build: app From 9faddab0942fa3da98181dadd26b7e56909832c0 Mon Sep 17 00:00:00 2001 From: Evan Van Dam Date: Fri, 2 Apr 2021 16:57:03 -0700 Subject: [PATCH 012/253] Add entrypoint property to plugin.yml --- plugin.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugin.yml b/plugin.yml index b6ed39b0..334a52ee 100644 --- a/plugin.yml +++ b/plugin.yml @@ -74,6 +74,8 @@ configuration: type: boolean mount-buildkite-agent: type: boolean + entrypoint: + type: [ string, array ] oneOf: - required: - run From 8eb0095ca79dc29ce0769346e1489d8b792d2d5f Mon Sep 17 00:00:00 2001 From: Evan Van Dam Date: Fri, 2 Apr 2021 17:10:45 -0700 Subject: [PATCH 013/253] make entrypoint a string --- plugin.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin.yml b/plugin.yml index 334a52ee..50169921 100644 --- a/plugin.yml +++ b/plugin.yml @@ -75,7 +75,7 @@ configuration: mount-buildkite-agent: type: boolean entrypoint: - type: [ string, array ] + type: string oneOf: - required: - run From 226f3b34415239fae7dc5d04dd9a2c693522f55a Mon Sep 17 00:00:00 2001 From: "boomper-bot[bot]" <40539397+boomper-bot[bot]@users.noreply.github.com> Date: Tue, 20 Jul 2021 02:42:03 +0000 Subject: [PATCH 014/253] Bump README.md for v3.8.0 release --- README.md | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 60938c02..d5bf116a 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.8.0: run: app ``` @@ -26,7 +26,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.8.0: run: app config: docker-compose.tests.yml env: @@ -39,7 +39,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.8.0: run: app config: - docker-compose.yml @@ -53,7 +53,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.7.0: + - docker-compose#v3.8.0: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -61,7 +61,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.7.0: + - docker-compose#v3.8.0: run: app ``` @@ -70,7 +70,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.8.0: run: app command: ["custom", "command", "values"] ``` @@ -86,7 +86,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.8.0: run: app ``` @@ -104,7 +104,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.8.0: run: app volumes: - "./dist:/app/dist" @@ -125,7 +125,7 @@ this plugin offers a `environment` block of it's own: steps: - command: generate-dist.sh plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.8.0: run: app env: - BUILDKITE_BUILD_NUMBER @@ -145,7 +145,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.8.0: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -162,7 +162,7 @@ To speed up run steps that use the same service/image (such as steps that run in steps: - label: ":docker: Build" plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.8.0: build: app image-repository: index.docker.io/myorg/myrepo @@ -172,7 +172,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.8.0: run: app ``` @@ -188,7 +188,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.8.0: build: - app - tests @@ -200,7 +200,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.8.0: run: tests ``` @@ -212,7 +212,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.8.0: push: app ``` @@ -224,7 +224,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.7.0: + - docker-compose#v3.8.0: push: app ``` @@ -236,7 +236,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.7.0: + - docker-compose#v3.8.0: push: - first-service - second-service @@ -250,7 +250,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.7.0: + - docker-compose#v3.8.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -264,14 +264,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker: Build an image" plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.8.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v3.7.0: + - docker-compose#v3.8.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest From 20634b794ceb2679457b8a6ce4ecb20e9221ab25 Mon Sep 17 00:00:00 2001 From: patrick-vanstee Date: Tue, 3 Aug 2021 13:38:39 -0400 Subject: [PATCH 015/253] Support omitting compose file version --- lib/shared.bash | 9 ++++++--- tests/composefiles/docker-compose.v1.0.yml | 3 ++- tests/docker-compose-config.bats | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/shared.bash b/lib/shared.bash index 375cae28..53c1bbdf 100644 --- a/lib/shared.bash +++ b/lib/shared.bash @@ -128,7 +128,7 @@ function build_image_override_file() { # Checks that a specific version of docker-compose supports cache_from function docker_compose_supports_cache_from() { local version="$1" - if [[ -z "$version" || "$version" == 1* || "$version" =~ ^(2|3)(\.[01])?$ ]] ; then + if [[ "$version" == 1* || "$version" =~ ^(2|3)(\.[01])?$ ]] ; then return 1 fi } @@ -138,14 +138,17 @@ function docker_compose_supports_cache_from() { function build_image_override_file_with_version() { local version="$1" - if [[ -z "$version" ]]; then + if [[ "$version" == 1* ]] ; then echo "The 'build' option can only be used with Compose file versions 2.0 and above." echo "For more information on Docker Compose configuration file versions, see:" echo "https://docs.docker.com/compose/compose-file/compose-versioning/#versioning" exit 1 fi - printf "version: '%s'\\n" "$version" + if [[ -n "$version" ]]; then + printf "version: '%s'\\n" "$version" + fi + printf "services:\\n" shift diff --git a/tests/composefiles/docker-compose.v1.0.yml b/tests/composefiles/docker-compose.v1.0.yml index 94500c85..4f343829 100644 --- a/tests/composefiles/docker-compose.v1.0.yml +++ b/tests/composefiles/docker-compose.v1.0.yml @@ -1,2 +1,3 @@ helloworld: - build: . \ No newline at end of file + build: . +version: '1.0' diff --git a/tests/docker-compose-config.bats b/tests/docker-compose-config.bats index aee136bd..b653049e 100644 --- a/tests/docker-compose-config.bats +++ b/tests/docker-compose-config.bats @@ -54,7 +54,7 @@ load '../lib/shared' @test "Whether docker-compose supports cache_from directive" { run docker_compose_supports_cache_from "" - assert_failure + assert_success run docker_compose_supports_cache_from "1.0" assert_failure From f96a86b1e87eeb3286a5c9ec1dfccaa1e293614e Mon Sep 17 00:00:00 2001 From: Evan Van Dam Date: Tue, 28 Sep 2021 16:53:33 -0700 Subject: [PATCH 016/253] read $COMPOSE_FILE for default config file if present --- lib/shared.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/shared.bash b/lib/shared.bash index 375cae28..c712fc50 100644 --- a/lib/shared.bash +++ b/lib/shared.bash @@ -102,7 +102,7 @@ function docker_compose_config_files() { # Use a default if there are no config files specified if [[ -z "${config_files[*]:-}" ]] ; then - echo "docker-compose.yml" + echo "${COMPOSE_FILE:-docker-compose.yml}" return fi From cdffdf8caa9e5b6a006960441a9f0b696b4ee783 Mon Sep 17 00:00:00 2001 From: Evan Van Dam Date: Tue, 26 Oct 2021 16:11:05 -0700 Subject: [PATCH 017/253] add test and update README --- README.md | 14 +++++++++++++- tests/run.bats | 27 +++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d5bf116a..e0a177a8 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,18 @@ steps: - docker-compose.test.yml ``` +You can also specify the Docker Compose config file with [`$COMPOSE_FILE`](https://docs.docker.com/compose/reference/envvars/#compose_file): + +```yml +env: + COMPOSE_FILE: docker-compose.yml +steps: + - command: test.sh + plugins: + - docker-compose#v3.8.0: + run: app +``` + You can leverage the [docker-login plugin](https://github.com/buildkite-plugins/docker-login-buildkite-plugin) in tandem for authenticating with a registry. For example, the following will build and push an image to a private repo, and pull from that private repo in subsequent run commands: ```yml @@ -299,7 +311,7 @@ Pull down multiple pre-built images. By default only the service that is being r ### `config` (optional) -The file name of the Docker Compose configuration file to use. Can also be a list of filenames. +The file name of the Docker Compose configuration file to use. Can also be a list of filenames. If `$COMPOSE_FILE` is set, it will be used if `config` is not specified. Default: `docker-compose.yml` diff --git a/tests/run.bats b/tests/run.bats index d03bf1e2..f7def5c2 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -335,6 +335,33 @@ export BUILDKITE_JOB_ID=1111 unstub buildkite-agent } +@test "Run with a prebuilt image and custom config file set from COMPOSE_FILE" { + export COMPOSE_FILE=tests/composefiles/docker-compose.v2.0.yml + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + + stub docker-compose \ + "-f tests/composefiles/docker-compose.v2.0.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "-f tests/composefiles/docker-compose.v2.0.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f tests/composefiles/docker-compose.v2.0.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v2.0.yml : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v2.0.yml : echo myimage" + + run $PWD/hooks/command + + assert_success + assert_output --partial "ran myservice" + unstub docker-compose + unstub buildkite-agent +} + @test "Run with a single prebuilt image, no retry on failed pull" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice From 68344f15a430b642fd0cb57235001bd133fb366e Mon Sep 17 00:00:00 2001 From: "boomper-bot[bot]" <40539397+boomper-bot[bot]@users.noreply.github.com> Date: Wed, 27 Oct 2021 17:17:17 +0000 Subject: [PATCH 018/253] Bump README.md for v3.9.0 release --- README.md | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index e0a177a8..f49de681 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v3.8.0: + - docker-compose#v3.9.0: run: app ``` @@ -26,7 +26,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v3.8.0: + - docker-compose#v3.9.0: run: app config: docker-compose.tests.yml env: @@ -39,7 +39,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v3.8.0: + - docker-compose#v3.9.0: run: app config: - docker-compose.yml @@ -54,7 +54,7 @@ env: steps: - command: test.sh plugins: - - docker-compose#v3.8.0: + - docker-compose#v3.9.0: run: app ``` @@ -65,7 +65,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.8.0: + - docker-compose#v3.9.0: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -73,7 +73,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.8.0: + - docker-compose#v3.9.0: run: app ``` @@ -82,7 +82,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v3.8.0: + - docker-compose#v3.9.0: run: app command: ["custom", "command", "values"] ``` @@ -98,7 +98,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.8.0: + - docker-compose#v3.9.0: run: app ``` @@ -116,7 +116,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.8.0: + - docker-compose#v3.9.0: run: app volumes: - "./dist:/app/dist" @@ -137,7 +137,7 @@ this plugin offers a `environment` block of it's own: steps: - command: generate-dist.sh plugins: - - docker-compose#v3.8.0: + - docker-compose#v3.9.0: run: app env: - BUILDKITE_BUILD_NUMBER @@ -157,7 +157,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v3.8.0: + - docker-compose#v3.9.0: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -174,7 +174,7 @@ To speed up run steps that use the same service/image (such as steps that run in steps: - label: ":docker: Build" plugins: - - docker-compose#v3.8.0: + - docker-compose#v3.9.0: build: app image-repository: index.docker.io/myorg/myrepo @@ -184,7 +184,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.8.0: + - docker-compose#v3.9.0: run: app ``` @@ -200,7 +200,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v3.8.0: + - docker-compose#v3.9.0: build: - app - tests @@ -212,7 +212,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.8.0: + - docker-compose#v3.9.0: run: tests ``` @@ -224,7 +224,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v3.8.0: + - docker-compose#v3.9.0: push: app ``` @@ -236,7 +236,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.8.0: + - docker-compose#v3.9.0: push: app ``` @@ -248,7 +248,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.8.0: + - docker-compose#v3.9.0: push: - first-service - second-service @@ -262,7 +262,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.8.0: + - docker-compose#v3.9.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -276,14 +276,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker: Build an image" plugins: - - docker-compose#v3.8.0: + - docker-compose#v3.9.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v3.8.0: + - docker-compose#v3.9.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest From 36c5fbbf04ed494445679365624fa3e52955928c Mon Sep 17 00:00:00 2001 From: Orien Madgwick <_@orien.io> Date: Wed, 22 Dec 2021 18:08:04 +1100 Subject: [PATCH 019/253] Add test covering version reading from multiple compose files --- tests/docker-compose-config.bats | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/docker-compose-config.bats b/tests/docker-compose-config.bats index aee136bd..c23b6b3e 100644 --- a/tests/docker-compose-config.bats +++ b/tests/docker-compose-config.bats @@ -52,6 +52,14 @@ load '../lib/shared' assert_output "2.1" } +@test "Read version from first of two docker-compose files configured" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG_0="tests/composefiles/docker-compose.v2.1.yml" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG_1="tests/composefiles/docker-compose.v3.2.yml" + run docker_compose_config_version + assert_success + assert_output "2.1" +} + @test "Whether docker-compose supports cache_from directive" { run docker_compose_supports_cache_from "" assert_failure From 59c1fc9aeeb96dff54830b985ab138d9a0cdde96 Mon Sep 17 00:00:00 2001 From: Orien Madgwick <_@orien.io> Date: Wed, 22 Dec 2021 18:09:53 +1100 Subject: [PATCH 020/253] Fix version reading when compose file has muliple 'version' keys --- lib/shared.bash | 2 +- .../composefiles/docker-compose.v3.2.with-version-arg.yml | 8 ++++++++ tests/docker-compose-config.bats | 7 +++++++ 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 tests/composefiles/docker-compose.v3.2.with-version-arg.yml diff --git a/lib/shared.bash b/lib/shared.bash index c712fc50..8a12c5a5 100644 --- a/lib/shared.bash +++ b/lib/shared.bash @@ -115,7 +115,7 @@ function docker_compose_config_files() { # Returns the version from the output of docker_compose_config function docker_compose_config_version() { IFS=$'\n' read -r -a config <<< "$(docker_compose_config_files)" - awk '/^\s*version:/ { print $2; }' < "${config[0]}" | sed "s/[\"']//g" + awk '/^\s*version:/ { print $2; exit; }' < "${config[0]}" | sed "s/[\"']//g" } # Build an docker-compose file that overrides the image for a set of diff --git a/tests/composefiles/docker-compose.v3.2.with-version-arg.yml b/tests/composefiles/docker-compose.v3.2.with-version-arg.yml new file mode 100644 index 00000000..94cf74b4 --- /dev/null +++ b/tests/composefiles/docker-compose.v3.2.with-version-arg.yml @@ -0,0 +1,8 @@ +version: "3.2" + +services: + helloworld: + build: + context: . + args: + version: 73.976.3 diff --git a/tests/docker-compose-config.bats b/tests/docker-compose-config.bats index c23b6b3e..06e8ea2a 100644 --- a/tests/docker-compose-config.bats +++ b/tests/docker-compose-config.bats @@ -60,6 +60,13 @@ load '../lib/shared' assert_output "2.1" } +@test "Read version given docker-compose file with argument named version" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG_0="tests/composefiles/docker-compose.v3.2.with-version-arg.yml" + run docker_compose_config_version + assert_success + assert_output "3.2" +} + @test "Whether docker-compose supports cache_from directive" { run docker_compose_supports_cache_from "" assert_failure From 094a2d8fcbd7bc9b973d82f1b75c6834e337ec2c Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Mon, 7 Mar 2022 16:38:33 +0000 Subject: [PATCH 021/253] Update buildkite plugin shellcheck to v1.2.0 --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 4ef335ae..baea9ad8 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -7,7 +7,7 @@ env: steps: - label: ":shell: Shellcheck" plugins: - shellcheck#v1.1.2: + shellcheck#v1.2.0: files: - hooks/** - lib/** From fb62e335743134ffc4d3e9dac114909ca7d70110 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Mon, 7 Mar 2022 16:38:38 +0000 Subject: [PATCH 022/253] Update buildkite plugin plugin-linter to v3 --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 4ef335ae..ad484904 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -15,7 +15,7 @@ steps: - label: ":sparkles: Lint" plugins: - plugin-linter#v2.0.0: + plugin-linter#v3.0.0: id: docker-compose - label: ":bash: Tests" From 88a00b1b9c69fac2752e63a5a714656a3ea30a00 Mon Sep 17 00:00:00 2001 From: Gregory McIntyre Date: Thu, 17 Mar 2022 10:28:18 +1100 Subject: [PATCH 023/253] Copy the mount-ssh-agent option from the docker plugin --- README.md | 6 ++++++ commands/run.sh | 22 ++++++++++++++++++++++ plugin.yml | 2 ++ tests/run.bats | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+) diff --git a/README.md b/README.md index f49de681..144a63c2 100644 --- a/README.md +++ b/README.md @@ -371,6 +371,12 @@ Whether to match the user ID and group ID for the container user to the user ID Using this option ensures that any files created on shared mounts from within the container will be accessible to the host user. It is otherwise common to accidentally create root-owned files that Buildkite will be unable to remove, since containers by default run as the root user. +### `mount-ssh-agent` (optional, run-only, boolean) + +Whether to automatically mount the ssh-agent socket from the host agent machine into the container (at `/ssh-agent`and `/root/.ssh/known_hosts` respectively), allowing git operations to work correctly. + +Default: `false` + ### `mount-buildkite-agent` (optional, run-only, boolean) Whether to automatically mount the `buildkite-agent` binary and associated environment variables from the host agent machine into the container. diff --git a/commands/run.sh b/commands/run.sh index a9bb70e4..aa5a86d2 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -8,6 +8,7 @@ run_service="$(plugin_read_config RUN)" container_name="$(docker_compose_project_name)_${run_service}_build_${BUILDKITE_BUILD_NUMBER}" override_file="docker-compose.buildkite-${BUILDKITE_BUILD_NUMBER}-override.yml" pull_retries="$(plugin_read_config PULL_RETRIES "0")" +mount_ssh_agent='' expand_headers_on_error() { echo "^^^ +++" @@ -161,6 +162,27 @@ if [[ -n "$(plugin_read_config ENTRYPOINT)" ]] ; then run_params+=("$(plugin_read_config ENTRYPOINT)") fi +# Mount ssh-agent socket and known_hosts +if [[ "${BUILDKITE_PLUGIN_DOCKER_COMPOSE_MOUNT_SSH_AGENT:-$mount_ssh_agent}" =~ ^(true|on|1)$ ]] ; then + if [[ -z "${SSH_AUTH_SOCK:-}" ]] ; then + echo "+++ 🚨 \$SSH_AUTH_SOCK isn't set, has ssh-agent started?" + exit 1 + fi + if [[ ! -S "${SSH_AUTH_SOCK}" ]] ; then + echo "+++ 🚨 There isn't any file at ${SSH_AUTH_SOCK}, has ssh-agent started?" + exit 1 + fi + if [[ ! -S "${SSH_AUTH_SOCK}" ]] ; then + echo "+++ 🚨 The file at ${SSH_AUTH_SOCK} isn't a socket, has ssh-agent started?" + exit 1 + fi + run_params+=( + "--env" "SSH_AUTH_SOCK=/ssh-agent" + "--volume" "${SSH_AUTH_SOCK}:/ssh-agent" + "--volume" "${HOME}/.ssh/known_hosts:/root/.ssh/known_hosts" + ) +fi + # Optionally handle the mount-buildkite-agent option if [[ "$(plugin_read_config MOUNT_BUILDKITE_AGENT "false")" == "true" ]]; then if [[ -z "${BUILDKITE_AGENT_BINARY_PATH:-}" ]] ; then diff --git a/plugin.yml b/plugin.yml index 50169921..b0a1d41f 100644 --- a/plugin.yml +++ b/plugin.yml @@ -72,6 +72,8 @@ configuration: type: string propagate-uid-gid: type: boolean + mount-ssh-agent: + type: boolean mount-buildkite-agent: type: boolean entrypoint: diff --git a/tests/run.bats b/tests/run.bats index f7def5c2..14fb3a01 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -911,3 +911,36 @@ export BUILDKITE_JOB_ID=1111 unstub docker-compose unstub buildkite-agent } + +@test "Run with mount-ssh-agent" { + export SSH_AUTH_SOCK=/tmp/ssh_auth_sock + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="echo hello world" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_MOUNT_SSH_AGENT=true + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm --env SSH_AUTH_SOCK=/ssh-agent --volume /tmp/ssh_auth_sock:/ssh-agent --volume /root/.ssh/known_hosts:/root/.ssh/known_hosts myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + apk add netcat-openbsd + nc -lkvU $SSH_AUTH_SOCK & + + run $PWD/hooks/command + + kill %1 + + assert_success + assert_output --partial "built myservice" + assert_output --partial "ran myservice" + unstub docker-compose + unstub buildkite-agent +} From 649a1a823db703fa4086257c64a17da498d15c12 Mon Sep 17 00:00:00 2001 From: Gregory McIntyre Date: Thu, 17 Mar 2022 10:47:43 +1100 Subject: [PATCH 024/253] Get docker compose CLI usage right --- commands/run.sh | 6 +++--- tests/run.bats | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/commands/run.sh b/commands/run.sh index aa5a86d2..1fc39aeb 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -177,9 +177,9 @@ if [[ "${BUILDKITE_PLUGIN_DOCKER_COMPOSE_MOUNT_SSH_AGENT:-$mount_ssh_agent}" =~ exit 1 fi run_params+=( - "--env" "SSH_AUTH_SOCK=/ssh-agent" - "--volume" "${SSH_AUTH_SOCK}:/ssh-agent" - "--volume" "${HOME}/.ssh/known_hosts:/root/.ssh/known_hosts" + "-e" "SSH_AUTH_SOCK=/ssh-agent" + "-v" "${SSH_AUTH_SOCK}:/ssh-agent" + "-v" "${HOME}/.ssh/known_hosts:/root/.ssh/known_hosts" ) fi diff --git a/tests/run.bats b/tests/run.bats index 14fb3a01..edf0b550 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -926,7 +926,7 @@ export BUILDKITE_JOB_ID=1111 stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ - "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm --env SSH_AUTH_SOCK=/ssh-agent --volume /tmp/ssh_auth_sock:/ssh-agent --volume /root/.ssh/known_hosts:/root/.ssh/known_hosts myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" + "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm -e SSH_AUTH_SOCK=/ssh-agent -v /tmp/ssh_auth_sock:/ssh-agent -v /root/.ssh/known_hosts:/root/.ssh/known_hosts myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" From 1b721403c2b6b33a99ed86ffbcc9d0fb65732cdc Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Wed, 13 May 2020 15:38:20 -0400 Subject: [PATCH 025/253] Allow for building an image with multiple --cache-from images --- README.md | 64 ++++++++++++++++++++++++++++++---- commands/build.sh | 55 +++++++++++++++++++++++++---- commands/run.sh | 2 +- lib/shared.bash | 9 +++-- tests/build.bats | 36 +++++++++++++++++++ tests/image-override-file.bats | 43 ++++++++++++++++------- 6 files changed, 181 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index f49de681..8e61c8c5 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,7 @@ You can use the [environment key in docker-compose.yml](https://docs.docker.com/ variables from outside docker-compose. If you want to add extra environment above what is declared in your `docker-compose.yml`, -this plugin offers a `environment` block of it's own: +this plugin offers a `environment` block of its own: ```yml steps: @@ -151,7 +151,7 @@ Note how the values in the list can either be just a key (so the value is source You can use the [build args key in docker-compose.yml](https://docs.docker.com/compose/compose-file/#args) to set specific build arguments when building an image. -Alternatively, if you want to set build arguments when pre-building an image, this plugin offers an `args` block of it's own: +Alternatively, if you want to set build arguments when pre-building an image, this plugin offers an `args` block of its own: ```yml steps: @@ -264,8 +264,8 @@ steps: username: xyz - docker-compose#v3.9.0: push: - - app:index.docker.io/myorg/myrepo/myapp - - app:index.docker.io/myorg/myrepo/myapp:latest + - app:index.docker.io/myorg/myrepo/myapp + - app:index.docker.io/myorg/myrepo/myapp:latest ``` ## Reusing caches from images @@ -285,10 +285,62 @@ steps: plugins: - docker-compose#v3.9.0: push: - - app:index.docker.io/myorg/myrepo/myapp - - app:index.docker.io/myorg/myrepo/myapp:latest + - app:index.docker.io/myorg/myrepo/myapp + - app:index.docker.io/myorg/myrepo/myapp:latest ``` +#### Multiple cache-from values + +This plugin allows for the value of `cache-from` to be a string or a list. If it's a list, as below, then the first successfully pulled image will be used. + +```yaml +steps: + - label: ":docker Build an image" + plugins: + - docker-compose#v3.2.0: + build: app + image-repository: index.docker.io/myorg/myrepo + cache-from: + - app:index.docker.io/myorg/myrepo/myapp:my-branch + - app:index.docker.io/myorg/myrepo/myapp:latest + - wait + - label: ":docker: Push to final repository" + plugins: + - docker-compose#v3.2.0: + push: + - app:index.docker.io/myorg/myrepo/myapp + - app:index.docker.io/myorg/myrepo/myapp:my-branch + - app:index.docker.io/myorg/myrepo/myapp:latest +``` + +You may actually want to build your image with multiple cache-from values, for instance, with the cached images of multiple stages in a multi-stage build. +By adding a grouping tag to the end of a cache-from list item, this plugin can differentiate between groups within which only the first successfully downloaded image should be used. +This way, not all of the images need to be downloaded and used as cache, and also not just the first. + +```yaml +steps: + - label: ":docker Build an image" + plugins: + - docker-compose#v3.2.0: + build: app + image-repository: index.docker.io/myorg/myrepo + cache-from: + - app:index.docker.io/myorg/myrepo/myapp-intermediate-target:this-build-number:intermediate + - app:index.docker.io/myorg/myrepo/myapp:my-branch + - app:index.docker.io/myorg/myrepo/myapp:latest + - wait + - label: ":docker: Push to final repository" + plugins: + - docker-compose#v3.2.0: + push: + - app:index.docker.io/myorg/myrepo/myapp + - app:index.docker.io/myorg/myrepo/myapp:my-branch + - app:index.docker.io/myorg/myrepo/myapp:latest +``` + +In the example above, the `myapp-intermediate-target:this-build-number` is one group named "intermediate", and `myapp:my-branch` and `myapp:latest` +are another (with a default name). The first successfully downloaded image in each group will be used as a cache. + ## Configuration ### `build` diff --git a/commands/build.sh b/commands/build.sh index cce943dd..18dd2415 100755 --- a/commands/build.sh +++ b/commands/build.sh @@ -12,6 +12,24 @@ service_name_cache_from_var() { echo "cache_from__${service_name//-/_}" } +service_name_group_name_cache_from_var() { + local service_name="$1" + local group_index="$2" + echo "group_cache_from__${service_name//-/_}__${group_index//-/_}" +} + +count_of_named_array() { + local tmp="$1[@]" + local copy=( "${!tmp}" ) + echo "${#copy[@]}" +} + +named_array_values() { + local tmp="$1[@]" + local copy=( "${!tmp}" ) + echo "${copy[@]}" +} + if [[ -z "$image_repository" ]] ; then echo "+++ ⚠️ Build step missing image-repository setting" echo "This build step has no image-repository set. Without an image-repository, the Docker image won't be pushed to a repository, and won't be automatically used by any run steps." @@ -23,16 +41,34 @@ if [[ "$(plugin_read_config NO_CACHE "false")" == "false" ]] ; then for line in $(plugin_read_list CACHE_FROM) ; do IFS=':' read -r -a tokens <<< "$line" service_name=${tokens[0]} - service_image=$(IFS=':'; echo "${tokens[*]:1}") + service_image=$(IFS=':'; echo "${tokens[*]:1:2}") + cache_from_group_name=$(IFS=':'; echo "${tokens[*]:3}") + if [[ -z "$cache_from_group_name" ]]; then + cache_from_group_name=":default:" + fi + # The variable with this name will hold an array of group names: cache_image_name="$(service_name_cache_from_var "$service_name")" if [[ -n ${!cache_image_name+x} ]]; then - continue # skipping since there's already a pulled cache image for this service + if [[ "$(named_array_values ${cache_image_name})" =~ "${cache_from_group_name}" ]]; then + continue # skipping since there's already a pulled cache image for this service+group + fi fi echo "~~~ :docker: Pulling cache image for $service_name" if retry "$pull_retries" plugin_prompt_and_run docker pull "$service_image" ; then - printf -v "$cache_image_name" "%s" "$service_image" + if [[ -z "${!cache_image_name+x}" ]]; then + declare -a "$cache_image_name" + cache_image_length=0 + else + cache_image_length="$(count_of_named_array "${cache_image_name}")" + fi + + declare "$cache_image_name+=( $cache_from_group_name )" + # The variable with this name will hold the image for the this group + # (based on index into the array of group names): + cache_from_group_var="$(service_name_group_name_cache_from_var "$service_name" "${cache_image_length}")" + printf -v "$cache_from_group_var" "%s" "$service_image" else echo "!!! :docker: Pull failed. $service_image will not be used as a cache for $service_name" fi @@ -55,9 +91,15 @@ for service_name in $(plugin_read_list BUILD) ; do cache_from_var="$(service_name_cache_from_var "${service_name}")" if [[ -n "${!cache_from_var-}" ]]; then - build_images+=("${!cache_from_var}") + cache_from_length="$(count_of_named_array "${cache_from_var}")" + build_images+=("${cache_from_length}") + + for i in $(seq 0 "$((cache_from_length-1))"); do + cache_from_group_var="$(service_name_group_name_cache_from_var "$service_name" "$i")" + build_images+=("${!cache_from_group_var}") + done else - build_images+=("") + build_images+=(0) fi done @@ -104,6 +146,7 @@ if [[ -n "$image_repository" ]] ; then done # pop-off the last build image - build_images=("${build_images[@]:3}") + # 3 for service, image, num_cache_from; plus num_cache_from + build_images=("${build_images[@]:(3 + ${build_images[2]})}") done fi diff --git a/commands/run.sh b/commands/run.sh index a9bb70e4..64f0f1b4 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -41,7 +41,7 @@ prebuilt_services=() for service_name in "${prebuilt_candidates[@]}" ; do if prebuilt_image=$(get_prebuilt_image "$service_name") ; then echo "~~~ :docker: Found a pre-built image for $service_name" - prebuilt_service_overrides+=("$service_name" "$prebuilt_image" "") + prebuilt_service_overrides+=("$service_name" "$prebuilt_image" 0) prebuilt_services+=("$service_name") # If it's prebuilt, we need to pull it down diff --git a/lib/shared.bash b/lib/shared.bash index c712fc50..e6e646f7 100644 --- a/lib/shared.bash +++ b/lib/shared.bash @@ -134,7 +134,7 @@ function docker_compose_supports_cache_from() { } # Build an docker-compose file that overrides the image for a specific -# docker-compose version and set of [ service, image, cache_from ] tuples +# docker-compose version and set of [ service, image, num_cache_from, cache_from1, cache_from2, ... ] tuples function build_image_override_file_with_version() { local version="$1" @@ -153,7 +153,7 @@ function build_image_override_file_with_version() { printf " %s:\\n" "$1" printf " image: %s\\n" "$2" - if [[ -n "$3" ]] ; then + if [[ "$3" -gt 0 ]] ; then if ! docker_compose_supports_cache_from "$version" ; then echo "Unsupported Docker Compose config file version: $version" echo "The 'cache_from' option can only be used with Compose file versions 2.2 or 3.2 and above." @@ -164,7 +164,10 @@ function build_image_override_file_with_version() { printf " build:\\n" printf " cache_from:\\n" - printf " - %s\\n" "$3" + for cache_from_i in $(seq 4 "$((3 + $3))"); do + printf " - %s\\n" "${!cache_from_i}" + done + shift "$3" fi shift 3 diff --git a/tests/build.bats b/tests/build.bats index 50a00b45..361735df 100644 --- a/tests/build.bats +++ b/tests/build.bats @@ -318,6 +318,42 @@ load '../lib/shared' unstub docker-compose } +@test "Build with several cache-from image groups for one service" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=helloworld:my.repository/myservice_cache:build-target-build-1:target1 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_1=helloworld:my.repository/myservice_cache:build-target-latest:target1 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_2=helloworld:my.repository/myservice_cache:install-target-build-1:target2 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_3=helloworld:my.repository/myservice_cache:branch-name + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_4=helloworld:my.repository/myservice_cache:latest + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "pull my.repository/myservice_cache:build-target-build-1 : echo pulled cache image build-target" \ + "pull my.repository/myservice_cache:install-target-build-1 : echo pulled cache image install-target" \ + "pull my.repository/myservice_cache:branch-name : echo pulled cache image branch-name" + + stub docker-compose \ + "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" + + run $PWD/hooks/command + + assert_success + assert_output --partial "pulled cache image build-target" + assert_output --partial "pulled cache image install-target" + assert_output --partial "pulled cache image branch-name" + assert_output --partial "- my.repository/myservice_cache:build-target-build-1" + refute_output --partial "- my.repository/myservice_cache:build-target-latest" + assert_output --partial "- my.repository/myservice_cache:install-target-build-1" + assert_output --partial "- my.repository/myservice_cache:branch-name" + refute_output --partial "- my.repository/myservice_cache:latest" + assert_output --partial "built helloworld" + unstub docker + unstub docker-compose +} + @test "Build with several cache-from images for one service with first image being not available" { export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" export BUILDKITE_JOB_ID=1111 diff --git a/tests/image-override-file.bats b/tests/image-override-file.bats index 1386afbb..5af1ac5b 100644 --- a/tests/image-override-file.bats +++ b/tests/image-override-file.bats @@ -32,61 +32,80 @@ services: EOF ) -@test "Build an docker-compose override file" { - run build_image_override_file_with_version "2.1" "myservice" "newimage:1.0.0" "" +myservice_override_file4=$(cat <<-EOF +version: '3.2' +services: + myservice: + image: newimage:1.0.0 + build: + cache_from: + - my.repository/myservice:latest + - my.repository/myservice:target +EOF +) + +@test "Build a docker-compose override file" { + run build_image_override_file_with_version "2.1" "myservice" "newimage:1.0.0" 0 assert_success assert_output "$myservice_override_file1" } -@test "Build an docker-compose override file with multiple entries" { +@test "Build a docker-compose override file with multiple entries" { run build_image_override_file_with_version "2.1" \ - "myservice1" "newimage1:1.0.0" "" \ - "myservice2" "newimage2:1.0.0" "" + "myservice1" "newimage1:1.0.0" 0 \ + "myservice2" "newimage2:1.0.0" 0 assert_success assert_output "$myservice_override_file2" } @test "Build a docker-compose file with cache-from" { - run build_image_override_file_with_version "3.2" "myservice" "newimage:1.0.0" "my.repository/myservice:latest" + run build_image_override_file_with_version "3.2" "myservice" "newimage:1.0.0" 1 "my.repository/myservice:latest" assert_success assert_output "$myservice_override_file3" } +@test "Build a docker-compose file with multiple cache-from entries" { + run build_image_override_file_with_version "3.2" "myservice" "newimage:1.0.0" 2 "my.repository/myservice:latest" "my.repository/myservice:target" + + assert_success + assert_output "$myservice_override_file4" +} + @test "Build a docker-compose file with cache-from and compose-file version 2" { - run build_image_override_file_with_version "2" "myservice" "newimage:1.0.0" "my.repository/myservice:latest" + run build_image_override_file_with_version "2" "myservice" "newimage:1.0.0" 1 "my.repository/myservice:latest" assert_failure } @test "Build a docker-compose file with cache-from and compose-file version 2.0" { - run build_image_override_file_with_version "2.0" "myservice" "newimage:1.0.0" "my.repository/myservice:latest" + run build_image_override_file_with_version "2.0" "myservice" "newimage:1.0.0" 1 "my.repository/myservice:latest" assert_failure } @test "Build a docker-compose file with cache-from and compose-file version 2.1" { - run build_image_override_file_with_version "2.1" "myservice" "newimage:1.0.0" "my.repository/myservice:latest" + run build_image_override_file_with_version "2.1" "myservice" "newimage:1.0.0" 1 "my.repository/myservice:latest" assert_failure } @test "Build a docker-compose file with cache-from and compose-file version 3" { - run build_image_override_file_with_version "3" "myservice" "newimage:1.0.0" "my.repository/myservice:latest" + run build_image_override_file_with_version "3" "myservice" "newimage:1.0.0" 1 "my.repository/myservice:latest" assert_failure } @test "Build a docker-compose file with cache-from and compose-file version 3.0" { - run build_image_override_file_with_version "3.0" "myservice" "newimage:1.0.0" "my.repository/myservice:latest" + run build_image_override_file_with_version "3.0" "myservice" "newimage:1.0.0" 1 "my.repository/myservice:latest" assert_failure } @test "Build a docker-compose file with cache-from and compose-file version 3.1" { - run build_image_override_file_with_version "3.1" "myservice" "newimage:1.0.0" "my.repository/myservice:latest" + run build_image_override_file_with_version "3.1" "myservice" "newimage:1.0.0" 1 "my.repository/myservice:latest" assert_failure } From 2bc400e6b7442f31407f87d747b75217e60b5f13 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Sun, 10 Apr 2022 19:53:51 -0400 Subject: [PATCH 026/253] Documentation improvements for multiple cache-froms --- README.md | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 8e61c8c5..b54a229c 100644 --- a/README.md +++ b/README.md @@ -319,26 +319,37 @@ This way, not all of the images need to be downloaded and used as cache, and als ```yaml steps: - - label: ":docker Build an image" + - label: ":docker: Build Intermediate Image" plugins: - docker-compose#v3.2.0: - build: app - image-repository: index.docker.io/myorg/myrepo + build: + - myservice_intermediate # docker-compose.yml is the same as myservice but has `target: intermediate` + image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} + image-repository: index.docker.io/myorg/myrepo/myservice_intermediate cache-from: - - app:index.docker.io/myorg/myrepo/myapp-intermediate-target:this-build-number:intermediate - - app:index.docker.io/myorg/myrepo/myapp:my-branch - - app:index.docker.io/myorg/myrepo/myapp:latest + - myservice_intermediate:index.docker.io/myorg/myrepo/myservice_intermediate:${BUILDKITE_BRANCH} + - myservice_intermediate:index.docker.io/myorg/myrepo/myservice_intermediate:latest + push: + - myservice_intermediate:index.docker.io/myorg/myrepo/myservice_intermediate:${BUILDKITE_BRANCH} - wait - - label: ":docker: Push to final repository" + + - label: ":docker: Build Final Image" plugins: - docker-compose#v3.2.0: + build: + - myservice + image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} + image-repository: index.docker.io/myorg/myrepo + cache-from: + - myservice:index.docker.io/myorg/myrepo/myservice_intermediate:buildkite-build-${BUILDKITE_BUILD_NUMBER}:intermediate # built in step above + - myservice:index.docker.io/myorg/myrepo/myservice:${BUILDKITE_BRANCH} + - myservice:index.docker.io/myorg/myrepo/myservice:latest push: - - app:index.docker.io/myorg/myrepo/myapp - - app:index.docker.io/myorg/myrepo/myapp:my-branch - - app:index.docker.io/myorg/myrepo/myapp:latest + - myservice:index.docker.io/myorg/myrepo/myservice:${BUILDKITE_BRANCH} + - myservice:index.docker.io/myorg/myrepo/myservice:latest ``` -In the example above, the `myapp-intermediate-target:this-build-number` is one group named "intermediate", and `myapp:my-branch` and `myapp:latest` +In the example above, the `myservice_intermediate:buildkite-build-${BUILDKITE_BUILD_NUMBER}` is one group named "intermediate", and `myservice:${BUILDKITE_BRANCH}` and `myservice:latest` are another (with a default name). The first successfully downloaded image in each group will be used as a cache. ## Configuration From af3655227962e60a00f5729d79fa7c4fe5e5ffbb Mon Sep 17 00:00:00 2001 From: Pol Date: Mon, 11 Apr 2022 11:08:02 -0300 Subject: [PATCH 027/253] Update readme to trigger build small wording change to trigger a new build --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b54a229c..7d030222 100644 --- a/README.md +++ b/README.md @@ -314,7 +314,7 @@ steps: ``` You may actually want to build your image with multiple cache-from values, for instance, with the cached images of multiple stages in a multi-stage build. -By adding a grouping tag to the end of a cache-from list item, this plugin can differentiate between groups within which only the first successfully downloaded image should be used. +Adding a grouping tag to the end of a cache-from list item allows this plugin to differentiate between groups within which only the first successfully downloaded image should be used. This way, not all of the images need to be downloaded and used as cache, and also not just the first. ```yaml From 810c5afe38580d3140ebb884f052925cbb485ba3 Mon Sep 17 00:00:00 2001 From: Jeremy Bumsted Date: Tue, 12 Apr 2022 12:18:16 -0700 Subject: [PATCH 028/253] WIP - add tests for running multiple commands - tests pass, but fail when trying to unstub commands --- tests/multiple-commands.bats | 46 ++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 tests/multiple-commands.bats diff --git a/tests/multiple-commands.bats b/tests/multiple-commands.bats new file mode 100644 index 00000000..edf4f4b4 --- /dev/null +++ b/tests/multiple-commands.bats @@ -0,0 +1,46 @@ +#!/usr/bin/env bats + +load '/usr/local/lib/bats/load.bash' +load '../lib/shared' + + export DOCKER_COMPOSE_STUB_DEBUG=/dev/stdout + export BUILDKITE_AGENT_STUB_DEBUG=/dev/stdout +# export BATS_MOCK_TMPDIR=$PWD + + +@test "Build and run in a single step" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="echo hello world" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_REPOSITORY=my.repository/llamas + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml push myservice : echo pushed myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --rm myservice /bin/sh -e -c echo hello world : echo ran myservice" + + stub buildkite-agent \ + "meta-data set docker-compose-plugin-built-image-tag-myservice my.repository/llamas:test-myservice-build-1 : echo set meta-data" \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : echo meta-data exists" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "ran myservice" + + #unstub docker-compose + #unstub buildkite-agent +} + + + + + From 22f83b07d21cfd3b6e8ed71fb000cd8c1d5d44f6 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Wed, 13 Apr 2022 20:48:30 -0400 Subject: [PATCH 029/253] Bump to latest docker-compose plugin version --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7d030222..a67d78bd 100644 --- a/README.md +++ b/README.md @@ -297,7 +297,7 @@ This plugin allows for the value of `cache-from` to be a string or a list. If it steps: - label: ":docker Build an image" plugins: - - docker-compose#v3.2.0: + - docker-compose#v3.9.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: @@ -306,7 +306,7 @@ steps: - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v3.2.0: + - docker-compose#v3.9.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:my-branch @@ -321,7 +321,7 @@ This way, not all of the images need to be downloaded and used as cache, and als steps: - label: ":docker: Build Intermediate Image" plugins: - - docker-compose#v3.2.0: + - docker-compose#v3.9.0: build: - myservice_intermediate # docker-compose.yml is the same as myservice but has `target: intermediate` image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} @@ -335,7 +335,7 @@ steps: - label: ":docker: Build Final Image" plugins: - - docker-compose#v3.2.0: + - docker-compose#v3.9.0: build: - myservice image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} From 46230c592ad1d9e860ea891b98e27ac48a0694c7 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Wed, 13 Apr 2022 20:49:12 -0400 Subject: [PATCH 030/253] Match as regex, not literally --- commands/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/build.sh b/commands/build.sh index 18dd2415..48900f2d 100755 --- a/commands/build.sh +++ b/commands/build.sh @@ -50,7 +50,7 @@ if [[ "$(plugin_read_config NO_CACHE "false")" == "false" ]] ; then cache_image_name="$(service_name_cache_from_var "$service_name")" if [[ -n ${!cache_image_name+x} ]]; then - if [[ "$(named_array_values ${cache_image_name})" =~ "${cache_from_group_name}" ]]; then + if [[ "$(named_array_values ${cache_image_name})" =~ ${cache_from_group_name} ]]; then continue # skipping since there's already a pulled cache image for this service+group fi fi From 1954c325db703a2ef8931a6746eb278c85d8dbf1 Mon Sep 17 00:00:00 2001 From: Pol Date: Mon, 18 Apr 2022 14:58:13 -0300 Subject: [PATCH 031/253] Update README.md Small text change to trigger a new build --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a67d78bd..2a537205 100644 --- a/README.md +++ b/README.md @@ -315,7 +315,7 @@ steps: You may actually want to build your image with multiple cache-from values, for instance, with the cached images of multiple stages in a multi-stage build. Adding a grouping tag to the end of a cache-from list item allows this plugin to differentiate between groups within which only the first successfully downloaded image should be used. -This way, not all of the images need to be downloaded and used as cache, and also not just the first. +This way, not all images need to be downloaded and used as cache, not just the first. ```yaml steps: From 565bf7358e69d4f49c92132698bb7ec77ac3a38c Mon Sep 17 00:00:00 2001 From: Suma Naidu <79238214+nsuma8989@users.noreply.github.com> Date: Fri, 22 Apr 2022 13:09:15 -0700 Subject: [PATCH 032/253] fix for regex error --- commands/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/build.sh b/commands/build.sh index 48900f2d..91ca055f 100755 --- a/commands/build.sh +++ b/commands/build.sh @@ -50,7 +50,7 @@ if [[ "$(plugin_read_config NO_CACHE "false")" == "false" ]] ; then cache_image_name="$(service_name_cache_from_var "$service_name")" if [[ -n ${!cache_image_name+x} ]]; then - if [[ "$(named_array_values ${cache_image_name})" =~ ${cache_from_group_name} ]]; then + if [[ "$(named_array_values "${cache_image_name}")" =~ ${cache_from_group_name} ]]; then continue # skipping since there's already a pulled cache image for this service+group fi fi From e10e7f7d50be3b24fdc94b7847282a8c27dfde4c Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Sun, 24 Apr 2022 18:27:32 -0400 Subject: [PATCH 033/253] Fixing example config to match latest schema --- README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2a537205..ceed063b 100644 --- a/README.md +++ b/README.md @@ -322,8 +322,7 @@ steps: - label: ":docker: Build Intermediate Image" plugins: - docker-compose#v3.9.0: - build: - - myservice_intermediate # docker-compose.yml is the same as myservice but has `target: intermediate` + build: myservice_intermediate # docker-compose.yml is the same as myservice but has `target: intermediate` image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo/myservice_intermediate cache-from: @@ -336,8 +335,7 @@ steps: - label: ":docker: Build Final Image" plugins: - docker-compose#v3.9.0: - build: - - myservice + build: myservice image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo cache-from: From acdf35db377757da4da47155c8e4e1d7c32faf06 Mon Sep 17 00:00:00 2001 From: Jeremy Bumsted Date: Mon, 25 Apr 2022 15:18:36 -0700 Subject: [PATCH 034/253] fix README version numbers --- README.md | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index ed28d68a..a0d4c59a 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: run: app ``` @@ -26,7 +26,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: run: app config: docker-compose.tests.yml env: @@ -39,7 +39,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: run: app config: - docker-compose.yml @@ -53,7 +53,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -61,7 +61,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: run: app ``` @@ -70,7 +70,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: run: app command: ["custom", "command", "values"] ``` @@ -86,7 +86,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: run: app ``` @@ -104,7 +104,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: run: app volumes: - "./dist:/app/dist" @@ -125,7 +125,7 @@ this plugin offers a `environment` block of it's own: steps: - command: generate-dist.sh plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: run: app env: - BUILDKITE_BUILD_NUMBER @@ -145,7 +145,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -162,7 +162,7 @@ To speed up run steps that use the same service/image (such as steps that run in steps: - label: ":docker: Build" plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: build: app image-repository: index.docker.io/myorg/myrepo @@ -172,7 +172,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: run: app ``` @@ -188,7 +188,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: build: - app - tests @@ -200,7 +200,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: run: tests ``` @@ -212,7 +212,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: push: app ``` @@ -224,7 +224,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: push: app ``` @@ -236,7 +236,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: push: - first-service - second-service @@ -250,7 +250,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -264,14 +264,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker Build an image" plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest From b5acb3b80035ab459cbaca73197d6819a7ce79fd Mon Sep 17 00:00:00 2001 From: Suma Naidu <79238214+nsuma8989@users.noreply.github.com> Date: Wed, 11 May 2022 16:43:38 -0700 Subject: [PATCH 035/253] fix for linter error --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ceed063b..31e76306 100644 --- a/README.md +++ b/README.md @@ -328,6 +328,7 @@ steps: cache-from: - myservice_intermediate:index.docker.io/myorg/myrepo/myservice_intermediate:${BUILDKITE_BRANCH} - myservice_intermediate:index.docker.io/myorg/myrepo/myservice_intermediate:latest + run: test push: - myservice_intermediate:index.docker.io/myorg/myrepo/myservice_intermediate:${BUILDKITE_BRANCH} - wait From 9c5bca1387d67cb36c32c51253f5364880dea67d Mon Sep 17 00:00:00 2001 From: Jeremy Bumsted Date: Wed, 11 May 2022 18:23:09 -0700 Subject: [PATCH 036/253] fix tests --- tests/multiple-commands.bats | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/tests/multiple-commands.bats b/tests/multiple-commands.bats index edf4f4b4..e88c8b2f 100644 --- a/tests/multiple-commands.bats +++ b/tests/multiple-commands.bats @@ -2,12 +2,12 @@ load '/usr/local/lib/bats/load.bash' load '../lib/shared' +load '../lib/metadata' - export DOCKER_COMPOSE_STUB_DEBUG=/dev/stdout - export BUILDKITE_AGENT_STUB_DEBUG=/dev/stdout +# export DOCKER_COMPOSE_STUB_DEBUG=/dev/tty +# export BUILDKITE_AGENT_STUB_DEBUG=/dev/tty # export BATS_MOCK_TMPDIR=$PWD - @test "Build and run in a single step" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice @@ -20,27 +20,29 @@ load '../lib/shared' export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false stub docker-compose \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml push myservice : echo pushed myservice" \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --rm myservice /bin/sh -e -c echo hello world : echo ran myservice" + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml push myservice : echo pushed myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" stub buildkite-agent \ - "meta-data set docker-compose-plugin-built-image-tag-myservice my.repository/llamas:test-myservice-build-1 : echo set meta-data" \ - "meta-data exists docker-compose-plugin-built-image-tag-myservice : echo meta-data exists" + "meta-data set docker-compose-plugin-built-image-tag-myservice my.repository/llamas:test-myservice-build-1 : echo set meta-data" \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo got meta-data" run $PWD/hooks/command + unstub docker-compose + unstub buildkite-agent + assert_success assert_output --partial "built myservice" - assert_output --partial "ran myservice" - - #unstub docker-compose - #unstub buildkite-agent + assert_output --partial "pushed myservice" + assert_output --partial "pulled myservice" + assert_output --partial "ran myservice dependencies" + assert_output --partial "ran myservice" } - - From 6865586c4bd6b4a9c68bc8ddcf3b46ad5c744805 Mon Sep 17 00:00:00 2001 From: Gregory McIntyre Date: Fri, 27 May 2022 10:26:59 +1000 Subject: [PATCH 037/253] Change to provoke a build --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 144a63c2..f3957127 100644 --- a/README.md +++ b/README.md @@ -377,6 +377,7 @@ Whether to automatically mount the ssh-agent socket from the host agent machine Default: `false` + ### `mount-buildkite-agent` (optional, run-only, boolean) Whether to automatically mount the `buildkite-agent` binary and associated environment variables from the host agent machine into the container. From d9976979d0b990cff677b7d6952222ed894a65f2 Mon Sep 17 00:00:00 2001 From: Gregory McIntyre Date: Fri, 27 May 2022 10:27:05 +1000 Subject: [PATCH 038/253] Change back --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index f3957127..144a63c2 100644 --- a/README.md +++ b/README.md @@ -377,7 +377,6 @@ Whether to automatically mount the ssh-agent socket from the host agent machine Default: `false` - ### `mount-buildkite-agent` (optional, run-only, boolean) Whether to automatically mount the `buildkite-agent` binary and associated environment variables from the host agent machine into the container. From 2c95843f7cbf51ee3a04064455a70f0fe783a86c Mon Sep 17 00:00:00 2001 From: Suma Naidu <79238214+nsuma8989@users.noreply.github.com> Date: Tue, 31 May 2022 13:16:11 -0700 Subject: [PATCH 039/253] Update README.md --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index 31e76306..b6c69657 100644 --- a/README.md +++ b/README.md @@ -328,11 +328,7 @@ steps: cache-from: - myservice_intermediate:index.docker.io/myorg/myrepo/myservice_intermediate:${BUILDKITE_BRANCH} - myservice_intermediate:index.docker.io/myorg/myrepo/myservice_intermediate:latest - run: test - push: - - myservice_intermediate:index.docker.io/myorg/myrepo/myservice_intermediate:${BUILDKITE_BRANCH} - wait - - label: ":docker: Build Final Image" plugins: - docker-compose#v3.9.0: From 970b15d9036d5a14199c9cce69dd0c4c8da4c69e Mon Sep 17 00:00:00 2001 From: "boomper-bot[bot]" <40539397+boomper-bot[bot]@users.noreply.github.com> Date: Wed, 29 Jun 2022 22:09:41 +0000 Subject: [PATCH 040/253] Bump README.md for v3.10.0 release --- README.md | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 144a63c2..86178c7f 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v3.9.0: + - docker-compose#v3.10.0: run: app ``` @@ -26,7 +26,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v3.9.0: + - docker-compose#v3.10.0: run: app config: docker-compose.tests.yml env: @@ -39,7 +39,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v3.9.0: + - docker-compose#v3.10.0: run: app config: - docker-compose.yml @@ -54,7 +54,7 @@ env: steps: - command: test.sh plugins: - - docker-compose#v3.9.0: + - docker-compose#v3.10.0: run: app ``` @@ -65,7 +65,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.9.0: + - docker-compose#v3.10.0: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -73,7 +73,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.9.0: + - docker-compose#v3.10.0: run: app ``` @@ -82,7 +82,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v3.9.0: + - docker-compose#v3.10.0: run: app command: ["custom", "command", "values"] ``` @@ -98,7 +98,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.9.0: + - docker-compose#v3.10.0: run: app ``` @@ -116,7 +116,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.9.0: + - docker-compose#v3.10.0: run: app volumes: - "./dist:/app/dist" @@ -137,7 +137,7 @@ this plugin offers a `environment` block of it's own: steps: - command: generate-dist.sh plugins: - - docker-compose#v3.9.0: + - docker-compose#v3.10.0: run: app env: - BUILDKITE_BUILD_NUMBER @@ -157,7 +157,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v3.9.0: + - docker-compose#v3.10.0: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -174,7 +174,7 @@ To speed up run steps that use the same service/image (such as steps that run in steps: - label: ":docker: Build" plugins: - - docker-compose#v3.9.0: + - docker-compose#v3.10.0: build: app image-repository: index.docker.io/myorg/myrepo @@ -184,7 +184,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.9.0: + - docker-compose#v3.10.0: run: app ``` @@ -200,7 +200,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v3.9.0: + - docker-compose#v3.10.0: build: - app - tests @@ -212,7 +212,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.9.0: + - docker-compose#v3.10.0: run: tests ``` @@ -224,7 +224,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v3.9.0: + - docker-compose#v3.10.0: push: app ``` @@ -236,7 +236,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.9.0: + - docker-compose#v3.10.0: push: app ``` @@ -248,7 +248,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.9.0: + - docker-compose#v3.10.0: push: - first-service - second-service @@ -262,7 +262,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.9.0: + - docker-compose#v3.10.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -276,14 +276,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker: Build an image" plugins: - - docker-compose#v3.9.0: + - docker-compose#v3.10.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v3.9.0: + - docker-compose#v3.10.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest From 3aea26a0c53b00ca9cf1ed8b5b4a00297150ed69 Mon Sep 17 00:00:00 2001 From: Vladimir Lyzo Date: Fri, 16 Sep 2022 13:41:15 +1000 Subject: [PATCH 041/253] Add docker-compose version 2 opt-in support --- lib/shared.bash | 7 ++++++- tests/build.bats | 17 +++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/lib/shared.bash b/lib/shared.bash index c712fc50..766ddd43 100644 --- a/lib/shared.bash +++ b/lib/shared.bash @@ -173,7 +173,12 @@ function build_image_override_file_with_version() { # Runs the docker-compose command, scoped to the project, with the given arguments function run_docker_compose() { - local command=(docker-compose) + local command + if [[ "$(plugin_read_config CLI_VERSION "1")" == "1" ]] ; then + command=(docker-compose) + else + command=(docker compose) + fi if [[ "$(plugin_read_config VERBOSE "false")" == "true" ]] ; then command+=(--verbose) diff --git a/tests/build.bats b/tests/build.bats index 50a00b45..51e27f53 100644 --- a/tests/build.bats +++ b/tests/build.bats @@ -75,6 +75,23 @@ load '../lib/shared' unstub docker-compose } +@test "Build with docker-compose v2" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLI_VERSION=2 + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + unstub docker +} + @test "Build with a repository" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice From 46e5caabc30e0c63486f89ae82e4d8a38f88414f Mon Sep 17 00:00:00 2001 From: Vladimir Lyzo Date: Fri, 16 Sep 2022 14:59:38 +1000 Subject: [PATCH 042/253] Update README --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 86178c7f..5ec80ea5 100644 --- a/README.md +++ b/README.md @@ -479,6 +479,10 @@ Select when to upload container logs. The default is `on-error`. +### `cli-version` (optional) + +If set to `1`, plugin will use `docker-compose` executable, else `docker compose` will be used. + ## Developing To run the tests: From dea62086a9820dd08962ad0270f3dcb229eea55a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Fri, 16 Sep 2022 17:57:54 -0300 Subject: [PATCH 043/253] Fix linter issues (example had build and push in the same step) --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index b6c69657..96a30760 100644 --- a/README.md +++ b/README.md @@ -339,9 +339,7 @@ steps: - myservice:index.docker.io/myorg/myrepo/myservice_intermediate:buildkite-build-${BUILDKITE_BUILD_NUMBER}:intermediate # built in step above - myservice:index.docker.io/myorg/myrepo/myservice:${BUILDKITE_BRANCH} - myservice:index.docker.io/myorg/myrepo/myservice:latest - push: - - myservice:index.docker.io/myorg/myrepo/myservice:${BUILDKITE_BRANCH} - - myservice:index.docker.io/myorg/myrepo/myservice:latest + ``` In the example above, the `myservice_intermediate:buildkite-build-${BUILDKITE_BUILD_NUMBER}` is one group named "intermediate", and `myservice:${BUILDKITE_BRANCH}` and `myservice:latest` From b769afc9c29edcf05977290247daa62fae0817ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Fri, 16 Sep 2022 18:01:50 -0300 Subject: [PATCH 044/253] Forcing old version of the plugin-tester to avoid breaking tests --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8023abd9..fe299c6b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ version: '2' services: tests: - image: buildkite/plugin-tester + image: buildkite/plugin-tester:v2.0.0 volumes: - ".:/plugin" From 5173e1f82ead5ca733c52672ac63e41fa16c1c59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Fri, 16 Sep 2022 19:49:08 -0300 Subject: [PATCH 045/253] Added group name to output and better clarification of the options in the readme --- README.md | 3 +-- commands/build.sh | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 96a30760..ed229a6e 100644 --- a/README.md +++ b/README.md @@ -314,8 +314,7 @@ steps: ``` You may actually want to build your image with multiple cache-from values, for instance, with the cached images of multiple stages in a multi-stage build. -Adding a grouping tag to the end of a cache-from list item allows this plugin to differentiate between groups within which only the first successfully downloaded image should be used. -This way, not all images need to be downloaded and used as cache, not just the first. +Adding a grouping tag to the end of a cache-from list item allows this plugin to differentiate between groups within which only the first successfully downloaded image should be used (those elements that don't have a group specified will make a separate `:default:` group of its own). This way, not all images need to be downloaded and used as cache, not just the first. ```yaml steps: diff --git a/commands/build.sh b/commands/build.sh index 91ca055f..cf660eeb 100755 --- a/commands/build.sh +++ b/commands/build.sh @@ -55,7 +55,7 @@ if [[ "$(plugin_read_config NO_CACHE "false")" == "false" ]] ; then fi fi - echo "~~~ :docker: Pulling cache image for $service_name" + echo "~~~ :docker: Pulling cache image for $service_name (group ${cache_from_group_name})" if retry "$pull_retries" plugin_prompt_and_run docker pull "$service_image" ; then if [[ -z "${!cache_image_name+x}" ]]; then declare -a "$cache_image_name" From 5f05983392727e8598cbf84ac808b6419d6ebf19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Fri, 16 Sep 2022 19:50:00 -0300 Subject: [PATCH 046/253] Added two more tests for more comprehensive tests of the feature --- tests/build.bats | 76 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/tests/build.bats b/tests/build.bats index 361735df..c95388c4 100644 --- a/tests/build.bats +++ b/tests/build.bats @@ -354,6 +354,82 @@ load '../lib/shared' unstub docker-compose } +@test "Build with several cache-from image groups for one service with failures" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=helloworld:my.repository/myservice_cache:build-target-build-1:target1 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_1=helloworld:my.repository/myservice_cache:build-target-latest:target1 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_2=helloworld:my.repository/myservice_cache:install-target-build-1:target2 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_3=helloworld:my.repository/myservice_cache:branch-name + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_4=helloworld:my.repository/myservice_cache:latest + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "pull my.repository/myservice_cache:build-target-build-1 : exit 1" \ + "pull my.repository/myservice_cache:build-target-latest : echo pulled cache image build-target-latest" \ + "pull my.repository/myservice_cache:install-target-build-1 : echo pulled cache image install-target" \ + "pull my.repository/myservice_cache:branch-name : echo pulled cache image branch-name" + + stub docker-compose \ + "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" + + run $PWD/hooks/command + + assert_success + assert_output --partial "pulled cache image build-target-latest" + assert_output --partial "pulled cache image install-target" + assert_output --partial "pulled cache image branch-name" + assert_output --partial "- my.repository/myservice_cache:build-target-latest" + assert_output --partial "- my.repository/myservice_cache:build-target-latest" + assert_output --partial "- my.repository/myservice_cache:install-target-build-1" + assert_output --partial "- my.repository/myservice_cache:branch-name" + refute_output --partial "- my.repository/myservice_cache:latest" + assert_output --partial "built helloworld" + unstub docker + unstub docker-compose +} + +@test "Build with several cache-from image groups out of order" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=helloworld:my.repository/myservice_cache:branch-name + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_1=helloworld:my.repository/myservice_cache:build-target-build-1:target1 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_2=helloworld:my.repository/myservice_cache:install-target-build-1:target2 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_3=helloworld:my.repository/myservice_cache:build-target-latest:target1 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_4=helloworld:my.repository/myservice_cache:latest + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "pull my.repository/myservice_cache:branch-name : exit 1" \ + "pull my.repository/myservice_cache:build-target-build-1 : exit 1" \ + "pull my.repository/myservice_cache:install-target-build-1 : echo pulled cache image install-target" \ + "pull my.repository/myservice_cache:build-target-latest : echo pulled cache image build-target-latest" \ + "pull my.repository/myservice_cache:latest : echo pulled cache image branch-name-latest" + + + stub docker-compose \ + "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" + + run $PWD/hooks/command + + assert_success + assert_output --partial "pulled cache image build-target-latest" + assert_output --partial "pulled cache image install-target" + assert_output --partial "pulled cache image branch-name" + assert_output --partial "- my.repository/myservice_cache:build-target-latest" + assert_output --partial "- my.repository/myservice_cache:build-target-latest" + assert_output --partial "- my.repository/myservice_cache:install-target-build-1" + refute_output --partial "- my.repository/myservice_cache:branch-name" + assert_output --partial "- my.repository/myservice_cache:latest" + assert_output --partial "built helloworld" + unstub docker + unstub docker-compose +} + @test "Build with several cache-from images for one service with first image being not available" { export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" export BUILDKITE_JOB_ID=1111 From 3b2933725432dd4fe8a62446e5c39fde2969964b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Fri, 16 Sep 2022 21:36:12 -0300 Subject: [PATCH 047/253] allow build and run at the same time --- plugin.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/plugin.yml b/plugin.yml index b0a1d41f..93ae0e4d 100644 --- a/plugin.yml +++ b/plugin.yml @@ -79,10 +79,11 @@ configuration: entrypoint: type: string oneOf: - - required: - - run - - required: - - build + - anyOf: + - required: + - run + - required: + - build - required: - push additionalProperties: false From fa62f5009915c339eb7212dc2da5e4a78b924960 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Fri, 16 Sep 2022 21:43:08 -0300 Subject: [PATCH 048/253] Add example in documentation about build & run --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 649eee60..96dc36e8 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,19 @@ steps: run: app ``` +Alternatively, you can do everything in a single step: +```yml +steps: + - command: test.sh + plugins: + - docker-login#v2.0.1: + username: xyz + - docker-compose#v3.11.0: + build: app + image-repository: index.docker.io/myorg/myrepo + run: app +``` + If you want to control how your command is passed to docker-compose, you can use the command parameter on the plugin directly: ```yml From a3b35d1fe23265d9f70e7ccae408052aacad45e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Fri, 16 Sep 2022 21:50:41 -0300 Subject: [PATCH 049/253] pin plugin tester to older version --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8023abd9..fe299c6b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ version: '2' services: tests: - image: buildkite/plugin-tester + image: buildkite/plugin-tester:v2.0.0 volumes: - ".:/plugin" From 68ce423c8563b45c96a60e1eb2458d9e9461d696 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Fri, 16 Sep 2022 22:35:40 -0300 Subject: [PATCH 050/253] Pin plugin tester to latest version --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8023abd9..eb680534 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ version: '2' services: tests: - image: buildkite/plugin-tester + image: buildkite/plugin-tester:v3.0.0 volumes: - ".:/plugin" From 832ed0d551124555602df470692579626276ba00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 17 Sep 2022 02:49:17 -0300 Subject: [PATCH 051/253] Add workaround for huge bats-mock bug --- lib/shared.bash | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/shared.bash b/lib/shared.bash index c712fc50..b847ed6a 100644 --- a/lib/shared.bash +++ b/lib/shared.bash @@ -1,5 +1,12 @@ #!/bin/bash +BATS_MOCK_FILE='/usr/local/lib/bats/bats-mock/binstub' +if ! grep -q '#BUG-CORRECTED' $BATS_MOCK_FILE; then + echo 'Applied fix to bats-mock' >&3 + sed -i "$(sed -n '/\${!_STUB_RESULT}/ =' $BATS_MOCK_FILE | tail -n 1)"' s/${!_STUB_RESULT}/$status/' $BATS_MOCK_FILE + echo '#BUG-CORRECTED' >> $BATS_MOCK_FILE +fi + # Show a prompt for a command function plugin_prompt() { if [[ -z "${HIDE_PROMPT:-}" ]] ; then From 858b5ea3ecd410a40d8588f811a5cab6c978fb83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 17 Sep 2022 02:58:29 -0300 Subject: [PATCH 052/253] Corrected tests in output.bats --- tests/output.bats | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/output.bats b/tests/output.bats index 5cd617e3..3139f7ca 100644 --- a/tests/output.bats +++ b/tests/output.bats @@ -34,9 +34,9 @@ load '../lib/run' stub docker \ "ps -a --filter label=com.docker.compose.project=buildkite1111 -q : cat tests/fixtures/id-multiple-services.txt" \ - "inspect -f {{if\ ne\ 0\ .State.ExitCode}}{{.Name}}.{{.State.ExitCode}}{{\ end\ }} 456456 : echo 456456.1" \ - "ps -a --filter label=com.docker.compose.project=buildkite1111 --format : cat tests/fixtures/service-id-exit-multiple-services-failed.txt" \ - "ps -a --filter label=com.docker.compose.project=buildkite1111 --format : cat tests/fixtures/id-service-multiple-services.txt" \ + "inspect -f {{if\ ne\ 0\ .State.ExitCode}}{{.Name}}.{{.State.ExitCode}}{{\ end\ }} 456456 789789 : echo 456456.1" \ + "ps -a --filter label=com.docker.compose.project=buildkite1111 --format \* : cat tests/fixtures/service-id-exit-multiple-services-failed.txt" \ + "ps -a --filter label=com.docker.compose.project=buildkite1111 --format \* : cat tests/fixtures/id-service-multiple-services.txt" \ "inspect --format={{.State.ExitCode}} 456456 : echo 1" \ "logs --timestamps --tail 5 456456 : exit 0" \ "logs -t 456456 : exit 0" \ @@ -71,13 +71,13 @@ load '../lib/run' stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice echo 'hello world' : echo ran myservice" stub docker \ "ps -a --filter label=com.docker.compose.project=buildkite1111 -q : cat tests/fixtures/id-multiple-services.txt" \ "inspect -f {{if\ ne\ 0\ .State.ExitCode}}{{.Name}}.{{.State.ExitCode}}{{\ end\ }} 456456 789789 : echo" \ - "ps -a --filter : cat tests/fixtures/id-service-multiple-services.txt" \ + "ps -a --filter label=com.docker.compose.project=buildkite1111 --format \* : cat tests/fixtures/id-service-multiple-services.txt" \ "inspect --format={{.State.ExitCode}} 456456 : echo 0" \ "inspect --format={{.State.ExitCode}} 789789 : echo 0" From d127286e875b16d80cf1afab2a0c6b7e9904858c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 17 Sep 2022 03:17:49 -0300 Subject: [PATCH 053/253] Corrected tests --- tests/run.bats | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/run.bats b/tests/run.bats index edf0b550..d30908f7 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -99,7 +99,7 @@ load '../lib/run' stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ - "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'sh -c \'echo hello world\'' : echo ran myservice" + "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c $'sh -c \'echo hello world\'' : echo ran myservice" stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" @@ -127,7 +127,7 @@ cmd3" stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ - "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'cmd1\ncmd2\ncmd3' : echo ran myservice" + "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c $'cmd1\ncmd2\ncmd3' : echo ran myservice" stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" @@ -155,7 +155,7 @@ cmd3" stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ - "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice echo hello world' : echo ran myservice" + "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice echo 'hello world' : echo ran myservice" stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" @@ -733,7 +733,7 @@ export BUILDKITE_JOB_ID=1111 stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ - "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --user=1000 myservice /bin/sh -e -c 'sh -c \'whoami\'' : echo ran myservice" + "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --user=1000 --rm myservice /bin/sh -e -c $'sh -c \'whoami\'' : echo ran myservice" stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" @@ -755,12 +755,12 @@ export BUILDKITE_JOB_ID=1111 export BUILDKITE_COMMAND="sh -c 'whoami'" export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false - export BUILDKITE_PLUGIN_DOCKER_COMPOSE_USER="1000" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_USER="1000:1001" stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ - "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --user=1000:1000 myservice /bin/sh -e -c 'sh -c \'whoami\'' : echo ran myservice" + "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --user=1000:1001 --rm myservice /bin/sh -e -c $'sh -c \'whoami\'' : echo ran myservice" stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" @@ -787,7 +787,7 @@ export BUILDKITE_JOB_ID=1111 stub docker-compose \ "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 myservice /bin/sh -e -c 'pwd' : echo ran myservice without tty" + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 myservice /bin/sh -e -c $'pwd' : echo ran myservice without tty" stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : echo myimage" \ From 93d5f018a18ee40bc0b9f76459b6fdd80d3203fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 17 Sep 2022 03:18:24 -0300 Subject: [PATCH 054/253] Added a basic test for propagate UID/GIDs option --- tests/run.bats | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/run.bats b/tests/run.bats index d30908f7..94daae9a 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -774,6 +774,29 @@ export BUILDKITE_JOB_ID=1111 unstub buildkite-agent } +@test "Fail with custom user and propagate UIDs" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="sh -c 'whoami'" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_USER="1000:1001" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PROPAGATE_UID_GID="true" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run $PWD/hooks/command + + assert_failure + assert_output --partial "Error" + assert_output --partial "Can't set both user and propagate-uid-gid" + unstub buildkite-agent +} + + @test "Run without --rm" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice From 62bf4a9ce44bfaf1d74979ed9e9b07317bdc59f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 17 Sep 2022 03:45:00 -0300 Subject: [PATCH 055/253] Added better condition for bug workaround --- lib/shared.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/shared.bash b/lib/shared.bash index b847ed6a..eac6dfcd 100644 --- a/lib/shared.bash +++ b/lib/shared.bash @@ -1,7 +1,7 @@ #!/bin/bash BATS_MOCK_FILE='/usr/local/lib/bats/bats-mock/binstub' -if ! grep -q '#BUG-CORRECTED' $BATS_MOCK_FILE; then +if [[ -e $BATS_MOCK_FILE ]] && ! grep -q '#BUG-CORRECTED' $BATS_MOCK_FILE; then echo 'Applied fix to bats-mock' >&3 sed -i "$(sed -n '/\${!_STUB_RESULT}/ =' $BATS_MOCK_FILE | tail -n 1)"' s/${!_STUB_RESULT}/$status/' $BATS_MOCK_FILE echo '#BUG-CORRECTED' >> $BATS_MOCK_FILE From f6b6d66352736351e6e44cb46e2bddea43d4f90a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 17 Sep 2022 19:02:21 -0300 Subject: [PATCH 056/253] Remove workaround now that the bug has been fixed --- docker-compose.yml | 2 +- lib/shared.bash | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index eb680534..0ed315f5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ version: '2' services: tests: - image: buildkite/plugin-tester:v3.0.0 + image: buildkite/plugin-tester:v3.0.1 volumes: - ".:/plugin" diff --git a/lib/shared.bash b/lib/shared.bash index eac6dfcd..c712fc50 100644 --- a/lib/shared.bash +++ b/lib/shared.bash @@ -1,12 +1,5 @@ #!/bin/bash -BATS_MOCK_FILE='/usr/local/lib/bats/bats-mock/binstub' -if [[ -e $BATS_MOCK_FILE ]] && ! grep -q '#BUG-CORRECTED' $BATS_MOCK_FILE; then - echo 'Applied fix to bats-mock' >&3 - sed -i "$(sed -n '/\${!_STUB_RESULT}/ =' $BATS_MOCK_FILE | tail -n 1)"' s/${!_STUB_RESULT}/$status/' $BATS_MOCK_FILE - echo '#BUG-CORRECTED' >> $BATS_MOCK_FILE -fi - # Show a prompt for a command function plugin_prompt() { if [[ -z "${HIDE_PROMPT:-}" ]] ; then From 67fafb33b40ac7cde13f02fb8bacf13d3db147ab Mon Sep 17 00:00:00 2001 From: Pol Date: Sat, 17 Sep 2022 19:44:07 -0300 Subject: [PATCH 057/253] Add wait step to fix race condition Pre build steps use metadata with the same key leading to a race condition --- .buildkite/pipeline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index fb53e971..7528c334 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -114,6 +114,8 @@ steps: config: tests/composefiles/docker-compose.v2.1.yml commmand: ["/hello"] + - wait: ~ + - label: prebuild with custom image-name key: prebuild-custom-image-name env: From 71b9bcb9ee98efe0e4e86817dcb312b3b90fa3b6 Mon Sep 17 00:00:00 2001 From: Vladimir Lyzo Date: Mon, 19 Sep 2022 09:43:06 +1000 Subject: [PATCH 058/253] Update README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matías Bellone --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5ec80ea5..67700742 100644 --- a/README.md +++ b/README.md @@ -481,7 +481,7 @@ The default is `on-error`. ### `cli-version` (optional) -If set to `1`, plugin will use `docker-compose` executable, else `docker compose` will be used. +If set to `2`, plugin will use `docker compose` to execute commands; otherwise it will default to version `1` using `docker-compose` instead. ## Developing From 26fe4d12937e36e67f306ceb7227e45e1199d9f6 Mon Sep 17 00:00:00 2001 From: Vladimir Lyzo Date: Mon, 19 Sep 2022 09:51:16 +1000 Subject: [PATCH 059/253] Add test for v1 and add property --- plugin.yml | 2 ++ tests/build.bats | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/plugin.yml b/plugin.yml index b0a1d41f..b36c4051 100644 --- a/plugin.yml +++ b/plugin.yml @@ -78,6 +78,8 @@ configuration: type: boolean entrypoint: type: string + cli-version: + type: string oneOf: - required: - run diff --git a/tests/build.bats b/tests/build.bats index 51e27f53..65986aaf 100644 --- a/tests/build.bats +++ b/tests/build.bats @@ -75,6 +75,23 @@ load '../lib/shared' unstub docker-compose } +@test "Build with docker-compose and v1 is set explicitly " { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLI_VERSION=1 + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + unstub docker +} + @test "Build with docker-compose v2" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice From b6581ae7c57c9b93c57ef6fa9df9f17107a475cc Mon Sep 17 00:00:00 2001 From: Vladimir Lyzo Date: Mon, 19 Sep 2022 10:04:49 +1000 Subject: [PATCH 060/253] Add options to property --- plugin.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugin.yml b/plugin.yml index b36c4051..dfcecb56 100644 --- a/plugin.yml +++ b/plugin.yml @@ -80,6 +80,9 @@ configuration: type: string cli-version: type: string + enum: + - 1 + - 2 oneOf: - required: - run From 077a1e37f1c6f121c1cf629e65753dd876148023 Mon Sep 17 00:00:00 2001 From: Vladimir Lyzo Date: Mon, 19 Sep 2022 10:07:50 +1000 Subject: [PATCH 061/253] Fix test --- tests/build.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/build.bats b/tests/build.bats index 65986aaf..c1072e7e 100644 --- a/tests/build.bats +++ b/tests/build.bats @@ -89,7 +89,7 @@ load '../lib/shared' assert_success assert_output --partial "built myservice" - unstub docker + unstub docker-compose } @test "Build with docker-compose v2" { From dbed4f99fba31921b3e4f1ac7534638a5199236e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sun, 18 Sep 2022 21:18:18 -0300 Subject: [PATCH 062/253] Added new test for versionless config file --- tests/composefiles/docker-compose.no-version.yml | 2 ++ tests/docker-compose-config.bats | 7 +++++++ 2 files changed, 9 insertions(+) create mode 100644 tests/composefiles/docker-compose.no-version.yml diff --git a/tests/composefiles/docker-compose.no-version.yml b/tests/composefiles/docker-compose.no-version.yml new file mode 100644 index 00000000..2f420f69 --- /dev/null +++ b/tests/composefiles/docker-compose.no-version.yml @@ -0,0 +1,2 @@ +helloworld: + build: . diff --git a/tests/docker-compose-config.bats b/tests/docker-compose-config.bats index b653049e..a647b577 100644 --- a/tests/docker-compose-config.bats +++ b/tests/docker-compose-config.bats @@ -52,6 +52,13 @@ load '../lib/shared' assert_output "2.1" } +@test "Read version from docker-compose file with empty version" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.no-version.yml" + run docker_compose_config_version + assert_success + assert_output "" +} + @test "Whether docker-compose supports cache_from directive" { run docker_compose_supports_cache_from "" assert_success From 14f0c4cf09eebb455ec807e48d0800fdabe0e5f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sun, 18 Sep 2022 22:15:44 -0300 Subject: [PATCH 063/253] Make version more robust: only find the most top-level version --- lib/shared.bash | 2 +- ...docker-compose.v2.0.with-version-arg-and-whitespace.yml | 7 +++++++ .../composefiles/docker-compose.v3.2.with-version-arg.yml | 4 ++-- tests/docker-compose-config.bats | 7 +++++++ 4 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 tests/composefiles/docker-compose.v2.0.with-version-arg-and-whitespace.yml diff --git a/lib/shared.bash b/lib/shared.bash index 8a12c5a5..ed945625 100644 --- a/lib/shared.bash +++ b/lib/shared.bash @@ -115,7 +115,7 @@ function docker_compose_config_files() { # Returns the version from the output of docker_compose_config function docker_compose_config_version() { IFS=$'\n' read -r -a config <<< "$(docker_compose_config_files)" - awk '/^\s*version:/ { print $2; exit; }' < "${config[0]}" | sed "s/[\"']//g" + grep 'version' < "${config[0]}" | sort -r | awk '/^\s*version:/ { print $2; exit; }' | sed "s/[\"']//g" } # Build an docker-compose file that overrides the image for a set of diff --git a/tests/composefiles/docker-compose.v2.0.with-version-arg-and-whitespace.yml b/tests/composefiles/docker-compose.v2.0.with-version-arg-and-whitespace.yml new file mode 100644 index 00000000..dc2e9643 --- /dev/null +++ b/tests/composefiles/docker-compose.v2.0.with-version-arg-and-whitespace.yml @@ -0,0 +1,7 @@ + services: + helloworld: + build: + context: . + args: + version: 73.976.3 + version: '2.0' \ No newline at end of file diff --git a/tests/composefiles/docker-compose.v3.2.with-version-arg.yml b/tests/composefiles/docker-compose.v3.2.with-version-arg.yml index 94cf74b4..3256547f 100644 --- a/tests/composefiles/docker-compose.v3.2.with-version-arg.yml +++ b/tests/composefiles/docker-compose.v3.2.with-version-arg.yml @@ -1,8 +1,8 @@ -version: "3.2" - services: helloworld: build: context: . args: version: 73.976.3 + +version: "3.2" \ No newline at end of file diff --git a/tests/docker-compose-config.bats b/tests/docker-compose-config.bats index 06e8ea2a..eb1b27d1 100644 --- a/tests/docker-compose-config.bats +++ b/tests/docker-compose-config.bats @@ -67,6 +67,13 @@ load '../lib/shared' assert_output "3.2" } +@test "Read version given docker-compose file with argument named version and whitespace" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG_0="tests/composefiles/docker-compose.v2.0.with-version-arg-and-whitespace.yml" + run docker_compose_config_version + assert_success + assert_output "2.0" +} + @test "Whether docker-compose supports cache_from directive" { run docker_compose_supports_cache_from "" assert_failure From 296882db86df0e7ca59865635db57af28cfec4ca Mon Sep 17 00:00:00 2001 From: Vladimir Lyzo Date: Mon, 19 Sep 2022 14:18:49 +1000 Subject: [PATCH 064/253] Apply proposed correction --- lib/shared.bash | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/shared.bash b/lib/shared.bash index 766ddd43..d4105c78 100644 --- a/lib/shared.bash +++ b/lib/shared.bash @@ -173,10 +173,8 @@ function build_image_override_file_with_version() { # Runs the docker-compose command, scoped to the project, with the given arguments function run_docker_compose() { - local command - if [[ "$(plugin_read_config CLI_VERSION "1")" == "1" ]] ; then - command=(docker-compose) - else + local command=(docker-compose) + if [[ "$(plugin_read_config CLI_VERSION "1")" == "2" ]] ; then command=(docker compose) fi From 96287d8e95066ee768f453ec422a690530a6ab02 Mon Sep 17 00:00:00 2001 From: Opal Symes Date: Wed, 29 Jun 2022 13:10:33 +1200 Subject: [PATCH 065/253] Upload logs when services fail to start Fixes Container logs are not uploaded when dependencies fail to start #327 --- commands/run.sh | 54 ++++++++++++++++++----------------------- lib/run.bash | 39 +++++++++++++++++++++++++++++ tests/output.bats | 62 +++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 117 insertions(+), 38 deletions(-) diff --git a/commands/run.sh b/commands/run.sh index 1fc39aeb..9af11c64 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -241,15 +241,35 @@ fi # Start up service dependencies in a different header to keep the main run with less noise if [[ "$(plugin_read_config DEPENDENCIES "true")" == "true" ]] ; then + exitcode=0 + echo "~~~ :docker: Starting dependencies" if [[ ${#up_params[@]} -gt 0 ]] ; then - run_docker_compose "${up_params[@]}" up -d --scale "${run_service}=0" "${run_service}" + if ! run_docker_compose "${up_params[@]}" up -d --scale "${run_service}=0" "${run_service}"; then + exitcode=1 + fi else - run_docker_compose up -d --scale "${run_service}=0" "${run_service}" + if ! run_docker_compose up -d --scale "${run_service}=0" "${run_service}"; then + exitcode=1 + fi fi # Sometimes docker-compose leaves unfinished ansi codes echo + + if [[ $exitcode -ne 0 ]] ; then + # Dependent services failed to start. + echo "^^^ +++" + echo "+++ 🚨 Failed to start dependencies" + + if [[ -n "${BUILDKITE_AGENT_ACCESS_TOKEN:-}" ]] ; then + print_failed_container_information + + upload_container_logs "$run_service" + fi + + return $exitcode + fi fi shell=() @@ -354,35 +374,9 @@ fi if [[ -n "${BUILDKITE_AGENT_ACCESS_TOKEN:-}" ]] ; then if [[ "$(plugin_read_config CHECK_LINKED_CONTAINERS "true")" != "false" ]] ; then + print_failed_container_information - # Get list of failed containers - containers=() - while read -r container ; do - [[ -n "$container" ]] && containers+=("$container") - done <<< "$(docker_ps_by_project -q)" - - failed_containers=() - if [[ 0 != "${#containers[@]}" ]] ; then - while read -r container ; do - [[ -n "$container" ]] && failed_containers+=("$container") - done <<< "$(docker inspect -f '{{if ne 0 .State.ExitCode}}{{.Name}}.{{.State.ExitCode}}{{ end }}' \ - "${containers[@]}")" - fi - - if [[ 0 != "${#failed_containers[@]}" ]] ; then - echo "+++ :warning: Some containers had non-zero exit codes" - docker_ps_by_project \ - --format 'table {{.Label "com.docker.compose.service"}}\t{{ .ID }}\t{{ .Status }}' - fi - - check_linked_containers_and_save_logs \ - "$run_service" "docker-compose-logs" \ - "$(plugin_read_config UPLOAD_CONTAINER_LOGS "on-error")" - - if [[ -d "docker-compose-logs" ]] && test -n "$(find docker-compose-logs/ -maxdepth 1 -name '*.log' -print)"; then - echo "~~~ Uploading linked container logs" - buildkite-agent artifact upload "docker-compose-logs/*.log" - fi + upload_container_logs "$run_service" fi fi diff --git a/lib/run.bash b/lib/run.bash index bd057ab7..3c88be92 100644 --- a/lib/run.bash +++ b/lib/run.bash @@ -88,3 +88,42 @@ expand_relative_volume_path() { echo "${path/.\//$pwd/}" } + +# Prints information about the failed containers. +function print_failed_container_information() { + # Get list of failed containers + containers=() + while read -r container ; do + [[ -n "$container" ]] && containers+=("$container") + done <<< "$(docker_ps_by_project -q)" + + failed_containers=() + if [[ 0 != "${#containers[@]}" ]] ; then + while read -r container ; do + [[ -n "$container" ]] && failed_containers+=("$container") + done <<< "$(docker inspect -f '{{if ne 0 .State.ExitCode}}{{.Name}}.{{.State.ExitCode}}{{ end }}' \ + "${containers[@]}")" + fi + + if [[ 0 != "${#failed_containers[@]}" ]] ; then + echo "+++ :warning: Some containers had non-zero exit codes" + docker_ps_by_project \ + --format 'table {{.Label "com.docker.compose.service"}}\t{{ .ID }}\t{{ .Status }}' + fi +} + +# Uploads the container's logs, respecting the `UPLOAD_CONTAINER_LOGS` option +function upload_container_logs() { + run_service="$1" + + if [[ -n "${BUILDKITE_AGENT_ACCESS_TOKEN:-}" ]] ; then + check_linked_containers_and_save_logs \ + "$run_service" "docker-compose-logs" \ + "$(plugin_read_config UPLOAD_CONTAINER_LOGS "on-error")" + + if [[ -d "docker-compose-logs" ]] && test -n "$(find docker-compose-logs/ -maxdepth 1 -name '*.log' -print)"; then + echo "~~~ Uploading linked container logs" + buildkite-agent artifact upload "docker-compose-logs/*.log" + fi + fi +} diff --git a/tests/output.bats b/tests/output.bats index 3139f7ca..c4a6f07a 100644 --- a/tests/output.bats +++ b/tests/output.bats @@ -29,8 +29,8 @@ load '../lib/run' stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ - "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice echo 'hello world' : echo ran myservice" + "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice echo 'hello world' : echo ran myservice command" stub docker \ "ps -a --filter label=com.docker.compose.project=buildkite1111 -q : cat tests/fixtures/id-multiple-services.txt" \ @@ -46,7 +46,51 @@ load '../lib/run' assert_success assert_output --partial "built myservice" - assert_output --partial "ran myservice" + assert_output --partial "ran myservice dependencies" + assert_output --partial "ran myservice command" + assert_output --partial "Some containers had non-zero exit codes" + unstub buildkite-agent + unstub docker-compose + unstub docker +} + +@test "Logs: Detect dependent services KO" { + # Test for Issue #327, Container logs are not uploaded when services fail to start. + export BUILDKITE_AGENT_ACCESS_TOKEN="123123" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_COMMAND_0=echo + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_COMMAND_1="hello world" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=true + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" \ + "artifact upload : exit 0" + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : exit 1" \ + + stub docker \ + "ps -a --filter label=com.docker.compose.project=buildkite1111 -q : cat tests/fixtures/id-multiple-services.txt" \ + "inspect -f {{if\ ne\ 0\ .State.ExitCode}}{{.Name}}.{{.State.ExitCode}}{{\ end\ }} 456456 789789 : echo 456456.1" \ + "ps -a --filter label=com.docker.compose.project=buildkite1111 --format \* : cat tests/fixtures/service-id-exit-multiple-services-failed.txt" \ + "ps -a --filter label=com.docker.compose.project=buildkite1111 --format \* : cat tests/fixtures/id-service-multiple-services.txt" \ + "inspect --format={{.State.ExitCode}} 456456 : echo 1" \ + "logs --timestamps --tail 5 456456 : exit 0" \ + "logs -t 456456 : exit 0" \ + "inspect --format={{.State.ExitCode}} 789789 : echo 0" + + run $PWD/hooks/command + + assert_failure + assert_output --partial "built myservice" + assert_output --partial "Failed to start dependencies" assert_output --partial "Some containers had non-zero exit codes" unstub buildkite-agent unstub docker-compose @@ -72,7 +116,7 @@ load '../lib/run' stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ - "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice echo 'hello world' : echo ran myservice" + "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice echo 'hello world' : echo ran myservice command" stub docker \ "ps -a --filter label=com.docker.compose.project=buildkite1111 -q : cat tests/fixtures/id-multiple-services.txt" \ @@ -85,7 +129,8 @@ load '../lib/run' assert_success assert_output --partial "built myservice" - assert_output --partial "ran myservice" + assert_output --partial "ran myservice dependencies" + assert_output --partial "ran myservice command" refute_output --partial "Some containers had non-zero exit codes" unstub docker unstub docker-compose @@ -113,8 +158,8 @@ load '../lib/run' stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ - "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice echo 'hello world' : echo ran myservice" + "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice echo 'hello world' : echo ran myservice command" stub docker \ "ps -a --filter label=com.docker.compose.project=buildkite1111 -q : echo" \ @@ -124,7 +169,8 @@ load '../lib/run' assert_success assert_output --partial "built myservice" - assert_output --partial "ran myservice" + assert_output --partial "ran myservice dependencies" + assert_output --partial "ran myservice command" refute_output --partial "Uploading linked container logs" unstub docker unstub docker-compose From 81d574974e41708ce735cdad7352737e87e4f702 Mon Sep 17 00:00:00 2001 From: Opal Symes Date: Thu, 22 Sep 2022 17:49:37 +1200 Subject: [PATCH 066/253] Updates & fixes from code review --- commands/run.sh | 32 ++++++++++++++------------------ tests/output.bats | 6 +++--- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/commands/run.sh b/commands/run.sh index 9af11c64..f1e4c653 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -239,37 +239,33 @@ elif [[ ! -f "$override_file" ]]; then echo fi -# Start up service dependencies in a different header to keep the main run with less noise +dependency_exitcode=0 if [[ "$(plugin_read_config DEPENDENCIES "true")" == "true" ]] ; then - exitcode=0 + # Start up service dependencies in a different header to keep the main run with less noise echo "~~~ :docker: Starting dependencies" if [[ ${#up_params[@]} -gt 0 ]] ; then - if ! run_docker_compose "${up_params[@]}" up -d --scale "${run_service}=0" "${run_service}"; then - exitcode=1 - fi + run_docker_compose "${up_params[@]}" up -d --scale "${run_service}=0" "${run_service}" || dependency_exitcode=$? else - if ! run_docker_compose up -d --scale "${run_service}=0" "${run_service}"; then - exitcode=1 - fi + run_docker_compose up -d --scale "${run_service}=0" "${run_service}" || dependency_exitcode=$? fi # Sometimes docker-compose leaves unfinished ansi codes echo +fi - if [[ $exitcode -ne 0 ]] ; then - # Dependent services failed to start. - echo "^^^ +++" - echo "+++ 🚨 Failed to start dependencies" - - if [[ -n "${BUILDKITE_AGENT_ACCESS_TOKEN:-}" ]] ; then - print_failed_container_information +if [[ $dependency_exitcode -ne 0 ]] ; then + # Dependent services failed to start. + echo "^^^ +++" + echo "+++ 🚨 Failed to start dependencies" - upload_container_logs "$run_service" - fi + if [[ -n "${BUILDKITE_AGENT_ACCESS_TOKEN:-}" ]] ; then + print_failed_container_information - return $exitcode + upload_container_logs "$run_service" fi + + return $exitcode fi shell=() diff --git a/tests/output.bats b/tests/output.bats index c4a6f07a..a7952cb7 100644 --- a/tests/output.bats +++ b/tests/output.bats @@ -25,7 +25,7 @@ load '../lib/run' stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" \ - "artifact upload : exit 0" + "artifact upload docker-compose-logs/\*.log : exit 0" stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ @@ -70,7 +70,7 @@ load '../lib/run' stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" \ - "artifact upload : exit 0" + "artifact upload docker-compose-logs/\*.log : exit 0" stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ @@ -163,7 +163,7 @@ load '../lib/run' stub docker \ "ps -a --filter label=com.docker.compose.project=buildkite1111 -q : echo" \ - "ps -a --filter : cat tests/fixtures/id-service-no-services.txt" + "ps -a --filter label=com.docker.compose.project=buildkite1111 --format '{{.ID}}\\t{{.Label \"com.docker.compose.service\"}}' : cat tests/fixtures/id-service-no-services.txt" run $PWD/hooks/command From 6a0db48e44e97642e04d7ddb84b98eb763c697f4 Mon Sep 17 00:00:00 2001 From: Opal Symes Date: Fri, 23 Sep 2022 10:31:31 +1200 Subject: [PATCH 067/253] Fix variable name --- commands/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/run.sh b/commands/run.sh index bcdaa892..9f15be28 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -265,7 +265,7 @@ if [[ $dependency_exitcode -ne 0 ]] ; then upload_container_logs "$run_service" fi - return $exitcode + return $dependency_exitcode fi shell=() From 59bb87d7250a72596c9f74009449d8de8ca6b0a5 Mon Sep 17 00:00:00 2001 From: Opal Symes Date: Fri, 23 Sep 2022 12:11:32 +1200 Subject: [PATCH 068/253] Update versions to v3.11.1 --- README.md | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 092519ae..d53066f5 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v3.10.0: + - docker-compose#v3.11.1: run: app ``` @@ -26,7 +26,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v3.10.0: + - docker-compose#v3.11.1: run: app config: docker-compose.tests.yml env: @@ -39,7 +39,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v3.10.0: + - docker-compose#v3.11.1: run: app config: - docker-compose.yml @@ -54,7 +54,7 @@ env: steps: - command: test.sh plugins: - - docker-compose#v3.10.0: + - docker-compose#v3.11.1: run: app ``` @@ -65,7 +65,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.10.0: + - docker-compose#v3.11.1: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -73,7 +73,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.10.0: + - docker-compose#v3.11.1: run: app ``` @@ -82,7 +82,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v3.10.0: + - docker-compose#v3.11.1: run: app command: ["custom", "command", "values"] ``` @@ -98,7 +98,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.10.0: + - docker-compose#v3.11.1: run: app ``` @@ -116,7 +116,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.10.0: + - docker-compose#v3.11.1: run: app volumes: - "./dist:/app/dist" @@ -137,7 +137,7 @@ this plugin offers a `environment` block of its own: steps: - command: generate-dist.sh plugins: - - docker-compose#v3.10.0: + - docker-compose#v3.11.1: run: app env: - BUILDKITE_BUILD_NUMBER @@ -157,7 +157,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v3.10.0: + - docker-compose#v3.11.1: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -174,7 +174,7 @@ To speed up run steps that use the same service/image (such as steps that run in steps: - label: ":docker: Build" plugins: - - docker-compose#v3.10.0: + - docker-compose#v3.11.1: build: app image-repository: index.docker.io/myorg/myrepo @@ -184,7 +184,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.10.0: + - docker-compose#v3.11.1: run: app ``` @@ -200,7 +200,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v3.10.0: + - docker-compose#v3.11.1: build: - app - tests @@ -212,7 +212,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.10.0: + - docker-compose#v3.11.1: run: tests ``` @@ -224,7 +224,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v3.10.0: + - docker-compose#v3.11.1: push: app ``` @@ -236,7 +236,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.10.0: + - docker-compose#v3.11.1: push: app ``` @@ -248,7 +248,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.10.0: + - docker-compose#v3.11.1: push: - first-service - second-service @@ -262,7 +262,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.10.0: + - docker-compose#v3.11.1: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -276,14 +276,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker: Build an image" plugins: - - docker-compose#v3.10.0: + - docker-compose#v3.11.1: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v3.10.0: + - docker-compose#v3.11.1: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest From 819fd57485224b4ef3e72a945c598ceae7316277 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Fri, 23 Sep 2022 16:54:44 -0300 Subject: [PATCH 069/253] Add warning about using step-level command array --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index d53066f5..8765d1a9 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,8 @@ steps: run: app ``` +:warning: Warning: you should not use this plugin with an array of commands at the step level. Execute a script in your repository, a single command separated by `;` or the plugin's [`command` option](#command-optional-run-only-array) instead. + You can also specify a custom Docker Compose config file and what environment to pass through if you need: From 1cfa9c048fb1922e0e7568011bc78a9b40376948 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Fri, 23 Sep 2022 16:55:05 -0300 Subject: [PATCH 070/253] Add warning on multiline commands to plugin execution output --- commands/run.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/commands/run.sh b/commands/run.sh index 9f15be28..eaa14c00 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -339,6 +339,12 @@ if [[ ${#shell[@]} -gt 0 ]] ; then fi if [[ -n "${BUILDKITE_COMMAND}" ]] ; then + if [[ $(echo "$BUILDKITE_COMMAND" | wc -l) -gt 1 ]]; then + # An array of commands in the step will be a single string with multiple lines + # This breaks a lot of things here so we will print a warning for user to be aware + echo "⚠️ Warning: The command received has multiple lines." + echo "⚠️ The Docker Compose Plugin does not correctly support step-level array commands." + fi run_params+=("${BUILDKITE_COMMAND}") display_command+=("'${BUILDKITE_COMMAND}'") elif [[ ${#command[@]} -gt 0 ]] ; then From 2135751bb647ffec683beb64b57e79578248bb6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Fri, 23 Sep 2022 17:00:52 -0300 Subject: [PATCH 071/253] Add output checking to tests --- tests/run.bats | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/run.bats b/tests/run.bats index 94daae9a..7b445310 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -54,6 +54,7 @@ load '../lib/run' run $PWD/hooks/command assert_success + refute_output --partial "The Docker Compose Plugin does not correctly support step-level array commands" assert_output --partial "built myservice" assert_output --partial "ran myservice" unstub docker-compose @@ -135,6 +136,7 @@ cmd3" run $PWD/hooks/command assert_success + assert_output --partial "The Docker Compose Plugin does not correctly support step-level array commands" assert_output --partial "built myservice" assert_output --partial "ran myservice" unstub docker-compose @@ -163,6 +165,7 @@ cmd3" run $PWD/hooks/command assert_success + refute_output --partial "The Docker Compose Plugin does not correctly support step-level array commands" assert_output --partial "built myservice" assert_output --partial "ran myservice" unstub docker-compose From 3bd45a989750fa63340db6a9cc1dc5ef37919c49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Fri, 23 Sep 2022 20:05:41 -0300 Subject: [PATCH 072/253] Deactivate renovate dependency dashboard --- renovate.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/renovate.json b/renovate.json index da407de5..1b4560b6 100644 --- a/renovate.json +++ b/renovate.json @@ -1,6 +1,7 @@ { "extends": [ - "config:base" + "config:base", + ":disableDependencyDashboard" ], "docker-compose": { "digest": { From bbf2fa37a6792de01ccdbf6568535859b9719131 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 24 Sep 2022 03:49:03 -0300 Subject: [PATCH 073/253] Added option to expand vars in volume configurations --- README.md | 8 ++++++++ lib/run.bash | 11 ++++++++++- plugin.yml | 3 +++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8765d1a9..bc146ff0 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,8 @@ steps: - "./dist:/app/dist" ``` +If you want to use environment variables in the `volumes` element, you will need to activate the (unsafe) option `expand-volume-vars`. + ## Environment By default, docker-compose makes whatever environment variables it gets available for @@ -462,6 +464,12 @@ A list of volumes to mount into the container. If a matching volume exists in th Additionally, volumes may be specified via the agent environment variable `BUILDKITE_DOCKER_DEFAULT_VOLUMES`, a `;` (semicolon) delimited list of mounts in the `-v` syntax. (Ex. `buildkite:/buildkite;./app:/app`). +### `expand-volume-vars` (optional, boolean, run only, unsafe) + +When set to true, it will activate interpolation of variables in the elements of the `volumes` configuration array. When turned off (the default), attempting to use variables will fail as the literal `$VARIABLE_NAME` string will be passed to the `-v` option. + +:warning: **Important:** this is considered an unsafe option as the most compatible way to achieve this is to run the strings through `eval` which could lead to arbitrary code execution or information leaking if you don't have complete control of the pipeline + ### `graceful-shutdown` (optional, run only) Gracefully shuts down all containers via 'docker-compose stop`. diff --git a/lib/run.bash b/lib/run.bash index 3c88be92..91697fdf 100644 --- a/lib/run.bash +++ b/lib/run.bash @@ -76,7 +76,14 @@ check_linked_containers_and_save_logs() { # # "./foo:/foo" => "/buildkite/builds/.../foo:/foo" expand_relative_volume_path() { - local path="$1" + local path + + if [[ "$(plugin_read_config EXPAND_VOLUME_VARS 'false')" == "true" ]]; then + path=$(eval echo "$1") + else + path="$1" + fi + local pwd="$PWD" # docker-compose's -v expects native paths on windows, so convert back. @@ -86,6 +93,8 @@ expand_relative_volume_path() { pwd="$(cygpath -w "$PWD")" fi + + echo "${path/.\//$pwd/}" } diff --git a/plugin.yml b/plugin.yml index b0a1d41f..b6478e0a 100644 --- a/plugin.yml +++ b/plugin.yml @@ -46,6 +46,8 @@ configuration: volumes: type: [ string, array ] minimum: 1 + expand-volume-vars: + type: boolean command: type: array skip-checkout: @@ -95,6 +97,7 @@ configuration: push-retries: [ push ] cache-from: [ build ] volumes: [ run ] + expand-volume-vars: [ volumes ] leave-volumes: [ run ] use-aliases: [ run ] dependencies: [ run ] From db2f7d6e02114ecdbc5ea89ff2302f29ccfe8813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 24 Sep 2022 03:49:14 -0300 Subject: [PATCH 074/253] Add tests to the new volume option --- tests/run.bats | 104 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/tests/run.bats b/tests/run.bats index 7b445310..916705f9 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -610,6 +610,110 @@ export BUILDKITE_JOB_ID=1111 unstub buildkite-agent } +@test "Run with volumes with variables" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_0="\$SUPER_VARIABLE:/mnt" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_1="/:\$OTHER_VARIABLE" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_2="\$RELATIVE_VARIABLE:/srv" + + export SUPER_VARIABLE='/test/path' + export OTHER_VARIABLE='/path/tested' + export RELATIVE_VARIABLE='./path' + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 -v \\\$SUPER_VARIABLE:/mnt -v /:\\\$OTHER_VARIABLE -v \\\$RELATIVE_VARIABLE:/srv --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with volumes" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run $PWD/hooks/command + + assert_success + assert_output --partial "ran myservice with volumes" + unstub docker-compose + unstub buildkite-agent +} + + +@test "Run with volumes with variables but option turned off" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_EXPAND_VOLUME_VARS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_0="\$SUPER_VARIABLE:/mnt" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_1="/:\$OTHER_VARIABLE" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_2="\$RELATIVE_VARIABLE:/srv" + + + export SUPER_VARIABLE='/test/path' + export OTHER_VARIABLE='/path/tested' + export RELATIVE_VARIABLE='./path' + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 -v \\\$SUPER_VARIABLE:/mnt -v /:\\\$OTHER_VARIABLE -v \\\$RELATIVE_VARIABLE:/srv --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with volumes" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run $PWD/hooks/command + + assert_success + assert_output --partial "ran myservice with volumes" + unstub docker-compose + unstub buildkite-agent +} + +@test "Run with volumes with variables and option turned on" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=true + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_EXPAND_VOLUME_VARS=true + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_0="\$SUPER_VARIABLE:/mnt" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_1="/:\$OTHER_VARIABLE" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_2="\$RELATIVE_VARIABLE:/srv" + + + export SUPER_VARIABLE='/test/path' + export OTHER_VARIABLE='/path/tested' + export RELATIVE_VARIABLE='./path' + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 -v /test/path:/mnt -v /:/path/tested -v $PWD/path:/srv --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with volumes" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run $PWD/hooks/command + + assert_success + assert_output --partial "ran myservice with volumes" + unstub docker-compose + unstub buildkite-agent +} + @test "Run with default volumes" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice From c88e071e6d13a5c9f1c7a061bd5d3ab61c28913d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sun, 25 Sep 2022 20:58:59 -0300 Subject: [PATCH 075/253] Simplified multiple-commands test file --- tests/multiple-commands.bats | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/tests/multiple-commands.bats b/tests/multiple-commands.bats index e88c8b2f..9224cd09 100644 --- a/tests/multiple-commands.bats +++ b/tests/multiple-commands.bats @@ -8,23 +8,26 @@ load '../lib/metadata' # export BUILDKITE_AGENT_STUB_DEBUG=/dev/tty # export BATS_MOCK_TMPDIR=$PWD +# General pipeline variables +export BUILDKITE_BUILD_NUMBER=1 +export BUILDKITE_COMMAND="pwd" +export BUILDKITE_JOB_ID=12 +export BUILDKITE_PIPELINE_SLUG=test + + @test "Build and run in a single step" { - export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice - export BUILDKITE_PIPELINE_SLUG=test - export BUILDKITE_BUILD_NUMBER=1 - export BUILDKITE_COMMAND="echo hello world" export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_REPOSITORY=my.repository/llamas export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false stub docker-compose \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml push myservice : echo pushed myservice" \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" + "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" \ + "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml push myservice : echo pushed myservice" \ + "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo ran dependencies" \ + "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml run --name buildkite12_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" stub buildkite-agent \ "meta-data set docker-compose-plugin-built-image-tag-myservice my.repository/llamas:test-myservice-build-1 : echo set meta-data" \ @@ -40,7 +43,7 @@ load '../lib/metadata' assert_output --partial "built myservice" assert_output --partial "pushed myservice" assert_output --partial "pulled myservice" - assert_output --partial "ran myservice dependencies" + assert_output --partial "ran dependencies" assert_output --partial "ran myservice" } From 29bacf53a19d6c8f8128c25d5add8018fc3fb8c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sun, 25 Sep 2022 21:00:06 -0300 Subject: [PATCH 076/253] It is now possible to run any combination of commands --- plugin.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/plugin.yml b/plugin.yml index 93ae0e4d..1086916a 100644 --- a/plugin.yml +++ b/plugin.yml @@ -78,12 +78,11 @@ configuration: type: boolean entrypoint: type: string - oneOf: - - anyOf: - - required: - - run - - required: - - build + anyOf: + - required: + - run + - required: + - build - required: - push additionalProperties: false From c62227ebcfd5fe4978dad4ba17b9637b274ee5ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sun, 25 Sep 2022 21:15:23 -0300 Subject: [PATCH 077/253] Did a read-through of the documentation --- README.md | 54 +++++++++++++++++++++--------------------------------- 1 file changed, 21 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 8765d1a9..15d914ea 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,18 @@ steps: run: app ``` +If you want to control how your command is passed to docker-compose, you can use the command parameter on the plugin directly: + +```yml +steps: + - plugins: + - docker-compose#v3.11.2: + run: app + command: ["custom", "command", "values"] +``` + +## Authenticated registries + You can leverage the [docker-login plugin](https://github.com/buildkite-plugins/docker-login-buildkite-plugin) in tandem for authenticating with a registry. For example, the following will build and push an image to a private repo, and pull from that private repo in subsequent run commands: ```yml @@ -79,15 +91,7 @@ steps: run: app ``` -If you want to control how your command is passed to docker-compose, you can use the command parameter on the plugin directly: - -```yml -steps: - - plugins: - - docker-compose#v3.11.1: - run: app - command: ["custom", "command", "values"] -``` +Note, you will need to add the configuration to all steps in which you use this plugin. ## Artifacts @@ -166,11 +170,11 @@ steps: - MY_CUSTOM_ARG=panda ``` -Note that the values in the list must be a KEY=VALUE pair. +Note that the values in the list must be a `KEY=VALUE` pair. ## Pre-building the image -To speed up run steps that use the same service/image (such as steps that run in parallel), you can add a pre-build step to your pipeline: +If you have multiple steps that use the same service/image (such as steps that run in parallel), you can use this plugin in a specific `build` step to your pipeline. That will set specific metadata in the pipeline for this plugin to use in `run` steps afterwards: ```yml steps: @@ -190,7 +194,7 @@ steps: run: app ``` -All `run` steps for the service `app` will automatically pull and use the pre-built image. +All `run` steps for the service `app` will automatically pull and use the pre-built image. Without this, each `Test %n` job would build its own instead. ## Building multiple images @@ -226,19 +230,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v3.11.1: - push: app -``` - -If you need to authenticate to the repository to push (e.g. when pushing to Docker Hub), use the Docker Login plugin: - -```yml -steps: - - label: ":docker: Push" - plugins: - - docker-login#v2.0.1: - username: xyz - - docker-compose#v3.11.1: + - docker-compose#v3.11.2: push: app ``` @@ -248,9 +240,7 @@ To push multiple images, you can use a list: steps: - label: ":docker: Push" plugins: - - docker-login#v2.0.1: - username: xyz - - docker-compose#v3.11.1: + - docker-compose#v3.11.2: push: - first-service - second-service @@ -262,9 +252,7 @@ If you want to push to a specific location (that's not defined as the `image` in steps: - label: ":docker: Push" plugins: - - docker-login#v2.0.1: - username: xyz - - docker-compose#v3.11.1: + - docker-compose#v3.11.2: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -350,7 +338,7 @@ are another (with a default name). The first successfully downloaded image in ea ### `build` -The name of a service to build and store, allowing following pipeline steps to run faster as they won't need to build the image. The step’s `command` will be ignored and does not need to be specified. +The name of a service to build and store, allowing following pipeline steps to run faster as they won't need to build the image. The step's `command` will be ignored and does not need to be specified. Either a single service or multiple services can be provided as an array. @@ -360,7 +348,7 @@ The name of the service the command should be run within. If the docker-compose ### `push` -A list of services to push in the format `service:image:tag`. If an image has been pre-built with the build step, that image will be re-tagged, otherwise docker-compose's built in push operation will be used. +A list of services to push in the format `service:image:tag`. If an image has been pre-built with the build step, that image will be re-tagged, otherwise docker-compose's built-in push operation will be used. ### `pull` (optional, run only) From f47d49e89ba3ad44d46dcdcf620c7b397ee64d10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sun, 25 Sep 2022 21:16:26 -0300 Subject: [PATCH 078/253] Updated version in preparation for next release --- README.md | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 15d914ea..88996047 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: run: app ``` @@ -28,7 +28,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: run: app config: docker-compose.tests.yml env: @@ -41,7 +41,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: run: app config: - docker-compose.yml @@ -56,7 +56,7 @@ env: steps: - command: test.sh plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: run: app ``` @@ -65,7 +65,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v3.11.2: + - docker-compose#v3.11.3: run: app command: ["custom", "command", "values"] ``` @@ -79,7 +79,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -87,7 +87,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: run: app ``` @@ -104,7 +104,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: run: app ``` @@ -122,7 +122,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: run: app volumes: - "./dist:/app/dist" @@ -143,7 +143,7 @@ this plugin offers a `environment` block of its own: steps: - command: generate-dist.sh plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: run: app env: - BUILDKITE_BUILD_NUMBER @@ -163,7 +163,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -180,7 +180,7 @@ If you have multiple steps that use the same service/image (such as steps that r steps: - label: ":docker: Build" plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: build: app image-repository: index.docker.io/myorg/myrepo @@ -190,7 +190,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: run: app ``` @@ -206,7 +206,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: build: - app - tests @@ -218,7 +218,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: run: tests ``` @@ -230,7 +230,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v3.11.2: + - docker-compose#v3.11.3: push: app ``` @@ -240,7 +240,7 @@ To push multiple images, you can use a list: steps: - label: ":docker: Push" plugins: - - docker-compose#v3.11.2: + - docker-compose#v3.11.3: push: - first-service - second-service @@ -252,7 +252,7 @@ If you want to push to a specific location (that's not defined as the `image` in steps: - label: ":docker: Push" plugins: - - docker-compose#v3.11.2: + - docker-compose#v3.11.3: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -266,14 +266,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker: Build an image" plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest From 9622b0f54adc37deab57d861a9ba5561730af35a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sun, 25 Sep 2022 23:32:23 -0300 Subject: [PATCH 079/253] Added tests for multiple-commands --- tests/multiple-commands.bats | 126 ++++++++++++++++++++++++++++++++--- 1 file changed, 116 insertions(+), 10 deletions(-) diff --git a/tests/multiple-commands.bats b/tests/multiple-commands.bats index 9224cd09..39aa0736 100644 --- a/tests/multiple-commands.bats +++ b/tests/multiple-commands.bats @@ -15,12 +15,12 @@ export BUILDKITE_JOB_ID=12 export BUILDKITE_PIPELINE_SLUG=test -@test "Build and run in a single step" { +@test "Build and run" { export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + + # necessary for build export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_REPOSITORY=my.repository/llamas - export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false - export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false stub docker-compose \ "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" \ @@ -28,24 +28,130 @@ export BUILDKITE_PIPELINE_SLUG=test "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo ran dependencies" \ "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml run --name buildkite12_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" + + # these commands simulate metadata for a specific value by using an intermediate-file + stub buildkite-agent \ + "meta-data set docker-compose-plugin-built-image-tag-myservice \* : echo \$4 > /tmp/build-run-metadata" \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : test -f /tmp/build-run-metadata" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : cat /tmp/build-run-metadata" + + run $PWD/hooks/command + + assert_success + assert_output --partial "Building services myservice" + assert_output --partial "Pushing built images to my.repository/llamas" + assert_output --partial "Found a pre-built image for myservice" + assert_output --partial "Starting dependencies" + assert_output --partial "ran myservice" + + unstub docker-compose + unstub buildkite-agent +} + +@test "Build and push" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PUSH=myservice + + # necessary for build + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_REPOSITORY=my.repository/llamas + + stub docker-compose \ + "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" \ + "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml push myservice : echo build-pushed myservice" \ + "-f docker-compose.yml -p buildkite12 config : echo ''" \ + "-f docker-compose.yml -p buildkite12 push myservice : echo push-pushed myservice" + # these commands simulate metadata for a specific value by using an intermediate-file stub buildkite-agent \ - "meta-data set docker-compose-plugin-built-image-tag-myservice my.repository/llamas:test-myservice-build-1 : echo set meta-data" \ - "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ - "meta-data get docker-compose-plugin-built-image-tag-myservice : echo got meta-data" + "meta-data set docker-compose-plugin-built-image-tag-myservice \* : echo \$4 > /tmp/build-push-metadata" \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : test -f /tmp/build-push-metadata" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : cat /tmp/build-push-metadata" + + stub docker \ + "pull my.repository/llamas:test-myservice-build-1 : echo pulled pre-built image" \ + "tag my.repository/llamas:test-myservice-build-1 buildkite12_myservice : echo re-tagged pre-built image" run $PWD/hooks/command + assert_success + + assert_output --partial "Building services myservice" + assert_output --partial "Pushing built images to my.repository/llamas" + assert_output --partial "Pulling pre-built service myservice" + assert_output --partial "Tagging pre-built service myservice" + assert_output --partial "Pushing images for myservice" + unstub docker-compose unstub buildkite-agent +} + +@test "Run and push without pre-built image" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PUSH=myservice + + stub docker-compose \ + "-f docker-compose.yml -p buildkite12 build --pull myservice : echo built myservice" \ + "-f docker-compose.yml -p buildkite12 up -d --scale myservice=0 myservice : echo ran dependencies" \ + "-f docker-compose.yml -p buildkite12 run --name buildkite12_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" \ + "-f docker-compose.yml -p buildkite12 config : echo ''" \ + "-f docker-compose.yml -p buildkite12 build myservice : echo built-2 myservice" \ + "-f docker-compose.yml -p buildkite12 push myservice : echo pushed myservice" + + # these make sure that the image is not pre-built + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run $PWD/hooks/command assert_success - assert_output --partial "built myservice" - assert_output --partial "pushed myservice" - assert_output --partial "pulled myservice" - assert_output --partial "ran dependencies" + + assert_output --partial "Building Docker Compose Service: myservice" + assert_output --partial "No pre-built image found from a previous " + assert_output --partial "Starting dependencies" assert_output --partial "ran myservice" + assert_output --partial "Building myservice" + assert_output --partial "Pushing images for myservice" + + unstub docker-compose + unstub buildkite-agent } +@test "Run and push with pre-built image" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PUSH=myservice + + stub docker-compose \ + "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo ran dependencies" \ + "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml run --name buildkite12_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" \ + "-f docker-compose.yml -p buildkite12 config : echo ''" \ + "-f docker-compose.yml -p buildkite12 push myservice : echo pushed myservice" + + # these make sure that the image is not pre-built + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myservice-tag" \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myservice-tag" + + stub docker \ + "pull myservice-tag : echo pulled pre-built image" \ + "tag myservice-tag buildkite12_myservice : echo re-tagged pre-built image" + + run $PWD/hooks/command + + assert_success + + refute_output --partial "Building services myservice" + assert_output --partial "Found a pre-built image for myservice" + assert_output --partial "Pulling services myservice" + assert_output --partial "Starting dependencies" + assert_output --partial "Pulling pre-built service myservice" + assert_output --partial "Pushing images for myservice" + + unstub docker-compose + unstub buildkite-agent +} From e0357ad0c3b7f7dc5a8664613374bf8157e6eba4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sun, 25 Sep 2022 23:32:38 -0300 Subject: [PATCH 080/253] Reworked command documentation a bit --- README.md | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 88996047..571356df 100644 --- a/README.md +++ b/README.md @@ -336,20 +336,56 @@ are another (with a default name). The first successfully downloaded image in ea ## Configuration -### `build` +### Main Commands + +You will need to specify at least one of the following to use this extension. + +#### `build` The name of a service to build and store, allowing following pipeline steps to run faster as they won't need to build the image. The step's `command` will be ignored and does not need to be specified. Either a single service or multiple services can be provided as an array. -### `run` +#### `run` The name of the service the command should be run within. If the docker-compose command would usually be `docker-compose run app test.sh` then the value would be `app`. -### `push` +#### `push` A list of services to push in the format `service:image:tag`. If an image has been pre-built with the build step, that image will be re-tagged, otherwise docker-compose's built-in push operation will be used. +#### Known issues + +##### Run & Push + +A basic pipeline similar to the following: + +```yaml +steps: + - label: ":docker: Run & Push" + plugins: + - docker-compose#v3.12.0: + run: myservice + push: myservice +``` + +Will cause the image to be built twice (once before running and once before pushing) unless there was a previous `build` step that set the appropriate metadata. + +##### Run & Push + +A basic pipeline similar to the following: + +```yaml +steps: + - label: ":docker: Build & Push" + plugins: + - docker-compose#v3.12.0: + build: myservice + push: myservice +``` + +Will cause the image to be pushed twice (once by the build step and another by the push step) + ### `pull` (optional, run only) Pull down multiple pre-built images. By default only the service that is being run will be pulled down, but this allows multiple images to be specified to handle prebuilt dependent images. From ae189c59f13002848dfeb94c75aca1c8a70a0fe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sun, 25 Sep 2022 23:36:36 -0300 Subject: [PATCH 081/253] Updated Readme version once more --- README.md | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 571356df..fb830350 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: run: app ``` @@ -28,7 +28,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: run: app config: docker-compose.tests.yml env: @@ -41,7 +41,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: run: app config: - docker-compose.yml @@ -56,7 +56,7 @@ env: steps: - command: test.sh plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: run: app ``` @@ -65,7 +65,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: run: app command: ["custom", "command", "values"] ``` @@ -79,7 +79,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -87,7 +87,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: run: app ``` @@ -104,7 +104,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: run: app ``` @@ -122,7 +122,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: run: app volumes: - "./dist:/app/dist" @@ -143,7 +143,7 @@ this plugin offers a `environment` block of its own: steps: - command: generate-dist.sh plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: run: app env: - BUILDKITE_BUILD_NUMBER @@ -163,7 +163,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -180,7 +180,7 @@ If you have multiple steps that use the same service/image (such as steps that r steps: - label: ":docker: Build" plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: build: app image-repository: index.docker.io/myorg/myrepo @@ -190,7 +190,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: run: app ``` @@ -206,7 +206,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: build: - app - tests @@ -218,7 +218,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: run: tests ``` @@ -230,7 +230,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: push: app ``` @@ -240,7 +240,7 @@ To push multiple images, you can use a list: steps: - label: ":docker: Push" plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: push: - first-service - second-service @@ -252,7 +252,7 @@ If you want to push to a specific location (that's not defined as the `image` in steps: - label: ":docker: Push" plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -266,14 +266,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker: Build an image" plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -287,7 +287,7 @@ This plugin allows for the value of `cache-from` to be a string or a list. If it steps: - label: ":docker Build an image" plugins: - - docker-compose#v3.9.0: + - docker-compose#v3.12.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: @@ -296,7 +296,7 @@ steps: - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v3.9.0: + - docker-compose#v3.12.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:my-branch @@ -310,7 +310,7 @@ Adding a grouping tag to the end of a cache-from list item allows this plugin to steps: - label: ":docker: Build Intermediate Image" plugins: - - docker-compose#v3.9.0: + - docker-compose#v3.12.0: build: myservice_intermediate # docker-compose.yml is the same as myservice but has `target: intermediate` image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo/myservice_intermediate @@ -320,7 +320,7 @@ steps: - wait - label: ":docker: Build Final Image" plugins: - - docker-compose#v3.9.0: + - docker-compose#v3.12.0: build: myservice image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo From e923c0281f4a0da1b7b83f2642d40ad1e372e9f8 Mon Sep 17 00:00:00 2001 From: Vladimir Lyzo Date: Mon, 26 Sep 2022 15:51:57 +1000 Subject: [PATCH 082/253] add build push run test for v2 --- tests/build.bats | 17 - tests/v2/build.bats | 487 ++++++++++++++++++++++ tests/v2/push.bats | 177 ++++++++ tests/v2/run.bats | 973 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 1637 insertions(+), 17 deletions(-) create mode 100644 tests/v2/build.bats create mode 100644 tests/v2/push.bats create mode 100644 tests/v2/run.bats diff --git a/tests/build.bats b/tests/build.bats index c1072e7e..51117d2b 100644 --- a/tests/build.bats +++ b/tests/build.bats @@ -92,23 +92,6 @@ load '../lib/shared' unstub docker-compose } -@test "Build with docker-compose v2" { - export BUILDKITE_JOB_ID=1111 - export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice - export BUILDKITE_PIPELINE_SLUG=test - export BUILDKITE_BUILD_NUMBER=1 - export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLI_VERSION=2 - - stub docker \ - "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" - - run $PWD/hooks/command - - assert_success - assert_output --partial "built myservice" - unstub docker -} - @test "Build with a repository" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice diff --git a/tests/v2/build.bats b/tests/v2/build.bats new file mode 100644 index 00000000..2322435d --- /dev/null +++ b/tests/v2/build.bats @@ -0,0 +1,487 @@ +#!/usr/bin/env bats + +load '/usr/local/lib/bats/load.bash' +load '../../lib/shared' + +# export DOCKER_COMPOSE_STUB_DEBUG=/dev/stdout +# export BUILDKITE_AGENT_STUB_DEBUG=/dev/stdout +# export BATS_MOCK_TMPDIR=$PWD + +setup_file() { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLI_VERSION=2 +} + +@test "Build without a repository" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" + + run $PWD/hooks/command + + unstub docker + assert_success + assert_output --partial "built myservice" +} + +@test "Build with no-cache" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_NO_CACHE=true + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --no-cache myservice : echo built myservice" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + unstub docker +} + +@test "Build with parallel" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_PARALLEL=true + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --parallel myservice : echo built myservice" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + unstub docker +} + +@test "Build with build args" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_ARGS_0=MYARG=0 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_ARGS_1=MYARG=1 + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --build-arg MYARG=0 --build-arg MYARG=1 myservice : echo built myservice" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + unstub docker +} + +@test "Build with a repository" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_REPOSITORY=my.repository/llamas + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml push myservice : echo pushed myservice" \ + + stub buildkite-agent \ + "meta-data set docker-compose-plugin-built-image-tag-myservice my.repository/llamas:test-myservice-build-1 : echo set image metadata for myservice" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "pushed myservice" + assert_output --partial "set image metadata for myservice" + unstub docker + unstub buildkite-agent +} + +@test "Build with a repository and multiple build aliases" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_REPOSITORY=my.repository/llamas + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_ALIAS_0=myservice-1 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_ALIAS_1=myservice-2 + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml push myservice : echo pushed myservice" \ + + stub buildkite-agent \ + "meta-data set docker-compose-plugin-built-image-tag-myservice my.repository/llamas:test-myservice-build-1 : echo set image metadata for myservice" \ + "meta-data set docker-compose-plugin-built-image-tag-myservice-1 my.repository/llamas:test-myservice-build-1 : echo set image metadata for myservice-1" \ + "meta-data set docker-compose-plugin-built-image-tag-myservice-2 my.repository/llamas:test-myservice-build-1 : echo set image metadata for myservice-2" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "pushed myservice" + assert_output --partial "set image metadata for myservice" + assert_output --partial "set image metadata for myservice-1" + assert_output --partial "set image metadata for myservice-2" + unstub docker + unstub buildkite-agent +} + +@test "Build with a repository and push retries" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_REPOSITORY=my.repository/llamas + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PUSH_RETRIES=3 + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml push myservice : exit 1" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml push myservice : exit 1" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml push myservice : echo pushed myservice" \ + + stub buildkite-agent \ + "meta-data set docker-compose-plugin-built-image-tag-myservice my.repository/llamas:test-myservice-build-1 : echo set image metadata for myservice" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "pushed myservice" + assert_output --partial "set image metadata for myservice" + unstub docker + unstub buildkite-agent +} + +@test "Build with a repository and custom config file" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG=tests/composefiles/docker-compose.v2.0.yml + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_REPOSITORY=my.repository/llamas + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "compose -f tests/composefiles/docker-compose.v2.0.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" \ + "compose -f tests/composefiles/docker-compose.v2.0.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml push myservice : echo pushed myservice" \ + + stub buildkite-agent \ + "meta-data set docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v2.0.yml my.repository/llamas:test-myservice-build-1 : echo set image metadata for myservice" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "pushed myservice" + assert_output --partial "set image metadata for myservice" + unstub docker + unstub buildkite-agent +} + +@test "Build with a repository and multiple custom config files" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG_0=tests/composefiles/docker-compose.v2.0.yml + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG_1=tests/composefiles/docker-compose.v2.1.yml + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_REPOSITORY=my.repository/llamas + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "compose -f tests/composefiles/docker-compose.v2.0.yml -f tests/composefiles/docker-compose.v2.1.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" \ + "compose -f tests/composefiles/docker-compose.v2.0.yml -f tests/composefiles/docker-compose.v2.1.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml push myservice : echo pushed myservice" \ + + stub buildkite-agent \ + "meta-data set docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v2.0.yml-tests/composefiles/docker-compose.v2.1.yml my.repository/llamas:test-myservice-build-1 : echo set image metadata for myservice" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "pushed myservice" + assert_output --partial "set image metadata for myservice" + unstub docker + unstub buildkite-agent +} + +@test "Build with a repository and multiple services" { + export BUILDKITE_JOB_ID=1112 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=myservice1 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_1=myservice2 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_REPOSITORY=my.repository/llamas + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "compose -f docker-compose.yml -p buildkite1112 -f docker-compose.buildkite-1-override.yml build --pull myservice1 myservice2 : echo built all services" \ + "compose -f docker-compose.yml -p buildkite1112 -f docker-compose.buildkite-1-override.yml push myservice1 myservice2 : echo pushed all services" \ + + stub buildkite-agent \ + "meta-data set docker-compose-plugin-built-image-tag-myservice1 my.repository/llamas:test-myservice1-build-1 : echo set image metadata for myservice1" \ + "meta-data set docker-compose-plugin-built-image-tag-myservice2 my.repository/llamas:test-myservice2-build-1 : echo set image metadata for myservice2" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built all services" + assert_output --partial "pushed all services" + assert_output --partial "set image metadata for myservice1" + assert_output --partial "set image metadata for myservice2" + unstub docker + unstub buildkite-agent +} + +@test "Build with a docker-compose v1.0 configuration file" { + export BUILDKITE_JOB_ID=1112 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v1.0.yml" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + run $PWD/hooks/command + + assert_failure + assert_output --partial "Compose file versions 2.0 and above" +} + +@test "Build with a cache-from image" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=helloworld:my.repository/myservice_cache:latest + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "pull my.repository/myservice_cache:latest : echo pulled cache image" \ + "compose -f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" + + run $PWD/hooks/command + + assert_success + assert_output --partial "pulled cache image" + assert_output --partial "- my.repository/myservice_cache:latest" + assert_output --partial "built helloworld" + unstub docker +} + +@test "Build with a cache-from image with no-cache also set" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=helloworld:my.repository/myservice_cache:latest + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_NO_CACHE=true + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "compose -f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --no-cache helloworld : echo built helloworld" + + run $PWD/hooks/command + + assert_success + refute_output --partial "pulled cache image" + refute_output --partial "- my.repository/myservice_cache:latest" + assert_output --partial "built helloworld" + unstub docker +} + +@test "Build with several cache-from images for one service" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=helloworld:my.repository/myservice_cache:branch-name + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_1=helloworld:my.repository/myservice_cache:latest + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "pull my.repository/myservice_cache:branch-name : echo pulled cache image" \ + "compose -f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" + + run $PWD/hooks/command + + assert_success + assert_output --partial "pulled cache image" + assert_output --partial "- my.repository/myservice_cache:branch-name" + refute_output --partial "- my.repository/myservice_cache:latest" + assert_output --partial "built helloworld" + unstub docker +} + +@test "Build with several cache-from images for one service with first image being not available" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=helloworld:my.repository/myservice_cache:branch-name + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_1=helloworld:my.repository/myservice_cache:latest + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "pull my.repository/myservice_cache:branch-name : exit 1" \ + "pull my.repository/myservice_cache:latest : echo pulled cache image" \ + "compose -f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" + + run $PWD/hooks/command + + assert_success + assert_output --partial "pulled cache image" + refute_output --partial "- my.repository/myservice_cache:branch-name" + assert_output --partial "- my.repository/myservice_cache:latest" + assert_output --partial "built helloworld" + unstub docker +} + +@test "Build with a cache-from image when pulling of the cache-from image failed" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=helloworld:my.repository/myservice_cache:latest + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "pull my.repository/myservice_cache:latest : exit 1" \ + "compose -f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" + + run $PWD/hooks/command + + assert_success + assert_output --partial "my.repository/myservice_cache:latest will not be used as a cache for helloworld" + refute_output --partial "- my.repository/myservice_cache:latest" + assert_output --partial "built helloworld" + unstub docker +} + +@test "Build with a cache-from image with hyphen" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=hello-world + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=hello-world:my.repository/my-service_cache:latest + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "pull my.repository/my-service_cache:latest : echo pulled cache image" \ + "compose -f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull hello-world : echo built hello-world" + + run $PWD/hooks/command + + assert_success + assert_output --partial "pulled cache image" + assert_output --partial "- my.repository/my-service_cache:latest" + assert_output --partial "built hello-world" + unstub docker +} + +@test "Build with a cache-from image retry on failing pull" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=helloworld:my.repository/myservice_cache:latest + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PULL_RETRIES=3 + + stub docker \ + "pull my.repository/myservice_cache:latest : exit 1" \ + "pull my.repository/myservice_cache:latest : exit 1" \ + "pull my.repository/myservice_cache:latest : echo pulled cache image" \ + "compose -f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" + + run $PWD/hooks/command + + assert_success + assert_output --partial "pulled cache image" + assert_output --partial "- my.repository/myservice_cache:latest" + assert_output --partial "built helloworld" + unstub docker +} + +@test "Build with a custom image-name" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_NAME=my-llamas-image + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_REPOSITORY=my.repository/llamas + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml push myservice : echo pushed myservice" \ + + stub buildkite-agent \ + "meta-data set docker-compose-plugin-built-image-tag-myservice my.repository/llamas:my-llamas-image : echo set image metadata for myservice" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "pushed myservice" + assert_output --partial "set image metadata for myservice" + unstub docker + unstub buildkite-agent +} + +@test "Build with a custom image-name and a config" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_NAME=my-llamas-image + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_REPOSITORY=my.repository/llamas + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "compose -f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" \ + "compose -f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml push myservice : echo pushed myservice" \ + + stub buildkite-agent \ + "meta-data set docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v3.2.yml my.repository/llamas:my-llamas-image : echo set image metadata for myservice" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "pushed myservice" + assert_output --partial "set image metadata for myservice" + unstub docker + unstub buildkite-agent +} + +@test "Build multiple images with custom image-names" { + export BUILDKITE_JOB_ID=1112 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=myservice1 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_1=myservice2 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_NAME_0=my-llamas-image-1 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_NAME_1=my-llamas-image-2 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_REPOSITORY=my.repository/llamas + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "compose -f docker-compose.yml -p buildkite1112 -f docker-compose.buildkite-1-override.yml build --pull myservice1 myservice2 : echo built all services" \ + "compose -f docker-compose.yml -p buildkite1112 -f docker-compose.buildkite-1-override.yml push myservice1 myservice2 : echo pushed all services" \ + + stub buildkite-agent \ + "meta-data set docker-compose-plugin-built-image-tag-myservice1 my.repository/llamas:my-llamas-image-1 : echo set image metadata for myservice1" \ + "meta-data set docker-compose-plugin-built-image-tag-myservice2 my.repository/llamas:my-llamas-image-2 : echo set image metadata for myservice2" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built all services" + assert_output --partial "pushed all services" + assert_output --partial "set image metadata for myservice1" + assert_output --partial "set image metadata for myservice2" + unstub docker + unstub buildkite-agent +} diff --git a/tests/v2/push.bats b/tests/v2/push.bats new file mode 100644 index 00000000..f1a185da --- /dev/null +++ b/tests/v2/push.bats @@ -0,0 +1,177 @@ +#!/usr/bin/env bats + +load '/usr/local/lib/bats/load.bash' +load '../../lib/shared' + +# export DOCKER_COMPOSE_STUB_DEBUG=/dev/tty +# export DOCKER_STUB_DEBUG=/dev/tty +# export BUILDKITE_AGENT_STUB_DEBUG=/dev/tty +# export BATS_MOCK_TMPDIR=$PWD + +setup_file() { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLI_VERSION=2 +} + +@test "Push a single service with an image in it's config" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PUSH=app + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-app : exit 1" + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 config : cat $PWD/tests/composefiles/docker-compose.config.v3.2.yml" \ + "compose -f docker-compose.yml -p buildkite1111 push app : echo pushed app" \ + "image inspect somewhere.dkr.ecr.some-region.amazonaws.com/blah : exit 0" + + run $PWD/hooks/command + + assert_success + assert_output --partial ":warning: Skipping build" + assert_output --partial "pushed app" + unstub docker + unstub buildkite-agent +} + +@test "Push two services with target repositories and tags" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PUSH_0=myservice1:my.repository/myservice1 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PUSH_1=myservice2:my.repository/myservice2:llamas + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 config : echo blah " \ + "compose -f docker-compose.yml -p buildkite1111 build myservice1 : echo blah " \ + "compose -f docker-compose.yml -p buildkite1111 config : echo blah " \ + "compose -f docker-compose.yml -p buildkite1111 build myservice2 : echo blah " \ + "image inspect buildkite1111_myservice1 : exit 1" \ + "tag buildkite1111_myservice1 my.repository/myservice1 : echo tagging image1" \ + "push my.repository/myservice1 : echo pushing myservice1 image" \ + "image inspect buildkite1111_myservice2 : exit 1" \ + "tag buildkite1111_myservice2 my.repository/myservice2:llamas : echo tagging image2" \ + "push my.repository/myservice2:llamas : echo pushing myservice2 image" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice1 : exit 1" \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice2 : exit 1" + + run $PWD/hooks/command + + assert_success + assert_output --partial "tagging image1" + assert_output --partial "pushing myservice1 image" + assert_output --partial "tagging image2" + assert_output --partial "pushing myservice2 image" + unstub docker + unstub buildkite-agent +} + +@test "Push a prebuilt image with a repository and a tag" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PUSH=myservice:my.repository/myservice:llamas + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "pull myimage : echo pulled prebuilt image" \ + "tag myimage buildkite1111_myservice : echo " \ + "tag buildkite1111_myservice my.repository/myservice:llamas : echo tagged image" \ + "push my.repository/myservice:llamas : echo pushed myservice" + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 config : echo blah" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run $PWD/hooks/command + + assert_success + assert_output --partial "pulled prebuilt image" + assert_output --partial "tagged image" + assert_output --partial "pushed myservice" + unstub docker + unstub docker + unstub buildkite-agent +} + +@test "Push a prebuilt image to multiple tags" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PUSH_0=myservice:my.repository/myservice:llamas + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PUSH_1=myservice:my.repository/myservice:latest + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PUSH_2=myservice:my.repository/myservice:alpacas + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "pull prebuilt : echo pulled prebuilt image" \ + "tag prebuilt buildkite1111_myservice : echo " \ + "tag buildkite1111_myservice my.repository/myservice:llamas : echo tagged image1" \ + "push my.repository/myservice:llamas : echo pushed myservice1" \ + "tag prebuilt buildkite1111_myservice : echo " \ + "tag buildkite1111_myservice my.repository/myservice:latest : echo tagged image2" \ + "push my.repository/myservice:latest : echo pushed myservice2" \ + "tag prebuilt buildkite1111_myservice : echo " \ + "tag buildkite1111_myservice my.repository/myservice:alpacas : echo tagged image3" \ + "push my.repository/myservice:alpacas : echo pushed myservice3" + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 config : echo blah" \ + "compose -f docker-compose.yml -p buildkite1111 config : echo blah" \ + "compose -f docker-compose.yml -p buildkite1111 config : echo blah" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo prebuilt" \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo prebuilt" \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo prebuilt" + + run $PWD/hooks/command + + assert_success + assert_output --partial "pulled prebuilt image" + assert_output --partial "tagged image1" + assert_output --partial "pushed myservice1" + assert_output --partial "tagged image2" + assert_output --partial "pushed myservice2" + assert_output --partial "tagged image3" + assert_output --partial "pushed myservice3" + unstub docker + unstub docker + unstub buildkite-agent +} + +@test "Push a single service that needs to be built" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PUSH=helper:my.repository/helper:llamas + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-helper : exit 1" + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 config : cat $PWD/tests/composefiles/docker-compose.config.v3.2.yml" \ + "compose -f docker-compose.yml -p buildkite1111 build helper : echo built helper" + + stub docker \ + "image inspect buildkite1111_helper : exit 1" \ + "tag buildkite1111_helper my.repository/helper:llamas : echo tagged helper" \ + "push my.repository/helper:llamas : echo pushed helper" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built helper" + assert_output --partial "tagged helper" + assert_output --partial "pushed helper" + unstub docker + unstub docker + unstub buildkite-agent +} diff --git a/tests/v2/run.bats b/tests/v2/run.bats new file mode 100644 index 00000000..5d28fae0 --- /dev/null +++ b/tests/v2/run.bats @@ -0,0 +1,973 @@ +#!/usr/bin/env bats + +load '/usr/local/lib/bats/load.bash' +load '../../lib/shared' +load '../../lib/run' + +# export DOCKER_COMPOSE_STUB_DEBUG=/dev/tty +# export BUILDKITE_AGENT_STUB_DEBUG=/dev/tty +# export BATS_MOCK_TMPDIR=$PWD + +setup_file() { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLI_VERSION=2 +} + +@test "Run without a prebuilt image" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="echo hello world" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "ran myservice" + unstub docker + unstub buildkite-agent +} + +@test "Run without a prebuilt image and an empty command" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "ran myservice" + unstub docker + unstub buildkite-agent +} + +@test "Run without a prebuilt image and a custom workdir" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_WORKDIR=/test_workdir + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --workdir=/test_workdir --rm myservice : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "ran myservice" + unstub docker + unstub buildkite-agent +} + +@test "Run without a prebuilt image with a quoted command" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="sh -c 'echo hello world'" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c $'sh -c \'echo hello world\'' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "ran myservice" + unstub docker + unstub buildkite-agent +} + +@test "Run without a prebuilt image with a multi-line command" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="cmd1 +cmd2 +cmd3" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c $'cmd1\ncmd2\ncmd3' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "ran myservice" + unstub docker + unstub buildkite-agent +} + +@test "Run without a prebuilt image with a command config" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_COMMAND_0=echo + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_COMMAND_1="hello world" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice echo 'hello world' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "ran myservice" + unstub docker + unstub buildkite-agent +} + +@test "Run without a prebuilt image with custom env" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_ENV_0=MYENV=0 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_ENV_1=MYENV + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_ENVIRONMENT_0=MYENV=2 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_ENVIRONMENT_1=MYENV + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_ENVIRONMENT_2=ANOTHER="this is a long string with spaces; and semi-colons" + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 -e MYENV=0 -e MYENV -e MYENV=2 -e MYENV -e ANOTHER=this\ is\ a\ long\ string\ with\ spaces\;\ and\ semi-colons --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run $PWD/hooks/command + + assert_success + assert_output --partial "ran myservice" + unstub docker + unstub buildkite-agent +} + +@test "Run without a prebuilt image with no-cache" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_NO_CACHE=true + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="echo hello world" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 build --pull --no-cache myservice : echo built myservice" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "ran myservice" + unstub docker + unstub buildkite-agent +} + +@test "Run without a prebuilt image with build args" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_ARGS_0=MYARG=0 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_ARGS_1=MYARG=1 + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="echo hello world" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 build --pull --build-arg MYARG=0 --build-arg MYARG=1 myservice : echo built myservice" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "ran myservice" + unstub docker + unstub buildkite-agent +} + +@test "Run with a prebuilt image" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run $PWD/hooks/command + + assert_success + assert_output --partial "ran myservice" + unstub docker + unstub buildkite-agent +} + +@test "Run with a prebuilt image and custom config file" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG=tests/composefiles/docker-compose.v2.0.yml + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + + stub docker \ + "compose -f tests/composefiles/docker-compose.v2.0.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "compose -f tests/composefiles/docker-compose.v2.0.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f tests/composefiles/docker-compose.v2.0.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v2.0.yml : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v2.0.yml : echo myimage" + + run $PWD/hooks/command + + assert_success + assert_output --partial "ran myservice" + unstub docker + unstub buildkite-agent +} + +@test "Run with a prebuilt image and multiple custom config files" { +export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG_0=tests/composefiles/docker-compose.v2.0.yml + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG_1=tests/composefiles/docker-compose.v2.1.yml + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + + stub docker \ + "compose -f tests/composefiles/docker-compose.v2.0.yml -f tests/composefiles/docker-compose.v2.1.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "compose -f tests/composefiles/docker-compose.v2.0.yml -f tests/composefiles/docker-compose.v2.1.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 : echo pulled myservice" \ + "compose -f tests/composefiles/docker-compose.v2.0.yml -f tests/composefiles/docker-compose.v2.1.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v2.0.yml-tests/composefiles/docker-compose.v2.1.yml : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v2.0.yml-tests/composefiles/docker-compose.v2.1.yml : echo myimage" + + run $PWD/hooks/command + + assert_success + assert_output --partial "ran myservice" + unstub docker + unstub buildkite-agent +} + +@test "Run with a prebuilt image and custom config file set from COMPOSE_FILE" { + export COMPOSE_FILE=tests/composefiles/docker-compose.v2.0.yml + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + + stub docker \ + "compose -f tests/composefiles/docker-compose.v2.0.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "compose -f tests/composefiles/docker-compose.v2.0.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f tests/composefiles/docker-compose.v2.0.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v2.0.yml : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v2.0.yml : echo myimage" + + run $PWD/hooks/command + + assert_success + assert_output --partial "ran myservice" + unstub docker + unstub buildkite-agent +} + +@test "Run with a single prebuilt image, no retry on failed pull" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : exit 2" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run $PWD/hooks/command + + assert_failure + assert_output --partial "Exited with 2" + unstub docker + unstub buildkite-agent +} + +@test "Run with a single prebuilt image, retry on failed pull" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PULL_RETRIES=3 + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : exit 2" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run $PWD/hooks/command + + assert_success + assert_output --partial "pulled myservice" + assert_output --partial "ran myservice" + unstub docker + unstub buildkite-agent +} + +@test "Run without a TTY" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_TTY=false + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 -T --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice without tty" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run $PWD/hooks/command + + assert_success + assert_output --partial "ran myservice without tty" + unstub docker + unstub buildkite-agent +} + +@test "Run without dependencies" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_DEPENDENCIES=false + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --no-deps --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice without dependencies" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run $PWD/hooks/command + + assert_success + assert_output --partial "ran myservice without dependencies" + unstub docker + unstub buildkite-agent +} + +@test "Run without ansi output" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_ANSI=false + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml --no-ansi run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice without ansi output" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run $PWD/hooks/command + + assert_success + assert_output --partial "ran myservice without ansi output" + unstub docker + unstub buildkite-agent +} + +@test "Run with use aliases" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_USE_ALIASES=true + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --use-aliases --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with use aliases output" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run $PWD/hooks/command + + assert_success + assert_output --partial "ran myservice with use aliases output" + unstub docker + unstub buildkite-agent +} + +@test "Run with a volumes option" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_0="./dist:/app/dist" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_1="./pkg:/app/pkg" + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 -v $PWD/dist:/app/dist -v $PWD/pkg:/app/pkg --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with volumes" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run $PWD/hooks/command + + assert_success + assert_output --partial "ran myservice with volumes" + unstub docker + unstub buildkite-agent +} + +@test "Run with an external volume" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES="buildkite:/buildkite" + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 -v buildkite:/buildkite --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with volumes" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run $PWD/hooks/command + + assert_success + assert_output --partial "ran myservice with volumes" + unstub docker + unstub buildkite-agent +} + +@test "Run with default volumes, extra delimiters" { + # Tests introduction of extra delimiters, as would occur if + # EXPORT BUILDKITE_DOCKER_DEFAULT_VOLUMES="new:mount; ${BUILDKITE_DOCKER_DEFAULT_VOLUMES:-}" + # was used with no existing value + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_DOCKER_DEFAULT_VOLUMES="buildkite:/buildkite; ./dist:/app/dist;; ; ;" + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 -v buildkite:/buildkite -v $PWD/dist:/app/dist --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with volumes" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run $PWD/hooks/command + + assert_success + assert_output --partial "ran myservice with volumes" + unstub docker + unstub buildkite-agent +} + +@test "Run with default volumes" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_DOCKER_DEFAULT_VOLUMES="buildkite:/buildkite;./dist:/app/dist" + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 -v buildkite:/buildkite -v $PWD/dist:/app/dist --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with volumes" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run $PWD/hooks/command + + assert_success + assert_output --partial "ran myservice with volumes" + unstub docker + unstub buildkite-agent +} + +@test "Run with multiple config files" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="echo hello world" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG_0="llamas1.yml" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG_1="llamas2.yml" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG_2="llamas3.yml" + + stub docker \ + "compose -f llamas1.yml -f llamas2.yml -f llamas3.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "compose -f llamas1.yml -f llamas2.yml -f llamas3.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "compose -f llamas1.yml -f llamas2.yml -f llamas3.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice-llamas1.yml-llamas2.yml-llamas3.yml : exit 1" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "ran myservice" + unstub docker + unstub buildkite-agent +} + +@test "Run with a failure should expand previous group" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : exit 2" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run $PWD/hooks/command + + assert_failure + assert_output --partial "^^^ +++" + assert_output --partial "Failed to run command, exited with 2" + unstub docker + unstub buildkite-agent +} + +@test "Run with multiple prebuilt images and multiple pulls" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice1 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PULL_0=myservice1 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PULL_1=myservice2 + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull --parallel myservice1 myservice2 : echo pulled myservice1 and myservice2" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice1=0 myservice1 : echo started dependencies for myservice1" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice1_build_1 --rm myservice1 /bin/sh -e -c 'pwd' : echo ran myservice1" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice1 : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice1 : echo myimage1" \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice2 : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice2 : echo myimage2" + + run $PWD/hooks/command + + assert_success + assert_output --partial "pulled myservice1 and myservice2" + assert_output --partial "ran myservice1" + unstub docker + unstub buildkite-agent +} + +@test "Run without a prebuilt image and a custom user" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="sh -c 'whoami'" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_USER="1000" + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --user=1000 --rm myservice /bin/sh -e -c $'sh -c \'whoami\'' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "ran myservice" + unstub docker + unstub buildkite-agent +} + +@test "Run without a prebuilt image and a custom user and group" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="sh -c 'whoami'" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_USER="1000:1001" + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --user=1000:1001 --rm myservice /bin/sh -e -c $'sh -c \'whoami\'' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "ran myservice" + unstub docker + unstub buildkite-agent +} + +@test "Fail with custom user and propagate UIDs" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="sh -c 'whoami'" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_USER="1000:1001" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PROPAGATE_UID_GID="true" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run $PWD/hooks/command + + assert_failure + assert_output --partial "Error" + assert_output --partial "Can't set both user and propagate-uid-gid" + unstub buildkite-agent +} + + +@test "Run without --rm" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RM=false + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 myservice /bin/sh -e -c $'pwd' : echo ran myservice without tty" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : echo myimage" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run $PWD/hooks/command + + assert_success + assert_output --partial "ran myservice without tty" + unstub docker + unstub buildkite-agent +} + +@test "Run with custom entrypoint" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_ENTRYPOINT="my custom entrypoint" + + ENTRYPOINT='--entrypoint\ \"my\ custom\ entrypoint\"' + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm --entrypoint 'my custom entrypoint' myservice : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "ran myservice" + unstub docker + unstub buildkite-agent +} + +@test "Run with mount-buildkite-agent enabled" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_MOUNT_BUILDKITE_AGENT=true + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm -e BUILDKITE_JOB_ID -e BUILDKITE_BUILD_ID -e BUILDKITE_AGENT_ACCESS_TOKEN -v $BATS_MOCK_TMPDIR/bin/buildkite-agent:/usr/bin/buildkite-agent myservice : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "ran myservice" + unstub docker + unstub buildkite-agent +} + +@test "Run with various build arguments" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="echo hello world" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_NO_CACHE=true + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_PARALLEL=true + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 build --pull --no-cache --parallel myservice : echo built myservice" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "ran myservice" + unstub docker + unstub buildkite-agent +} + +@test "Run with git-mirrors" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="echo hello world" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_REPO_MIRROR=/tmp/sample-mirror + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 -v /tmp/sample-mirror:/tmp/sample-mirror:ro --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "ran myservice" + unstub docker + unstub buildkite-agent +} + +@test "Run with mount-ssh-agent" { + export SSH_AUTH_SOCK=/tmp/ssh_auth_sock + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="echo hello world" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_MOUNT_SSH_AGENT=true + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm -e SSH_AUTH_SOCK=/ssh-agent -v /tmp/ssh_auth_sock:/ssh-agent -v /root/.ssh/known_hosts:/root/.ssh/known_hosts myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + apk add netcat-openbsd + nc -lkvU $SSH_AUTH_SOCK & + + run $PWD/hooks/command + + kill %1 + + assert_success + assert_output --partial "built myservice" + assert_output --partial "ran myservice" + unstub docker + unstub buildkite-agent +} From 311d9bcb6173e6da4515f3113c04b827099bdff1 Mon Sep 17 00:00:00 2001 From: Vladimir Lyzo Date: Mon, 26 Sep 2022 15:53:09 +1000 Subject: [PATCH 083/253] Enable recursive tests --- .buildkite/pipeline.yml | 1 + README.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 7528c334..323d867f 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -22,6 +22,7 @@ steps: plugins: ${BUILDKITE_PULL_REQUEST_REPO:-$BUILDKITE_REPO}#${BUILDKITE_COMMIT}: run: tests + commmand: bats -r tests # The rest of the steps are integration tests diff --git a/README.md b/README.md index 67700742..ccb66b26 100644 --- a/README.md +++ b/README.md @@ -488,7 +488,7 @@ If set to `2`, plugin will use `docker compose` to execute commands; otherwise i To run the tests: ```bash -docker-compose run --rm tests +docker-compose run --rm tests bats -r tests ``` ## License From fc6aae3786d565e1ed013bbcc51b5364d699dc4c Mon Sep 17 00:00:00 2001 From: Vladimir Lyzo Date: Mon, 26 Sep 2022 16:08:10 +1000 Subject: [PATCH 084/253] Fix readme lint --- README.md | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 9f91d401..293dc2d1 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.2: run: app ``` @@ -28,7 +28,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.2: run: app config: docker-compose.tests.yml env: @@ -41,7 +41,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.2: run: app config: - docker-compose.yml @@ -56,7 +56,7 @@ env: steps: - command: test.sh plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.2: run: app ``` @@ -67,7 +67,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.11.1: + - docker-compose#v3.11.2: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -75,7 +75,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.11.1: + - docker-compose#v3.11.2: run: app ``` @@ -84,7 +84,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.2: run: app command: ["custom", "command", "values"] ``` @@ -100,7 +100,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.2: run: app ``` @@ -118,7 +118,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.2: run: app volumes: - "./dist:/app/dist" @@ -139,7 +139,7 @@ this plugin offers a `environment` block of its own: steps: - command: generate-dist.sh plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.2: run: app env: - BUILDKITE_BUILD_NUMBER @@ -159,7 +159,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.2: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -176,7 +176,7 @@ To speed up run steps that use the same service/image (such as steps that run in steps: - label: ":docker: Build" plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.2: build: app image-repository: index.docker.io/myorg/myrepo @@ -186,7 +186,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.2: run: app ``` @@ -202,7 +202,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.2: build: - app - tests @@ -214,7 +214,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.2: run: tests ``` @@ -226,7 +226,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.2: push: app ``` @@ -238,7 +238,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.11.1: + - docker-compose#v3.11.2: push: app ``` @@ -250,7 +250,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.11.1: + - docker-compose#v3.11.2: push: - first-service - second-service @@ -264,7 +264,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.11.1: + - docker-compose#v3.11.2: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -278,14 +278,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker: Build an image" plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.2: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.2: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest From 51120c66d95fb21bd37273bfc734c641cf53fcd3 Mon Sep 17 00:00:00 2001 From: Vladimir Lyzo Date: Mon, 26 Sep 2022 16:12:43 +1000 Subject: [PATCH 085/253] Fix recursive bats --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 323d867f..475be7fb 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -22,7 +22,7 @@ steps: plugins: ${BUILDKITE_PULL_REQUEST_REPO:-$BUILDKITE_REPO}#${BUILDKITE_COMMIT}: run: tests - commmand: bats -r tests + command: ["bats", "-r", "tests"] # The rest of the steps are integration tests From 7a70f3a498ba7fc491316220149eec4e5c205e29 Mon Sep 17 00:00:00 2001 From: Vladimir Lyzo Date: Mon, 26 Sep 2022 16:43:06 +1000 Subject: [PATCH 086/253] Fix tests --- tests/v2/push.bats | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/tests/v2/push.bats b/tests/v2/push.bats index f1a185da..6aaf4099 100644 --- a/tests/v2/push.bats +++ b/tests/v2/push.bats @@ -23,8 +23,8 @@ setup_file() { stub docker \ "compose -f docker-compose.yml -p buildkite1111 config : cat $PWD/tests/composefiles/docker-compose.config.v3.2.yml" \ - "compose -f docker-compose.yml -p buildkite1111 push app : echo pushed app" \ - "image inspect somewhere.dkr.ecr.some-region.amazonaws.com/blah : exit 0" + "image inspect somewhere.dkr.ecr.some-region.amazonaws.com/blah : exit 0" \ + "compose -f docker-compose.yml -p buildkite1111 push app : echo pushed app" run $PWD/hooks/command @@ -44,13 +44,13 @@ setup_file() { stub docker \ "compose -f docker-compose.yml -p buildkite1111 config : echo blah " \ - "compose -f docker-compose.yml -p buildkite1111 build myservice1 : echo blah " \ - "compose -f docker-compose.yml -p buildkite1111 config : echo blah " \ - "compose -f docker-compose.yml -p buildkite1111 build myservice2 : echo blah " \ "image inspect buildkite1111_myservice1 : exit 1" \ + "compose -f docker-compose.yml -p buildkite1111 build myservice1 : echo blah " \ "tag buildkite1111_myservice1 my.repository/myservice1 : echo tagging image1" \ "push my.repository/myservice1 : echo pushing myservice1 image" \ + "compose -f docker-compose.yml -p buildkite1111 config : echo blah " \ "image inspect buildkite1111_myservice2 : exit 1" \ + "compose -f docker-compose.yml -p buildkite1111 build myservice2 : echo blah " \ "tag buildkite1111_myservice2 my.repository/myservice2:llamas : echo tagging image2" \ "push my.repository/myservice2:llamas : echo pushing myservice2 image" @@ -76,14 +76,12 @@ setup_file() { export BUILDKITE_BUILD_NUMBER=1 stub docker \ + "compose -f docker-compose.yml -p buildkite1111 config : echo blah" \ "pull myimage : echo pulled prebuilt image" \ "tag myimage buildkite1111_myservice : echo " \ "tag buildkite1111_myservice my.repository/myservice:llamas : echo tagged image" \ "push my.repository/myservice:llamas : echo pushed myservice" - stub docker \ - "compose -f docker-compose.yml -p buildkite1111 config : echo blah" - stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" @@ -95,7 +93,6 @@ setup_file() { assert_output --partial "tagged image" assert_output --partial "pushed myservice" unstub docker - unstub docker unstub buildkite-agent } @@ -108,22 +105,20 @@ setup_file() { export BUILDKITE_BUILD_NUMBER=1 stub docker \ + "compose -f docker-compose.yml -p buildkite1111 config : echo blah" \ "pull prebuilt : echo pulled prebuilt image" \ "tag prebuilt buildkite1111_myservice : echo " \ "tag buildkite1111_myservice my.repository/myservice:llamas : echo tagged image1" \ "push my.repository/myservice:llamas : echo pushed myservice1" \ + "compose -f docker-compose.yml -p buildkite1111 config : echo blah" \ "tag prebuilt buildkite1111_myservice : echo " \ "tag buildkite1111_myservice my.repository/myservice:latest : echo tagged image2" \ "push my.repository/myservice:latest : echo pushed myservice2" \ + "compose -f docker-compose.yml -p buildkite1111 config : echo blah" \ "tag prebuilt buildkite1111_myservice : echo " \ "tag buildkite1111_myservice my.repository/myservice:alpacas : echo tagged image3" \ "push my.repository/myservice:alpacas : echo pushed myservice3" - stub docker \ - "compose -f docker-compose.yml -p buildkite1111 config : echo blah" \ - "compose -f docker-compose.yml -p buildkite1111 config : echo blah" \ - "compose -f docker-compose.yml -p buildkite1111 config : echo blah" - stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo prebuilt" \ @@ -143,7 +138,6 @@ setup_file() { assert_output --partial "tagged image3" assert_output --partial "pushed myservice3" unstub docker - unstub docker unstub buildkite-agent } @@ -158,10 +152,8 @@ setup_file() { stub docker \ "compose -f docker-compose.yml -p buildkite1111 config : cat $PWD/tests/composefiles/docker-compose.config.v3.2.yml" \ - "compose -f docker-compose.yml -p buildkite1111 build helper : echo built helper" - - stub docker \ "image inspect buildkite1111_helper : exit 1" \ + "compose -f docker-compose.yml -p buildkite1111 build helper : echo built helper" \ "tag buildkite1111_helper my.repository/helper:llamas : echo tagged helper" \ "push my.repository/helper:llamas : echo pushed helper" @@ -172,6 +164,5 @@ setup_file() { assert_output --partial "tagged helper" assert_output --partial "pushed helper" unstub docker - unstub docker unstub buildkite-agent } From d20c0682c34681d08eb3efde6d67973dca59bcf1 Mon Sep 17 00:00:00 2001 From: Vladimir Lyzo Date: Tue, 27 Sep 2022 09:02:24 +1000 Subject: [PATCH 087/253] Explicitly specify tests/v2 --- .buildkite/pipeline.yml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 475be7fb..a61a76e5 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -22,7 +22,7 @@ steps: plugins: ${BUILDKITE_PULL_REQUEST_REPO:-$BUILDKITE_REPO}#${BUILDKITE_COMMIT}: run: tests - command: ["bats", "-r", "tests"] + command: ["bats", "tests", "tests/v2"] # The rest of the steps are integration tests diff --git a/README.md b/README.md index 293dc2d1..a1dd0563 100644 --- a/README.md +++ b/README.md @@ -545,7 +545,7 @@ If set to `2`, plugin will use `docker compose` to execute commands; otherwise i To run the tests: ```bash -docker-compose run --rm tests bats -r tests +docker-compose run --rm tests bats tests tests/v2 ``` ## License From 8a96bc73fb04d6236200247a52fda036ac6234d5 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 28 Sep 2022 22:18:59 +0000 Subject: [PATCH 088/253] Update buildkite plugin shellcheck to v1.3.0 --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index a61a76e5..71546977 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -7,7 +7,7 @@ env: steps: - label: ":shell: Shellcheck" plugins: - shellcheck#v1.2.0: + shellcheck#v1.3.0: files: - hooks/** - lib/** From bf4931419bff574ee510bebe0afa1059b7c386a3 Mon Sep 17 00:00:00 2001 From: Pol Date: Fri, 30 Sep 2022 00:05:49 -0300 Subject: [PATCH 089/253] Update version on README --- README.md | 52 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index b7331ffd..83b3d5a1 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v3.12.0: + - docker-compose#v3.13.0: run: app ``` @@ -28,7 +28,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v3.12.0: + - docker-compose#v3.13.0: run: app config: docker-compose.tests.yml env: @@ -41,7 +41,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v3.12.0: + - docker-compose#v3.13.0: run: app config: - docker-compose.yml @@ -56,7 +56,7 @@ env: steps: - command: test.sh plugins: - - docker-compose#v3.12.0: + - docker-compose#v3.13.0: run: app ``` @@ -65,7 +65,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v3.12.0: + - docker-compose#v3.13.0: run: app command: ["custom", "command", "values"] ``` @@ -79,7 +79,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.12.0: + - docker-compose#v3.13.0: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -87,7 +87,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.12.0: + - docker-compose#v3.13.0: run: app ``` @@ -104,7 +104,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.12.0: + - docker-compose#v3.13.0: run: app ``` @@ -122,7 +122,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.12.0: + - docker-compose#v3.13.0: run: app volumes: - "./dist:/app/dist" @@ -145,7 +145,7 @@ this plugin offers a `environment` block of its own: steps: - command: generate-dist.sh plugins: - - docker-compose#v3.12.0: + - docker-compose#v3.13.0: run: app env: - BUILDKITE_BUILD_NUMBER @@ -165,7 +165,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v3.12.0: + - docker-compose#v3.13.0: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -182,7 +182,7 @@ If you have multiple steps that use the same service/image (such as steps that r steps: - label: ":docker: Build" plugins: - - docker-compose#v3.12.0: + - docker-compose#v3.13.0: build: app image-repository: index.docker.io/myorg/myrepo @@ -192,7 +192,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.12.0: + - docker-compose#v3.13.0: run: app ``` @@ -208,7 +208,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v3.12.0: + - docker-compose#v3.13.0: build: - app - tests @@ -220,7 +220,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.12.0: + - docker-compose#v3.13.0: run: tests ``` @@ -232,7 +232,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v3.12.0: + - docker-compose#v3.13.0: push: app ``` @@ -242,7 +242,7 @@ To push multiple images, you can use a list: steps: - label: ":docker: Push" plugins: - - docker-compose#v3.12.0: + - docker-compose#v3.13.0: push: - first-service - second-service @@ -254,7 +254,7 @@ If you want to push to a specific location (that's not defined as the `image` in steps: - label: ":docker: Push" plugins: - - docker-compose#v3.12.0: + - docker-compose#v3.13.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -268,14 +268,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker: Build an image" plugins: - - docker-compose#v3.12.0: + - docker-compose#v3.13.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v3.12.0: + - docker-compose#v3.13.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -289,7 +289,7 @@ This plugin allows for the value of `cache-from` to be a string or a list. If it steps: - label: ":docker Build an image" plugins: - - docker-compose#v3.12.0: + - docker-compose#v3.13.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: @@ -298,7 +298,7 @@ steps: - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v3.12.0: + - docker-compose#v3.13.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:my-branch @@ -312,7 +312,7 @@ Adding a grouping tag to the end of a cache-from list item allows this plugin to steps: - label: ":docker: Build Intermediate Image" plugins: - - docker-compose#v3.12.0: + - docker-compose#v3.13.0: build: myservice_intermediate # docker-compose.yml is the same as myservice but has `target: intermediate` image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo/myservice_intermediate @@ -322,7 +322,7 @@ steps: - wait - label: ":docker: Build Final Image" plugins: - - docker-compose#v3.12.0: + - docker-compose#v3.13.0: build: myservice image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo @@ -366,7 +366,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Run & Push" plugins: - - docker-compose#v3.12.0: + - docker-compose#v3.13.0: run: myservice push: myservice ``` @@ -381,7 +381,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Build & Push" plugins: - - docker-compose#v3.12.0: + - docker-compose#v3.13.0: build: myservice push: myservice ``` From d460a79e493f0fc1286dbc7105bc022f7c5338ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 3 Oct 2022 18:16:18 -0300 Subject: [PATCH 090/253] Updated version in readme in preparation for next release --- README.md | 52 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 83b3d5a1..c11ca3c2 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v3.13.0: + - docker-compose#v4.0.0: run: app ``` @@ -28,7 +28,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v3.13.0: + - docker-compose#v4.0.0: run: app config: docker-compose.tests.yml env: @@ -41,7 +41,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v3.13.0: + - docker-compose#v4.0.0: run: app config: - docker-compose.yml @@ -56,7 +56,7 @@ env: steps: - command: test.sh plugins: - - docker-compose#v3.13.0: + - docker-compose#v4.0.0: run: app ``` @@ -65,7 +65,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v3.13.0: + - docker-compose#v4.0.0: run: app command: ["custom", "command", "values"] ``` @@ -79,7 +79,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.13.0: + - docker-compose#v4.0.0: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -87,7 +87,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.13.0: + - docker-compose#v4.0.0: run: app ``` @@ -104,7 +104,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.13.0: + - docker-compose#v4.0.0: run: app ``` @@ -122,7 +122,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.13.0: + - docker-compose#v4.0.0: run: app volumes: - "./dist:/app/dist" @@ -145,7 +145,7 @@ this plugin offers a `environment` block of its own: steps: - command: generate-dist.sh plugins: - - docker-compose#v3.13.0: + - docker-compose#v4.0.0: run: app env: - BUILDKITE_BUILD_NUMBER @@ -165,7 +165,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v3.13.0: + - docker-compose#v4.0.0: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -182,7 +182,7 @@ If you have multiple steps that use the same service/image (such as steps that r steps: - label: ":docker: Build" plugins: - - docker-compose#v3.13.0: + - docker-compose#v4.0.0: build: app image-repository: index.docker.io/myorg/myrepo @@ -192,7 +192,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.13.0: + - docker-compose#v4.0.0: run: app ``` @@ -208,7 +208,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v3.13.0: + - docker-compose#v4.0.0: build: - app - tests @@ -220,7 +220,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.13.0: + - docker-compose#v4.0.0: run: tests ``` @@ -232,7 +232,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v3.13.0: + - docker-compose#v4.0.0: push: app ``` @@ -242,7 +242,7 @@ To push multiple images, you can use a list: steps: - label: ":docker: Push" plugins: - - docker-compose#v3.13.0: + - docker-compose#v4.0.0: push: - first-service - second-service @@ -254,7 +254,7 @@ If you want to push to a specific location (that's not defined as the `image` in steps: - label: ":docker: Push" plugins: - - docker-compose#v3.13.0: + - docker-compose#v4.0.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -268,14 +268,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker: Build an image" plugins: - - docker-compose#v3.13.0: + - docker-compose#v4.0.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v3.13.0: + - docker-compose#v4.0.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -289,7 +289,7 @@ This plugin allows for the value of `cache-from` to be a string or a list. If it steps: - label: ":docker Build an image" plugins: - - docker-compose#v3.13.0: + - docker-compose#v4.0.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: @@ -298,7 +298,7 @@ steps: - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v3.13.0: + - docker-compose#v4.0.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:my-branch @@ -312,7 +312,7 @@ Adding a grouping tag to the end of a cache-from list item allows this plugin to steps: - label: ":docker: Build Intermediate Image" plugins: - - docker-compose#v3.13.0: + - docker-compose#v4.0.0: build: myservice_intermediate # docker-compose.yml is the same as myservice but has `target: intermediate` image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo/myservice_intermediate @@ -322,7 +322,7 @@ steps: - wait - label: ":docker: Build Final Image" plugins: - - docker-compose#v3.13.0: + - docker-compose#v4.0.0: build: myservice image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo @@ -366,7 +366,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Run & Push" plugins: - - docker-compose#v3.13.0: + - docker-compose#v4.0.0: run: myservice push: myservice ``` @@ -381,7 +381,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Build & Push" plugins: - - docker-compose#v3.13.0: + - docker-compose#v4.0.0: build: myservice push: myservice ``` From 82f3617578582a920a9900cdec04ca05a4c73d2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Thu, 6 Oct 2022 00:28:00 -0300 Subject: [PATCH 091/253] Sorted dependencies in plugin.yml --- plugin.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/plugin.yml b/plugin.yml index 4fa99451..c385d1fb 100644 --- a/plugin.yml +++ b/plugin.yml @@ -94,21 +94,21 @@ configuration: - push additionalProperties: false dependencies: - pull: [ run ] - image-repository: [ build ] - image-name: [ build ] + ansi: [ run ] + cache-from: [ build ] + dependencies: [ run ] env: [ run ] environment: [ run ] - push-retries: [ push ] - cache-from: [ build ] - volumes: [ run ] expand-volume-vars: [ volumes ] + image-repository: [ build ] + image-name: [ build ] leave-volumes: [ run ] - use-aliases: [ run ] - dependencies: [ run ] - ansi: [ run ] + mount-buildkite-agent: [ run ] + propagate-uid-gid: [ run ] + pull: [ run ] + push-retries: [ push ] tty: [ run ] - workdir: [ run ] + use-aliases: [ run ] user: [ run ] - propagate-uid-gid: [ run ] - mount-buildkite-agent: [ run ] + volumes: [ run ] + workdir: [ run ] From 6ec2c32b4ebf29497667a3d565eabdf9ecf9180a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Thu, 6 Oct 2022 00:37:33 -0300 Subject: [PATCH 092/253] Corrected minor shellcheck warning --- tests/build.bats | 50 ++++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/tests/build.bats b/tests/build.bats index 15628d21..1c29181c 100644 --- a/tests/build.bats +++ b/tests/build.bats @@ -16,7 +16,7 @@ load '../lib/shared' stub docker-compose \ "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" - run $PWD/hooks/command + run "$PWD"/hooks/command unstub docker-compose assert_success @@ -33,7 +33,7 @@ load '../lib/shared' stub docker-compose \ "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --no-cache myservice : echo built myservice" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -50,7 +50,7 @@ load '../lib/shared' stub docker-compose \ "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --parallel myservice : echo built myservice" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -68,7 +68,7 @@ load '../lib/shared' stub docker-compose \ "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --build-arg MYARG=0 --build-arg MYARG=1 myservice : echo built myservice" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -85,7 +85,7 @@ load '../lib/shared' stub docker-compose \ "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -106,7 +106,7 @@ load '../lib/shared' stub buildkite-agent \ "meta-data set docker-compose-plugin-built-image-tag-myservice my.repository/llamas:test-myservice-build-1 : echo set image metadata for myservice" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -134,7 +134,7 @@ load '../lib/shared' "meta-data set docker-compose-plugin-built-image-tag-myservice-1 my.repository/llamas:test-myservice-build-1 : echo set image metadata for myservice-1" \ "meta-data set docker-compose-plugin-built-image-tag-myservice-2 my.repository/llamas:test-myservice-build-1 : echo set image metadata for myservice-2" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -163,7 +163,7 @@ load '../lib/shared' stub buildkite-agent \ "meta-data set docker-compose-plugin-built-image-tag-myservice my.repository/llamas:test-myservice-build-1 : echo set image metadata for myservice" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -188,7 +188,7 @@ load '../lib/shared' stub buildkite-agent \ "meta-data set docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v2.0.yml my.repository/llamas:test-myservice-build-1 : echo set image metadata for myservice" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -214,7 +214,7 @@ load '../lib/shared' stub buildkite-agent \ "meta-data set docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v2.0.yml-tests/composefiles/docker-compose.v2.1.yml my.repository/llamas:test-myservice-build-1 : echo set image metadata for myservice" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -240,7 +240,7 @@ load '../lib/shared' "meta-data set docker-compose-plugin-built-image-tag-myservice1 my.repository/llamas:test-myservice1-build-1 : echo set image metadata for myservice1" \ "meta-data set docker-compose-plugin-built-image-tag-myservice2 my.repository/llamas:test-myservice2-build-1 : echo set image metadata for myservice2" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built all services" @@ -258,7 +258,7 @@ load '../lib/shared' export BUILDKITE_PIPELINE_SLUG=test export BUILDKITE_BUILD_NUMBER=1 - run $PWD/hooks/command + run "$PWD"/hooks/command assert_failure assert_output --partial "Compose file versions 2.0 and above" @@ -278,7 +278,7 @@ load '../lib/shared' stub docker-compose \ "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "pulled cache image" @@ -300,7 +300,7 @@ load '../lib/shared' stub docker-compose \ "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --no-cache helloworld : echo built helloworld" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success refute_output --partial "pulled cache image" @@ -324,7 +324,7 @@ load '../lib/shared' stub docker-compose \ "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "pulled cache image" @@ -355,7 +355,7 @@ load '../lib/shared' stub docker-compose \ "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "pulled cache image build-target" @@ -392,7 +392,7 @@ load '../lib/shared' stub docker-compose \ "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "pulled cache image build-target-latest" @@ -431,7 +431,7 @@ load '../lib/shared' stub docker-compose \ "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "pulled cache image build-target-latest" @@ -463,7 +463,7 @@ load '../lib/shared' stub docker-compose \ "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "pulled cache image" @@ -488,7 +488,7 @@ load '../lib/shared' stub docker-compose \ "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "my.repository/myservice_cache:latest will not be used as a cache for helloworld" @@ -512,7 +512,7 @@ load '../lib/shared' stub docker-compose \ "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull hello-world : echo built hello-world" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "pulled cache image" @@ -539,7 +539,7 @@ load '../lib/shared' stub docker-compose \ "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "pulled cache image" @@ -564,7 +564,7 @@ load '../lib/shared' stub buildkite-agent \ "meta-data set docker-compose-plugin-built-image-tag-myservice my.repository/llamas:my-llamas-image : echo set image metadata for myservice" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -590,7 +590,7 @@ load '../lib/shared' stub buildkite-agent \ "meta-data set docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v3.2.yml my.repository/llamas:my-llamas-image : echo set image metadata for myservice" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -618,7 +618,7 @@ load '../lib/shared' "meta-data set docker-compose-plugin-built-image-tag-myservice1 my.repository/llamas:my-llamas-image-1 : echo set image metadata for myservice1" \ "meta-data set docker-compose-plugin-built-image-tag-myservice2 my.repository/llamas:my-llamas-image-2 : echo set image metadata for myservice2" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built all services" From 6bdaf2e8589ba8caf4007a91cd71530cd49ba05f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Thu, 6 Oct 2022 00:37:57 -0300 Subject: [PATCH 093/253] Added run option skip-pull --- commands/run.sh | 2 +- plugin.yml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/commands/run.sh b/commands/run.sh index eaa14c00..d63e5ad8 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -69,7 +69,7 @@ elif [[ ${#pull_services[@]} -eq 1 ]] ; then fi # Pull down specified services -if [[ ${#pull_services[@]} -gt 0 ]] ; then +if [[ ${#pull_services[@]} -gt 0 ]] && [[ "$(plugin_read_config SKIP_PULL "false")" == "false" ]]; then echo "~~~ :docker: Pulling services ${pull_services[0]}" retry "$pull_retries" run_docker_compose "${pull_params[@]}" diff --git a/plugin.yml b/plugin.yml index c385d1fb..e273caa6 100644 --- a/plugin.yml +++ b/plugin.yml @@ -70,6 +70,8 @@ configuration: type: string rm: type: boolean + skip-pull: + type: boolean upload-container-logs: type: string propagate-uid-gid: @@ -107,6 +109,7 @@ configuration: propagate-uid-gid: [ run ] pull: [ run ] push-retries: [ push ] + skip-pull: [ run ] tty: [ run ] use-aliases: [ run ] user: [ run ] From bdbf53ee55f44accb2fe41a93f826d3b75cb7b0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Thu, 6 Oct 2022 00:58:12 -0300 Subject: [PATCH 094/253] Made condition turn on with specific value --- commands/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/run.sh b/commands/run.sh index d63e5ad8..11ac6427 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -69,7 +69,7 @@ elif [[ ${#pull_services[@]} -eq 1 ]] ; then fi # Pull down specified services -if [[ ${#pull_services[@]} -gt 0 ]] && [[ "$(plugin_read_config SKIP_PULL "false")" == "false" ]]; then +if [[ ${#pull_services[@]} -gt 0 ]] && [[ "$(plugin_read_config SKIP_PULL "false")" != "true" ]]; then echo "~~~ :docker: Pulling services ${pull_services[0]}" retry "$pull_retries" run_docker_compose "${pull_params[@]}" From 6365558d9263f3e3038c0a626cc96406d9e14277 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Thu, 6 Oct 2022 01:05:40 -0300 Subject: [PATCH 095/253] Added documentation for option --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c11ca3c2..18e68a65 100644 --- a/README.md +++ b/README.md @@ -390,7 +390,7 @@ Will cause the image to be pushed twice (once by the build step and another by t ### `pull` (optional, run only) -Pull down multiple pre-built images. By default only the service that is being run will be pulled down, but this allows multiple images to be specified to handle prebuilt dependent images. +Pull down multiple pre-built images. By default only the service that is being run will be pulled down, but this allows multiple images to be specified to handle prebuilt dependent images. Note that pulling will be skipped if the `skip-pull` option is activated. ### `config` (optional) @@ -440,6 +440,10 @@ Example: `[ "powershell", "-Command" ]` Whether to skip the repository checkout phase. This is useful for steps that use a pre-built image. This will fail if there is no pre-built image. +### `skip-pull` (optional, run only) + +Completely avoid running any `pull` command. Images being used will need to be present in the machine from before or have been built in the same step. Could be useful to avoid hitting rate limits when you can be sure the operation is unnecessary. Note that it is possible other commands run in the plugin's lifecycle will trigger a pull of necessary images. + ### `workdir` (optional, run only) Specify the container working directory via `docker-compose run --workdir`. From 6241660cd6a9ef6ecc1fa871c4378157ea195b93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Thu, 6 Oct 2022 01:09:33 -0300 Subject: [PATCH 096/253] Corrected shellcheck warnings in run.bats --- tests/run.bats | 78 ++++++++++++++++++++++++-------------------------- 1 file changed, 38 insertions(+), 40 deletions(-) diff --git a/tests/run.bats b/tests/run.bats index 916705f9..9389eb58 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -25,7 +25,7 @@ load '../lib/run' stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -51,7 +51,7 @@ load '../lib/run' stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success refute_output --partial "The Docker Compose Plugin does not correctly support step-level array commands" @@ -79,7 +79,7 @@ load '../lib/run' stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -105,7 +105,7 @@ load '../lib/run' stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -133,7 +133,7 @@ cmd3" stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "The Docker Compose Plugin does not correctly support step-level array commands" @@ -162,7 +162,7 @@ cmd3" stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success refute_output --partial "The Docker Compose Plugin does not correctly support step-level array commands" @@ -194,7 +194,7 @@ cmd3" stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice" @@ -220,7 +220,7 @@ cmd3" stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -248,7 +248,7 @@ cmd3" stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -275,7 +275,7 @@ cmd3" "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice" @@ -302,7 +302,7 @@ cmd3" "meta-data exists docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v2.0.yml : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v2.0.yml : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice" @@ -330,7 +330,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v2.0.yml-tests/composefiles/docker-compose.v2.1.yml : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v2.0.yml-tests/composefiles/docker-compose.v2.1.yml : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice" @@ -357,7 +357,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v2.0.yml : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v2.0.yml : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice" @@ -381,7 +381,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_failure assert_output --partial "Exited with 2" @@ -409,7 +409,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "pulled myservice" @@ -437,7 +437,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice without tty" @@ -463,7 +463,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice without dependencies" @@ -490,7 +490,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice without ansi output" @@ -517,7 +517,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice with use aliases output" @@ -545,7 +545,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice with volumes" @@ -572,7 +572,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice with volumes" @@ -602,7 +602,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice with volumes" @@ -635,7 +635,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice with volumes" @@ -671,7 +671,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice with volumes" @@ -706,7 +706,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice with volumes" @@ -733,7 +733,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice with volumes" @@ -761,7 +761,7 @@ export BUILDKITE_JOB_ID=1111 stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice-llamas1.yml-llamas2.yml-llamas3.yml : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -787,7 +787,7 @@ export BUILDKITE_JOB_ID=1111 stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_failure assert_output --partial "^^^ +++" @@ -818,7 +818,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice2 : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice2 : echo myimage2" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "pulled myservice1 and myservice2" @@ -845,7 +845,7 @@ export BUILDKITE_JOB_ID=1111 stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -872,7 +872,7 @@ export BUILDKITE_JOB_ID=1111 stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -895,7 +895,7 @@ export BUILDKITE_JOB_ID=1111 stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_failure assert_output --partial "Error" @@ -923,7 +923,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice : echo myimage" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice without tty" @@ -941,8 +941,6 @@ export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false export BUILDKITE_PLUGIN_DOCKER_COMPOSE_ENTRYPOINT="my custom entrypoint" - ENTRYPOINT='--entrypoint\ \"my\ custom\ entrypoint\"' - stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ @@ -951,7 +949,7 @@ export BUILDKITE_JOB_ID=1111 stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -978,7 +976,7 @@ export BUILDKITE_JOB_ID=1111 stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -1006,7 +1004,7 @@ export BUILDKITE_JOB_ID=1111 stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -1033,7 +1031,7 @@ export BUILDKITE_JOB_ID=1111 stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -1064,7 +1062,7 @@ export BUILDKITE_JOB_ID=1111 apk add netcat-openbsd nc -lkvU $SSH_AUTH_SOCK & - run $PWD/hooks/command + run "$PWD"/hooks/command kill %1 From d9ccb60818571bc67d4bf0030aa39244ef6cb455 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Thu, 6 Oct 2022 01:19:34 -0300 Subject: [PATCH 097/253] Add test for new option --- tests/run.bats | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/run.bats b/tests/run.bats index 9389eb58..9f7dbe85 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -498,6 +498,33 @@ export BUILDKITE_JOB_ID=1111 unstub buildkite-agent } + +@test "Run without pull" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_SKIP_PULL=true + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice without pull" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "Running /bin/sh -e -c 'pwd' in service myservice" + assert_output --partial "ran myservice without pull" + + unstub docker-compose + unstub buildkite-agent +} + @test "Run with use aliases" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice From 36983425631acc5f7711e6780d6b3c755bbf40ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Thu, 6 Oct 2022 02:11:27 -0300 Subject: [PATCH 098/253] Clarified skup-checkout option (closes #232) --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 18e68a65..794ca8c4 100644 --- a/README.md +++ b/README.md @@ -438,7 +438,9 @@ Example: `[ "powershell", "-Command" ]` ### `skip-checkout` (optional, run only) -Whether to skip the repository checkout phase. This is useful for steps that use a pre-built image. This will fail if there is no pre-built image. +Whether to skip the repository checkout phase. This is useful for steps that use a pre-built image and will fail if there is no pre-built image. + +**Important**: as the code repository will not be available in the step, you need to ensure that the docker compose file(s) are present in some way (like using artifacts) ### `skip-pull` (optional, run only) From 763b3b0f731dc70880d40200eb47e6a0915c8a3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Thu, 6 Oct 2022 18:53:34 -0300 Subject: [PATCH 099/253] Moved --no-ansi option to general command --- commands/run.sh | 14 -------------- lib/shared.bash | 12 ++++++++++++ tests/run.bats | 6 +++--- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/commands/run.sh b/commands/run.sh index eaa14c00..ae394814 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -72,14 +72,6 @@ fi if [[ ${#pull_services[@]} -gt 0 ]] ; then echo "~~~ :docker: Pulling services ${pull_services[0]}" retry "$pull_retries" run_docker_compose "${pull_params[@]}" - - # Sometimes docker-compose pull leaves unfinished ansi codes - echo -fi - -# Optionally disable ansi output -if [[ "$(plugin_read_config ANSI "true")" == "false" ]] ; then - run_params+=(--no-ansi) fi # We set a predictable container name so we can find it and inspect it later on @@ -234,9 +226,6 @@ elif [[ ! -f "$override_file" ]]; then # for when an image and a build is defined in the docker-compose.ymk file, otherwise we try and # pull an image that doesn't exist run_docker_compose build "${build_params[@]}" "$run_service" - - # Sometimes docker-compose pull leaves unfinished ansi codes - echo fi dependency_exitcode=0 @@ -249,9 +238,6 @@ if [[ "$(plugin_read_config DEPENDENCIES "true")" == "true" ]] ; then else run_docker_compose up -d --scale "${run_service}=0" "${run_service}" || dependency_exitcode=$? fi - - # Sometimes docker-compose leaves unfinished ansi codes - echo fi if [[ $dependency_exitcode -ne 0 ]] ; then diff --git a/lib/shared.bash b/lib/shared.bash index dc5faf62..4f190c67 100644 --- a/lib/shared.bash +++ b/lib/shared.bash @@ -17,8 +17,16 @@ function plugin_prompt() { # Shows the command being run, and runs it function plugin_prompt_and_run() { + local exit_code + plugin_prompt "$@" "$@" + exit_code=$? + + # Sometimes docker-compose pull leaves unfinished ansi codes + echo + + return $exit_code } # Shows the command about to be run, and exits if it fails @@ -188,6 +196,10 @@ function run_docker_compose() { command+=(--verbose) fi + if [[ "$(plugin_read_config ANSI "true")" == "false" ]] ; then + command+=(--no-ansi) + fi + for file in $(docker_compose_config_files) ; do command+=(-f "$file") done diff --git a/tests/run.bats b/tests/run.bats index 916705f9..14ad90c5 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -482,9 +482,9 @@ export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_ANSI=false stub docker-compose \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml --no-ansi run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice without ansi output" + "--no-ansi -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "--no-ansi -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "--no-ansi -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice without ansi output" stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ From 6adba1c6064276e90df3049d88be66dbc5dfeb9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Thu, 6 Oct 2022 19:10:27 -0300 Subject: [PATCH 100/253] Corrected tests in v2 --- tests/v2/run.bats | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/v2/run.bats b/tests/v2/run.bats index 5d28fae0..2b3ae173 100644 --- a/tests/v2/run.bats +++ b/tests/v2/run.bats @@ -483,9 +483,9 @@ export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_ANSI=false stub docker \ - "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ - "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ - "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml --no-ansi run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice without ansi output" + "compose --no-ansi -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "compose --no-ansi -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "compose --no-ansi -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice without ansi output" stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ From 1983ce05ec5eab23eab22868381a515647386edf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 8 Oct 2022 03:14:59 -0300 Subject: [PATCH 101/253] Replace periods for better variable names --- commands/build.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/commands/build.sh b/commands/build.sh index cf660eeb..5d69948c 100755 --- a/commands/build.sh +++ b/commands/build.sh @@ -7,15 +7,24 @@ push_retries="$(plugin_read_config PUSH_RETRIES "0")" override_file="docker-compose.buildkite-${BUILDKITE_BUILD_NUMBER}-override.yml" build_images=() +normalize_var_name() { + local orig_value="$1" + # POSIX variable names should match [a-zA-Z_][a-zA-Z0-9_]* + # service names and the like also allow periods and dashes + no_periods="${orig_value//./_}" + no_dashes="${no_periods//-/_}" + echo "${no_dashes}" +} + service_name_cache_from_var() { local service_name="$1" - echo "cache_from__${service_name//-/_}" + echo "cache_from__$(normalize_var_name ${service_name})" } service_name_group_name_cache_from_var() { local service_name="$1" local group_index="$2" - echo "group_cache_from__${service_name//-/_}__${group_index//-/_}" + echo "group_cache_from__$(normalize_var_name ${service_name})__$(normalize_var_name ${group_index})" } count_of_named_array() { From 10573f6acfb4e943c377f9f77632fe0f5405133c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 8 Oct 2022 03:15:15 -0300 Subject: [PATCH 102/253] Added tests for services with periods --- tests/build.bats | 25 +++++++++++++++++++++++++ tests/v2/build.bats | 21 +++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/tests/build.bats b/tests/build.bats index 15628d21..aa89415b 100644 --- a/tests/build.bats +++ b/tests/build.bats @@ -522,6 +522,31 @@ load '../lib/shared' unstub docker-compose } +@test "Build with a service name and cache-from with period" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=hello.world + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=hello.world:my.repository/my-service_cache:latest + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "pull my.repository/my-service_cache:latest : echo pulled cache image" + + stub docker-compose \ + "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull \* : echo built \$9" + + run $PWD/hooks/command + + assert_success + assert_output --partial "pulled cache image" + assert_output --partial "- my.repository/my-service_cache:latest" + assert_output --partial "built hello.world" + unstub docker + unstub docker-compose +} + + @test "Build with a cache-from image retry on failing pull" { export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" export BUILDKITE_JOB_ID=1111 diff --git a/tests/v2/build.bats b/tests/v2/build.bats index 2322435d..05a98e65 100644 --- a/tests/v2/build.bats +++ b/tests/v2/build.bats @@ -382,6 +382,27 @@ setup_file() { unstub docker } +@test "Build with a service name and cache-from with period" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=hello.world + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=hello.world:my.repository/my-service_cache:latest + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "pull my.repository/my-service_cache:latest : echo pulled cache image" \ + "compose -f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull \* : echo built \${10}" + + run $PWD/hooks/command + + assert_success + assert_output --partial "pulled cache image" + assert_output --partial "- my.repository/my-service_cache:latest" + assert_output --partial "built hello.world" + unstub docker +} + @test "Build with a cache-from image retry on failing pull" { export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" export BUILDKITE_JOB_ID=1111 From 183a1b637064ac57f1288daa078be067c74982ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 8 Oct 2022 21:55:49 -0300 Subject: [PATCH 103/253] Corrected shellcheck errors --- commands/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/commands/build.sh b/commands/build.sh index 5d69948c..8158f0e9 100755 --- a/commands/build.sh +++ b/commands/build.sh @@ -18,13 +18,13 @@ normalize_var_name() { service_name_cache_from_var() { local service_name="$1" - echo "cache_from__$(normalize_var_name ${service_name})" + echo "cache_from__$(normalize_var_name "${service_name}")" } service_name_group_name_cache_from_var() { local service_name="$1" local group_index="$2" - echo "group_cache_from__$(normalize_var_name ${service_name})__$(normalize_var_name ${group_index})" + echo "group_cache_from__$(normalize_var_name "${service_name}")__$(normalize_var_name "${group_index}")" } count_of_named_array() { From 072dfe54daedc53365da3268804560cb87d6cac3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 10 Oct 2022 02:56:29 -0300 Subject: [PATCH 104/253] Implemented propagate-environment option --- commands/run.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/commands/run.sh b/commands/run.sh index eaa14c00..f72946fc 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -92,6 +92,19 @@ done <<< "$(printf '%s\n%s' \ "$(plugin_read_list ENV)" \ "$(plugin_read_list ENVIRONMENT)")" +# Propagate all environment variables into the container if requested +if [[ "$(plugin_read_config PROPAGATE_ENVIRONMENT "false")" =~ ^(true|on|1)$ ]] ; then + if [[ -n "${BUILDKITE_ENV_FILE:-}" ]] ; then + # Read in the env file and convert to --env params for docker + # This is because --env-file doesn't support newlines or quotes per https://docs.docker.com/compose/env-file/#syntax-rules + while read -r var; do + args+=( --env "${var%%=*}" ) + done < "$BUILDKITE_ENV_FILE" + else + echo -n "🚨 Not propagating environment variables to container as \$BUILDKITE_ENV_FILE is not set" + fi +fi + while IFS=$'\n' read -r vol ; do [[ -n "${vol:-}" ]] && run_params+=("-v" "$(expand_relative_volume_path "$vol")") done <<< "$(plugin_read_list VOLUMES)" From 02392c8faa270582051549d851b19d1e81b1e822 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 10 Oct 2022 02:57:06 -0300 Subject: [PATCH 105/253] Added spec and documentation for new option --- README.md | 24 ++++++++++++++++++++++-- plugin.yml | 2 ++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c11ca3c2..b719a146 100644 --- a/README.md +++ b/README.md @@ -135,8 +135,9 @@ If you want to use environment variables in the `volumes` element, you will need By default, docker-compose makes whatever environment variables it gets available for interpolation of docker-compose.yml, but it doesn't pass them in to your containers. -You can use the [environment key in docker-compose.yml](https://docs.docker.com/compose/environment-variables/) to either set specific environment vars or "pass through" environment -variables from outside docker-compose. +You can use the [environment key in docker-compose.yml](https://docs.docker.com/compose/environment-variables/) to either set specific environment vars or "pass through" environment variables from outside docker-compose. + +### Specific values If you want to add extra environment above what is declared in your `docker-compose.yml`, this plugin offers a `environment` block of its own: @@ -155,6 +156,19 @@ steps: Note how the values in the list can either be just a key (so the value is sourced from the environment) or a KEY=VALUE pair. +### Pipeline variables + +Alternatively, you can have the plugin add all environment variables defined for the job by the agent as defined in [`BUILDKITE_ENV_FILE`](https://buildkite.com/docs/pipelines/environment-variables#BUILDKITE_ENV_FILE) activating the `propagate-environment` option: + +```yml +steps: + - command: use-vars.sh + plugins: + - docker-compose#v4.0.0: + run: app + propagate-environment: true +``` + ## Build Arguments You can use the [build args key in docker-compose.yml](https://docs.docker.com/compose/compose-file/#args) to set specific build arguments when building an image. @@ -422,6 +436,12 @@ A list of KEY=VALUE that are passed through as build arguments when image is bei A list of either KEY or KEY=VALUE that are passed through as environment variables to the container. +### `propagate-environment` (optional, boolean) + +Whether or not to automatically propagate all pipeline environment variables into the run container. Avoiding the need to be specified with environment. + +**Important**: only pipeline variables will automatically be propagated (what you see in the Buildkite UI). Variables set in proceeding hook scripts will not be propagated to the container. + ### `command` (optional, run only, array) Sets the command for the Docker image, and defaults the `shell` option to `false`. Useful if the Docker image has an entrypoint, or doesn't contain a shell. diff --git a/plugin.yml b/plugin.yml index 4fa99451..c07eb7d9 100644 --- a/plugin.yml +++ b/plugin.yml @@ -74,6 +74,8 @@ configuration: type: string propagate-uid-gid: type: boolean + propagate-environment: + type: boolean mount-ssh-agent: type: boolean mount-buildkite-agent: From a7b3ed07fb00b501f6ce5ca1a6a558060aafb861 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 10 Oct 2022 03:08:57 -0300 Subject: [PATCH 106/253] Corrected variable in implementation --- commands/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/commands/run.sh b/commands/run.sh index f72946fc..9291dc7f 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -98,8 +98,8 @@ if [[ "$(plugin_read_config PROPAGATE_ENVIRONMENT "false")" =~ ^(true|on|1)$ ]] # Read in the env file and convert to --env params for docker # This is because --env-file doesn't support newlines or quotes per https://docs.docker.com/compose/env-file/#syntax-rules while read -r var; do - args+=( --env "${var%%=*}" ) - done < "$BUILDKITE_ENV_FILE" + run_params+=("-e" "${var%%=*}") + done < "${BUILDKITE_ENV_FILE}" else echo -n "🚨 Not propagating environment variables to container as \$BUILDKITE_ENV_FILE is not set" fi From a073cf656f7f1e47884afe02ee581a7b4c387270 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 10 Oct 2022 03:13:34 -0300 Subject: [PATCH 107/253] Added tests for new option --- tests/run.bats | 60 +++++++++++++++++++++++++++++++++++++++++++++++ tests/v2/run.bats | 60 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+) diff --git a/tests/run.bats b/tests/run.bats index 916705f9..ed55baf1 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -257,6 +257,66 @@ cmd3" unstub buildkite-agent } +@test "Run with a prebuilt image and propagate environment but no BUILDKITE_ENV_FILE" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PROPAGATE_ENVIRONMENT=true + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "Running /bin/sh -e -c 'pwd' in service myservice" + assert_output --partial "Not propagating environment variables to container" + + unstub docker-compose + unstub buildkite-agent +} + +@test "Run with a prebuilt image and propagate environment" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_ENV_FILE=/tmp/test_env + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PROPAGATE_ENVIRONMENT=true + + echo "VAR0=1" > "${BUILDKITE_ENV_FILE}" + echo "VAR2=lalala" >> "${BUILDKITE_ENV_FILE}" + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 -e \* -e \* --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with vars \${11} and \${13}" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "Running /bin/sh -e -c 'pwd' in service myservice" + assert_output --partial "ran myservice with vars VAR0 and VAR2" + + unstub docker-compose + unstub buildkite-agent + + rm "${BUILDKITE_ENV_FILE}" +} + @test "Run with a prebuilt image" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice diff --git a/tests/v2/run.bats b/tests/v2/run.bats index 5d28fae0..e49d4fdb 100644 --- a/tests/v2/run.bats +++ b/tests/v2/run.bats @@ -258,6 +258,66 @@ cmd3" unstub buildkite-agent } +@test "Run with a prebuilt image and propagate environment but no BUILDKITE_ENV_FILE" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PROPAGATE_ENVIRONMENT=true + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "Running /bin/sh -e -c 'pwd' in service myservice" + assert_output --partial "Not propagating environment variables to container" + + unstub docker + unstub buildkite-agent +} + +@test "Run with a prebuilt image and propagate environment" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_ENV_FILE=/tmp/test_env + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PROPAGATE_ENVIRONMENT=true + + echo "VAR0=1" > "${BUILDKITE_ENV_FILE}" + echo "VAR2=lalala" >> "${BUILDKITE_ENV_FILE}" + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 -e \* -e \* --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with vars \${12} and \${14}" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "Running /bin/sh -e -c 'pwd' in service myservice" + assert_output --partial "ran myservice with vars VAR0 and VAR2" + + unstub docker + unstub buildkite-agent + + rm "${BUILDKITE_ENV_FILE}" +} + @test "Run with a prebuilt image" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice From eac8c7021a6af509534279fd75e0a045d5079588 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 11 Oct 2022 19:19:32 -0300 Subject: [PATCH 108/253] Corrected valid values for image-name option --- plugin.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin.yml b/plugin.yml index 788256e8..05287636 100644 --- a/plugin.yml +++ b/plugin.yml @@ -35,7 +35,7 @@ configuration: image-repository: type: string image-name: - type: string + type: [ string, array ] pull-retries: type: integer push-retries: From e533b0b646f89fc7a9cfb6dde0ece6f47d9532fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 11 Oct 2022 19:20:04 -0300 Subject: [PATCH 109/253] Validate tag names --- commands/build.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/commands/build.sh b/commands/build.sh index 8158f0e9..0966b0a9 100755 --- a/commands/build.sh +++ b/commands/build.sh @@ -90,6 +90,12 @@ fi service_idx=0 for service_name in $(plugin_read_list BUILD) ; do image_name=$(build_image_name "${service_name}" "${service_idx}") + + if ! [[ "$image_name" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]]; then + echo "🚨 ${image_name} is not a valid tag name" + exit 1 + fi + service_idx=$((service_idx+1)) if [[ -n "$image_repository" ]] ; then From 7e7ee96be2cee4a536c68c9d6140e7c2caf8b00e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 11 Oct 2022 19:20:18 -0300 Subject: [PATCH 110/253] Added tests for invalid tags --- tests/build.bats | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/build.bats b/tests/build.bats index 77e4efe5..b5fce0d2 100644 --- a/tests/build.bats +++ b/tests/build.bats @@ -599,6 +599,46 @@ load '../lib/shared' unstub buildkite-agent } +@test "Build with an invalid image-name (start with hyphen) " { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_NAME=-llamas-image + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_REPOSITORY=my.repository/llamas + export BUILDKITE_BUILD_NUMBER=1 + + run "$PWD"/hooks/command + + assert_failure + assert_output --partial "-llamas-image is not a valid tag name" +} + +@test "Build with an invalid image-name (start with period) " { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_NAME=.llamas-image + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_REPOSITORY=my.repository/llamas + export BUILDKITE_BUILD_NUMBER=1 + + run "$PWD"/hooks/command + + assert_failure + assert_output --partial ".llamas-image is not a valid tag name" +} + +@test "Build with an invalid image-name (too long) " { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + # numbers from 1 to 69 result in 129 characters + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_NAME="$(seq 69 | tr -d "\n")" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_REPOSITORY=my.repository/llamas + export BUILDKITE_BUILD_NUMBER=1 + + run "$PWD"/hooks/command + + assert_failure + assert_output --partial "is not a valid tag name" +} + @test "Build with a custom image-name and a config" { export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" export BUILDKITE_JOB_ID=1111 From 36d1719b9c8b4c53c9976bab820650672fd9f5ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 11 Oct 2022 19:23:37 -0300 Subject: [PATCH 111/253] Added test to v2 --- tests/v2/build.bats | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/v2/build.bats b/tests/v2/build.bats index 05a98e65..304105cd 100644 --- a/tests/v2/build.bats +++ b/tests/v2/build.bats @@ -452,6 +452,46 @@ setup_file() { unstub buildkite-agent } +@test "Build with an invalid image-name (start with hyphen) " { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_NAME=-llamas-image + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_REPOSITORY=my.repository/llamas + export BUILDKITE_BUILD_NUMBER=1 + + run "$PWD"/hooks/command + + assert_failure + assert_output --partial "-llamas-image is not a valid tag name" +} + +@test "Build with an invalid image-name (start with period) " { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_NAME=.llamas-image + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_REPOSITORY=my.repository/llamas + export BUILDKITE_BUILD_NUMBER=1 + + run "$PWD"/hooks/command + + assert_failure + assert_output --partial ".llamas-image is not a valid tag name" +} + +@test "Build with an invalid image-name (too long) " { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + # numbers from 1 to 69 result in 129 characters + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_NAME="$(seq 69 | tr -d "\n")" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_REPOSITORY=my.repository/llamas + export BUILDKITE_BUILD_NUMBER=1 + + run "$PWD"/hooks/command + + assert_failure + assert_output --partial "is not a valid tag name" +} + @test "Build with a custom image-name and a config" { export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" export BUILDKITE_JOB_ID=1111 From 630e5fb9914dc99cd202c653d551a31c9b009700 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 11 Oct 2022 19:36:08 -0300 Subject: [PATCH 112/253] Added validation to cache-from images --- commands/build.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/commands/build.sh b/commands/build.sh index 0966b0a9..eefefc1d 100755 --- a/commands/build.sh +++ b/commands/build.sh @@ -51,6 +51,13 @@ if [[ "$(plugin_read_config NO_CACHE "false")" == "false" ]] ; then IFS=':' read -r -a tokens <<< "$line" service_name=${tokens[0]} service_image=$(IFS=':'; echo "${tokens[*]:1:2}") + service_tag=${tokens[2]} + + if ! [[ "$service_tag" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]]; then + echo "🚨 cache-from ${service_image} has an invalid tag so it will be ignored" + continue + fi + cache_from_group_name=$(IFS=':'; echo "${tokens[*]:3}") if [[ -z "$cache_from_group_name" ]]; then cache_from_group_name=":default:" From 4ee75303c7c52516a0ee10bc0f451d9069f880cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 11 Oct 2022 19:36:43 -0300 Subject: [PATCH 113/253] Added tests for invalid cache image tags --- tests/build.bats | 21 +++++++++++++++++++++ tests/v2/build.bats | 21 +++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/tests/build.bats b/tests/build.bats index b5fce0d2..be40072f 100644 --- a/tests/build.bats +++ b/tests/build.bats @@ -309,6 +309,27 @@ load '../lib/shared' unstub docker-compose } +@test "Build with an invalid cache-from tag" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=helloworld:my.repository/myservice_cache:-latest + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker-compose \ + "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" + + run "$PWD"/hooks/command + + assert_success + refute_output --partial "pulled cache image" + refute_output --partial "- my.repository/myservice_cache:-latest" + assert_output --partial "invalid tag so it will be ignored" + assert_output --partial "built helloworld" + unstub docker-compose +} + @test "Build with several cache-from images for one service" { export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" export BUILDKITE_JOB_ID=1111 diff --git a/tests/v2/build.bats b/tests/v2/build.bats index 304105cd..e5733b7e 100644 --- a/tests/v2/build.bats +++ b/tests/v2/build.bats @@ -272,6 +272,27 @@ setup_file() { unstub docker } +@test "Build with an invalid cache-from tag" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=helloworld:my.repository/myservice_cache:-latest + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "compose -f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" + + run "$PWD"/hooks/command + + assert_success + refute_output --partial "pulled cache image" + refute_output --partial "- my.repository/myservice_cache:-latest" + assert_output --partial "invalid tag so it will be ignored" + assert_output --partial "built helloworld" + unstub docker +} + @test "Build with a cache-from image with no-cache also set" { export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" export BUILDKITE_JOB_ID=1111 From d1ad107b5f1d6aca0b22163cdb6d5fb1f33e1714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 11 Oct 2022 19:50:30 -0300 Subject: [PATCH 114/253] Refactored tag validation to shared function --- lib/shared.bash | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/shared.bash b/lib/shared.bash index 4f190c67..583cb122 100644 --- a/lib/shared.bash +++ b/lib/shared.bash @@ -258,3 +258,13 @@ function is_windows() { function is_macos() { [[ "$OSTYPE" =~ ^(darwin) ]] } + +function validate_tag { + local tag=$1 + + if [[ "$tag" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]]; then + return 0 + else + return 1 + fi +} \ No newline at end of file From 4d13a454e24e144caca8011496aa4c1af7fa31b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 11 Oct 2022 19:50:53 -0300 Subject: [PATCH 115/253] Added validation to push command as well --- commands/push.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/commands/push.sh b/commands/push.sh index 564418da..6aecd365 100755 --- a/commands/push.sh +++ b/commands/push.sh @@ -21,6 +21,14 @@ for line in $(plugin_read_list PUSH) ; do service_name=${tokens[0]} service_image=$(compose_image_for_service "$service_name") + # push in the form of service:repo:tag + if [[ ${#tokens[@]} -gt 2 ]]; then + if ! validate_tag "${tokens[2]}"; then + echo "🚨 specified image to push ${line} has an invalid tag so it will be ignored" + continue + fi + fi + # Pull down prebuilt image if one exists if prebuilt_image=$(get_prebuilt_image "$service_name") ; then From 4069dd4f492f4d1c963fbac302eb065abf52a564 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 11 Oct 2022 19:51:17 -0300 Subject: [PATCH 116/253] Use refactored function in build command --- commands/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/commands/build.sh b/commands/build.sh index eefefc1d..7a687be9 100755 --- a/commands/build.sh +++ b/commands/build.sh @@ -53,7 +53,7 @@ if [[ "$(plugin_read_config NO_CACHE "false")" == "false" ]] ; then service_image=$(IFS=':'; echo "${tokens[*]:1:2}") service_tag=${tokens[2]} - if ! [[ "$service_tag" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]]; then + if ! validate_tag "$service_tag"; then echo "🚨 cache-from ${service_image} has an invalid tag so it will be ignored" continue fi @@ -98,7 +98,7 @@ service_idx=0 for service_name in $(plugin_read_list BUILD) ; do image_name=$(build_image_name "${service_name}" "${service_idx}") - if ! [[ "$image_name" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]]; then + if ! validate_tag "$image_name"; then echo "🚨 ${image_name} is not a valid tag name" exit 1 fi From 63c26c15f91535e755f194dc56377c655571cbdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 11 Oct 2022 19:56:39 -0300 Subject: [PATCH 117/253] Added tests for invalid push tags --- tests/push.bats | 18 ++++++++++++++++++ tests/v2/push.bats | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/tests/push.bats b/tests/push.bats index 49003647..cefe7996 100644 --- a/tests/push.bats +++ b/tests/push.bats @@ -101,6 +101,24 @@ load '../lib/shared' unstub buildkite-agent } +@test "Push a prebuilt image with an invalid tag" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PUSH=myservice:my.repository/myservice:-llamas + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 config : echo blah" + + run $PWD/hooks/command + + assert_success + refute_output --partial "pulled prebuilt image" + refute_output --partial "tagged image" + assert_output --partial "invalid tag" + unstub docker-compose +} + @test "Push a prebuilt image to multiple tags" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PUSH_0=myservice:my.repository/myservice:llamas diff --git a/tests/v2/push.bats b/tests/v2/push.bats index 6aaf4099..9feb4f95 100644 --- a/tests/v2/push.bats +++ b/tests/v2/push.bats @@ -96,6 +96,24 @@ setup_file() { unstub buildkite-agent } +@test "Push a prebuilt image with an invalid tag" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PUSH=myservice:my.repository/myservice:-llamas + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 config : echo blah" + + run $PWD/hooks/command + + assert_success + refute_output --partial "pulled prebuilt image" + refute_output --partial "tagged image" + assert_output --partial "invalid tag" + unstub docker +} + @test "Push a prebuilt image to multiple tags" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PUSH_0=myservice:my.repository/myservice:llamas From 14f1640a980681817cd5faef7940e0b554d3c3bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 11 Oct 2022 19:59:36 -0300 Subject: [PATCH 118/253] Updated version in Readme to next bugfix version --- README.md | 54 +++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 2217f8e3..f1e52336 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v4.0.0: + - docker-compose#v4.1.1: run: app ``` @@ -28,7 +28,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v4.0.0: + - docker-compose#v4.1.1: run: app config: docker-compose.tests.yml env: @@ -41,7 +41,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v4.0.0: + - docker-compose#v4.1.1: run: app config: - docker-compose.yml @@ -56,7 +56,7 @@ env: steps: - command: test.sh plugins: - - docker-compose#v4.0.0: + - docker-compose#v4.1.1: run: app ``` @@ -65,7 +65,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v4.0.0: + - docker-compose#v4.1.1: run: app command: ["custom", "command", "values"] ``` @@ -79,7 +79,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.0.0: + - docker-compose#v4.1.1: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -87,7 +87,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.0.0: + - docker-compose#v4.1.1: run: app ``` @@ -104,7 +104,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.0.0: + - docker-compose#v4.1.1: run: app ``` @@ -122,7 +122,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.0.0: + - docker-compose#v4.1.1: run: app volumes: - "./dist:/app/dist" @@ -146,7 +146,7 @@ this plugin offers a `environment` block of its own: steps: - command: generate-dist.sh plugins: - - docker-compose#v4.0.0: + - docker-compose#v4.1.1: run: app env: - BUILDKITE_BUILD_NUMBER @@ -164,7 +164,7 @@ Alternatively, you can have the plugin add all environment variables defined for steps: - command: use-vars.sh plugins: - - docker-compose#v4.0.0: + - docker-compose#v4.1.1: run: app propagate-environment: true ``` @@ -179,7 +179,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v4.0.0: + - docker-compose#v4.1.1: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -196,7 +196,7 @@ If you have multiple steps that use the same service/image (such as steps that r steps: - label: ":docker: Build" plugins: - - docker-compose#v4.0.0: + - docker-compose#v4.1.1: build: app image-repository: index.docker.io/myorg/myrepo @@ -206,7 +206,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.0.0: + - docker-compose#v4.1.1: run: app ``` @@ -222,7 +222,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v4.0.0: + - docker-compose#v4.1.1: build: - app - tests @@ -234,7 +234,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.0.0: + - docker-compose#v4.1.1: run: tests ``` @@ -246,7 +246,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v4.0.0: + - docker-compose#v4.1.1: push: app ``` @@ -256,7 +256,7 @@ To push multiple images, you can use a list: steps: - label: ":docker: Push" plugins: - - docker-compose#v4.0.0: + - docker-compose#v4.1.1: push: - first-service - second-service @@ -268,7 +268,7 @@ If you want to push to a specific location (that's not defined as the `image` in steps: - label: ":docker: Push" plugins: - - docker-compose#v4.0.0: + - docker-compose#v4.1.1: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -282,14 +282,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker: Build an image" plugins: - - docker-compose#v4.0.0: + - docker-compose#v4.1.1: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.0.0: + - docker-compose#v4.1.1: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -303,7 +303,7 @@ This plugin allows for the value of `cache-from` to be a string or a list. If it steps: - label: ":docker Build an image" plugins: - - docker-compose#v4.0.0: + - docker-compose#v4.1.1: build: app image-repository: index.docker.io/myorg/myrepo cache-from: @@ -312,7 +312,7 @@ steps: - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.0.0: + - docker-compose#v4.1.1: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:my-branch @@ -326,7 +326,7 @@ Adding a grouping tag to the end of a cache-from list item allows this plugin to steps: - label: ":docker: Build Intermediate Image" plugins: - - docker-compose#v4.0.0: + - docker-compose#v4.1.1: build: myservice_intermediate # docker-compose.yml is the same as myservice but has `target: intermediate` image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo/myservice_intermediate @@ -336,7 +336,7 @@ steps: - wait - label: ":docker: Build Final Image" plugins: - - docker-compose#v4.0.0: + - docker-compose#v4.1.1: build: myservice image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo @@ -380,7 +380,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Run & Push" plugins: - - docker-compose#v4.0.0: + - docker-compose#v4.1.1: run: myservice push: myservice ``` @@ -395,7 +395,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Build & Push" plugins: - - docker-compose#v4.0.0: + - docker-compose#v4.1.1: build: myservice push: myservice ``` From d9d0bb051eba3ed1116422248e41c5826672ba04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Wed, 12 Oct 2022 18:01:18 -0300 Subject: [PATCH 119/253] Sorted plugin properties alphabetically --- plugin.yml | 86 +++++++++++++++++++++++++++--------------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/plugin.yml b/plugin.yml index 05287636..108bb9a4 100644 --- a/plugin.yml +++ b/plugin.yml @@ -14,81 +14,81 @@ configuration: push: type: [ string, array ] minimum: 1 - pull: + ansi: + type: boolean + args: type: [ string, array ] minimum: 1 - config: + build-alias: type: [ string, array ] minimum: 1 - env: + cache-from: type: [ string, array ] minimum: 1 - environment: + cli-version: + type: string + enum: + - 1 + - 2 + command: + type: array + config: type: [ string, array ] minimum: 1 - args: + dependencies: + type: boolean + entrypoint: + type: string + env: type: [ string, array ] minimum: 1 - build-alias: + environment: type: [ string, array ] minimum: 1 + expand-volume-vars: + type: boolean image-repository: type: string image-name: type: [ string, array ] - pull-retries: - type: integer - push-retries: - type: integer - cache-from: - type: [ string, array ] - minimum: 1 - volumes: - type: [ string, array ] - minimum: 1 - expand-volume-vars: - type: boolean - command: - type: array - skip-checkout: - type: boolean leave-volumes: type: boolean - no-cache: - type: boolean - use-aliases: + mount-buildkite-agent: type: boolean - tty: + mount-ssh-agent: type: boolean - dependencies: + no-cache: type: boolean - ansi: + propagate-environment: type: boolean - verbose: + propagate-uid-gid: type: boolean - workdir: - type: string + pull: + type: [ string, array ] + minimum: 1 + pull-retries: + type: integer + push-retries: + type: integer rm: type: boolean + skip-checkout: + type: boolean skip-pull: type: boolean + tty: + type: boolean upload-container-logs: type: string - propagate-uid-gid: - type: boolean - propagate-environment: - type: boolean - mount-ssh-agent: + use-aliases: type: boolean - mount-buildkite-agent: + verbose: type: boolean - entrypoint: - type: string - cli-version: + volumes: + type: [ string, array ] + minimum: 1 + workdir: type: string - enum: - - 1 - - 2 anyOf: - required: - run From c45678374da6f9916c38dfd6b8c641d42a204f18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Wed, 12 Oct 2022 18:21:09 -0300 Subject: [PATCH 120/253] Added target option --- commands/build.sh | 5 +++++ plugin.yml | 3 +++ 2 files changed, 8 insertions(+) diff --git a/commands/build.sh b/commands/build.sh index 7a687be9..b382a771 100755 --- a/commands/build.sh +++ b/commands/build.sh @@ -147,6 +147,11 @@ if [[ "$(plugin_read_config BUILD_PARALLEL "false")" == "true" ]] ; then build_params+=(--parallel) fi +target="$(plugin_read_config TARGET "")" +if [[ -n "$target" ]] ; then + build_params+=(--target "$target") +fi + while read -r arg ; do [[ -n "${arg:-}" ]] && build_params+=("--build-arg" "${arg}") done <<< "$(plugin_read_list ARGS)" diff --git a/plugin.yml b/plugin.yml index 108bb9a4..872c4a88 100644 --- a/plugin.yml +++ b/plugin.yml @@ -76,6 +76,8 @@ configuration: type: boolean skip-pull: type: boolean + target: + type: string tty: type: boolean upload-container-logs: @@ -112,6 +114,7 @@ configuration: pull: [ run ] push-retries: [ push ] skip-pull: [ run ] + target: [ build ] tty: [ run ] use-aliases: [ run ] user: [ run ] From 8422b242f517c214ea00704f60a5758a9c3991bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Wed, 12 Oct 2022 18:21:55 -0300 Subject: [PATCH 121/253] Added tests on new option --- tests/build.bats | 20 ++++++++++++++++++++ tests/v2/build.bats | 20 ++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/tests/build.bats b/tests/build.bats index be40072f..5b09d0c8 100644 --- a/tests/build.bats +++ b/tests/build.bats @@ -714,3 +714,23 @@ load '../lib/shared' unstub docker-compose unstub buildkite-agent } + +@test "Build with target" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_TARGET=intermediate + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --target \* \* : echo built \${11} with target \${10}" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "with target intermediate" + + unstub docker-compose +} \ No newline at end of file diff --git a/tests/v2/build.bats b/tests/v2/build.bats index e5733b7e..ba96142d 100644 --- a/tests/v2/build.bats +++ b/tests/v2/build.bats @@ -567,3 +567,23 @@ setup_file() { unstub docker unstub buildkite-agent } + +@test "Build with target" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_TARGET=intermediate + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --target \* \* : echo built \${12} with target \${11}" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "with target intermediate" + + unstub docker +} \ No newline at end of file From 148de07a2fa4aab383f44dc0aa4b4bcdc88b8c51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Wed, 12 Oct 2022 18:25:17 -0300 Subject: [PATCH 122/253] Corrected all shellcheck warnings --- tests/build.bats | 4 ++-- tests/v2/build.bats | 46 ++++++++++++++++++++++----------------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/tests/build.bats b/tests/build.bats index 5b09d0c8..30efdf35 100644 --- a/tests/build.bats +++ b/tests/build.bats @@ -557,7 +557,7 @@ load '../lib/shared' stub docker-compose \ "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull \* : echo built \$9" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "pulled cache image" @@ -649,7 +649,7 @@ load '../lib/shared' @test "Build with an invalid image-name (too long) " { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice - # numbers from 1 to 69 result in 129 characters + # shellcheck disable=SC2155 # numbers from 1 to 69 result in 129 characters export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_NAME="$(seq 69 | tr -d "\n")" export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_REPOSITORY=my.repository/llamas export BUILDKITE_BUILD_NUMBER=1 diff --git a/tests/v2/build.bats b/tests/v2/build.bats index ba96142d..a61bed27 100644 --- a/tests/v2/build.bats +++ b/tests/v2/build.bats @@ -20,7 +20,7 @@ setup_file() { stub docker \ "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" - run $PWD/hooks/command + run "$PWD"/hooks/command unstub docker assert_success @@ -37,7 +37,7 @@ setup_file() { stub docker \ "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --no-cache myservice : echo built myservice" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -54,7 +54,7 @@ setup_file() { stub docker \ "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --parallel myservice : echo built myservice" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -72,7 +72,7 @@ setup_file() { stub docker \ "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --build-arg MYARG=0 --build-arg MYARG=1 myservice : echo built myservice" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -93,7 +93,7 @@ setup_file() { stub buildkite-agent \ "meta-data set docker-compose-plugin-built-image-tag-myservice my.repository/llamas:test-myservice-build-1 : echo set image metadata for myservice" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -121,7 +121,7 @@ setup_file() { "meta-data set docker-compose-plugin-built-image-tag-myservice-1 my.repository/llamas:test-myservice-build-1 : echo set image metadata for myservice-1" \ "meta-data set docker-compose-plugin-built-image-tag-myservice-2 my.repository/llamas:test-myservice-build-1 : echo set image metadata for myservice-2" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -150,7 +150,7 @@ setup_file() { stub buildkite-agent \ "meta-data set docker-compose-plugin-built-image-tag-myservice my.repository/llamas:test-myservice-build-1 : echo set image metadata for myservice" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -175,7 +175,7 @@ setup_file() { stub buildkite-agent \ "meta-data set docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v2.0.yml my.repository/llamas:test-myservice-build-1 : echo set image metadata for myservice" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -201,7 +201,7 @@ setup_file() { stub buildkite-agent \ "meta-data set docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v2.0.yml-tests/composefiles/docker-compose.v2.1.yml my.repository/llamas:test-myservice-build-1 : echo set image metadata for myservice" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -227,7 +227,7 @@ setup_file() { "meta-data set docker-compose-plugin-built-image-tag-myservice1 my.repository/llamas:test-myservice1-build-1 : echo set image metadata for myservice1" \ "meta-data set docker-compose-plugin-built-image-tag-myservice2 my.repository/llamas:test-myservice2-build-1 : echo set image metadata for myservice2" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built all services" @@ -245,7 +245,7 @@ setup_file() { export BUILDKITE_PIPELINE_SLUG=test export BUILDKITE_BUILD_NUMBER=1 - run $PWD/hooks/command + run "$PWD"/hooks/command assert_failure assert_output --partial "Compose file versions 2.0 and above" @@ -263,7 +263,7 @@ setup_file() { "pull my.repository/myservice_cache:latest : echo pulled cache image" \ "compose -f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "pulled cache image" @@ -305,7 +305,7 @@ setup_file() { stub docker \ "compose -f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --no-cache helloworld : echo built helloworld" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success refute_output --partial "pulled cache image" @@ -327,7 +327,7 @@ setup_file() { "pull my.repository/myservice_cache:branch-name : echo pulled cache image" \ "compose -f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "pulled cache image" @@ -351,7 +351,7 @@ setup_file() { "pull my.repository/myservice_cache:latest : echo pulled cache image" \ "compose -f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "pulled cache image" @@ -373,7 +373,7 @@ setup_file() { "pull my.repository/myservice_cache:latest : exit 1" \ "compose -f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "my.repository/myservice_cache:latest will not be used as a cache for helloworld" @@ -394,7 +394,7 @@ setup_file() { "pull my.repository/my-service_cache:latest : echo pulled cache image" \ "compose -f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull hello-world : echo built hello-world" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "pulled cache image" @@ -415,7 +415,7 @@ setup_file() { "pull my.repository/my-service_cache:latest : echo pulled cache image" \ "compose -f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull \* : echo built \${10}" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "pulled cache image" @@ -439,7 +439,7 @@ setup_file() { "pull my.repository/myservice_cache:latest : echo pulled cache image" \ "compose -f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "pulled cache image" @@ -463,7 +463,7 @@ setup_file() { stub buildkite-agent \ "meta-data set docker-compose-plugin-built-image-tag-myservice my.repository/llamas:my-llamas-image : echo set image metadata for myservice" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -502,7 +502,7 @@ setup_file() { @test "Build with an invalid image-name (too long) " { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice - # numbers from 1 to 69 result in 129 characters + # shellcheck disable=SC2155 # numbers from 1 to 69 result in 129 characters export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_NAME="$(seq 69 | tr -d "\n")" export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_REPOSITORY=my.repository/llamas export BUILDKITE_BUILD_NUMBER=1 @@ -529,7 +529,7 @@ setup_file() { stub buildkite-agent \ "meta-data set docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v3.2.yml my.repository/llamas:my-llamas-image : echo set image metadata for myservice" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -557,7 +557,7 @@ setup_file() { "meta-data set docker-compose-plugin-built-image-tag-myservice1 my.repository/llamas:my-llamas-image-1 : echo set image metadata for myservice1" \ "meta-data set docker-compose-plugin-built-image-tag-myservice2 my.repository/llamas:my-llamas-image-2 : echo set image metadata for myservice2" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built all services" From dfc2402cd55a3dc6ff502cc37c42b496dc2f25d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Wed, 12 Oct 2022 18:29:07 -0300 Subject: [PATCH 123/253] Added documentation for target option --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f1e52336..5bdd7843 100644 --- a/README.md +++ b/README.md @@ -464,7 +464,7 @@ Whether to skip the repository checkout phase. This is useful for steps that use ### `skip-pull` (optional, run only) -Completely avoid running any `pull` command. Images being used will need to be present in the machine from before or have been built in the same step. Could be useful to avoid hitting rate limits when you can be sure the operation is unnecessary. Note that it is possible other commands run in the plugin's lifecycle will trigger a pull of necessary images. +Completely avoid running any `pull` command. Images being used will need to be present in the machine from before or have been built in the same step. Could be useful to avoid hitting rate limits when you can be sure the operation is unnecessary. Note that it is possible other commands run in the plugin's lifecycle will trigger a pull of necessary images. ### `workdir` (optional, run only) @@ -508,6 +508,12 @@ This option can also be configured on the agent machine using the environment va A list of images to pull caches from in the format `service:index.docker.io/myorg/myrepo/myapp:tag` before building, ignoring any failures. If multiple images are listed for a service, the first one to successfully pull will be used. Requires docker-compose file version `3.2+`. +### `target` (optional, build only) + +Allow for intermediate builds with `--target VALUE` options. + +Note that there is a single build command run for all services so the target value will apply to all of them. + ### `volumes` (optional, run only) A list of volumes to mount into the container. If a matching volume exists in the Docker Compose config file, this option will override that definition. From ea1cdf8a9b40b0f841524f69d038b0590966c5df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Thu, 13 Oct 2022 02:48:28 -0300 Subject: [PATCH 124/253] Added very basic buildkit support --- README.md | 6 ++++++ commands/build.sh | 6 ++++++ plugin.yml | 3 +++ 3 files changed, 15 insertions(+) diff --git a/README.md b/README.md index 5bdd7843..eaf9d6f3 100644 --- a/README.md +++ b/README.md @@ -612,6 +612,12 @@ To run the tests: docker-compose run --rm tests bats tests tests/v2 ``` +### `buildkit` (optional, build only, boolean) + +Assuming you have a compatible docker available in the agent, activating this option would setup the environment for the `docker-compose build` call to use BuildKit. Note that if you are using `cli-version` 2, you are already using buildkit by default. + +You may want to also add `BUILDKIT_INLINE_CACHE=1` to your build arguments (`args` option in this plugin), but know that [there are known issues with it](https://github.com/moby/buildkit/issues/2274). + ## License MIT (see [LICENSE](LICENSE)) diff --git a/commands/build.sh b/commands/build.sh index b382a771..0748a108 100755 --- a/commands/build.sh +++ b/commands/build.sh @@ -44,6 +44,12 @@ if [[ -z "$image_repository" ]] ; then echo "This build step has no image-repository set. Without an image-repository, the Docker image won't be pushed to a repository, and won't be automatically used by any run steps." fi +if [[ "$(plugin_read_config BUILDKIT "false")" == "true" ]]; then + export DOCKER_BUILDKIT=1 + export COMPOSE_DOCKER_CLI_BUILD=1 + export BUILDKIT_PROGRESS=plain +fi + # Read any cache-from parameters provided and pull down those images first # If no-cache is set skip pulling the cache-from images if [[ "$(plugin_read_config NO_CACHE "false")" == "false" ]] ; then diff --git a/plugin.yml b/plugin.yml index 872c4a88..7919ac9b 100644 --- a/plugin.yml +++ b/plugin.yml @@ -22,6 +22,8 @@ configuration: build-alias: type: [ string, array ] minimum: 1 + buildkit: + type: boolean cache-from: type: [ string, array ] minimum: 1 @@ -101,6 +103,7 @@ configuration: additionalProperties: false dependencies: ansi: [ run ] + buildkit: [ build ] cache-from: [ build ] dependencies: [ run ] env: [ run ] From cf4493636721e11c42de56c793da18436e16152e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Thu, 13 Oct 2022 03:08:49 -0300 Subject: [PATCH 125/253] Added --ssh option --- README.md | 6 +++++- commands/build.sh | 12 ++++++++++-- plugin.yml | 3 +++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index eaf9d6f3..f2aa88f8 100644 --- a/README.md +++ b/README.md @@ -614,10 +614,14 @@ docker-compose run --rm tests bats tests tests/v2 ### `buildkit` (optional, build only, boolean) -Assuming you have a compatible docker available in the agent, activating this option would setup the environment for the `docker-compose build` call to use BuildKit. Note that if you are using `cli-version` 2, you are already using buildkit by default. +Assuming you have a compatible docker installation and configuration in the agent, activating this option would setup the environment for the `docker-compose build` call to use BuildKit. Note that if you are using `cli-version` 2, you are already using buildkit by default. You may want to also add `BUILDKIT_INLINE_CACHE=1` to your build arguments (`args` option in this plugin), but know that [there are known issues with it](https://github.com/moby/buildkit/issues/2274). +### `ssh` (optional, build only, boolean) + +When enabled, it will add the `--ssh` option to the build command. Note that it assumes you have a compatible docker installation and configuration in the agent (meaning you are using BuildKit and it is correctly setup). + ## License MIT (see [LICENSE](LICENSE)) diff --git a/commands/build.sh b/commands/build.sh index 0748a108..6801d1b5 100755 --- a/commands/build.sh +++ b/commands/build.sh @@ -143,7 +143,7 @@ while read -r line ; do [[ -n "$line" ]] && services+=("$line") done <<< "$(plugin_read_list BUILD)" -build_params=(--pull) +build_params=(build --pull) if [[ "$(plugin_read_config NO_CACHE "false")" == "true" ]] ; then build_params+=(--no-cache) @@ -153,6 +153,14 @@ if [[ "$(plugin_read_config BUILD_PARALLEL "false")" == "true" ]] ; then build_params+=(--parallel) fi +if [[ "$(plugin_read_config SSH "false")" == "true" ]] ; then + if [[ "${DOCKER_BUILDKIT}" != "1" ]]; then + echo "🚨 You can not use the ssh option if you are not using buildkit" + exit 1 + fi + build_params+=(--ssh) +fi + target="$(plugin_read_config TARGET "")" if [[ -n "$target" ]] ; then build_params+=(--target "$target") @@ -163,7 +171,7 @@ while read -r arg ; do done <<< "$(plugin_read_list ARGS)" echo "+++ :docker: Building services ${services[*]}" -run_docker_compose -f "$override_file" build "${build_params[@]}" "${services[@]}" +run_docker_compose -f "$override_file" "${build_params[@]}" "${services[@]}" if [[ -n "$image_repository" ]] ; then echo "~~~ :docker: Pushing built images to $image_repository" diff --git a/plugin.yml b/plugin.yml index 7919ac9b..c2136ecb 100644 --- a/plugin.yml +++ b/plugin.yml @@ -78,6 +78,8 @@ configuration: type: boolean skip-pull: type: boolean + ssh: + type: boolean target: type: string tty: @@ -117,6 +119,7 @@ configuration: pull: [ run ] push-retries: [ push ] skip-pull: [ run ] + ssh: [ buildkit ] target: [ build ] tty: [ run ] use-aliases: [ run ] From 0d32deccf177d97bf660c68f8d1717b00c757337 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 15 Oct 2022 02:15:05 -0300 Subject: [PATCH 126/253] Added tests and corrected compatibility issue --- commands/build.sh | 2 +- tests/build.bats | 36 ++++++++++++++++++++++++++++++++++++ tests/v2/build.bats | 20 ++++++++++++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/commands/build.sh b/commands/build.sh index 6801d1b5..57d1c859 100755 --- a/commands/build.sh +++ b/commands/build.sh @@ -154,7 +154,7 @@ if [[ "$(plugin_read_config BUILD_PARALLEL "false")" == "true" ]] ; then fi if [[ "$(plugin_read_config SSH "false")" == "true" ]] ; then - if [[ "${DOCKER_BUILDKIT}" != "1" ]]; then + if [[ "${DOCKER_BUILDKIT:-}" != "1" && "${BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLI_VERSION:-}" != "2" ]]; then echo "🚨 You can not use the ssh option if you are not using buildkit" exit 1 fi diff --git a/tests/build.bats b/tests/build.bats index 30efdf35..1d3b7531 100644 --- a/tests/build.bats +++ b/tests/build.bats @@ -732,5 +732,41 @@ load '../lib/shared' assert_output --partial "built myservice" assert_output --partial "with target intermediate" + unstub docker-compose +} + +@test "Build with ssh option (but no buildkit)" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_SSH=true + + run "$PWD"/hooks/command + + assert_failure + assert_output --partial "You can not use the ssh option if you are not using buildkit" + refute_output --partial "built myservice" +} + +@test "Build with ssh option and buildkit" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILDKIT=true + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_SSH=true + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --ssh \* : echo built \${10} with ssh" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "with ssh" + unstub docker-compose } \ No newline at end of file diff --git a/tests/v2/build.bats b/tests/v2/build.bats index a61bed27..30389a69 100644 --- a/tests/v2/build.bats +++ b/tests/v2/build.bats @@ -585,5 +585,25 @@ setup_file() { assert_output --partial "built myservice" assert_output --partial "with target intermediate" + unstub docker +} + +@test "Build with ssh option" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_SSH=true + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --ssh \* : echo built \${11} with ssh" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "with ssh" + unstub docker } \ No newline at end of file From be871aa4d9612678567f882e0a75f1f14c95b02c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 17 Oct 2022 18:27:02 -0300 Subject: [PATCH 127/253] Allow for tag-less cache-from --- commands/build.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/commands/build.sh b/commands/build.sh index b382a771..bf69d673 100755 --- a/commands/build.sh +++ b/commands/build.sh @@ -51,7 +51,11 @@ if [[ "$(plugin_read_config NO_CACHE "false")" == "false" ]] ; then IFS=':' read -r -a tokens <<< "$line" service_name=${tokens[0]} service_image=$(IFS=':'; echo "${tokens[*]:1:2}") - service_tag=${tokens[2]} + if [ ${#tokens[@]} -gt 2 ]; then + service_tag=${tokens[2]} + else + service_tag="latest" + fi if ! validate_tag "$service_tag"; then echo "🚨 cache-from ${service_image} has an invalid tag so it will be ignored" From 1b0913111038b260a19f5a2550bc66c1d8f03fc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 17 Oct 2022 18:27:16 -0300 Subject: [PATCH 128/253] Added tests for tag-less cache-from --- tests/build.bats | 25 +++++++++++++++++++++++++ tests/v2/build.bats | 22 ++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/tests/build.bats b/tests/build.bats index 30efdf35..d0af63a5 100644 --- a/tests/build.bats +++ b/tests/build.bats @@ -330,6 +330,31 @@ load '../lib/shared' unstub docker-compose } +@test "Build with a cache-from image with no tag" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=helloworld:my.repository/myservice_cache + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" + + stub docker \ + "pull my.repository/myservice_cache : echo pulled cache image" + + stub docker-compose \ + "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "pulled cache image" + assert_output --partial "- my.repository/myservice_cache" + assert_output --partial "built helloworld" + unstub docker + unstub docker-compose +} + @test "Build with several cache-from images for one service" { export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" export BUILDKITE_JOB_ID=1111 diff --git a/tests/v2/build.bats b/tests/v2/build.bats index a61bed27..298a3d74 100644 --- a/tests/v2/build.bats +++ b/tests/v2/build.bats @@ -293,6 +293,28 @@ setup_file() { unstub docker } +@test "Build with a cache-from image with no tag" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=helloworld:my.repository/myservice_cache + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" + + stub docker \ + "pull my.repository/myservice_cache : echo pulled cache image" \ + "compose -f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "pulled cache image" + assert_output --partial "- my.repository/myservice_cache" + assert_output --partial "built helloworld" + unstub docker +} + @test "Build with a cache-from image with no-cache also set" { export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" export BUILDKITE_JOB_ID=1111 From 73f1b8896a295a8c4d189c2564ec8debbdbfa63a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 17 Oct 2022 18:42:09 -0300 Subject: [PATCH 129/253] Updated plugin version in preparation for next release --- README.md | 54 +++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index f2aa88f8..f939336b 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v4.1.1: + - docker-compose#v4.3.0: run: app ``` @@ -28,7 +28,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v4.1.1: + - docker-compose#v4.3.0: run: app config: docker-compose.tests.yml env: @@ -41,7 +41,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v4.1.1: + - docker-compose#v4.3.0: run: app config: - docker-compose.yml @@ -56,7 +56,7 @@ env: steps: - command: test.sh plugins: - - docker-compose#v4.1.1: + - docker-compose#v4.3.0: run: app ``` @@ -65,7 +65,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v4.1.1: + - docker-compose#v4.3.0: run: app command: ["custom", "command", "values"] ``` @@ -79,7 +79,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.1.1: + - docker-compose#v4.3.0: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -87,7 +87,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.1.1: + - docker-compose#v4.3.0: run: app ``` @@ -104,7 +104,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.1.1: + - docker-compose#v4.3.0: run: app ``` @@ -122,7 +122,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.1.1: + - docker-compose#v4.3.0: run: app volumes: - "./dist:/app/dist" @@ -146,7 +146,7 @@ this plugin offers a `environment` block of its own: steps: - command: generate-dist.sh plugins: - - docker-compose#v4.1.1: + - docker-compose#v4.3.0: run: app env: - BUILDKITE_BUILD_NUMBER @@ -164,7 +164,7 @@ Alternatively, you can have the plugin add all environment variables defined for steps: - command: use-vars.sh plugins: - - docker-compose#v4.1.1: + - docker-compose#v4.3.0: run: app propagate-environment: true ``` @@ -179,7 +179,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v4.1.1: + - docker-compose#v4.3.0: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -196,7 +196,7 @@ If you have multiple steps that use the same service/image (such as steps that r steps: - label: ":docker: Build" plugins: - - docker-compose#v4.1.1: + - docker-compose#v4.3.0: build: app image-repository: index.docker.io/myorg/myrepo @@ -206,7 +206,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.1.1: + - docker-compose#v4.3.0: run: app ``` @@ -222,7 +222,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v4.1.1: + - docker-compose#v4.3.0: build: - app - tests @@ -234,7 +234,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.1.1: + - docker-compose#v4.3.0: run: tests ``` @@ -246,7 +246,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v4.1.1: + - docker-compose#v4.3.0: push: app ``` @@ -256,7 +256,7 @@ To push multiple images, you can use a list: steps: - label: ":docker: Push" plugins: - - docker-compose#v4.1.1: + - docker-compose#v4.3.0: push: - first-service - second-service @@ -268,7 +268,7 @@ If you want to push to a specific location (that's not defined as the `image` in steps: - label: ":docker: Push" plugins: - - docker-compose#v4.1.1: + - docker-compose#v4.3.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -282,14 +282,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker: Build an image" plugins: - - docker-compose#v4.1.1: + - docker-compose#v4.3.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.1.1: + - docker-compose#v4.3.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -303,7 +303,7 @@ This plugin allows for the value of `cache-from` to be a string or a list. If it steps: - label: ":docker Build an image" plugins: - - docker-compose#v4.1.1: + - docker-compose#v4.3.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: @@ -312,7 +312,7 @@ steps: - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.1.1: + - docker-compose#v4.3.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:my-branch @@ -326,7 +326,7 @@ Adding a grouping tag to the end of a cache-from list item allows this plugin to steps: - label: ":docker: Build Intermediate Image" plugins: - - docker-compose#v4.1.1: + - docker-compose#v4.3.0: build: myservice_intermediate # docker-compose.yml is the same as myservice but has `target: intermediate` image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo/myservice_intermediate @@ -336,7 +336,7 @@ steps: - wait - label: ":docker: Build Final Image" plugins: - - docker-compose#v4.1.1: + - docker-compose#v4.3.0: build: myservice image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo @@ -380,7 +380,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Run & Push" plugins: - - docker-compose#v4.1.1: + - docker-compose#v4.3.0: run: myservice push: myservice ``` @@ -395,7 +395,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Build & Push" plugins: - - docker-compose#v4.1.1: + - docker-compose#v4.3.0: build: myservice push: myservice ``` From 1e79bb2e1ebb56916842666b03b1c5ce4fa337c1 Mon Sep 17 00:00:00 2001 From: Francois Campbell Date: Sun, 16 Oct 2022 18:49:04 -0700 Subject: [PATCH 130/253] Add mount-checkout support --- README.md | 10 +++++++++- commands/run.sh | 22 ++++++++++++++++++++-- plugin.yml | 2 ++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f939336b..88fe261d 100644 --- a/README.md +++ b/README.md @@ -468,7 +468,9 @@ Completely avoid running any `pull` command. Images being used will need to be p ### `workdir` (optional, run only) -Specify the container working directory via `docker-compose run --workdir`. +Specify the container working directory via `docker-compose run --workdir`. The default is `/workdir`. This path is also used by `mount-checkout` to determine where to mount the checkout in the container. + +Example: `/app` ### `user` (optional, run only) @@ -492,6 +494,12 @@ Whether to automatically mount the `buildkite-agent` binary and associated envir Default: `false` +### `mount-checkout` (optional, boolean) + +Whether to automatically mount the current working directory which contains your checked out codebase. Mounts onto `/workdir`, unless `workdir` is set, in which case that will be used. + +Default: `true` + ### `pull-retries` (optional) A number of times to retry failed docker pull. Defaults to 0. diff --git a/commands/run.sh b/commands/run.sh index b78f01a1..56a215d8 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -9,6 +9,7 @@ container_name="$(docker_compose_project_name)_${run_service}_build_${BUILDKITE_ override_file="docker-compose.buildkite-${BUILDKITE_BUILD_NUMBER}-override.yml" pull_retries="$(plugin_read_config PULL_RETRIES "0")" mount_ssh_agent='' +mount_checkout="$(plugin_read_config MOUNT_CHECKOUT "true")" expand_headers_on_error() { echo "^^^ +++" @@ -115,10 +116,16 @@ if [[ -n "${BUILDKITE_REPO_MIRROR:-}" ]]; then fi tty_default='true' +workdir_default="/workdir" +pwd_default="$PWD" # Set operating system specific defaults if is_windows ; then tty_default='false' + workdir_default="C:\\workdir" + # escaping /C is a necessary workaround for an issue with Git for Windows 2.24.1.2 + # https://github.com/git-for-windows/git/issues/2442 + pwd_default="$(cmd.exe //C "echo %CD%")" fi # Optionally disable allocating a TTY @@ -131,8 +138,19 @@ if [[ "$(plugin_read_config DEPENDENCIES "true")" == "false" ]] ; then run_params+=(--no-deps) fi -if [[ -n "$(plugin_read_config WORKDIR)" ]] ; then - run_params+=("--workdir=$(plugin_read_config WORKDIR)") +workdir='' + +if [[ -n "$(plugin_read_config WORKDIR)" ]] || [[ "${mount_checkout}" == "true" ]]; then + workdir="$(plugin_read_config WORKDIR "$workdir_default")" +fi + +if [[ -n "${workdir}" ]] ; then + run_params+=("--workdir=${workdir}") +fi + +# By default, mount $PWD onto $WORKDIR +if [[ "${mount_checkout}" == "true" ]] ; then + run_params+=("-v" "${pwd_default}:${workdir}") fi # Can't set both user and propagate-uid-gid diff --git a/plugin.yml b/plugin.yml index c2136ecb..8f855c55 100644 --- a/plugin.yml +++ b/plugin.yml @@ -59,6 +59,8 @@ configuration: type: boolean mount-ssh-agent: type: boolean + mount-checkout: + type: boolean no-cache: type: boolean propagate-environment: From a2dc3f60600c93482da37942d2dce97d815c2e38 Mon Sep 17 00:00:00 2001 From: Francois Campbell Date: Mon, 17 Oct 2022 13:41:26 -0700 Subject: [PATCH 131/253] PR comments, dont make it on by default --- README.md | 4 ++-- commands/run.sh | 2 +- plugin.yml | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 88fe261d..afaab7d5 100644 --- a/README.md +++ b/README.md @@ -494,11 +494,11 @@ Whether to automatically mount the `buildkite-agent` binary and associated envir Default: `false` -### `mount-checkout` (optional, boolean) +### `mount-checkout` (optional, run-only, boolean) Whether to automatically mount the current working directory which contains your checked out codebase. Mounts onto `/workdir`, unless `workdir` is set, in which case that will be used. -Default: `true` +Default: `false` ### `pull-retries` (optional) diff --git a/commands/run.sh b/commands/run.sh index 56a215d8..5784ed47 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -9,7 +9,7 @@ container_name="$(docker_compose_project_name)_${run_service}_build_${BUILDKITE_ override_file="docker-compose.buildkite-${BUILDKITE_BUILD_NUMBER}-override.yml" pull_retries="$(plugin_read_config PULL_RETRIES "0")" mount_ssh_agent='' -mount_checkout="$(plugin_read_config MOUNT_CHECKOUT "true")" +mount_checkout="$(plugin_read_config MOUNT_CHECKOUT "false")" expand_headers_on_error() { echo "^^^ +++" diff --git a/plugin.yml b/plugin.yml index 8f855c55..b511865f 100644 --- a/plugin.yml +++ b/plugin.yml @@ -117,6 +117,7 @@ configuration: image-name: [ build ] leave-volumes: [ run ] mount-buildkite-agent: [ run ] + mount-checkout: [ run ] propagate-uid-gid: [ run ] pull: [ run ] push-retries: [ push ] From d58a85c164c7ee014283ba27646fa1a2f3df46ed Mon Sep 17 00:00:00 2001 From: Francois Campbell Date: Wed, 19 Oct 2022 09:08:50 -0700 Subject: [PATCH 132/253] Add tests --- tests/run.bats | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/tests/run.bats b/tests/run.bats index 2b4ef823..9e9aea9f 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -1159,3 +1159,84 @@ export BUILDKITE_JOB_ID=1111 unstub docker-compose unstub buildkite-agent } + +@test "mount-checkout is not enabled by default" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice without mount-checkout" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "ran myservice without mount-checkout" + unstub docker-compose + unstub buildkite-agent +} + +@test "Run with mount-checkout set with default workdir" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_MOUNT_CHECKOUT=true + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --workdir=/workdir -v $PWD:/workdir --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with mount-checkout" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "ran myservice with mount-checkout" + unstub docker-compose + unstub buildkite-agent +} + +@test "Run with mount-checkout set with custom workdir" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_MOUNT_CHECKOUT=true + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_WORKDIR="/custom_workdir" + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --workdir=$BUILDKITE_PLUGIN_DOCKER_COMPOSE_WORKDIR -v $PWD:$BUILDKITE_PLUGIN_DOCKER_COMPOSE_WORKDIR --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with mount-checkout" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "ran myservice with mount-checkout" + unstub docker-compose + unstub buildkite-agent +} From 5836d15a38b20cfb33a1a9dce319722af7944d19 Mon Sep 17 00:00:00 2001 From: Francois Campbell Date: Wed, 19 Oct 2022 15:32:19 -0700 Subject: [PATCH 133/253] edit test to set explicit false --- tests/run.bats | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/run.bats b/tests/run.bats index 9e9aea9f..3f8e2e53 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -1160,7 +1160,7 @@ export BUILDKITE_JOB_ID=1111 unstub buildkite-agent } -@test "mount-checkout is not enabled by default" { +@test "Run without mount-checkout doesn't set volume" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice export BUILDKITE_PIPELINE_SLUG=test @@ -1168,6 +1168,7 @@ export BUILDKITE_JOB_ID=1111 export BUILDKITE_COMMAND=pwd export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_MOUNT_CHECKOUT=false stub docker-compose \ "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ From c70ff6f5d03c2996bf3fa36e4772d118c9b28925 Mon Sep 17 00:00:00 2001 From: Tomasz Gieniusz Date: Thu, 20 Oct 2022 12:35:01 +1100 Subject: [PATCH 134/253] Fix push without prebuild when using docker-compose v2 --- lib/push.bash | 7 ++++++- tests/v2/push.bats | 28 ++++++++++++++-------------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/lib/push.bash b/lib/push.bash index 7714352f..d69ee8fa 100644 --- a/lib/push.bash +++ b/lib/push.bash @@ -20,8 +20,13 @@ compose_image_for_service() { default_compose_image_for_service() { local service="$1" + + local separator="_" + if [[ "$(plugin_read_config CLI_VERSION "1")" == "2" ]] ; then + separator="-" + fi - printf '%s_%s\n' "$(docker_compose_project_name)" "$service" + printf '%s%s%s\n' "$(docker_compose_project_name)" "$separator" "$service" } docker_image_exists() { diff --git a/tests/v2/push.bats b/tests/v2/push.bats index 9feb4f95..d0ef5ae1 100644 --- a/tests/v2/push.bats +++ b/tests/v2/push.bats @@ -44,14 +44,14 @@ setup_file() { stub docker \ "compose -f docker-compose.yml -p buildkite1111 config : echo blah " \ - "image inspect buildkite1111_myservice1 : exit 1" \ + "image inspect buildkite1111-myservice1 : exit 1" \ "compose -f docker-compose.yml -p buildkite1111 build myservice1 : echo blah " \ - "tag buildkite1111_myservice1 my.repository/myservice1 : echo tagging image1" \ + "tag buildkite1111-myservice1 my.repository/myservice1 : echo tagging image1" \ "push my.repository/myservice1 : echo pushing myservice1 image" \ "compose -f docker-compose.yml -p buildkite1111 config : echo blah " \ - "image inspect buildkite1111_myservice2 : exit 1" \ + "image inspect buildkite1111-myservice2 : exit 1" \ "compose -f docker-compose.yml -p buildkite1111 build myservice2 : echo blah " \ - "tag buildkite1111_myservice2 my.repository/myservice2:llamas : echo tagging image2" \ + "tag buildkite1111-myservice2 my.repository/myservice2:llamas : echo tagging image2" \ "push my.repository/myservice2:llamas : echo pushing myservice2 image" stub buildkite-agent \ @@ -78,8 +78,8 @@ setup_file() { stub docker \ "compose -f docker-compose.yml -p buildkite1111 config : echo blah" \ "pull myimage : echo pulled prebuilt image" \ - "tag myimage buildkite1111_myservice : echo " \ - "tag buildkite1111_myservice my.repository/myservice:llamas : echo tagged image" \ + "tag myimage buildkite1111-myservice : echo " \ + "tag buildkite1111-myservice my.repository/myservice:llamas : echo tagged image" \ "push my.repository/myservice:llamas : echo pushed myservice" stub buildkite-agent \ @@ -125,16 +125,16 @@ setup_file() { stub docker \ "compose -f docker-compose.yml -p buildkite1111 config : echo blah" \ "pull prebuilt : echo pulled prebuilt image" \ - "tag prebuilt buildkite1111_myservice : echo " \ - "tag buildkite1111_myservice my.repository/myservice:llamas : echo tagged image1" \ + "tag prebuilt buildkite1111-myservice : echo " \ + "tag buildkite1111-myservice my.repository/myservice:llamas : echo tagged image1" \ "push my.repository/myservice:llamas : echo pushed myservice1" \ "compose -f docker-compose.yml -p buildkite1111 config : echo blah" \ - "tag prebuilt buildkite1111_myservice : echo " \ - "tag buildkite1111_myservice my.repository/myservice:latest : echo tagged image2" \ + "tag prebuilt buildkite1111-myservice : echo " \ + "tag buildkite1111-myservice my.repository/myservice:latest : echo tagged image2" \ "push my.repository/myservice:latest : echo pushed myservice2" \ "compose -f docker-compose.yml -p buildkite1111 config : echo blah" \ - "tag prebuilt buildkite1111_myservice : echo " \ - "tag buildkite1111_myservice my.repository/myservice:alpacas : echo tagged image3" \ + "tag prebuilt buildkite1111-myservice : echo " \ + "tag buildkite1111-myservice my.repository/myservice:alpacas : echo tagged image3" \ "push my.repository/myservice:alpacas : echo pushed myservice3" stub buildkite-agent \ @@ -170,9 +170,9 @@ setup_file() { stub docker \ "compose -f docker-compose.yml -p buildkite1111 config : cat $PWD/tests/composefiles/docker-compose.config.v3.2.yml" \ - "image inspect buildkite1111_helper : exit 1" \ + "image inspect buildkite1111-helper : exit 1" \ "compose -f docker-compose.yml -p buildkite1111 build helper : echo built helper" \ - "tag buildkite1111_helper my.repository/helper:llamas : echo tagged helper" \ + "tag buildkite1111-helper my.repository/helper:llamas : echo tagged helper" \ "push my.repository/helper:llamas : echo pushed helper" run $PWD/hooks/command From d969fd86ce41bd41cf385aafbc4bdb15e5072c07 Mon Sep 17 00:00:00 2001 From: Francois Campbell Date: Wed, 19 Oct 2022 21:49:24 -0700 Subject: [PATCH 135/253] Update README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matías Bellone --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index afaab7d5..4b5054ea 100644 --- a/README.md +++ b/README.md @@ -468,7 +468,7 @@ Completely avoid running any `pull` command. Images being used will need to be p ### `workdir` (optional, run only) -Specify the container working directory via `docker-compose run --workdir`. The default is `/workdir`. This path is also used by `mount-checkout` to determine where to mount the checkout in the container. +Specify the container working directory via `docker-compose run --workdir`. This option is also used by [`mount-checkout`](#mount-checkout-optional-run-only-boolean) to determine where to mount the checkout in the container. Example: `/app` From 3e59a2d2ce08e68deb4ec5e43cd523e9873ece3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 22 Oct 2022 13:30:50 -0300 Subject: [PATCH 136/253] Allow mount-checkout be a path --- commands/run.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/commands/run.sh b/commands/run.sh index 5784ed47..2c705936 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -10,6 +10,7 @@ override_file="docker-compose.buildkite-${BUILDKITE_BUILD_NUMBER}-override.yml" pull_retries="$(plugin_read_config PULL_RETRIES "0")" mount_ssh_agent='' mount_checkout="$(plugin_read_config MOUNT_CHECKOUT "false")" +workdir='' expand_headers_on_error() { echo "^^^ +++" @@ -138,20 +139,23 @@ if [[ "$(plugin_read_config DEPENDENCIES "true")" == "false" ]] ; then run_params+=(--no-deps) fi -workdir='' - if [[ -n "$(plugin_read_config WORKDIR)" ]] || [[ "${mount_checkout}" == "true" ]]; then workdir="$(plugin_read_config WORKDIR "$workdir_default")" fi +if [[ "${mount_checkout}" == "true" ]]; then + run_params+=("-v" "${pwd_default}:${workdir}") +elif [[ "${mount_checkout}" ~= ^/.*$ ]]; then + run_params+=("-v" "${pwd_default}:${mount_checkout}") +elif [[ "${mount_checkout}" != "false" ]]; then + echo -n "🚨 mount-checkout should be either true or an absolute path to use as a mountpoint" + exit 1 +fi + if [[ -n "${workdir}" ]] ; then run_params+=("--workdir=${workdir}") fi -# By default, mount $PWD onto $WORKDIR -if [[ "${mount_checkout}" == "true" ]] ; then - run_params+=("-v" "${pwd_default}:${workdir}") -fi # Can't set both user and propagate-uid-gid if [[ -n "$(plugin_read_config USER)" ]] && [[ -n "$(plugin_read_config PROPAGATE_UID_GID)" ]]; then From 331dea1ceedb4147484d71c2537a482b2306e0b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 22 Oct 2022 13:33:02 -0300 Subject: [PATCH 137/253] Updated documentation --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4b5054ea..ce4a55a6 100644 --- a/README.md +++ b/README.md @@ -468,7 +468,7 @@ Completely avoid running any `pull` command. Images being used will need to be p ### `workdir` (optional, run only) -Specify the container working directory via `docker-compose run --workdir`. This option is also used by [`mount-checkout`](#mount-checkout-optional-run-only-boolean) to determine where to mount the checkout in the container. +Specify the container working directory via `docker-compose run --workdir`. This option is also used by [`mount-checkout`](#mount-checkout-optional-run-only-boolean) if it doesn't specify where to mount the checkout in the container. Example: `/app` @@ -494,9 +494,11 @@ Whether to automatically mount the `buildkite-agent` binary and associated envir Default: `false` -### `mount-checkout` (optional, run-only, boolean) +### `mount-checkout` (optional, run-only, string or boolean) -Whether to automatically mount the current working directory which contains your checked out codebase. Mounts onto `/workdir`, unless `workdir` is set, in which case that will be used. +The absolute path where to mount the current working directory which contains your checked out codebase. + +If set to `true` it will mount onto `/workdir`, unless `workdir` is set, in which case that will be used. Default: `false` From a43b60654a54ca112526d23e3b46cd5d88c09a91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 22 Oct 2022 13:34:20 -0300 Subject: [PATCH 138/253] Corrected regex operator in conditional --- commands/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/run.sh b/commands/run.sh index 2c705936..4e56ad13 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -145,7 +145,7 @@ fi if [[ "${mount_checkout}" == "true" ]]; then run_params+=("-v" "${pwd_default}:${workdir}") -elif [[ "${mount_checkout}" ~= ^/.*$ ]]; then +elif [[ "${mount_checkout}" =~ ^/.*$ ]]; then run_params+=("-v" "${pwd_default}:${mount_checkout}") elif [[ "${mount_checkout}" != "false" ]]; then echo -n "🚨 mount-checkout should be either true or an absolute path to use as a mountpoint" From 128b85f40649e30c5217d7b672abd2e7c97123be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 22 Oct 2022 13:42:17 -0300 Subject: [PATCH 139/253] Moved workdir back to what it was to avoid altering option order --- commands/run.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/commands/run.sh b/commands/run.sh index 4e56ad13..cc0f36c0 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -143,6 +143,10 @@ if [[ -n "$(plugin_read_config WORKDIR)" ]] || [[ "${mount_checkout}" == "true" workdir="$(plugin_read_config WORKDIR "$workdir_default")" fi +if [[ -n "${workdir}" ]] ; then + run_params+=("--workdir=${workdir}") +fi + if [[ "${mount_checkout}" == "true" ]]; then run_params+=("-v" "${pwd_default}:${workdir}") elif [[ "${mount_checkout}" =~ ^/.*$ ]]; then @@ -152,11 +156,6 @@ elif [[ "${mount_checkout}" != "false" ]]; then exit 1 fi -if [[ -n "${workdir}" ]] ; then - run_params+=("--workdir=${workdir}") -fi - - # Can't set both user and propagate-uid-gid if [[ -n "$(plugin_read_config USER)" ]] && [[ -n "$(plugin_read_config PROPAGATE_UID_GID)" ]]; then echo "+++ Error: Can't set both user and propagate-uid-gid" From ad6a39fc35d6e9f206ab5a10d8f28cec734e4e6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 22 Oct 2022 13:51:57 -0300 Subject: [PATCH 140/253] Updated tests and added new ones --- tests/run.bats | 119 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 117 insertions(+), 2 deletions(-) diff --git a/tests/run.bats b/tests/run.bats index 3f8e2e53..4f4918c8 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -1187,7 +1187,7 @@ export BUILDKITE_JOB_ID=1111 unstub buildkite-agent } -@test "Run with mount-checkout set with default workdir" { +@test "Run with mount-checkout set to true" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice export BUILDKITE_PIPELINE_SLUG=test @@ -1214,7 +1214,7 @@ export BUILDKITE_JOB_ID=1111 unstub buildkite-agent } -@test "Run with mount-checkout set with custom workdir" { +@test "Run with mount-checkout set to true with custom workdir" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice export BUILDKITE_PIPELINE_SLUG=test @@ -1241,3 +1241,118 @@ export BUILDKITE_JOB_ID=1111 unstub docker-compose unstub buildkite-agent } + +@test "Run with mount-checkout set to specific path" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_COMMAND=pwd + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_MOUNT_CHECKOUT="/special" + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f \* pull myservice : echo pulled myservice" \ + "-f docker-compose.yml -p buildkite1111 -f \* up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "-f docker-compose.yml -p buildkite1111 -f \* run --name buildkite1111_myservice_build_1 -v \* --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with mount-checkout on \${11}" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "ran myservice with mount-checkout on /plugin:/special" + + unstub docker-compose + unstub buildkite-agent +} + +@test "Run with mount-checkout set to specific path and workdir set" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_COMMAND=pwd + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_MOUNT_CHECKOUT="/special" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_WORKDIR="/custom_workdir" + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f \* pull myservice : echo pulled myservice" \ + "-f docker-compose.yml -p buildkite1111 -f \* up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "-f docker-compose.yml -p buildkite1111 -f \* run --name buildkite1111_myservice_build_1 \* -v \* --rm myservice /bin/sh -e -c 'pwd' : echo echo ran myservice with mount-checkout on \${12}" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "ran myservice with mount-checkout on /plugin:/special" + assert_output --partial "--workdir=/custom_workdir" + + unstub docker-compose + unstub buildkite-agent +} + +@test "Run with mount-checkout set something else" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_COMMAND=pwd + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_MOUNT_CHECKOUT="not_absolute" + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f \* pull myservice : echo pulled myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run "$PWD"/hooks/command + + assert_failure + assert_output --partial "mount-checkout should be either true or an absolute path to use as a mountpoint" + + unstub docker-compose + unstub buildkite-agent +} + +@test "Run with mount-checkout set something else and workdir set" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_COMMAND=pwd + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_MOUNT_CHECKOUT="not-absolute" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_WORKDIR="/custom_workdir" + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f \* pull myservice : echo pulled myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run "$PWD"/hooks/command + + assert_failure + assert_output --partial "mount-checkout should be either true or an absolute path to use as a mountpoint" + + unstub docker-compose + unstub buildkite-agent +} From 0b45fd4c332e3db5d4c78b71431945b2b6ec1989 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 22 Oct 2022 13:57:09 -0300 Subject: [PATCH 141/253] Corrected shellcheck warnings --- tests/v2/run.bats | 72 +++++++++++++++++++++++------------------------ 1 file changed, 35 insertions(+), 37 deletions(-) diff --git a/tests/v2/run.bats b/tests/v2/run.bats index 9de0ff8d..eabbefa2 100644 --- a/tests/v2/run.bats +++ b/tests/v2/run.bats @@ -29,7 +29,7 @@ setup_file() { stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -55,7 +55,7 @@ setup_file() { stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -82,7 +82,7 @@ setup_file() { stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -108,7 +108,7 @@ setup_file() { stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -136,7 +136,7 @@ cmd3" stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -164,7 +164,7 @@ cmd3" stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -195,7 +195,7 @@ cmd3" stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice" @@ -221,7 +221,7 @@ cmd3" stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -249,7 +249,7 @@ cmd3" stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -336,7 +336,7 @@ cmd3" "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice" @@ -363,7 +363,7 @@ cmd3" "meta-data exists docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v2.0.yml : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v2.0.yml : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice" @@ -391,7 +391,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v2.0.yml-tests/composefiles/docker-compose.v2.1.yml : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v2.0.yml-tests/composefiles/docker-compose.v2.1.yml : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice" @@ -418,7 +418,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v2.0.yml : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice-tests/composefiles/docker-compose.v2.0.yml : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice" @@ -442,7 +442,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_failure assert_output --partial "Exited with 2" @@ -470,7 +470,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "pulled myservice" @@ -498,7 +498,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice without tty" @@ -524,7 +524,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice without dependencies" @@ -551,7 +551,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice without ansi output" @@ -578,7 +578,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice with use aliases output" @@ -606,7 +606,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice with volumes" @@ -633,7 +633,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice with volumes" @@ -663,7 +663,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice with volumes" @@ -690,7 +690,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice with volumes" @@ -718,7 +718,7 @@ export BUILDKITE_JOB_ID=1111 stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice-llamas1.yml-llamas2.yml-llamas3.yml : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -744,7 +744,7 @@ export BUILDKITE_JOB_ID=1111 stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_failure assert_output --partial "^^^ +++" @@ -775,7 +775,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice2 : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice2 : echo myimage2" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "pulled myservice1 and myservice2" @@ -802,7 +802,7 @@ export BUILDKITE_JOB_ID=1111 stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -829,7 +829,7 @@ export BUILDKITE_JOB_ID=1111 stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -852,7 +852,7 @@ export BUILDKITE_JOB_ID=1111 stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_failure assert_output --partial "Error" @@ -880,7 +880,7 @@ export BUILDKITE_JOB_ID=1111 "meta-data exists docker-compose-plugin-built-image-tag-myservice : echo myimage" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "ran myservice without tty" @@ -898,8 +898,6 @@ export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false export BUILDKITE_PLUGIN_DOCKER_COMPOSE_ENTRYPOINT="my custom entrypoint" - ENTRYPOINT='--entrypoint\ \"my\ custom\ entrypoint\"' - stub docker \ "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ @@ -908,7 +906,7 @@ export BUILDKITE_JOB_ID=1111 stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -935,7 +933,7 @@ export BUILDKITE_JOB_ID=1111 stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -963,7 +961,7 @@ export BUILDKITE_JOB_ID=1111 stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -990,7 +988,7 @@ export BUILDKITE_JOB_ID=1111 stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -1021,7 +1019,7 @@ export BUILDKITE_JOB_ID=1111 apk add netcat-openbsd nc -lkvU $SSH_AUTH_SOCK & - run $PWD/hooks/command + run "$PWD"/hooks/command kill %1 From a78983357d4225c14caabd96a2cda98709dcf98c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 22 Oct 2022 14:04:19 -0300 Subject: [PATCH 142/253] Added mount-checkout options to v2 --- tests/v2/run.bats | 203 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 203 insertions(+) diff --git a/tests/v2/run.bats b/tests/v2/run.bats index eabbefa2..46abd364 100644 --- a/tests/v2/run.bats +++ b/tests/v2/run.bats @@ -1029,3 +1029,206 @@ export BUILDKITE_JOB_ID=1111 unstub docker unstub buildkite-agent } + +@test "Run without mount-checkout doesn't set volume" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_COMMAND=pwd + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_MOUNT_CHECKOUT=false + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f \* pull myservice : echo pulled myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f \* up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f \* run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice without mount-checkout" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "ran myservice without mount-checkout" + + unstub docker + unstub buildkite-agent +} + +@test "Run with mount-checkout set to true" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_COMMAND=pwd + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_MOUNT_CHECKOUT=true + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f \* pull myservice : echo pulled myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f \* up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f \* run --name buildkite1111_myservice_build_1 --workdir=/workdir -v /plugin:/workdir --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with mount-checkout" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "ran myservice with mount-checkout" + + unstub docker + unstub buildkite-agent +} + +@test "Run with mount-checkout set to true with custom workdir" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_COMMAND=pwd + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_MOUNT_CHECKOUT=true + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_WORKDIR="/custom_workdir" + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f \* pull myservice : echo pulled myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f \* up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f \* run --name buildkite1111_myservice_build_1 \* -v \* --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with mount-checkout on \${13}" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "ran myservice with mount-checkout on /plugin:/custom_workdir" + + unstub docker + unstub buildkite-agent +} + +@test "Run with mount-checkout set to specific path" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_COMMAND=pwd + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_MOUNT_CHECKOUT="/special" + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f \* pull myservice : echo pulled myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f \* up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f \* run --name buildkite1111_myservice_build_1 -v \* --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with mount-checkout on \${12}" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "ran myservice with mount-checkout on /plugin:/special" + + unstub docker + unstub buildkite-agent +} + +@test "Run with mount-checkout set to specific path and workdir set" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_COMMAND=pwd + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_MOUNT_CHECKOUT="/special" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_WORKDIR="/custom_workdir" + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f \* pull myservice : echo pulled myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f \* up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f \* run --name buildkite1111_myservice_build_1 \* -v \* --rm myservice /bin/sh -e -c 'pwd' : echo echo ran myservice with mount-checkout on \${13}" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "ran myservice with mount-checkout on /plugin:/special" + assert_output --partial "--workdir=/custom_workdir" + + unstub docker + unstub buildkite-agent +} + +@test "Run with mount-checkout set something else" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_COMMAND=pwd + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_MOUNT_CHECKOUT="not_absolute" + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f \* pull myservice : echo pulled myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run "$PWD"/hooks/command + + assert_failure + assert_output --partial "mount-checkout should be either true or an absolute path to use as a mountpoint" + + unstub docker + unstub buildkite-agent +} + +@test "Run with mount-checkout set something else and workdir set" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_COMMAND=pwd + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_MOUNT_CHECKOUT="not-absolute" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_WORKDIR="/custom_workdir" + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f \* pull myservice : echo pulled myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run "$PWD"/hooks/command + + assert_failure + assert_output --partial "mount-checkout should be either true or an absolute path to use as a mountpoint" + + unstub docker + unstub buildkite-agent +} From 922db41964329e34bab09ee2b12e0f22b5342780 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 22 Oct 2022 14:05:03 -0300 Subject: [PATCH 143/253] Corrected definition of option in plugin.yml --- plugin.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin.yml b/plugin.yml index b511865f..71025495 100644 --- a/plugin.yml +++ b/plugin.yml @@ -60,7 +60,7 @@ configuration: mount-ssh-agent: type: boolean mount-checkout: - type: boolean + type: [ boolean, string ] no-cache: type: boolean propagate-environment: From b647262748403975f4800201a21473592edef228 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 22 Oct 2022 14:06:35 -0300 Subject: [PATCH 144/253] Updated version to next upcoming one --- README.md | 54 +++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index ce4a55a6..48b6d5e0 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v4.3.0: + - docker-compose#v4.5.0: run: app ``` @@ -28,7 +28,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v4.3.0: + - docker-compose#v4.5.0: run: app config: docker-compose.tests.yml env: @@ -41,7 +41,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v4.3.0: + - docker-compose#v4.5.0: run: app config: - docker-compose.yml @@ -56,7 +56,7 @@ env: steps: - command: test.sh plugins: - - docker-compose#v4.3.0: + - docker-compose#v4.5.0: run: app ``` @@ -65,7 +65,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v4.3.0: + - docker-compose#v4.5.0: run: app command: ["custom", "command", "values"] ``` @@ -79,7 +79,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.3.0: + - docker-compose#v4.5.0: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -87,7 +87,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.3.0: + - docker-compose#v4.5.0: run: app ``` @@ -104,7 +104,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.3.0: + - docker-compose#v4.5.0: run: app ``` @@ -122,7 +122,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.3.0: + - docker-compose#v4.5.0: run: app volumes: - "./dist:/app/dist" @@ -146,7 +146,7 @@ this plugin offers a `environment` block of its own: steps: - command: generate-dist.sh plugins: - - docker-compose#v4.3.0: + - docker-compose#v4.5.0: run: app env: - BUILDKITE_BUILD_NUMBER @@ -164,7 +164,7 @@ Alternatively, you can have the plugin add all environment variables defined for steps: - command: use-vars.sh plugins: - - docker-compose#v4.3.0: + - docker-compose#v4.5.0: run: app propagate-environment: true ``` @@ -179,7 +179,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v4.3.0: + - docker-compose#v4.5.0: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -196,7 +196,7 @@ If you have multiple steps that use the same service/image (such as steps that r steps: - label: ":docker: Build" plugins: - - docker-compose#v4.3.0: + - docker-compose#v4.5.0: build: app image-repository: index.docker.io/myorg/myrepo @@ -206,7 +206,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.3.0: + - docker-compose#v4.5.0: run: app ``` @@ -222,7 +222,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v4.3.0: + - docker-compose#v4.5.0: build: - app - tests @@ -234,7 +234,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.3.0: + - docker-compose#v4.5.0: run: tests ``` @@ -246,7 +246,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v4.3.0: + - docker-compose#v4.5.0: push: app ``` @@ -256,7 +256,7 @@ To push multiple images, you can use a list: steps: - label: ":docker: Push" plugins: - - docker-compose#v4.3.0: + - docker-compose#v4.5.0: push: - first-service - second-service @@ -268,7 +268,7 @@ If you want to push to a specific location (that's not defined as the `image` in steps: - label: ":docker: Push" plugins: - - docker-compose#v4.3.0: + - docker-compose#v4.5.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -282,14 +282,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker: Build an image" plugins: - - docker-compose#v4.3.0: + - docker-compose#v4.5.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.3.0: + - docker-compose#v4.5.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -303,7 +303,7 @@ This plugin allows for the value of `cache-from` to be a string or a list. If it steps: - label: ":docker Build an image" plugins: - - docker-compose#v4.3.0: + - docker-compose#v4.5.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: @@ -312,7 +312,7 @@ steps: - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.3.0: + - docker-compose#v4.5.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:my-branch @@ -326,7 +326,7 @@ Adding a grouping tag to the end of a cache-from list item allows this plugin to steps: - label: ":docker: Build Intermediate Image" plugins: - - docker-compose#v4.3.0: + - docker-compose#v4.5.0: build: myservice_intermediate # docker-compose.yml is the same as myservice but has `target: intermediate` image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo/myservice_intermediate @@ -336,7 +336,7 @@ steps: - wait - label: ":docker: Build Final Image" plugins: - - docker-compose#v4.3.0: + - docker-compose#v4.5.0: build: myservice image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo @@ -380,7 +380,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Run & Push" plugins: - - docker-compose#v4.3.0: + - docker-compose#v4.5.0: run: myservice push: myservice ``` @@ -395,7 +395,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Build & Push" plugins: - - docker-compose#v4.3.0: + - docker-compose#v4.5.0: build: myservice push: myservice ``` From a7c75647d735fc8d0c9d8deb02845df1cee412b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 12 Nov 2022 00:43:46 -0300 Subject: [PATCH 145/253] Add new secrets option to plugin spec --- plugin.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugin.yml b/plugin.yml index 71025495..d08e1c46 100644 --- a/plugin.yml +++ b/plugin.yml @@ -82,6 +82,10 @@ configuration: type: boolean ssh: type: boolean + secrets: + type: array + items: + type: string target: type: string tty: @@ -122,6 +126,7 @@ configuration: pull: [ run ] push-retries: [ push ] skip-pull: [ run ] + secrets: [ buildkit, build ] ssh: [ buildkit ] target: [ build ] tty: [ run ] From 2f9af14c10deedbce69581343875c9ba370b3db4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 12 Nov 2022 00:49:40 -0300 Subject: [PATCH 146/253] Add --secrets parameters on build --- commands/build.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/commands/build.sh b/commands/build.sh index 640d39e4..ddd6b04d 100755 --- a/commands/build.sh +++ b/commands/build.sh @@ -157,6 +157,11 @@ if [[ "$(plugin_read_config BUILD_PARALLEL "false")" == "true" ]] ; then build_params+=(--parallel) fi +# Parse the list of secrets to pass on to build command +while read -r line ; do + [[ -n "$line" ]] && build_params+=("--secret" "$line") +done <<< "$(plugin_read_list SECRETS)" + if [[ "$(plugin_read_config SSH "false")" == "true" ]] ; then if [[ "${DOCKER_BUILDKIT:-}" != "1" && "${BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLI_VERSION:-}" != "2" ]]; then echo "🚨 You can not use the ssh option if you are not using buildkit" From dd9f5e89dce3ecbc4a3da7be4bada9f64c2e291c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 12 Nov 2022 00:56:08 -0300 Subject: [PATCH 147/253] Added tests for new option --- tests/build.bats | 21 +++++++++++++++++++++ tests/v2/build.bats | 22 ++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/tests/build.bats b/tests/build.bats index 5edd4f5d..0cb6a2cb 100644 --- a/tests/build.bats +++ b/tests/build.bats @@ -793,5 +793,26 @@ load '../lib/shared' assert_output --partial "built myservice" assert_output --partial "with ssh" + unstub docker-compose +} + +@test "Build with secrets" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_SECRETS_0='id=test,file=~/.test' + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_SECRETS_1='id=SECRET_VAR' + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --secret \* --secret \* \* : echo built \${13} with secrets \${10} and \${12}" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "with secrets id=test,file=~/.test and id=SECRET_VAR" + unstub docker-compose } \ No newline at end of file diff --git a/tests/v2/build.bats b/tests/v2/build.bats index e45de20a..2a1d2255 100644 --- a/tests/v2/build.bats +++ b/tests/v2/build.bats @@ -627,5 +627,27 @@ setup_file() { assert_output --partial "built myservice" assert_output --partial "with ssh" + unstub docker +} + + +@test "Build with secrets" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_SECRETS_0='id=test,file=~/.test' + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_SECRETS_1='id=SECRET_VAR' + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --secret \* --secret \* \* : echo built \${14} with secrets \${11} and \${13}" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "with secrets id=test,file=~/.test and id=SECRET_VAR" + unstub docker } \ No newline at end of file From c309f541d39b5e4ab38c6efa766cbaf914f8b958 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 12 Nov 2022 00:56:22 -0300 Subject: [PATCH 148/253] Added documentation for new option (and corrected ordering --- README.md | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 48b6d5e0..eb5dbbe1 100644 --- a/README.md +++ b/README.md @@ -614,14 +614,6 @@ The default is `on-error`. If set to `2`, plugin will use `docker compose` to execute commands; otherwise it will default to version `1` using `docker-compose` instead. -## Developing - -To run the tests: - -```bash -docker-compose run --rm tests bats tests tests/v2 -``` - ### `buildkit` (optional, build only, boolean) Assuming you have a compatible docker installation and configuration in the agent, activating this option would setup the environment for the `docker-compose build` call to use BuildKit. Note that if you are using `cli-version` 2, you are already using buildkit by default. @@ -632,6 +624,18 @@ You may want to also add `BUILDKIT_INLINE_CACHE=1` to your build arguments (`arg When enabled, it will add the `--ssh` option to the build command. Note that it assumes you have a compatible docker installation and configuration in the agent (meaning you are using BuildKit and it is correctly setup). +### `secrets` (optional, build only, array of strings) + +All elements in this array will be passed literally to the `build` command as parameters of the [`--secrets` option](https://docs.docker.com/engine/reference/commandline/buildx_build/#secret). Note that you must have BuildKit enabled for this option to have any effect and special `RUN` stanzas in your Dockerfile to actually make use of them. + +## Developing + +To run the tests: + +```bash +docker-compose run --rm tests bats tests tests/v2 +``` + ## License MIT (see [LICENSE](LICENSE)) From 57a4b0110079dcc980658864513cc18a2cb37ee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 12 Nov 2022 00:57:29 -0300 Subject: [PATCH 149/253] Updated version for upcoming release --- README.md | 54 +++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index eb5dbbe1..f7793529 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v4.5.0: + - docker-compose#v4.6.0: run: app ``` @@ -28,7 +28,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v4.5.0: + - docker-compose#v4.6.0: run: app config: docker-compose.tests.yml env: @@ -41,7 +41,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v4.5.0: + - docker-compose#v4.6.0: run: app config: - docker-compose.yml @@ -56,7 +56,7 @@ env: steps: - command: test.sh plugins: - - docker-compose#v4.5.0: + - docker-compose#v4.6.0: run: app ``` @@ -65,7 +65,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v4.5.0: + - docker-compose#v4.6.0: run: app command: ["custom", "command", "values"] ``` @@ -79,7 +79,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.5.0: + - docker-compose#v4.6.0: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -87,7 +87,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.5.0: + - docker-compose#v4.6.0: run: app ``` @@ -104,7 +104,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.5.0: + - docker-compose#v4.6.0: run: app ``` @@ -122,7 +122,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.5.0: + - docker-compose#v4.6.0: run: app volumes: - "./dist:/app/dist" @@ -146,7 +146,7 @@ this plugin offers a `environment` block of its own: steps: - command: generate-dist.sh plugins: - - docker-compose#v4.5.0: + - docker-compose#v4.6.0: run: app env: - BUILDKITE_BUILD_NUMBER @@ -164,7 +164,7 @@ Alternatively, you can have the plugin add all environment variables defined for steps: - command: use-vars.sh plugins: - - docker-compose#v4.5.0: + - docker-compose#v4.6.0: run: app propagate-environment: true ``` @@ -179,7 +179,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v4.5.0: + - docker-compose#v4.6.0: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -196,7 +196,7 @@ If you have multiple steps that use the same service/image (such as steps that r steps: - label: ":docker: Build" plugins: - - docker-compose#v4.5.0: + - docker-compose#v4.6.0: build: app image-repository: index.docker.io/myorg/myrepo @@ -206,7 +206,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.5.0: + - docker-compose#v4.6.0: run: app ``` @@ -222,7 +222,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v4.5.0: + - docker-compose#v4.6.0: build: - app - tests @@ -234,7 +234,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.5.0: + - docker-compose#v4.6.0: run: tests ``` @@ -246,7 +246,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v4.5.0: + - docker-compose#v4.6.0: push: app ``` @@ -256,7 +256,7 @@ To push multiple images, you can use a list: steps: - label: ":docker: Push" plugins: - - docker-compose#v4.5.0: + - docker-compose#v4.6.0: push: - first-service - second-service @@ -268,7 +268,7 @@ If you want to push to a specific location (that's not defined as the `image` in steps: - label: ":docker: Push" plugins: - - docker-compose#v4.5.0: + - docker-compose#v4.6.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -282,14 +282,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker: Build an image" plugins: - - docker-compose#v4.5.0: + - docker-compose#v4.6.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.5.0: + - docker-compose#v4.6.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -303,7 +303,7 @@ This plugin allows for the value of `cache-from` to be a string or a list. If it steps: - label: ":docker Build an image" plugins: - - docker-compose#v4.5.0: + - docker-compose#v4.6.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: @@ -312,7 +312,7 @@ steps: - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.5.0: + - docker-compose#v4.6.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:my-branch @@ -326,7 +326,7 @@ Adding a grouping tag to the end of a cache-from list item allows this plugin to steps: - label: ":docker: Build Intermediate Image" plugins: - - docker-compose#v4.5.0: + - docker-compose#v4.6.0: build: myservice_intermediate # docker-compose.yml is the same as myservice but has `target: intermediate` image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo/myservice_intermediate @@ -336,7 +336,7 @@ steps: - wait - label: ":docker: Build Final Image" plugins: - - docker-compose#v4.5.0: + - docker-compose#v4.6.0: build: myservice image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo @@ -380,7 +380,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Run & Push" plugins: - - docker-compose#v4.5.0: + - docker-compose#v4.6.0: run: myservice push: myservice ``` @@ -395,7 +395,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Build & Push" plugins: - - docker-compose#v4.5.0: + - docker-compose#v4.6.0: build: myservice push: myservice ``` From 3710e14baa9a8bef0fa65866f9aef37570ff0b5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sun, 13 Nov 2022 21:31:55 -0300 Subject: [PATCH 150/253] Added wait option documentation and to spec --- README.md | 6 ++++++ plugin.yml | 1 + 2 files changed, 7 insertions(+) diff --git a/README.md b/README.md index 48b6d5e0..163998d1 100644 --- a/README.md +++ b/README.md @@ -572,6 +572,12 @@ If set to false, doesn't start linked services. The default is `true`. +### `wait` (optional, run only) + +Whether to wait for dependencies to be up (and healthy if possible) when starting them up. It translates to using [`--wait` in the docker-compose up] command. + +Defaults to `false`. + ### `ansi` (optional, run only) If set to false, disables the ansi output from containers. diff --git a/plugin.yml b/plugin.yml index 71025495..edb40f51 100644 --- a/plugin.yml +++ b/plugin.yml @@ -128,4 +128,5 @@ configuration: use-aliases: [ run ] user: [ run ] volumes: [ run ] + wait: [ run ] workdir: [ run ] From acf42ba9ad97adea942dd16988941e273aee5714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sun, 13 Nov 2022 21:32:17 -0300 Subject: [PATCH 151/253] Implemented --wait (and simplified a bit the code) --- commands/run.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/commands/run.sh b/commands/run.sh index cc0f36c0..d3fb45a4 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -262,16 +262,17 @@ elif [[ ! -f "$override_file" ]]; then run_docker_compose build "${build_params[@]}" "$run_service" fi +up_params+=("up") # this ensures that the array has elements to avoid issues with bash 4.3 + +if [[ "$(plugin_read_config WAIT "false")" == "true" ]] ; then + up_params+=("--wait") +fi + dependency_exitcode=0 if [[ "$(plugin_read_config DEPENDENCIES "true")" == "true" ]] ; then - # Start up service dependencies in a different header to keep the main run with less noise echo "~~~ :docker: Starting dependencies" - if [[ ${#up_params[@]} -gt 0 ]] ; then - run_docker_compose "${up_params[@]}" up -d --scale "${run_service}=0" "${run_service}" || dependency_exitcode=$? - else - run_docker_compose up -d --scale "${run_service}=0" "${run_service}" || dependency_exitcode=$? - fi + run_docker_compose "${up_params[@]}" -d --scale "${run_service}=0" "${run_service}" || dependency_exitcode=$? fi if [[ $dependency_exitcode -ne 0 ]] ; then From 25c1c33043bbfc7253c6fecaa1fec8cd48c0ff2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sun, 13 Nov 2022 21:32:37 -0300 Subject: [PATCH 152/253] Added tests for new feature --- tests/run.bats | 28 ++++++++++++++++++++++++++++ tests/v2/run.bats | 28 ++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/tests/run.bats b/tests/run.bats index 4f4918c8..5f3f35cf 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -1356,3 +1356,31 @@ export BUILDKITE_JOB_ID=1111 unstub docker-compose unstub buildkite-agent } + +@test "Run waiting for dependencies" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="echo hello world" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_WAIT=true + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "-f docker-compose.yml -p buildkite1111 up --wait -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "Starting dependencies" + assert_output --partial "ran myservice" + + unstub docker-compose + unstub buildkite-agent +} \ No newline at end of file diff --git a/tests/v2/run.bats b/tests/v2/run.bats index 46abd364..316b45bb 100644 --- a/tests/v2/run.bats +++ b/tests/v2/run.bats @@ -1232,3 +1232,31 @@ export BUILDKITE_JOB_ID=1111 unstub docker unstub buildkite-agent } + +@test "Run waiting for dependencies" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="echo hello world" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_WAIT=true + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "compose -f docker-compose.yml -p buildkite1111 up --wait -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "ran myservice" + + unstub docker + unstub buildkite-agent +} \ No newline at end of file From 6c58916d4a88d60d7db5ea2d5057645cfa7ecefd Mon Sep 17 00:00:00 2001 From: Trevor North Date: Wed, 16 Nov 2022 14:49:58 +0000 Subject: [PATCH 153/253] Add support for exposing service ports on run --- README.md | 6 ++++++ commands/run.sh | 5 +++++ tests/run.bats | 27 +++++++++++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/README.md b/README.md index 794eb210..827e66fa 100644 --- a/README.md +++ b/README.md @@ -606,6 +606,12 @@ The default is `true`. Sets the `--entrypoint` argument when running `docker-compose`. +### `service-ports` (optional, run only) + +If set to true, docker compose will run with the service ports enabled and mapped to the host. Equivalent to `--service-ports` in docker-compose. + +The default is `false`. + ### `upload-container-logs` (optional, run only) Select when to upload container logs. diff --git a/commands/run.sh b/commands/run.sh index d3fb45a4..c0008df7 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -231,6 +231,11 @@ if [[ -n "${BUILDKITE_AGENT_BINARY_PATH:-}" ]] ; then ) fi +# Optionally expose service ports +if [[ "$(plugin_read_config SERVICE_PORTS "false")" == "true" ]]; then + run_params+=(--service-ports) +fi + run_params+=("$run_service") build_params=(--pull) diff --git a/tests/run.bats b/tests/run.bats index 5f3f35cf..f50d73a3 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -1381,6 +1381,33 @@ export BUILDKITE_JOB_ID=1111 assert_output --partial "Starting dependencies" assert_output --partial "ran myservice" + unstub docker-compose + unstub buildkite-agent +} + +@test "Run with --service-ports" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_SERVICE_PORTS=true + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm --service-ports myservice /bin/sh -e -c $'pwd' : echo ran myservice without tty" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : echo myimage" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "ran myservice without tty" unstub docker-compose unstub buildkite-agent } \ No newline at end of file From 125230727a2cdabd9a3542780f713358b618ef55 Mon Sep 17 00:00:00 2001 From: Trevor North Date: Wed, 16 Nov 2022 17:25:05 +0000 Subject: [PATCH 154/253] Add v2 test and property validation --- plugin.yml | 2 ++ tests/v2/run.bats | 27 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/plugin.yml b/plugin.yml index 9c4ddcbc..473b6f1b 100644 --- a/plugin.yml +++ b/plugin.yml @@ -76,6 +76,8 @@ configuration: type: integer rm: type: boolean + service-ports: + type: boolean skip-checkout: type: boolean skip-pull: diff --git a/tests/v2/run.bats b/tests/v2/run.bats index 316b45bb..711019df 100644 --- a/tests/v2/run.bats +++ b/tests/v2/run.bats @@ -1257,6 +1257,33 @@ export BUILDKITE_JOB_ID=1111 assert_output --partial "built myservice" assert_output --partial "ran myservice" + unstub docker + unstub buildkite-agent +} + +@test "Run with --service-ports" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_SERVICE_PORTS=true + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm --service-ports myservice /bin/sh -e -c $'pwd' : echo ran myservice without tty" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : echo myimage" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "ran myservice without tty" unstub docker unstub buildkite-agent } \ No newline at end of file From 53ddaf2198615822ab273ec6329756483e7f4268 Mon Sep 17 00:00:00 2001 From: Trevor North Date: Wed, 16 Nov 2022 22:19:30 +0000 Subject: [PATCH 155/253] service-port depends on run --- plugin.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/plugin.yml b/plugin.yml index 473b6f1b..238b5041 100644 --- a/plugin.yml +++ b/plugin.yml @@ -127,6 +127,7 @@ configuration: propagate-uid-gid: [ run ] pull: [ run ] push-retries: [ push ] + service-ports: [ run ] skip-pull: [ run ] secrets: [ buildkit, build ] ssh: [ buildkit ] From 71ee4802091eb81d465dd248f181234183852139 Mon Sep 17 00:00:00 2001 From: Trevor North Date: Wed, 16 Nov 2022 22:19:54 +0000 Subject: [PATCH 156/253] Bump version in readme examples --- README.md | 54 +++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 827e66fa..f406117c 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v4.6.0: + - docker-compose#v4.7.0: run: app ``` @@ -28,7 +28,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v4.6.0: + - docker-compose#v4.7.0: run: app config: docker-compose.tests.yml env: @@ -41,7 +41,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v4.6.0: + - docker-compose#v4.7.0: run: app config: - docker-compose.yml @@ -56,7 +56,7 @@ env: steps: - command: test.sh plugins: - - docker-compose#v4.6.0: + - docker-compose#v4.7.0: run: app ``` @@ -65,7 +65,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v4.6.0: + - docker-compose#v4.7.0: run: app command: ["custom", "command", "values"] ``` @@ -79,7 +79,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.6.0: + - docker-compose#v4.7.0: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -87,7 +87,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.6.0: + - docker-compose#v4.7.0: run: app ``` @@ -104,7 +104,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.6.0: + - docker-compose#v4.7.0: run: app ``` @@ -122,7 +122,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.6.0: + - docker-compose#v4.7.0: run: app volumes: - "./dist:/app/dist" @@ -146,7 +146,7 @@ this plugin offers a `environment` block of its own: steps: - command: generate-dist.sh plugins: - - docker-compose#v4.6.0: + - docker-compose#v4.7.0: run: app env: - BUILDKITE_BUILD_NUMBER @@ -164,7 +164,7 @@ Alternatively, you can have the plugin add all environment variables defined for steps: - command: use-vars.sh plugins: - - docker-compose#v4.6.0: + - docker-compose#v4.7.0: run: app propagate-environment: true ``` @@ -179,7 +179,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v4.6.0: + - docker-compose#v4.7.0: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -196,7 +196,7 @@ If you have multiple steps that use the same service/image (such as steps that r steps: - label: ":docker: Build" plugins: - - docker-compose#v4.6.0: + - docker-compose#v4.7.0: build: app image-repository: index.docker.io/myorg/myrepo @@ -206,7 +206,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.6.0: + - docker-compose#v4.7.0: run: app ``` @@ -222,7 +222,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v4.6.0: + - docker-compose#v4.7.0: build: - app - tests @@ -234,7 +234,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.6.0: + - docker-compose#v4.7.0: run: tests ``` @@ -246,7 +246,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v4.6.0: + - docker-compose#v4.7.0: push: app ``` @@ -256,7 +256,7 @@ To push multiple images, you can use a list: steps: - label: ":docker: Push" plugins: - - docker-compose#v4.6.0: + - docker-compose#v4.7.0: push: - first-service - second-service @@ -268,7 +268,7 @@ If you want to push to a specific location (that's not defined as the `image` in steps: - label: ":docker: Push" plugins: - - docker-compose#v4.6.0: + - docker-compose#v4.7.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -282,14 +282,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker: Build an image" plugins: - - docker-compose#v4.6.0: + - docker-compose#v4.7.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.6.0: + - docker-compose#v4.7.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -303,7 +303,7 @@ This plugin allows for the value of `cache-from` to be a string or a list. If it steps: - label: ":docker Build an image" plugins: - - docker-compose#v4.6.0: + - docker-compose#v4.7.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: @@ -312,7 +312,7 @@ steps: - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.6.0: + - docker-compose#v4.7.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:my-branch @@ -326,7 +326,7 @@ Adding a grouping tag to the end of a cache-from list item allows this plugin to steps: - label: ":docker: Build Intermediate Image" plugins: - - docker-compose#v4.6.0: + - docker-compose#v4.7.0: build: myservice_intermediate # docker-compose.yml is the same as myservice but has `target: intermediate` image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo/myservice_intermediate @@ -336,7 +336,7 @@ steps: - wait - label: ":docker: Build Final Image" plugins: - - docker-compose#v4.6.0: + - docker-compose#v4.7.0: build: myservice image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo @@ -380,7 +380,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Run & Push" plugins: - - docker-compose#v4.6.0: + - docker-compose#v4.7.0: run: myservice push: myservice ``` @@ -395,7 +395,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Build & Push" plugins: - - docker-compose#v4.6.0: + - docker-compose#v4.7.0: build: myservice push: myservice ``` From ceed5696becf6f42090f1a7e4eeff9f159f52482 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 21 Nov 2022 22:25:23 -0300 Subject: [PATCH 157/253] add new option for cache-from separator --- plugin.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugin.yml b/plugin.yml index 238b5041..a4836bf7 100644 --- a/plugin.yml +++ b/plugin.yml @@ -27,6 +27,10 @@ configuration: cache-from: type: [ string, array ] minimum: 1 + separator-cache-from: + type: string + minLength: 1 + maxLength: 1 cli-version: type: string enum: @@ -115,6 +119,7 @@ configuration: ansi: [ run ] buildkit: [ build ] cache-from: [ build ] + cache-from-separator: [ cache-from ] dependencies: [ run ] env: [ run ] environment: [ run ] From f021db42715a77c5190ddd47518c2f424c358b63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 21 Nov 2022 22:25:44 -0300 Subject: [PATCH 158/253] Implement new seprator option --- commands/build.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/commands/build.sh b/commands/build.sh index ddd6b04d..e2e1505a 100755 --- a/commands/build.sh +++ b/commands/build.sh @@ -4,6 +4,7 @@ set -ueo pipefail image_repository="$(plugin_read_config IMAGE_REPOSITORY)" pull_retries="$(plugin_read_config PULL_RETRIES "0")" push_retries="$(plugin_read_config PUSH_RETRIES "0")" +separator="$(plugin_read_config SEPARATOR_CACHE_FROM ":")" override_file="docker-compose.buildkite-${BUILDKITE_BUILD_NUMBER}-override.yml" build_images=() @@ -54,7 +55,7 @@ fi # If no-cache is set skip pulling the cache-from images if [[ "$(plugin_read_config NO_CACHE "false")" == "false" ]] ; then for line in $(plugin_read_list CACHE_FROM) ; do - IFS=':' read -r -a tokens <<< "$line" + IFS="${separator}" read -r -a tokens <<< "$line" service_name=${tokens[0]} service_image=$(IFS=':'; echo "${tokens[*]:1:2}") if [ ${#tokens[@]} -gt 2 ]; then @@ -68,7 +69,7 @@ if [[ "$(plugin_read_config NO_CACHE "false")" == "false" ]] ; then continue fi - cache_from_group_name=$(IFS=':'; echo "${tokens[*]:3}") + cache_from_group_name=$(echo "${tokens[*]:3}") if [[ -z "$cache_from_group_name" ]]; then cache_from_group_name=":default:" fi From ba4d1f1754dc8d1955dff11cd39d1de50d4b5ca2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 21 Nov 2022 22:25:54 -0300 Subject: [PATCH 159/253] Add tests for new option --- tests/build.bats | 26 ++++++++++++++++++++++++++ tests/v2/build.bats | 23 +++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/tests/build.bats b/tests/build.bats index 0cb6a2cb..9f049f12 100644 --- a/tests/build.bats +++ b/tests/build.bats @@ -288,6 +288,32 @@ load '../lib/shared' unstub docker-compose } +@test "Build with a cache-from image and custom separator" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_PIPELINE_SLUG=test + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0='helloworld#my.repository:port/myservice_cache#latest' + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_SEPARATOR='#' + + stub docker \ + "pull my.repository:port/myservice_cache:latest : echo pulled cache image" + + stub docker-compose \ + "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "pulled cache image" + assert_output --partial "- my.repository:port/myservice_cache:latest" + assert_output --partial "built helloworld" + unstub docker + unstub docker-compose +} + @test "Build with a cache-from image with no-cache also set" { export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" export BUILDKITE_JOB_ID=1111 diff --git a/tests/v2/build.bats b/tests/v2/build.bats index 2a1d2255..9fc590e1 100644 --- a/tests/v2/build.bats +++ b/tests/v2/build.bats @@ -272,6 +272,29 @@ setup_file() { unstub docker } +@test "Build with a cache-from image and custom separator" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_PIPELINE_SLUG=test + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0='helloworld#my.repository:port/myservice_cache#latest' + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_SEPARATOR_CACHE_FROM='#' + + stub docker \ + "pull my.repository:port/myservice_cache:latest : echo pulled cache image" \ + "compose -f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "pulled cache image" + assert_output --partial "- my.repository:port/myservice_cache:latest" + assert_output --partial "built helloworld" + unstub docker +} + @test "Build with an invalid cache-from tag" { export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" export BUILDKITE_JOB_ID=1111 From c282fc716a63bf17d9aae0fbc3a0047dd65c9e59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 21 Nov 2022 22:26:14 -0300 Subject: [PATCH 160/253] Add documentation for separator and more examples --- README.md | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index f406117c..843a6508 100644 --- a/README.md +++ b/README.md @@ -295,6 +295,10 @@ steps: - app:index.docker.io/myorg/myrepo/myapp:latest ``` +**Important**: if your registry URL contains a port, you will need to take the following into account: +* specify the `separator-cache-from` option to change the colon character to something else (like `#`) +* you will have to specify tags in the `push` elements (or the plugin will try to validate everything after the port as a tag) + #### Multiple cache-from values This plugin allows for the value of `cache-from` to be a string or a list. If it's a list, as below, then the first successfully pulled image will be used. @@ -306,17 +310,17 @@ steps: - docker-compose#v4.7.0: build: app image-repository: index.docker.io/myorg/myrepo + separator-cache-from: "#" cache-from: - - app:index.docker.io/myorg/myrepo/myapp:my-branch - - app:index.docker.io/myorg/myrepo/myapp:latest + - "app#myregistry:port/myrepo/myapp#my-branch" + - "app#myregistry:port/myrepo/myapp#latest" - wait - label: ":docker: Push to final repository" plugins: - docker-compose#v4.7.0: push: - - app:index.docker.io/myorg/myrepo/myapp - - app:index.docker.io/myorg/myrepo/myapp:my-branch - - app:index.docker.io/myorg/myrepo/myapp:latest + - app:myregistry:port/myrepo/myapp:my-branch + - app:myregistry:port/myrepo/myapp:latest ``` You may actually want to build your image with multiple cache-from values, for instance, with the cached images of multiple stages in a multi-stage build. @@ -518,6 +522,14 @@ This option can also be configured on the agent machine using the environment va A list of images to pull caches from in the format `service:index.docker.io/myorg/myrepo/myapp:tag` before building, ignoring any failures. If multiple images are listed for a service, the first one to successfully pull will be used. Requires docker-compose file version `3.2+`. +### `separator-cache-from` (optional, build only, single character) + +A single character that specifies the character to use for splitting elements in the `cache-from` option. + +By default it is `:` which should not be a problem unless your registry's URL contains a port, in which case you will have to use this option to specify a different character. + +**Important**: the tag to use is its own field, so you will have to specify elements like `service#registry:port/myrepo/myapp#tag#group` + ### `target` (optional, build only) Allow for intermediate builds with `--target VALUE` options. From 15f0df55df827bacac5cb60953a7b3cb768f81c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 21 Nov 2022 22:26:34 -0300 Subject: [PATCH 161/253] Assume the last element in push is the tag --- commands/push.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/commands/push.sh b/commands/push.sh index 6aecd365..bc96fb63 100755 --- a/commands/push.sh +++ b/commands/push.sh @@ -22,8 +22,9 @@ for line in $(plugin_read_list PUSH) ; do service_image=$(compose_image_for_service "$service_name") # push in the form of service:repo:tag + # if the registry contains a port this means that the tag is mandatory if [[ ${#tokens[@]} -gt 2 ]]; then - if ! validate_tag "${tokens[2]}"; then + if ! validate_tag "${tokens[-1]}"; then echo "🚨 specified image to push ${line} has an invalid tag so it will be ignored" continue fi From 4e738abfc60cdd4909ab149c55b831ca71d6262f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 21 Nov 2022 22:27:31 -0300 Subject: [PATCH 162/253] Update version for next release --- README.md | 54 +++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 843a6508..47447cff 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v4.7.0: + - docker-compose#v4.8.0: run: app ``` @@ -28,7 +28,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v4.7.0: + - docker-compose#v4.8.0: run: app config: docker-compose.tests.yml env: @@ -41,7 +41,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v4.7.0: + - docker-compose#v4.8.0: run: app config: - docker-compose.yml @@ -56,7 +56,7 @@ env: steps: - command: test.sh plugins: - - docker-compose#v4.7.0: + - docker-compose#v4.8.0: run: app ``` @@ -65,7 +65,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v4.7.0: + - docker-compose#v4.8.0: run: app command: ["custom", "command", "values"] ``` @@ -79,7 +79,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.7.0: + - docker-compose#v4.8.0: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -87,7 +87,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.7.0: + - docker-compose#v4.8.0: run: app ``` @@ -104,7 +104,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.7.0: + - docker-compose#v4.8.0: run: app ``` @@ -122,7 +122,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.7.0: + - docker-compose#v4.8.0: run: app volumes: - "./dist:/app/dist" @@ -146,7 +146,7 @@ this plugin offers a `environment` block of its own: steps: - command: generate-dist.sh plugins: - - docker-compose#v4.7.0: + - docker-compose#v4.8.0: run: app env: - BUILDKITE_BUILD_NUMBER @@ -164,7 +164,7 @@ Alternatively, you can have the plugin add all environment variables defined for steps: - command: use-vars.sh plugins: - - docker-compose#v4.7.0: + - docker-compose#v4.8.0: run: app propagate-environment: true ``` @@ -179,7 +179,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v4.7.0: + - docker-compose#v4.8.0: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -196,7 +196,7 @@ If you have multiple steps that use the same service/image (such as steps that r steps: - label: ":docker: Build" plugins: - - docker-compose#v4.7.0: + - docker-compose#v4.8.0: build: app image-repository: index.docker.io/myorg/myrepo @@ -206,7 +206,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.7.0: + - docker-compose#v4.8.0: run: app ``` @@ -222,7 +222,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v4.7.0: + - docker-compose#v4.8.0: build: - app - tests @@ -234,7 +234,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.7.0: + - docker-compose#v4.8.0: run: tests ``` @@ -246,7 +246,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v4.7.0: + - docker-compose#v4.8.0: push: app ``` @@ -256,7 +256,7 @@ To push multiple images, you can use a list: steps: - label: ":docker: Push" plugins: - - docker-compose#v4.7.0: + - docker-compose#v4.8.0: push: - first-service - second-service @@ -268,7 +268,7 @@ If you want to push to a specific location (that's not defined as the `image` in steps: - label: ":docker: Push" plugins: - - docker-compose#v4.7.0: + - docker-compose#v4.8.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -282,14 +282,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker: Build an image" plugins: - - docker-compose#v4.7.0: + - docker-compose#v4.8.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.7.0: + - docker-compose#v4.8.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -307,7 +307,7 @@ This plugin allows for the value of `cache-from` to be a string or a list. If it steps: - label: ":docker Build an image" plugins: - - docker-compose#v4.7.0: + - docker-compose#v4.8.0: build: app image-repository: index.docker.io/myorg/myrepo separator-cache-from: "#" @@ -317,7 +317,7 @@ steps: - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.7.0: + - docker-compose#v4.8.0: push: - app:myregistry:port/myrepo/myapp:my-branch - app:myregistry:port/myrepo/myapp:latest @@ -330,7 +330,7 @@ Adding a grouping tag to the end of a cache-from list item allows this plugin to steps: - label: ":docker: Build Intermediate Image" plugins: - - docker-compose#v4.7.0: + - docker-compose#v4.8.0: build: myservice_intermediate # docker-compose.yml is the same as myservice but has `target: intermediate` image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo/myservice_intermediate @@ -340,7 +340,7 @@ steps: - wait - label: ":docker: Build Final Image" plugins: - - docker-compose#v4.7.0: + - docker-compose#v4.8.0: build: myservice image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo @@ -384,7 +384,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Run & Push" plugins: - - docker-compose#v4.7.0: + - docker-compose#v4.8.0: run: myservice push: myservice ``` @@ -399,7 +399,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Build & Push" plugins: - - docker-compose#v4.7.0: + - docker-compose#v4.8.0: build: myservice push: myservice ``` From d7ff4c2a15671be4208590ed4a5cb84329d7b3f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 21 Nov 2022 23:53:22 -0300 Subject: [PATCH 163/253] revert to old group name logic (not exactly the same as I expected it to be) --- commands/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/build.sh b/commands/build.sh index e2e1505a..0ad9a025 100755 --- a/commands/build.sh +++ b/commands/build.sh @@ -69,7 +69,7 @@ if [[ "$(plugin_read_config NO_CACHE "false")" == "false" ]] ; then continue fi - cache_from_group_name=$(echo "${tokens[*]:3}") + cache_from_group_name=$(IFS=':'; echo "${tokens[*]:3}") if [[ -z "$cache_from_group_name" ]]; then cache_from_group_name=":default:" fi From cd6f73ba97e94c780f93d538426627edc4a505e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 21 Nov 2022 23:53:30 -0300 Subject: [PATCH 164/253] Correct variable name in test --- tests/build.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/build.bats b/tests/build.bats index 9f049f12..363597a1 100644 --- a/tests/build.bats +++ b/tests/build.bats @@ -296,7 +296,7 @@ load '../lib/shared' export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0='helloworld#my.repository:port/myservice_cache#latest' - export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_SEPARATOR='#' + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_SEPARATOR_CACHE_FROM='#' stub docker \ "pull my.repository:port/myservice_cache:latest : echo pulled cache image" From 43117b1d275ee5f51b400a853136f1e148e1a0d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 21 Nov 2022 23:59:51 -0300 Subject: [PATCH 165/253] Added more complicated cache-from test with separator --- tests/build.bats | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/build.bats b/tests/build.bats index 363597a1..1b8b0ffa 100644 --- a/tests/build.bats +++ b/tests/build.bats @@ -480,6 +480,46 @@ load '../lib/shared' unstub docker-compose } +@test "Build with several cache-from image groups for one service with failures and separator" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=helloworld#my.repository:port/myservice_cache#build-target-build-1#target1 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_1=helloworld#my.repository:port/myservice_cache#build-target-latest#target1 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_2=helloworld#my.repository:port/myservice_cache#install-target-build-1#target2 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_3=helloworld#my.repository:port/myservice_cache#branch-name + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_4=helloworld#my.repository:port/myservice_cache#latest + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_SEPARATOR_CACHE_FROM='#' + + stub docker \ + "pull my.repository:port/myservice_cache:build-target-build-1 : exit 1" \ + "pull my.repository:port/myservice_cache:build-target-latest : echo pulled cache image build-target-latest" \ + "pull my.repository:port/myservice_cache:install-target-build-1 : echo pulled cache image install-target" \ + "pull my.repository:port/myservice_cache:branch-name : echo pulled cache image branch-name" + + stub docker-compose \ + "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "pulled cache image build-target-latest" + assert_output --partial "pulled cache image install-target" + assert_output --partial "pulled cache image branch-name" + assert_output --partial "- my.repository:port/myservice_cache:build-target-latest" + assert_output --partial "- my.repository:port/myservice_cache:build-target-latest" + assert_output --partial "- my.repository:port/myservice_cache:install-target-build-1" + assert_output --partial "- my.repository:port/myservice_cache:branch-name" + refute_output --partial "- my.repository:port/myservice_cache:latest" + assert_output --partial "built helloworld" + + unstub docker + unstub docker-compose +} + @test "Build with several cache-from image groups out of order" { export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" export BUILDKITE_JOB_ID=1111 From 764b4bdcb271e19aa11dd8b99e9d5a8eea120eb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Bellone?= Date: Thu, 24 Nov 2022 16:21:32 -0300 Subject: [PATCH 166/253] Added new option to plugin --- plugin.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugin.yml b/plugin.yml index 238b5041..f96af54f 100644 --- a/plugin.yml +++ b/plugin.yml @@ -63,6 +63,8 @@ configuration: type: [ boolean, string ] no-cache: type: boolean + pre-run-dependencies: + type: boolean propagate-environment: type: boolean propagate-uid-gid: @@ -124,6 +126,7 @@ configuration: leave-volumes: [ run ] mount-buildkite-agent: [ run ] mount-checkout: [ run ] + pre-run-dependencies: [ run ] propagate-uid-gid: [ run ] pull: [ run ] push-retries: [ push ] From 2784e435cdd5c2b9ee5d5ba307c1d7e24a8d9c3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Bellone?= Date: Thu, 24 Nov 2022 16:36:45 -0300 Subject: [PATCH 167/253] Implemented feature to skip pre-run for dependencies --- commands/run.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/commands/run.sh b/commands/run.sh index c0008df7..9ea24ab7 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -119,6 +119,7 @@ fi tty_default='true' workdir_default="/workdir" pwd_default="$PWD" +run_dependencies="true" # Set operating system specific defaults if is_windows ; then @@ -137,6 +138,9 @@ fi # Optionally disable dependencies if [[ "$(plugin_read_config DEPENDENCIES "true")" == "false" ]] ; then run_params+=(--no-deps) + run_dependencies="false" +elif [[ "$(plugin_read_config PRE_RUN_DEPENDENCIES "true")" == "false" ]]; then + run_dependencies="false" fi if [[ -n "$(plugin_read_config WORKDIR)" ]] || [[ "${mount_checkout}" == "true" ]]; then @@ -274,7 +278,7 @@ if [[ "$(plugin_read_config WAIT "false")" == "true" ]] ; then fi dependency_exitcode=0 -if [[ "$(plugin_read_config DEPENDENCIES "true")" == "true" ]] ; then +if [[ "${run_dependencies}" == "true" ]] ; then # Start up service dependencies in a different header to keep the main run with less noise echo "~~~ :docker: Starting dependencies" run_docker_compose "${up_params[@]}" -d --scale "${run_service}=0" "${run_service}" || dependency_exitcode=$? From 1ead3e192daa251f76411ad095b6318bf2f1fc9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Bellone?= Date: Thu, 24 Nov 2022 16:39:40 -0300 Subject: [PATCH 168/253] Add documentation for the option --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f406117c..388ebb7f 100644 --- a/README.md +++ b/README.md @@ -568,10 +568,14 @@ The default is `true` on unix, `false` on windows ### `dependencies` (optional, run only) -If set to false, doesn't start linked services. +If set to false, runs with `--no-deps` and doesn't start linked services. The default is `true`. +### `pre-run-dependencies` (optional, run only) + +If `dependencies` are activated (which is the default), you can skip starting them up before the main container by setting this option to `false`. This is useful if you want compose to take care of that on its own at the expense of messier output in the run step. + ### `wait` (optional, run only) Whether to wait for dependencies to be up (and healthy if possible) when starting them up. It translates to using [`--wait` in the docker-compose up] command. From 049ae3ce697009a82a1af88ba2d37a355dba65bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Bellone?= Date: Thu, 24 Nov 2022 16:39:52 -0300 Subject: [PATCH 169/253] Added tests for the option --- tests/run.bats | 26 ++++++++++++++++++++++++++ tests/v2/run.bats | 27 +++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/tests/run.bats b/tests/run.bats index f50d73a3..3cd4b623 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -531,6 +531,32 @@ export BUILDKITE_JOB_ID=1111 unstub buildkite-agent } +@test "Run with dependencies but in a single step" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PRE_RUN_DEPENDENCIES=false + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with dependencies" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "ran myservice with dependencies" + unstub docker-compose + unstub buildkite-agent +} + @test "Run without ansi output" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice diff --git a/tests/v2/run.bats b/tests/v2/run.bats index 711019df..b2349b66 100644 --- a/tests/v2/run.bats +++ b/tests/v2/run.bats @@ -532,6 +532,33 @@ export BUILDKITE_JOB_ID=1111 unstub buildkite-agent } +@test "Run with dependencies but in a single step" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PRE_RUN_DEPENDENCIES=false + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with dependencies" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "ran myservice with dependencies" + + unstub docker + unstub buildkite-agent +} + @test "Run without ansi output" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice From 6b5dd30390d726d4cc04f9a4c67669663bce78a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 12 Dec 2022 21:53:36 -0300 Subject: [PATCH 170/253] Added compatibility option to plugin spec and documentation --- README.md | 12 ++++++++++++ plugin.yml | 2 ++ 2 files changed, 14 insertions(+) diff --git a/README.md b/README.md index f406117c..b86aec0d 100644 --- a/README.md +++ b/README.md @@ -602,6 +602,18 @@ If set to true, docker compose will remove the primary container after run. Equi The default is `true`. +### `compatibility` (optional, run only) + +If set to true, docker compose will run the `up` command with compatibility mode. Equivalent to `--compatibility` in docker-compose. + +The default is `false`. + +Note that [the effect of this option changes depending on your docker compose CLI version](https://docs.docker.com/compose/cli-command-compatibility/#flags-that-will-not-be-implemented): +* in v1 it translates (composefile) v3 deploy keys to their non-swarm (composefile) v2 equivalents +* in v2 it will revert some behaviour to v1 as well, including (but not limited to): + - [Character separator for container names](https://github.com/docker/compose/blob/a0acc20d883ce22b8b0c65786e3bea1328809bbd/cmd/compose/compose.go#L181) + - [Not normalizing compose models (when running `config`)](https://github.com/docker/compose/blob/2e7644ff21f9ca0ea6fb5e8d41d4f6af32cd7e20/cmd/compose/convert.go#L69) (but it shouldn't apply here because it is only used in the `up` command) + ### `entrypoint` (optional, run only) Sets the `--entrypoint` argument when running `docker-compose`. diff --git a/plugin.yml b/plugin.yml index 238b5041..4741b1e7 100644 --- a/plugin.yml +++ b/plugin.yml @@ -27,6 +27,8 @@ configuration: cache-from: type: [ string, array ] minimum: 1 + compatibility: + type: boolean cli-version: type: string enum: From 706d9dcd04896415713fcfc8304df2c66605b4d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 12 Dec 2022 21:54:00 -0300 Subject: [PATCH 171/253] Implemented compatibility flag when running compose up --- commands/run.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/commands/run.sh b/commands/run.sh index c0008df7..a5618655 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -182,6 +182,11 @@ if [[ "$(plugin_read_config RM "true")" == "true" ]]; then run_params+=(--rm) fi +# Enable compatibility mode for v3 files +if [[ "$(plugin_read_config COMPATIBILITY "false")" == "true" ]]; then + run_params+=(--compatibility) +fi + # Optionally sets --entrypoint if [[ -n "$(plugin_read_config ENTRYPOINT)" ]] ; then run_params+=(--entrypoint) From be45f2215a0ff6d8b723bf9678e59ecb196ff266 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 12 Dec 2022 21:54:20 -0300 Subject: [PATCH 172/253] Added tests for v1 --- tests/run.bats | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/run.bats b/tests/run.bats index f50d73a3..48a39d96 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -612,6 +612,33 @@ export BUILDKITE_JOB_ID=1111 unstub buildkite-agent } +@test "Run with compatibility mode" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_COMPATIBILITY=true + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm --compatibility myservice /bin/sh -e -c 'pwd' : echo ran myservice with use aliases output" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "ran myservice with use aliases output" + unstub docker-compose + unstub buildkite-agent +} + @test "Run with a volumes option" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice From b2bbc9760a8c4734faa0f65675d720e2b59d07b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 13 Dec 2022 03:29:02 -0300 Subject: [PATCH 173/253] Moved implementation to shared command as it is a global flag --- commands/run.sh | 5 ----- lib/shared.bash | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/commands/run.sh b/commands/run.sh index a5618655..c0008df7 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -182,11 +182,6 @@ if [[ "$(plugin_read_config RM "true")" == "true" ]]; then run_params+=(--rm) fi -# Enable compatibility mode for v3 files -if [[ "$(plugin_read_config COMPATIBILITY "false")" == "true" ]]; then - run_params+=(--compatibility) -fi - # Optionally sets --entrypoint if [[ -n "$(plugin_read_config ENTRYPOINT)" ]] ; then run_params+=(--entrypoint) diff --git a/lib/shared.bash b/lib/shared.bash index 583cb122..986eaf9a 100644 --- a/lib/shared.bash +++ b/lib/shared.bash @@ -200,6 +200,11 @@ function run_docker_compose() { command+=(--no-ansi) fi + # Enable compatibility mode for v3 files + if [[ "$(plugin_read_config COMPATIBILITY "false")" == "true" ]]; then + command+=(--compatibility) + fi + for file in $(docker_compose_config_files) ; do command+=(-f "$file") done From a581b8ef067196c4c04d7bdccd883c808212119a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 13 Dec 2022 03:30:30 -0300 Subject: [PATCH 174/253] Added doc clarification and push compatibility due to separator --- README.md | 4 ++-- lib/push.bash | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b86aec0d..2d7bae74 100644 --- a/README.md +++ b/README.md @@ -604,7 +604,7 @@ The default is `true`. ### `compatibility` (optional, run only) -If set to true, docker compose will run the `up` command with compatibility mode. Equivalent to `--compatibility` in docker-compose. +If set to true, all docker compose commands will rum with compatibility mode. Equivalent to `--compatibility` in docker-compose. The default is `false`. @@ -612,7 +612,7 @@ Note that [the effect of this option changes depending on your docker compose CL * in v1 it translates (composefile) v3 deploy keys to their non-swarm (composefile) v2 equivalents * in v2 it will revert some behaviour to v1 as well, including (but not limited to): - [Character separator for container names](https://github.com/docker/compose/blob/a0acc20d883ce22b8b0c65786e3bea1328809bbd/cmd/compose/compose.go#L181) - - [Not normalizing compose models (when running `config`)](https://github.com/docker/compose/blob/2e7644ff21f9ca0ea6fb5e8d41d4f6af32cd7e20/cmd/compose/convert.go#L69) (but it shouldn't apply here because it is only used in the `up` command) + - [Not normalizing compose models (when running `config`)](https://github.com/docker/compose/blob/2e7644ff21f9ca0ea6fb5e8d41d4f6af32cd7e20/cmd/compose/convert.go#L69) ### `entrypoint` (optional, run only) diff --git a/lib/push.bash b/lib/push.bash index d69ee8fa..1025c974 100644 --- a/lib/push.bash +++ b/lib/push.bash @@ -22,7 +22,7 @@ default_compose_image_for_service() { local service="$1" local separator="_" - if [[ "$(plugin_read_config CLI_VERSION "1")" == "2" ]] ; then + if [[ "$(plugin_read_config CLI_VERSION "1")" == "2" ]] && [[ "$(plugin_read_config COMPATIBILITY "false")" != "true" ]] ; then separator="-" fi From b272e0daad8ed92811b1fdfdadb5bfeab1d12e05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 13 Dec 2022 03:31:07 -0300 Subject: [PATCH 175/253] Updated test --- tests/run.bats | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/run.bats b/tests/run.bats index 48a39d96..fe7fa648 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -623,9 +623,9 @@ export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_COMPATIBILITY=true stub docker-compose \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm --compatibility myservice /bin/sh -e -c 'pwd' : echo ran myservice with use aliases output" + "--compatibility -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "--compatibility -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "--compatibility -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with use aliases output" stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ From 211da9018b3e213589cda82cd56f0df713625b36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 13 Dec 2022 03:42:49 -0300 Subject: [PATCH 176/253] Added v2 tests --- tests/v2/push.bats | 30 ++++++++++++++++++++++++++++++ tests/v2/run.bats | 28 ++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/tests/v2/push.bats b/tests/v2/push.bats index d0ef5ae1..f6d109af 100644 --- a/tests/v2/push.bats +++ b/tests/v2/push.bats @@ -96,6 +96,36 @@ setup_file() { unstub buildkite-agent } +@test "Push a prebuilt image with a repository and a tag in compatibility mode" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PUSH=myservice:my.repository/myservice:llamas + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_COMPATIBILITY=true + + stub docker \ + "compose --compatibility -f docker-compose.yml -p buildkite1111 config : echo blah" \ + "pull myimage : echo pulled prebuilt image" \ + "tag myimage buildkite1111_myservice : echo " \ + "tag buildkite1111_myservice my.repository/myservice:llamas : echo tagged image" \ + "push my.repository/myservice:llamas : echo pushed myservice" + + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run $PWD/hooks/command + + assert_success + assert_output --partial "pulled prebuilt image" + assert_output --partial "tagged image" + assert_output --partial "pushed myservice" + unstub docker + unstub buildkite-agent +} + @test "Push a prebuilt image with an invalid tag" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PUSH=myservice:my.repository/myservice:-llamas diff --git a/tests/v2/run.bats b/tests/v2/run.bats index 711019df..58fc252e 100644 --- a/tests/v2/run.bats +++ b/tests/v2/run.bats @@ -586,6 +586,34 @@ export BUILDKITE_JOB_ID=1111 unstub buildkite-agent } + +@test "Run with compatibility mode" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_COMPATIBILITY=true + + stub docker \ + "compose --compatibility -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "compose --compatibility -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "compose --compatibility -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with use aliases output" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "ran myservice with use aliases output" + unstub docker + unstub buildkite-agent +} + @test "Run with a volumes option" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice From 414b9254fd149bce5f00cc6acc78e9ecdc66f27d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 17 Dec 2022 16:09:51 -0300 Subject: [PATCH 177/253] Updated version in readme in preparation for next release --- README.md | 54 +++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 7f7b6462..82d3bf0e 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v4.8.0: + - docker-compose#v4.9.0: run: app ``` @@ -28,7 +28,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v4.8.0: + - docker-compose#v4.9.0: run: app config: docker-compose.tests.yml env: @@ -41,7 +41,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v4.8.0: + - docker-compose#v4.9.0: run: app config: - docker-compose.yml @@ -56,7 +56,7 @@ env: steps: - command: test.sh plugins: - - docker-compose#v4.8.0: + - docker-compose#v4.9.0: run: app ``` @@ -65,7 +65,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v4.8.0: + - docker-compose#v4.9.0: run: app command: ["custom", "command", "values"] ``` @@ -79,7 +79,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.8.0: + - docker-compose#v4.9.0: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -87,7 +87,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.8.0: + - docker-compose#v4.9.0: run: app ``` @@ -104,7 +104,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.8.0: + - docker-compose#v4.9.0: run: app ``` @@ -122,7 +122,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.8.0: + - docker-compose#v4.9.0: run: app volumes: - "./dist:/app/dist" @@ -146,7 +146,7 @@ this plugin offers a `environment` block of its own: steps: - command: generate-dist.sh plugins: - - docker-compose#v4.8.0: + - docker-compose#v4.9.0: run: app env: - BUILDKITE_BUILD_NUMBER @@ -164,7 +164,7 @@ Alternatively, you can have the plugin add all environment variables defined for steps: - command: use-vars.sh plugins: - - docker-compose#v4.8.0: + - docker-compose#v4.9.0: run: app propagate-environment: true ``` @@ -179,7 +179,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v4.8.0: + - docker-compose#v4.9.0: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -196,7 +196,7 @@ If you have multiple steps that use the same service/image (such as steps that r steps: - label: ":docker: Build" plugins: - - docker-compose#v4.8.0: + - docker-compose#v4.9.0: build: app image-repository: index.docker.io/myorg/myrepo @@ -206,7 +206,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.8.0: + - docker-compose#v4.9.0: run: app ``` @@ -222,7 +222,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v4.8.0: + - docker-compose#v4.9.0: build: - app - tests @@ -234,7 +234,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.8.0: + - docker-compose#v4.9.0: run: tests ``` @@ -246,7 +246,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v4.8.0: + - docker-compose#v4.9.0: push: app ``` @@ -256,7 +256,7 @@ To push multiple images, you can use a list: steps: - label: ":docker: Push" plugins: - - docker-compose#v4.8.0: + - docker-compose#v4.9.0: push: - first-service - second-service @@ -268,7 +268,7 @@ If you want to push to a specific location (that's not defined as the `image` in steps: - label: ":docker: Push" plugins: - - docker-compose#v4.8.0: + - docker-compose#v4.9.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -282,14 +282,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker: Build an image" plugins: - - docker-compose#v4.8.0: + - docker-compose#v4.9.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.8.0: + - docker-compose#v4.9.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -307,7 +307,7 @@ This plugin allows for the value of `cache-from` to be a string or a list. If it steps: - label: ":docker Build an image" plugins: - - docker-compose#v4.8.0: + - docker-compose#v4.9.0: build: app image-repository: index.docker.io/myorg/myrepo separator-cache-from: "#" @@ -317,7 +317,7 @@ steps: - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.8.0: + - docker-compose#v4.9.0: push: - app:myregistry:port/myrepo/myapp:my-branch - app:myregistry:port/myrepo/myapp:latest @@ -330,7 +330,7 @@ Adding a grouping tag to the end of a cache-from list item allows this plugin to steps: - label: ":docker: Build Intermediate Image" plugins: - - docker-compose#v4.8.0: + - docker-compose#v4.9.0: build: myservice_intermediate # docker-compose.yml is the same as myservice but has `target: intermediate` image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo/myservice_intermediate @@ -340,7 +340,7 @@ steps: - wait - label: ":docker: Build Final Image" plugins: - - docker-compose#v4.8.0: + - docker-compose#v4.9.0: build: myservice image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo @@ -384,7 +384,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Run & Push" plugins: - - docker-compose#v4.8.0: + - docker-compose#v4.9.0: run: myservice push: myservice ``` @@ -399,7 +399,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Build & Push" plugins: - - docker-compose#v4.8.0: + - docker-compose#v4.9.0: build: myservice push: myservice ``` From bbb2dba73e6edc31fd3682580145385d5899fbc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Wed, 4 Jan 2023 11:56:39 -0300 Subject: [PATCH 178/253] Updated tests with missing argument in stubs --- tests/run.bats | 42 +++++++++++++++++++++--------------------- tests/v2/run.bats | 42 +++++++++++++++++++++--------------------- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/tests/run.bats b/tests/run.bats index 199660c7..124f16c3 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -19,7 +19,7 @@ load '../lib/run' stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" stub buildkite-agent \ @@ -45,7 +45,7 @@ load '../lib/run' stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice : echo ran myservice" stub buildkite-agent \ @@ -73,7 +73,7 @@ load '../lib/run' stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --workdir=/test_workdir --rm myservice : echo ran myservice" stub buildkite-agent \ @@ -99,7 +99,7 @@ load '../lib/run' stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c $'sh -c \'echo hello world\'' : echo ran myservice" stub buildkite-agent \ @@ -127,7 +127,7 @@ cmd3" stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c $'cmd1\ncmd2\ncmd3' : echo ran myservice" stub buildkite-agent \ @@ -156,7 +156,7 @@ cmd3" stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice echo 'hello world' : echo ran myservice" stub buildkite-agent \ @@ -188,7 +188,7 @@ cmd3" stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 -e MYENV=0 -e MYENV -e MYENV=2 -e MYENV -e ANOTHER=this\ is\ a\ long\ string\ with\ spaces\;\ and\ semi-colons --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" stub buildkite-agent \ @@ -214,7 +214,7 @@ cmd3" stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull --no-cache myservice : echo built myservice" \ - "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" stub buildkite-agent \ @@ -242,7 +242,7 @@ cmd3" stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull --build-arg MYARG=0 --build-arg MYARG=1 myservice : echo built myservice" \ - "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" stub buildkite-agent \ @@ -267,7 +267,7 @@ cmd3" stub docker-compose \ "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" stub buildkite-agent \ @@ -298,7 +298,7 @@ cmd3" stub docker-compose \ "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 -e \* -e \* --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with vars \${11} and \${13}" stub buildkite-agent \ @@ -328,7 +328,7 @@ cmd3" stub docker-compose \ "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" stub buildkite-agent \ @@ -355,7 +355,7 @@ cmd3" stub docker-compose \ "-f tests/composefiles/docker-compose.v2.0.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ - "-f tests/composefiles/docker-compose.v2.0.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f tests/composefiles/docker-compose.v2.0.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "-f tests/composefiles/docker-compose.v2.0.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" stub buildkite-agent \ @@ -383,7 +383,7 @@ export BUILDKITE_JOB_ID=1111 stub docker-compose \ "-f tests/composefiles/docker-compose.v2.0.yml -f tests/composefiles/docker-compose.v2.1.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ - "-f tests/composefiles/docker-compose.v2.0.yml -f tests/composefiles/docker-compose.v2.1.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 : echo pulled myservice" \ + "-f tests/composefiles/docker-compose.v2.0.yml -f tests/composefiles/docker-compose.v2.1.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo pulled myservice" \ "-f tests/composefiles/docker-compose.v2.0.yml -f tests/composefiles/docker-compose.v2.1.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" stub buildkite-agent \ @@ -410,7 +410,7 @@ export BUILDKITE_JOB_ID=1111 stub docker-compose \ "-f tests/composefiles/docker-compose.v2.0.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ - "-f tests/composefiles/docker-compose.v2.0.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f tests/composefiles/docker-compose.v2.0.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "-f tests/composefiles/docker-compose.v2.0.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" stub buildkite-agent \ @@ -1083,7 +1083,7 @@ export BUILDKITE_JOB_ID=1111 stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm --entrypoint 'my custom entrypoint' myservice : echo ran myservice" stub buildkite-agent \ @@ -1110,7 +1110,7 @@ export BUILDKITE_JOB_ID=1111 stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm -e BUILDKITE_JOB_ID -e BUILDKITE_BUILD_ID -e BUILDKITE_AGENT_ACCESS_TOKEN -v $BATS_MOCK_TMPDIR/bin/buildkite-agent:/usr/bin/buildkite-agent myservice : echo ran myservice" stub buildkite-agent \ @@ -1138,7 +1138,7 @@ export BUILDKITE_JOB_ID=1111 stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull --no-cache --parallel myservice : echo built myservice" \ - "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" stub buildkite-agent \ @@ -1165,7 +1165,7 @@ export BUILDKITE_JOB_ID=1111 stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 -v /tmp/sample-mirror:/tmp/sample-mirror:ro --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" stub buildkite-agent \ @@ -1193,7 +1193,7 @@ export BUILDKITE_JOB_ID=1111 stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm -e SSH_AUTH_SOCK=/ssh-agent -v /tmp/ssh_auth_sock:/ssh-agent -v /root/.ssh/known_hosts:/root/.ssh/known_hosts myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" stub buildkite-agent \ @@ -1422,7 +1422,7 @@ export BUILDKITE_JOB_ID=1111 stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "-f docker-compose.yml -p buildkite1111 up --wait -d --scale myservice=0 : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 up --wait -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" stub buildkite-agent \ diff --git a/tests/v2/run.bats b/tests/v2/run.bats index fe85c769..693faa5a 100644 --- a/tests/v2/run.bats +++ b/tests/v2/run.bats @@ -23,7 +23,7 @@ setup_file() { stub docker \ "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" stub buildkite-agent \ @@ -49,7 +49,7 @@ setup_file() { stub docker \ "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice : echo ran myservice" stub buildkite-agent \ @@ -76,7 +76,7 @@ setup_file() { stub docker \ "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --workdir=/test_workdir --rm myservice : echo ran myservice" stub buildkite-agent \ @@ -102,7 +102,7 @@ setup_file() { stub docker \ "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c $'sh -c \'echo hello world\'' : echo ran myservice" stub buildkite-agent \ @@ -130,7 +130,7 @@ cmd3" stub docker \ "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c $'cmd1\ncmd2\ncmd3' : echo ran myservice" stub buildkite-agent \ @@ -158,7 +158,7 @@ cmd3" stub docker \ "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice echo 'hello world' : echo ran myservice" stub buildkite-agent \ @@ -189,7 +189,7 @@ cmd3" stub docker \ "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 -e MYENV=0 -e MYENV -e MYENV=2 -e MYENV -e ANOTHER=this\ is\ a\ long\ string\ with\ spaces\;\ and\ semi-colons --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" stub buildkite-agent \ @@ -215,7 +215,7 @@ cmd3" stub docker \ "compose -f docker-compose.yml -p buildkite1111 build --pull --no-cache myservice : echo built myservice" \ - "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" stub buildkite-agent \ @@ -243,7 +243,7 @@ cmd3" stub docker \ "compose -f docker-compose.yml -p buildkite1111 build --pull --build-arg MYARG=0 --build-arg MYARG=1 myservice : echo built myservice" \ - "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" stub buildkite-agent \ @@ -268,7 +268,7 @@ cmd3" stub docker \ "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ - "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" stub buildkite-agent \ @@ -299,7 +299,7 @@ cmd3" stub docker \ "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ - "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 -e \* -e \* --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with vars \${12} and \${14}" stub buildkite-agent \ @@ -329,7 +329,7 @@ cmd3" stub docker \ "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ - "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" stub buildkite-agent \ @@ -356,7 +356,7 @@ cmd3" stub docker \ "compose -f tests/composefiles/docker-compose.v2.0.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ - "compose -f tests/composefiles/docker-compose.v2.0.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f tests/composefiles/docker-compose.v2.0.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "compose -f tests/composefiles/docker-compose.v2.0.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" stub buildkite-agent \ @@ -384,7 +384,7 @@ export BUILDKITE_JOB_ID=1111 stub docker \ "compose -f tests/composefiles/docker-compose.v2.0.yml -f tests/composefiles/docker-compose.v2.1.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ - "compose -f tests/composefiles/docker-compose.v2.0.yml -f tests/composefiles/docker-compose.v2.1.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 : echo pulled myservice" \ + "compose -f tests/composefiles/docker-compose.v2.0.yml -f tests/composefiles/docker-compose.v2.1.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo pulled myservice" \ "compose -f tests/composefiles/docker-compose.v2.0.yml -f tests/composefiles/docker-compose.v2.1.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" stub buildkite-agent \ @@ -411,7 +411,7 @@ export BUILDKITE_JOB_ID=1111 stub docker \ "compose -f tests/composefiles/docker-compose.v2.0.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ - "compose -f tests/composefiles/docker-compose.v2.0.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f tests/composefiles/docker-compose.v2.0.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "compose -f tests/composefiles/docker-compose.v2.0.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" stub buildkite-agent \ @@ -955,7 +955,7 @@ export BUILDKITE_JOB_ID=1111 stub docker \ "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm --entrypoint 'my custom entrypoint' myservice : echo ran myservice" stub buildkite-agent \ @@ -982,7 +982,7 @@ export BUILDKITE_JOB_ID=1111 stub docker \ "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm -e BUILDKITE_JOB_ID -e BUILDKITE_BUILD_ID -e BUILDKITE_AGENT_ACCESS_TOKEN -v $BATS_MOCK_TMPDIR/bin/buildkite-agent:/usr/bin/buildkite-agent myservice : echo ran myservice" stub buildkite-agent \ @@ -1010,7 +1010,7 @@ export BUILDKITE_JOB_ID=1111 stub docker \ "compose -f docker-compose.yml -p buildkite1111 build --pull --no-cache --parallel myservice : echo built myservice" \ - "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" stub buildkite-agent \ @@ -1037,7 +1037,7 @@ export BUILDKITE_JOB_ID=1111 stub docker \ "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 -v /tmp/sample-mirror:/tmp/sample-mirror:ro --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" stub buildkite-agent \ @@ -1065,7 +1065,7 @@ export BUILDKITE_JOB_ID=1111 stub docker \ "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm -e SSH_AUTH_SOCK=/ssh-agent -v /tmp/ssh_auth_sock:/ssh-agent -v /root/.ssh/known_hosts:/root/.ssh/known_hosts myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" stub buildkite-agent \ @@ -1300,7 +1300,7 @@ export BUILDKITE_JOB_ID=1111 stub docker \ "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "compose -f docker-compose.yml -p buildkite1111 up --wait -d --scale myservice=0 : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 up --wait -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" stub buildkite-agent \ From cbd1288ec254caf6756a546c85a5a4b7f0c2d129 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 10 Jan 2023 21:37:07 -0300 Subject: [PATCH 179/253] Update tester version --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0ed315f5..2a9c3a38 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ version: '2' services: tests: - image: buildkite/plugin-tester:v3.0.1 + image: buildkite/plugin-tester:v4.0.0 volumes: - ".:/plugin" From fe8739a7f40c833451ae64e5f94764311cad3ce5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 30 Jan 2023 20:45:10 -0300 Subject: [PATCH 180/253] Added ability to change the known_hosts mount path --- commands/run.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/commands/run.sh b/commands/run.sh index 9ea24ab7..005899eb 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -8,7 +8,6 @@ run_service="$(plugin_read_config RUN)" container_name="$(docker_compose_project_name)_${run_service}_build_${BUILDKITE_BUILD_NUMBER}" override_file="docker-compose.buildkite-${BUILDKITE_BUILD_NUMBER}-override.yml" pull_retries="$(plugin_read_config PULL_RETRIES "0")" -mount_ssh_agent='' mount_checkout="$(plugin_read_config MOUNT_CHECKOUT "false")" workdir='' @@ -193,23 +192,26 @@ if [[ -n "$(plugin_read_config ENTRYPOINT)" ]] ; then fi # Mount ssh-agent socket and known_hosts -if [[ "${BUILDKITE_PLUGIN_DOCKER_COMPOSE_MOUNT_SSH_AGENT:-$mount_ssh_agent}" =~ ^(true|on|1)$ ]] ; then +if [[ ! "${BUILDKITE_PLUGIN_DOCKER_COMPOSE_MOUNT_SSH_AGENT:-false}" = 'false' ]] ; then if [[ -z "${SSH_AUTH_SOCK:-}" ]] ; then echo "+++ 🚨 \$SSH_AUTH_SOCK isn't set, has ssh-agent started?" exit 1 fi if [[ ! -S "${SSH_AUTH_SOCK}" ]] ; then - echo "+++ 🚨 There isn't any file at ${SSH_AUTH_SOCK}, has ssh-agent started?" + echo "+++ 🚨 The file at ${SSH_AUTH_SOCK} does not exist or is not a socket, was ssh-agent started?" exit 1 fi - if [[ ! -S "${SSH_AUTH_SOCK}" ]] ; then - echo "+++ 🚨 The file at ${SSH_AUTH_SOCK} isn't a socket, has ssh-agent started?" - exit 1 + + if [[ "${BUILDKITE_PLUGIN_DOCKER_COMPOSE_MOUNT_SSH_AGENT:-''}" =~ ^(true|on|1)$ ]]; then + MOUNT_PATH=/root + else + MOUNT_PATH="${BUILDKITE_PLUGIN_DOCKER_COMPOSE_MOUNT_SSH_AGENT}" fi + run_params+=( "-e" "SSH_AUTH_SOCK=/ssh-agent" "-v" "${SSH_AUTH_SOCK}:/ssh-agent" - "-v" "${HOME}/.ssh/known_hosts:/root/.ssh/known_hosts" + "-v" "${HOME}/.ssh/known_hosts:${MOUNT_PATH}/.ssh/known_hosts" ) fi From 17fc1336efae1818d4512b44fcf2dad88e6981c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 30 Jan 2023 20:45:29 -0300 Subject: [PATCH 181/253] Updated option in plugin.yml and documentation --- README.md | 4 ++-- plugin.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 82d3bf0e..c99fb455 100644 --- a/README.md +++ b/README.md @@ -486,9 +486,9 @@ Whether to match the user ID and group ID for the container user to the user ID Using this option ensures that any files created on shared mounts from within the container will be accessible to the host user. It is otherwise common to accidentally create root-owned files that Buildkite will be unable to remove, since containers by default run as the root user. -### `mount-ssh-agent` (optional, run-only, boolean) +### `mount-ssh-agent` (optional, run-only, boolean or string) -Whether to automatically mount the ssh-agent socket from the host agent machine into the container (at `/ssh-agent`and `/root/.ssh/known_hosts` respectively), allowing git operations to work correctly. +Whether to mount the ssh-agent socket (at `/ssh-agent`) from the host agent machine into the container or not. Instead of just `true` or `false`, you can specify absolute path in the container for the home directory of the user used to run on which the agent's `.ssh/known_hosts` will be mounted (by default, `/root`). Default: `false` diff --git a/plugin.yml b/plugin.yml index 78525af7..3455f4d3 100644 --- a/plugin.yml +++ b/plugin.yml @@ -60,7 +60,7 @@ configuration: mount-buildkite-agent: type: boolean mount-ssh-agent: - type: boolean + type: [ boolean, string ] mount-checkout: type: [ boolean, string ] no-cache: From 57e3563b5e9dfc08ef7fb45076596d1be8a8ba75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 30 Jan 2023 20:45:47 -0300 Subject: [PATCH 182/253] Added new tests for ssh-agent option variations --- tests/run.bats | 35 +++++++++++++++++++++++++++++++++++ tests/v2/run.bats | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/tests/run.bats b/tests/run.bats index 124f16c3..05ba0845 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -1204,9 +1204,44 @@ export BUILDKITE_JOB_ID=1111 run "$PWD"/hooks/command + assert_success + kill %1 + assert_output --partial "built myservice" + assert_output --partial "ran myservice" + unstub docker-compose + unstub buildkite-agent +} + +@test "Run with mount-ssh-agent on particular folder" { + export SSH_AUTH_SOCK=/tmp/ssh_auth_sock + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="echo hello world" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_MOUNT_SSH_AGENT=/tmp/test + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm -e SSH_AUTH_SOCK=/ssh-agent -v /tmp/ssh_auth_sock:/ssh-agent -v /root/.ssh/known_hosts:/tmp/test/.ssh/known_hosts myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + apk add netcat-openbsd + nc -lkvU $SSH_AUTH_SOCK & + + run "$PWD"/hooks/command + assert_success + + kill %1 + assert_output --partial "built myservice" assert_output --partial "ran myservice" unstub docker-compose diff --git a/tests/v2/run.bats b/tests/v2/run.bats index 693faa5a..f859dc88 100644 --- a/tests/v2/run.bats +++ b/tests/v2/run.bats @@ -1085,6 +1085,39 @@ export BUILDKITE_JOB_ID=1111 unstub buildkite-agent } +@test "Run with mount-ssh-agent on particular folder" { + export SSH_AUTH_SOCK=/tmp/ssh_auth_sock + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="echo hello world" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_MOUNT_SSH_AGENT=/tmp/test + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm -e SSH_AUTH_SOCK=/ssh-agent -v /tmp/ssh_auth_sock:/ssh-agent -v /root/.ssh/known_hosts:/tmp/test/.ssh/known_hosts myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + apk add netcat-openbsd + nc -lkvU $SSH_AUTH_SOCK & + + run "$PWD"/hooks/command + + kill %1 + + assert_success + assert_output --partial "built myservice" + assert_output --partial "ran myservice" + unstub docker + unstub buildkite-agent +} + @test "Run without mount-checkout doesn't set volume" { export BUILDKITE_BUILD_NUMBER=1 export BUILDKITE_JOB_ID=1111 From 8d016790658a5e07f22c7b151b7f8c10a87455aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 30 Jan 2023 21:34:33 -0300 Subject: [PATCH 183/253] Update version in readme in preparation for next release --- README.md | 54 +++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index c99fb455..d3770cea 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v4.9.0: + - docker-compose#v4.10.0: run: app ``` @@ -28,7 +28,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v4.9.0: + - docker-compose#v4.10.0: run: app config: docker-compose.tests.yml env: @@ -41,7 +41,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v4.9.0: + - docker-compose#v4.10.0: run: app config: - docker-compose.yml @@ -56,7 +56,7 @@ env: steps: - command: test.sh plugins: - - docker-compose#v4.9.0: + - docker-compose#v4.10.0: run: app ``` @@ -65,7 +65,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v4.9.0: + - docker-compose#v4.10.0: run: app command: ["custom", "command", "values"] ``` @@ -79,7 +79,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.9.0: + - docker-compose#v4.10.0: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -87,7 +87,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.9.0: + - docker-compose#v4.10.0: run: app ``` @@ -104,7 +104,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.9.0: + - docker-compose#v4.10.0: run: app ``` @@ -122,7 +122,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.9.0: + - docker-compose#v4.10.0: run: app volumes: - "./dist:/app/dist" @@ -146,7 +146,7 @@ this plugin offers a `environment` block of its own: steps: - command: generate-dist.sh plugins: - - docker-compose#v4.9.0: + - docker-compose#v4.10.0: run: app env: - BUILDKITE_BUILD_NUMBER @@ -164,7 +164,7 @@ Alternatively, you can have the plugin add all environment variables defined for steps: - command: use-vars.sh plugins: - - docker-compose#v4.9.0: + - docker-compose#v4.10.0: run: app propagate-environment: true ``` @@ -179,7 +179,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v4.9.0: + - docker-compose#v4.10.0: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -196,7 +196,7 @@ If you have multiple steps that use the same service/image (such as steps that r steps: - label: ":docker: Build" plugins: - - docker-compose#v4.9.0: + - docker-compose#v4.10.0: build: app image-repository: index.docker.io/myorg/myrepo @@ -206,7 +206,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.9.0: + - docker-compose#v4.10.0: run: app ``` @@ -222,7 +222,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v4.9.0: + - docker-compose#v4.10.0: build: - app - tests @@ -234,7 +234,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.9.0: + - docker-compose#v4.10.0: run: tests ``` @@ -246,7 +246,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v4.9.0: + - docker-compose#v4.10.0: push: app ``` @@ -256,7 +256,7 @@ To push multiple images, you can use a list: steps: - label: ":docker: Push" plugins: - - docker-compose#v4.9.0: + - docker-compose#v4.10.0: push: - first-service - second-service @@ -268,7 +268,7 @@ If you want to push to a specific location (that's not defined as the `image` in steps: - label: ":docker: Push" plugins: - - docker-compose#v4.9.0: + - docker-compose#v4.10.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -282,14 +282,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker: Build an image" plugins: - - docker-compose#v4.9.0: + - docker-compose#v4.10.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.9.0: + - docker-compose#v4.10.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -307,7 +307,7 @@ This plugin allows for the value of `cache-from` to be a string or a list. If it steps: - label: ":docker Build an image" plugins: - - docker-compose#v4.9.0: + - docker-compose#v4.10.0: build: app image-repository: index.docker.io/myorg/myrepo separator-cache-from: "#" @@ -317,7 +317,7 @@ steps: - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.9.0: + - docker-compose#v4.10.0: push: - app:myregistry:port/myrepo/myapp:my-branch - app:myregistry:port/myrepo/myapp:latest @@ -330,7 +330,7 @@ Adding a grouping tag to the end of a cache-from list item allows this plugin to steps: - label: ":docker: Build Intermediate Image" plugins: - - docker-compose#v4.9.0: + - docker-compose#v4.10.0: build: myservice_intermediate # docker-compose.yml is the same as myservice but has `target: intermediate` image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo/myservice_intermediate @@ -340,7 +340,7 @@ steps: - wait - label: ":docker: Build Final Image" plugins: - - docker-compose#v4.9.0: + - docker-compose#v4.10.0: build: myservice image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo @@ -384,7 +384,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Run & Push" plugins: - - docker-compose#v4.9.0: + - docker-compose#v4.10.0: run: myservice push: myservice ``` @@ -399,7 +399,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Build & Push" plugins: - - docker-compose#v4.9.0: + - docker-compose#v4.10.0: build: myservice push: myservice ``` From 889a474a099d044b2f75cb0d0e6c088e22b109a4 Mon Sep 17 00:00:00 2001 From: paul <423357+toothbrush@users.noreply.github.com> Date: Tue, 21 Feb 2023 11:02:34 +1100 Subject: [PATCH 184/253] README: remove extraneous space. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d3770cea..57410f52 100644 --- a/README.md +++ b/README.md @@ -464,7 +464,7 @@ Example: `[ "powershell", "-Command" ]` Whether to skip the repository checkout phase. This is useful for steps that use a pre-built image and will fail if there is no pre-built image. -**Important**: as the code repository will not be available in the step, you need to ensure that the docker compose file(s) are present in some way (like using artifacts) +**Important**: as the code repository will not be available in the step, you need to ensure that the docker compose file(s) are present in some way (like using artifacts) ### `skip-pull` (optional, run only) From b54be7bd3a5ab48ce7407abee42f5a8df4c6cc1f Mon Sep 17 00:00:00 2001 From: paul david <423357+toothbrush@users.noreply.github.com> Date: Mon, 20 Feb 2023 15:16:50 +1100 Subject: [PATCH 185/253] cli-version: Support string or integer. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matías Bellone --- plugin.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/plugin.yml b/plugin.yml index 3455f4d3..fd36d20c 100644 --- a/plugin.yml +++ b/plugin.yml @@ -30,10 +30,11 @@ configuration: compatibility: type: boolean cli-version: - type: string - enum: - - 1 - - 2 + oneOf: + - type: string + enum: [ "1", "2" ] + - type: integer + enum: [ 1, 2 ] command: type: array config: From 6682e2f24751e5a5bc017cd6af833a912bdaf830 Mon Sep 17 00:00:00 2001 From: paul <423357+toothbrush@users.noreply.github.com> Date: Mon, 20 Feb 2023 16:31:46 +1100 Subject: [PATCH 186/253] README(cli-version): Document that we can use int or string. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 57410f52..d3f89f8f 100644 --- a/README.md +++ b/README.md @@ -650,9 +650,9 @@ Select when to upload container logs. The default is `on-error`. -### `cli-version` (optional) +### `cli-version` (optional, string or integer) -If set to `2`, plugin will use `docker compose` to execute commands; otherwise it will default to version `1` using `docker-compose` instead. +If set to `2`, plugin will use `docker compose` to execute commands; otherwise it will default to version `1`, using `docker-compose` instead. ### `buildkit` (optional, build only, boolean) From d339795acdd9a6a2faeb61ac28b002f6a56db268 Mon Sep 17 00:00:00 2001 From: paul <423357+toothbrush@users.noreply.github.com> Date: Tue, 21 Feb 2023 11:05:04 +1100 Subject: [PATCH 187/253] README: Bump to v4.10.1 in preparation of release. --- README.md | 54 +++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index d3f89f8f..dde8136c 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v4.10.0: + - docker-compose#v4.10.1: run: app ``` @@ -28,7 +28,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v4.10.0: + - docker-compose#v4.10.1: run: app config: docker-compose.tests.yml env: @@ -41,7 +41,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v4.10.0: + - docker-compose#v4.10.1: run: app config: - docker-compose.yml @@ -56,7 +56,7 @@ env: steps: - command: test.sh plugins: - - docker-compose#v4.10.0: + - docker-compose#v4.10.1: run: app ``` @@ -65,7 +65,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v4.10.0: + - docker-compose#v4.10.1: run: app command: ["custom", "command", "values"] ``` @@ -79,7 +79,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.10.0: + - docker-compose#v4.10.1: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -87,7 +87,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.10.0: + - docker-compose#v4.10.1: run: app ``` @@ -104,7 +104,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.10.0: + - docker-compose#v4.10.1: run: app ``` @@ -122,7 +122,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.10.0: + - docker-compose#v4.10.1: run: app volumes: - "./dist:/app/dist" @@ -146,7 +146,7 @@ this plugin offers a `environment` block of its own: steps: - command: generate-dist.sh plugins: - - docker-compose#v4.10.0: + - docker-compose#v4.10.1: run: app env: - BUILDKITE_BUILD_NUMBER @@ -164,7 +164,7 @@ Alternatively, you can have the plugin add all environment variables defined for steps: - command: use-vars.sh plugins: - - docker-compose#v4.10.0: + - docker-compose#v4.10.1: run: app propagate-environment: true ``` @@ -179,7 +179,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v4.10.0: + - docker-compose#v4.10.1: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -196,7 +196,7 @@ If you have multiple steps that use the same service/image (such as steps that r steps: - label: ":docker: Build" plugins: - - docker-compose#v4.10.0: + - docker-compose#v4.10.1: build: app image-repository: index.docker.io/myorg/myrepo @@ -206,7 +206,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.10.0: + - docker-compose#v4.10.1: run: app ``` @@ -222,7 +222,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v4.10.0: + - docker-compose#v4.10.1: build: - app - tests @@ -234,7 +234,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.10.0: + - docker-compose#v4.10.1: run: tests ``` @@ -246,7 +246,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v4.10.0: + - docker-compose#v4.10.1: push: app ``` @@ -256,7 +256,7 @@ To push multiple images, you can use a list: steps: - label: ":docker: Push" plugins: - - docker-compose#v4.10.0: + - docker-compose#v4.10.1: push: - first-service - second-service @@ -268,7 +268,7 @@ If you want to push to a specific location (that's not defined as the `image` in steps: - label: ":docker: Push" plugins: - - docker-compose#v4.10.0: + - docker-compose#v4.10.1: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -282,14 +282,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker: Build an image" plugins: - - docker-compose#v4.10.0: + - docker-compose#v4.10.1: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.10.0: + - docker-compose#v4.10.1: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -307,7 +307,7 @@ This plugin allows for the value of `cache-from` to be a string or a list. If it steps: - label: ":docker Build an image" plugins: - - docker-compose#v4.10.0: + - docker-compose#v4.10.1: build: app image-repository: index.docker.io/myorg/myrepo separator-cache-from: "#" @@ -317,7 +317,7 @@ steps: - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.10.0: + - docker-compose#v4.10.1: push: - app:myregistry:port/myrepo/myapp:my-branch - app:myregistry:port/myrepo/myapp:latest @@ -330,7 +330,7 @@ Adding a grouping tag to the end of a cache-from list item allows this plugin to steps: - label: ":docker: Build Intermediate Image" plugins: - - docker-compose#v4.10.0: + - docker-compose#v4.10.1: build: myservice_intermediate # docker-compose.yml is the same as myservice but has `target: intermediate` image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo/myservice_intermediate @@ -340,7 +340,7 @@ steps: - wait - label: ":docker: Build Final Image" plugins: - - docker-compose#v4.10.0: + - docker-compose#v4.10.1: build: myservice image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo @@ -384,7 +384,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Run & Push" plugins: - - docker-compose#v4.10.0: + - docker-compose#v4.10.1: run: myservice push: myservice ``` @@ -399,7 +399,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Build & Push" plugins: - - docker-compose#v4.10.0: + - docker-compose#v4.10.1: build: myservice push: myservice ``` From 85b933209297e22a0701a5eccaafa62820ed7ef9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 22 Feb 2023 04:33:36 +0000 Subject: [PATCH 188/253] Update buildkite plugin plugin-linter to v3.1.0 --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 71546977..101e89b9 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -15,7 +15,7 @@ steps: - label: ":sparkles: Lint" plugins: - plugin-linter#v3.0.0: + plugin-linter#v3.1.0: id: docker-compose - label: ":bash: Tests" From aa41ab7ff585eb0dd863017a519ca48fbc8b7476 Mon Sep 17 00:00:00 2001 From: Francois Campbell Date: Mon, 27 Feb 2023 13:17:19 -0500 Subject: [PATCH 189/253] Add labels to run container --- README.md | 13 +++++++++++++ commands/run.sh | 12 ++++++++++++ 2 files changed, 25 insertions(+) diff --git a/README.md b/README.md index dde8136c..4ba3d62b 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,19 @@ steps: propagate-environment: true ``` +## Container Labels + +When running a command, the plugin will automatically use add the following Docker labels to the container specified in the `run` option: +- `com.buildkite.pipeline_name=${BUILDKITE_PIPELINE_NAME}` +- `com.buildkite.pipeline_slug=${BUILDKITE_PIPELINE_SLUG}` +- `com.buildkite.build_number=${BUILDKITE_BUILD_NUMBER}` +- `com.buildkite.job_id=${BUILDKITE_JOB_ID}` +- `com.buildkite.job_label=${BUILDKITE_LABEL}` +- `com.buildkite.step_key=${BUILDKITE_STEP_KEY}` +- `com.buildkite.agent_name=${BUILDKITE_AGENT_NAME}` +- `com.buildkite.agent_id=${BUILDKITE_AGENT_ID}` + + ## Build Arguments You can use the [build args key in docker-compose.yml](https://docs.docker.com/compose/compose-file/#args) to set specific build arguments when building an image. diff --git a/commands/run.sh b/commands/run.sh index 005899eb..c58c7454 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -78,6 +78,18 @@ fi # We set a predictable container name so we can find it and inspect it later on run_params+=("run" "--name" "$container_name") +# Add useful labels to run container +run_params+=( + "--label" "com.buildkite.pipeline_name=${BUILDKITE_PIPELINE_NAME}" + "--label" "com.buildkite.pipeline_slug=${BUILDKITE_PIPELINE_SLUG}" + "--label" "com.buildkite.build_number=${BUILDKITE_BUILD_NUMBER}" + "--label" "com.buildkite.job_id=${BUILDKITE_JOB_ID}" + "--label" "com.buildkite.job_label=${BUILDKITE_LABEL}" + "--label" "com.buildkite.step_key=${BUILDKITE_STEP_KEY}" + "--label" "com.buildkite.agent_name=${BUILDKITE_AGENT_NAME}" + "--label" "com.buildkite.agent_id=${BUILDKITE_AGENT_ID}" +) + # append env vars provided in ENV or ENVIRONMENT, these are newline delimited while IFS=$'\n' read -r env ; do [[ -n "${env:-}" ]] && run_params+=("-e" "${env}") From 547d80981f3ad35b071113d4367ecdad6424ddfc Mon Sep 17 00:00:00 2001 From: Francois Campbell Date: Mon, 27 Feb 2023 13:21:00 -0500 Subject: [PATCH 190/253] add example --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 4ba3d62b..005cc0cc 100644 --- a/README.md +++ b/README.md @@ -181,6 +181,11 @@ When running a command, the plugin will automatically use add the following Dock - `com.buildkite.agent_name=${BUILDKITE_AGENT_NAME}` - `com.buildkite.agent_id=${BUILDKITE_AGENT_ID}` +These labels can make it easier to query containers on hosts using `docker ps` for example: + +```bash +docker ps --filter "label=com.buildkite.job_label=Run tests" +``` ## Build Arguments From 335479c9abff0e37b737a03b5eb90f62f93bbb07 Mon Sep 17 00:00:00 2001 From: Francois Campbell Date: Mon, 27 Feb 2023 13:37:48 -0500 Subject: [PATCH 191/253] fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 005cc0cc..494741a9 100644 --- a/README.md +++ b/README.md @@ -171,7 +171,7 @@ steps: ## Container Labels -When running a command, the plugin will automatically use add the following Docker labels to the container specified in the `run` option: +When running a command, the plugin will automatically add the following Docker labels to the container specified in the `run` option: - `com.buildkite.pipeline_name=${BUILDKITE_PIPELINE_NAME}` - `com.buildkite.pipeline_slug=${BUILDKITE_PIPELINE_SLUG}` - `com.buildkite.build_number=${BUILDKITE_BUILD_NUMBER}` From 7c29ccb116b7b6ca1768f50e9227216051c430fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Bellone?= Date: Thu, 2 Mar 2023 02:02:09 -0300 Subject: [PATCH 192/253] Add clarifications on volume variables --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index dde8136c..b9e58e7f 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,7 @@ steps: - "./dist:/app/dist" ``` -If you want to use environment variables in the `volumes` element, you will need to activate the (unsafe) option `expand-volume-vars`. +If you want to use environment variables in the `volumes` element, you will need to activate the (unsafe) option `expand-volume-vars` (and most likely escape it using `$$VARIABLE_NAME`). ## Environment @@ -548,6 +548,8 @@ When set to true, it will activate interpolation of variables in the elements of :warning: **Important:** this is considered an unsafe option as the most compatible way to achieve this is to run the strings through `eval` which could lead to arbitrary code execution or information leaking if you don't have complete control of the pipeline +Note that rules regarding [environment variable interpolation](https://buildkite.com/docs/pipelines/environment-variables#runtime-variable-interpolation) apply here. That means that `$VARIABLE_NAME` is resolved at pipeline upload time, whereas `$$VARIABLE_NAME` will be at run time. All things being equal, you likely want to use `$$VARIABLE_NAME` on the variables mentioned in this option. + ### `graceful-shutdown` (optional, run only) Gracefully shuts down all containers via 'docker-compose stop`. From b407ce048b04e3ed16ddf0776daa5f1905a631b1 Mon Sep 17 00:00:00 2001 From: Francois Campbell Date: Fri, 3 Mar 2023 14:16:46 -0500 Subject: [PATCH 193/253] make it a flag --- commands/run.sh | 24 +++++++++++++----------- plugin.yml | 2 ++ tests/multiple-commands.bats | 14 ++++++++------ tests/output.bats | 3 +++ tests/push.bats | 4 ++++ tests/run.bats | 6 +++++- tests/v2/run.bats | 3 ++- 7 files changed, 37 insertions(+), 19 deletions(-) diff --git a/commands/run.sh b/commands/run.sh index c58c7454..51a49924 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -78,17 +78,19 @@ fi # We set a predictable container name so we can find it and inspect it later on run_params+=("run" "--name" "$container_name") -# Add useful labels to run container -run_params+=( - "--label" "com.buildkite.pipeline_name=${BUILDKITE_PIPELINE_NAME}" - "--label" "com.buildkite.pipeline_slug=${BUILDKITE_PIPELINE_SLUG}" - "--label" "com.buildkite.build_number=${BUILDKITE_BUILD_NUMBER}" - "--label" "com.buildkite.job_id=${BUILDKITE_JOB_ID}" - "--label" "com.buildkite.job_label=${BUILDKITE_LABEL}" - "--label" "com.buildkite.step_key=${BUILDKITE_STEP_KEY}" - "--label" "com.buildkite.agent_name=${BUILDKITE_AGENT_NAME}" - "--label" "com.buildkite.agent_id=${BUILDKITE_AGENT_ID}" -) +if [[ "$(plugin_read_config RUN_LABELS "true")" =~ ^(true|on|1)$ ]]; then + # Add useful labels to run container + run_params+=( + "--label" "com.buildkite.pipeline_name=${BUILDKITE_PIPELINE_NAME}" + "--label" "com.buildkite.pipeline_slug=${BUILDKITE_PIPELINE_SLUG}" + "--label" "com.buildkite.build_number=${BUILDKITE_BUILD_NUMBER}" + "--label" "com.buildkite.job_id=${BUILDKITE_JOB_ID}" + "--label" "com.buildkite.job_label=${BUILDKITE_LABEL}" + "--label" "com.buildkite.step_key=${BUILDKITE_STEP_KEY}" + "--label" "com.buildkite.agent_name=${BUILDKITE_AGENT_NAME}" + "--label" "com.buildkite.agent_id=${BUILDKITE_AGENT_ID}" + ) +fi # append env vars provided in ENV or ENVIRONMENT, these are newline delimited while IFS=$'\n' read -r env ; do diff --git a/plugin.yml b/plugin.yml index fd36d20c..9cf6dab7 100644 --- a/plugin.yml +++ b/plugin.yml @@ -81,6 +81,8 @@ configuration: type: integer rm: type: boolean + run-labels: + type: boolean separator-cache-from: type: string minLength: 1 diff --git a/tests/multiple-commands.bats b/tests/multiple-commands.bats index 39aa0736..7e2a71fa 100644 --- a/tests/multiple-commands.bats +++ b/tests/multiple-commands.bats @@ -8,12 +8,14 @@ load '../lib/metadata' # export BUILDKITE_AGENT_STUB_DEBUG=/dev/tty # export BATS_MOCK_TMPDIR=$PWD -# General pipeline variables -export BUILDKITE_BUILD_NUMBER=1 -export BUILDKITE_COMMAND="pwd" -export BUILDKITE_JOB_ID=12 -export BUILDKITE_PIPELINE_SLUG=test - +setup_file() { + # General pipeline variables + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="pwd" + export BUILDKITE_JOB_ID=12 + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN_LABELS="false" +} @test "Build and run" { export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice diff --git a/tests/output.bats b/tests/output.bats index a7952cb7..f2845553 100644 --- a/tests/output.bats +++ b/tests/output.bats @@ -9,6 +9,9 @@ load '../lib/run' # export DOCKER_STUB_DEBUG=/dev/tty # export BATS_MOCK_TMPDIR=$PWD +setup_file() { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN_LABELS="false" +} @test "Logs: Detect some containers KO" { export BUILDKITE_AGENT_ACCESS_TOKEN="123123" diff --git a/tests/push.bats b/tests/push.bats index cefe7996..0d71f238 100644 --- a/tests/push.bats +++ b/tests/push.bats @@ -8,6 +8,10 @@ load '../lib/shared' # export BUILDKITE_AGENT_STUB_DEBUG=/dev/tty # export BATS_MOCK_TMPDIR=$PWD +setup_file() { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN_LABELS="false" +} + @test "Push a single service with an image in it's config" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PUSH=app diff --git a/tests/run.bats b/tests/run.bats index 05ba0845..61bd6784 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -8,6 +8,10 @@ load '../lib/run' # export BUILDKITE_AGENT_STUB_DEBUG=/dev/tty # export BATS_MOCK_TMPDIR=$PWD +setup_file() { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN_LABELS="false" +} + @test "Run without a prebuilt image" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice @@ -1498,4 +1502,4 @@ export BUILDKITE_JOB_ID=1111 assert_output --partial "ran myservice without tty" unstub docker-compose unstub buildkite-agent -} \ No newline at end of file +} diff --git a/tests/v2/run.bats b/tests/v2/run.bats index f859dc88..3856736e 100644 --- a/tests/v2/run.bats +++ b/tests/v2/run.bats @@ -10,6 +10,7 @@ load '../../lib/run' setup_file() { export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLI_VERSION=2 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN_LABELS="false" } @test "Run without a prebuilt image" { @@ -1374,4 +1375,4 @@ export BUILDKITE_JOB_ID=1111 assert_output --partial "ran myservice without tty" unstub docker unstub buildkite-agent -} \ No newline at end of file +} From 9d628ebd199adfd0cc958f7ee8889d62f991c2ad Mon Sep 17 00:00:00 2001 From: Francois Campbell Date: Fri, 3 Mar 2023 15:42:21 -0500 Subject: [PATCH 194/253] Add test --- tests/run.bats | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/run.bats b/tests/run.bats index 61bd6784..0a1bf3f5 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -1503,3 +1503,42 @@ export BUILDKITE_JOB_ID=1111 unstub docker-compose unstub buildkite-agent } + +@test "Run with Docker labels" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN_LABELS="true" + export BUILDKITE_PIPELINE_NAME="label-test" + export BUILDKITE_LABEL="Testjob" + export BUILDKITE_STEP_KEY="test-job" + export BUILDKITE_AGENT_NAME="agent" + export BUILDKITE_AGENT_ID="1234" + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 \ + --label com.buildkite.pipeline_name=${BUILDKITE_PIPELINE_NAME} \ + --label com.buildkite.pipeline_slug=${BUILDKITE_PIPELINE_SLUG} \ + --label com.buildkite.build_number=${BUILDKITE_BUILD_NUMBER} \ + --label com.buildkite.job_id=${BUILDKITE_JOB_ID} \ + --label com.buildkite.job_label=${BUILDKITE_LABEL} \ + --label com.buildkite.step_key=${BUILDKITE_STEP_KEY} \ + --label com.buildkite.agent_name=${BUILDKITE_AGENT_NAME} \ + --label com.buildkite.agent_id=${BUILDKITE_AGENT_ID} \ + --rm myservice /bin/sh -e -c $'pwd' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : echo myimage" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "ran myservice" + unstub docker-compose + unstub buildkite-agent +} From f7909e603ad47e046a76e1a2e7523b054d735480 Mon Sep 17 00:00:00 2001 From: Francois Campbell Date: Fri, 3 Mar 2023 15:49:25 -0500 Subject: [PATCH 195/253] update readme to add flag --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 494741a9..61544fcd 100644 --- a/README.md +++ b/README.md @@ -187,6 +187,8 @@ These labels can make it easier to query containers on hosts using `docker ps` f docker ps --filter "label=com.buildkite.job_label=Run tests" ``` +This behaviour can be disabled with the `run-labels: false` option. + ## Build Arguments You can use the [build args key in docker-compose.yml](https://docs.docker.com/compose/compose-file/#args) to set specific build arguments when building an image. @@ -636,6 +638,12 @@ If set to true, docker compose will remove the primary container after run. Equi The default is `true`. +### `run-labels` (optional, run only) + +If set to true, adds useful Docker labels to the primary container. See [Container Labels](#container-labels) for more info. + +The default is `true`. + ### `compatibility` (optional, run only) If set to true, all docker compose commands will rum with compatibility mode. Equivalent to `--compatibility` in docker-compose. From 9eac919abb44d5363b0f4be58ce6fb4a577aceaa Mon Sep 17 00:00:00 2001 From: Francois Campbell Date: Mon, 6 Mar 2023 14:22:58 -0500 Subject: [PATCH 196/253] remove unnecessary setup_file, order env vars in test --- tests/push.bats | 4 ---- tests/run.bats | 17 ++++++++++------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/tests/push.bats b/tests/push.bats index 0d71f238..cefe7996 100644 --- a/tests/push.bats +++ b/tests/push.bats @@ -8,10 +8,6 @@ load '../lib/shared' # export BUILDKITE_AGENT_STUB_DEBUG=/dev/tty # export BATS_MOCK_TMPDIR=$PWD -setup_file() { - export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN_LABELS="false" -} - @test "Push a single service with an image in it's config" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PUSH=app diff --git a/tests/run.bats b/tests/run.bats index 0a1bf3f5..b0754d2b 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -1505,17 +1505,20 @@ export BUILDKITE_JOB_ID=1111 } @test "Run with Docker labels" { - export BUILDKITE_JOB_ID=1111 - export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice - export BUILDKITE_PIPELINE_SLUG=test + # Pipeline vars + export BUILDKITE_AGENT_ID="1234" + export BUILDKITE_AGENT_NAME="agent" export BUILDKITE_BUILD_NUMBER=1 export BUILDKITE_COMMAND=pwd - export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN_LABELS="true" - export BUILDKITE_PIPELINE_NAME="label-test" + export BUILDKITE_JOB_ID=1111 export BUILDKITE_LABEL="Testjob" + export BUILDKITE_PIPELINE_NAME="label-test" + export BUILDKITE_PIPELINE_SLUG=test export BUILDKITE_STEP_KEY="test-job" - export BUILDKITE_AGENT_NAME="agent" - export BUILDKITE_AGENT_ID="1234" + + # Plugin config + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN_LABELS="true" stub docker-compose \ "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ From eff1b6329c3fc6e11ce999dcc998c748304f382f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Bellone?= Date: Thu, 9 Mar 2023 18:51:17 -0300 Subject: [PATCH 197/253] Remove additional space Co-authored-by: Pol (Paula) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b9e58e7f..f40e581d 100644 --- a/README.md +++ b/README.md @@ -548,7 +548,7 @@ When set to true, it will activate interpolation of variables in the elements of :warning: **Important:** this is considered an unsafe option as the most compatible way to achieve this is to run the strings through `eval` which could lead to arbitrary code execution or information leaking if you don't have complete control of the pipeline -Note that rules regarding [environment variable interpolation](https://buildkite.com/docs/pipelines/environment-variables#runtime-variable-interpolation) apply here. That means that `$VARIABLE_NAME` is resolved at pipeline upload time, whereas `$$VARIABLE_NAME` will be at run time. All things being equal, you likely want to use `$$VARIABLE_NAME` on the variables mentioned in this option. +Note that rules regarding [environment variable interpolation](https://buildkite.com/docs/pipelines/environment-variables#runtime-variable-interpolation) apply here. That means that `$VARIABLE_NAME` is resolved at pipeline upload time, whereas `$$VARIABLE_NAME` will be at run time. All things being equal, you likely want to use `$$VARIABLE_NAME` on the variables mentioned in this option. ### `graceful-shutdown` (optional, run only) From 4bf5f3454ee08ab0cefaa571fad66613ad82584a Mon Sep 17 00:00:00 2001 From: "Pol (Paula)" Date: Thu, 9 Mar 2023 19:40:09 -0300 Subject: [PATCH 198/253] Update release version --- README.md | 54 +++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 91b8fce0..62df84ba 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v4.10.1: + - docker-compose#v4.11.0: run: app ``` @@ -28,7 +28,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v4.10.1: + - docker-compose#v4.11.0: run: app config: docker-compose.tests.yml env: @@ -41,7 +41,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v4.10.1: + - docker-compose#v4.11.0: run: app config: - docker-compose.yml @@ -56,7 +56,7 @@ env: steps: - command: test.sh plugins: - - docker-compose#v4.10.1: + - docker-compose#v4.11.0: run: app ``` @@ -65,7 +65,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v4.10.1: + - docker-compose#v4.11.0: run: app command: ["custom", "command", "values"] ``` @@ -79,7 +79,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.10.1: + - docker-compose#v4.11.0: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -87,7 +87,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.10.1: + - docker-compose#v4.11.0: run: app ``` @@ -104,7 +104,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.10.1: + - docker-compose#v4.11.0: run: app ``` @@ -122,7 +122,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.10.1: + - docker-compose#v4.11.0: run: app volumes: - "./dist:/app/dist" @@ -146,7 +146,7 @@ this plugin offers a `environment` block of its own: steps: - command: generate-dist.sh plugins: - - docker-compose#v4.10.1: + - docker-compose#v4.11.0: run: app env: - BUILDKITE_BUILD_NUMBER @@ -164,7 +164,7 @@ Alternatively, you can have the plugin add all environment variables defined for steps: - command: use-vars.sh plugins: - - docker-compose#v4.10.1: + - docker-compose#v4.11.0: run: app propagate-environment: true ``` @@ -199,7 +199,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v4.10.1: + - docker-compose#v4.11.0: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -216,7 +216,7 @@ If you have multiple steps that use the same service/image (such as steps that r steps: - label: ":docker: Build" plugins: - - docker-compose#v4.10.1: + - docker-compose#v4.11.0: build: app image-repository: index.docker.io/myorg/myrepo @@ -226,7 +226,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.10.1: + - docker-compose#v4.11.0: run: app ``` @@ -242,7 +242,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v4.10.1: + - docker-compose#v4.11.0: build: - app - tests @@ -254,7 +254,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.10.1: + - docker-compose#v4.11.0: run: tests ``` @@ -266,7 +266,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v4.10.1: + - docker-compose#v4.11.0: push: app ``` @@ -276,7 +276,7 @@ To push multiple images, you can use a list: steps: - label: ":docker: Push" plugins: - - docker-compose#v4.10.1: + - docker-compose#v4.11.0: push: - first-service - second-service @@ -288,7 +288,7 @@ If you want to push to a specific location (that's not defined as the `image` in steps: - label: ":docker: Push" plugins: - - docker-compose#v4.10.1: + - docker-compose#v4.11.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -302,14 +302,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker: Build an image" plugins: - - docker-compose#v4.10.1: + - docker-compose#v4.11.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.10.1: + - docker-compose#v4.11.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -327,7 +327,7 @@ This plugin allows for the value of `cache-from` to be a string or a list. If it steps: - label: ":docker Build an image" plugins: - - docker-compose#v4.10.1: + - docker-compose#v4.11.0: build: app image-repository: index.docker.io/myorg/myrepo separator-cache-from: "#" @@ -337,7 +337,7 @@ steps: - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.10.1: + - docker-compose#v4.11.0: push: - app:myregistry:port/myrepo/myapp:my-branch - app:myregistry:port/myrepo/myapp:latest @@ -350,7 +350,7 @@ Adding a grouping tag to the end of a cache-from list item allows this plugin to steps: - label: ":docker: Build Intermediate Image" plugins: - - docker-compose#v4.10.1: + - docker-compose#v4.11.0: build: myservice_intermediate # docker-compose.yml is the same as myservice but has `target: intermediate` image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo/myservice_intermediate @@ -360,7 +360,7 @@ steps: - wait - label: ":docker: Build Final Image" plugins: - - docker-compose#v4.10.1: + - docker-compose#v4.11.0: build: myservice image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo @@ -404,7 +404,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Run & Push" plugins: - - docker-compose#v4.10.1: + - docker-compose#v4.11.0: run: myservice push: myservice ``` @@ -419,7 +419,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Build & Push" plugins: - - docker-compose#v4.10.1: + - docker-compose#v4.11.0: build: myservice push: myservice ``` From 6a17807cd01edc90cc968650fdbe266eee62759b Mon Sep 17 00:00:00 2001 From: Lar Van Der Jagt Date: Tue, 4 Apr 2023 10:41:44 -0400 Subject: [PATCH 199/253] Update heading in README to reflect contents Looks like a copy/paste of the Run & Push headline didn't get updated for the Build & Push contents that follow. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 62df84ba..a9fb55da 100644 --- a/README.md +++ b/README.md @@ -411,7 +411,7 @@ steps: Will cause the image to be built twice (once before running and once before pushing) unless there was a previous `build` step that set the appropriate metadata. -##### Run & Push +##### Build & Push A basic pipeline similar to the following: From 4e08409cfaffd8e927f927a9d623cff4fe49c4e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 4 Apr 2023 15:41:29 -0300 Subject: [PATCH 200/253] Correct definition and documentation of ssh option --- README.md | 4 ++-- plugin.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a9fb55da..0c9b787f 100644 --- a/README.md +++ b/README.md @@ -688,9 +688,9 @@ Assuming you have a compatible docker installation and configuration in the agen You may want to also add `BUILDKIT_INLINE_CACHE=1` to your build arguments (`args` option in this plugin), but know that [there are known issues with it](https://github.com/moby/buildkit/issues/2274). -### `ssh` (optional, build only, boolean) +### `ssh` (optional, build only, boolean or string) -When enabled, it will add the `--ssh` option to the build command. Note that it assumes you have a compatible docker installation and configuration in the agent (meaning you are using BuildKit and it is correctly setup). +It will add the `--ssh` option to the build command with the passed value (if `true` it will use `default`). Note that it assumes you have a compatible docker installation and configuration in the agent (meaning you are using BuildKit and it is correctly setup). ### `secrets` (optional, build only, array of strings) diff --git a/plugin.yml b/plugin.yml index 9cf6dab7..631fc311 100644 --- a/plugin.yml +++ b/plugin.yml @@ -94,7 +94,7 @@ configuration: skip-pull: type: boolean ssh: - type: boolean + type: [ boolean, string ] secrets: type: array items: From b776c2b8ce6d8a1bc937e6a1522f4d8d44536729 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 4 Apr 2023 15:41:47 -0300 Subject: [PATCH 201/253] Implemented change in ssh option --- commands/build.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/commands/build.sh b/commands/build.sh index 0ad9a025..1a4f336f 100755 --- a/commands/build.sh +++ b/commands/build.sh @@ -163,12 +163,18 @@ while read -r line ; do [[ -n "$line" ]] && build_params+=("--secret" "$line") done <<< "$(plugin_read_list SECRETS)" -if [[ "$(plugin_read_config SSH "false")" == "true" ]] ; then +if [[ "$(plugin_read_config SSH "false")" != "false" ]] ; then if [[ "${DOCKER_BUILDKIT:-}" != "1" && "${BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLI_VERSION:-}" != "2" ]]; then echo "🚨 You can not use the ssh option if you are not using buildkit" exit 1 fi - build_params+=(--ssh) + + SSH_CONTEXT="$(plugin_read_config SSH)" + if [[ "${SSH_CONTEXT}" == "true" ]]; then + # ssh option was a boolean + SSH_CONTEXT='default' + fi + build_params+=(--ssh "${SSH_CONTEXT}") fi target="$(plugin_read_config TARGET "")" From 789f4f190d289918ec1da09c72b9446bbad36cbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 4 Apr 2023 15:42:01 -0300 Subject: [PATCH 202/253] Corrected tests (and added a new one) --- tests/build.bats | 25 +++++++++++++++++++++++-- tests/v2/build.bats | 23 +++++++++++++++++++++-- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/tests/build.bats b/tests/build.bats index 1b8b0ffa..074c9d0d 100644 --- a/tests/build.bats +++ b/tests/build.bats @@ -841,7 +841,7 @@ load '../lib/shared' refute_output --partial "built myservice" } -@test "Build with ssh option and buildkit" { +@test "Build with ssh option as true and buildkit" { export BUILDKITE_BUILD_NUMBER=1 export BUILDKITE_JOB_ID=1111 export BUILDKITE_PIPELINE_SLUG=test @@ -851,7 +851,28 @@ load '../lib/shared' export BUILDKITE_PLUGIN_DOCKER_COMPOSE_SSH=true stub docker-compose \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --ssh \* : echo built \${10} with ssh" + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --ssh default \* : echo built \${11} with ssh" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "with ssh" + + unstub docker-compose +} + +@test "Build with ssh option as string and buildkit" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILDKIT=true + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_SSH=context + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --ssh context \* : echo built \${11} with ssh" run "$PWD"/hooks/command diff --git a/tests/v2/build.bats b/tests/v2/build.bats index 9fc590e1..4189e190 100644 --- a/tests/v2/build.bats +++ b/tests/v2/build.bats @@ -633,7 +633,7 @@ setup_file() { unstub docker } -@test "Build with ssh option" { +@test "Build with ssh option as boolean" { export BUILDKITE_BUILD_NUMBER=1 export BUILDKITE_JOB_ID=1111 export BUILDKITE_PIPELINE_SLUG=test @@ -642,7 +642,7 @@ setup_file() { export BUILDKITE_PLUGIN_DOCKER_COMPOSE_SSH=true stub docker \ - "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --ssh \* : echo built \${11} with ssh" + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --ssh default \* : echo built \${11} with ssh" run "$PWD"/hooks/command @@ -653,6 +653,25 @@ setup_file() { unstub docker } +@test "Build with ssh option as string" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_SSH=context + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --ssh context \* : echo built \${11} with ssh" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "with ssh" + + unstub docker +} @test "Build with secrets" { export BUILDKITE_BUILD_NUMBER=1 From 27817aff191a60befd274f4483318b63e17a6042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 4 Apr 2023 15:48:43 -0300 Subject: [PATCH 203/253] Correct parameter used for image in v2 tests --- tests/v2/build.bats | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/v2/build.bats b/tests/v2/build.bats index 4189e190..4b6e961f 100644 --- a/tests/v2/build.bats +++ b/tests/v2/build.bats @@ -642,7 +642,7 @@ setup_file() { export BUILDKITE_PLUGIN_DOCKER_COMPOSE_SSH=true stub docker \ - "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --ssh default \* : echo built \${11} with ssh" + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --ssh default \* : echo built \${12} with ssh" run "$PWD"/hooks/command @@ -662,7 +662,7 @@ setup_file() { export BUILDKITE_PLUGIN_DOCKER_COMPOSE_SSH=context stub docker \ - "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --ssh context \* : echo built \${11} with ssh" + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --ssh context \* : echo built \${12} with ssh" run "$PWD"/hooks/command From c8a6cde98890ff1feed2f7d5776a30ab817aa8a5 Mon Sep 17 00:00:00 2001 From: Lar Van Der Jagt Date: Fri, 7 Apr 2023 14:52:37 -0400 Subject: [PATCH 204/253] Add skip-pull support to build phase Reuses the existing `skip-pull` option to conditionally remove the `--pull` flag from the generated docker compose command. --- README.md | 2 +- commands/build.sh | 6 +++++- plugin.yml | 2 +- tests/build.bats | 19 ++++++++++++++++++- tests/v2/build.bats | 19 ++++++++++++++++++- 5 files changed, 43 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a9fb55da..9de1b4fc 100644 --- a/README.md +++ b/README.md @@ -486,7 +486,7 @@ Whether to skip the repository checkout phase. This is useful for steps that use **Important**: as the code repository will not be available in the step, you need to ensure that the docker compose file(s) are present in some way (like using artifacts) -### `skip-pull` (optional, run only) +### `skip-pull` (optional, build and run only) Completely avoid running any `pull` command. Images being used will need to be present in the machine from before or have been built in the same step. Could be useful to avoid hitting rate limits when you can be sure the operation is unnecessary. Note that it is possible other commands run in the plugin's lifecycle will trigger a pull of necessary images. diff --git a/commands/build.sh b/commands/build.sh index 0ad9a025..ae6fb95f 100755 --- a/commands/build.sh +++ b/commands/build.sh @@ -148,7 +148,11 @@ while read -r line ; do [[ -n "$line" ]] && services+=("$line") done <<< "$(plugin_read_list BUILD)" -build_params=(build --pull) +build_params=(build) + +if [[ ! "$(plugin_read_config SKIP_PULL "false")" == "true" ]] ; then + build_params+=(--pull) +fi if [[ "$(plugin_read_config NO_CACHE "false")" == "true" ]] ; then build_params+=(--no-cache) diff --git a/plugin.yml b/plugin.yml index 9cf6dab7..0da0b270 100644 --- a/plugin.yml +++ b/plugin.yml @@ -141,7 +141,7 @@ configuration: pull: [ run ] push-retries: [ push ] service-ports: [ run ] - skip-pull: [ run ] + skip-pull: [ build, run ] secrets: [ buildkit, build ] ssh: [ buildkit ] target: [ build ] diff --git a/tests/build.bats b/tests/build.bats index 1b8b0ffa..144dce30 100644 --- a/tests/build.bats +++ b/tests/build.bats @@ -881,4 +881,21 @@ load '../lib/shared' assert_output --partial "with secrets id=test,file=~/.test and id=SECRET_VAR" unstub docker-compose -} \ No newline at end of file +} + +@test "Build without pull" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_SKIP_PULL=true + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build myservice : echo built myservice" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "built myservice" + unstub docker-compose +} diff --git a/tests/v2/build.bats b/tests/v2/build.bats index 9fc590e1..10f99913 100644 --- a/tests/v2/build.bats +++ b/tests/v2/build.bats @@ -673,4 +673,21 @@ setup_file() { assert_output --partial "with secrets id=test,file=~/.test and id=SECRET_VAR" unstub docker -} \ No newline at end of file +} + +@test "Build without pull" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_SKIP_PULL=true + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build myservice : echo built myservice" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "built myservice" + unstub docker +} From 384db3509e3c71a5a9ad50437221872b7e827254 Mon Sep 17 00:00:00 2001 From: "Pol (Paula)" Date: Mon, 10 Apr 2023 16:58:00 -0300 Subject: [PATCH 205/253] Update plugin version in README --- README.md | 54 +++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 577b2a3b..81a7fc80 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v4.11.0: + - docker-compose#v4.12.0: run: app ``` @@ -28,7 +28,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v4.11.0: + - docker-compose#v4.12.0: run: app config: docker-compose.tests.yml env: @@ -41,7 +41,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v4.11.0: + - docker-compose#v4.12.0: run: app config: - docker-compose.yml @@ -56,7 +56,7 @@ env: steps: - command: test.sh plugins: - - docker-compose#v4.11.0: + - docker-compose#v4.12.0: run: app ``` @@ -65,7 +65,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v4.11.0: + - docker-compose#v4.12.0: run: app command: ["custom", "command", "values"] ``` @@ -79,7 +79,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.11.0: + - docker-compose#v4.12.0: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -87,7 +87,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.11.0: + - docker-compose#v4.12.0: run: app ``` @@ -104,7 +104,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.11.0: + - docker-compose#v4.12.0: run: app ``` @@ -122,7 +122,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.11.0: + - docker-compose#v4.12.0: run: app volumes: - "./dist:/app/dist" @@ -146,7 +146,7 @@ this plugin offers a `environment` block of its own: steps: - command: generate-dist.sh plugins: - - docker-compose#v4.11.0: + - docker-compose#v4.12.0: run: app env: - BUILDKITE_BUILD_NUMBER @@ -164,7 +164,7 @@ Alternatively, you can have the plugin add all environment variables defined for steps: - command: use-vars.sh plugins: - - docker-compose#v4.11.0: + - docker-compose#v4.12.0: run: app propagate-environment: true ``` @@ -199,7 +199,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v4.11.0: + - docker-compose#v4.12.0: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -216,7 +216,7 @@ If you have multiple steps that use the same service/image (such as steps that r steps: - label: ":docker: Build" plugins: - - docker-compose#v4.11.0: + - docker-compose#v4.12.0: build: app image-repository: index.docker.io/myorg/myrepo @@ -226,7 +226,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.11.0: + - docker-compose#v4.12.0: run: app ``` @@ -242,7 +242,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v4.11.0: + - docker-compose#v4.12.0: build: - app - tests @@ -254,7 +254,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.11.0: + - docker-compose#v4.12.0: run: tests ``` @@ -266,7 +266,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v4.11.0: + - docker-compose#v4.12.0: push: app ``` @@ -276,7 +276,7 @@ To push multiple images, you can use a list: steps: - label: ":docker: Push" plugins: - - docker-compose#v4.11.0: + - docker-compose#v4.12.0: push: - first-service - second-service @@ -288,7 +288,7 @@ If you want to push to a specific location (that's not defined as the `image` in steps: - label: ":docker: Push" plugins: - - docker-compose#v4.11.0: + - docker-compose#v4.12.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -302,14 +302,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker: Build an image" plugins: - - docker-compose#v4.11.0: + - docker-compose#v4.12.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.11.0: + - docker-compose#v4.12.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -327,7 +327,7 @@ This plugin allows for the value of `cache-from` to be a string or a list. If it steps: - label: ":docker Build an image" plugins: - - docker-compose#v4.11.0: + - docker-compose#v4.12.0: build: app image-repository: index.docker.io/myorg/myrepo separator-cache-from: "#" @@ -337,7 +337,7 @@ steps: - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.11.0: + - docker-compose#v4.12.0: push: - app:myregistry:port/myrepo/myapp:my-branch - app:myregistry:port/myrepo/myapp:latest @@ -350,7 +350,7 @@ Adding a grouping tag to the end of a cache-from list item allows this plugin to steps: - label: ":docker: Build Intermediate Image" plugins: - - docker-compose#v4.11.0: + - docker-compose#v4.12.0: build: myservice_intermediate # docker-compose.yml is the same as myservice but has `target: intermediate` image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo/myservice_intermediate @@ -360,7 +360,7 @@ steps: - wait - label: ":docker: Build Final Image" plugins: - - docker-compose#v4.11.0: + - docker-compose#v4.12.0: build: myservice image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo @@ -404,7 +404,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Run & Push" plugins: - - docker-compose#v4.11.0: + - docker-compose#v4.12.0: run: myservice push: myservice ``` @@ -419,7 +419,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Build & Push" plugins: - - docker-compose#v4.11.0: + - docker-compose#v4.12.0: build: myservice push: myservice ``` From 08ea89cf9ebb7bc185a1f9ab50943fa26cdfe926 Mon Sep 17 00:00:00 2001 From: Songyu-Wang Date: Fri, 5 May 2023 00:49:05 -0500 Subject: [PATCH 206/253] This is a default message. I am too lazy to type anything --- README.md | 6 ++++++ commands/run.sh | 4 ++++ plugin.yml | 3 +++ tests/v2/run.bats | 29 +++++++++++++++++++++++++++++ 4 files changed, 42 insertions(+) diff --git a/README.md b/README.md index 81a7fc80..1d6918e5 100644 --- a/README.md +++ b/README.md @@ -634,6 +634,12 @@ Sets `docker-compose` to run with `--verbose` The default is `false`. +### `quiet-pull` (optional, run only) + +Sets `docker-compose up` to run with `--quiet-pull`. There are other commands that can accept `--quiet-pull` or equivalent. They have not been implemented yet. + +The default is `false`. + ### `rm` (optional, run only) If set to true, docker compose will remove the primary container after run. Equivalent to `--rm` in docker-compose. diff --git a/commands/run.sh b/commands/run.sh index 51a49924..5fdb1d8d 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -293,6 +293,10 @@ if [[ "$(plugin_read_config WAIT "false")" == "true" ]] ; then up_params+=("--wait") fi +if [[ "$(plugin_read_config QUIET_PULL "false")" == "true" ]] ; then + up_params+=("--quiet-pull") +fi + dependency_exitcode=0 if [[ "${run_dependencies}" == "true" ]] ; then # Start up service dependencies in a different header to keep the main run with less noise diff --git a/plugin.yml b/plugin.yml index 606936ff..441a037a 100644 --- a/plugin.yml +++ b/plugin.yml @@ -79,6 +79,8 @@ configuration: type: integer push-retries: type: integer + quiet-pull: + type: boolean rm: type: boolean run-labels: @@ -140,6 +142,7 @@ configuration: propagate-uid-gid: [ run ] pull: [ run ] push-retries: [ push ] + quiet-pull: [ run ] service-ports: [ run ] skip-pull: [ build, run ] secrets: [ buildkit, build ] diff --git a/tests/v2/run.bats b/tests/v2/run.bats index 3856736e..c835f38b 100644 --- a/tests/v2/run.bats +++ b/tests/v2/run.bats @@ -1376,3 +1376,32 @@ export BUILDKITE_JOB_ID=1111 unstub docker unstub buildkite-agent } + +@test "Run with --quiet-pull" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="echo hello world" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_WAIT=true + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "compose -f docker-compose.yml -p buildkite1111 up --quiet-pull -d --scale myservice=0 myservice : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "built myservice" + refute_output --partial "Pulling" + assert_output --partial "ran myservice" + + unstub docker + unstub buildkite-agent +} \ No newline at end of file From 70ca4a8123dec5354a7a70cdaaba042736ec6663 Mon Sep 17 00:00:00 2001 From: Songyu-Wang Date: Fri, 5 May 2023 00:52:34 -0500 Subject: [PATCH 207/253] Update run.bats --- tests/v2/run.bats | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/v2/run.bats b/tests/v2/run.bats index c835f38b..e2ddca86 100644 --- a/tests/v2/run.bats +++ b/tests/v2/run.bats @@ -1385,7 +1385,7 @@ export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false - export BUILDKITE_PLUGIN_DOCKER_COMPOSE_WAIT=true + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_QUIET_PULL=true stub docker \ "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ @@ -1404,4 +1404,4 @@ export BUILDKITE_JOB_ID=1111 unstub docker unstub buildkite-agent -} \ No newline at end of file +} From 192e661ae5a185598e1879bec2fdd48eaed20d27 Mon Sep 17 00:00:00 2001 From: Songyu-Wang Date: Fri, 5 May 2023 00:58:19 -0500 Subject: [PATCH 208/253] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1d6918e5..ccb513c5 100644 --- a/README.md +++ b/README.md @@ -636,7 +636,7 @@ The default is `false`. ### `quiet-pull` (optional, run only) -Sets `docker-compose up` to run with `--quiet-pull`. There are other commands that can accept `--quiet-pull` or equivalent. They have not been implemented yet. +Sets `docker-compose up` to run with `--quiet-pull`. There are other commands that can accept `--quiet-pull` or equivalents. They have not been implemented yet. The default is `false`. From 3bf8c9c3666a028fd521da4bec1f8136a26a740b Mon Sep 17 00:00:00 2001 From: Alex Martani Date: Fri, 5 May 2023 10:18:22 -0700 Subject: [PATCH 209/253] `env-propagation-list`: programmatically propagate envvars to the container --- README.md | 6 +++++- commands/run.sh | 13 +++++++++++++ plugin.yml | 2 ++ tests/run.bats | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 81a7fc80..744865e8 100644 --- a/README.md +++ b/README.md @@ -460,11 +460,15 @@ A list of KEY=VALUE that are passed through as build arguments when image is bei A list of either KEY or KEY=VALUE that are passed through as environment variables to the container. +### `env-propagation-list` (optional, string) + +If you set this to `VALUE`, and `VALUE` is an environment variable containing a space-separated list of environment variables such as `A B C D`, then A, B, C, and D will all be propagated to the container. This is helpful when you've set up an `environment` hook to export secrets as environment variables, and you'd also like to programmatically ensure that secrets get propagated to containers, instead of listing them all out. + ### `propagate-environment` (optional, boolean) Whether or not to automatically propagate all pipeline environment variables into the run container. Avoiding the need to be specified with environment. -**Important**: only pipeline variables will automatically be propagated (what you see in the Buildkite UI). Variables set in proceeding hook scripts will not be propagated to the container. +\* Caveat: only environment variables listed in $BUILDKITE_ENV_FILE will be propagated. This does not include e.g. variables that you exported in an `environment` hook. If you wish for those to be propagated, try `env-propagation-list`. ### `command` (optional, run only, array) diff --git a/commands/run.sh b/commands/run.sh index 51a49924..32fdf3d4 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -112,6 +112,19 @@ if [[ "$(plugin_read_config PROPAGATE_ENVIRONMENT "false")" =~ ^(true|on|1)$ ]] fi fi +# If requested, propagate a set of env vars as listed in a given env var to the +# container. +if [[ -n "$(plugin_read_config ENV_PROPAGATION_LIST)" ]]; then + env_propagation_list_var="$(plugin_read_config ENV_PROPAGATION_LIST)" + if [[ -z "${!env_propagation_list_var:-}" ]]; then + echo -n "env-propagation-list desired, but ${env_propagation_list_var} is not defined!" + exit 1 + fi + for var in ${!env_propagation_list_var}; do + run_params+=("-e" "$var") + done +fi + while IFS=$'\n' read -r vol ; do [[ -n "${vol:-}" ]] && run_params+=("-v" "$(expand_relative_volume_path "$vol")") done <<< "$(plugin_read_list VOLUMES)" diff --git a/plugin.yml b/plugin.yml index 606936ff..94fa18e5 100644 --- a/plugin.yml +++ b/plugin.yml @@ -50,6 +50,8 @@ configuration: environment: type: [ string, array ] minimum: 1 + env-propagation-list: + type: string expand-volume-vars: type: boolean image-repository: diff --git a/tests/run.bats b/tests/run.bats index b0754d2b..80c9467c 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -1545,3 +1545,51 @@ export BUILDKITE_JOB_ID=1111 unstub docker-compose unstub buildkite-agent } + +@test "Run with a list of propagated env vars" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_ENV_PROPAGATION_LIST="LIST_OF_VARS" + export LIST_OF_VARS="VAR_A VAR_B VAR_C" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="echo hello world" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 -e VAR_A -e VAR_B -e VAR_C --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "ran myservice" + unstub docker-compose + unstub buildkite-agent +} + +@test "Run with a list of propagated env vars - unless you forgot to define the variable" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_ENV_PROPAGATION_LIST="LIST_OF_VARS" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="echo hello world" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run "$PWD"/hooks/command + + assert_failure + assert_output --partial "env-propagation-list desired, but LIST_OF_VARS is not defined!" + unstub buildkite-agent +} From 8f430ce034bb0215de2367582386c96d4abda559 Mon Sep 17 00:00:00 2001 From: Songyu-Wang Date: Fri, 5 May 2023 19:53:49 -0500 Subject: [PATCH 210/253] Update README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matías Bellone --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ccb513c5..fc3be0a8 100644 --- a/README.md +++ b/README.md @@ -636,7 +636,8 @@ The default is `false`. ### `quiet-pull` (optional, run only) -Sets `docker-compose up` to run with `--quiet-pull`. There are other commands that can accept `--quiet-pull` or equivalents. They have not been implemented yet. + +Start up dependencies with `--quiet-pull` to prevent even more logs during that portion of the execution. The default is `false`. From 27a10d31ed3327f66b965f6a1c55c2fc8e80e45f Mon Sep 17 00:00:00 2001 From: Songyu-Wang Date: Fri, 5 May 2023 19:55:54 -0500 Subject: [PATCH 211/253] Update run.bats --- tests/run.bats | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/run.bats b/tests/run.bats index b0754d2b..ca953798 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -1545,3 +1545,33 @@ export BUILDKITE_JOB_ID=1111 unstub docker-compose unstub buildkite-agent } + +@test "Run with --quiet-pull" { + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="echo hello world" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PIPELINE_SLUG=test + + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_QUIET_PULL=true + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "compose -f docker-compose.yml -p buildkite1111 up --quiet-pull -d --scale myservice=0 myservice : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "built myservice" + refute_output --partial "Pulling" + assert_output --partial "ran myservice" + + unstub docker + unstub buildkite-agent +} + From 78e4a98e06911b1f399771242de7e44f026a6992 Mon Sep 17 00:00:00 2001 From: Songyu-Wang Date: Fri, 5 May 2023 19:56:22 -0500 Subject: [PATCH 212/253] Update run.bats --- tests/run.bats | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/run.bats b/tests/run.bats index ca953798..066f4102 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -1574,4 +1574,3 @@ export BUILDKITE_JOB_ID=1111 unstub docker unstub buildkite-agent } - From 202484fe1720f7624e07ccee5ae7cddd4a9e294a Mon Sep 17 00:00:00 2001 From: Songyu-Wang Date: Fri, 5 May 2023 19:57:14 -0500 Subject: [PATCH 213/253] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index fc3be0a8..f29f942f 100644 --- a/README.md +++ b/README.md @@ -636,7 +636,6 @@ The default is `false`. ### `quiet-pull` (optional, run only) - Start up dependencies with `--quiet-pull` to prevent even more logs during that portion of the execution. The default is `false`. From 51f43c37e1e648f7e00e2e4dd6457d49f0c7309d Mon Sep 17 00:00:00 2001 From: Songyu-Wang Date: Fri, 5 May 2023 20:03:31 -0500 Subject: [PATCH 214/253] Update run.bats --- tests/run.bats | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/run.bats b/tests/run.bats index 066f4102..5a8314cf 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -1556,10 +1556,10 @@ export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false export BUILDKITE_PLUGIN_DOCKER_COMPOSE_QUIET_PULL=true - stub docker \ - "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "compose -f docker-compose.yml -p buildkite1111 up --quiet-pull -d --scale myservice=0 myservice : echo ran myservice dependencies" \ - "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "-f docker-compose.yml -p buildkite1111 up --quite-pull -d --scale myservice=0 myservice : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" stub buildkite-agent \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" From 4de0608f0c2b93854001be75c56b8674fd967b99 Mon Sep 17 00:00:00 2001 From: Songyu-Wang Date: Fri, 5 May 2023 20:04:18 -0500 Subject: [PATCH 215/253] Update run.bats --- tests/run.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/run.bats b/tests/run.bats index 5a8314cf..be331384 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -1571,6 +1571,6 @@ export BUILDKITE_JOB_ID=1111 refute_output --partial "Pulling" assert_output --partial "ran myservice" - unstub docker + unstub docker-compose unstub buildkite-agent } From 5901018358285ab7111a684bd7226f7c827ebe33 Mon Sep 17 00:00:00 2001 From: Songyu-Wang Date: Fri, 5 May 2023 20:09:13 -0500 Subject: [PATCH 216/253] Update run.bats --- tests/run.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/run.bats b/tests/run.bats index be331384..70760e2f 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -1558,7 +1558,7 @@ export BUILDKITE_JOB_ID=1111 stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ - "-f docker-compose.yml -p buildkite1111 up --quite-pull -d --scale myservice=0 myservice : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 up --quiet-pull -d --scale myservice=0 myservice : echo ran myservice dependencies" \ "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" stub buildkite-agent \ From 5f1a51dfc40b960e0420faee6671f90e31a24acb Mon Sep 17 00:00:00 2001 From: Alex Martani Date: Sat, 6 May 2023 12:06:40 -0700 Subject: [PATCH 217/253] Update README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matías Bellone --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 744865e8..72014ea0 100644 --- a/README.md +++ b/README.md @@ -468,7 +468,7 @@ If you set this to `VALUE`, and `VALUE` is an environment variable containing a Whether or not to automatically propagate all pipeline environment variables into the run container. Avoiding the need to be specified with environment. -\* Caveat: only environment variables listed in $BUILDKITE_ENV_FILE will be propagated. This does not include e.g. variables that you exported in an `environment` hook. If you wish for those to be propagated, try `env-propagation-list`. +**Important**: only pipeline environment variables will be propagated (what you see in the BuildKite UI, those listed in `$BUILDKITE_ENV_FILE`). This does not include variables exported in preceeding `environment` hooks. If you wish for those to be propagated you will need to list them specifically or use `env-propagation-list`. ### `command` (optional, run only, array) From ce8a063e2dadce1e32a470df241b1e7cf14d4c98 Mon Sep 17 00:00:00 2001 From: Alex Martani Date: Sat, 6 May 2023 19:47:24 +0000 Subject: [PATCH 218/253] Add tests for docker compose v2 --- tests/v2/run.bats | 48 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/v2/run.bats b/tests/v2/run.bats index e2ddca86..1684c74a 100644 --- a/tests/v2/run.bats +++ b/tests/v2/run.bats @@ -1405,3 +1405,51 @@ export BUILDKITE_JOB_ID=1111 unstub docker unstub buildkite-agent } + +@test "Run with a list of propagated env vars" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_ENV_PROPAGATION_LIST="LIST_OF_VARS" + export LIST_OF_VARS="VAR_A VAR_B VAR_C" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="echo hello world" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 -e VAR_A -e VAR_B -e VAR_C --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "ran myservice" + unstub docker + unstub buildkite-agent +} + +@test "Run with a list of propagated env vars - unless you forgot to define the variable" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_ENV_PROPAGATION_LIST="LIST_OF_VARS" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="echo hello world" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run "$PWD"/hooks/command + + assert_failure + assert_output --partial "env-propagation-list desired, but LIST_OF_VARS is not defined!" + unstub buildkite-agent +} From 545ff50c31ae6d62197534338c78d5819aaad65a Mon Sep 17 00:00:00 2001 From: raylu <90059+raylu@users.noreply.github.com> Date: Tue, 2 May 2023 16:07:33 -0700 Subject: [PATCH 219/253] Remove orphaned containers in cleanup This stops the main/run container if it's still running. See #389. --- lib/run.bash | 6 +++--- tests/cleanup.bats | 2 +- tests/docker-compose-cleanup.bats | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/run.bash b/lib/run.bash index 91697fdf..f531292a 100644 --- a/lib/run.bash +++ b/lib/run.bash @@ -18,9 +18,9 @@ compose_cleanup() { # Stop and remove all the linked services and network if [[ "$(plugin_read_config LEAVE_VOLUMES 'false')" == "false" ]]; then - run_docker_compose down --volumes || true + run_docker_compose down --remove-orphans --volumes || true else - run_docker_compose down || true + run_docker_compose down --remove-orphans || true fi } @@ -72,7 +72,7 @@ check_linked_containers_and_save_logs() { } # docker-compose's -v arguments don't do local path expansion like the .yml -# versions do. So we add very simple support, for the common and basic case. +# versions do. So we add very simple support for the common and basic case. # # "./foo:/foo" => "/buildkite/builds/.../foo:/foo" expand_relative_volume_path() { diff --git a/tests/cleanup.bats b/tests/cleanup.bats index fd0be0f4..3b0e58a7 100644 --- a/tests/cleanup.bats +++ b/tests/cleanup.bats @@ -19,7 +19,7 @@ load '../lib/run' stub docker-compose \ "-f docker-compose.yml -p buildkite1111 kill : echo killing containers" \ "-f docker-compose.yml -p buildkite1111 rm --force -v : echo removing stopped containers" \ - "-f docker-compose.yml -p buildkite1111 down --volumes : echo removing everything" + "-f docker-compose.yml -p buildkite1111 down --remove-orphans --volumes : echo removing everything" run $PWD/hooks/pre-exit diff --git a/tests/docker-compose-cleanup.bats b/tests/docker-compose-cleanup.bats index e30e104a..37a2dea3 100644 --- a/tests/docker-compose-cleanup.bats +++ b/tests/docker-compose-cleanup.bats @@ -13,7 +13,7 @@ load '../lib/run' assert_success assert_equal "${lines[0]}" "kill" assert_equal "${lines[1]}" "rm --force -v" - assert_equal "${lines[2]}" "down --volumes" + assert_equal "${lines[2]}" "down --remove-orphans --volumes" } @test "Possible to gracefully shutdown containers in docker-compose cleanup" { @@ -26,7 +26,7 @@ load '../lib/run' assert_success assert_equal "${lines[0]}" "stop" assert_equal "${lines[1]}" "rm --force -v" - assert_equal "${lines[2]}" "down --volumes" + assert_equal "${lines[2]}" "down --remove-orphans --volumes" } @test "Possible to skip volume destruction in docker-compose cleanup" { @@ -39,5 +39,5 @@ load '../lib/run' assert_success assert_equal "${lines[0]}" "kill" assert_equal "${lines[1]}" "rm --force" - assert_equal "${lines[2]}" "down" -} \ No newline at end of file + assert_equal "${lines[2]}" "down --remove-orphans" +} From 74b7b8c8532f1c21ee01f7602fb099d82b06c057 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 22 May 2023 19:56:49 -0300 Subject: [PATCH 220/253] Added trapping of signals to stop running command --- commands/run.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/commands/run.sh b/commands/run.sh index 5b51047d..b7bd9e17 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -422,6 +422,8 @@ fi set +e ( + trap 'run_docker_compose stop ${container_name}' INT TERM + echo "+++ :docker: Running ${display_command[*]:-} in service $run_service" >&2 run_docker_compose "${run_params[@]}" ) From 9d8942c607cc4f0946f65bf01d54996b1d24412e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 22 May 2023 20:02:32 -0300 Subject: [PATCH 221/253] Corrected signal names in trap --- commands/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/run.sh b/commands/run.sh index b7bd9e17..23c099e2 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -422,7 +422,7 @@ fi set +e ( - trap 'run_docker_compose stop ${container_name}' INT TERM + trap 'run_docker_compose stop ${container_name}' SIGINT SIGTERM echo "+++ :docker: Running ${display_command[*]:-} in service $run_service" >&2 run_docker_compose "${run_params[@]}" From 85565a81177db16a923de1c74bf9b39b134781be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 22 May 2023 20:05:29 -0300 Subject: [PATCH 222/253] Setting trap outside of subshell --- commands/run.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/commands/run.sh b/commands/run.sh index 23c099e2..347dfc34 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -417,14 +417,15 @@ elif [[ ${#command[@]} -gt 0 ]] ; then done fi +trap 'run_docker_compose stop ${container_name}' SIGINT SIGTERM + # Disable -e outside of the subshell; since the subshell returning a failure # would exit the parent shell (here) early. set +e ( - trap 'run_docker_compose stop ${container_name}' SIGINT SIGTERM - echo "+++ :docker: Running ${display_command[*]:-} in service $run_service" >&2 + run_docker_compose "${run_params[@]}" ) From 09fbfb1b9f20a5fbacb797b823eb281bb40f5cb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Mon, 22 May 2023 20:10:30 -0300 Subject: [PATCH 223/253] Made it a function and avoid failing unnecessarily --- commands/run.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/commands/run.sh b/commands/run.sh index 347dfc34..0734bf34 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -417,7 +417,12 @@ elif [[ ${#command[@]} -gt 0 ]] ; then done fi -trap 'run_docker_compose stop ${container_name}' SIGINT SIGTERM +ensure_stopped() { + echo ':warning: Signal received, stopping container' + run_docker_compose stop "${container_name}" || true +} + +trap ensure_stopped SIGINT SIGTERM # Disable -e outside of the subshell; since the subshell returning a failure # would exit the parent shell (here) early. From 4f004f6a72deed2f71effb4765f0c3016dc6ad85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 23 May 2023 01:34:59 -0300 Subject: [PATCH 224/253] Remove subshell (Im quite confident it is not really necessary) --- commands/run.sh | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/commands/run.sh b/commands/run.sh index 0734bf34..77ef5272 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -424,16 +424,11 @@ ensure_stopped() { trap ensure_stopped SIGINT SIGTERM -# Disable -e outside of the subshell; since the subshell returning a failure -# would exit the parent shell (here) early. +# Disable -e to prevent cancelling step if the command fails for whatever reason set +e -( - echo "+++ :docker: Running ${display_command[*]:-} in service $run_service" >&2 - - run_docker_compose "${run_params[@]}" -) - +echo "+++ :docker: Running ${display_command[*]:-} in service $run_service" >&2 +run_docker_compose "${run_params[@]}" exitcode=$? # Restore -e as an option. From d0e426b68b70d4016054d310bd1310e7761ccb9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 23 May 2023 02:29:05 -0300 Subject: [PATCH 225/253] re instanted subshell (v2 appears to ignore signals) --- commands/run.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/commands/run.sh b/commands/run.sh index 77ef5272..4baaf77b 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -426,11 +426,11 @@ trap ensure_stopped SIGINT SIGTERM # Disable -e to prevent cancelling step if the command fails for whatever reason set +e - -echo "+++ :docker: Running ${display_command[*]:-} in service $run_service" >&2 -run_docker_compose "${run_params[@]}" -exitcode=$? - +( # subshell is necessary to trap signals (compose v2 fails to stop otherwise) + echo "+++ :docker: Running ${display_command[*]:-} in service $run_service" >&2 + run_docker_compose "${run_params[@]}" + exitcode=$? +) # Restore -e as an option. set -e From 7e786739f7fb2bd2a951dcdcce249fce95986851 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 23 May 2023 02:32:02 -0300 Subject: [PATCH 226/253] Set exitcode variable in trap to avoid further errors --- commands/run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/commands/run.sh b/commands/run.sh index 4baaf77b..e74ddaf7 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -420,6 +420,7 @@ fi ensure_stopped() { echo ':warning: Signal received, stopping container' run_docker_compose stop "${container_name}" || true + exitcode=-1 } trap ensure_stopped SIGINT SIGTERM From 540574a8f97607258485c506adfc033d738b30bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 23 May 2023 02:35:39 -0300 Subject: [PATCH 227/253] Better error message --- commands/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/run.sh b/commands/run.sh index e74ddaf7..a70dbabf 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -418,7 +418,7 @@ elif [[ ${#command[@]} -gt 0 ]] ; then fi ensure_stopped() { - echo ':warning: Signal received, stopping container' + echo '+++ :warning: Signal received, stopping container' run_docker_compose stop "${container_name}" || true exitcode=-1 } From 8e0e2e24804ea519c36752b954da40ac1a685080 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 23 May 2023 02:39:36 -0300 Subject: [PATCH 228/253] Better messaging on job cancellation --- commands/run.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/commands/run.sh b/commands/run.sh index a70dbabf..56d10a85 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -420,7 +420,7 @@ fi ensure_stopped() { echo '+++ :warning: Signal received, stopping container' run_docker_compose stop "${container_name}" || true - exitcode=-1 + exitcode='TRAP' } trap ensure_stopped SIGINT SIGTERM @@ -435,7 +435,11 @@ set +e # Restore -e as an option. set -e -if [[ $exitcode -ne 0 ]] ; then +if [[ $exitcode = "TRAP" ]]; then + # command failed due to cancellation signal, prevent printing more messages + # mask the exit code + exitcode=0 +elif [[ $exitcode -ne 0 ]] ; then echo "^^^ +++" echo "+++ :warning: Failed to run command, exited with $exitcode, run params:" echo "${run_params[@]}" @@ -449,4 +453,4 @@ if [[ -n "${BUILDKITE_AGENT_ACCESS_TOKEN:-}" ]] ; then fi fi -return $exitcode +return "$exitcode" From 33811c69aed0de202655c89d834dc99615ceae29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 23 May 2023 02:43:47 -0300 Subject: [PATCH 229/253] Added sigquit to signals being handled --- commands/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/run.sh b/commands/run.sh index 56d10a85..55e0f765 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -423,7 +423,7 @@ ensure_stopped() { exitcode='TRAP' } -trap ensure_stopped SIGINT SIGTERM +trap ensure_stopped SIGINT SIGTERM SIGQUIT # Disable -e to prevent cancelling step if the command fails for whatever reason set +e From 9888cb632b475a985ac84dbcd9ca3bcb1bab8c03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 23 May 2023 02:48:01 -0300 Subject: [PATCH 230/253] Corrected container to be stopped --- commands/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/run.sh b/commands/run.sh index 55e0f765..82df9b44 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -419,7 +419,7 @@ fi ensure_stopped() { echo '+++ :warning: Signal received, stopping container' - run_docker_compose stop "${container_name}" || true + run_docker_compose stop "${run_service}" || true exitcode='TRAP' } From 61bcd539940662a17956dcae79eb60cff7c080af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 23 May 2023 02:58:29 -0300 Subject: [PATCH 231/253] Try to print the last log lines (if missing) --- commands/run.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/commands/run.sh b/commands/run.sh index 82df9b44..a033cd71 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -420,6 +420,8 @@ fi ensure_stopped() { echo '+++ :warning: Signal received, stopping container' run_docker_compose stop "${run_service}" || true + echo '--- Last log lines (may be missing above)' + run_docker_compose logs "${run_service}" || true exitcode='TRAP' } From 8efd64033ad6d0bec00c9bfd7a656b70a18e1594 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 23 May 2023 03:33:42 -0300 Subject: [PATCH 232/253] Need to use docker logs directly to get logs :( --- commands/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/commands/run.sh b/commands/run.sh index a033cd71..577ac750 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -420,8 +420,8 @@ fi ensure_stopped() { echo '+++ :warning: Signal received, stopping container' run_docker_compose stop "${run_service}" || true - echo '--- Last log lines (may be missing above)' - run_docker_compose logs "${run_service}" || true + echo '--- Last log lines that may be missing above (will be empty if container was already removed)' + docker logs "${container_name}" || true exitcode='TRAP' } From e737e995ac313edf3d01220b8717e81945bb05a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 23 May 2023 03:40:49 -0300 Subject: [PATCH 233/253] Service is not running so need to use docker for stopping as well --- commands/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/run.sh b/commands/run.sh index 577ac750..acb4dcf5 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -419,7 +419,7 @@ fi ensure_stopped() { echo '+++ :warning: Signal received, stopping container' - run_docker_compose stop "${run_service}" || true + docker stop "${container_name}" || true echo '--- Last log lines that may be missing above (will be empty if container was already removed)' docker logs "${container_name}" || true exitcode='TRAP' From 716c4adee2f8af1f992f741206915a5ffb0a561f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 23 May 2023 03:51:57 -0300 Subject: [PATCH 234/253] Better logging, add back error exit code --- commands/run.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/commands/run.sh b/commands/run.sh index acb4dcf5..b99b2b56 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -420,7 +420,7 @@ fi ensure_stopped() { echo '+++ :warning: Signal received, stopping container' docker stop "${container_name}" || true - echo '--- Last log lines that may be missing above (will be empty if container was already removed)' + echo '~~~ Last log lines that may be missing above (if container was not already removed)' docker logs "${container_name}" || true exitcode='TRAP' } @@ -439,8 +439,7 @@ set -e if [[ $exitcode = "TRAP" ]]; then # command failed due to cancellation signal, prevent printing more messages - # mask the exit code - exitcode=0 + exitcode=-1 elif [[ $exitcode -ne 0 ]] ; then echo "^^^ +++" echo "+++ :warning: Failed to run command, exited with $exitcode, run params:" From 50a2f2cc85e06630307ce03adecb7e4fdd442474 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 23 May 2023 16:55:29 -0300 Subject: [PATCH 235/253] Removed subshell once more as previous error was not related to what I thought it was) --- commands/run.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/commands/run.sh b/commands/run.sh index b99b2b56..13a7fc05 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -429,11 +429,11 @@ trap ensure_stopped SIGINT SIGTERM SIGQUIT # Disable -e to prevent cancelling step if the command fails for whatever reason set +e -( # subshell is necessary to trap signals (compose v2 fails to stop otherwise) - echo "+++ :docker: Running ${display_command[*]:-} in service $run_service" >&2 - run_docker_compose "${run_params[@]}" - exitcode=$? -) + +echo "+++ :docker: Running ${display_command[*]:-} in service $run_service" >&2 +run_docker_compose "${run_params[@]}" +exitcode=$? + # Restore -e as an option. set -e From 2592c374c90538ab026e2685e647f9a914b9a8ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 23 May 2023 16:58:27 -0300 Subject: [PATCH 236/253] Nope, subshell is really necessary :shrug: This reverts commit 50a2f2cc85e06630307ce03adecb7e4fdd442474. --- commands/run.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/commands/run.sh b/commands/run.sh index 13a7fc05..b99b2b56 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -429,11 +429,11 @@ trap ensure_stopped SIGINT SIGTERM SIGQUIT # Disable -e to prevent cancelling step if the command fails for whatever reason set +e - -echo "+++ :docker: Running ${display_command[*]:-} in service $run_service" >&2 -run_docker_compose "${run_params[@]}" -exitcode=$? - +( # subshell is necessary to trap signals (compose v2 fails to stop otherwise) + echo "+++ :docker: Running ${display_command[*]:-} in service $run_service" >&2 + run_docker_compose "${run_params[@]}" + exitcode=$? +) # Restore -e as an option. set -e From 3e3a714f32b1f9f458f46d69db644390c5b4073d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 23 May 2023 17:15:30 -0300 Subject: [PATCH 237/253] Changed variable handling due to issues with subshell --- commands/run.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/commands/run.sh b/commands/run.sh index b99b2b56..734a9635 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -422,7 +422,7 @@ ensure_stopped() { docker stop "${container_name}" || true echo '~~~ Last log lines that may be missing above (if container was not already removed)' docker logs "${container_name}" || true - exitcode='TRAP' + trapped='TRAP' } trap ensure_stopped SIGINT SIGTERM SIGQUIT @@ -437,8 +437,8 @@ set +e # Restore -e as an option. set -e -if [[ $exitcode = "TRAP" ]]; then - # command failed due to cancellation signal, prevent printing more messages +if [[ "${trapped:-}" = "TRAP" ]]; then + # command failed due to cancellation signal, make sure there is an error but no further output exitcode=-1 elif [[ $exitcode -ne 0 ]] ; then echo "^^^ +++" From d8b3cc138661cf87a8664e41bb9d74a48a343c86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 23 May 2023 17:17:47 -0300 Subject: [PATCH 238/253] Move exitcode setting outside subshell as it was before :facepalm: --- commands/run.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/commands/run.sh b/commands/run.sh index 734a9635..8a2f2e0e 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -422,7 +422,7 @@ ensure_stopped() { docker stop "${container_name}" || true echo '~~~ Last log lines that may be missing above (if container was not already removed)' docker logs "${container_name}" || true - trapped='TRAP' + exitcode='TRAP' } trap ensure_stopped SIGINT SIGTERM SIGQUIT @@ -432,12 +432,13 @@ set +e ( # subshell is necessary to trap signals (compose v2 fails to stop otherwise) echo "+++ :docker: Running ${display_command[*]:-} in service $run_service" >&2 run_docker_compose "${run_params[@]}" - exitcode=$? ) +exitcode=$? + # Restore -e as an option. set -e -if [[ "${trapped:-}" = "TRAP" ]]; then +if [[ $exitcode = "TRAP" ]]; then # command failed due to cancellation signal, make sure there is an error but no further output exitcode=-1 elif [[ $exitcode -ne 0 ]] ; then From c79c77d8c4ccac3535ac01cf229ae69361e009e6 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 30 May 2023 15:51:55 +0000 Subject: [PATCH 239/253] Update buildkite plugin plugin-linter to v3.2.0 --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 101e89b9..409016a9 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -15,7 +15,7 @@ steps: - label: ":sparkles: Lint" plugins: - plugin-linter#v3.1.0: + plugin-linter#v3.2.0: id: docker-compose - label: ":bash: Tests" From 2d882b3ec96cadfe2b9429449f680229871a5f6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 30 May 2023 14:08:14 -0300 Subject: [PATCH 240/253] Correct loading of bats libraries --- tests/build.bats | 2 +- tests/cleanup.bats | 2 +- tests/docker-compose-cleanup.bats | 2 +- tests/docker-compose-config.bats | 2 +- tests/docker-compose-images.bats | 2 +- tests/docker-compose-verbosity.bats | 2 +- tests/image-override-file.bats | 2 +- tests/logs.bats | 2 +- tests/metadata.bats | 2 +- tests/multiple-commands.bats | 2 +- tests/output.bats | 2 +- tests/plugin-config.bats | 2 +- tests/project-name.bats | 2 +- tests/push.bats | 2 +- tests/run.bats | 2 +- tests/v2/build.bats | 2 +- tests/v2/push.bats | 2 +- tests/v2/run.bats | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/build.bats b/tests/build.bats index b30614c4..8ed91f13 100644 --- a/tests/build.bats +++ b/tests/build.bats @@ -1,6 +1,6 @@ #!/usr/bin/env bats -load '/usr/local/lib/bats/load.bash' +load "${BATS_PLUGIN_PATH}/load.bash" load '../lib/shared' # export DOCKER_COMPOSE_STUB_DEBUG=/dev/stdout diff --git a/tests/cleanup.bats b/tests/cleanup.bats index 3b0e58a7..19c47d55 100644 --- a/tests/cleanup.bats +++ b/tests/cleanup.bats @@ -1,6 +1,6 @@ #!/usr/bin/env bats -load '/usr/local/lib/bats/load.bash' +load "${BATS_PLUGIN_PATH}/load.bash" load '../lib/shared' load '../lib/run' diff --git a/tests/docker-compose-cleanup.bats b/tests/docker-compose-cleanup.bats index 37a2dea3..6d00322a 100644 --- a/tests/docker-compose-cleanup.bats +++ b/tests/docker-compose-cleanup.bats @@ -1,6 +1,6 @@ #!/usr/bin/env bats -load '/usr/local/lib/bats/load.bash' +load "${BATS_PLUGIN_PATH}/load.bash" load '../lib/shared' load '../lib/run' diff --git a/tests/docker-compose-config.bats b/tests/docker-compose-config.bats index 2284cb96..910d2531 100644 --- a/tests/docker-compose-config.bats +++ b/tests/docker-compose-config.bats @@ -1,6 +1,6 @@ #!/usr/bin/env bats -load '/usr/local/lib/bats/load.bash' +load "${BATS_PLUGIN_PATH}/load.bash" load '../lib/shared' @test "Read docker-compose config when none exists" { diff --git a/tests/docker-compose-images.bats b/tests/docker-compose-images.bats index dca55923..a47838c2 100644 --- a/tests/docker-compose-images.bats +++ b/tests/docker-compose-images.bats @@ -2,7 +2,7 @@ # export DOCKER_COMPOSE_STUB_DEBUG=/dev/tty -load '/usr/local/lib/bats/load.bash' +load "${BATS_PLUGIN_PATH}/load.bash" load '../lib/shared' load '../lib/push' diff --git a/tests/docker-compose-verbosity.bats b/tests/docker-compose-verbosity.bats index 2240c474..75cb1769 100644 --- a/tests/docker-compose-verbosity.bats +++ b/tests/docker-compose-verbosity.bats @@ -1,6 +1,6 @@ #!/usr/bin/env bats -load '/usr/local/lib/bats/load.bash' +load "${BATS_PLUGIN_PATH}/load.bash" load '../lib/shared' load '../lib/run' diff --git a/tests/image-override-file.bats b/tests/image-override-file.bats index 5af1ac5b..cd052fac 100644 --- a/tests/image-override-file.bats +++ b/tests/image-override-file.bats @@ -1,6 +1,6 @@ #!/usr/bin/env bats -load '/usr/local/lib/bats/load.bash' +load "${BATS_PLUGIN_PATH}/load.bash" load '../lib/shared' myservice_override_file1=$(cat <<-EOF diff --git a/tests/logs.bats b/tests/logs.bats index bd11416d..e3037d81 100644 --- a/tests/logs.bats +++ b/tests/logs.bats @@ -1,6 +1,6 @@ #!/usr/bin/env bats -load '/usr/local/lib/bats/load.bash' +load "${BATS_PLUGIN_PATH}/load.bash" load '../lib/run' load '../lib/shared' diff --git a/tests/metadata.bats b/tests/metadata.bats index 80c9f1a7..d520c73f 100644 --- a/tests/metadata.bats +++ b/tests/metadata.bats @@ -2,7 +2,7 @@ # export BUILDKITE_AGENT_STUB_DEBUG=/dev/tty -load '/usr/local/lib/bats/load.bash' +load "${BATS_PLUGIN_PATH}/load.bash" load '../lib/shared' load '../lib/metadata' diff --git a/tests/multiple-commands.bats b/tests/multiple-commands.bats index 7e2a71fa..fdebe77c 100644 --- a/tests/multiple-commands.bats +++ b/tests/multiple-commands.bats @@ -1,6 +1,6 @@ #!/usr/bin/env bats -load '/usr/local/lib/bats/load.bash' +load "${BATS_PLUGIN_PATH}/load.bash" load '../lib/shared' load '../lib/metadata' diff --git a/tests/output.bats b/tests/output.bats index f2845553..4e1619cf 100644 --- a/tests/output.bats +++ b/tests/output.bats @@ -1,6 +1,6 @@ #!/usr/bin/env bats -load '/usr/local/lib/bats/load.bash' +load "${BATS_PLUGIN_PATH}/load.bash" load '../lib/shared' load '../lib/run' diff --git a/tests/plugin-config.bats b/tests/plugin-config.bats index a9b3c046..cf590b71 100644 --- a/tests/plugin-config.bats +++ b/tests/plugin-config.bats @@ -1,6 +1,6 @@ #!/usr/bin/env bats -load '/usr/local/lib/bats/load.bash' +load "${BATS_PLUGIN_PATH}/load.bash" load '../lib/shared' @test "Read existing config without default" { diff --git a/tests/project-name.bats b/tests/project-name.bats index 4678b4e8..18f00e7c 100644 --- a/tests/project-name.bats +++ b/tests/project-name.bats @@ -1,6 +1,6 @@ #!/usr/bin/env bats -load '/usr/local/lib/bats/load.bash' +load "${BATS_PLUGIN_PATH}/load.bash" load '../lib/shared' @test "Project name comes from BUILDKITE_JOB_ID" { diff --git a/tests/push.bats b/tests/push.bats index cefe7996..a87a8742 100644 --- a/tests/push.bats +++ b/tests/push.bats @@ -1,6 +1,6 @@ #!/usr/bin/env bats -load '/usr/local/lib/bats/load.bash' +load "${BATS_PLUGIN_PATH}/load.bash" load '../lib/shared' # export DOCKER_COMPOSE_STUB_DEBUG=/dev/tty diff --git a/tests/run.bats b/tests/run.bats index 2102adcd..1e004ea2 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -1,6 +1,6 @@ #!/usr/bin/env bats -load '/usr/local/lib/bats/load.bash' +load "${BATS_PLUGIN_PATH}/load.bash" load '../lib/shared' load '../lib/run' diff --git a/tests/v2/build.bats b/tests/v2/build.bats index 21ca38ec..535679ec 100644 --- a/tests/v2/build.bats +++ b/tests/v2/build.bats @@ -1,6 +1,6 @@ #!/usr/bin/env bats -load '/usr/local/lib/bats/load.bash' +load "${BATS_PLUGIN_PATH}/load.bash" load '../../lib/shared' # export DOCKER_COMPOSE_STUB_DEBUG=/dev/stdout diff --git a/tests/v2/push.bats b/tests/v2/push.bats index f6d109af..d85ea02f 100644 --- a/tests/v2/push.bats +++ b/tests/v2/push.bats @@ -1,6 +1,6 @@ #!/usr/bin/env bats -load '/usr/local/lib/bats/load.bash' +load "${BATS_PLUGIN_PATH}/load.bash" load '../../lib/shared' # export DOCKER_COMPOSE_STUB_DEBUG=/dev/tty diff --git a/tests/v2/run.bats b/tests/v2/run.bats index 1684c74a..e34f6718 100644 --- a/tests/v2/run.bats +++ b/tests/v2/run.bats @@ -1,6 +1,6 @@ #!/usr/bin/env bats -load '/usr/local/lib/bats/load.bash' +load "${BATS_PLUGIN_PATH}/load.bash" load '../../lib/shared' load '../../lib/run' From 2d3d6f783aeca93a331979e45f5f2eb2168924cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 30 May 2023 14:09:33 -0300 Subject: [PATCH 241/253] Updated tester image version --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2a9c3a38..7b38f95b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ version: '2' services: tests: - image: buildkite/plugin-tester:v4.0.0 + image: buildkite/plugin-tester:v4.1.0 volumes: - ".:/plugin" From 7c55e9edc69a249a38d49446493fb2ed9a1f1d25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Tue, 30 May 2023 14:17:24 -0300 Subject: [PATCH 242/253] Correct shellcheck warnings --- tests/cleanup.bats | 2 +- tests/docker-compose-cleanup.bats | 12 +++++------- tests/logs.bats | 26 +++++++------------------- tests/multiple-commands.bats | 8 ++++---- tests/output.bats | 8 ++++---- tests/push.bats | 12 ++++++------ tests/v2/push.bats | 14 +++++++------- 7 files changed, 34 insertions(+), 48 deletions(-) diff --git a/tests/cleanup.bats b/tests/cleanup.bats index 19c47d55..adb69f18 100644 --- a/tests/cleanup.bats +++ b/tests/cleanup.bats @@ -21,7 +21,7 @@ load '../lib/run' "-f docker-compose.yml -p buildkite1111 rm --force -v : echo removing stopped containers" \ "-f docker-compose.yml -p buildkite1111 down --remove-orphans --volumes : echo removing everything" - run $PWD/hooks/pre-exit + run "$PWD"/hooks/pre-exit assert_success assert_output --partial "Cleaning up after docker-compose" diff --git a/tests/docker-compose-cleanup.bats b/tests/docker-compose-cleanup.bats index 6d00322a..a8843434 100644 --- a/tests/docker-compose-cleanup.bats +++ b/tests/docker-compose-cleanup.bats @@ -4,10 +4,14 @@ load "${BATS_PLUGIN_PATH}/load.bash" load '../lib/shared' load '../lib/run' -@test "Default cleanup of docker-compose" { +setup () { run_docker_compose() { + # shellcheck disable=2317 # funtion used by loaded scripts echo "$@" } +} + +@test "Default cleanup of docker-compose" { run compose_cleanup assert_success @@ -18,9 +22,6 @@ load '../lib/run' @test "Possible to gracefully shutdown containers in docker-compose cleanup" { export BUILDKITE_PLUGIN_DOCKER_COMPOSE_GRACEFUL_SHUTDOWN=1 - run_docker_compose() { - echo "$@" - } run compose_cleanup assert_success @@ -31,9 +32,6 @@ load '../lib/run' @test "Possible to skip volume destruction in docker-compose cleanup" { export BUILDKITE_PLUGIN_DOCKER_COMPOSE_LEAVE_VOLUMES=1 - run_docker_compose() { - echo "$@" - } run compose_cleanup assert_success diff --git a/tests/logs.bats b/tests/logs.bats index e3037d81..3d05d845 100644 --- a/tests/logs.bats +++ b/tests/logs.bats @@ -8,16 +8,20 @@ load '../lib/shared' # export CHECK_LINKED_CONTAINERS_AND_SAVE_LOGS_STUB_DEBUG=/dev/tty # export DOCKER_STUB_DEBUG=/dev/tty -@test "Upload log settings: on-error" { - export LOG_DIR="docker-compose-logs" - +setup () { function docker_ps_by_project() { + # shellcheck disable=2317 # funtion used by loaded scripts cat tests/fixtures/id-service-multiple-services.txt } function plugin_prompt_and_run() { + # shellcheck disable=2317 # funtion used by loaded scripts echo "ran plugin_prompt_and_run" } +} + +@test "Upload log settings: on-error" { + export LOG_DIR="docker-compose-logs" stub docker \ "inspect --format={{.State.ExitCode}} 456456 : echo 1" \ @@ -36,14 +40,6 @@ load '../lib/shared' @test "Upload log settings: always" { export LOG_DIR="docker-compose-logs" - function docker_ps_by_project() { - cat tests/fixtures/id-service-multiple-services.txt - } - - function plugin_prompt_and_run() { - echo "ran plugin_prompt_and_run" - } - stub docker \ "inspect --format={{.State.ExitCode}} 456456 : echo 1" \ "logs -t 456456 : echo got logs for failed" \ @@ -62,14 +58,6 @@ load '../lib/shared' @test "Upload log settings: never" { export LOG_DIR="docker-compose-logs" - function docker_ps_by_project() { - cat tests/fixtures/id-service-multiple-services.txt - } - - function plugin_prompt_and_run() { - echo "ran plugin_prompt_and_run" - } - run check_linked_containers_and_save_logs \ "main" "/tmp/docker-compose-logs" "never" diff --git a/tests/multiple-commands.bats b/tests/multiple-commands.bats index fdebe77c..f4989bb5 100644 --- a/tests/multiple-commands.bats +++ b/tests/multiple-commands.bats @@ -37,7 +37,7 @@ setup_file() { "meta-data exists docker-compose-plugin-built-image-tag-myservice : test -f /tmp/build-run-metadata" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : cat /tmp/build-run-metadata" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "Building services myservice" @@ -73,7 +73,7 @@ setup_file() { "pull my.repository/llamas:test-myservice-build-1 : echo pulled pre-built image" \ "tag my.repository/llamas:test-myservice-build-1 buildkite12_myservice : echo re-tagged pre-built image" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success @@ -104,7 +104,7 @@ setup_file() { "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" \ "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success @@ -142,7 +142,7 @@ setup_file() { "pull myservice-tag : echo pulled pre-built image" \ "tag myservice-tag buildkite12_myservice : echo re-tagged pre-built image" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success diff --git a/tests/output.bats b/tests/output.bats index 4e1619cf..f5318b6f 100644 --- a/tests/output.bats +++ b/tests/output.bats @@ -45,7 +45,7 @@ setup_file() { "logs -t 456456 : exit 0" \ "inspect --format={{.State.ExitCode}} 789789 : echo 0" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -89,7 +89,7 @@ setup_file() { "logs -t 456456 : exit 0" \ "inspect --format={{.State.ExitCode}} 789789 : echo 0" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_failure assert_output --partial "built myservice" @@ -128,7 +128,7 @@ setup_file() { "inspect --format={{.State.ExitCode}} 456456 : echo 0" \ "inspect --format={{.State.ExitCode}} 789789 : echo 0" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" @@ -168,7 +168,7 @@ setup_file() { "ps -a --filter label=com.docker.compose.project=buildkite1111 -q : echo" \ "ps -a --filter label=com.docker.compose.project=buildkite1111 --format '{{.ID}}\\t{{.Label \"com.docker.compose.service\"}}' : cat tests/fixtures/id-service-no-services.txt" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built myservice" diff --git a/tests/push.bats b/tests/push.bats index a87a8742..459aa6b6 100644 --- a/tests/push.bats +++ b/tests/push.bats @@ -24,7 +24,7 @@ load '../lib/shared' stub docker \ "image inspect somewhere.dkr.ecr.some-region.amazonaws.com/blah : exit 0" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial ":warning: Skipping build" @@ -59,7 +59,7 @@ load '../lib/shared' "meta-data exists docker-compose-plugin-built-image-tag-myservice1 : exit 1" \ "meta-data exists docker-compose-plugin-built-image-tag-myservice2 : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "tagging image1" @@ -90,7 +90,7 @@ load '../lib/shared' "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "pulled prebuilt image" @@ -110,7 +110,7 @@ load '../lib/shared' stub docker-compose \ "-f docker-compose.yml -p buildkite1111 config : echo blah" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success refute_output --partial "pulled prebuilt image" @@ -152,7 +152,7 @@ load '../lib/shared' "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo prebuilt" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "pulled prebuilt image" @@ -185,7 +185,7 @@ load '../lib/shared' "tag buildkite1111_helper my.repository/helper:llamas : echo tagged helper" \ "push my.repository/helper:llamas : echo pushed helper" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built helper" diff --git a/tests/v2/push.bats b/tests/v2/push.bats index d85ea02f..4bef2dbc 100644 --- a/tests/v2/push.bats +++ b/tests/v2/push.bats @@ -26,7 +26,7 @@ setup_file() { "image inspect somewhere.dkr.ecr.some-region.amazonaws.com/blah : exit 0" \ "compose -f docker-compose.yml -p buildkite1111 push app : echo pushed app" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial ":warning: Skipping build" @@ -58,7 +58,7 @@ setup_file() { "meta-data exists docker-compose-plugin-built-image-tag-myservice1 : exit 1" \ "meta-data exists docker-compose-plugin-built-image-tag-myservice2 : exit 1" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "tagging image1" @@ -86,7 +86,7 @@ setup_file() { "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "pulled prebuilt image" @@ -116,7 +116,7 @@ setup_file() { "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "pulled prebuilt image" @@ -135,7 +135,7 @@ setup_file() { stub docker \ "compose -f docker-compose.yml -p buildkite1111 config : echo blah" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success refute_output --partial "pulled prebuilt image" @@ -175,7 +175,7 @@ setup_file() { "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ "meta-data get docker-compose-plugin-built-image-tag-myservice : echo prebuilt" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "pulled prebuilt image" @@ -205,7 +205,7 @@ setup_file() { "tag buildkite1111-helper my.repository/helper:llamas : echo tagged helper" \ "push my.repository/helper:llamas : echo pushed helper" - run $PWD/hooks/command + run "$PWD"/hooks/command assert_success assert_output --partial "built helper" From 07dd03a5be71e347fed106e5f505b94c217a68df Mon Sep 17 00:00:00 2001 From: "Pol (Paula)" Date: Thu, 8 Jun 2023 14:31:24 -0300 Subject: [PATCH 243/253] update docker-compose version in Readme --- README.md | 54 +++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 624479b5..48db3e41 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v4.12.0: + - docker-compose#v4.13.0: run: app ``` @@ -28,7 +28,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v4.12.0: + - docker-compose#v4.13.0: run: app config: docker-compose.tests.yml env: @@ -41,7 +41,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v4.12.0: + - docker-compose#v4.13.0: run: app config: - docker-compose.yml @@ -56,7 +56,7 @@ env: steps: - command: test.sh plugins: - - docker-compose#v4.12.0: + - docker-compose#v4.13.0: run: app ``` @@ -65,7 +65,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v4.12.0: + - docker-compose#v4.13.0: run: app command: ["custom", "command", "values"] ``` @@ -79,7 +79,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.12.0: + - docker-compose#v4.13.0: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -87,7 +87,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.12.0: + - docker-compose#v4.13.0: run: app ``` @@ -104,7 +104,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.12.0: + - docker-compose#v4.13.0: run: app ``` @@ -122,7 +122,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.12.0: + - docker-compose#v4.13.0: run: app volumes: - "./dist:/app/dist" @@ -146,7 +146,7 @@ this plugin offers a `environment` block of its own: steps: - command: generate-dist.sh plugins: - - docker-compose#v4.12.0: + - docker-compose#v4.13.0: run: app env: - BUILDKITE_BUILD_NUMBER @@ -164,7 +164,7 @@ Alternatively, you can have the plugin add all environment variables defined for steps: - command: use-vars.sh plugins: - - docker-compose#v4.12.0: + - docker-compose#v4.13.0: run: app propagate-environment: true ``` @@ -199,7 +199,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v4.12.0: + - docker-compose#v4.13.0: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -216,7 +216,7 @@ If you have multiple steps that use the same service/image (such as steps that r steps: - label: ":docker: Build" plugins: - - docker-compose#v4.12.0: + - docker-compose#v4.13.0: build: app image-repository: index.docker.io/myorg/myrepo @@ -226,7 +226,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.12.0: + - docker-compose#v4.13.0: run: app ``` @@ -242,7 +242,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v4.12.0: + - docker-compose#v4.13.0: build: - app - tests @@ -254,7 +254,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.12.0: + - docker-compose#v4.13.0: run: tests ``` @@ -266,7 +266,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v4.12.0: + - docker-compose#v4.13.0: push: app ``` @@ -276,7 +276,7 @@ To push multiple images, you can use a list: steps: - label: ":docker: Push" plugins: - - docker-compose#v4.12.0: + - docker-compose#v4.13.0: push: - first-service - second-service @@ -288,7 +288,7 @@ If you want to push to a specific location (that's not defined as the `image` in steps: - label: ":docker: Push" plugins: - - docker-compose#v4.12.0: + - docker-compose#v4.13.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -302,14 +302,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker: Build an image" plugins: - - docker-compose#v4.12.0: + - docker-compose#v4.13.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.12.0: + - docker-compose#v4.13.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -327,7 +327,7 @@ This plugin allows for the value of `cache-from` to be a string or a list. If it steps: - label: ":docker Build an image" plugins: - - docker-compose#v4.12.0: + - docker-compose#v4.13.0: build: app image-repository: index.docker.io/myorg/myrepo separator-cache-from: "#" @@ -337,7 +337,7 @@ steps: - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.12.0: + - docker-compose#v4.13.0: push: - app:myregistry:port/myrepo/myapp:my-branch - app:myregistry:port/myrepo/myapp:latest @@ -350,7 +350,7 @@ Adding a grouping tag to the end of a cache-from list item allows this plugin to steps: - label: ":docker: Build Intermediate Image" plugins: - - docker-compose#v4.12.0: + - docker-compose#v4.13.0: build: myservice_intermediate # docker-compose.yml is the same as myservice but has `target: intermediate` image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo/myservice_intermediate @@ -360,7 +360,7 @@ steps: - wait - label: ":docker: Build Final Image" plugins: - - docker-compose#v4.12.0: + - docker-compose#v4.13.0: build: myservice image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo @@ -404,7 +404,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Run & Push" plugins: - - docker-compose#v4.12.0: + - docker-compose#v4.13.0: run: myservice push: myservice ``` @@ -419,7 +419,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Build & Push" plugins: - - docker-compose#v4.12.0: + - docker-compose#v4.13.0: build: myservice push: myservice ``` From 75113039ea8e6c39bc7b0bcfa5084b711195100e Mon Sep 17 00:00:00 2001 From: "Pol (Paula)" Date: Wed, 21 Jun 2023 17:59:31 -0300 Subject: [PATCH 244/253] Update version to next release --- README.md | 54 +++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 48db3e41..36740236 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v4.13.0: + - docker-compose#v4.14.0: run: app ``` @@ -28,7 +28,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v4.13.0: + - docker-compose#v4.14.0: run: app config: docker-compose.tests.yml env: @@ -41,7 +41,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v4.13.0: + - docker-compose#v4.14.0: run: app config: - docker-compose.yml @@ -56,7 +56,7 @@ env: steps: - command: test.sh plugins: - - docker-compose#v4.13.0: + - docker-compose#v4.14.0: run: app ``` @@ -65,7 +65,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v4.13.0: + - docker-compose#v4.14.0: run: app command: ["custom", "command", "values"] ``` @@ -79,7 +79,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.13.0: + - docker-compose#v4.14.0: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -87,7 +87,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.13.0: + - docker-compose#v4.14.0: run: app ``` @@ -104,7 +104,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.13.0: + - docker-compose#v4.14.0: run: app ``` @@ -122,7 +122,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.13.0: + - docker-compose#v4.14.0: run: app volumes: - "./dist:/app/dist" @@ -146,7 +146,7 @@ this plugin offers a `environment` block of its own: steps: - command: generate-dist.sh plugins: - - docker-compose#v4.13.0: + - docker-compose#v4.14.0: run: app env: - BUILDKITE_BUILD_NUMBER @@ -164,7 +164,7 @@ Alternatively, you can have the plugin add all environment variables defined for steps: - command: use-vars.sh plugins: - - docker-compose#v4.13.0: + - docker-compose#v4.14.0: run: app propagate-environment: true ``` @@ -199,7 +199,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v4.13.0: + - docker-compose#v4.14.0: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -216,7 +216,7 @@ If you have multiple steps that use the same service/image (such as steps that r steps: - label: ":docker: Build" plugins: - - docker-compose#v4.13.0: + - docker-compose#v4.14.0: build: app image-repository: index.docker.io/myorg/myrepo @@ -226,7 +226,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.13.0: + - docker-compose#v4.14.0: run: app ``` @@ -242,7 +242,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v4.13.0: + - docker-compose#v4.14.0: build: - app - tests @@ -254,7 +254,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.13.0: + - docker-compose#v4.14.0: run: tests ``` @@ -266,7 +266,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v4.13.0: + - docker-compose#v4.14.0: push: app ``` @@ -276,7 +276,7 @@ To push multiple images, you can use a list: steps: - label: ":docker: Push" plugins: - - docker-compose#v4.13.0: + - docker-compose#v4.14.0: push: - first-service - second-service @@ -288,7 +288,7 @@ If you want to push to a specific location (that's not defined as the `image` in steps: - label: ":docker: Push" plugins: - - docker-compose#v4.13.0: + - docker-compose#v4.14.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -302,14 +302,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker: Build an image" plugins: - - docker-compose#v4.13.0: + - docker-compose#v4.14.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.13.0: + - docker-compose#v4.14.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -327,7 +327,7 @@ This plugin allows for the value of `cache-from` to be a string or a list. If it steps: - label: ":docker Build an image" plugins: - - docker-compose#v4.13.0: + - docker-compose#v4.14.0: build: app image-repository: index.docker.io/myorg/myrepo separator-cache-from: "#" @@ -337,7 +337,7 @@ steps: - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.13.0: + - docker-compose#v4.14.0: push: - app:myregistry:port/myrepo/myapp:my-branch - app:myregistry:port/myrepo/myapp:latest @@ -350,7 +350,7 @@ Adding a grouping tag to the end of a cache-from list item allows this plugin to steps: - label: ":docker: Build Intermediate Image" plugins: - - docker-compose#v4.13.0: + - docker-compose#v4.14.0: build: myservice_intermediate # docker-compose.yml is the same as myservice but has `target: intermediate` image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo/myservice_intermediate @@ -360,7 +360,7 @@ steps: - wait - label: ":docker: Build Final Image" plugins: - - docker-compose#v4.13.0: + - docker-compose#v4.14.0: build: myservice image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo @@ -404,7 +404,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Run & Push" plugins: - - docker-compose#v4.13.0: + - docker-compose#v4.14.0: run: myservice push: myservice ``` @@ -419,7 +419,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Build & Push" plugins: - - docker-compose#v4.13.0: + - docker-compose#v4.14.0: build: myservice push: myservice ``` From 1bfc082c0080c4b913ecbee6a1fad4d8172d2541 Mon Sep 17 00:00:00 2001 From: Francois Campbell Date: Wed, 11 Oct 2023 22:20:43 -0400 Subject: [PATCH 245/253] Collapse logging of run args by default --- README.md | 8 ++++++++ commands/run.sh | 7 ++++++- plugin.yml | 2 ++ tests/run.bats | 53 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 36740236..9a0f0e1f 100644 --- a/README.md +++ b/README.md @@ -430,6 +430,14 @@ Will cause the image to be pushed twice (once by the build step and another by t Pull down multiple pre-built images. By default only the service that is being run will be pulled down, but this allows multiple images to be specified to handle prebuilt dependent images. Note that pulling will be skipped if the `skip-pull` option is activated. +### `collapse-run-log-group` (optional, boolean, run only) + +Whether to collapse or expand the log group that is created for the output of `docker-compose run`. When this setting is `true`, the output is collected into a `---` group, when `false` the output is collected into a `+++` group. Setting this to `false` can be useful to highlight your command's output if it does not create its own `+++` group. + +For more information see [Managing log output](https://buildkite.com/docs/pipelines/managing-log-output). + +Default `true` + ### `config` (optional) The file name of the Docker Compose configuration file to use. Can also be a list of filenames. If `$COMPOSE_FILE` is set, it will be used if `config` is not specified. diff --git a/commands/run.sh b/commands/run.sh index 8a2f2e0e..ac25b9de 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -427,10 +427,15 @@ ensure_stopped() { trap ensure_stopped SIGINT SIGTERM SIGQUIT +if [[ "${BUILDKITE_PLUGIN_DOCKER_COMPOSE_COLLAPSE_RUN_LOG_GROUP:-true}" = "true" ]]; then + group_type="---" +else + group_type="+++" +fi # Disable -e to prevent cancelling step if the command fails for whatever reason set +e ( # subshell is necessary to trap signals (compose v2 fails to stop otherwise) - echo "+++ :docker: Running ${display_command[*]:-} in service $run_service" >&2 + echo "${group_type} :docker: Running ${display_command[*]:-} in service $run_service" >&2 run_docker_compose "${run_params[@]}" ) exitcode=$? diff --git a/plugin.yml b/plugin.yml index cef5cf8e..8eb4291a 100644 --- a/plugin.yml +++ b/plugin.yml @@ -37,6 +37,8 @@ configuration: enum: [ 1, 2 ] command: type: array + collapse-run-log-group: + type: boolean config: type: [ string, array ] minimum: 1 diff --git a/tests/run.bats b/tests/run.bats index 1e004ea2..9e79c8d2 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -1622,3 +1622,56 @@ export BUILDKITE_JOB_ID=1111 assert_output --partial "env-propagation-list desired, but LIST_OF_VARS is not defined!" unstub buildkite-agent } + +@test "Run with collapsed run log group by default" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="echo hello world" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "--- :docker: Running /bin/sh -e -c 'echo hello world' in service myservice" + unstub docker-compose + unstub buildkite-agent +} + +@test "Run with expanded run log group" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="echo hello world" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_COLLAPSE_RUN_LOG_GROUP=false + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ + "-f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "+++ :docker: Running /bin/sh -e -c 'echo hello world' in service myservice" + unstub docker-compose + unstub buildkite-agent + + +} From 962d5c7d0832942da31a27df2c85811fde12680b Mon Sep 17 00:00:00 2001 From: Eugene Date: Thu, 12 Oct 2023 10:09:42 -0700 Subject: [PATCH 246/253] Add --skip-pull support to v2 run command The `docker-compose` plugin currently seems to ignore the `skip-pull` option. For example, this configuration: ``` steps: - command: bundle exec rubocop plugins: - docker-compose#v4.14.0: run: ruby skip-pull: true config: docker-compose.yml cli-version: 2 ``` Results in a command that includes the `--pull` flag, something like this: ``` docker compose -f docker-compose.yml -p buildkite0123456789 build --pull ruby ``` This PR updates run.sh to fully support the `skip-pull` option. The same configuration as above now results in the `--pull` flag being omitted from the generated command: ``` docker compose -f docker-compose.yml -p buildkite0123456789 build ruby ``` --- commands/run.sh | 7 ++++++- tests/v2/run.bats | 27 +++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/commands/run.sh b/commands/run.sh index 8a2f2e0e..3782b307 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -271,7 +271,12 @@ fi run_params+=("$run_service") -build_params=(--pull) +build_params=() + +# Only pull if SKIP_PULL is not true +if [[ ! "$(plugin_read_config SKIP_PULL "false")" == "true" ]] ; then + build_params+=(--pull) +fi if [[ "$(plugin_read_config NO_CACHE "false")" == "true" ]] ; then build_params+=(--no-cache) diff --git a/tests/v2/run.bats b/tests/v2/run.bats index e34f6718..d8dcf64e 100644 --- a/tests/v2/run.bats +++ b/tests/v2/run.bats @@ -259,6 +259,33 @@ cmd3" unstub buildkite-agent } +@test "Run without a prebuilt image without pulling" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="echo hello world" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_SKIP_PULL=true + + stub docker \ + "compose -f docker-compose.yml -p buildkite1111 build myservice : echo built myservice" \ + "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ + "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run "$PWD"/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "ran myservice" + unstub docker + unstub buildkite-agent +} + @test "Run with a prebuilt image and propagate environment but no BUILDKITE_ENV_FILE" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PIPELINE_SLUG=test From 335f48ab37aacfb42bb013945661794c7360050b Mon Sep 17 00:00:00 2001 From: "Pol (Paula)" Date: Thu, 12 Oct 2023 19:17:58 -0300 Subject: [PATCH 247/253] Update cache-from info --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 36740236..7f40cd79 100644 --- a/README.md +++ b/README.md @@ -544,7 +544,8 @@ This option can also be configured on the agent machine using the environment va ### `cache-from` (optional, build only) -A list of images to pull caches from in the format `service:index.docker.io/myorg/myrepo/myapp:tag` before building, ignoring any failures. If multiple images are listed for a service, the first one to successfully pull will be used. Requires docker-compose file version `3.2+`. +A list of images to pull caches from in the format `service:index.docker.io/myorg/myrepo/myapp:tag:group` before building, ignoring any failures. The parameters `service` and `image-repo` are mandatory, without them it won't work. If multiple images are listed for a service, the first one to successfully pull will be used. Adding a grouping tag to the end of a `cache-from` list item allows this plugin to differentiate between groups within which only the first successfully downloaded image should be used (those elements that don't have a group specified will make a separate `:default:` group of its own). This way, not all images need to be downloaded and used as cache, not just the first. +Requires docker-compose file version `3.2+`. ### `separator-cache-from` (optional, build only, single character) From 9c1cfbee373a40f4211addda096c55355dd4cdde Mon Sep 17 00:00:00 2001 From: "Pol (Paula)" Date: Thu, 12 Oct 2023 19:28:27 -0300 Subject: [PATCH 248/253] rephrase explanation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7f40cd79..2d1efbd9 100644 --- a/README.md +++ b/README.md @@ -544,7 +544,7 @@ This option can also be configured on the agent machine using the environment va ### `cache-from` (optional, build only) -A list of images to pull caches from in the format `service:index.docker.io/myorg/myrepo/myapp:tag:group` before building, ignoring any failures. The parameters `service` and `image-repo` are mandatory, without them it won't work. If multiple images are listed for a service, the first one to successfully pull will be used. Adding a grouping tag to the end of a `cache-from` list item allows this plugin to differentiate between groups within which only the first successfully downloaded image should be used (those elements that don't have a group specified will make a separate `:default:` group of its own). This way, not all images need to be downloaded and used as cache, not just the first. +A list of images to attempt pulling before building in the format `service:index.docker.io/myorg/myrepo/myapp:tag:group`, ignoring any failures, to allow docker to re-use layers. The parameters `service` and `image-repo` are mandatory, without them it won't work. For each combination of service and group, it will attempt to pull each in order until one is successful (the rest will be ignored). Those elements that don't have a group specified will use a `:default:` group. Requires docker-compose file version `3.2+`. ### `separator-cache-from` (optional, build only, single character) From 10b47e7bcf64a7a28288d59a169ac3e658b3a523 Mon Sep 17 00:00:00 2001 From: Francois Campbell Date: Mon, 16 Oct 2023 14:56:07 -0400 Subject: [PATCH 249/253] flip default --- README.md | 4 ++-- commands/run.sh | 2 +- tests/run.bats | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 9a0f0e1f..4a7d81b3 100644 --- a/README.md +++ b/README.md @@ -432,11 +432,11 @@ Pull down multiple pre-built images. By default only the service that is being r ### `collapse-run-log-group` (optional, boolean, run only) -Whether to collapse or expand the log group that is created for the output of `docker-compose run`. When this setting is `true`, the output is collected into a `---` group, when `false` the output is collected into a `+++` group. Setting this to `false` can be useful to highlight your command's output if it does not create its own `+++` group. +Whether to collapse or expand the log group that is created for the output of `docker-compose run`. When this setting is `true`, the output is collected into a `---` group, when `false` the output is collected into a `+++` group. Setting this to `true` can be useful to de-emphasize plugin output if your command creates its own `+++` group. For more information see [Managing log output](https://buildkite.com/docs/pipelines/managing-log-output). -Default `true` +Default `false` ### `config` (optional) diff --git a/commands/run.sh b/commands/run.sh index ac25b9de..c24240e4 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -427,7 +427,7 @@ ensure_stopped() { trap ensure_stopped SIGINT SIGTERM SIGQUIT -if [[ "${BUILDKITE_PLUGIN_DOCKER_COMPOSE_COLLAPSE_RUN_LOG_GROUP:-true}" = "true" ]]; then +if [[ "${BUILDKITE_PLUGIN_DOCKER_COMPOSE_COLLAPSE_RUN_LOG_GROUP:-false}" = "true" ]]; then group_type="---" else group_type="+++" diff --git a/tests/run.bats b/tests/run.bats index 9e79c8d2..3569addd 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -1623,7 +1623,7 @@ export BUILDKITE_JOB_ID=1111 unstub buildkite-agent } -@test "Run with collapsed run log group by default" { +@test "Run with expanded run log group by default" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice export BUILDKITE_PIPELINE_SLUG=test @@ -1643,12 +1643,12 @@ export BUILDKITE_JOB_ID=1111 run "$PWD"/hooks/command assert_success - assert_output --partial "--- :docker: Running /bin/sh -e -c 'echo hello world' in service myservice" + assert_output --partial "+++ :docker: Running /bin/sh -e -c 'echo hello world' in service myservice" unstub docker-compose unstub buildkite-agent } -@test "Run with expanded run log group" { +@test "Run with collapsed run log group" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice export BUILDKITE_PIPELINE_SLUG=test @@ -1656,7 +1656,7 @@ export BUILDKITE_JOB_ID=1111 export BUILDKITE_COMMAND="echo hello world" export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false - export BUILDKITE_PLUGIN_DOCKER_COMPOSE_COLLAPSE_RUN_LOG_GROUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_COLLAPSE_RUN_LOG_GROUP=true stub docker-compose \ "-f docker-compose.yml -p buildkite1111 build --pull myservice : echo built myservice" \ @@ -1669,7 +1669,7 @@ export BUILDKITE_JOB_ID=1111 run "$PWD"/hooks/command assert_success - assert_output --partial "+++ :docker: Running /bin/sh -e -c 'echo hello world' in service myservice" + assert_output --partial "--- :docker: Running /bin/sh -e -c 'echo hello world' in service myservice" unstub docker-compose unstub buildkite-agent From 70aa7f79d55c6851bbcf787fc5977ad946acc0d9 Mon Sep 17 00:00:00 2001 From: Francois Campbell Date: Mon, 23 Oct 2023 10:26:44 -0400 Subject: [PATCH 250/253] Update README.md --- README.md | 54 +++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 4a7d81b3..c3e05f7e 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v4.14.0: + - docker-compose#v4.15.0: run: app ``` @@ -28,7 +28,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v4.14.0: + - docker-compose#v4.15.0: run: app config: docker-compose.tests.yml env: @@ -41,7 +41,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v4.14.0: + - docker-compose#v4.15.0: run: app config: - docker-compose.yml @@ -56,7 +56,7 @@ env: steps: - command: test.sh plugins: - - docker-compose#v4.14.0: + - docker-compose#v4.15.0: run: app ``` @@ -65,7 +65,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v4.14.0: + - docker-compose#v4.15.0: run: app command: ["custom", "command", "values"] ``` @@ -79,7 +79,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.14.0: + - docker-compose#v4.15.0: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -87,7 +87,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v4.14.0: + - docker-compose#v4.15.0: run: app ``` @@ -104,7 +104,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.14.0: + - docker-compose#v4.15.0: run: app ``` @@ -122,7 +122,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v4.14.0: + - docker-compose#v4.15.0: run: app volumes: - "./dist:/app/dist" @@ -146,7 +146,7 @@ this plugin offers a `environment` block of its own: steps: - command: generate-dist.sh plugins: - - docker-compose#v4.14.0: + - docker-compose#v4.15.0: run: app env: - BUILDKITE_BUILD_NUMBER @@ -164,7 +164,7 @@ Alternatively, you can have the plugin add all environment variables defined for steps: - command: use-vars.sh plugins: - - docker-compose#v4.14.0: + - docker-compose#v4.15.0: run: app propagate-environment: true ``` @@ -199,7 +199,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v4.14.0: + - docker-compose#v4.15.0: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -216,7 +216,7 @@ If you have multiple steps that use the same service/image (such as steps that r steps: - label: ":docker: Build" plugins: - - docker-compose#v4.14.0: + - docker-compose#v4.15.0: build: app image-repository: index.docker.io/myorg/myrepo @@ -226,7 +226,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.14.0: + - docker-compose#v4.15.0: run: app ``` @@ -242,7 +242,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v4.14.0: + - docker-compose#v4.15.0: build: - app - tests @@ -254,7 +254,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v4.14.0: + - docker-compose#v4.15.0: run: tests ``` @@ -266,7 +266,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v4.14.0: + - docker-compose#v4.15.0: push: app ``` @@ -276,7 +276,7 @@ To push multiple images, you can use a list: steps: - label: ":docker: Push" plugins: - - docker-compose#v4.14.0: + - docker-compose#v4.15.0: push: - first-service - second-service @@ -288,7 +288,7 @@ If you want to push to a specific location (that's not defined as the `image` in steps: - label: ":docker: Push" plugins: - - docker-compose#v4.14.0: + - docker-compose#v4.15.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -302,14 +302,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker: Build an image" plugins: - - docker-compose#v4.14.0: + - docker-compose#v4.15.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.14.0: + - docker-compose#v4.15.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -327,7 +327,7 @@ This plugin allows for the value of `cache-from` to be a string or a list. If it steps: - label: ":docker Build an image" plugins: - - docker-compose#v4.14.0: + - docker-compose#v4.15.0: build: app image-repository: index.docker.io/myorg/myrepo separator-cache-from: "#" @@ -337,7 +337,7 @@ steps: - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v4.14.0: + - docker-compose#v4.15.0: push: - app:myregistry:port/myrepo/myapp:my-branch - app:myregistry:port/myrepo/myapp:latest @@ -350,7 +350,7 @@ Adding a grouping tag to the end of a cache-from list item allows this plugin to steps: - label: ":docker: Build Intermediate Image" plugins: - - docker-compose#v4.14.0: + - docker-compose#v4.15.0: build: myservice_intermediate # docker-compose.yml is the same as myservice but has `target: intermediate` image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo/myservice_intermediate @@ -360,7 +360,7 @@ steps: - wait - label: ":docker: Build Final Image" plugins: - - docker-compose#v4.14.0: + - docker-compose#v4.15.0: build: myservice image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo @@ -404,7 +404,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Run & Push" plugins: - - docker-compose#v4.14.0: + - docker-compose#v4.15.0: run: myservice push: myservice ``` @@ -419,7 +419,7 @@ A basic pipeline similar to the following: steps: - label: ":docker: Build & Push" plugins: - - docker-compose#v4.14.0: + - docker-compose#v4.15.0: build: myservice push: myservice ``` From d86f2095d154556e43ab782e4e789e6c3ac9576c Mon Sep 17 00:00:00 2001 From: Brian Reath Date: Thu, 10 Sep 2020 10:49:09 -0500 Subject: [PATCH 251/253] Add --include-deps option to Docker image pulls and retries to Docker service startups --- commands/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/commands/run.sh b/commands/run.sh index 722f2573..a0aae523 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -64,9 +64,9 @@ fi # If there are multiple services to pull, run it in parallel (although this is now the default) if [[ ${#pull_services[@]} -gt 1 ]] ; then - pull_params+=("pull" "--parallel" "${pull_services[@]}") + pull_params+=("pull" "--parallel" "--include-deps" "${pull_services[@]}") elif [[ ${#pull_services[@]} -eq 1 ]] ; then - pull_params+=("pull" "${pull_services[0]}") + pull_params+=("pull" "--include-deps" "${pull_services[0]}") fi # Pull down specified services From 3612a724f53fb2e78dee88b78a5c63071203a667 Mon Sep 17 00:00:00 2001 From: David Kutalek Date: Wed, 8 May 2024 13:46:22 -0700 Subject: [PATCH 252/253] merge changes from branch --- commands/run.sh | 31 +++++++++++------------- hooks/pre-exit | 2 +- lib/log.bash | 40 +++++++++++++++++++++++++++++++ lib/run.bash | 27 +++++++++++++++------ tests/cleanup.bats | 4 ++-- tests/docker-compose-cleanup.bats | 19 +++++++-------- 6 files changed, 86 insertions(+), 37 deletions(-) create mode 100644 lib/log.bash mode change 100644 => 100755 tests/cleanup.bats diff --git a/commands/run.sh b/commands/run.sh index a0aae523..4edd9cbe 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -1,6 +1,8 @@ #!/bin/bash set -ueo pipefail +. "$DIR/../lib/log.bash" + # Run takes a service name, pulls down any pre-built image for that name # and then runs docker-compose run a generated project name @@ -423,35 +425,30 @@ elif [[ ${#command[@]} -gt 0 ]] ; then fi ensure_stopped() { - echo '+++ :warning: Signal received, stopping container' - docker stop "${container_name}" || true + log 143 + echo '+++ :warning: Signal received, stopping container gracefully' + # docker stop "${container_name}" || true + compose_cleanup ${run_service} echo '~~~ Last log lines that may be missing above (if container was not already removed)' docker logs "${container_name}" || true - exitcode='TRAP' + exit 143 } -trap ensure_stopped SIGINT SIGTERM SIGQUIT +trap 'ensure_stopped "$?"' SIGINT SIGTERM SIGQUIT if [[ "${BUILDKITE_PLUGIN_DOCKER_COMPOSE_COLLAPSE_RUN_LOG_GROUP:-false}" = "true" ]]; then group_type="---" else group_type="+++" fi -# Disable -e to prevent cancelling step if the command fails for whatever reason -set +e -( # subshell is necessary to trap signals (compose v2 fails to stop otherwise) + +exitcode=0 +( echo "${group_type} :docker: Running ${display_command[*]:-} in service $run_service" >&2 run_docker_compose "${run_params[@]}" -) -exitcode=$? - -# Restore -e as an option. -set -e +) || exitcode=$? -if [[ $exitcode = "TRAP" ]]; then - # command failed due to cancellation signal, make sure there is an error but no further output - exitcode=-1 -elif [[ $exitcode -ne 0 ]] ; then +if [[ $exitcode -ne 0 ]] ; then echo "^^^ +++" echo "+++ :warning: Failed to run command, exited with $exitcode, run params:" echo "${run_params[@]}" @@ -465,4 +462,4 @@ if [[ -n "${BUILDKITE_AGENT_ACCESS_TOKEN:-}" ]] ; then fi fi -return "$exitcode" +return "$exitcode" \ No newline at end of file diff --git a/hooks/pre-exit b/hooks/pre-exit index f3bd61d4..8f1b4d82 100755 --- a/hooks/pre-exit +++ b/hooks/pre-exit @@ -15,5 +15,5 @@ if [[ -n "$(plugin_read_list RUN)" ]] && [[ "$(plugin_read_config CLEANUP "true" . "$DIR/../lib/run.bash" echo "~~~ :docker: Cleaning up after docker-compose" >&2 - compose_cleanup + compose_cleanup "" fi diff --git a/lib/log.bash b/lib/log.bash new file mode 100644 index 00000000..d66e4980 --- /dev/null +++ b/lib/log.bash @@ -0,0 +1,40 @@ +#!/bin/bash + +log() { + msg="SIG $1 received, process exiting" + echo "${msg}" + buildkite-agent meta-data set "dd_tags.job-signal-${BUILDKITE_STEP_KEY}" "$1" + buildkite-agent meta-data set "dd_tags.step-error-code-${BUILDKITE_STEP_KEY}" "$1" + buildkite-agent meta-data set "dd_tags.job-error-code-${BUILDKITE_JOB_ID}" "$1" + + echo "$(pidof buildkite-agent) is the pid of the buildkite agent" || true + + send_job_signaled_to_dd "${msg}" "${1}" +} + +send_job_signaled_to_dd() { + send_event_to_dd '{ "title": "Job '"${BUILDKITE_STEP_KEY}"' received signal", "text": "'"${1}"'", "alert_type": "error", "tags": [ "ci:job_signal", "exit_status:'"${2}"'", "job_name:'"${BUILDKITE_STEP_KEY}"'", "build_id:'"${BUILDKITE_BUILD_ID}"'", "branch:'"${BUILDKITE_BRANCH}"'", "hs_source:docker_compose_plugin", "env:ci" ] }' +} + +send_event_to_dd() { + if command -v curl >/dev/null 2>&1; then + echo "Using curl to send event to Datadog" + curl -X POST "https://api.datadoghq.com/api/v1/events" \ + -H "Accept: application/json" \ + -H "Content-Type: application/json" \ + -H "DD-API-KEY: ${DD_API_KEY}" \ + -d "$1" + elif command -v wget >/dev/null 2>&1; then + echo "Using wget to send event to Datadog" + wget \ + --header="Accept: application/json" \ + --header="Content-Type: application/json" \ + --header="DD-API-KEY: ${DD_API_KEY}" \ + --post-data="$1" \ + --output-document - \ + https://api.datadoghq.com/api/v1/events + else + echo "No suitable network tool found to send event to Datadog" + exit 1 + fi +} diff --git a/lib/run.bash b/lib/run.bash index f531292a..522ef069 100644 --- a/lib/run.bash +++ b/lib/run.bash @@ -1,14 +1,15 @@ #!/bin/bash -compose_cleanup() { - if [[ "$(plugin_read_config GRACEFUL_SHUTDOWN 'false')" == "false" ]]; then - # Send all containers a SIGKILL - run_docker_compose kill || true - else - # Send all containers a friendly SIGTERM, followed by a SIGKILL after exceeding the stop_grace_period - run_docker_compose stop || true +kill_or_wait_for_stop() { + + if [[ "$(plugin_read_config GRACEFUL_SHUTDOWN 'false')" == "true" ]]; then + # This will block until the container exits + run_docker_compose wait "$1" + container_exit_code=$? + echo "exit code was $container_exit_code" fi + # This will kill the container if it hasn't exited yet # `compose down` doesn't support force removing images if [[ "$(plugin_read_config LEAVE_VOLUMES 'false')" == "false" ]]; then run_docker_compose rm --force -v || true @@ -24,6 +25,18 @@ compose_cleanup() { fi } +compose_cleanup() { + kill_or_wait_for_stop "$1" & + sleep 1 + + # No need to call kill directly for GRACEFUL_SHUTDOWN == false since rm --force will send the same kill signal + if [[ "$(plugin_read_config GRACEFUL_SHUTDOWN 'false')" == "true" ]]; then + echo "graceful shutdown was true, stopping ${1}" + # Send all containers a friendly SIGTERM, followed by a SIGKILL after exceeding the stop_grace_period + run_docker_compose stop "$1" || true + fi +} + # Checks for failed containers and writes logs for them the the provided dir check_linked_containers_and_save_logs() { local service="$1" diff --git a/tests/cleanup.bats b/tests/cleanup.bats old mode 100644 new mode 100755 index adb69f18..1bec6d49 --- a/tests/cleanup.bats +++ b/tests/cleanup.bats @@ -17,13 +17,13 @@ load '../lib/run' export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=true stub docker-compose \ - "-f docker-compose.yml -p buildkite1111 kill : echo killing containers" \ - "-f docker-compose.yml -p buildkite1111 rm --force -v : echo removing stopped containers" \ + "-f docker-compose.yml -p buildkite1111 rm --force -v : echo killing and removing stopped containers" \ "-f docker-compose.yml -p buildkite1111 down --remove-orphans --volumes : echo removing everything" run "$PWD"/hooks/pre-exit assert_success assert_output --partial "Cleaning up after docker-compose" + unstub docker-compose } diff --git a/tests/docker-compose-cleanup.bats b/tests/docker-compose-cleanup.bats index a8843434..7cd3ada6 100644 --- a/tests/docker-compose-cleanup.bats +++ b/tests/docker-compose-cleanup.bats @@ -15,19 +15,19 @@ setup () { run compose_cleanup assert_success - assert_equal "${lines[0]}" "kill" - assert_equal "${lines[1]}" "rm --force -v" - assert_equal "${lines[2]}" "down --remove-orphans --volumes" + assert_equal "${lines[0]}" "rm --force -v" + assert_equal "${lines[1]}" "down --remove-orphans --volumes" } @test "Possible to gracefully shutdown containers in docker-compose cleanup" { - export BUILDKITE_PLUGIN_DOCKER_COMPOSE_GRACEFUL_SHUTDOWN=1 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_GRACEFUL_SHUTDOWN="true" run compose_cleanup assert_success - assert_equal "${lines[0]}" "stop" - assert_equal "${lines[1]}" "rm --force -v" - assert_equal "${lines[2]}" "down --remove-orphans --volumes" + assert_output --partial "wait" + assert_equal "${lines[1]}" "exit code was 0" + assert_equal "${lines[2]}" "rm --force -v" + assert_equal "${lines[3]}" "down --remove-orphans --volumes" } @test "Possible to skip volume destruction in docker-compose cleanup" { @@ -35,7 +35,6 @@ setup () { run compose_cleanup assert_success - assert_equal "${lines[0]}" "kill" - assert_equal "${lines[1]}" "rm --force" - assert_equal "${lines[2]}" "down --remove-orphans" + assert_equal "${lines[0]}" "rm --force" + assert_equal "${lines[1]}" "down --remove-orphans" } From 78291486627ed8515a2ad404ba5d8193f025d2fb Mon Sep 17 00:00:00 2001 From: David Kutalek Date: Tue, 11 Jun 2024 11:58:20 -0700 Subject: [PATCH 253/253] prevent process restarts when sig term received --- commands/run.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/commands/run.sh b/commands/run.sh index 4edd9cbe..e7926369 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -426,9 +426,9 @@ fi ensure_stopped() { log 143 - echo '+++ :warning: Signal received, stopping container gracefully' - # docker stop "${container_name}" || true - compose_cleanup ${run_service} + echo '+++ :warning: Signal received, stopping container' + docker stop "${container_name}" || true + # compose_cleanup ${run_service} # This works but causes the docker process time to restart before the container dies. This must be a docker setting somewhere causing the restart, but I can't find it echo '~~~ Last log lines that may be missing above (if container was not already removed)' docker logs "${container_name}" || true exit 143