diff --git a/.github/docker-memory-monitor/action.yml b/.github/docker-memory-monitor/action.yml index 1909cc56..e127f083 100644 --- a/.github/docker-memory-monitor/action.yml +++ b/.github/docker-memory-monitor/action.yml @@ -26,9 +26,27 @@ runs: echo echo "== container OOM status ==" + # Status alone hides two cases: a container that restarted between samples, and one that + # reports healthy but no longer serves (the healthcheck runs inside the container, + # against localhost). for c in $(docker ps -aq 2>/dev/null); do docker inspect "$c" \ - --format 'Name={{.Name}} Status={{.State.Status}} OOMKilled={{.State.OOMKilled}} ExitCode={{.State.ExitCode}} Memory={{.HostConfig.Memory}} MemorySwap={{.HostConfig.MemorySwap}}' + --format 'Name={{.Name}} Status={{.State.Status}} OOMKilled={{.State.OOMKilled}} ExitCode={{.State.ExitCode}} RestartCount={{.RestartCount}} StartedAt={{.State.StartedAt}} Health={{if .State.Health}}{{.State.Health.Status}}/{{.State.Health.FailingStreak}}{{else}}none{{end}} Memory={{.HostConfig.Memory}} MemorySwap={{.HostConfig.MemorySwap}}' + done + + # Whether the stack still answers from outside, and how fast. Container state and pod + # readiness both stay green when a process is listening but no longer serving, so this is + # the only sample that tells the two apart. Same URLs in both environments. + echo + echo "== service reachability (http_code / total_time_s) ==" + for svc in "kibana https://localhost:5601/api/status" "es https://localhost:9200/_cluster/health"; do + name="${svc%% *}"; url="${svc#* }" + printf '%s ' "$name" + # -w still prints its line when curl fails (http_code=000), so no fallback echo. + curl -sk -o /dev/null -u kibana:kibana \ + --max-time 10 \ + -w 'http_code=%{http_code} total_time=%{time_total} connect_time=%{time_connect}\n' \ + "$url" 2>/dev/null || true done echo @@ -47,6 +65,18 @@ runs: echo "== kubectl OOM events ==" kubectl get events -A --field-selector=reason=OOMKilling 2>/dev/null || true + # Restarts, probe failures and evictions, which reason=OOMKilling above does not cover. + # + # Printed with lastTimestamp rather than the default relative AGE: this loop re-dumps the + # whole event list every 10s, so without an absolute timestamp old startup events are + # indistinguishable from live ones. + echo + echo "== kubectl warning events (newest last, absolute timestamps) ==" + kubectl get events -A --field-selector=type=Warning \ + --sort-by=.lastTimestamp \ + -o custom-columns='LAST:.lastTimestamp,COUNT:.count,NS:.metadata.namespace,REASON:.reason,OBJECT:.involvedObject.name,MESSAGE:.message' \ + 2>/dev/null | tail -25 || true + echo sleep 10 done diff --git a/.github/workflows/all-e2e-tests.yml b/.github/workflows/all-e2e-tests.yml index fb1e412b..96b688df 100644 --- a/.github/workflows/all-e2e-tests.yml +++ b/.github/workflows/all-e2e-tests.yml @@ -4,20 +4,31 @@ on: workflow_dispatch: {} schedule: - cron: '0 0 * * *' + # Only the two long-lived branches. Feature branches are covered by their pull request, and + # listing them here would run everything twice for every push to an open PR. + push: + branches: [master, develop] pull_request: types: [opened, synchronize, reopened] +env: + APPLY_RESOURCE_LIMITS: "auto" + jobs: # ========================================== - # E2E TESTS - MASTER BRANCH + # E2E TESTS - RELEASED (PROD) PLUGIN IMAGES # ========================================== - master-e2e-tests: - name: "๐Ÿ”ฌ Master E2E Tests" + # Runs against the RELEASED plugin images (`--mode prod`, ror-latest). The signal is "the shipped + # plugins still pass the suite": master itself, PRs targeting it, and the nightly schedule. + # + # Keep the matrix in sync with ELK_VERSIONS in prepare-dev-images. + prod-e2e-tests: + name: "๐Ÿ”ฌ E2E Tests (released plugins)" if: > github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || - github.ref == 'refs/heads/master' || - (github.base_ref == 'master' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false)) + (github.event_name == 'push' && github.ref == 'refs/heads/master') || + (github.event_name == 'pull_request' && github.base_ref == 'master' && github.event.pull_request.head.repo.fork == false) runs-on: ubuntu-latest strategy: fail-fast: false @@ -47,11 +58,11 @@ jobs: uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 with: max_attempts: 2 - timeout_minutes: 60 + timeout_minutes: 35 retry_wait_seconds: 120 retry_on: any command: | - ./runner.sh --run e2e --env ${{ matrix.env }} --elk ${{ matrix.version }} + ./runner.sh --run e2e --env ${{ matrix.env }} --elk ${{ matrix.version }} --mode prod env: ROR_ACTIVATION_KEY: ${{ secrets.ROR_KBN_LICENSE }} ELECTRON_EXTRA_LAUNCH_ARGS: '--disable-gpu' @@ -62,6 +73,7 @@ jobs: action: stop - name: S3 Upload Videos & show logs if: failure() + continue-on-error: true uses: ./.github/upload-videos with: access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -71,14 +83,16 @@ jobs: # ========================================== # BOOTSTRAP TESTS # ========================================== + # Same condition as prod-e2e-tests, which this gates on โ€” kept identical so the two cannot drift. + # Bootstrap uses released images (no --mode), as it did before. master-bootstrap-tests: name: "๐Ÿš€ Bootstrap Tests" if: > github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || - github.ref == 'refs/heads/master' || - (github.base_ref == 'master' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false)) - needs: master-e2e-tests + (github.event_name == 'push' && github.ref == 'refs/heads/master') || + (github.event_name == 'pull_request' && github.base_ref == 'master' && github.event.pull_request.head.repo.fork == false) + needs: prod-e2e-tests runs-on: ubuntu-latest strategy: fail-fast: false @@ -117,23 +131,69 @@ jobs: action: stop # ========================================== - # E2E TESTS - DEVELOP BRANCH + # DEV IMAGE PREPARATION - DEVELOP AND NON-MASTER PRs # ========================================== - develop-e2e-tests: - name: "๐Ÿงช Develop E2E Tests" + # Builds branch-matched dev images of both plugins for dev-e2e-tests, tagged per run. One dispatch + # per plugin covers the whole matrix, since both pre-build workflows accept a version list. + # `target_branch` is passed verbatim; both fall back to `develop` if the branch is not there. + # Fork PRs are excluded: the dispatch needs secrets GitHub does not expose to them. + prepare-dev-images: + name: "๐Ÿ—๏ธ Prepare dev images" if: > - github.ref == 'refs/heads/develop' || - (github.base_ref == 'develop' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false)) + (github.event_name == 'push' && github.ref == 'refs/heads/develop') || + (github.event_name == 'pull_request' && github.base_ref != 'master' && github.event.pull_request.head.repo.fork == false) + runs-on: ubuntu-latest + timeout-minutes: 90 + outputs: + run_tag: ${{ steps.prepare.outputs.run_tag }} + versions: ${{ steps.prepare.outputs.versions }} + steps: + - name: Checkout code + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + persist-credentials: false + - name: Dispatch and await ROR plugin pre-builds + id: prepare + env: + # Mirrors the prod-e2e-tests matrix โ€” keep both in sync. + ELK_VERSIONS: "9.4.4 9.3.8 8.19.19 7.17.29" + # On a PR, head_ref is the source branch โ€” the plugins must be built from the PR head, not + # from the merge ref github.ref points at. On a push, head_ref is empty and ref_name is + # the branch that was pushed (develop). + TARGET_BRANCH: ${{ github.head_ref || github.ref_name }} + ES_REPO_GH_TOKEN: ${{ secrets.ES_REPO_GH_TOKEN }} + KBN_REPO_GH_TOKEN: ${{ secrets.KBN_REPO_GH_TOKEN }} + run: | + set -euo pipefail + . ci/prebuild-images-lib.sh + + # Unique per attempt: a re-run must not silently reuse the previous attempt's images. + RUN_TAG="run-${{ github.run_id }}-${{ github.run_attempt }}" + { + echo "run_tag=$RUN_TAG" + echo "versions=$(printf '%s\n' $ELK_VERSIONS | jq -Rcn '[inputs]')" + } >> "$GITHUB_OUTPUT" + + dispatch_prebuild_images "$ELK_VERSIONS" "$TARGET_BRANCH" "$RUN_TAG" + wait_for_prebuild_images "$ELK_VERSIONS" "$RUN_TAG" + + # ========================================== + # E2E TESTS - PRE-BUILD (DEV) PLUGIN IMAGES + # ========================================== + # Every push to develop and every non-fork pull request that does not target master, against the + # per-run, branch-matched dev images produced by prepare-dev-images. Skipped automatically when + # that job is skipped. + dev-e2e-tests: + name: "๐Ÿงช E2E Tests (pre-build plugins)" + needs: prepare-dev-images runs-on: ubuntu-latest strategy: fail-fast: false matrix: - version: ["9.4.4", "9.3.8", "8.19.19", "7.17.29"] + version: ${{ fromJSON(needs.prepare-dev-images.outputs.versions) }} env: [docker, eck-2.16.1, eck-3.4.1] env: - ROR_ES_VERSION: "latest" - ROR_KBN_VERSION: "latest" - MODE: 'dev' + ROR_IMAGE_TAG: ${{ needs.prepare-dev-images.outputs.run_tag }} steps: - name: Checkout code uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 @@ -157,11 +217,11 @@ jobs: uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 with: max_attempts: 2 - timeout_minutes: 60 + timeout_minutes: 35 retry_wait_seconds: 120 retry_on: any command: | - ./runner.sh --run e2e --env ${{ matrix.env }} --elk ${{ matrix.version }} --ror-es ${{ env.ROR_ES_VERSION }} --ror-kbn ${{ env.ROR_KBN_VERSION }} --mode ${{ env.MODE }} + ./runner.sh --run e2e --env ${{ matrix.env }} --elk ${{ matrix.version }} --ror-es ${{ env.ROR_IMAGE_TAG }} --ror-kbn ${{ env.ROR_IMAGE_TAG }} --mode dev env: ROR_ACTIVATION_KEY: ${{ secrets.ROR_KBN_LICENSE }} @@ -172,7 +232,9 @@ jobs: action: stop - name: S3 Upload Videos & show logs if: failure() + continue-on-error: true uses: ./.github/upload-videos - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + with: + access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + endpoint_url: ${{ secrets.AWS_ENDPOINT_URL }} diff --git a/.github/workflows/targeted-e2e-tests.yml b/.github/workflows/targeted-e2e-tests.yml index 2fb686bd..e421a920 100644 --- a/.github/workflows/targeted-e2e-tests.yml +++ b/.github/workflows/targeted-e2e-tests.yml @@ -25,6 +25,9 @@ on: default: 'prod' type: string +env: + APPLY_RESOURCE_LIMITS: "auto" + jobs: e2e-tests: name: "๐Ÿ”ฌ E2E Tests" @@ -77,6 +80,7 @@ jobs: - name: S3 Upload Videos & show logs if: failure() + continue-on-error: true uses: ./.github/upload-videos with: access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} diff --git a/ci/prebuild-images-lib.sh b/ci/prebuild-images-lib.sh new file mode 100644 index 00000000..80e395ad --- /dev/null +++ b/ci/prebuild-images-lib.sh @@ -0,0 +1,499 @@ +# Helpers for producing ROR plugin dev images: trigger the pre-build workflow in a plugin repo, then +# wait for the image it publishes to appear in the registry. +# +# Sourced, not executed. +# +# Needs `gh` and `jq` to dispatch and to follow the run, and `docker` to probe the registry, each +# only when the matching function is called. +# +# Nothing relies on the caller using `set -e`: every function returns non-zero on failure, and the +# ones that call others pass that status on with `|| return $?`. + +# Do nothing if this file was already sourced. +if [ -n "${_ROR_PREBUILD_IMAGES_LIB_SOURCED:-}" ]; then + return 0 2>/dev/null || true +fi +_ROR_PREBUILD_IMAGES_LIB_SOURCED=1 + +# --- Coordinates ------------------------------------------------------------------------------- +# Every value below can be overridden from the environment; the defaults are the real ones. + +# Each plugin publishes its pre-build images from a manually-triggered GitHub Actions workflow. +ROR_KBN_GH_REPO="${ROR_KBN_GH_REPO:-sscarduzio/readonlyrest_kbn}" +ROR_KBN_PUBLISH_WORKFLOW="${ROR_KBN_PUBLISH_WORKFLOW:-publish-pre-builds.yml}" + +ROR_ES_GH_REPO="${ROR_ES_GH_REPO:-sscarduzio/elasticsearch-readonlyrest-plugin}" +ROR_ES_PUBLISH_WORKFLOW="${ROR_ES_PUBLISH_WORKFLOW:-publish-pre-builds.yml}" + +# Which ref the workflow file itself is read from. That is a different thing from which sources get +# built, which is the target branch passed to the workflow as an input. It matters when the workflow +# is being changed on a branch, so that the branch's own copy runs. +# +# "auto" uses the target branch when it exists in the plugin repo, otherwise the fallback ref below. +# Set a literal ref to pin one, or empty to let gh use the repo's default branch. +# +# This cannot make an unregistered workflow dispatchable: GitHub only allows dispatching a workflow +# whose file is on the repo's default branch, and gh looks it up there before applying --ref. +ROR_ES_PUBLISH_WORKFLOW_REF="${ROR_ES_PUBLISH_WORKFLOW_REF-auto}" +ROR_KBN_PUBLISH_WORKFLOW_REF="${ROR_KBN_PUBLISH_WORKFLOW_REF-auto}" +ROR_ES_PUBLISH_WORKFLOW_FALLBACK_REF="${ROR_ES_PUBLISH_WORKFLOW_FALLBACK_REF:-develop}" +ROR_KBN_PUBLISH_WORKFLOW_FALLBACK_REF="${ROR_KBN_PUBLISH_WORKFLOW_FALLBACK_REF:-develop}" + +ROR_ES_DEV_IMAGE_REPO="${ROR_ES_DEV_IMAGE_REPO:-beshultd/elasticsearch-readonlyrest-dev}" +ROR_KBN_DEV_IMAGE_REPO="${ROR_KBN_DEV_IMAGE_REPO:-beshultd/kibana-readonlyrest-dev}" + +# How long to wait for a published image before giving up. ES gets the longer limit because its +# build takes longer than the Kibana one. +ROR_ES_WAIT_TIMEOUT_SECONDS="${ROR_ES_WAIT_TIMEOUT_SECONDS:-$((45 * 60))}" +ROR_KBN_WAIT_TIMEOUT_SECONDS="${ROR_KBN_WAIT_TIMEOUT_SECONDS:-$((30 * 60))}" + +# Every poll costs one registry manifest request, and Docker Hub counts those against the pull rate +# limit (100 per 6h for an unauthenticated IP, which is what CI is unless a docker login step is +# added). Eight images at this interval stay well inside that; halving it would not. +ROR_PREBUILD_POLL_INTERVAL_SECONDS="${ROR_PREBUILD_POLL_INTERVAL_SECONDS:-60}" + +# Grace period after the plugin run reports success, before concluding that the run finished without +# publishing the tag we asked for. Covers the gap between the push and the tag being readable. +ROR_PREBUILD_POST_SUCCESS_GRACE_SECONDS="${ROR_PREBUILD_POST_SUCCESS_GRACE_SECONDS:-120}" + +# --- Image reference helpers ------------------------------------------------------------------- + +# Full image reference for a dev image, given a stack version and a tag. +ror_es_dev_image() { echo "${ROR_ES_DEV_IMAGE_REPO}:${1}-ror-${2}"; } +ror_kbn_dev_image() { echo "${ROR_KBN_DEV_IMAGE_REPO}:${1}-ror-${2}"; } + +# Whatever the last _probe_dev_image call failed with. Kept so the wait loop can report the real +# reason instead of a bare timeout. +ROR_PREBUILD_LAST_PROBE_ERROR="" + +# Asks the registry whether a tag exists, without pulling it. Returns: +# 0 - the tag is there +# 1 - the registry answered and the tag is not there +# 2 - the question could not be asked: rate limit, auth, network, or a repo name that does not +# resolve +# +# Telling 1 and 2 apart is the whole point. Both look like "not published yet" to a plain +# `docker manifest inspect >/dev/null 2>&1`, so a throttled or unauthorised probe silently spins for +# the entire timeout and then blames the plugin workflow for something that happened here. +_probe_dev_image() { + local IMAGE=$1 OUTPUT STATUS=0 + + OUTPUT=$(docker manifest inspect "$IMAGE" 2>&1) || STATUS=$? + ROR_PREBUILD_LAST_PROBE_ERROR="" + [ "$STATUS" -eq 0 ] && return 0 + + ROR_PREBUILD_LAST_PROBE_ERROR=$OUTPUT + + # A missing tag is worded differently depending on the registry and the CLI version; anything else + # (`toomanyrequests`, `unauthorized`, `denied`, a DNS or TLS error) means the probe itself failed. + case "$OUTPUT" in + *"manifest unknown"* | *"no such manifest"* | *"not found"* | *MANIFEST_UNKNOWN*) return 1 ;; + esac + return 2 +} + +# Checks whether an image tag exists in the remote registry without pulling it. Kept for callers that +# only want a yes/no; use _probe_dev_image when the reason matters. +docker_image_exists() { + local STATUS=0 + _probe_dev_image "$1" || STATUS=$? + [ "$STATUS" -eq 0 ] +} + +# The pre-build run each dispatch started, so the wait can follow it. Empty until dispatched, and +# empty for a wait that runs without a dispatch in the same shell. +ROR_ES_PREBUILD_RUN_ID="${ROR_ES_PREBUILD_RUN_ID:-}" +ROR_ES_PREBUILD_RUN_URL="${ROR_ES_PREBUILD_RUN_URL:-}" +ROR_KBN_PREBUILD_RUN_ID="${ROR_KBN_PREBUILD_RUN_ID:-}" +ROR_KBN_PREBUILD_RUN_URL="${ROR_KBN_PREBUILD_RUN_URL:-}" + +# Turns a space- or comma-separated version list into space-separated tokens, and rejects anything +# that is not X.Y.Z or X.Y.Z-qualifier. +normalize_elk_versions() { + if [ "$#" -lt 1 ] || [ -z "${1// /}" ]; then + echo "ERROR: no ELK versions given" >&2 + return 1 + fi + + local VERSIONS + VERSIONS=$(echo "$1" | tr ',' ' ' | tr -s '[:space:]' ' ' | sed 's/^ //; s/ $//') + + local VERSION + for VERSION in $VERSIONS; do + if ! [[ $VERSION =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9]+)?$ ]]; then + echo "ERROR: invalid ELK version '$VERSION'. Expected format: X.Y.Z" >&2 + return 2 + fi + done + + echo "$VERSIONS" +} + +# --- Dispatch ---------------------------------------------------------------------------------- +# +# Dispatching only queues the workflow and returns; use the wait helpers below to block on the +# result. +# +# It is safe to dispatch even when the image already exists. The workflow skips the build when the +# sources have not changed and only re-tags the image it already published. +# +# The target branch does not have to exist in the plugin repo โ€” the workflow falls back to `develop` +# โ€” so the current branch name can always be passed as it is. +# +# Versions may be passed as a list, so one dispatch can cover several versions. + +# An ISO8601 UTC timestamp N minutes in the past, for filtering `gh run list` output. Goes through +# epoch arithmetic because GNU (`date -d @epoch`) and BSD (`date -r epoch`) spell the same thing +# differently and this file runs on both. +_iso8601_minutes_ago() { + local EPOCH + EPOCH=$(( $(date -u +%s) - $1 * 60 )) + date -u -d "@$EPOCH" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || + date -u -r "$EPOCH" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null +} + +# Dispatching does not tell us which run it created, so the run has to be found afterwards: the +# newest one of that workflow created since we asked. Echoes " ". +# +# Best-effort โ€” a failure here only costs the fast-fail in the wait loop, so it is never fatal. The +# `--limit 20` window and the timestamp filter can both miss under heavy concurrency in the plugin +# repo; that is the same trade-off. +# Usage: _locate_prebuild_run +_locate_prebuild_run() { + local REPO=$1 WORKFLOW=$2 TOKEN=$3 SINCE=$4 ATTEMPT RUN + + # Comparing the timestamps as strings is exact: gh and `date` above both emit + # YYYY-MM-DDTHH:MM:SSZ, which sorts lexicographically. + # + # The sleep comes first on purpose. GitHub takes a moment to register a dispatched run, and the + # window is backdated, so asking immediately can return somebody else's slightly older run of the + # same workflow and latch onto it. Waiting first makes our run โ€” the newest โ€” the one `last` picks. + for ATTEMPT in 1 2 3 4 5 6; do + sleep 5 + RUN=$(GH_TOKEN="$TOKEN" gh run list -R "$REPO" --workflow "$WORKFLOW" --limit 20 \ + --json databaseId,url,createdAt 2>/dev/null | + jq -r --arg since "$SINCE" \ + '[.[] | select(.createdAt >= $since)] | sort_by(.createdAt) | last // empty + | "\(.databaseId) \(.url)"' 2>/dev/null) || RUN="" + if [ -n "$RUN" ]; then + echo "$RUN" + return 0 + fi + done + return 1 +} + +# The run's outcome as one word: `running`, `success`, another conclusion (`failure`, `cancelled`, +# `timed_out`, ...), or `unknown` when it cannot be read. Never fails, so a flaky API call degrades +# to "keep waiting" rather than aborting a healthy wait. +# Usage: _prebuild_run_state +_prebuild_run_state() { + local REPO=$1 TOKEN=$2 RUN_ID=$3 JSON STATE + + JSON=$(GH_TOKEN="$TOKEN" gh run view "$RUN_ID" -R "$REPO" --json status,conclusion 2>/dev/null) || { + echo unknown + return 0 + } + STATE=$(echo "$JSON" | jq -r 'if .status != "completed" then "running" else (.conclusion // "unknown") end' 2>/dev/null) || + STATE=unknown + echo "${STATE:-unknown}" +} + +# Set by _dispatch_prebuild_workflow so the plugin-specific wrappers below can stash the run they +# just started. Empty when the run could not be identified. +_ROR_LAST_DISPATCH_RUN_ID="" +_ROR_LAST_DISPATCH_RUN_URL="" + +# Runs `gh workflow run` for one plugin. The plugins differ only in repo, workflow, token and input +# names, so the caller supplies the `-f key=value` pairs. +# +# Usage: _dispatch_prebuild_workflow