From e2cfe3ac9af46000df52440c97d8c5150ac9bf52 Mon Sep 17 00:00:00 2001 From: jeremydixon22 Date: Tue, 11 Aug 2026 11:18:46 -0400 Subject: [PATCH] Use explicit Azure trigger sync for partial inventories --- ...t-validate-azure-durable-functions-live.sh | 31 ++++++++---- .../validate-azure-durable-functions-live.sh | 48 ++++++++++++++----- 2 files changed, 58 insertions(+), 21 deletions(-) diff --git a/scripts/test-validate-azure-durable-functions-live.sh b/scripts/test-validate-azure-durable-functions-live.sh index c851da2..0309241 100755 --- a/scripts/test-validate-azure-durable-functions-live.sh +++ b/scripts/test-validate-azure-durable-functions-live.sh @@ -61,11 +61,17 @@ case "$command" in "functionapp keys list "*) printf '%s\n' '{"functionKeys":{"default":"test-function-key-never-log"},"masterKey":"test-master-key-never-log"}' ;; - "functionapp restart "*) - touch "$VYRAL_TEST_STATE/restart-issued" + "account show "*) + printf 'test-subscription\n' + ;; + "rest --method post --url "*"/syncfunctiontriggers?api-version=2016-08-01 "*) + if [[ "${VYRAL_TEST_SYNC_FAIL:-false}" == true ]]; then + exit 1 + fi + touch "$VYRAL_TEST_STATE/trigger-sync-issued" ;; "cosmosdb sql container show "*) - if [[ "${VYRAL_TEST_FUNCTION_COUNT:-7}" != 7 && "${VYRAL_TEST_RECOVER_AFTER_RESTART:-false}" != true ]]; then + if [[ "${VYRAL_TEST_FUNCTION_COUNT:-7}" != 7 && "${VYRAL_TEST_RECOVER_AFTER_SYNC:-false}" != true ]]; then exit 1 fi ;; @@ -114,7 +120,7 @@ done case "$url" in */admin/functions | */admin/functions/) function_count="${VYRAL_TEST_FUNCTION_COUNT:-7}" - if [[ "${VYRAL_TEST_RECOVER_AFTER_RESTART:-false}" == true && -e "$VYRAL_TEST_STATE/restart-issued" ]]; then + if [[ "${VYRAL_TEST_RECOVER_AFTER_SYNC:-false}" == true && -e "$VYRAL_TEST_STATE/trigger-sync-issued" ]]; then function_count=7 fi case "$function_count" in @@ -182,8 +188,9 @@ run_case() { local expected_status="$3" local expected_result="$4" local expected_stage="$5" - local expected_restart="$6" - local recover_after_restart="${7:-false}" + local expected_sync="$6" + local recover_after_sync="${7:-false}" + local sync_fail="${8:-false}" local state="$work/state-$name" local receipt="$work/receipts/$name.json" local output="$work/$name.log" @@ -194,7 +201,8 @@ run_case() { TMPDIR="$work/temp" \ VYRAL_TEST_STATE="$state" \ VYRAL_TEST_FUNCTION_COUNT="$function_count" \ - VYRAL_TEST_RECOVER_AFTER_RESTART="$recover_after_restart" \ + VYRAL_TEST_RECOVER_AFTER_SYNC="$recover_after_sync" \ + VYRAL_TEST_SYNC_FAIL="$sync_fail" \ VYRAL_AZURE_LIVE_RESOURCE_GROUP=test-disposable \ VYRAL_AZURE_LIVE_COSMOS_ACCOUNT=test-cosmos \ VYRAL_AZURE_COSMOS_CONNECTION_STRING='AccountEndpoint=https://fixture.invalid;AccountKey=fixture;' \ @@ -209,7 +217,10 @@ run_case() { [[ "$(jq -r '.cleanup.result' "$receipt")" == passed ]] [[ "$(jq -r '.cleanup.functionApp' "$receipt")" == deleted ]] [[ "$(jq -r '.cleanup.storageAccount' "$receipt")" == deleted ]] - [[ "$(jq -r '.recovery.partialInventoryRestartAttempted' "$receipt")" == "$expected_restart" ]] + [[ "$(jq -r '.recovery.partialInventoryTriggerSyncAttempted' "$receipt")" == "$expected_sync" ]] + [[ "$(jq -r '.recovery.partialInventoryRestartAttempted' "$receipt")" == false ]] + [[ "$(jq -r '.diagnostics.deploymentAttempts' "$receipt")" == 1 ]] + [[ "$(jq -r '.diagnostics.packagedFunctionNames | length' "$receipt")" == 7 ]] [[ "$(stat -c '%a' "$receipt")" == 600 ]] ! grep -Fq 'test-function-key-never-log' "$output" ! grep -Fq 'test-master-key-never-log' "$output" @@ -223,6 +234,7 @@ run_case() { [[ "$(jq -r '.failure.stage' "$receipt")" == "$expected_stage" ]] [[ "$(jq -r '.failure.discoveredFunctionCount' "$receipt")" == "$function_count" ]] [[ "$(jq -r '.failure.expectedFunctionCount' "$receipt")" == 7 ]] + [[ "$(jq -r '.diagnostics.runtimeFunctionNames | length' "$receipt")" == "$function_count" ]] [[ "$(jq -r '.cleanup.statusContainer' "$receipt")" == not-created ]] fi } @@ -230,7 +242,8 @@ run_case() { run_case success 7 0 passed complete false run_case discovery-failure 0 1 failed function-discovery false run_case incomplete-discovery 1 1 failed function-discovery true -run_case restart-recovery 1 0 passed complete true true +run_case trigger-sync-recovery 1 0 passed complete true true +run_case trigger-sync-failure 1 1 failed function-discovery-trigger-sync true false true if find "$work/temp" -mindepth 1 -print -quit | grep -q .; then echo 'Azure live script left temporary residue.' >&2 diff --git a/scripts/validate-azure-durable-functions-live.sh b/scripts/validate-azure-durable-functions-live.sh index f5bc35e..849d430 100755 --- a/scripts/validate-azure-durable-functions-live.sh +++ b/scripts/validate-azure-durable-functions-live.sh @@ -51,7 +51,11 @@ FUNCTION_KEY_AVAILABLE=false ENDPOINT_READY=false DISCOVERED_FUNCTION_COUNT=0 READINESS_HTTP_CODE="not-attempted" +# Preserve the legacy receipt field so evidence from the earlier restart recovery remains comparable. HOST_RESTART_ATTEMPTED=false +TRIGGER_SYNC_ATTEMPTED=false +DISCOVERED_FUNCTION_NAMES_JSON='[]' +DEPLOYMENT_ATTEMPTS=0 FAILURE_STAGE="publish" cleanup() { @@ -125,8 +129,12 @@ cleanup() { --argjson function_key_available "$FUNCTION_KEY_AVAILABLE" \ --argjson endpoint_ready "$ENDPOINT_READY" \ --argjson host_restart_attempted "$HOST_RESTART_ATTEMPTED" \ + --argjson trigger_sync_attempted "$TRIGGER_SYNC_ATTEMPTED" \ --argjson discovered_function_count "$DISCOVERED_FUNCTION_COUNT" \ + --argjson discovered_function_names "$DISCOVERED_FUNCTION_NAMES_JSON" \ --argjson expected_function_count "$EXPECTED_FUNCTION_COUNT" \ + --argjson expected_function_names "$EXPECTED_FUNCTION_NAMES_JSON" \ + --argjson deployment_attempts "$DEPLOYMENT_ATTEMPTS" \ --argjson cleanup_passed "$([[ "$cleanup_failed" == false ]] && echo true || echo false)" \ '{ schemaVersion: 1, @@ -150,8 +158,14 @@ cleanup() { liveAssertions: $assertions }, recovery: { + partialInventoryTriggerSyncAttempted: $trigger_sync_attempted, partialInventoryRestartAttempted: $host_restart_attempted }, + diagnostics: { + deploymentAttempts: $deployment_attempts, + packagedFunctionNames: $expected_function_names, + runtimeFunctionNames: $discovered_function_names + }, failure: ( if $result == "passed" then null else { @@ -229,10 +243,9 @@ az functionapp config appsettings set --resource-group "$VYRAL_AZURE_LIVE_RESOUR # ready, so retry the operation instead of paying a fixed delay on every successful run. FAILURE_STAGE="deployment" deployed=false -deployment_attempts=0 deployment_diagnostic="$WORK_ROOT/deployment-attempt.log" for _ in $(seq 1 6); do - deployment_attempts=$((deployment_attempts + 1)) + DEPLOYMENT_ATTEMPTS=$((DEPLOYMENT_ATTEMPTS + 1)) if az functionapp deployment source config-zip --resource-group "$VYRAL_AZURE_LIVE_RESOURCE_GROUP" \ --name "$FUNCTION" --src "$WORK_ROOT/app.zip" --timeout 600 --only-show-errors --output none \ 2>"$deployment_diagnostic"; then @@ -242,11 +255,11 @@ for _ in $(seq 1 6); do sleep 20 done if [[ "$deployed" != true ]]; then - echo "azure-durable-functions-live-deployment=failed attempts:${deployment_attempts}" >&2 + echo "azure-durable-functions-live-deployment=failed attempts:${DEPLOYMENT_ATTEMPTS}" >&2 false fi DEPLOYMENT_PASSED=true -echo "azure-durable-functions-live-deployment=passed attempts:${deployment_attempts}" +echo "azure-durable-functions-live-deployment=passed attempts:${DEPLOYMENT_ATTEMPTS}" FAILURE_STAGE="function-key" function_key="" @@ -289,6 +302,12 @@ for _ in $(seq 1 48); do partial_inventory_observations=$((partial_inventory_observations + 1)) fi fi + DISCOVERED_FUNCTION_NAMES_JSON="$(jq -cer ' + if type == "array" and all(.[]; (.name? | type) == "string") + then [.[].name] | sort + else [] + end + ' <<<"$admin_payload" 2>/dev/null || echo '[]')" inventory_matches="$(jq -r --argjson expected "$EXPECTED_FUNCTION_NAMES_JSON" ' if type == "array" and all(.[]; (.name? | type) == "string") then ([.[].name] | sort) == ($expected | sort) @@ -300,17 +319,22 @@ for _ in $(seq 1 48); do FUNCTIONS_DISCOVERED=true break fi - if [[ "$HOST_RESTART_ATTEMPTED" == false && "$partial_inventory_observations" -ge 6 ]]; then - FAILURE_STAGE="function-discovery-restart" - HOST_RESTART_ATTEMPTED=true - if ! az functionapp restart --resource-group "$VYRAL_AZURE_LIVE_RESOURCE_GROUP" \ - --name "$FUNCTION" --only-show-errors --output none \ - 2>"$WORK_ROOT/function-restart.log"; then - echo 'azure-durable-functions-live-host-restart=failed' >&2 + if [[ "$TRIGGER_SYNC_ATTEMPTED" == false && "$partial_inventory_observations" -ge 6 ]]; then + FAILURE_STAGE="function-discovery-trigger-sync" + TRIGGER_SYNC_ATTEMPTED=true + subscription_id="$(az account show --query id --output tsv --only-show-errors)" + [[ -n "$subscription_id" ]] + if ! az rest --method post \ + --url "https://management.azure.com/subscriptions/${subscription_id}/resourceGroups/${VYRAL_AZURE_LIVE_RESOURCE_GROUP}/providers/Microsoft.Web/sites/${FUNCTION}/syncfunctiontriggers?api-version=2016-08-01" \ + --only-show-errors --output none 2>"$WORK_ROOT/function-trigger-sync.log"; then + unset subscription_id + echo 'azure-durable-functions-live-trigger-sync=failed' >&2 false fi - echo 'azure-durable-functions-live-host-restart=issued' + unset subscription_id + echo 'azure-durable-functions-live-trigger-sync=issued' DISCOVERED_FUNCTION_COUNT=0 + DISCOVERED_FUNCTION_NAMES_JSON='[]' partial_inventory_observations=0 FAILURE_STAGE="function-discovery" sleep 10