Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 22 additions & 9 deletions scripts/test-validate-azure-durable-functions-live.sh
Original file line number Diff line number Diff line change
Expand Up @@ -61,11 +61,17 @@ case "$command" in
"functionapp keys list "*)
printf '%s\n' '{"functionKeys":{"default":"test-function-key-never-log"},"masterKey":"test-master-key-never-log"}'
;;
"functionapp restart "*)
touch "$VYRAL_TEST_STATE/restart-issued"
"account show "*)
printf 'test-subscription\n'
;;
"rest --method post --url "*"/syncfunctiontriggers?api-version=2016-08-01 "*)
if [[ "${VYRAL_TEST_SYNC_FAIL:-false}" == true ]]; then
exit 1
fi
touch "$VYRAL_TEST_STATE/trigger-sync-issued"
;;
"cosmosdb sql container show "*)
if [[ "${VYRAL_TEST_FUNCTION_COUNT:-7}" != 7 && "${VYRAL_TEST_RECOVER_AFTER_RESTART:-false}" != true ]]; then
if [[ "${VYRAL_TEST_FUNCTION_COUNT:-7}" != 7 && "${VYRAL_TEST_RECOVER_AFTER_SYNC:-false}" != true ]]; then
exit 1
fi
;;
Expand Down Expand Up @@ -114,7 +120,7 @@ done
case "$url" in
*/admin/functions | */admin/functions/)
function_count="${VYRAL_TEST_FUNCTION_COUNT:-7}"
if [[ "${VYRAL_TEST_RECOVER_AFTER_RESTART:-false}" == true && -e "$VYRAL_TEST_STATE/restart-issued" ]]; then
if [[ "${VYRAL_TEST_RECOVER_AFTER_SYNC:-false}" == true && -e "$VYRAL_TEST_STATE/trigger-sync-issued" ]]; then
function_count=7
fi
case "$function_count" in
Expand Down Expand Up @@ -182,8 +188,9 @@ run_case() {
local expected_status="$3"
local expected_result="$4"
local expected_stage="$5"
local expected_restart="$6"
local recover_after_restart="${7:-false}"
local expected_sync="$6"
local recover_after_sync="${7:-false}"
local sync_fail="${8:-false}"
local state="$work/state-$name"
local receipt="$work/receipts/$name.json"
local output="$work/$name.log"
Expand All @@ -194,7 +201,8 @@ run_case() {
TMPDIR="$work/temp" \
VYRAL_TEST_STATE="$state" \
VYRAL_TEST_FUNCTION_COUNT="$function_count" \
VYRAL_TEST_RECOVER_AFTER_RESTART="$recover_after_restart" \
VYRAL_TEST_RECOVER_AFTER_SYNC="$recover_after_sync" \
VYRAL_TEST_SYNC_FAIL="$sync_fail" \
VYRAL_AZURE_LIVE_RESOURCE_GROUP=test-disposable \
VYRAL_AZURE_LIVE_COSMOS_ACCOUNT=test-cosmos \
VYRAL_AZURE_COSMOS_CONNECTION_STRING='AccountEndpoint=https://fixture.invalid;AccountKey=fixture;' \
Expand All @@ -209,7 +217,10 @@ run_case() {
[[ "$(jq -r '.cleanup.result' "$receipt")" == passed ]]
[[ "$(jq -r '.cleanup.functionApp' "$receipt")" == deleted ]]
[[ "$(jq -r '.cleanup.storageAccount' "$receipt")" == deleted ]]
[[ "$(jq -r '.recovery.partialInventoryRestartAttempted' "$receipt")" == "$expected_restart" ]]
[[ "$(jq -r '.recovery.partialInventoryTriggerSyncAttempted' "$receipt")" == "$expected_sync" ]]
[[ "$(jq -r '.recovery.partialInventoryRestartAttempted' "$receipt")" == false ]]
[[ "$(jq -r '.diagnostics.deploymentAttempts' "$receipt")" == 1 ]]
[[ "$(jq -r '.diagnostics.packagedFunctionNames | length' "$receipt")" == 7 ]]
[[ "$(stat -c '%a' "$receipt")" == 600 ]]
! grep -Fq 'test-function-key-never-log' "$output"
! grep -Fq 'test-master-key-never-log' "$output"
Expand All @@ -223,14 +234,16 @@ run_case() {
[[ "$(jq -r '.failure.stage' "$receipt")" == "$expected_stage" ]]
[[ "$(jq -r '.failure.discoveredFunctionCount' "$receipt")" == "$function_count" ]]
[[ "$(jq -r '.failure.expectedFunctionCount' "$receipt")" == 7 ]]
[[ "$(jq -r '.diagnostics.runtimeFunctionNames | length' "$receipt")" == "$function_count" ]]
[[ "$(jq -r '.cleanup.statusContainer' "$receipt")" == not-created ]]
fi
}

run_case success 7 0 passed complete false
run_case discovery-failure 0 1 failed function-discovery false
run_case incomplete-discovery 1 1 failed function-discovery true
run_case restart-recovery 1 0 passed complete true true
run_case trigger-sync-recovery 1 0 passed complete true true
run_case trigger-sync-failure 1 1 failed function-discovery-trigger-sync true false true

if find "$work/temp" -mindepth 1 -print -quit | grep -q .; then
echo 'Azure live script left temporary residue.' >&2
Expand Down
48 changes: 36 additions & 12 deletions scripts/validate-azure-durable-functions-live.sh
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,11 @@ FUNCTION_KEY_AVAILABLE=false
ENDPOINT_READY=false
DISCOVERED_FUNCTION_COUNT=0
READINESS_HTTP_CODE="not-attempted"
# Preserve the legacy receipt field so evidence from the earlier restart recovery remains comparable.
HOST_RESTART_ATTEMPTED=false
TRIGGER_SYNC_ATTEMPTED=false
DISCOVERED_FUNCTION_NAMES_JSON='[]'
DEPLOYMENT_ATTEMPTS=0
FAILURE_STAGE="publish"

cleanup() {
Expand Down Expand Up @@ -125,8 +129,12 @@ cleanup() {
--argjson function_key_available "$FUNCTION_KEY_AVAILABLE" \
--argjson endpoint_ready "$ENDPOINT_READY" \
--argjson host_restart_attempted "$HOST_RESTART_ATTEMPTED" \
--argjson trigger_sync_attempted "$TRIGGER_SYNC_ATTEMPTED" \
--argjson discovered_function_count "$DISCOVERED_FUNCTION_COUNT" \
--argjson discovered_function_names "$DISCOVERED_FUNCTION_NAMES_JSON" \
--argjson expected_function_count "$EXPECTED_FUNCTION_COUNT" \
--argjson expected_function_names "$EXPECTED_FUNCTION_NAMES_JSON" \
--argjson deployment_attempts "$DEPLOYMENT_ATTEMPTS" \
--argjson cleanup_passed "$([[ "$cleanup_failed" == false ]] && echo true || echo false)" \
'{
schemaVersion: 1,
Expand All @@ -150,8 +158,14 @@ cleanup() {
liveAssertions: $assertions
},
recovery: {
partialInventoryTriggerSyncAttempted: $trigger_sync_attempted,
partialInventoryRestartAttempted: $host_restart_attempted
},
diagnostics: {
deploymentAttempts: $deployment_attempts,
packagedFunctionNames: $expected_function_names,
runtimeFunctionNames: $discovered_function_names
},
failure: (
if $result == "passed" then null
else {
Expand Down Expand Up @@ -229,10 +243,9 @@ az functionapp config appsettings set --resource-group "$VYRAL_AZURE_LIVE_RESOUR
# ready, so retry the operation instead of paying a fixed delay on every successful run.
FAILURE_STAGE="deployment"
deployed=false
deployment_attempts=0
deployment_diagnostic="$WORK_ROOT/deployment-attempt.log"
for _ in $(seq 1 6); do
deployment_attempts=$((deployment_attempts + 1))
DEPLOYMENT_ATTEMPTS=$((DEPLOYMENT_ATTEMPTS + 1))
if az functionapp deployment source config-zip --resource-group "$VYRAL_AZURE_LIVE_RESOURCE_GROUP" \
--name "$FUNCTION" --src "$WORK_ROOT/app.zip" --timeout 600 --only-show-errors --output none \
2>"$deployment_diagnostic"; then
Expand All @@ -242,11 +255,11 @@ for _ in $(seq 1 6); do
sleep 20
done
if [[ "$deployed" != true ]]; then
echo "azure-durable-functions-live-deployment=failed attempts:${deployment_attempts}" >&2
echo "azure-durable-functions-live-deployment=failed attempts:${DEPLOYMENT_ATTEMPTS}" >&2
false
fi
DEPLOYMENT_PASSED=true
echo "azure-durable-functions-live-deployment=passed attempts:${deployment_attempts}"
echo "azure-durable-functions-live-deployment=passed attempts:${DEPLOYMENT_ATTEMPTS}"

FAILURE_STAGE="function-key"
function_key=""
Expand Down Expand Up @@ -289,6 +302,12 @@ for _ in $(seq 1 48); do
partial_inventory_observations=$((partial_inventory_observations + 1))
fi
fi
DISCOVERED_FUNCTION_NAMES_JSON="$(jq -cer '
if type == "array" and all(.[]; (.name? | type) == "string")
then [.[].name] | sort
else []
end
' <<<"$admin_payload" 2>/dev/null || echo '[]')"
inventory_matches="$(jq -r --argjson expected "$EXPECTED_FUNCTION_NAMES_JSON" '
if type == "array" and all(.[]; (.name? | type) == "string")
then ([.[].name] | sort) == ($expected | sort)
Expand All @@ -300,17 +319,22 @@ for _ in $(seq 1 48); do
FUNCTIONS_DISCOVERED=true
break
fi
if [[ "$HOST_RESTART_ATTEMPTED" == false && "$partial_inventory_observations" -ge 6 ]]; then
FAILURE_STAGE="function-discovery-restart"
HOST_RESTART_ATTEMPTED=true
if ! az functionapp restart --resource-group "$VYRAL_AZURE_LIVE_RESOURCE_GROUP" \
--name "$FUNCTION" --only-show-errors --output none \
2>"$WORK_ROOT/function-restart.log"; then
echo 'azure-durable-functions-live-host-restart=failed' >&2
if [[ "$TRIGGER_SYNC_ATTEMPTED" == false && "$partial_inventory_observations" -ge 6 ]]; then
FAILURE_STAGE="function-discovery-trigger-sync"
TRIGGER_SYNC_ATTEMPTED=true
subscription_id="$(az account show --query id --output tsv --only-show-errors)"
[[ -n "$subscription_id" ]]
if ! az rest --method post \
--url "https://management.azure.com/subscriptions/${subscription_id}/resourceGroups/${VYRAL_AZURE_LIVE_RESOURCE_GROUP}/providers/Microsoft.Web/sites/${FUNCTION}/syncfunctiontriggers?api-version=2016-08-01" \
--only-show-errors --output none 2>"$WORK_ROOT/function-trigger-sync.log"; then
unset subscription_id
echo 'azure-durable-functions-live-trigger-sync=failed' >&2
false
fi
echo 'azure-durable-functions-live-host-restart=issued'
unset subscription_id
echo 'azure-durable-functions-live-trigger-sync=issued'
DISCOVERED_FUNCTION_COUNT=0
DISCOVERED_FUNCTION_NAMES_JSON='[]'
partial_inventory_observations=0
FAILURE_STAGE="function-discovery"
sleep 10
Expand Down
Loading