Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions PUBLIC-EXPORT-MANIFEST.json
Original file line number Diff line number Diff line change
Expand Up @@ -1669,7 +1669,7 @@
{
"mode": "755",
"path": "scripts/test-validate-azure-durable-functions-live.sh",
"sha256": "2519a905220d15b173f7298cfbc44c3ad3370c61b8d21b2a28611617a6ba83a8"
"sha256": "18cf5d42c9890426f3a8fc6fb40367d2464e833778096ea1a59ede6c74f94059"
},
{
"mode": "755",
Expand Down Expand Up @@ -1714,7 +1714,7 @@
{
"mode": "755",
"path": "scripts/validate-azure-durable-functions-live.sh",
"sha256": "50b7860448693186b0733bb0ab9ea9badd34f1dd7f94cb0f7ceecd7853424a14"
"sha256": "39ac593760fb253dd3e42ba9648a4d09d219ee4a2501270ff3edd17dac17d80f"
},
{
"mode": "755",
Expand Down Expand Up @@ -4274,5 +4274,5 @@
],
"schemaVersion": 1,
"sourceDirty": false,
"treeSha256": "9d58c326e9fa62cee7633fdd2d6c0e61e647a8fcc953f0a5c615d01945c31126"
"treeSha256": "1533913decd63d279f8aeb085ef2a81db539f4e48ab4e7959179c0f6646dd45f"
}
22 changes: 17 additions & 5 deletions scripts/test-validate-azure-durable-functions-live.sh
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,11 @@ case "$command" in
"functionapp keys list "*)
printf '%s\n' '{"functionKeys":{"default":"test-function-key-never-log"},"masterKey":"test-master-key-never-log"}'
;;
"functionapp restart "*)
touch "$VYRAL_TEST_STATE/restart-issued"
;;
"cosmosdb sql container show "*)
if [[ "${VYRAL_TEST_FUNCTION_COUNT:-7}" != 7 ]]; then
if [[ "${VYRAL_TEST_FUNCTION_COUNT:-7}" != 7 && "${VYRAL_TEST_RECOVER_AFTER_RESTART:-false}" != true ]]; then
exit 1
fi
;;
Expand Down Expand Up @@ -110,7 +113,11 @@ done

case "$url" in
*/admin/functions | */admin/functions/)
case "${VYRAL_TEST_FUNCTION_COUNT:-7}" in
function_count="${VYRAL_TEST_FUNCTION_COUNT:-7}"
if [[ "${VYRAL_TEST_RECOVER_AFTER_RESTART:-false}" == true && -e "$VYRAL_TEST_STATE/restart-issued" ]]; then
function_count=7
fi
case "$function_count" in
7)
printf '%s\n' '[{"name":"VyralAzureDurableSmokeCancel"},{"name":"VyralAzureDurableSmokeGet"},{"name":"VyralAzureDurableSmokeRaiseEvent"},{"name":"VyralAzureDurableSmokeStart"},{"name":"VyralExecutionRuntimeOrchestrator"},{"name":"VyralExecutionRuntimeStart"},{"name":"VyralExecutionRuntimeStep"}]'
;;
Expand Down Expand Up @@ -175,6 +182,8 @@ run_case() {
local expected_status="$3"
local expected_result="$4"
local expected_stage="$5"
local expected_restart="$6"
local recover_after_restart="${7:-false}"
local state="$work/state-$name"
local receipt="$work/receipts/$name.json"
local output="$work/$name.log"
Expand All @@ -185,6 +194,7 @@ run_case() {
TMPDIR="$work/temp" \
VYRAL_TEST_STATE="$state" \
VYRAL_TEST_FUNCTION_COUNT="$function_count" \
VYRAL_TEST_RECOVER_AFTER_RESTART="$recover_after_restart" \
VYRAL_AZURE_LIVE_RESOURCE_GROUP=test-disposable \
VYRAL_AZURE_LIVE_COSMOS_ACCOUNT=test-cosmos \
VYRAL_AZURE_COSMOS_CONNECTION_STRING='AccountEndpoint=https://fixture.invalid;AccountKey=fixture;' \
Expand All @@ -199,6 +209,7 @@ run_case() {
[[ "$(jq -r '.cleanup.result' "$receipt")" == passed ]]
[[ "$(jq -r '.cleanup.functionApp' "$receipt")" == deleted ]]
[[ "$(jq -r '.cleanup.storageAccount' "$receipt")" == deleted ]]
[[ "$(jq -r '.recovery.partialInventoryRestartAttempted' "$receipt")" == "$expected_restart" ]]
[[ "$(stat -c '%a' "$receipt")" == 600 ]]
! grep -Fq 'test-function-key-never-log' "$output"
! grep -Fq 'test-master-key-never-log' "$output"
Expand All @@ -216,9 +227,10 @@ run_case() {
fi
}

run_case success 7 0 passed complete
run_case discovery-failure 0 1 failed function-discovery
run_case incomplete-discovery 1 1 failed function-discovery
run_case success 7 0 passed complete false
run_case discovery-failure 0 1 failed function-discovery false
run_case incomplete-discovery 1 1 failed function-discovery true
run_case restart-recovery 1 0 passed complete true true

if find "$work/temp" -mindepth 1 -print -quit | grep -q .; then
echo 'Azure live script left temporary residue.' >&2
Expand Down
28 changes: 26 additions & 2 deletions scripts/validate-azure-durable-functions-live.sh
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ FUNCTION_KEY_AVAILABLE=false
ENDPOINT_READY=false
DISCOVERED_FUNCTION_COUNT=0
READINESS_HTTP_CODE="not-attempted"
HOST_RESTART_ATTEMPTED=false
FAILURE_STAGE="publish"

cleanup() {
Expand Down Expand Up @@ -123,6 +124,7 @@ cleanup() {
--argjson functions_discovered "$FUNCTIONS_DISCOVERED" \
--argjson function_key_available "$FUNCTION_KEY_AVAILABLE" \
--argjson endpoint_ready "$ENDPOINT_READY" \
--argjson host_restart_attempted "$HOST_RESTART_ATTEMPTED" \
--argjson discovered_function_count "$DISCOVERED_FUNCTION_COUNT" \
--argjson expected_function_count "$EXPECTED_FUNCTION_COUNT" \
--argjson cleanup_passed "$([[ "$cleanup_failed" == false ]] && echo true || echo false)" \
Expand All @@ -147,6 +149,9 @@ cleanup() {
durableTimer: $timer,
liveAssertions: $assertions
},
recovery: {
partialInventoryRestartAttempted: $host_restart_attempted
},
failure: (
if $result == "passed" then null
else {
Expand Down Expand Up @@ -272,13 +277,17 @@ base_url="https://${FUNCTION}.azurewebsites.net"
# snapshot while a Flex host is still indexing. Query the authenticated runtime inventory instead;
# it both wakes the deployed host and proves the exact functions that host can execute.
FAILURE_STAGE="function-discovery"
for _ in $(seq 1 72); do
partial_inventory_observations=0
for _ in $(seq 1 48); do
admin_payload="$(curl --config "$ADMIN_CURL_CONFIG" -sS --fail \
"$base_url/admin/functions/" 2>/dev/null || true)"
--max-time 10 "$base_url/admin/functions/" 2>/dev/null || true)"
candidate_count="$(jq -r 'if type == "array" then length else 0 end' \
<<<"$admin_payload" 2>/dev/null || echo 0)"
if [[ "$candidate_count" =~ ^[0-9]+$ ]]; then
DISCOVERED_FUNCTION_COUNT="$candidate_count"
if (( candidate_count > 0 && candidate_count < EXPECTED_FUNCTION_COUNT )); then
partial_inventory_observations=$((partial_inventory_observations + 1))
fi
fi
inventory_matches="$(jq -r --argjson expected "$EXPECTED_FUNCTION_NAMES_JSON" '
if type == "array" and all(.[]; (.name? | type) == "string")
Expand All @@ -291,6 +300,21 @@ for _ in $(seq 1 72); do
FUNCTIONS_DISCOVERED=true
break
fi
if [[ "$HOST_RESTART_ATTEMPTED" == false && "$partial_inventory_observations" -ge 6 ]]; then
FAILURE_STAGE="function-discovery-restart"
HOST_RESTART_ATTEMPTED=true
if ! az functionapp restart --resource-group "$VYRAL_AZURE_LIVE_RESOURCE_GROUP" \
--name "$FUNCTION" --only-show-errors --output none \
2>"$WORK_ROOT/function-restart.log"; then
echo 'azure-durable-functions-live-host-restart=failed' >&2
false
fi
echo 'azure-durable-functions-live-host-restart=issued'
DISCOVERED_FUNCTION_COUNT=0
partial_inventory_observations=0
FAILURE_STAGE="function-discovery"
sleep 10
fi
sleep 5
done
echo "azure-durable-functions-live-discovery=count:${DISCOVERED_FUNCTION_COUNT}"
Expand Down
Loading