Skip to content

RDKEMW-21581: Network connection recovery script should handle IPv4-only/IPv6-only networks #570

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tukken-comcast wants to merge 5 commits into
base: develop
Choose a base branch
from
Open

RDKEMW-21581: Network connection recovery script should handle IPv4-only/IPv6-only networks #570

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5620603
DELIA-70624: Updated Script to reset counters only if gateway is enab…
Jun 29, 2026
1691ade
DELIA-70624: networkConnectionRecovery: presence-aware packet-loss re…
tukken-comcast Jul 3, 2026
6a1174b
DELIA-70624: networkConnectionRecovery: add log() helper and per-call…
tukken-comcast Jul 3, 2026
037420b
DELIA-70624: networkConnectionRecovery: log route/loss snapshot on st…
tukken-comcast Jul 3, 2026
99cfab1
DELIA-70624: remove debug logging and make script POSIX/busybox-sh safe
tukken-comcast Jul 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
197 changes: 123 additions & 74 deletions lib/rdk/networkConnectionRecovery.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,16 @@
. $RDK_PATH/utils.sh

if [ -f /lib/rdk/t2Shared_api.sh ]; then
source /lib/rdk/t2Shared_api.sh
. /lib/rdk/t2Shared_api.sh
fi

logsFile=$LOG_PATH/ConnectionStats.txt
dnsFile="/etc/resolv.dnsmasq"
wifiStateFile="/tmp/wifi-on"
packetsLostipv4=0
packetsLostipv6=0
ipv4PacketLoss=0
ipv6PacketLoss=0
ipv4GwPresent=0
ipv6GwPresent=0
lossThreshold=10
lnfSSIDConnected=0
lnfPskSSID=A16746DF2466410CA2ED9FB2E32FE7D9
Expand All @@ -45,6 +47,7 @@ pingInterval=0.2 #Interval between pings
wifiResetWaitTime=180
currentTime=0
tmpFile="/tmp/.Connection.txt"
prevStateFile="/tmp/.ncr_laststate"
wifiDriverErrors=0

##RFC parameters that can be customized
Expand All @@ -59,6 +62,12 @@ WifiReassociateTolerance=100
dnsFailures=0
maxdnsFailures=3

# Append a timestamped line to $logsFile.
log()
{
echo "$(/bin/timestamp) $*" >> "$logsFile"
}

StoreTotmpFile()
{
[ -f "$tmpFile" ] && rm "$tmpFile"
Expand Down Expand Up @@ -124,8 +133,8 @@ checkWifiConnected()
{
[ ! -f "$wifiStateFile" ] && return 0
strBuffer=$(wpa_cli status 2> /dev/null)
[[ ! "$strBuffer" =~ "wpa_state=COMPLETED" ]] && return 0
[[ "$strBuffer" =~ "$lnfPskSSID" ]] || [[ "$strBuffer" =~ "$lnfEnterpriseSSID" ]] && lnfSSIDConnected=1 && return 0
case "$strBuffer" in *"wpa_state=COMPLETED"*) ;; *) return 0 ;; esac
case "$strBuffer" in *"$lnfPskSSID"*|*"$lnfEnterpriseSSID"*) lnfSSIDConnected=1 ; return 0 ;; esac
return 1
}

Expand All @@ -138,25 +147,25 @@ checkEthernetConnected()
ret=$?
if [ $ret -eq 0 ] ; then
if [ "$lnfSSIDConnected" = "1" ]; then
echo "$(/bin/timestamp) TELEMETRY_WIFI_CONNECTED_LNF" >> "$logsFile"
log "TELEMETRY_WIFI_CONNECTED_LNF"
#Reset count when lnf ssid is connected
count=0
t2CountNotify "SYST_INFO_WIFIConn"
else
#Skip printing wifi not connected log for the first time
[ $count -gt 0 ] && echo "$(/bin/timestamp) TELEMETRY_WIFI_NOT_CONNECTED" >> "$logsFile"
[ $count -gt 0 ] && log "TELEMETRY_WIFI_NOT_CONNECTED"
count=$((count + 1))
fi
return 0
else
echo "$(/bin/timestamp) TELEMETRY_WIFI_CONNECTED" >> "$logsFile"
log "TELEMETRY_WIFI_CONNECTED"
#Reset count when connectivity is good
count=0
t2CountNotify "SYST_INFO_WIFIConn"
return 0
fi
else
echo "$(/bin/timestamp) TELEMETRY_ETHERNET_CONNECTED" >> "$logsFile"
log "TELEMETRY_ETHERNET_CONNECTED"
#Reset count when connectivity is good
count=0
t2CountNotify "SYST_INFO_ETHConn"
Expand All @@ -179,7 +188,7 @@ printWifiDetails()

wifiReassociate()
{
echo "$(/bin/timestamp) Packet Loss WiFi Reassociating" >> "$logsFile"
log "Packet Loss WiFi Reassociating"
t2CountNotify "WIFIV_ERR_reassoc"
wpa_cli reassociate
#set IsWifiReassociated to 1 after wifi reassociation
Expand All @@ -190,17 +199,18 @@ checkWifiDrvErrors()
{
dir=$(find /sys/kernel/debug/ieee80211 -type d -maxdepth 1 | sed '1d')
if [ -z "$dir" ] ; then
echo "$(/bin/timestamp) phy directory not in /sys/kernel/debug/ieee80211" >> "$logsFile"
log "phy directory not in /sys/kernel/debug/ieee80211"
elif [ ! -f "$dir"/ath10k/fw_stats ]; then
echo "$(/bin/timestamp) fw_stats file not in /sys/kernel/debug/ieee80211/$dir/ath10k/" >> "$logsFile"
log "fw_stats file not in $dir/ath10k/"
else
cat "$dir"/ath10k/fw_stats > /dev/null 2>&1
if [[ $? -ne 0 ]]; then
echo "$(/bin/timestamp) Cant open file /sys/kernel/debug/ieee80211/$dir/ath10k/ status=$?" >> "$logsFile"
catStatus=$?
if [ "$catStatus" -ne 0 ]; then
log "Can't open file $dir/ath10k/fw_stats status=$catStatus"
else
#Reset tmp variables to 0 when there is no wifi driver issue
FirstWifiDriverIssueTime=0
["$IsWifiReassociated" -eq 0 ] && IsWifiReset=0 #$IsWifiReassociated=1 indicates wifi reassociation done already and still packetloss happens hence don't make IsWifiReset=0
[ "$IsWifiReassociated" -eq 0 ] && IsWifiReset=0 #$IsWifiReassociated=1 indicates wifi reassociation done already and still packetloss happens hence don't make IsWifiReset=0
return 0
fi
fi
Expand All @@ -212,17 +222,21 @@ checkWifiDrvErrors()
checkPacketLoss()
{
version=$1
packetLoss=""
currentTime=$(($(date +%s)))

if [ -f "/tmp/checkpacketloss" ] ; then
if [ "$version" = "V4" ] ; then
gwIp=$(cat /tmp/checkpacketloss)
pingCmd="ping"
else
gwIp=""
fi
else
if [ "$version" = "V4" ] ; then
gwIp=$(/sbin/ip -4 route | awk '/default/ { print $3 }' | head -n1 | awk '{print $1;}')
pingCmd="ping"
gwIp_interface=$(/sbin/ip -4 route | awk '/default/ { print $5 }' | head -n1 | awk '{print $1;}')
pingCmd="ping -I $gwIp_interface"
elif [ "$version" = "V6" ] ; then
gwIp=$(/sbin/ip -6 route | awk '/default/ { print $3 }' | head -n1 | awk '{print $1;}')
Comment thread
tukken-comcast marked this conversation as resolved.
gwIp_interface=$(/sbin/ip -6 route | awk '/default/ { print $5 }' | head -n1 | awk '{print $1;}')
Expand All @@ -232,79 +246,114 @@ checkPacketLoss()

if [ "$gwIp" != "" ] && [ "$gwIp" != "dev" ] ; then
gwResponse=$($pingCmd -c "$pingCount" -i "$pingInterval" "$gwIp")
ret=$(echo "$gwResponse" | grep "packet"|awk '{print $7}'|cut -d'%' -f1)
packetLoss=$(echo "$gwResponse" | grep "packet"|awk '{print $7}'|cut -d'%' -f1)

if [ "$version" = "V4" ] ; then
packetsLostipv4=$ret
ipv4PacketLoss=$packetLoss
ipv4GwPresent=1
elif [ "$version" = "V6" ] ; then
packetsLostipv6=$ret
ipv6PacketLoss=$packetLoss
ipv6GwPresent=1
fi

gwResponseTime=$(echo "$gwResponse" | sed '$!d;s|.*/\([0-9.]*\)/.*|\1|')
#Notify 100% gateway packet loss on every run, independent of GatewayLoggingInterval,
#so the marker is not skipped when the run cadence is shorter than GatewayLoggingInterval.
if [ "$packetLoss" = "100" ] ; then
log "$version SYST_WARN_GW100PERC_PACKETLOSS, gateway $gwIp"
t2CountNotify "SYST_WARN_GW100PERC_PACKETLOSS"
fi
if [ "$(($GatewayLogTimeStamp+$GatewayLoggingInterval))" -le "$currentTime" ] ; then
echo "$(/bin/timestamp) $version gateway = $gwIp " >> "$logsFile"
if [ "$ret" = "100" ] ; then
echo "$(/bin/timestamp) TELEMETRY_GATEWAY_RESPONSE_TIME:NR,$gwIp" >> "$logsFile"
echo "$(/bin/timestamp) Current Packet loss is SYST_WARN_GW100PERC_PACKETLOSS"
t2CountNotify "SYST_WARN_GW100PERC_PACKETLOSS"
if [ "$packetLoss" = "100" ] ; then
log "TELEMETRY_GATEWAY_RESPONSE_TIME:NR,$gwIp"
else
echo "$(/bin/timestamp) TELEMETRY_GATEWAY_RESPONSE_TIME:$gwResponseTime,$gwIp" >> "$logsFile"
log "TELEMETRY_GATEWAY_RESPONSE_TIME:$gwResponseTime,$gwIp"
fi
echo "$(/bin/timestamp) TELEMETRY_GATEWAY_PACKET_LOSS:$ret,$gwIp" >> "$logsFile"
log "TELEMETRY_GATEWAY_PACKET_LOSS:$packetLoss,$gwIp"
fi
else
if [ "$(($GatewayLogTimeStamp+$GatewayLoggingInterval))" -le "$currentTime" ] ; then
echo "$(/bin/timestamp) TELEMETRY_GATEWAY_NO_ROUTE_$version" >> "$logsFile"
log "TELEMETRY_GATEWAY_NO_ROUTE_$version"
t2CountNotify "WIFIV_INFO_NO${version}ROUTE"
fi
fi

[ "$(($GatewayLogTimeStamp+$GatewayLoggingInterval))" -le "$currentTime" ] && GatewayLogTimeStamp=$(($(date +%s)))

#Send telemetry notification for 20%,30%....90% packet loss
if [ "$packetsLostipv4" -gt "$lossThreshold" ] || [ "$packetsLostipv6" -gt "$lossThreshold" ] ; then
echo "$(/bin/timestamp) Packet loss more than $lossThreshold% observed." >> "$logsFile"
if [ "$packetsLostipv4" -ne 100 ] && [ "$packetsLostipv6" -ne 100 ]; then
for i in {1..9}; do
if ([ "$packetsLostipv4" -ge $((i*10)) ] && [ "$packetsLostipv4" -lt $((i*10+10)) ]) || ([ "$packetsLostipv6" -ge $((i*10)) ] && [ "$packetsLostipv6" -lt $((i*10+10)) ]); then
echo "$(/bin/timestamp) Current Packet loss is WIFIV_WARN_PL_"$((i*10))"PERC" >> "$logsFile"
t2CountNotify "WIFIV_WARN_PL_"$((i*10))"PERC"
break
fi
done
fi
else
if [ "$packetsLostipv4" -ne 0 ] && [ "$packetsLostipv6" -ne 0 ]; then
#Send telemetry notification for 20%,30%....90% packet loss
if [ -n "$packetLoss" ] ; then
if [ "$packetLoss" -gt "$lossThreshold" ] ; then
log "$version packet loss more than $lossThreshold% observed (packetLoss=$packetLoss)"
if [ "$packetLoss" -ne 100 ] ; then
for i in 1 2 3 4 5 6 7 8 9; do
if [ "$packetLoss" -ge $((i*10)) ] && [ "$packetLoss" -lt $((i*10+10)) ] ; then
log "$version packet loss is WIFIV_WARN_PL_$((i*10))PERC"
t2CountNotify "WIFIV_WARN_PL_"$((i*10))"PERC"
break
fi
done
fi
elif [ "$packetLoss" -ne 0 ] ; then
#Send telemetry notification for 10% packet loss
echo "$(/bin/timestamp) Current Packet loss is WIFIV_WARN_PL_10PERC" >> "$logsFile"
log "$version packet loss is WIFIV_WARN_PL_10PERC"
t2CountNotify "WIFIV_WARN_PL_10PERC"
fi
fi

if [ "$packetsLostipv4" -ge "$WifiReassociateTolerance" ] && [ "$packetsLostipv6" -ge "$WifiReassociateTolerance" ]; then
echo "$(/bin/timestamp) ${WifiReassociateTolerance}% Packet loss is observed for both ipv4 and ipv6." >> "$logsFile"
#Note down $FirstPacketLossTime when threshold packetloss is detected for the first time
[ "$FirstPacketLossTime" -eq 0 ] && FirstPacketLossTime=$(($(date +%s)))
#Note down $PacketLossLogTimeStamp when PacketLossLogTimeStamp is 0
[ "$PacketLossLogTimeStamp" -eq 0 ] && PacketLossLogTimeStamp=$(($(date +%s)))
#Note down $EthernetLogTimeStamp when EthernetLogTimeStamp is 0 and ethernet connected
[ "$IsEthernetConnected" -eq 1 ] && [ "$EthernetLogTimeStamp" -eq 0 ] && EthernetLogTimeStamp=$(($(date +%s)))
return 1
fi
#Evaluate the packet-loss trigger only on the V6 call, once both stacks have
#been probed (globals set during the preceding V4 call persist here). Classify
#each routed stack as good (loss < tolerance = acceptable connectivity) or bad
#(loss >= tolerance); a stack with no default route or an unparseable ping
#result is ignored. Because any acceptable stack means we should not tear down
#L2, recovery is warranted only when NO routed stack is good and at least one
#routed stack is bad:
# - dual-stack (IPv4 + IPv6): both must be at/above tolerance to return 1
# - IPv4-only : IPv4 packet loss alone controls the result
# - IPv6-only : IPv6 packet loss alone controls the result
if [ "$version" = "V6" ] ; then
#Log the per-stack route/loss snapshot only when it changes from the previous
#run, so field logs capture every transition without flooding on every run.
currState="ipv4Route=$ipv4GwPresent ipv6Route=$ipv6GwPresent ipv4PacketLoss=$ipv4PacketLoss ipv6PacketLoss=$ipv6PacketLoss"
prevState=""
[ -f "$prevStateFile" ] && prevState=$(cat "$prevStateFile")
if [ "$currState" != "$prevState" ] ; then
log "network state changed: ($currState)"
echo "$currState" > "$prevStateFile"
fi

#Reset tmp parameters to default values only after both V4 and V6 are measured (on V6 call).
#Resetting on V4 check alone would be premature because packetsLostipv6 is still 0 (script init)
#at that point, causing FirstPacketLossTime to be incorrectly cleared before V6 is measured.
#Reset if either V4 or V6 is below the reassociate tolerance, indicating recovery on at least one path.
if [ "$version" = "V6" ] && { [ "$packetsLostipv4" -lt "$WifiReassociateTolerance" ] || [ "$packetsLostipv6" -lt "$WifiReassociateTolerance" ]; }; then
echo "$(/bin/timestamp) [DEBUG_NCR] checkPacketLoss: BELOW TOLERANCE returning 0 - resetting FirstPacketLossTime/PacketLossLogTimeStamp/IsWifiReassociated. wifiDriverErrors=$wifiDriverErrors" >> "$logsFile"
FirstPacketLossTime=0
PacketLossLogTimeStamp=0
EthernetLogTimeStamp=0
IsWifiReassociated=0
[ "$wifiDriverErrors" -eq 0 ] && IsWifiReset=0 #Make IsWifiReset=0 only when there is no wifidriverissue
else
echo "$(/bin/timestamp) [DEBUG_NCR] checkPacketLoss: BELOW TOLERANCE returning 0 - skipping reset (version=$version, waiting for V6 measurement). wifiDriverErrors=$wifiDriverErrors" >> "$logsFile"
anyGood=0
anyBad=0
if [ "$ipv4GwPresent" -eq 1 ] && [ -n "$ipv4PacketLoss" ] ; then
if [ "$ipv4PacketLoss" -ge "$WifiReassociateTolerance" ] ; then anyBad=1 ; else anyGood=1 ; fi
fi
if [ "$ipv6GwPresent" -eq 1 ] && [ -n "$ipv6PacketLoss" ] ; then
if [ "$ipv6PacketLoss" -ge "$WifiReassociateTolerance" ] ; then anyBad=1 ; else anyGood=1 ; fi
fi

if [ "$anyGood" -eq 0 ] && [ "$anyBad" -eq 1 ] ; then
#No routed stack has acceptable connectivity and at least one is bad -> recover.
log "Packet loss is observed on all routed IP stacks (ipv4Route=$ipv4GwPresent ipv6Route=$ipv6GwPresent ipv4PacketLoss=$ipv4PacketLoss ipv6PacketLoss=$ipv6PacketLoss tolerance=${WifiReassociateTolerance}%)"
#Note down $FirstPacketLossTime when threshold packetloss is detected for the first time
[ "$FirstPacketLossTime" -eq 0 ] && FirstPacketLossTime=$(($(date +%s)))
#Note down $PacketLossLogTimeStamp when PacketLossLogTimeStamp is 0
[ "$PacketLossLogTimeStamp" -eq 0 ] && PacketLossLogTimeStamp=$(($(date +%s)))
#Note down $EthernetLogTimeStamp when EthernetLogTimeStamp is 0 and ethernet connected
[ "$IsEthernetConnected" -eq 1 ] && [ "$EthernetLogTimeStamp" -eq 0 ] && EthernetLogTimeStamp=$(($(date +%s)))
return 1
elif [ "$anyGood" -eq 1 ] ; then
#At least one routed stack has acceptable connectivity (below tolerance) -> clear packet-loss state.
# log "[DEBUG_NCR] checkPacketLoss: acceptable connectivity on a routed stack (ipv4PacketLoss=$ipv4PacketLoss ipv6PacketLoss=$ipv6PacketLoss) - resetting FirstPacketLossTime/PacketLossLogTimeStamp/IsWifiReassociated. wifiDriverErrors=$wifiDriverErrors"
FirstPacketLossTime=0
PacketLossLogTimeStamp=0
EthernetLogTimeStamp=0
IsWifiReassociated=0
[ "$wifiDriverErrors" -eq 0 ] && IsWifiReset=0 #Make IsWifiReset=0 only when there is no wifidriverissue
else
#No routed stack was measurable (no route / unparseable) -> skip reset so a total loss of
#routes does not wipe an in-progress packet-loss timer.
# log "[DEBUG_NCR] checkPacketLoss: no routed-stack measurement (version=$version) - skipping reset. wifiDriverErrors=$wifiDriverErrors"
:
fi
fi

return 0
Expand All @@ -324,29 +373,29 @@ wifiReset()
#Set IsWifiReset to 1 after wifi reset
IsWifiReset=1
StoreTotmpFile
echo "$(/bin/timestamp) Start WiFi Reset. !!!!!!!!!!!!!!" >> "$logsFile"
log "Start WiFi Reset. !!!!!!!!!!!!!!"

systemctl restart wifi.service
echo "$(/bin/timestamp) WiFi Reset done as part of Recovery. !!!!!!!!!!!!!!" >> "$logsFile"
log "WiFi Reset done as part of Recovery. !!!!!!!!!!!!!!"
exit 0
}

checkDnsFile()
{
if [ -f "$dnsFile" ] ; then
if [ $(tr -d ' \r\n\t' < $dnsFile | wc -c ) -eq 0 ] ; then
echo "$(/bin/timestamp) DNS File($dnsFile) is empty" >> "$logsFile"
log "DNS File($dnsFile) is empty"
t2CountNotify "SYST_ERR_DNSFileEmpty"
gwIpv4=$(/sbin/ip -4 route show default | awk 'NR==1 {print $3; exit}')
gwIpv6=$(/sbin/ip -6 route show default | awk 'NR==1 {print $3; exit}')
routeIpv4=$(/sbin/ip -4 route)
routeIpv6=$(/sbin/ip -6 route)
routeIpv6=$(/sbin/ip -6 route)
if [ "$gwIpv4" != "" ] || [ "$gwIpv6" != "" ] ; then
dnsFailures=$((dnsFailures + 1))
case $routeIpv4 in
*"error"*)
dnsFailures=0
;;
;;
esac
case $routeIpv6 in
*"error"*)
Expand All @@ -358,7 +407,7 @@ checkDnsFile()
fi

if [ "$dnsFailures" -gt "$maxdnsFailures" ] ; then
echo "$(/bin/timestamp) Restarting udhcpc to recover" >> "$logsFile"
log "Restarting udhcpc to recover"
InterfaceList="$ethernet_interface $WIFI_INTERFACE"
for interface in $InterfaceList
do
Expand All @@ -376,15 +425,15 @@ checkDnsFile()
dnsFailures=0
fi
else
echo "$(/bin/timestamp) DNS File is not there $dnsFile" >> "$logsFile"
log "DNS File is not there $dnsFile"
fi
}

checkRfc()
{
rfcWifiResetEnable="$(tr181 Device.DeviceInfo.X_RDKCENTRAL-COM_RFC.Feature.WiFiReset.Enable 2>&1 > /dev/null)"
if [ "$rfcWifiResetEnable" = "true" ] ; then
echo "$(/bin/timestamp) WiFiReset RFC is true " >> "$logsFile"
log "WiFiReset RFC is true "
rfcEthernetLoggingInterval="$(tr181 Device.DeviceInfo.X_RDKCENTRAL-COM_RFC.Feature.WiFiReset.EthernetLoggingInterval 2>&1 > /dev/null)"
if [ ! -z "$rfcEthernetLoggingInterval" ] && [ "$rfcEthernetLoggingInterval" != 0 ] ; then
EthernetLoggingInterval="$rfcEthernetLoggingInterval"
Expand Down Expand Up @@ -427,7 +476,7 @@ LoadFromtmpFile
if [ "$IsWifiReset" -eq 1 ] ; then
currentTime=$(($(date +%s)))
if [ "$(($WifiResetTime+$wifiResetWaitTime))" -gt "$currentTime" ] ; then
echo "$(/bin/timestamp) Skip all checks since wifi reset is done recently" >> "$logsFile"
log "Skip all checks since wifi reset is done recently"
exit 0
fi
fi
Expand Down