diff --git a/data/test.txt b/data/test.txt new file mode 100644 index 0000000..d65aae4 --- /dev/null +++ b/data/test.txt @@ -0,0 +1,76 @@ +103 +WWUS83 KUNR 021906 +RFWUNR + +URGENT - FIRE WEATHER MESSAGE +National Weather Service Rapid City SD +106 PM MDT Thu Oct 2 2025 + +...RED FLAG WARNING IN EFFECT FROM FRIDAY AFTERNOON THROUGH +FRIDAY EVENING FOR PORTIONS OF NORTHEAST WY AND SOUTHWEST SD... + +.Hot temperatures and breezy southwest winds are expected Friday +for much of the area. RH's will fall to around 10 to 15 percent +across northeast WY and much of southwest SD in the afternoon +with breezy southwest winds developing, as winds may gust to +25 to 30 mph. Given dry fuels and these conditions, a fire +weather watch has been upgraded to a Red Flag Warning for Friday +afternoon for much of northeast WY and southwest SD. + +SDZ320-022015- +/O.CAN.KUNR.FW.A.0010.251003T1800Z-251004T0100Z/ +Central Black Hills- +106 PM MDT Thu Oct 2 2025 + +...FIRE WEATHER WATCH IS CANCELLED FOR GUSTY WINDS AND LOW +RELATIVE HUMIDITY FOR FIRE WEATHER ZONE 320... + +The National Weather Service in Rapid City has cancelled the Fire +Weather Watch. + +Near critical fire weather conditions will still exist Friday +afternoon. + +$$ + +SDZ321-322-324>326-329-332-WYZ314>317-031015- +/O.UPG.KUNR.FW.A.0010.251003T1800Z-251004T0100Z/ +/O.NEW.KUNR.FW.W.0011.251003T1800Z-251004T0100Z/ +Southern Black Hills-Fall River County Area-Eastern Foot Hills- +Custer County Plains-Pine Ridge Area-West Central Plains- +Badlands Area-Northern Campbell-Southern Campbell- +Crook County Plains-Weston County Plains- +106 PM MDT Thu Oct 2 2025 + +...RED FLAG WARNING IN EFFECT FROM NOON TO 7 PM MDT FRIDAY FOR +GUSTY WINDS AND LOW RELATIVE HUMIDITY FOR FIRE WEATHER ZONES 314, +315, 316, 317, 321, 322, 324, 325, 326, 329, AND 332... + +The National Weather Service in Rapid City has issued a Red Flag +Warning for gusty winds and low relative humidity, which is in +effect from noon to 7 PM MDT Friday. The Fire Weather Watch is no +longer in effect. + +* AFFECTED AREA...Fire Weather Zones 314 Northern Campbell, 315 + Southern Campbell, 316 Crook County Plains, 317 Weston County + Plains, 321 Southern Black Hills, 322 Fall River County Area, + 324 Eastern Foot Hills, 325 Custer County Plains, 326 Pine + Ridge Area, 329 West Central Plains and 332 Badlands Area. + +* WINDS...Southwest 10 to 20 mph with gusts up to 30 mph. + +* RELATIVE HUMIDITY...As low as 13 percent. + +* IMPACTS...The combination of gusty winds and low relative + humidity would produce critical fire weather conditions. + +PRECAUTIONARY/PREPAREDNESS ACTIONS... + +A Red Flag Warning means that critical fire weather conditions +are either occurring now, or will shortly. A combination of +strong winds, low relative humidity, and warm temperatures can +contribute to extreme fire behavior. + +&& + +$$ diff --git a/data/test/awips/tor/TOR-W-KOUN-14-2025-1.txt b/data/test/awips/tor/TOR-W-KOUN-14-2025-1.txt new file mode 100644 index 0000000..81b3bfc --- /dev/null +++ b/data/test/awips/tor/TOR-W-KOUN-14-2025-1.txt @@ -0,0 +1,61 @@ +802 +WFUS54 KOUN 200120 +TOROUN +OKC019-067-085-200200- +/O.NEW.KOUN.TO.W.0014.250420T0120Z-250420T0200Z/ + +BULLETIN - EAS ACTIVATION REQUESTED +Tornado Warning +National Weather Service Norman OK +820 PM CDT Sat Apr 19 2025 + +The National Weather Service in Norman has issued a + +* Tornado Warning for... + Northeastern Jefferson County in southern Oklahoma... + Northwestern Love County in southern Oklahoma... + Southwestern Carter County in southern Oklahoma... + +* Until 900 PM CDT. + +* At 820 PM CDT, a severe thunderstorm capable of producing a tornado + was located 5 miles northwest of Rubottom, moving northeast at 20 + mph. + + HAZARD...Tornado and ping pong ball size hail. + + SOURCE...Radar indicated rotation. + + IMPACT...Flying debris will be dangerous to those caught without + shelter. Mobile homes will be damaged or destroyed. + Damage to roofs, windows, and vehicles will occur. Tree + damage is likely. + +* Locations impacted include... + Lone Grove, Healdton, Wilson, Ringling, Leon, Rubottom, + Burneyville, Cornish, Courtney, and Petersburg. + +PRECAUTIONARY/PREPAREDNESS ACTIONS... + +TAKE COVER NOW! Move to a storm shelter, safe room or an interior +room on the lowest floor of a sturdy building. Avoid windows. If you +are outdoors, in a mobile home, or in a vehicle, move to the closest +substantial shelter and protect yourself from flying debris. + +Tornadoes are extremely difficult to see and confirm at night. Do not +wait to see or hear the tornado. TAKE COVER NOW! + +&& + +LAT...LON 3382 9744 3383 9746 3390 9745 3392 9749 + 3390 9755 3391 9759 3392 9760 3395 9759 + 3397 9762 3429 9762 3427 9719 3385 9733 + 3388 9733 3382 9737 +TIME...MOT...LOC 0120Z 208DEG 18KT 3398 9753 + +TORNADO...RADAR INDICATED +MAX HAIL SIZE...1.50 IN + +$$ + +Mahale \ No newline at end of file diff --git a/data/test/awips/tor/TOR-W-KOUN-14-2025-2.txt b/data/test/awips/tor/TOR-W-KOUN-14-2025-2.txt new file mode 100644 index 0000000..6105777 --- /dev/null +++ b/data/test/awips/tor/TOR-W-KOUN-14-2025-2.txt @@ -0,0 +1,57 @@ +922 +WWUS54 KOUN 200140 +SVSOUN + +Severe Weather Statement +National Weather Service Norman OK +840 PM CDT Sat Apr 19 2025 + +OKC019-067-085-200200- +/O.CON.KOUN.TO.W.0014.000000T0000Z-250420T0200Z/ +Jefferson OK-Love OK-Carter OK- +840 PM CDT Sat Apr 19 2025 + +...A TORNADO WARNING REMAINS IN EFFECT UNTIL 900 PM CDT FOR +NORTHEASTERN JEFFERSON...NORTHWESTERN LOVE AND SOUTHWESTERN CARTER +COUNTIES... + +At 840 PM CDT, a severe thunderstorm capable of producing a tornado +was located 4 miles north of Rubottom, moving northeast at 20 mph. + +HAZARD...Tornado and ping pong ball size hail. + +SOURCE...Radar indicated rotation. + +IMPACT...Flying debris will be dangerous to those caught without + shelter. Mobile homes will be damaged or destroyed. Damage + to roofs, windows, and vehicles will occur. Tree damage is + likely. + +Locations impacted include... +Lone Grove, Healdton, Wilson, Ringling, Leon, Rubottom, Burneyville, +Cornish, Courtney, and Petersburg. + +PRECAUTIONARY/PREPAREDNESS ACTIONS... + +TAKE COVER NOW! Move to a storm shelter, safe room, or an interior +room on the lowest floor of a sturdy building. Avoid windows. If you +are outdoors, in a mobile home, or in a vehicle, move to the closest +substantial shelter and protect yourself from flying debris. + +Tornadoes are extremely difficult to see and confirm at night. Do not +wait to see or hear the tornado. TAKE COVER NOW! + +&& + +LAT...LON 3386 9745 3390 9745 3392 9749 3390 9755 + 3391 9759 3392 9760 3395 9759 3397 9762 + 3429 9762 3427 9719 3386 9733 3388 9733 + 3386 9734 +TIME...MOT...LOC 0140Z 213DEG 17KT 3401 9748 + +TORNADO...RADAR INDICATED +MAX HAIL SIZE...1.50 IN + +$$ + +Mahale \ No newline at end of file diff --git a/data/test/awips/tor/TOR-W-KOUN-14-2025-3.txt b/data/test/awips/tor/TOR-W-KOUN-14-2025-3.txt new file mode 100644 index 0000000..f67df21 --- /dev/null +++ b/data/test/awips/tor/TOR-W-KOUN-14-2025-3.txt @@ -0,0 +1,71 @@ +910 +WWUS54 KOUN 200147 +SVSOUN + +Severe Weather Statement +National Weather Service Norman OK +847 PM CDT Sat Apr 19 2025 + +OKC067-200156- +/O.CAN.KOUN.TO.W.0014.000000T0000Z-250420T0200Z/ +Jefferson OK- +847 PM CDT Sat Apr 19 2025 + +...THE TORNADO WARNING FOR NORTHEASTERN JEFFERSON COUNTY IS +CANCELLED... + +The tornadic thunderstorm which prompted the warning has moved out +of the warned area. Therefore, the warning has been cancelled. + +LAT...LON 3395 9756 3398 9756 3398 9754 3401 9756 + 3429 9756 3427 9719 3393 9731 3392 9745 +TIME...MOT...LOC 0146Z 212DEG 19KT 3404 9743 + +$$ + +OKC019-085-200200- +/O.CON.KOUN.TO.W.0014.000000T0000Z-250420T0200Z/ +Love OK-Carter OK- +847 PM CDT Sat Apr 19 2025 + +...A TORNADO WARNING REMAINS IN EFFECT UNTIL 900 PM CDT FOR +NORTHWESTERN LOVE AND SOUTHWESTERN CARTER COUNTIES... + +At 846 PM CDT, a confirmed tornado was located 7 miles north of +Rubottom, moving northeast at 20 mph. + +HAZARD...Damaging tornado and ping pong ball size hail. + +SOURCE...Weather spotters confirmed tornado. + +IMPACT...Flying debris will be dangerous to those caught without + shelter. Mobile homes will be damaged or destroyed. Damage + to roofs, windows, and vehicles will occur. Tree damage is + likely. + +Locations impacted include... +Lone Grove, Healdton, Wilson, Rubottom, and Courtney. + +PRECAUTIONARY/PREPAREDNESS ACTIONS... + +To repeat, a tornado is on the ground. TAKE COVER NOW! Move to a +storm shelter, safe room, or an interior room on the lowest floor of +a sturdy building. Avoid windows. If you are outdoors, in a mobile +home, or in a vehicle, move to the closest substantial shelter and +protect yourself from flying debris. + +Tornadoes are extremely difficult to see and confirm at night. Do not +wait to see or hear the tornado. TAKE COVER NOW! + +&& + +LAT...LON 3395 9756 3398 9756 3398 9754 3401 9756 + 3429 9756 3427 9719 3393 9731 3392 9745 +TIME...MOT...LOC 0146Z 212DEG 19KT 3404 9743 + +TORNADO...OBSERVED +MAX HAIL SIZE...1.50 IN + +$$ + +Mahale \ No newline at end of file diff --git a/data/test/awips/winter weather/WW-Y-KPBZ-26-2025-1.txt b/data/test/awips/winter weather/WW-Y-KPBZ-26-2025-1.txt new file mode 100644 index 0000000..3103653 --- /dev/null +++ b/data/test/awips/winter weather/WW-Y-KPBZ-26-2025-1.txt @@ -0,0 +1,162 @@ +913 +WWUS41 KPBZ 251742 +WSWPBZ + +URGENT - WINTER WEATHER MESSAGE +National Weather Service Pittsburgh PA +1242 PM EST Thu Dec 25 2025 + +PAZ008-009-015-016-022-074-077-078-260315- +/O.UPG.KPBZ.WS.A.0011.251226T1200Z-251227T1200Z/ +/O.NEW.KPBZ.IS.W.0004.251226T1500Z-251227T1200Z/ +Venango-Forest-Clarion-Jefferson PA-Armstrong-Higher Elevations +of Westmoreland-Indiana-Higher Elevations of Indiana- +Including the cities of Indiana, Oil City, Punxsutawney, Kittanning, +Ford City, Donegal, Franklin, Tionesta, Brookville, Ligonier, Armagh, +and Clarion +1242 PM EST Thu Dec 25 2025 + +...ICE STORM WARNING IN EFFECT FROM 10 AM FRIDAY TO 7 AM EST +SATURDAY... + +* WHAT...Significant icing expected. Total snow accumulations up to + half an inch and ice accumulations between two tenths and three + tenths of an inch. + +* WHERE...Portions of northwest, southwest, and western Pennsylvania. + +* WHEN...From 10 AM Friday to 7 AM EST Saturday. + +* IMPACTS...Roads, and especially bridges and overpasses, will likely + become slick and hazardous. Power outages and tree damage are + likely due to the ice. Travel could be nearly impossible. The + hazardous conditions will impact the Friday post-holiday travel. + +PRECAUTIONARY/PREPAREDNESS ACTIONS... + +Persons should delay all travel if possible. If travel is absolutely +necessary, drive with extreme caution. Leave plenty of room between +you and the motorist ahead of you, and allow extra time to reach your +destination. Avoid sudden braking or acceleration, and be especially +cautious on hills or when making turns. Make sure your car is +winterized and in good working order. + +Please report ice accumulations or damage by calling 412-262-1988, +posting to the NWS Pittsburgh Facebook page, or using X +@NWSPittsburgh. + +&& + +$$ + +PAZ076-260315- +/O.UPG.KPBZ.WS.A.0011.251226T1200Z-251227T1200Z/ +/O.NEW.KPBZ.WW.Y.0026.251226T1300Z-251227T0600Z/ +Higher Elevations of Fayette- +Including the cities of Champion and Ohiopyle +1242 PM EST Thu Dec 25 2025 + +...WINTER WEATHER ADVISORY IN EFFECT FROM 8 AM FRIDAY TO 1 AM EST +SATURDAY... + +* WHAT...Freezing rain expected. Total snow accumulations up to half + an inch and ice accumulations up to two tenths of an inch. + +* WHERE...Higher Elevations of Fayette County. + +* WHEN...From 8 AM Friday to 1 AM EST Saturday. + +* IMPACTS...Roads, and especially bridges and overpasses, will likely + become slick and hazardous. Plan on slippery road conditions. The + hazardous conditions will impact the Friday post-holiday travel. + +PRECAUTIONARY/PREPAREDNESS ACTIONS... + +Slow down and use caution while traveling. The latest road conditions +can be obtained by calling 5 1 1. + +Please report snow or ice accumulations by calling 412-262-1988, +posting to the NWS Pittsburgh Facebook page, or using X +@NWSPittsburgh. + +&& + +$$ + +OHZ040-041-049-050-PAZ020-021-029-WVZ001-002-260315- +/O.NEW.KPBZ.WW.Y.0026.251226T1300Z-251226T2000Z/ +Carroll-Columbiana-Harrison-Jefferson OH-Beaver-Allegheny- +Washington-Hancock-Brooke- +Including the cities of Ambridge, Follansbee, Salem, Aliquippa, +Beaver Falls, Wellsburg, Cadiz, Washington, East Liverpool, +Canonsburg, Malvern, Columbiana, Monaca, Pittsburgh Metro Area, +Weirton, Carrollton, and Steubenville +1242 PM EST Thu Dec 25 2025 + +...WINTER WEATHER ADVISORY IN EFFECT FROM 8 AM TO 3 PM EST FRIDAY... + +* WHAT...Freezing rain expected. Total ice accumulations of a glaze + to one tenth of an inch. + +* WHERE...Portions of east central Ohio, southwest and western + Pennsylvania, and northern and the northern panhandle of West + Virginia. + +* WHEN...From 8 AM to 3 PM EST Friday. + +* IMPACTS...Roads, and especially bridges and overpasses, will likely + become slick and hazardous. Plan on slippery road conditions. The + hazardous conditions will impact the Friday post-holiday travel. + +PRECAUTIONARY/PREPAREDNESS ACTIONS... + +Slow down and use caution while traveling. The latest road conditions +can be obtained by calling 5 1 1. + +Please report snow or ice accumulations by calling 412-262-1988, +posting to the NWS Pittsburgh Facebook page, or using X +@NWSPittsburgh. + +&& + +$$ + +PAZ007-013-014-073-WVZ512-514-260315- +/O.NEW.KPBZ.WW.Y.0026.251226T1300Z-251227T0600Z/ +Mercer-Lawrence-Butler-Westmoreland-Eastern Preston-Eastern +Tucker- +Including the cities of Ellwood City, Sharon, Grove City, Monessen, +Latrobe, Murrysville, Butler, Lower Burrell, Rowlesburg, Hazelton, +Greensburg, Thomas, New Kensington, New Castle, Canaan Valley, Davis, +Terra Alta, and Hermitage +1242 PM EST Thu Dec 25 2025 + +...WINTER WEATHER ADVISORY IN EFFECT FROM 8 AM FRIDAY TO 1 AM EST +SATURDAY... + +* WHAT...Freezing rain expected. Total ice accumulations of one to + two tenths of an inch. + +* WHERE...Portions of northwest, southwest, and western Pennsylvania + and northern West Virginia. + +* WHEN...From 8 AM Friday to 1 AM EST Saturday. + +* IMPACTS...Roads, and especially bridges and overpasses, will likely + become slick and hazardous. Plan on slippery road conditions. The + hazardous conditions will impact the Friday post-holiday travel. + +PRECAUTIONARY/PREPAREDNESS ACTIONS... + +Slow down and use caution while traveling. The latest road conditions +can be obtained by calling 5 1 1. + +Please report snow or ice accumulations by calling 412-262-1988, +posting to the NWS Pittsburgh Facebook page, or using X +@NWSPittsburgh. + +&& + +$$ + +MLB diff --git a/data/test/awips/winter weather/WW-Y-KPBZ-26-2025-2.txt b/data/test/awips/winter weather/WW-Y-KPBZ-26-2025-2.txt new file mode 100644 index 0000000..b5acfeb --- /dev/null +++ b/data/test/awips/winter weather/WW-Y-KPBZ-26-2025-2.txt @@ -0,0 +1,128 @@ +875 +WWUS41 KPBZ 260028 +WSWPBZ + +URGENT - WINTER WEATHER MESSAGE +National Weather Service Pittsburgh PA +728 PM EST Thu Dec 25 2025 + +PAZ008-009-015-016-022-074-077-078-260900- +/O.CON.KPBZ.IS.W.0004.251226T1500Z-251227T1200Z/ +Venango-Forest-Clarion-Jefferson PA-Armstrong-Higher Elevations +of Westmoreland-Indiana-Higher Elevations of Indiana- +Including the cities of Indiana, Tionesta, Ligonier, Clarion, +Donegal, Punxsutawney, Kittanning, Brookville, Franklin, Armagh, Oil +City, and Ford City +728 PM EST Thu Dec 25 2025 + +...ICE STORM WARNING REMAINS IN EFFECT FROM 10 AM FRIDAY TO 7 AM EST +SATURDAY... + +* WHAT...Significant icing expected. Total snow accumulations up to + half an inch and ice accumulations between two tenths and three + tenths of an inch. + +* WHERE...Portions of northwest, southwest, and western Pennsylvania. + +* WHEN...From 10 AM Friday to 7 AM EST Saturday. + +* IMPACTS...Roads, and especially bridges and overpasses, will likely + become slick and hazardous. Power outages and tree damage are + likely due to the ice. Travel could be nearly impossible. The + hazardous conditions will impact the Friday post-holiday travel. + +PRECAUTIONARY/PREPAREDNESS ACTIONS... + +Persons should delay all travel if possible. If travel is absolutely +necessary, drive with extreme caution. Leave plenty of room between +you and the motorist ahead of you, and allow extra time to reach your +destination. Avoid sudden braking or acceleration, and be especially +cautious on hills or when making turns. Make sure your car is +winterized and in good working order. + +Please report ice accumulations or damage by calling 412-262-1988, +posting to the NWS Pittsburgh Facebook page, or using X +@NWSPittsburgh. + +&& + +$$ + +OHZ040-041-049-050-PAZ020-021-029-WVZ001-002-260900- +/O.CON.KPBZ.WW.Y.0026.251226T1300Z-251226T2000Z/ +Carroll-Columbiana-Harrison-Jefferson OH-Beaver-Allegheny- +Washington-Hancock-Brooke- +Including the cities of Canonsburg, Carrollton, Monaca, Wellsburg, +Cadiz, Washington, Steubenville, Aliquippa, Ambridge, East Liverpool, +Weirton, Columbiana, Malvern, Beaver Falls, Pittsburgh Metro Area, +Follansbee, and Salem +728 PM EST Thu Dec 25 2025 + +...WINTER WEATHER ADVISORY REMAINS IN EFFECT FROM 8 AM TO 3 PM EST +FRIDAY... + +* WHAT...Freezing rain expected. Total ice accumulations of a glaze + to one tenth of an inch. + +* WHERE...Portions of east central Ohio, southwest and western + Pennsylvania, and northern and the northern panhandle of West + Virginia. + +* WHEN...From 8 AM to 3 PM EST Friday. + +* IMPACTS...Roads, and especially bridges and overpasses, will likely + become slick and hazardous. Plan on slippery road conditions. The + hazardous conditions will impact the Friday post-holiday travel. + +PRECAUTIONARY/PREPAREDNESS ACTIONS... + +Slow down and use caution while traveling. The latest road conditions +can be obtained by calling 5 1 1. + +Please report snow or ice accumulations by calling 412-262-1988, +posting to the NWS Pittsburgh Facebook page, or using X +@NWSPittsburgh. + +&& + +$$ + +PAZ007-013-014-073-076-WVZ512-514-260900- +/O.CON.KPBZ.WW.Y.0026.251226T1300Z-251227T0600Z/ +Mercer-Lawrence-Butler-Westmoreland-Higher Elevations of Fayette- +Eastern Preston-Eastern Tucker- +Including the cities of Ohiopyle, Hazelton, Sharon, Murrysville, +Ellwood City, Lower Burrell, New Castle, Butler, Greensburg, Thomas, +Grove City, Davis, Champion, Rowlesburg, Canaan Valley, New +Kensington, Latrobe, Monessen, Terra Alta, and Hermitage +728 PM EST Thu Dec 25 2025 + +...WINTER WEATHER ADVISORY REMAINS IN EFFECT FROM 8 AM FRIDAY TO 1 AM +EST SATURDAY... + +* WHAT...Freezing rain expected. Total ice accumulations of one to + two tenths of an inch. + +* WHERE...Portions of northwest, southwest, and western Pennsylvania + and northern West Virginia. + +* WHEN...From 8 AM Friday to 1 AM EST Saturday. + +* IMPACTS...Roads, and especially bridges and overpasses, will likely + become slick and hazardous. Plan on slippery road conditions. The + hazardous conditions will impact the Friday post-holiday travel. + +PRECAUTIONARY/PREPAREDNESS ACTIONS... + +Slow down and use caution while traveling. The latest road conditions +can be obtained by calling 5 1 1. + +Please report snow or ice accumulations by calling 412-262-1988, +posting to the NWS Pittsburgh Facebook page, or using X +@NWSPittsburgh. + +&& + +$$ + +Rackley \ No newline at end of file diff --git a/data/test/awips/winter weather/WW-Y-KPBZ-26-2025-3.txt b/data/test/awips/winter weather/WW-Y-KPBZ-26-2025-3.txt new file mode 100644 index 0000000..a8b9d4e --- /dev/null +++ b/data/test/awips/winter weather/WW-Y-KPBZ-26-2025-3.txt @@ -0,0 +1,127 @@ +727 +WWUS41 KPBZ 260643 +WSWPBZ + +URGENT - WINTER WEATHER MESSAGE +National Weather Service Pittsburgh PA +143 AM EST Fri Dec 26 2025 + +PAZ008-009-015-016-022-074-077-078-261900- +/O.CON.KPBZ.IS.W.0004.251226T1500Z-251227T1200Z/ +Venango-Forest-Clarion-Jefferson PA-Armstrong-Higher Elevations +of Westmoreland-Indiana-Higher Elevations of Indiana- +Including the cities of Clarion, Ligonier, Punxsutawney, Armagh, +Indiana, Tionesta, Donegal, Franklin, Oil City, Kittanning, Ford +City, and Brookville +143 AM EST Fri Dec 26 2025 + +...ICE STORM WARNING REMAINS IN EFFECT FROM 10 AM THIS MORNING TO 7 +AM EST SATURDAY... + +* WHAT...Significant icing expected. Total snow accumulations less + than one inch and ice accumulations between two and three tenths of + an inch. + +* WHERE...Portions of western Pennsylvania. + +* WHEN...From 10 AM this morning to 7 AM EST Saturday. + +* IMPACTS...Roads, and especially bridges and overpasses, will likely + become slick and hazardous. Power outages and tree damage are + likely due to the ice. Travel could be nearly impossible. The + hazardous conditions will impact the Friday post-holiday travel. + +PRECAUTIONARY/PREPAREDNESS ACTIONS... + +Persons should delay all travel if possible. If travel is absolutely +necessary, drive with extreme caution. Leave plenty of room between +you and the motorist ahead of you, and allow extra time to reach your +destination. Avoid sudden braking or acceleration, and be especially +cautious on hills or when making turns. Make sure your car is +winterized and in good working order. + +Please report ice accumulations or damage by calling 412-262-1988, +posting to the NWS Pittsburgh Facebook page, or using X +@NWSPittsburgh. + +&& + +$$ + +OHZ040-041-049-050-PAZ020-021-029-WVZ001-002-261900- +/O.CON.KPBZ.WW.Y.0026.251226T1300Z-251226T2000Z/ +Carroll-Columbiana-Harrison-Jefferson OH-Beaver-Allegheny- +Washington-Hancock-Brooke- +Including the cities of Aliquippa, Washington, Carrollton, Malvern, +Wellsburg, Salem, Weirton, Columbiana, Follansbee, Monaca, Ambridge, +Pittsburgh Metro Area, Steubenville, Cadiz, Canonsburg, Beaver Falls, +and East Liverpool +143 AM EST Fri Dec 26 2025 + +...WINTER WEATHER ADVISORY REMAINS IN EFFECT FROM 8 AM THIS MORNING +TO 3 PM EST THIS AFTERNOON... + +* WHAT...Freezing rain expected. Total ice accumulations around a + light glaze. + +* WHERE...Portions of east central Ohio, southwest and western + Pennsylvania, and the northern panhandle of West Virginia. + +* WHEN...From 8 AM this morning to 3 PM EST this afternoon. + +* IMPACTS...Roads, and especially bridges and overpasses, will likely + become slick and hazardous. Plan on slippery road conditions. The + hazardous conditions will impact the Friday post-holiday travel. + +PRECAUTIONARY/PREPAREDNESS ACTIONS... + +Slow down and use caution while traveling. The latest road conditions +can be obtained by calling 5 1 1. + +Please report snow or ice accumulations by calling 412-262-1988, +posting to the NWS Pittsburgh Facebook page, or using X +@NWSPittsburgh. + +&& + +$$ + +PAZ007-013-014-073-076-WVZ512-514-261900- +/O.CON.KPBZ.WW.Y.0026.251226T1300Z-251227T0600Z/ +Mercer-Lawrence-Butler-Westmoreland-Higher Elevations of Fayette- +Eastern Preston-Eastern Tucker- +Including the cities of Latrobe, New Kensington, Grove City, Butler, +Thomas, Canaan Valley, Greensburg, New Castle, Lower Burrell, +Murrysville, Ohiopyle, Monessen, Davis, Terra Alta, Ellwood City, +Champion, Rowlesburg, Hazelton, Hermitage, and Sharon +143 AM EST Fri Dec 26 2025 + +...WINTER WEATHER ADVISORY REMAINS IN EFFECT FROM 8 AM THIS MORNING +TO 1 AM EST SATURDAY... + +* WHAT...Freezing rain expected. Total ice accumulation up to a tenth + of an inch. + +* WHERE...Portions of western Pennsylvania and northern West + Virginia. + +* WHEN...From 8 AM this morning to 1 AM EST Saturday. + +* IMPACTS...Roads, and especially bridges and overpasses, will likely + become slick and hazardous. Plan on slippery road conditions. The + hazardous conditions will impact the Friday post-holiday travel. + +PRECAUTIONARY/PREPAREDNESS ACTIONS... + +Slow down and use caution while traveling. The latest road conditions +can be obtained by calling 5 1 1. + +Please report snow or ice accumulations by calling 412-262-1988, +posting to the NWS Pittsburgh Facebook page, or using X +@NWSPittsburgh. + +&& + +$$ + +WM \ No newline at end of file diff --git a/data/test/awips/winter weather/WW-Y-KPBZ-26-2025-4.txt b/data/test/awips/winter weather/WW-Y-KPBZ-26-2025-4.txt new file mode 100644 index 0000000..9550218 --- /dev/null +++ b/data/test/awips/winter weather/WW-Y-KPBZ-26-2025-4.txt @@ -0,0 +1,120 @@ +237 +WWUS41 KPBZ 261846 +WSWPBZ + +URGENT - WINTER WEATHER MESSAGE +National Weather Service Pittsburgh PA +146 PM EST Fri Dec 26 2025 + +OHZ040-041-049-050-PAZ020-021-029-WVZ001-002-262000- +/O.CAN.KPBZ.WW.Y.0026.000000T0000Z-251226T2000Z/ +Carroll-Columbiana-Harrison-Jefferson OH-Beaver-Allegheny- +Washington-Hancock-Brooke- +Including the cities of Monaca, Pittsburgh Metro Area, Steubenville, +Wellsburg, Canonsburg, Aliquippa, Ambridge, Columbiana, Cadiz, Salem, +Beaver Falls, Follansbee, Washington, Carrollton, Malvern, Weirton, +and East Liverpool +146 PM EST Fri Dec 26 2025 + +...WINTER WEATHER ADVISORY IS CANCELLED... + +Temperatures have climbed above freezing, so the freezing rain threat +has ended. + +$$ + +PAZ073-262000- +/O.CAN.KPBZ.WW.Y.0026.000000T0000Z-251227T0600Z/ +Westmoreland- +Including the cities of Lower Burrell, Greensburg, New Kensington, +Monessen, Latrobe, and Murrysville +146 PM EST Fri Dec 26 2025 + +...WINTER WEATHER ADVISORY IS CANCELLED... + +Temperatures have climbed above freezing, so the freezing rain threat +has ended. + +$$ + +PAZ008-009-015-016-022-074-077-078-270615- +/O.CON.KPBZ.IS.W.0004.000000T0000Z-251227T1200Z/ +Venango-Forest-Clarion-Jefferson PA-Armstrong-Higher Elevations +of Westmoreland-Indiana-Higher Elevations of Indiana- +Including the cities of Oil City, Kittanning, Brookville, Armagh, +Franklin, Tionesta, Ford City, Punxsutawney, Ligonier, Clarion, +Indiana, and Donegal +146 PM EST Fri Dec 26 2025 + +...ICE STORM WARNING REMAINS IN EFFECT UNTIL 7 AM EST SATURDAY... + +* WHAT...Significant icing expected. Total snow accumulations less + than one inch and ice accumulations between two and three tenths of + an inch. + +* WHERE...Portions of northwest, southwest, and western Pennsylvania. + +* WHEN...Until 7 AM EST Saturday. + +* IMPACTS...Roads, and especially bridges and overpasses, will likely + become slick and hazardous. Power outages and tree damage are + likely due to the ice. Travel could be nearly impossible. The + hazardous conditions will impact the Friday evening commute. + +PRECAUTIONARY/PREPAREDNESS ACTIONS... + +Persons should delay all travel if possible. If travel is absolutely +necessary, drive with extreme caution and be prepared for sudden +changes in visibility. Leave plenty of room between you and the +motorist ahead of you, and allow extra time to reach your +destination. Avoid sudden braking or acceleration, and be especially +cautious on hills or when making turns. Make sure your car is +winterized and in good working order. + +Please report ice accumulations or damage by calling 412-262-1988, +posting to the NWS Pittsburgh Facebook page, or using X +@NWSPittsburgh. + +&& + +$$ + +PAZ007-013-014-076-WVZ512-514-270600- +/O.CON.KPBZ.WW.Y.0026.000000T0000Z-251227T0600Z/ +Mercer-Lawrence-Butler-Higher Elevations of Fayette-Eastern +Preston-Eastern Tucker- +Including the cities of Thomas, Grove City, Hazelton, Sharon, Davis, +Ellwood City, Ohiopyle, Terra Alta, Hermitage, Canaan Valley, +Rowlesburg, Butler, New Castle, and Champion +146 PM EST Fri Dec 26 2025 + +...WINTER WEATHER ADVISORY REMAINS IN EFFECT UNTIL 1 AM EST +SATURDAY... + +* WHAT...Freezing rain. Additional ice accumulations around one tenth + of an inch. + +* WHERE...Portions of northwest, southwest, and western Pennsylvania + and northern West Virginia. + +* WHEN...Until 1 AM EST Saturday. + +* IMPACTS...Roads, and especially bridges and overpasses, will likely + become slick and hazardous. Difficult travel conditions are + possible. The hazardous conditions will impact the Friday evening + commute. + +PRECAUTIONARY/PREPAREDNESS ACTIONS... + +Slow down and use caution while traveling. Prepare for possible power +outages. The latest road conditions can be obtained by calling 5 1 1. + +Please report snow or ice accumulations by calling 412-262-1988, +posting to the NWS Pittsburgh Facebook page, or using X +@NWSPittsburgh. + +&& + +$$ + +TC \ No newline at end of file diff --git a/data/test/awips/winter weather/WW-Y-KPBZ-26-2025-5.txt b/data/test/awips/winter weather/WW-Y-KPBZ-26-2025-5.txt new file mode 100644 index 0000000..fe70e27 --- /dev/null +++ b/data/test/awips/winter weather/WW-Y-KPBZ-26-2025-5.txt @@ -0,0 +1,100 @@ +334 +WWUS41 KPBZ 262250 +WSWPBZ + +URGENT - WINTER WEATHER MESSAGE +National Weather Service Pittsburgh PA +550 PM EST Fri Dec 26 2025 + +PAZ013-076-WVZ512-514-270000- +/O.CAN.KPBZ.WW.Y.0026.000000T0000Z-251227T0600Z/ +Lawrence-Higher Elevations of Fayette-Eastern Preston-Eastern +Tucker- +Including the cities of Rowlesburg, Terra Alta, Champion, Davis, +Thomas, Ellwood City, Ohiopyle, Hazelton, Canaan Valley, and New +Castle +550 PM EST Fri Dec 26 2025 + +...WINTER WEATHER ADVISORY IS CANCELLED... + +Temperatures have risen above freezing and will remain so overnight. +Rain to continue. + +$$ + +PAZ008-009-015-016-022-074-077-078-270700- +/O.CON.KPBZ.IS.W.0004.000000T0000Z-251227T1200Z/ +Venango-Forest-Clarion-Jefferson PA-Armstrong-Higher Elevations +of Westmoreland-Indiana-Higher Elevations of Indiana- +Including the cities of Tionesta, Donegal, Franklin, Punxsutawney, +Oil City, Ford City, Brookville, Indiana, Armagh, Kittanning, +Ligonier, and Clarion +550 PM EST Fri Dec 26 2025 + +...ICE STORM WARNING REMAINS IN EFFECT UNTIL 7 AM EST SATURDAY... + +* WHAT...Significant icing expected. Total snow accumulations less + than one inch and ice accumulations between two and three tenths of + an inch. + +* WHERE...Portions of western Pennsylvania. + +* WHEN...Until 7 AM EST Saturday. + +* IMPACTS...Roads, and especially bridges and overpasses, will likely + become slick and hazardous. Power outages and tree damage are + likely due to the ice. Travel could be nearly impossible. The + hazardous conditions will impact the Friday post-holiday travel. + +PRECAUTIONARY/PREPAREDNESS ACTIONS... + +Persons should delay all travel if possible. If travel is absolutely +necessary, drive with extreme caution. Leave plenty of room between +you and the motorist ahead of you, and allow extra time to reach your +destination. Avoid sudden braking or acceleration, and be especially +cautious on hills or when making turns. Make sure your car is +winterized and in good working order. + +Please report ice accumulations or damage by calling 412-262-1988, +posting to the NWS Pittsburgh Facebook page, or using X +@NWSPittsburgh. + +&& + +$$ + +PAZ007-014-270600- +/O.CON.KPBZ.WW.Y.0026.000000T0000Z-251227T0600Z/ +Mercer-Butler- +Including the cities of Grove City, Sharon, Butler, and Hermitage +550 PM EST Fri Dec 26 2025 + +...WINTER WEATHER ADVISORY REMAINS IN EFFECT UNTIL 1 AM EST +SATURDAY... + +* WHAT...Freezing rain expected. Total ice accumulation up to a tenth + of an inch. + +* WHERE...Portions of western Pennsylvania and northern West + Virginia. + +* WHEN...Until 1 AM EST Saturday. + +* IMPACTS...Roads, and especially bridges and overpasses, will likely + become slick and hazardous. Plan on slippery road conditions. The + hazardous conditions will impact the Friday post-holiday travel. + +PRECAUTIONARY/PREPAREDNESS ACTIONS... + +Slow down and use caution while traveling. The latest road conditions +can be obtained by calling 5 1 1. + +Please report snow or ice accumulations by calling 412-262-1988, +posting to the NWS Pittsburgh Facebook page, or using X +@NWSPittsburgh. + +&& + +$$ + +AK \ No newline at end of file diff --git a/data/test/awips/winter weather/WW-Y-KPBZ-26-2025-6.txt b/data/test/awips/winter weather/WW-Y-KPBZ-26-2025-6.txt new file mode 100644 index 0000000..da812a1 --- /dev/null +++ b/data/test/awips/winter weather/WW-Y-KPBZ-26-2025-6.txt @@ -0,0 +1,73 @@ +034 +WWUS41 KPBZ 270212 +WSWPBZ + +URGENT - WINTER WEATHER MESSAGE +National Weather Service Pittsburgh PA +912 PM EST Fri Dec 26 2025 + +PAZ074-270315- +/O.CAN.KPBZ.IS.W.0004.000000T0000Z-251227T1200Z/ +Higher Elevations of Westmoreland- +Including the cities of Ligonier and Donegal +912 PM EST Fri Dec 26 2025 + +...ICE STORM WARNING IS CANCELLED... + +Temperatures have risen above freezing and are expected to remain +there overnight. The threat for freezing rain has ended. + +$$ + +PAZ007-014-270315- +/O.CAN.KPBZ.WW.Y.0026.000000T0000Z-251227T0600Z/ +Mercer-Butler- +Including the cities of Grove City, Sharon, Butler, and Hermitage +912 PM EST Fri Dec 26 2025 + +...WINTER WEATHER ADVISORY IS CANCELLED... + +Temperatures have risen above freezing and are expected to remain +there overnight. The threat for freezing rain has ended. + +$$ + +PAZ008-009-015-016-022-077-078-271200- +/O.CON.KPBZ.IS.W.0004.000000T0000Z-251227T1200Z/ +Venango-Forest-Clarion-Jefferson PA-Armstrong-Indiana-Higher +Elevations of Indiana- +Including the cities of Tionesta, Franklin, Punxsutawney, Oil City, +Ford City, Brookville, Indiana, Armagh, Kittanning, and Clarion +912 PM EST Fri Dec 26 2025 + +...ICE STORM WARNING REMAINS IN EFFECT UNTIL 7 AM EST SATURDAY... + +* WHAT...Freezing drizzle to continue. Additional ice accumulations + of a light glaze to a few hundredths of an inch. + +* WHERE...Portions of western Pennsylvania. + +* WHEN...Until 7 AM EST Saturday. + +* IMPACTS...Roads, and especially bridges and overpasses, will likely + become slick and hazardous. Power outages and tree damage are + likely due to the ice. Travel could be nearly impossible. + +PRECAUTIONARY/PREPAREDNESS ACTIONS... + +Persons should delay all travel if possible. If travel is absolutely +necessary, drive with extreme caution. Leave plenty of room between +you and the motorist ahead of you, and allow extra time to reach your +destination. Avoid sudden braking or acceleration, and be especially +cautious on hills or when making turns. Make sure your car is +winterized and in good working order. + +Please report ice accumulations or damage by calling 412-262-1988, +posting to the NWS Pittsburgh Facebook page, or using X +@NWSPittsburgh. + +&& + +$$ + +AK diff --git a/database/schemas/vtec.sql b/database/schemas/vtec.sql index 63a93e3..99a2fa0 100644 --- a/database/schemas/vtec.sql +++ b/database/schemas/vtec.sql @@ -4,8 +4,7 @@ ALTER SCHEMA vtec OWNER TO mds; -- Phenomena types CREATE TABLE IF NOT EXISTS vtec.phenomena ( id char(2) PRIMARY KEY, - name varchar(64) NOT NULL, - description varchar(64) + name varchar(64) NOT NULL ); ALTER TABLE vtec.phenomena OWNER TO mds; GRANT SELECT ON TABLE vtec.phenomena TO awips_service; @@ -14,8 +13,7 @@ GRANT SELECT ON TABLE vtec.phenomena TO nobody, api_service; -- Significance levels CREATE TABLE IF NOT EXISTS vtec.significance ( id char(1) PRIMARY KEY, - name varchar(64) NOT NULL, - description varchar(64) + name varchar(64) NOT NULL ); ALTER TABLE vtec.significance OWNER TO mds; GRANT SELECT ON TABLE vtec.significance TO awips_service; @@ -24,47 +22,52 @@ GRANT SELECT ON TABLE vtec.significance TO nobody, api_service; -- Action types CREATE TABLE IF NOT EXISTS vtec.action ( id char(3) PRIMARY KEY, - name varchar(64) NOT NULL, - description varchar(64) + name varchar(64) NOT NULL ); ALTER TABLE vtec.action OWNER TO mds; GRANT SELECT ON TABLE vtec.action TO awips_service; GRANT SELECT ON TABLE vtec.action TO nobody, api_service; --- VTEC Event -- +-- VTEC Events CREATE TABLE IF NOT EXISTS vtec.events ( - id serial, - created_at timestamptz DEFAULT CURRENT_TIMESTAMP, - updated_at timestamptz DEFAULT CURRENT_TIMESTAMP, - issued timestamptz NOT NULL, - starts timestamptz, - expires timestamptz NOT NULL, - ends timestamptz DEFAULT NULL, - end_initial timestamptz DEFAULT NULL, - class char(1) NOT NULL, + -- Composite key phenomena char(2) NOT NULL REFERENCES vtec.phenomena(id), - wfo char(4) NOT NULL REFERENCES postgis.offices(icao), significance char(1) NOT NULL REFERENCES vtec.significance(id), + wfo char(4) NOT NULL REFERENCES postgis.offices(icao), event_number smallint NOT NULL, year smallint NOT NULL, + + -- Stuff + class char(1) NOT NULL, title varchar(128) NOT NULL, is_emergency boolean DEFAULT false, is_pds boolean DEFAULT false, - PRIMARY KEY (wfo, phenomena, significance, event_number, year) + + -- State + created_at timestamptz DEFAULT CURRENT_TIMESTAMP, + updated_at timestamptz DEFAULT CURRENT_TIMESTAMP, + issued timestamptz NOT NULL, + starts timestamptz, + expires timestamptz NOT NULL, + ends timestamptz DEFAULT NULL, + ends_initial timestamptz DEFAULT NULL, + + PRIMARY KEY (phenomena, significance, wfo, event_number, year) ) PARTITION BY LIST (year); ALTER TABLE vtec.events OWNER TO mds; GRANT ALL ON TABLE vtec.events TO awips_service; GRANT SELECT ON TABLE vtec.events TO nobody, api_service; --- VTEC UGC -- +-- VTEC UGC CREATE TABLE IF NOT EXISTS vtec.ugcs ( id serial, created_at timestamptz DEFAULT CURRENT_TIMESTAMP, updated_at timestamptz DEFAULT CURRENT_TIMESTAMP, - wfo char(4) NOT NULL, phenomena char(2) NOT NULL, significance char(1) NOT NULL, + wfo char(4) NOT NULL, event_number smallint NOT NULL, + year smallint NOT NULL, ugc integer NOT NULL REFERENCES postgis.ugcs(id), issued timestamptz NOT NULL, starts timestamptz DEFAULT NULL, @@ -72,35 +75,28 @@ CREATE TABLE IF NOT EXISTS vtec.ugcs ( ends timestamptz DEFAULT NULL, end_initial timestamptz DEFAULT NULL, action char(3) NOT NULL REFERENCES vtec.action(id), - year smallint NOT NULL, - FOREIGN KEY (wfo, phenomena, significance, event_number, year) - REFERENCES vtec.events(wfo, phenomena, significance, event_number, year) ON DELETE CASCADE, - PRIMARY KEY (wfo, phenomena, significance, event_number, year, ugc) + FOREIGN KEY (phenomena, significance, wfo, event_number, year) + REFERENCES vtec.events(phenomena, significance, wfo, event_number, year) ON DELETE CASCADE, + PRIMARY KEY (phenomena, significance, wfo, event_number, year, ugc) ) PARTITION BY LIST (year); ALTER TABLE vtec.ugcs OWNER TO mds; GRANT ALL ON TABLE vtec.ugcs TO awips_service; GRANT SELECT ON TABLE vtec.ugcs TO nobody, api_service; --- VTEC Event Updates -- +-- VTEC Updates CREATE TABLE IF NOT EXISTS vtec.updates ( - id serial, - created_at timestamptz DEFAULT CURRENT_TIMESTAMP, - issued timestamptz NOT NULL, - starts timestamptz DEFAULT NULL, - expires timestamptz NOT NULL, - ends timestamptz DEFAULT NULL, - text text NOT NULL, - product varchar(38) NOT NULL, - wfo char(4) NOT NULL, - action char(3) NOT NULL, - class char(1) NOT NULL, - phenomena char(2) NOT NULL, - significance char(1) NOT NULL, + id bigserial, + + -- Composite key + phenomena char(2) NOT NULL REFERENCES vtec.phenomena(id), + significance char(1) NOT NULL REFERENCES vtec.significance(id), + wfo char(4) NOT NULL REFERENCES postgis.offices(icao), event_number smallint NOT NULL, year smallint NOT NULL, - title varchar(128) NOT NULL, - is_emergency boolean DEFAULT false, - is_pds boolean DEFAULT false, + + class char(1) NOT NULL, + + -- Geospatial geom geometry(Polygon, 4326), direction int, location geometry(MultiPoint, 4326), @@ -108,6 +104,19 @@ CREATE TABLE IF NOT EXISTS vtec.updates ( speed_text varchar(30), tml_time timestamptz, ugc char(6)[], + + -- State + action char(3) NOT NULL REFERENCES vtec.action(id), + created_at timestamptz DEFAULT CURRENT_TIMESTAMP, + issued timestamptz NOT NULL, + starts timestamptz, + expires timestamptz NOT NULL, + ends timestamptz DEFAULT NULL, + title varchar(128) NOT NULL, + is_emergency boolean DEFAULT false, + is_pds boolean DEFAULT false, + + -- Event tags tornado varchar(64), damage varchar(64), hail_threat varchar(64), @@ -120,10 +129,15 @@ CREATE TABLE IF NOT EXISTS vtec.updates ( spout_tag varchar(64), snow_squall varchar(64), snow_squall_tag varchar(64), - PRIMARY KEY (wfo, phenomena, significance, event_number, year, id), - CONSTRAINT fk_vtec_event + + -- Porduct data + text text NOT NULL, + product varchar(38) NOT NULL, + + PRIMARY KEY (year, id), -- year first for partition pruning FOREIGN KEY (wfo, phenomena, significance, event_number, year) - REFERENCES vtec.events(wfo, phenomena, significance, event_number, year) ON DELETE CASCADE + REFERENCES vtec.events(wfo, phenomena, significance, event_number, year) + ON DELETE CASCADE ) PARTITION BY LIST (year); ALTER TABLE vtec.updates OWNER TO mds; GRANT ALL ON TABLE vtec.updates TO awips_service; diff --git a/database/schemas/warnings.sql b/database/schemas/warnings.sql index 93638f9..d9f1a39 100644 --- a/database/schemas/warnings.sql +++ b/database/schemas/warnings.sql @@ -2,25 +2,35 @@ CREATE SCHEMA IF NOT EXISTS warnings; ALTER SCHEMA warnings OWNER TO mds; CREATE TABLE IF NOT EXISTS warnings.warnings ( - id serial, + id bigserial, + + -- Combined key + phenomena char(2) NOT NULL, + significance char(1) NOT NULL, + wfo char(4) NOT NULL, + event_number smallint NOT NULL, + year smallint NOT NULL, + + -- State + action char(3) NOT NULL, + current boolean DEFAULT true, created_at timestamptz DEFAULT CURRENT_TIMESTAMP, updated_at timestamptz DEFAULT CURRENT_TIMESTAMP, issued timestamptz NOT NULL, starts timestamptz DEFAULT NULL, expires timestamptz NOT NULL, + expires_initial timestamptz DEFAULT NULL, ends timestamptz DEFAULT NULL, - end_initial timestamptz DEFAULT NULL, - text text NOT NULL, - wfo char(4) NOT NULL, - action char(3) NOT NULL, class char(1) NOT NULL, - phenomena char(2) NOT NULL, - significance char(1) NOT NULL, - event_number smallint NOT NULL, - year smallint NOT NULL, title varchar(128) NOT NULL, is_emergency boolean DEFAULT false, is_pds boolean DEFAULT false, + + -- Product data + text text NOT NULL, + product varchar(38) NOT NULL, + + -- Geospatial geom geometry(MultiPolygon, 4326), direction int, location geometry(MultiPoint, 4326), @@ -28,6 +38,8 @@ CREATE TABLE IF NOT EXISTS warnings.warnings ( speed_text varchar(30), tml_time timestamptz, ugc char(6)[], + + -- Tags tornado varchar(64), damage varchar(64), hail_threat varchar(64), @@ -40,6 +52,7 @@ CREATE TABLE IF NOT EXISTS warnings.warnings ( spout_tag varchar(64), snow_squall varchar(64), snow_squall_tag varchar(64), + PRIMARY KEY (wfo, phenomena, significance, event_number, year, id) ); CREATE INDEX IF NOT EXISTS warnings_issued ON warnings.warnings(issued); diff --git a/go.work.sum b/go.work.sum index 85d1573..c96a599 100644 --- a/go.work.sum +++ b/go.work.sum @@ -35,11 +35,11 @@ github.com/goccy/go-yaml v1.17.1/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7Lk github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= @@ -51,9 +51,7 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY= github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lufia/plan9stats v0.0.0-20240226150601-1dcf7310316a/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= @@ -81,7 +79,6 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgc github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/shirou/gopsutil/v3 v3.24.2/go.mod h1:tSg/594BcA+8UdQU2XcW803GWYgdtauFFPgJCJKZlVk= @@ -143,6 +140,7 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= diff --git a/pkg/awips/product.go b/pkg/awips/product.go index dfe609a..49e3299 100644 --- a/pkg/awips/product.go +++ b/pkg/awips/product.go @@ -227,6 +227,20 @@ func (product *Product) FindBroadcastInstructions() string { return bilRegexp.FindString(product.Text) } +func (product *Product) GetVTECs() map[string]VTEC { + vtecs := map[string]VTEC{} + + for _, segment := range product.Segments { + for _, vtec := range segment.VTEC { + if _, ok := vtecs[vtec.Original]; !ok { + vtecs[vtec.Original] = vtec + } + } + } + + return vtecs +} + func (segment *ProductSegment) HasVTEC() bool { return len(segment.VTEC) != 0 } diff --git a/pkg/db/ugc.go b/pkg/db/ugc.go index b13584a..1b3eded 100644 --- a/pkg/db/ugc.go +++ b/pkg/db/ugc.go @@ -151,6 +151,30 @@ func GetUGCUnionGeomSimplified(db *pgxpool.Pool, ugcs []string) (*geos.Geom, err return geom, nil } +func GetUGCUnionGeomSimplifiedTx(tx pgx.Tx, ugcs []string) (*geos.Geom, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + rows, err := tx.Query(ctx, ` + SELECT ST_Simplify(ST_Union(geom), 0.0025) FROM postgis.ugcs WHERE valid_to IS NULL AND ugc = ANY($1) + `, ugcs) + if err != nil { + return nil, err + } + + defer rows.Close() + + var geom *geos.Geom + if rows.Next() { + geom = &geos.Geom{} + if err := rows.Scan(&geom); err != nil { + return nil, err + } + } + + return geom, nil +} + func ScanUGC(row pgx.Row, ugc *models.UGC) error { return row.Scan( &ugc.ID, diff --git a/pkg/db/vtec.go b/pkg/db/vtec.go index 132febd..14f4b96 100644 --- a/pkg/db/vtec.go +++ b/pkg/db/vtec.go @@ -37,26 +37,52 @@ func FindVTECEvent(db *pgxpool.Pool, wfo string, phenomena string, significance return event, nil } +func FindVTECEventTX(tx pgx.Tx, wfo string, phenomena string, significance string, eventNumber int, year int) (*models.VTECEvent, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Lets check if the VTEC Event is already in the database + rows, err := tx.Query(ctx, ` + SELECT * FROM vtec.events WHERE + wfo = $1 AND phenomena = $2 AND significance = $3 AND event_number = $4 AND year = $5 + `, wfo, phenomena, significance, eventNumber, year) + if err != nil { + return nil, err + } + + defer rows.Close() + + // Scan the row into the event struct + var event *models.VTECEvent + if rows.Next() { + event = &models.VTECEvent{} + if err := ScanVTECEvent(rows, event); err != nil { + return nil, err + } + } + + return event, nil +} + // Scan a row into the VTEC Event struct func ScanVTECEvent(rows pgx.Rows, event *models.VTECEvent) error { return rows.Scan( - &event.ID, - &event.CreatedAt, - &event.UpdatedAt, - &event.Issued, - &event.Starts, - &event.Expires, - &event.Ends, - &event.EndInitial, - &event.Class, &event.Phenomena, - &event.WFO, &event.Significance, + &event.WFO, &event.EventNumber, &event.Year, + &event.Class, &event.Title, &event.IsEmergency, &event.IsPDS, + &event.CreatedAt, + &event.UpdatedAt, + &event.Issued, + &event.Starts, + &event.Expires, + &event.Ends, + &event.EndInitial, ) } @@ -64,7 +90,7 @@ func ScanVTECEvent(rows pgx.Rows, event *models.VTECEvent) error { func InsertVTECEvent(db *pgxpool.Pool, event *models.VTECEvent) error { rows, err := db.Query(context.Background(), ` - INSERT INTO vtec.events(issued, starts, expires, ends, end_initial, class, phenomena, wfo, + INSERT INTO vtec.events(issued, starts, expires, ends, ends_initial, class, phenomena, wfo, significance, event_number, year, title, is_emergency, is_pds) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14); `, event.Issued, event.Starts, event.Expires, event.Ends, event.EndInitial, event.Class, @@ -74,7 +100,6 @@ func InsertVTECEvent(db *pgxpool.Pool, event *models.VTECEvent) error { if err != nil { return err } - defer rows.Close() // Scan the row into the event struct @@ -184,6 +209,30 @@ func FindCurrentVTECEventUGCs(db *pgxpool.Pool, wfo string, phenomena string, si return ugcs, nil } +func FindCurrentVTECEventUGCsTX(tx pgx.Tx, wfo string, phenomena string, significance string, eventNumber int, year int, expires time.Time) ([]*models.VTECUGC, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + rows, err := tx.Query(ctx, ` + SELECT * FROM vtec.ugcs WHERE wfo = $1 AND phenomena = $2 AND significance = $3 AND event_number = $4 AND year = $5 AND action NOT IN ('CAN', 'UPG') AND expires > $6 + `, wfo, phenomena, significance, eventNumber, year, expires) + if err != nil { + return nil, err + } + defer rows.Close() + + var ugcs []*models.VTECUGC + for rows.Next() { + ugc := &models.VTECUGC{} + if err := ScanVTECUGC(rows, ugc); err != nil { + return nil, err + } + ugcs = append(ugcs, ugc) + } + + return ugcs, nil +} + func ScanVTECUGC(rows pgx.Rows, ugc *models.VTECUGC) error { return rows.Scan( &ugc.ID, diff --git a/pkg/db/warning.go b/pkg/db/warning.go index 5489c71..2319c52 100644 --- a/pkg/db/warning.go +++ b/pkg/db/warning.go @@ -12,24 +12,26 @@ import ( func ScanWarning(row pgx.Row, warning *models.Warning) error { return row.Scan( &warning.ID, + &warning.Phenomena, + &warning.Significance, + &warning.WFO, + &warning.EventNumber, + &warning.Year, + &warning.Action, + &warning.Current, &warning.CreatedAt, &warning.UpdatedAt, &warning.Issued, &warning.Starts, &warning.Expires, + &warning.ExpiresInitial, &warning.Ends, - &warning.EndInitial, - &warning.Text, - &warning.WFO, - &warning.Action, &warning.Class, - &warning.Phenomena, - &warning.Significance, - &warning.EventNumber, - &warning.Year, &warning.Title, &warning.IsEmergency, &warning.IsPDS, + &warning.Text, + &warning.Product, &warning.Geom, &warning.Direction, &warning.Location, @@ -126,28 +128,53 @@ func FindWarning(db *pgxpool.Pool, wfo string, phenomena string, significance st return warning, nil } +func FindWarningTx(tx pgx.Tx, wfo string, phenomena string, significance string, eventNumber int, year int) (*models.Warning, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + rows, err := tx.Query(ctx, ` + SELECT * FROM warnings.warnings WHERE wfo = $1 AND phenomena = $2 AND significance = $3 AND event_number = $4 AND year = $5 + `, wfo, phenomena, significance, eventNumber, year) + if err != nil { + return nil, err + } + + defer rows.Close() + + var warning *models.Warning + if rows.Next() { + warning = &models.Warning{} + if err := ScanWarning(rows, warning); err != nil { + return nil, err + } + } + + return warning, nil +} + func InsertWarning(db *pgxpool.Pool, warning *models.Warning) error { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() rows, err := db.Query(ctx, ` INSERT INTO warnings.warnings( - issued, starts, expires, ends, end_initial, text, + issued, starts, expires, ends, expires_initial, text, product, wfo, action, class, phenomena, significance, event_number, year, title, is_emergency, is_pds, geom, direction, location, speed, speed_text, tml_time, ugc, tornado, damage, hail_threat, hail_tag, wind_threat, wind_tag, flash_flood, rainfall_tag, flood_tag_dam, spout_tag, snow_squall, snow_squall_tag ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, - $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35 + $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36 ) `, warning.Issued, warning.Starts, warning.Expires, warning.Ends, - warning.EndInitial, + warning.ExpiresInitial, warning.Text, + warning.Product, warning.WFO, warning.Action, warning.Class, diff --git a/pkg/models/warning.go b/pkg/models/warning.go index fa5603f..51e2dab 100644 --- a/pkg/models/warning.go +++ b/pkg/models/warning.go @@ -9,57 +9,63 @@ import ( ) type Warning struct { - ID int `json:"id"` - CreatedAt time.Time `json:"created_at,omitzero"` - UpdatedAt time.Time `json:"updated_at,omitzero"` - Issued time.Time `json:"issued"` - Starts *time.Time `json:"starts,omitzero"` - Expires time.Time `json:"expires"` - Ends time.Time `json:"ends,omitzero"` - EndInitial time.Time `json:"end_initial,omitzero"` - Text string `json:"text"` - WFO string `json:"wfo"` - Action string `json:"action"` - Class string `json:"class"` - Phenomena string `json:"phenomena"` - Significance string `json:"significance"` - EventNumber int `json:"event_number"` - Year int `json:"year"` - Title string `json:"title"` - IsEmergency bool `json:"is_emergency"` - IsPDS bool `json:"is_pds"` - Geom *geos.Geom `json:"geom,omitempty"` - Direction *int `json:"direction"` - Location *geos.Geom `json:"location,omitempty"` - Speed *int `json:"speed"` - SpeedText *string `json:"speed_text"` - TMLTime *time.Time `json:"tml_time"` - UGC []string `json:"ugc"` - Tornado string `json:"tornado,omitempty"` - Damage string `json:"damage,omitempty"` - HailThreat string `json:"hail_threat,omitempty"` - HailTag string `json:"hail_tag,omitempty"` - WindThreat string `json:"wind_threat,omitempty"` - WindTag string `json:"wind_tag,omitempty"` - FlashFlood string `json:"flash_flood,omitempty"` - RainfallTag string `json:"rainfall_tag,omitempty"` - FloodTagDam string `json:"flood_tag_dam,omitempty"` - SpoutTag string `json:"spout_tag,omitempty"` - SnowSquall string `json:"snow_squall,omitempty"` - SnowSquallTag string `json:"snow_squall_tag,omitempty"` + ID int `json:"id"` + Phenomena string `json:"phenomena"` + Significance string `json:"significance"` + WFO string `json:"wfo"` + EventNumber int `json:"event_number"` + Year int `json:"year"` + Action string `json:"action"` + Current bool `json:"current"` + CreatedAt time.Time `json:"created_at,omitzero"` + UpdatedAt time.Time `json:"updated_at,omitzero"` + Issued time.Time `json:"issued"` + Starts *time.Time `json:"starts,omitzero"` + Expires time.Time `json:"expires"` + ExpiresInitial time.Time `json:"expires_initial,omitzero"` + Ends time.Time `json:"ends,omitzero"` + Class string `json:"class"` + Title string `json:"title"` + IsEmergency bool `json:"is_emergency"` + IsPDS bool `json:"is_pds"` + Text string `json:"text"` + Product string `json:"product"` + Geom *geos.Geom `json:"geom"` + Direction *int `json:"direction"` + Location *geos.Geom `json:"location"` + Speed *int `json:"speed"` + SpeedText *string `json:"speed_text"` + TMLTime *time.Time `json:"tml_time"` + UGC []string `json:"ugc"` + Tornado string `json:"tornado,omitempty"` + Damage string `json:"damage,omitempty"` + HailThreat string `json:"hail_threat,omitempty"` + HailTag string `json:"hail_tag,omitempty"` + WindThreat string `json:"wind_threat,omitempty"` + WindTag string `json:"wind_tag,omitempty"` + FlashFlood string `json:"flash_flood,omitempty"` + RainfallTag string `json:"rainfall_tag,omitempty"` + FloodTagDam string `json:"flood_tag_dam,omitempty"` + SpoutTag string `json:"spout_tag,omitempty"` + SnowSquall string `json:"snow_squall,omitempty"` + SnowSquallTag string `json:"snow_squall_tag,omitempty"` } func (warning *Warning) GenerateID() string { return fmt.Sprintf("%v-%v-%v-%04v-%v", warning.WFO, warning.Phenomena, warning.Significance, warning.EventNumber, warning.Year) } +func (warning *Warning) GenerateCompositeID() string { + return fmt.Sprintf("%s-%v", warning.GenerateID(), warning.ID) +} + func (w *Warning) MarshalJSON() ([]byte, error) { type Alias Warning // Use type alias to avoid recursion aux := struct { *Alias - Geom []byte `json:"geom,omitempty"` - Location []byte `json:"location,omitempty"` + Geom []byte `json:"geom"` + Location []byte `json:"location"` }{ Alias: (*Alias)(w), } @@ -80,8 +86,8 @@ func (w *Warning) UnmarshalJSON(data []byte) error { aux := struct { *Alias - Geom []byte `json:"geom,omitempty"` - Location []byte `json:"location,omitempty"` + Geom []byte `json:"geom"` + Location []byte `json:"location"` }{ Alias: (*Alias)(w), } diff --git a/scripts/shapefiler/counties.go b/scripts/shapefiler/counties.go index ed9ac5c..b6f168c 100644 --- a/scripts/shapefiler/counties.go +++ b/scripts/shapefiler/counties.go @@ -7,7 +7,7 @@ import ( "time" "github.com/everystreet/go-shapefile" - orbjson "github.com/paulmach/orb/geojson" + "github.com/twpayne/go-geos" ) func ParseCounties(scanner *shapefile.ZipScanner, t time.Time) error { @@ -25,7 +25,7 @@ func ParseCounties(scanner *shapefile.ZipScanner, t time.Time) error { return err } - ugcRecords := make([]UGC, info.NumRecords) + ugcRecords := make(map[string]UGC, info.NumRecords) count := 0 // Call Record() to get each record in turn, until either the end of the file, or an error occurs @@ -68,7 +68,7 @@ func ParseCounties(scanner *shapefile.ZipScanner, t time.Time) error { return err } - centre := [2]float64{lon, lat} + centre := geos.NewPoint([]float64{lon, lat}) cwaAttr, _ := record.Attributes.Field("CWA") cwa := fmt.Sprintf("%v", cwaAttr.Value()) @@ -86,7 +86,7 @@ func ParseCounties(scanner *shapefile.ZipScanner, t time.Time) error { Type: "C", Area: 0.0, Centre: centre, - Geometry: *mpolygon, + Geometry: mpolygon, CWA: cwaarr, IsMarine: false, IsFire: false, @@ -94,7 +94,7 @@ func ParseCounties(scanner *shapefile.ZipScanner, t time.Time) error { ValidTo: nil, } - ugcRecords[count] = ugc + ugcRecords[ugc.ID] = ugc count++ } @@ -105,43 +105,14 @@ func ParseCounties(scanner *shapefile.ZipScanner, t time.Time) error { return err } - out, err := ToSQL(ugcRecords) + records, err := ToCSV(ugcRecords) if err != nil { return err } - err = WriteToFile("counties.sql", []byte(out)) - if err != nil { - return err - } - - slog.Info(fmt.Sprintf("Wrote %d records to counties.sql\n", len(ugcRecords))) - - collection := orbjson.NewFeatureCollection() - - for _, ugc := range ugcRecords { - feature := orbjson.NewFeature(ugc.Geometry) - feature.Properties = map[string]interface{}{ - "id": ugc.ID, - "name": ugc.Name, - "state": ugc.State, - "type": ugc.Type, - "number": ugc.Number, - "is_marine": ugc.IsMarine, - "is_fire": ugc.IsFire, - "cwa": ugc.CWA, - } - collection.Append(feature) - } - - data, err := collection.MarshalJSON() - if err != nil { - return err - } - - WriteToFile("counties.geojson", data) + err = WriteToCSV("counties", records) - slog.Info(fmt.Sprintf("Wrote %d records to counties.geojson\n", len(ugcRecords))) + slog.Info(fmt.Sprintf("Wrote %d records to counties.csv\n", len(ugcRecords))) return err } diff --git a/scripts/shapefiler/cwa.go b/scripts/shapefiler/cwa.go index 7441f3f..1bc3ada 100644 --- a/scripts/shapefiler/cwa.go +++ b/scripts/shapefiler/cwa.go @@ -6,19 +6,18 @@ import ( "time" "github.com/everystreet/go-shapefile" - "github.com/paulmach/orb" - orbjson "github.com/paulmach/orb/geojson" + "github.com/twpayne/go-geos" ) type CWA struct { - ID string `json:"id"` - Name string `json:"name"` - Centre orb.Point `json:"centre"` - Geometry orb.Geometry `json:"geometry"` - Area float64 `json:"area"` - WFO string `json:"wfo"` - Region string `json:"region"` - ValidFrom time.Time `json:"valid_from"` + ID string `json:"id"` + Name string `json:"name"` + Centre *geos.Geom `json:"centre"` + Geometry *geos.Geom `json:"geometry"` + Area float64 `json:"area"` + WFO string `json:"wfo"` + Region string `json:"region"` + ValidFrom time.Time `json:"valid_from"` } func ParseCWA(scanner *shapefile.ZipScanner, t time.Time) error { @@ -67,7 +66,7 @@ func ParseCWA(scanner *shapefile.ZipScanner, t time.Time) error { return err } - location := [2]float64{lon, lat} + location := geos.NewPoint([]float64{lon, lat}) regionAttr, _ := record.Attributes.Field("REGION") region := fmt.Sprintf("%v", regionAttr.Value()) @@ -76,7 +75,7 @@ func ParseCWA(scanner *shapefile.ZipScanner, t time.Time) error { ID: id, Name: name, Centre: location, - Geometry: *mpolygon, + Geometry: mpolygon, Area: 0.0, WFO: id, Region: region, @@ -95,48 +94,32 @@ func ParseCWA(scanner *shapefile.ZipScanner, t time.Time) error { return err } - result := "INSERT INTO postgis.cwas(id, name, area, geom, wfo, region, valid_from) VALUES\n" + records := [][]string{} - for _, cwa := range cwaRecords { - geometry, err := orbjson.NewGeometry(cwa.Geometry).MarshalJSON() - if err != nil { - return err - } - - result += fmt.Sprintf("('%s', '%s', ST_Area(ST_GeomFromGeoJSON('%s')), ST_GeomFromGeoJSON('%s'), '%s', '%s', %v),\n", - cwa.ID, cwa.Name, geometry, geometry, cwa.WFO, cwa.Region, DateToString(&cwa.ValidFrom)) - } - - result = result[:len(result)-2] - - err = WriteToFile("cwa.sql", []byte(result)) - if err != nil { - return err - } - - slog.Info(fmt.Sprintf("Wrote %d records to cwa.sql\n", len(cwaRecords))) - - collection := orbjson.NewFeatureCollection() + header := []string{"id", "name", "area", "geom", "wfo", "region", "valid_from"} + records = append(records, header) for _, cwa := range cwaRecords { - feature := orbjson.NewFeature(cwa.Geometry) - feature.Properties = map[string]interface{}{ - "id": cwa.ID, - "name": cwa.Name, - "wfo": cwa.WFO, - "region": cwa.Region, + geometry := cwa.Geometry.ToWKT() + + record := []string{ + cwa.ID, + cwa.Name, + fmt.Sprintf("%f", cwa.Area), + geometry, + cwa.WFO, + cwa.Region, + DateToString(&cwa.ValidFrom), } - collection.Append(feature) + records = append(records, record) } - data, err := collection.MarshalJSON() + err = WriteToCSV("cwa", records) if err != nil { return err } - WriteToFile("cwa.geojson", data) - - slog.Info(fmt.Sprintf("Wrote %d records to cwa.geojson\n", len(cwaRecords))) + slog.Info(fmt.Sprintf("Wrote %d records to cwa.csv\n", len(cwaRecords))) - return err + return nil } diff --git a/scripts/shapefiler/fire.go b/scripts/shapefiler/fire.go index b6f3dda..2797fda 100644 --- a/scripts/shapefiler/fire.go +++ b/scripts/shapefiler/fire.go @@ -7,7 +7,7 @@ import ( "time" "github.com/everystreet/go-shapefile" - orbjson "github.com/paulmach/orb/geojson" + "github.com/twpayne/go-geos" ) func ParseFire(scanner *shapefile.ZipScanner, t time.Time) error { @@ -24,7 +24,7 @@ func ParseFire(scanner *shapefile.ZipScanner, t time.Time) error { return err } - ugcRecords := make([]UGC, info.NumRecords) + ugcRecords := make(map[string]UGC, info.NumRecords) count := 0 for { @@ -63,7 +63,7 @@ func ParseFire(scanner *shapefile.ZipScanner, t time.Time) error { return err } - centre := [2]float64{lon, lat} + centre := geos.NewPoint([]float64{lon, lat}) cwaAttr, _ := record.Attributes.Field("CWA") cwa := fmt.Sprintf("%v", cwaAttr.Value()) @@ -81,7 +81,7 @@ func ParseFire(scanner *shapefile.ZipScanner, t time.Time) error { Type: "Z", Area: 0.0, Centre: centre, - Geometry: *mpolygon, + Geometry: mpolygon, CWA: cwaArr, IsMarine: false, IsFire: true, @@ -89,7 +89,7 @@ func ParseFire(scanner *shapefile.ZipScanner, t time.Time) error { ValidTo: nil, } - ugcRecords[count] = ugc + ugcRecords[ugc.ID] = ugc count++ @@ -101,43 +101,17 @@ func ParseFire(scanner *shapefile.ZipScanner, t time.Time) error { return err } - out, err := ToSQL(ugcRecords) + records, err := ToCSV(ugcRecords) if err != nil { return err } - err = WriteToFile("firezones.sql", []byte(out)) + err = WriteToCSV("firezones", records) if err != nil { return err } - slog.Info(fmt.Sprintf("Wrote %d records to firezones.sql\n", len(ugcRecords))) - - collection := orbjson.NewFeatureCollection() - - for _, ugc := range ugcRecords { - feature := orbjson.NewFeature(ugc.Geometry) - feature.Properties = map[string]interface{}{ - "id": ugc.ID, - "name": ugc.Name, - "state": ugc.State, - "type": ugc.Type, - "number": ugc.Number, - "is_marine": ugc.IsMarine, - "is_fire": ugc.IsFire, - "cwa": ugc.CWA, - } - collection.Append(feature) - } - - data, err := collection.MarshalJSON() - if err != nil { - return err - } - - WriteToFile("firezones.geojson", data) - - slog.Info(fmt.Sprintf("Wrote %d records to firezones.geojson\n", len(ugcRecords))) + slog.Info(fmt.Sprintf("Wrote %d records to firezones.csv\n", len(ugcRecords))) return nil } diff --git a/scripts/shapefiler/go.mod b/scripts/shapefiler/go.mod index 3504a35..c69362a 100644 --- a/scripts/shapefiler/go.mod +++ b/scripts/shapefiler/go.mod @@ -7,17 +7,15 @@ toolchain go1.24.7 require ( github.com/everystreet/go-geojson/v2 v2.0.2 github.com/everystreet/go-shapefile v1.0.0 - github.com/paulmach/orb v0.11.1 + github.com/twpayne/go-geos v0.20.1 ) require ( github.com/golang/geo v0.0.0-20230421003525-6adc56603217 // indirect - github.com/google/go-cmp v0.7.0 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/stretchr/testify v1.11.1 // indirect - go.mongodb.org/mongo-driver v1.17.1 // indirect golang.org/x/net v0.44.0 // indirect golang.org/x/text v0.29.0 // indirect ) diff --git a/scripts/shapefiler/go.sum b/scripts/shapefiler/go.sum index 042f153..fc376b6 100644 --- a/scripts/shapefiler/go.sum +++ b/scripts/shapefiler/go.sum @@ -1,5 +1,7 @@ github.com/ajstarks/svgo v0.0.0-20210406150507-75cfd577ce75/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= github.com/alecthomas/kong v0.2.16/go.mod h1:kQOmtJgV+Lb4aj+I2LEn40cbtawdWJ9Y8QLq+lElKxE= +github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -11,19 +13,11 @@ github.com/everystreet/go-geojson/v2 v2.0.2 h1:Cr9yZ5qyh94s5m6Y+CXYQ8ao4tl8RykwG github.com/everystreet/go-geojson/v2 v2.0.2/go.mod h1:2DOMFHMJNGpgMCX87c2DJjVAlav0UMvcgx6t1DYA6kA= github.com/everystreet/go-shapefile v1.0.0 h1:RHXJ0Yhj2GD257zS17gjn1q5Ud1atOjDMK6m1h5qBpU= github.com/everystreet/go-shapefile v1.0.0/go.mod h1:bnFakeByxvu+JTjGvMLo0xCI+XLou3DxcYqbCat1atQ= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/geo v0.0.0-20230421003525-6adc56603217 h1:HKlyj6in2JV6wVkmQ4XmG/EIm+SCYlPZ+V4GWit7Z+I= github.com/golang/geo v0.0.0-20230421003525-6adc56603217/go.mod h1:8wI0hitZ3a1IxZfeH3/5I97CI8i5cLGsYe7xNhQGs9U= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -34,12 +28,8 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= -github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= -github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -54,59 +44,17 @@ github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= -github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= -go.mongodb.org/mongo-driver v1.17.1 h1:Wic5cJIwJgSpBhe3lx3+/RybR5PiYRMpVFgO7cOHyIM= -go.mongodb.org/mongo-driver v1.17.1/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +github.com/twpayne/go-geos v0.20.1 h1:Z1Itw0ms7bQGkXh7nLo+qrrv5Op3jBJoWmyZ2PMxxr8= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/scripts/shapefiler/marinezones.go b/scripts/shapefiler/marinezones.go index c427594..cea0e45 100644 --- a/scripts/shapefiler/marinezones.go +++ b/scripts/shapefiler/marinezones.go @@ -6,7 +6,7 @@ import ( "time" "github.com/everystreet/go-shapefile" - orbjson "github.com/paulmach/orb/geojson" + "github.com/twpayne/go-geos" ) func ParseMarineZones(scanner *shapefile.ZipScanner, t time.Time) error { @@ -24,7 +24,7 @@ func ParseMarineZones(scanner *shapefile.ZipScanner, t time.Time) error { return err } - ugcRecords := make([]UGC, info.NumRecords) + ugcRecords := make(map[string]UGC, info.NumRecords) count := 0 for { @@ -59,7 +59,7 @@ func ParseMarineZones(scanner *shapefile.ZipScanner, t time.Time) error { return err } - centre := [2]float64{lon, lat} + centre := geos.NewPoint([]float64{lon, lat}) cwaAttr, _ := record.Attributes.Field("WFO") cwa := fmt.Sprintf("%v", cwaAttr.Value()) @@ -77,7 +77,7 @@ func ParseMarineZones(scanner *shapefile.ZipScanner, t time.Time) error { Type: "Z", Area: 0.0, // Will be calculated in DB Centre: centre, - Geometry: *mpolygon, + Geometry: mpolygon, CWA: cwaArr, IsMarine: true, IsFire: false, @@ -85,7 +85,7 @@ func ParseMarineZones(scanner *shapefile.ZipScanner, t time.Time) error { ValidTo: nil, } - ugcRecords[count] = ugc + ugcRecords[ugc.ID] = ugc count++ @@ -97,43 +97,17 @@ func ParseMarineZones(scanner *shapefile.ZipScanner, t time.Time) error { return err } - out, err := ToSQL(ugcRecords) + records, err := ToCSV(ugcRecords) if err != nil { return err } - err = WriteToFile("marinezones.sql", []byte(out)) + err = WriteToCSV("marinezones", records) if err != nil { return err } - slog.Info(fmt.Sprintf("Wrote %d records to marinezones.sql\n", len(ugcRecords))) - - collection := orbjson.NewFeatureCollection() - - for _, ugc := range ugcRecords { - feature := orbjson.NewFeature(ugc.Geometry) - feature.Properties = map[string]interface{}{ - "id": ugc.ID, - "name": ugc.Name, - "state": ugc.State, - "type": ugc.Type, - "number": ugc.Number, - "is_marine": ugc.IsMarine, - "is_fire": ugc.IsFire, - "cwa": ugc.CWA, - } - collection.Append(feature) - } - - data, err := collection.MarshalJSON() - if err != nil { - return err - } - - WriteToFile("marinezones.geojson", data) - - slog.Info(fmt.Sprintf("Wrote %d records to marinezones.geojson\n", len(ugcRecords))) + slog.Info(fmt.Sprintf("Wrote %d records to marinezones.csv\n", len(ugcRecords))) return err } diff --git a/scripts/shapefiler/ugc.go b/scripts/shapefiler/ugc.go index 3c04b26..1c17355 100644 --- a/scripts/shapefiler/ugc.go +++ b/scripts/shapefiler/ugc.go @@ -5,35 +5,67 @@ import ( "strings" "time" - "github.com/paulmach/orb" - orbjson "github.com/paulmach/orb/geojson" + "github.com/twpayne/go-geos" ) type UGC struct { - ID string `json:"id"` - Name string `json:"name"` - State string `json:"state"` - Number string `json:"number"` - Type string `json:"type"` - Area float64 `json:"area"` - Centre orb.Point `json:"centre"` - Geometry orb.Geometry `json:"geometry"` - CWA []string `json:"cwa"` - IsMarine bool `json:"is_marine"` - IsFire bool `json:"is_fire"` - ValidFrom time.Time `json:"valid_from"` - ValidTo *time.Time `json:"valid_to"` + ID string `json:"id"` + Name string `json:"name"` + State string `json:"state"` + Number string `json:"number"` + Type string `json:"type"` + Area float64 `json:"area"` + Centre *geos.Geom `json:"centre"` + Geometry *geos.Geom `json:"geometry"` + CWA []string `json:"cwa"` + IsMarine bool `json:"is_marine"` + IsFire bool `json:"is_fire"` + ValidFrom time.Time `json:"valid_from"` + ValidTo *time.Time `json:"valid_to"` } -func ToSQL(ugcs []UGC) (string, error) { - result := "INSERT INTO postgis.ugcs(ugc, name, state, type, number, area, geom, cwa, is_marine, is_fire, valid_from) VALUES \n" +func ToCSV(ugcs map[string]UGC) ([][]string, error) { + + records := [][]string{} + + header := []string{"ugc", "name", "state", "type", "number", "area", "geom", "cwa", "is_marine", "is_fire", "valid_from"} + records = append(records, header) for _, ugc := range ugcs { + cwa := "{" + for i, c := range ugc.CWA { + cwa += fmt.Sprintf("\"%s\"", c) + if i < len(ugc.CWA)-1 { + cwa += "," + } + } + cwa += "}" - geometry, err := orbjson.NewGeometry(ugc.Geometry).MarshalJSON() - if err != nil { - return "", err + record := []string{ + ugc.ID, + ugc.Name, + ugc.State, + ugc.Type, + ugc.Number, + "0.0", + ugc.Geometry.ToWKT(), + cwa, + fmt.Sprintf("%v", ugc.IsMarine), + fmt.Sprintf("%v", ugc.IsFire), + DateToString(&ugc.ValidFrom), } + records = append(records, record) + } + + return records, nil +} + +func ToSQL(ugcs map[string]UGC) (string, error) { + result := "INSERT INTO postgis.ugcs(ugc, name, state, type, number, area, geom, cwa, is_marine, is_fire, valid_from) VALUES \n" + + for _, ugc := range ugcs { + + geometry := ugc.Geometry.ToWKT() ugc.Name = strings.ReplaceAll(ugc.Name, "'", "''") @@ -46,8 +78,8 @@ func ToSQL(ugcs []UGC) (string, error) { } cwa += "}'" - result += fmt.Sprintf("('%s', '%s', '%s', '%s', %s, ST_Area(ST_GeomFromGeoJSON('%s')), ST_GeomFromGeoJSON('%s'), %s, %v, %v, %s),\n", - ugc.ID, ugc.Name, ugc.State, ugc.Type, ugc.Number, string(geometry), string(geometry), cwa, ugc.IsMarine, ugc.IsFire, DateToString(&ugc.ValidFrom)) + result += fmt.Sprintf("('%s', '%s', '%s', '%s', %s, 0.0, ST_GeomFromWKT('%s'), %s, %v, %v, %s),\n", + ugc.ID, ugc.Name, ugc.State, ugc.Type, ugc.Number, geometry, cwa, ugc.IsMarine, ugc.IsFire, DateToString(&ugc.ValidFrom)) } result = result[:len(result)-2] diff --git a/scripts/shapefiler/utils.go b/scripts/shapefiler/utils.go index 4973e11..699c724 100644 --- a/scripts/shapefiler/utils.go +++ b/scripts/shapefiler/utils.go @@ -1,6 +1,7 @@ package main import ( + "encoding/csv" "errors" "fmt" "math" @@ -10,8 +11,7 @@ import ( "github.com/everystreet/go-geojson/v2" "github.com/everystreet/go-shapefile" - "github.com/paulmach/orb" - orbjson "github.com/paulmach/orb/geojson" + "github.com/twpayne/go-geos" ) func DateToString(date *time.Time) string { @@ -32,6 +32,44 @@ func getFloat(unk interface{}) (float64, error) { return fv.Float(), nil } +func getorCreateFile(filename string) (*os.File, error) { + _, err := os.Stat(filename) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return nil, err + } + } else { + err := os.Remove(filename) + + if err != nil { + return nil, err + } + + } + + return os.Create(filename) +} + +func WriteToCSV(name string, records [][]string) error { + file, err := getorCreateFile(fmt.Sprintf("%s.csv", name)) + if err != nil { + return err + } + defer file.Close() + + writer := csv.NewWriter(file) + defer writer.Flush() + + for _, record := range records { + err := writer.Write(record) + if err != nil { + return err + } + } + + return nil +} + func WriteToFile(filename string, contents []byte) error { var err error if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) { @@ -71,7 +109,7 @@ func CreateZipScanner(filename string) (*shapefile.ZipScanner, error) { return scanner, nil } -func GetShape(shape *geojson.Feature) (*orb.Geometry, error) { +func GetShape(shape *geojson.Feature) (*geos.Geom, error) { var mpolygon *geojson.Feature switch f := shape.Geometry.(type) { case *geojson.Polygon: @@ -87,7 +125,7 @@ func GetShape(shape *geojson.Feature) (*orb.Geometry, error) { panic(err) } - geom, err := orbjson.UnmarshalFeature(geometry) + geom, err := geos.NewGeomFromGeoJSON(string(geometry)) if err != nil { panic(err) } @@ -97,5 +135,5 @@ func GetShape(shape *geojson.Feature) (*orb.Geometry, error) { // return nil, fmt.Errorf("could not assert type of orb.Geometry to orb.MultiPolygon") // } - return &geom.Geometry, nil + return geom, nil } diff --git a/scripts/shapefiler/zones.go b/scripts/shapefiler/zones.go index bb4af96..83e0227 100644 --- a/scripts/shapefiler/zones.go +++ b/scripts/shapefiler/zones.go @@ -7,7 +7,7 @@ import ( "time" "github.com/everystreet/go-shapefile" - orbjson "github.com/paulmach/orb/geojson" + "github.com/twpayne/go-geos" ) func ParseZones(scanner *shapefile.ZipScanner, t time.Time) error { @@ -25,7 +25,7 @@ func ParseZones(scanner *shapefile.ZipScanner, t time.Time) error { return err } - ugcRecords := make([]UGC, info.NumRecords) + ugcRecords := make(map[string]UGC, info.NumRecords) count := 0 for { @@ -49,7 +49,7 @@ func ParseZones(scanner *shapefile.ZipScanner, t time.Time) error { stateAttr, _ := record.Attributes.Field("STATE") state := fmt.Sprintf("%v", stateAttr.Value()) - zonename, _ := record.Attributes.Field("SHORTNAME") + zonename, _ := record.Attributes.Field("NAME") name := fmt.Sprintf("%v", zonename.Value()) lonAttr, _ := record.Attributes.Field("LON") @@ -64,7 +64,7 @@ func ParseZones(scanner *shapefile.ZipScanner, t time.Time) error { return err } - centre := [2]float64{lon, lat} + centre := geos.NewPoint([]float64{lon, lat}) cwaAttr, _ := record.Attributes.Field("CWA") cwa := fmt.Sprintf("%v", cwaAttr.Value()) @@ -82,7 +82,7 @@ func ParseZones(scanner *shapefile.ZipScanner, t time.Time) error { Type: "Z", Area: 0.0, Centre: centre, - Geometry: *mpolygon, + Geometry: mpolygon, CWA: cwaArr, IsMarine: false, IsFire: false, @@ -90,7 +90,7 @@ func ParseZones(scanner *shapefile.ZipScanner, t time.Time) error { ValidTo: nil, } - ugcRecords[count] = ugc + ugcRecords[ugc.ID] = ugc count++ @@ -102,43 +102,17 @@ func ParseZones(scanner *shapefile.ZipScanner, t time.Time) error { return err } - out, err := ToSQL(ugcRecords) + records, err := ToCSV(ugcRecords) if err != nil { return err } - err = WriteToFile("zones.sql", []byte(out)) + err = WriteToCSV("zones", records) if err != nil { return err } - slog.Info(fmt.Sprintf("Wrote %d records to zones.sql\n", len(ugcRecords))) - - collection := orbjson.NewFeatureCollection() - - for _, ugc := range ugcRecords { - feature := orbjson.NewFeature(ugc.Geometry) - feature.Properties = map[string]interface{}{ - "id": ugc.ID, - "name": ugc.Name, - "state": ugc.State, - "type": ugc.Type, - "number": ugc.Number, - "is_marine": ugc.IsMarine, - "is_fire": ugc.IsFire, - "cwa": ugc.CWA, - } - collection.Append(feature) - } - - data, err := collection.MarshalJSON() - if err != nil { - return err - } - - WriteToFile("zones.geojson", data) - - slog.Info(fmt.Sprintf("Wrote %d records to zones.geojson\n", len(ugcRecords))) + slog.Info(fmt.Sprintf("Wrote %d records to zones.csv\n", len(ugcRecords))) return nil } diff --git a/services/live/warnings.go b/services/live/warnings.go index 96fcf91..8428f9c 100644 --- a/services/live/warnings.go +++ b/services/live/warnings.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "fmt" "sync" "time" @@ -16,43 +17,48 @@ import ( const WarningTopic string = "warnings" type Warning struct { - ID string `json:"id"` - UpdatedAt time.Time `json:"updatedAt,omitzero"` - Issued time.Time `json:"issued"` - Starts *time.Time `json:"starts,omitzero"` - Expires time.Time `json:"expires"` - Ends time.Time `json:"ends,omitzero"` - EndInitial time.Time `json:"endInitial,omitzero"` - Text string `json:"text"` - WFO string `json:"wfo"` - Action string `json:"action"` - Class string `json:"class"` - Phenomena string `json:"phenomena"` - Significance string `json:"significance"` - EventNumber int `json:"eventNumber"` - Year int `json:"year"` - Title string `json:"title"` - IsEmergency bool `json:"isEmergency"` - IsPDS bool `json:"isPDS"` - Geom *geos.Geom `json:"geom,omitempty"` - Direction *int `json:"direction"` - Location *geos.Geom `json:"location"` - Speed *int `json:"speed"` - SpeedText *string `json:"speedText"` - TMLTime *time.Time `json:"tmlTime"` - UGC map[string]UGC `json:"ugc"` - Tornado string `json:"tornado,omitempty"` - Damage string `json:"damage,omitempty"` - HailThreat string `json:"hailThreat,omitempty"` - HailTag string `json:"hailTag,omitempty"` - WindThreat string `json:"windThreat,omitempty"` - WindTag string `json:"windTag,omitempty"` - FlashFlood string `json:"flashFlood,omitempty"` - RainfallTag string `json:"rainfallTag,omitempty"` - FloodTagDam string `json:"floodTagDam,omitempty"` - SpoutTag string `json:"spoutTag,omitempty"` - SnowSquall string `json:"snowSquall,omitempty"` - SnowSquallTag string `json:"snowSquall_tag,omitempty"` + ID int `json:"id"` + WarningID string `json:"warningID"` + UpdatedAt time.Time `json:"updatedAt,omitzero"` + Issued time.Time `json:"issued"` + Starts *time.Time `json:"starts,omitzero"` + Expires time.Time `json:"expires"` + Ends time.Time `json:"ends,omitzero"` + ExpiresInitial time.Time `json:"expires_initial,omitzero"` + Text string `json:"text"` + WFO string `json:"wfo"` + Action string `json:"action"` + Class string `json:"class"` + Phenomena string `json:"phenomena"` + Significance string `json:"significance"` + EventNumber int `json:"eventNumber"` + Year int `json:"year"` + Title string `json:"title"` + IsEmergency bool `json:"isEmergency"` + IsPDS bool `json:"isPDS"` + Geom *geos.Geom `json:"geom,omitempty"` + Direction *int `json:"direction"` + Location *geos.Geom `json:"location"` + Speed *int `json:"speed"` + SpeedText *string `json:"speedText"` + TMLTime *time.Time `json:"tmlTime"` + UGC map[string]UGC `json:"ugc"` + Tornado string `json:"tornado,omitempty"` + Damage string `json:"damage,omitempty"` + HailThreat string `json:"hailThreat,omitempty"` + HailTag string `json:"hailTag,omitempty"` + WindThreat string `json:"windThreat,omitempty"` + WindTag string `json:"windTag,omitempty"` + FlashFlood string `json:"flashFlood,omitempty"` + RainfallTag string `json:"rainfallTag,omitempty"` + FloodTagDam string `json:"floodTagDam,omitempty"` + SpoutTag string `json:"spoutTag,omitempty"` + SnowSquall string `json:"snowSquall,omitempty"` + SnowSquallTag string `json:"snowSquall_tag,omitempty"` +} + +func (w *Warning) CompositeID() string { + return fmt.Sprintf("%s-%v", w.WarningID, w.ID) } func (w *Warning) MarshalJSON() ([]byte, error) { @@ -83,7 +89,7 @@ type WarningManager struct { hub *Hub rabbitQueue amqp.Queue - data map[string]*Warning + data map[string]map[int]*Warning subscribers map[*client]struct{} ticker *time.Ticker @@ -99,7 +105,7 @@ func NewWarningManager(hub *Hub) *WarningManager { store := &WarningManager{ hub: hub, - data: map[string]*Warning{}, + data: map[string]map[int]*Warning{}, subscribers: map[*client]struct{}{}, ticker: ticker, } @@ -147,7 +153,12 @@ func (manager *WarningManager) Load() error { for _, warning := range warnings { id := warning.GenerateID() - manager.data[id] = manager.modelToWarning(id, *warning) + _, ok := manager.data[id] + if !ok { + manager.data[id] = map[int]*Warning{warning.ID: manager.modelToWarning(*warning)} + } else { + manager.data[id][warning.ID] = manager.modelToWarning(*warning) + } } log.Debug().Int("size", len(manager.data)).Msg("loaded warning data") @@ -174,6 +185,7 @@ func (manager *WarningManager) Run() { for { select { case t := <-manager.ticker.C: + manager.ticker.Reset(60 * time.Second) manager.checkExpired(t) case message := <-d: warning := &models.Warning{} @@ -181,8 +193,7 @@ func (manager *WarningManager) Run() { log.Error().Err(err).Msg("failed to unmarshal warning message") continue } - id := message.MessageId - err := manager.handleUpdate(id, *warning) + err := manager.handleUpdate(*warning, message.Type) if err != nil { log.Error().Err(err).Msg("failed to handle warning update") continue @@ -200,8 +211,10 @@ func (manager *WarningManager) Subscribe(c *client) { manager.subscribers[c] = struct{}{} warnings := []*Warning{} - for _, w := range manager.data { - warnings = append(warnings, w) + for _, list := range manager.data { + for _, w := range list { + warnings = append(warnings, w) + } } // Marshal the warnings slice to JSON @@ -239,23 +252,13 @@ func (manager *WarningManager) Unsubscribe(c *client) { delete(manager.subscribers, c) } -func (manager *WarningManager) handleUpdate(id string, w models.Warning) error { +func (manager *WarningManager) handleUpdate(w models.Warning, eventType string) error { manager.mu.Lock() defer manager.mu.Unlock() - newWarning := manager.modelToWarning(id, w) + warning := manager.modelToWarning(w) - var eventType string - switch newWarning.Action { - case "NEW", "EXA", "EXB": - eventType = EnvelopeNew - case "CAN", "UPG", "EXP": - eventType = EnvelopeDelete - default: - eventType = EnvelopeUpdate - } - - warningBytes, err := json.Marshal(newWarning) + warningBytes, err := json.Marshal(warning) if err != nil { return err } @@ -263,7 +266,7 @@ func (manager *WarningManager) handleUpdate(id string, w models.Warning) error { envelope := Envelope{ Type: eventType, Product: WarningTopic, - ID: newWarning.ID, + ID: warning.CompositeID(), Timestamp: time.Now(), Data: warningBytes, } @@ -278,66 +281,18 @@ func (manager *WarningManager) handleUpdate(id string, w models.Warning) error { } // See if we have the warning already - warning, ok := manager.data[id] - if !ok { - if newWarning.Action == "CAN" || newWarning.Action == "UPG" || newWarning.Action == "EXP" { - return nil - } - manager.data[id] = newWarning - return nil - } - - if newWarning.Action == "CAN" || newWarning.Action == "UPG" || newWarning.Action == "EXP" { - codes := []string{} - for _, ugc := range newWarning.UGC { - _, ok := warning.UGC[ugc.Code] - if ok { - codes = append(codes, ugc.Code) - } - } - for _, code := range codes { - delete(warning.UGC, code) - } - - if len(warning.UGC) == 0 { - delete(manager.data, id) - return nil - } - } else { - for _, ugc := range newWarning.UGC { - _, ok := warning.UGC[ugc.Code] - if !ok { - warning.UGC[ugc.Code] = ugc + if _, ok := manager.data[warning.WarningID]; ok { + if eventType == streaming.EventDelete { + delete(manager.data, warning.WarningID) + } else { + if _, ok := manager.data[warning.WarningID][warning.ID]; ok { + manager.data[warning.WarningID][warning.ID] = warning } } + } else if eventType != streaming.EventDelete { + manager.data[warning.WarningID] = map[int]*Warning{warning.ID: warning} } - warning.Expires = newWarning.Expires - warning.Ends = newWarning.Ends - warning.Text = newWarning.Text - warning.Action = newWarning.Action - warning.Title = newWarning.Title - warning.IsEmergency = newWarning.IsEmergency - warning.IsPDS = newWarning.IsPDS - warning.Geom = newWarning.Geom - warning.Direction = newWarning.Direction - warning.Location = newWarning.Location - warning.Speed = newWarning.Speed - warning.SpeedText = newWarning.SpeedText - warning.TMLTime = newWarning.TMLTime - warning.Tornado = newWarning.Tornado - warning.Damage = newWarning.Damage - warning.HailThreat = newWarning.HailThreat - warning.HailTag = newWarning.HailTag - warning.WindThreat = newWarning.WindThreat - warning.WindTag = newWarning.WindTag - warning.FlashFlood = newWarning.FlashFlood - warning.RainfallTag = newWarning.RainfallTag - warning.FloodTagDam = newWarning.FloodTagDam - warning.SpoutTag = newWarning.SpoutTag - warning.SnowSquall = newWarning.SnowSquall - warning.SnowSquallTag = newWarning.SnowSquallTag - return nil } @@ -345,23 +300,47 @@ func (manager *WarningManager) checkExpired(t time.Time) { manager.mu.Lock() defer manager.mu.Unlock() - toDelete := []string{} - for id, warning := range manager.data { - if warning.Ends.Before(t) { - toDelete = append(toDelete, id) + toDelete := []*Warning{} + for _, bucket := range manager.data { + for _, warning := range bucket { + if warning.Ends.Before(t) { + toDelete = append(toDelete, warning) + } } } - for _, id := range toDelete { - delete(manager.data, id) - } - if len(toDelete) > 0 { + for _, warning := range toDelete { + delete(manager.data[warning.WarningID], warning.ID) + + warningBytes, err := json.Marshal(warning) + if err != nil { + log.Error().Err(err).Msg("failed to marshal warning for expired warning") + } + + envelope := Envelope{ + Type: EnvelopeDelete, + Product: WarningTopic, + ID: warning.CompositeID(), + Timestamp: time.Now(), + Data: warningBytes, + } + + envelopeBytes, err := json.Marshal(envelope) + if err != nil { + log.Error().Err(err).Msg("failed to marshal envelope for expired warning") + } + + for client := range manager.subscribers { + client.send <- envelopeBytes + } + } + log.Debug().Int("deleted", len(toDelete)).Msg("deleted expired warnings") } } -func (manager *WarningManager) modelToWarning(id string, w models.Warning) *Warning { +func (manager *WarningManager) modelToWarning(w models.Warning) *Warning { ugcs := map[string]UGC{} @@ -373,42 +352,43 @@ func (manager *WarningManager) modelToWarning(id string, w models.Warning) *Warn } return &Warning{ - ID: id, - UpdatedAt: w.UpdatedAt, - Issued: w.Issued, - Starts: w.Starts, - Expires: w.Expires, - Ends: w.Ends, - EndInitial: w.EndInitial, - Text: w.Text, - WFO: w.WFO, - Action: w.Action, - Class: w.Class, - Phenomena: w.Phenomena, - Significance: w.Significance, - EventNumber: w.EventNumber, - Year: w.Year, - Title: w.Title, - IsEmergency: w.IsEmergency, - IsPDS: w.IsPDS, - Geom: w.Geom, - Direction: w.Direction, - Location: w.Location, - Speed: w.Speed, - SpeedText: w.SpeedText, - TMLTime: w.TMLTime, - UGC: ugcs, - Tornado: w.Tornado, - Damage: w.Damage, - HailThreat: w.HailThreat, - HailTag: w.HailTag, - WindThreat: w.WindThreat, - WindTag: w.WindTag, - FlashFlood: w.FlashFlood, - RainfallTag: w.RainfallTag, - FloodTagDam: w.FloodTagDam, - SpoutTag: w.SpoutTag, - SnowSquall: w.SnowSquall, - SnowSquallTag: w.SnowSquallTag, + ID: w.ID, + WarningID: w.GenerateCompositeID(), + UpdatedAt: w.UpdatedAt, + Issued: w.Issued, + Starts: w.Starts, + Expires: w.Expires, + Ends: w.Ends, + ExpiresInitial: w.ExpiresInitial, + Text: w.Text, + WFO: w.WFO, + Action: w.Action, + Class: w.Class, + Phenomena: w.Phenomena, + Significance: w.Significance, + EventNumber: w.EventNumber, + Year: w.Year, + Title: w.Title, + IsEmergency: w.IsEmergency, + IsPDS: w.IsPDS, + Geom: w.Geom, + Direction: w.Direction, + Location: w.Location, + Speed: w.Speed, + SpeedText: w.SpeedText, + TMLTime: w.TMLTime, + UGC: ugcs, + Tornado: w.Tornado, + Damage: w.Damage, + HailThreat: w.HailThreat, + HailTag: w.HailTag, + WindThreat: w.WindThreat, + WindTag: w.WindTag, + FlashFlood: w.FlashFlood, + RainfallTag: w.RainfallTag, + FloodTagDam: w.FloodTagDam, + SpoutTag: w.SpoutTag, + SnowSquall: w.SnowSquall, + SnowSquallTag: w.SnowSquallTag, } } diff --git a/services/parse/awips/go.mod b/services/parse/awips/go.mod index f5a6f5c..00f3044 100644 --- a/services/parse/awips/go.mod +++ b/services/parse/awips/go.mod @@ -19,6 +19,7 @@ require ( require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect @@ -28,6 +29,7 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect @@ -39,11 +41,13 @@ require ( golang.org/x/sys v0.36.0 // indirect golang.org/x/text v0.29.0 // indirect google.golang.org/protobuf v1.36.8 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/pgx/v5 v5.7.6 github.com/spf13/pflag v1.0.10 // indirect + github.com/stretchr/testify v1.11.1 github.com/twpayne/pgx-geos v1.0.0 ) diff --git a/services/parse/awips/internal/db.go b/services/parse/awips/internal/db.go index 35ee468..3c781f0 100644 --- a/services/parse/awips/internal/db.go +++ b/services/parse/awips/internal/db.go @@ -2,7 +2,6 @@ package internal import ( "context" - "os" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" @@ -10,10 +9,10 @@ import ( pgxgeos "github.com/twpayne/pgx-geos" ) -func newDatabasePool() (*pgxpool.Pool, error) { +func newDatabasePool(url string) (*pgxpool.Pool, error) { ctx := context.Background() - config, err := pgxpool.ParseConfig(os.Getenv("DATABASE_URL")) + config, err := pgxpool.ParseConfig(url) if err != nil { return nil, err } diff --git a/services/parse/awips/internal/db_test.go b/services/parse/awips/internal/db_test.go new file mode 100644 index 0000000..9a9a5f8 --- /dev/null +++ b/services/parse/awips/internal/db_test.go @@ -0,0 +1,17 @@ +package internal + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewDatabasePool(t *testing.T) { + pool, err := newDatabasePool("postgres://mds:@localhost:5432/mds") + assert.NoError(t, err) + assert.NotNil(t, pool) + + err = pool.Ping(context.Background()) + assert.NoError(t, err) +} diff --git a/services/parse/awips/internal/local.go b/services/parse/awips/internal/local.go index ca2a874..60a1bf5 100644 --- a/services/parse/awips/internal/local.go +++ b/services/parse/awips/internal/local.go @@ -13,7 +13,7 @@ import ( func Local(path string, logLevel zerolog.Level) { zerolog.SetGlobalLevel(logLevel) - db, err := newDatabasePool() + db, err := newDatabasePool(os.Getenv("DATABASE_URL")) if err != nil { log.Error().Err(err).Msg("failed to initialise database") return @@ -57,7 +57,7 @@ func process(path string, db *pgxpool.Pool, rabbit *amqp.Channel) { } for _, f := range files { - process(f.Name(), db, rabbit) + process(path+f.Name(), db, rabbit) } } else { processFile(file, stat.Size(), db, rabbit) @@ -74,4 +74,6 @@ func processFile(file *os.File, size int64, db *pgxpool.Pool, rabbit *amqp.Chann text := string(data) HandleText(text, time.Now(), db, rabbit) + + time.Sleep(10 * time.Second) } diff --git a/services/parse/awips/internal/server.go b/services/parse/awips/internal/server.go index 94eba81..23bc9e8 100644 --- a/services/parse/awips/internal/server.go +++ b/services/parse/awips/internal/server.go @@ -3,6 +3,7 @@ package internal import ( "context" "net/http" + "os" "os/signal" "syscall" "time" @@ -69,7 +70,7 @@ func Server(logLevel zerolog.Level) { zerolog.SetGlobalLevel(logLevel) - db, err := newDatabasePool() + db, err := newDatabasePool(os.Getenv("DATABASE_URL")) if err != nil { log.Error().Err(err).Msg("failed to initialise database") return diff --git a/services/parse/awips/internal/vtec.go b/services/parse/awips/internal/vtec.go index 37d1017..cf58552 100644 --- a/services/parse/awips/internal/vtec.go +++ b/services/parse/awips/internal/vtec.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/jackc/pgx/v5" "github.com/metdatasystem/us/pkg/awips" "github.com/metdatasystem/us/pkg/db" "github.com/metdatasystem/us/pkg/models" @@ -12,10 +13,15 @@ import ( type vtecHandler struct { Handler + ctx context.Context + tx pgx.Tx + + publishedWarnings map[string]struct{} + // errorCollecting *ErrorCollector } func NewVTECHandler(handler *Handler) *vtecHandler { - return &vtecHandler{*handler} + return &vtecHandler{*handler, context.Background(), nil, map[string]struct{}{}} } // Handle a VTEC product @@ -24,6 +30,14 @@ func (handler *vtecHandler) Handle() error { product := handler.product log := handler.log + // Initialise transaction + tx, err := handler.db.BeginTx(handler.ctx, pgx.TxOptions{}) + if err != nil { + return err + } + handler.tx = tx + defer tx.Rollback(handler.ctx) + // Go through each segment... for _, segment := range handler.product.Segments { // ...and each VTEC line in the segment @@ -51,7 +65,7 @@ func (handler *vtecHandler) Handle() error { } // Try and find the event in the database - event, err := db.FindVTECEvent(handler.db, vtec.WFO, vtec.Phenomena, vtec.Significance, vtec.EventNumber, year) + event, err := db.FindVTECEventTX(handler.tx, vtec.WFO, vtec.Phenomena, vtec.Significance, vtec.EventNumber, year) if err != nil { log.Error().Err(err).Msg("failed to find vtec event") continue @@ -78,7 +92,13 @@ func (handler *vtecHandler) Handle() error { IsPDS: segment.IsPDS(), } - err = db.InsertVTECEvent(handler.db, event) + _, err := tx.Exec(handler.ctx, ` + INSERT INTO vtec.events(issued, starts, expires, ends, ends_initial, class, phenomena, wfo, + significance, event_number, year, title, is_emergency, is_pds) VALUES + ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14); + `, event.Issued, event.Starts, event.Expires, event.Ends, event.EndInitial, event.Class, + event.Phenomena, event.WFO, event.Significance, event.EventNumber, event.Year, event.Title, + event.IsEmergency, event.IsPDS) if err != nil { log.Error().Err(err).Msg("failed to insert vtec event") continue @@ -125,6 +145,10 @@ func (handler *vtecHandler) Handle() error { } + if err := tx.Commit(context.Background()); err != nil { + return err + } + return nil } @@ -240,7 +264,20 @@ func (handler *vtecHandler) createUpdate(segment *awips.ProductSegment, event *m SnowSquallTag: segment.Tags["snowSquallImpact"], } - err := db.InsertVTECUpdate(handler.db, update) + _, err := handler.tx.Exec(handler.ctx, ` + INSERT INTO vtec.updates(issued, starts, expires, ends, text, product, + wfo, action, class, phenomena, significance, event_number, year, title, + is_emergency, is_pds, geom, direction, location, speed, speed_text, tml_time, + ugc, tornado, damage, hail_threat, hail_tag, wind_threat, wind_tag, flash_flood, + rainfall_tag, flood_tag_dam, spout_tag, snow_squall, snow_squall_tag) + VALUES + ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, + $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35) + `, update.Issued, update.Starts, update.Expires, update.Ends, update.Text, update.Product, + update.WFO, update.Action, update.Class, update.Phenomena, update.Significance, update.EventNumber, update.Year, update.Title, + update.IsEmergency, update.IsPDS, update.Geom, update.Direction, update.Location, update.Speed, update.SpeedText, update.TMLTime, + update.UGC, update.Tornado, update.Damage, update.HailThreat, update.HailTag, update.WindThreat, update.WindTag, update.FlashFlood, + update.RainfallTag, update.FloodTagDam, update.SpoutTag, update.SnowSquall, update.SnowSquallTag) if err != nil { return err } @@ -270,7 +307,7 @@ func (handler *vtecHandler) ugcNew(segment *awips.ProductSegment, event *models. end = *vtec.End } - currentUGCs, err := db.FindCurrentVTECEventUGCs(handler.db, event.WFO, event.Phenomena, event.Significance, event.EventNumber, event.Year, expires) + currentUGCs, err := db.FindCurrentVTECEventUGCsTX(handler.tx, event.WFO, event.Phenomena, event.Significance, event.EventNumber, event.Year, expires) if err != nil { return err } @@ -291,7 +328,12 @@ func (handler *vtecHandler) ugcNew(segment *awips.ProductSegment, event *models. if current != nil { // If the product was reissued as a correction, delete the existing UGC since it may not be valid anymore if handler.product.IsCorrection() && current.Action == vtec.Action { - db.DeleteVTECUGC(handler.db, current) + _, err := handler.tx.Exec(handler.ctx, ` + DELETE FROM vtec.ugcs WHERE id = $1 + `, ugc.ID) + if err != nil { + log.Error().Err(err).Msg("failed to delete vtec.ugc entry") + } deleted++ } duplicates++ @@ -320,7 +362,12 @@ func (handler *vtecHandler) ugcNew(segment *awips.ProductSegment, event *models. Year: event.Year, } - err = db.InsertVTECUGC(handler.db, newUGC) + _, err := handler.tx.Exec(handler.ctx, ` + INSERT INTO vtec.ugcs(wfo, phenomena, significance, event_number, ugc, issued, starts, expires, ends, end_initial, action, year) VALUES + ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12); + `, newUGC.WFO, newUGC.Phenomena, newUGC.Significance, newUGC.EventNumber, newUGC.UGC, + newUGC.Issued, newUGC.Starts, newUGC.Expires, newUGC.Ends, newUGC.EndInitial, + newUGC.Action, newUGC.Year) if err != nil { log.Error().Err(err).Msg("failed to insert new vtec ugc") } @@ -339,14 +386,19 @@ func (handler *vtecHandler) ugcUpdate(segment *awips.ProductSegment, event *mode u = append(u, ugc.ID) } - return db.BulkUpdateUGCsById(handler.db, u, expires, end, vtec.Action, event.WFO, event.Phenomena, event.Significance, event.EventNumber, event.Year) + _, err := handler.tx.Exec(handler.ctx, ` + UPDATE vtec.ugcs SET expires = $1, ends = $2, action = $3 WHERE + wfo = $4 AND phenomena = $5 AND significance = $6 AND event_number = $7 AND year = $8 + AND ugc = ANY($9) + `, expires, end, vtec.Action, event.WFO, event.Phenomena, event.Significance, event.EventNumber, + event.Year, ugcs) + + return err } func (handler *vtecHandler) updateEvent(segment *awips.ProductSegment, event *models.VTECEvent, vtec awips.VTEC) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - _, err := handler.db.Exec(ctx, ` + _, err := handler.tx.Exec(handler.ctx, ` UPDATE vtec.events SET updated_at = CURRENT_TIMESTAMP, is_emergency = $6, is_pds = $7 WHERE wfo = $1 AND phenomena = $2 AND significance = $3 AND event_number = $4 AND year = $5 `, vtec.WFO, vtec.Phenomena, vtec.Significance, vtec.EventNumber, event.Year, segment.IsEmergency(), segment.IsPDS()) diff --git a/services/parse/awips/internal/vtec_test.go b/services/parse/awips/internal/vtec_test.go new file mode 100644 index 0000000..1c84fe6 --- /dev/null +++ b/services/parse/awips/internal/vtec_test.go @@ -0,0 +1,190 @@ +package internal + +import ( + "context" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/metdatasystem/us/pkg/awips" + "github.com/metdatasystem/us/pkg/db" + "github.com/metdatasystem/us/pkg/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const awipsTestDataPath = "../../../../data/test/awips/" + +func getTestFiles(path string) ([]string, error) { + dir, err := os.ReadDir(path) + if err != nil { + return nil, err + } + + var files []string + for _, entry := range dir { + if entry.IsDir() { + continue + } + files = append(files, entry.Name()) + } + return files, nil +} + +func readFile(t *testing.T, path string, filename string) string { + file, err := os.ReadFile(path) + assert.NoErrorf(t, err, "failed to read file %s", filename) + assert.Greaterf(t, len(file), 0, "%s is nil", filename) + return string(file) +} + +type testSuite struct { + db *pgxpool.Pool + files []string +} + +func initTestSuite(t *testing.T, folder string) (*testSuite, error) { + pool, err := newDatabasePool("postgres://mds:@localhost:5432/mds") + assert.NoError(t, err, "failed to create database pool") + assert.NotNil(t, pool, "database pool is nil") + + files, err := getTestFiles(awipsTestDataPath + folder) + assert.NoError(t, err, "failed to get test files") + assert.Greater(t, len(files), 0, "no test files found") + + suite := testSuite{ + db: pool, + files: files, + } + + return &suite, nil +} + +func (suite *testSuite) teardown() { + suite.db.Close() +} + +func TestTornadoWarning(t *testing.T) { + dir := "tor/" + + suite, err := initTestSuite(t, dir) + require.NoError(t, err, "failed to initialize test suite") + require.NotNil(t, suite, "test suite is nil") + t.Cleanup(func() { + defer suite.teardown() + + }) + + var year int + var v *awips.VTEC + + for _, filename := range suite.files { + text := readFile(t, awipsTestDataPath+dir+filename, filename) + + product, err := awips.New(text) + assert.NoErrorf(t, err, "failed to parse awips product from file %s", filename) + assert.NotNilf(t, product, "awips product from file %s is nil", filename) + + HandleText(text, time.Now(), suite.db, nil) + + for _, segments := range product.Segments { + for _, vtec := range segments.VTEC { + if year == 0 && vtec.Start != nil { + year = vtec.Start.Year() + } + if v == nil { + v = &vtec + } + + // Check VTEC event + rows, err := suite.db.Query(context.Background(), ` + SELECT * FROM vtec.events WHERE phenomena=$1 AND significance=$2 AND event_number=$3 AND wfo=$4 AND year=$5 + `, vtec.Phenomena, vtec.Significance, vtec.EventNumber, vtec.WFO, year) + assert.NoError(t, err, "failed to query vtec events") + if assert.True(t, rows.Next(), "no vtec event row returned") { + event := &models.VTECEvent{} + err = db.ScanVTECEvent(rows, event) + assert.NoError(t, err, "failed to scan vtec event") + } + + rows.Close() + + } + } + + } + + // Cleanup + _, err = suite.db.Exec(context.Background(), ` + DELETE FROM vtec.events WHERE phenomena=$1 AND significance=$2 AND event_number=$3 AND wfo=$4 AND year=$5 + `, v.Phenomena, v.Significance, v.EventNumber, v.WFO, year) + assert.NoError(t, err, "failed to delete vtec event") + + _, err = suite.db.Exec(context.Background(), ` + DELETE FROM warnings.warnings WHERE phenomena=$1 AND significance=$2 AND event_number=$3 AND wfo=$4 AND year=$5 + `, v.Phenomena, v.Significance, v.EventNumber, v.WFO, year) + assert.NoError(t, err, "failed to delete warnings") +} + +func TestWinterWeather(t *testing.T) { + dir := "winter weather/" + + suite, err := initTestSuite(t, dir) + require.NoError(t, err, "failed to initialize test suite") + require.NotNil(t, suite, "test suite is nil") + t.Cleanup(func() { + defer suite.teardown() + + }) + + var year int + var v *awips.VTEC + + for _, filename := range suite.files { + text := readFile(t, awipsTestDataPath+dir+filename, filename) + + product, err := awips.New(text) + assert.NoErrorf(t, err, "failed to parse awips product from file %s", filename) + assert.NotNilf(t, product, "awips product from file %s is nil", filename) + + HandleText(text, time.Now(), suite.db, nil) + + for _, segments := range product.Segments { + for _, vtec := range segments.VTEC { + if year == 0 && vtec.Start != nil { + year = vtec.Start.Year() + } + if v == nil { + v = &vtec + } + + // Check VTEC event + rows, err := suite.db.Query(context.Background(), ` + SELECT * FROM vtec.events WHERE phenomena=$1 AND significance=$2 AND event_number=$3 AND wfo=$4 AND year=$5 + `, vtec.Phenomena, vtec.Significance, vtec.EventNumber, vtec.WFO, year) + assert.NoError(t, err, "failed to query vtec events") + if assert.True(t, rows.Next(), "no vtec event row returned") { + event := &models.VTECEvent{} + err = db.ScanVTECEvent(rows, event) + assert.NoError(t, err, "failed to scan vtec event") + } + + rows.Close() + + } + } + + } + + // Cleanup + // _, err = suite.db.Exec(context.Background(), ` + // DELETE FROM vtec.events WHERE phenomena=$1 AND significance=$2 AND event_number=$3 AND wfo=$4 AND year=$5 + // `, v.Phenomena, v.Significance, v.EventNumber, v.WFO, year) + // assert.NoError(t, err, "failed to delete vtec event") + + // _, err = suite.db.Exec(context.Background(), ` + // DELETE FROM warnings.warnings WHERE phenomena=$1 AND significance=$2 AND event_number=$3 AND wfo=$4 AND year=$5 + // `, v.Phenomena, v.Significance, v.EventNumber, v.WFO, year) + // assert.NoError(t, err, "failed to delete warnings") +} diff --git a/services/parse/awips/internal/warning.go b/services/parse/awips/internal/warning.go index 31133f6..a9d51c3 100644 --- a/services/parse/awips/internal/warning.go +++ b/services/parse/awips/internal/warning.go @@ -2,7 +2,6 @@ package internal import ( "context" - "encoding/json" "time" "github.com/metdatasystem/us/pkg/awips" @@ -20,7 +19,7 @@ func (handler *vtecHandler) warning(segment *awips.ProductSegment, event *models yesterday := time.Now().Add(time.Hour * -24) if event.Ends.Before(yesterday) { - return nil + // return nil } ugcList := []string{} @@ -32,9 +31,8 @@ func (handler *vtecHandler) warning(segment *awips.ProductSegment, event *models if segment.LatLon != nil { coords := segment.LatLon.ToFloatClosing() geom = geos.NewPolygon([][][]float64{coords}) - } else { - - g, err := db.GetUGCUnionGeomSimplified(handler.db, ugcList) + } else if vtec.Action != "CAN" && vtec.Action != "UPG" && vtec.Action != "EXP" { + g, err := db.GetUGCUnionGeomSimplifiedTx(handler.tx, ugcList) if err != nil { return err } @@ -62,156 +60,203 @@ func (handler *vtecHandler) warning(segment *awips.ProductSegment, event *models tmlTime = &segment.TML.Time } - warning, err := db.FindWarning(handler.db, event.WFO, event.Phenomena, event.Significance, event.EventNumber, event.Year) - if err != nil { - return err + warning := &models.Warning{ + Issued: product.Issued, + Starts: event.Starts, + Expires: segment.UGC.Expires, + Ends: event.Ends, + ExpiresInitial: segment.UGC.Expires, + Text: segment.Text, + Product: handler.dbProduct.ProductID, + WFO: vtec.WFO, + Action: vtec.Action, + Class: vtec.Class, + Phenomena: vtec.Phenomena, + Significance: vtec.Significance, + EventNumber: vtec.EventNumber, + Year: event.Year, + Title: vtec.Title(segment.IsEmergency()), + IsEmergency: segment.IsEmergency(), + IsPDS: segment.IsPDS(), + Geom: geom, + Direction: direction, + Location: locations, + Speed: speed, + SpeedText: speedText, + TMLTime: tmlTime, + UGC: ugcList, + Tornado: segment.Tags["tornado"], + Damage: segment.Tags["damage"], + HailThreat: segment.Tags["hailThreat"], + HailTag: segment.Tags["hail"], + WindThreat: segment.Tags["windThreat"], + WindTag: segment.Tags["wind"], + FlashFlood: segment.Tags["flashFlood"], + RainfallTag: segment.Tags["expectedRainfall"], + FloodTagDam: segment.Tags["damFailure"], + SpoutTag: segment.Tags["spout"], + SnowSquall: segment.Tags["snowSquall"], + SnowSquallTag: segment.Tags["snowSquallImpact"], } - // Warning exists and can be updated - if warning != nil { - - warning.Expires = segment.UGC.Expires - warning.Ends = event.Ends - warning.Text = segment.Text - warning.Action = vtec.Action - warning.Title = vtec.Title(segment.IsEmergency()) - warning.IsEmergency = segment.IsEmergency() - warning.IsPDS = segment.IsPDS() - warning.Geom = geom - warning.Direction = direction - warning.Location = locations - warning.Speed = speed - warning.SpeedText = speedText - warning.TMLTime = tmlTime - warning.Tornado = segment.Tags["tornado"] - warning.Damage = segment.Tags["damage"] - warning.HailThreat = segment.Tags["hailThreat"] - warning.HailTag = segment.Tags["hail"] - warning.WindThreat = segment.Tags["windThreat"] - warning.WindTag = segment.Tags["wind"] - warning.FlashFlood = segment.Tags["flashFlood"] - warning.RainfallTag = segment.Tags["expectedRainfall"] - warning.FloodTagDam = segment.Tags["damFailure"] - warning.SpoutTag = segment.Tags["spout"] - warning.SnowSquall = segment.Tags["snowSquall"] - warning.SnowSquallTag = segment.Tags["snowSquallImpact"] - - // Publish before we override all the UGC data - err = handler.publishWarning(warning) + if _, ok := handler.publishedWarnings[warning.GenerateID()]; !ok { + rows, err := handler.tx.Query(handler.ctx, ` + UPDATE warnings.warnings SET expires_initial = $1, current = false, updated_at = CURRENT_TIMESTAMP + WHERE phenomena = $2 AND significance = $3 AND wfo = $4 AND event_number = $5 AND year = $6 AND current = true RETURNING id + `, warning.Issued, warning.Phenomena, warning.Significance, warning.WFO, warning.EventNumber, warning.Year) if err != nil { - log.Error().Err(err).Msg("failed to publish warning to kafka") - } - - // Convert warning.UGC into a map for fast lookups - existing := make(map[string]bool) - for _, v := range warning.UGC { - existing[v] = true + return err } - // Handle based on Action - switch warning.Action { - case "CAN", "UPG", "EXP": - // Remove elements that exist in newWarning.UGC - filtered := []string{} - toRemove := make(map[string]bool) - for _, v := range ugcList { - toRemove[v] = true + for rows.Next() { + var id int + err := rows.Scan(&id) + if err != nil { + log.Error().Err(err).Msg("failed to scan warning ID from update") + continue } - for _, v := range warning.UGC { - if !toRemove[v] { - filtered = append(filtered, v) - } + + tempW := *warning + tempW.ID = id + data, err := tempW.MarshalJSON() + if err != nil { + log.Error().Err(err).Msg("failed to marshal warning to publish from update") + continue } - warning.UGC = filtered - - default: - // Add new elements that are not already in warning.UGC - for _, v := range ugcList { - if !existing[v] { - warning.UGC = append(warning.UGC, v) - existing[v] = true - } + + err = handler.rabbit.PublishWithContext(context.Background(), + streaming.ExchangeLiveName, + "warning", + false, + false, + amqp091.Publishing{ + ContentType: "application/json", + MessageId: tempW.GenerateCompositeID(), + Timestamp: time.Now(), + Type: streaming.EventDelete, + AppId: "us.parse.awips", + Body: data, + }, + ) + if err != nil { + log.Error().Err(err).Msg("failed to publish warning delete") + continue } } + rows.Close() + } - if err := db.UpdateWarning(handler.db, warning); err != nil { - return err - } - } else { - - warning = &models.Warning{ - Issued: product.Issued, - Starts: event.Starts, - Expires: segment.UGC.Expires, - Ends: event.Ends, - EndInitial: event.EndInitial, - Text: segment.Text, - WFO: vtec.WFO, - Action: vtec.Action, - Class: vtec.Class, - Phenomena: vtec.Phenomena, - Significance: vtec.Significance, - EventNumber: vtec.EventNumber, - Year: event.Year, - Title: vtec.Title(segment.IsEmergency()), - IsEmergency: segment.IsEmergency(), - IsPDS: segment.IsPDS(), - Geom: geom, - Direction: direction, - Location: locations, - Speed: speed, - SpeedText: speedText, - TMLTime: tmlTime, - UGC: ugcList, - Tornado: segment.Tags["tornado"], - Damage: segment.Tags["damage"], - HailThreat: segment.Tags["hailThreat"], - HailTag: segment.Tags["hail"], - WindThreat: segment.Tags["windThreat"], - WindTag: segment.Tags["wind"], - FlashFlood: segment.Tags["flashFlood"], - RainfallTag: segment.Tags["expectedRainfall"], - FloodTagDam: segment.Tags["damFailure"], - SpoutTag: segment.Tags["spout"], - SnowSquall: segment.Tags["snowSquall"], - SnowSquallTag: segment.Tags["snowSquallImpact"], - } + current := true + if warning.Action == "CAN" || warning.Action == "UPG" || warning.Action == "EXP" { + current = false + } - err = handler.publishWarning(warning) - if err != nil { - log.Error().Err(err).Msg("failed to publish warning to kafka") - } + rows, err := handler.tx.Query(handler.ctx, ` + INSERT INTO warnings.warnings( + issued, starts, expires, ends, expires_initial, text, product, + wfo, action, current, class, phenomena, significance, event_number, year, + title, is_emergency, is_pds, geom, direction, location, speed, speed_text, tml_time, + ugc, tornado, damage, hail_threat, hail_tag, wind_threat, wind_tag, flash_flood, + rainfall_tag, flood_tag_dam, spout_tag, snow_squall, snow_squall_tag + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, + $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37 + ) RETURNING id + `, + warning.Issued, + warning.Starts, + warning.Expires, + warning.Ends, + warning.ExpiresInitial, + warning.Text, + warning.Product, + warning.WFO, + warning.Action, + current, + warning.Class, + warning.Phenomena, + warning.Significance, + warning.EventNumber, + warning.Year, + warning.Title, + warning.IsEmergency, + warning.IsPDS, + warning.Geom, + warning.Direction, + warning.Location, + warning.Speed, + warning.SpeedText, + warning.TMLTime, + warning.UGC, + warning.Tornado, + warning.Damage, + warning.HailThreat, + warning.HailTag, + warning.WindThreat, + warning.WindTag, + warning.FlashFlood, + warning.RainfallTag, + warning.FloodTagDam, + warning.SpoutTag, + warning.SnowSquall, + warning.SnowSquallTag, + ) + if err != nil { + return err + } - err = db.InsertWarning(handler.db, warning) - if err != nil { - return err + for rows.Next() { + var id int + if err := rows.Scan(&id); err != nil { + log.Error().Err(err).Msg("failed to scan warning ID from insert") + continue } + warning.ID = id + } + err = handler.handleWarningPublishing(warning) + if err != nil { + log.Error().Err(err).Msg("failed to publish warning") } return nil } -func (handler *vtecHandler) publishWarning(warning *models.Warning) error { - var eventType string +func (handler *vtecHandler) handleWarningPublishing(warning *models.Warning) error { + if handler.rabbit == nil { + log.Warn().Msg("handler missing RabbitMQ channel. Not publishing warning") + return nil + } + + warningId := warning.GenerateID() + _, ok := handler.publishedWarnings[warningId] + + var err error switch warning.Action { - case "NEW", "EXA", "EXB": - eventType = streaming.EventNew case "CAN", "UPG": - eventType = streaming.EventDelete + err = handler.publishWarning(warning, streaming.EventDelete) default: - eventType = streaming.EventUpdate + err = handler.publishWarning(warning, streaming.EventNew) } - data, err := json.Marshal(warning) if err != nil { return err } - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() + if !ok { + handler.publishedWarnings[warningId] = struct{}{} + } + + return nil +} + +func (handler *vtecHandler) publishWarning(warning *models.Warning, eventType string) error { + data, err := warning.MarshalJSON() + if err != nil { + return err + } - return handler.rabbit.PublishWithContext(ctx, + return handler.rabbit.PublishWithContext(context.Background(), streaming.ExchangeLiveName, "warning", false,