From 1dbc32c104537354cc2ec824cb7ce92b4b5d2cb1 Mon Sep 17 00:00:00 2001 From: Anson Date: Mon, 17 Sep 2018 00:12:11 +0100 Subject: [PATCH 1/3] Conver DB get to use a filter instead --- noaa.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/noaa.py b/noaa.py index a3ca99f..68197d8 100644 --- a/noaa.py +++ b/noaa.py @@ -50,11 +50,12 @@ def scrape_noaa(geolocator, database, db_query): print("[NOAA] Fetching list of stations") station_page = requests.get(STATION_LIST_URL) stations = [] + all_db = database.all() for match in re.finditer(STATION_LISTING_PATTERN, station_page.text): stat_id = match[1] stat_name = html.unescape(match[2]) - search = database.get(db_query.station.id_ == stat_id) - if search is None: + search = list(filter(lambda station: station['station'] == stat_id, all_db)) + if not search: station_info = requests.get(STATION_INFO_URL_FORMAT.format(stat_id)) # Get station latitude @@ -75,7 +76,7 @@ def scrape_noaa(geolocator, database, db_query): stations.append(station_object) database.insert({'station': station_object.to_dict()}) else: - stations.append(Station.from_dict(search['station'])) + stations.append(Station.from_dict(search[0]['station'])) return StationGlobe(stations, geolocator) def closest_station_coords(self, latitude, longitude): From 740d58a79f5f40f2afa8bfb7491c2970b63b58dc Mon Sep 17 00:00:00 2001 From: Anson Date: Mon, 17 Sep 2018 20:10:33 +0100 Subject: [PATCH 2/3] Fix filter function --- noaa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noaa.py b/noaa.py index 68197d8..2df1873 100644 --- a/noaa.py +++ b/noaa.py @@ -54,7 +54,7 @@ def scrape_noaa(geolocator, database, db_query): for match in re.finditer(STATION_LISTING_PATTERN, station_page.text): stat_id = match[1] stat_name = html.unescape(match[2]) - search = list(filter(lambda station: station['station'] == stat_id, all_db)) + search = list(filter(lambda station: station['station']['id_'] == stat_id, all_db)) if not search: station_info = requests.get(STATION_INFO_URL_FORMAT.format(stat_id)) From 4cb674c07140c86b31a5c6df3ceebe178be7fa7d Mon Sep 17 00:00:00 2001 From: Anson Date: Mon, 17 Sep 2018 20:14:02 +0100 Subject: [PATCH 3/3] Remove startup print statement --- noaa.py | 1 - 1 file changed, 1 deletion(-) diff --git a/noaa.py b/noaa.py index 2df1873..c964605 100644 --- a/noaa.py +++ b/noaa.py @@ -167,7 +167,6 @@ class NOAA(BotModule): def __init__(self): super().__init__() - print("Initializing NOAA bot module") self.station_globe = StationGlobe.scrape_noaa(geocoders.Nominatim(user_agent='scubot'), self.module_db, Query())