clerie/dwd-scraper
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Move decision which station is next to a function

Browse Source
This commit is contained in:
clerie 2022-02-28 17:17:10 +01:00
parent 6d3b85373e
commit 88e67bcd3e
1 changed files with 25 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
scraper/daily_import_climate.py
View File

@ -10,24 +10,17 @@ from config.db import db as config_db
class DwdScraper: class DwdScraper:
def run(self): def run(self):
last_station = self.get_station_to_update()
print("checking station", last_station)
with psycopg2.connect(config_db["uri"]) as conn: with psycopg2.connect(config_db["uri"]) as conn:
with conn.cursor() as cur: with conn.cursor() as cur:
cur.execute("SELECT min(dwd_last_update) FROM stations;") cur.execute("SELECT id, name, lat, lon, dwd_id, dwd_last_update FROM stations WHERE dwd_id = %s;", [last_station])
last_date = cur.fetchall()[0][0] current_station = cur.fetchone()
print(last_date)
if not last_date is None: curr_station_id = current_station[0]
cur.execute("SELECT id, name, lat, lon, dwd_id, dwd_last_update FROM stations WHERE dwd_last_update = %s LIMIT 1;", [last_date]) curr_station_dwd_id = current_station[4]
else:
cur.execute("SELECT id, name, lat, lon, dwd_id, dwd_last_update FROM stations WHERE dwd_last_update IS NULL LIMIT 1;")
last_station = cur.fetchone()
print(last_station)
curr_station_id = last_station[0]
curr_station_dwd_id = last_station[4]
print(curr_station_dwd_id) print(curr_station_dwd_id)
r = requests.get('https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/daily/kl/recent/tageswerte_KL_' + str(curr_station_dwd_id) + '_akt.zip', stream=True) r = requests.get('https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/daily/kl/recent/tageswerte_KL_' + str(curr_station_dwd_id) + '_akt.zip', stream=True)
@ -104,6 +97,25 @@ class DwdScraper:
cur.execute("UPDATE stations SET dwd_last_update = %s WHERE id = %s;", [datetime.datetime.today().isoformat(), curr_station_id]) cur.execute("UPDATE stations SET dwd_last_update = %s WHERE id = %s;", [datetime.datetime.today().isoformat(), curr_station_id])
conn.commit() conn.commit()
def get_station_to_update(self):
"""
Returns DWD ID of station that should get updated soon.
"""
with psycopg2.connect(config_db["uri"]) as conn:
with conn.cursor() as cur:
cur.execute("SELECT min(dwd_last_update) FROM stations;")
last_date = cur.fetchall()[0][0]
if not last_date is None:
cur.execute("SELECT id, name, lat, lon, dwd_id, dwd_last_update FROM stations WHERE dwd_last_update = %s LIMIT 1;", [last_date])
else:
cur.execute("SELECT id, name, lat, lon, dwd_id, dwd_last_update FROM stations WHERE dwd_last_update IS NULL LIMIT 1;")
last_station = cur.fetchone()
return last_station[4]
def cleanup_value(self, v): def cleanup_value(self, v):
if int(v) == -999: if int(v) == -999:
return None return None