clerie/dwd-scraper
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Move cleanup function to class

Browse Source
This commit is contained in:
clerie 2022-02-28 17:06:52 +01:00
parent bf638d84f3
commit 6d3b85373e
1 changed files with 21 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
scraper/daily_import_climate.py
View File

@ -10,11 +10,7 @@ from config.db import db as config_db
class DwdScraper: class DwdScraper:
def run(self): def run(self):
def cleanup_value(v):
if int(v) == -999:
return None
return v
with psycopg2.connect(config_db["uri"]) as conn: with psycopg2.connect(config_db["uri"]) as conn:
with conn.cursor() as cur: with conn.cursor() as cur:
@ -62,37 +58,37 @@ class DwdScraper:
b["date"] = str(b["date"]) b["date"] = str(b["date"])
b["date"] = datetime.date(int(b["date"][0:4]), int(b["date"][4:6]), int(b["date"][6:8])) b["date"] = datetime.date(int(b["date"][0:4]), int(b["date"][4:6]), int(b["date"][6:8]))
b["qn_3"], l = l.strip().split(";", 1) b["qn_3"], l = l.strip().split(";", 1)
b["qn_3"] = cleanup_value(int(b["qn_3"])) b["qn_3"] = self.cleanup_value(int(b["qn_3"]))
b["fx"], l = l.strip().split(";", 1) b["fx"], l = l.strip().split(";", 1)
b["fx"] = cleanup_value(float(b["fx"])) b["fx"] = self.cleanup_value(float(b["fx"]))
b["fm"], l = l.strip().split(";", 1) b["fm"], l = l.strip().split(";", 1)
b["fm"] = cleanup_value(float(b["fm"])) b["fm"] = self.cleanup_value(float(b["fm"]))
b["qn_4"], l = l.strip().split(";", 1) b["qn_4"], l = l.strip().split(";", 1)
b["qn_4"] = cleanup_value(int(b["qn_4"])) b["qn_4"] = self.cleanup_value(int(b["qn_4"]))
b["rsk"], l = l.strip().split(";", 1) b["rsk"], l = l.strip().split(";", 1)
b["rsk"] = cleanup_value(float(b["rsk"])) b["rsk"] = self.cleanup_value(float(b["rsk"]))
b["rskf"], l = l.strip().split(";", 1) b["rskf"], l = l.strip().split(";", 1)
b["rskf"] = cleanup_value(int(b["rskf"])) b["rskf"] = self.cleanup_value(int(b["rskf"]))
b["sdk"], l = l.strip().split(";", 1) b["sdk"], l = l.strip().split(";", 1)
b["sdk"] = cleanup_value(float(b["sdk"])) b["sdk"] = self.cleanup_value(float(b["sdk"]))
b["shk_tag"], l = l.strip().split(";", 1) b["shk_tag"], l = l.strip().split(";", 1)
b["shk_tag"] = cleanup_value(float(b["shk_tag"])) b["shk_tag"] = self.cleanup_value(float(b["shk_tag"]))
b["nm"], l = l.strip().split(";", 1) b["nm"], l = l.strip().split(";", 1)
b["nm"] = cleanup_value(float(b["nm"])) b["nm"] = self.cleanup_value(float(b["nm"]))
b["vpm"], l = l.strip().split(";", 1) b["vpm"], l = l.strip().split(";", 1)
b["vpm"] = cleanup_value(float(b["vpm"])) b["vpm"] = self.cleanup_value(float(b["vpm"]))
b["pm"], l = l.strip().split(";", 1) b["pm"], l = l.strip().split(";", 1)
b["pm"] = cleanup_value(float(b["pm"])) b["pm"] = self.cleanup_value(float(b["pm"]))
b["tmk"], l = l.strip().split(";", 1) b["tmk"], l = l.strip().split(";", 1)
b["tmk"] = cleanup_value(float(b["tmk"])) b["tmk"] = self.cleanup_value(float(b["tmk"]))
b["upm"], l = l.strip().split(";", 1) b["upm"], l = l.strip().split(";", 1)
b["upm"] = cleanup_value(float(b["upm"])) b["upm"] = self.cleanup_value(float(b["upm"]))
b["txk"], l = l.strip().split(";", 1) b["txk"], l = l.strip().split(";", 1)
b["txk"] = cleanup_value(float(b["txk"])) b["txk"] = self.cleanup_value(float(b["txk"]))
b["tnk"], l = l.strip().split(";", 1) b["tnk"], l = l.strip().split(";", 1)
b["tnk"] = cleanup_value(float(b["tnk"])) b["tnk"] = self.cleanup_value(float(b["tnk"]))
b["tgk"], l = l.strip().split(";", 1) b["tgk"], l = l.strip().split(";", 1)
b["tgk"] = cleanup_value(float(b["tgk"])) b["tgk"] = self.cleanup_value(float(b["tgk"]))
#print(b) #print(b)
print(curr_station_id, b["date"].isoformat()) print(curr_station_id, b["date"].isoformat())
@ -108,6 +104,11 @@ class DwdScraper:
cur.execute("UPDATE stations SET dwd_last_update = %s WHERE id = %s;", [datetime.datetime.today().isoformat(), curr_station_id]) cur.execute("UPDATE stations SET dwd_last_update = %s WHERE id = %s;", [datetime.datetime.today().isoformat(), curr_station_id])
conn.commit() conn.commit()
def cleanup_value(self, v):
if int(v) == -999:
return None
return v
if __name__ == "__main__": if __name__ == "__main__":
DwdScraper().run() DwdScraper().run()