diff --git a/scraper/daily_import_climate.py b/scraper/daily_import_climate.py index 99b7cc8..57f6830 100755 --- a/scraper/daily_import_climate.py +++ b/scraper/daily_import_climate.py @@ -10,11 +10,7 @@ from config.db import db as config_db class DwdScraper: def run(self): - def cleanup_value(v): - if int(v) == -999: - return None - return v with psycopg2.connect(config_db["uri"]) as conn: with conn.cursor() as cur: @@ -62,37 +58,37 @@ class DwdScraper: b["date"] = str(b["date"]) b["date"] = datetime.date(int(b["date"][0:4]), int(b["date"][4:6]), int(b["date"][6:8])) b["qn_3"], l = l.strip().split(";", 1) - b["qn_3"] = cleanup_value(int(b["qn_3"])) + b["qn_3"] = self.cleanup_value(int(b["qn_3"])) b["fx"], l = l.strip().split(";", 1) - b["fx"] = cleanup_value(float(b["fx"])) + b["fx"] = self.cleanup_value(float(b["fx"])) b["fm"], l = l.strip().split(";", 1) - b["fm"] = cleanup_value(float(b["fm"])) + b["fm"] = self.cleanup_value(float(b["fm"])) b["qn_4"], l = l.strip().split(";", 1) - b["qn_4"] = cleanup_value(int(b["qn_4"])) + b["qn_4"] = self.cleanup_value(int(b["qn_4"])) b["rsk"], l = l.strip().split(";", 1) - b["rsk"] = cleanup_value(float(b["rsk"])) + b["rsk"] = self.cleanup_value(float(b["rsk"])) b["rskf"], l = l.strip().split(";", 1) - b["rskf"] = cleanup_value(int(b["rskf"])) + b["rskf"] = self.cleanup_value(int(b["rskf"])) b["sdk"], l = l.strip().split(";", 1) - b["sdk"] = cleanup_value(float(b["sdk"])) + b["sdk"] = self.cleanup_value(float(b["sdk"])) b["shk_tag"], l = l.strip().split(";", 1) - b["shk_tag"] = cleanup_value(float(b["shk_tag"])) + b["shk_tag"] = self.cleanup_value(float(b["shk_tag"])) b["nm"], l = l.strip().split(";", 1) - b["nm"] = cleanup_value(float(b["nm"])) + b["nm"] = self.cleanup_value(float(b["nm"])) b["vpm"], l = l.strip().split(";", 1) - b["vpm"] = cleanup_value(float(b["vpm"])) + b["vpm"] = self.cleanup_value(float(b["vpm"])) b["pm"], l = l.strip().split(";", 1) - b["pm"] = cleanup_value(float(b["pm"])) + b["pm"] = self.cleanup_value(float(b["pm"])) b["tmk"], l = l.strip().split(";", 1) - b["tmk"] = cleanup_value(float(b["tmk"])) + b["tmk"] = self.cleanup_value(float(b["tmk"])) b["upm"], l = l.strip().split(";", 1) - b["upm"] = cleanup_value(float(b["upm"])) + b["upm"] = self.cleanup_value(float(b["upm"])) b["txk"], l = l.strip().split(";", 1) - b["txk"] = cleanup_value(float(b["txk"])) + b["txk"] = self.cleanup_value(float(b["txk"])) b["tnk"], l = l.strip().split(";", 1) - b["tnk"] = cleanup_value(float(b["tnk"])) + b["tnk"] = self.cleanup_value(float(b["tnk"])) b["tgk"], l = l.strip().split(";", 1) - b["tgk"] = cleanup_value(float(b["tgk"])) + b["tgk"] = self.cleanup_value(float(b["tgk"])) #print(b) print(curr_station_id, b["date"].isoformat()) @@ -108,6 +104,11 @@ class DwdScraper: cur.execute("UPDATE stations SET dwd_last_update = %s WHERE id = %s;", [datetime.datetime.today().isoformat(), curr_station_id]) conn.commit() + def cleanup_value(self, v): + if int(v) == -999: + return None + + return v if __name__ == "__main__": DwdScraper().run()