clerie/dwd-scraper
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Establish sql connection only after data got scraped successfully

Browse Source
This commit is contained in:
clerie 2022-02-28 17:31:10 +01:00
parent fc952670bb
commit 9fb1d81652
1 changed files with 65 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
scraper/daily_import_climate.py
View File

@ -13,6 +13,18 @@ class DwdScraper:
last_station = self.get_station_to_update() last_station = self.get_station_to_update()
print("checking station", last_station) print("checking station", last_station)
print(last_station)
r = requests.get('https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/daily/kl/recent/tageswerte_KL_' + str(last_station) + '_akt.zip', stream=True)
print(r.url)
if r.status_code != 200:
return
zip = zipfile.ZipFile(io.BytesIO(r.content))
files = zip.namelist()
print(files)
files_climate = [f for f in files if f.startswith("produkt_klima_")]
print(files_climate)
with psycopg2.connect(config_db["uri"]) as conn: with psycopg2.connect(config_db["uri"]) as conn:
with conn.cursor() as cur: with conn.cursor() as cur:
@ -22,19 +34,6 @@ class DwdScraper:
curr_station_id = current_station[0] curr_station_id = current_station[0]
curr_station_dwd_id = current_station[4] curr_station_dwd_id = current_station[4]
print(curr_station_dwd_id)
r = requests.get('https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/daily/kl/recent/tageswerte_KL_' + str(curr_station_dwd_id) + '_akt.zip', stream=True)
print(r.url)
if r.status_code == 200:
zip = zipfile.ZipFile(io.BytesIO(r.content))
files = zip.namelist()
print(files)
files_climate = [f for f in files if f.startswith("produkt_klima_")]
print(files_climate)
buffer = []
is_first_line = True is_first_line = True
for line in zip.open(files_climate[0]): for line in zip.open(files_climate[0]):