commit 68f52727adfa8c36bc0e7a459224366f0b7faa78
Author: clerie <git@clerie.de>
Date:   Wed Apr 8 11:14:45 2020 +0200

    Initial commit

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..0db3bed
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,142 @@
+# dwd-scraper custom
+scraper/config/db.py
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..6708866
--- /dev/null
+++ b/README.md
@@ -0,0 +1,18 @@
+# DWD Scraper
+Importiert Messdaten aus dem OpenData Programm des DWD in eine Datenbank.
+
+## Datenbank
+Als Datenbank wird PostgreSQL verwendet und das Datenbankschema liegt unter `sql/wetter.sql`.
+
+Zugangsdaten für die Datenbank müssen unter `scraper/config/db.py` abgelegt werden, das entsprechende Format ist unter `scraper/config/db.py.example` hinterlegt.
+
+## Scraper
+Als erstes müssen die Stationen importiert werden.
+```
+./scraper/daily_import_stations.py
+```
+
+Anschließend können darauf die Messergebnisse für die importierten Stationen eingelesen werden.
+```
+./scraper/daily_import_climate.py
+```
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..37ec460
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1 @@
+psycopg2-binary
diff --git a/scraper/config/__init__.py b/scraper/config/__init__.py
new file mode 100644
index 0000000..e5a0d9b
--- /dev/null
+++ b/scraper/config/__init__.py
@@ -0,0 +1 @@
+#!/usr/bin/env python3
diff --git a/scraper/config/db.py.example b/scraper/config/db.py.example
new file mode 100644
index 0000000..ee98e5b
--- /dev/null
+++ b/scraper/config/db.py.example
@@ -0,0 +1,4 @@
+#!/usr/bin/env python3
+
+db = {}
+db['uri'] = "postgresql://user:password@host:5432/database"
diff --git a/scraper/daily_import_climate.py b/scraper/daily_import_climate.py
new file mode 100755
index 0000000..17e516d
--- /dev/null
+++ b/scraper/daily_import_climate.py
@@ -0,0 +1,110 @@
+#! /usr/bin/env python3
+
+import psycopg2
+import requests
+import io
+import zipfile
+import datetime
+from config.db import db as config_db
+
+def cleanup_value(v):
+    if int(v) == -999:
+        return None
+
+    return v
+
+conn = psycopg2.connect(config_db["uri"])
+cur = conn.cursor()
+
+cur.execute("SELECT min(dwd_last_update) FROM stations;")
+last_date = cur.fetchall()[0][0]
+print(last_date)
+
+if not last_date is None:
+    cur.execute("SELECT id, name, lat, lon, dwd_id, dwd_last_update FROM stations WHERE dwd_last_update = %s LIMIT 1;", [last_date])
+else:
+    cur.execute("SELECT id, name, lat, lon, dwd_id, dwd_last_update FROM stations WHERE dwd_last_update IS NULL LIMIT 1;")
+last_station = cur.fetchone()
+print(last_station)
+
+curr_station_id = last_station[0]
+curr_station_dwd_id = last_station[4]
+
+print(curr_station_dwd_id)
+r = requests.get('https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/daily/kl/recent/tageswerte_KL_' + str(curr_station_dwd_id) + '_akt.zip', stream=True)
+print(r.url)
+if r.status_code == 200:
+    zip = zipfile.ZipFile(io.BytesIO(r.content))
+    files = zip.namelist()
+    print(files)
+    files_climate = [f for f in files if f.startswith("produkt_klima_")]
+    print(files_climate)
+
+
+    buffer = []
+
+    is_first_line = True
+
+    for line in zip.open(files_climate[0]):
+        l = line.decode('utf-8').strip()
+        if is_first_line:
+            is_first_line = False
+            continue
+
+        b = {}
+
+        b["dwd_id"], l = l.strip().split(";", 1)
+        b["dwd_id"] = str(b["dwd_id"])
+        b["date"], l = l.strip().split(";", 1)
+        b["date"] = str(b["date"])
+        b["date"] = datetime.date(int(b["date"][0:4]), int(b["date"][4:6]), int(b["date"][6:8]))
+        b["qn_3"], l = l.strip().split(";", 1)
+        b["qn_3"] = cleanup_value(int(b["qn_3"]))
+        b["fx"], l = l.strip().split(";", 1)
+        b["fx"] = cleanup_value(float(b["fx"]))
+        b["fm"], l = l.strip().split(";", 1)
+        b["fm"] = cleanup_value(float(b["fm"]))
+        b["qn_4"], l = l.strip().split(";", 1)
+        b["qn_4"] = cleanup_value(int(b["qn_4"]))
+        b["rsk"], l = l.strip().split(";", 1)
+        b["rsk"] = cleanup_value(float(b["rsk"]))
+        b["rskf"], l = l.strip().split(";", 1)
+        b["rskf"] = cleanup_value(int(b["rskf"]))
+        b["sdk"], l = l.strip().split(";", 1)
+        b["sdk"] = cleanup_value(float(b["sdk"]))
+        b["shk_tag"], l = l.strip().split(";", 1)
+        b["shk_tag"] = cleanup_value(float(b["shk_tag"]))
+        b["nm"], l = l.strip().split(";", 1)
+        b["nm"] = cleanup_value(float(b["nm"]))
+        b["vpm"], l = l.strip().split(";", 1)
+        b["vpm"] = cleanup_value(float(b["vpm"]))
+        b["pm"], l = l.strip().split(";", 1)
+        b["pm"] = cleanup_value(float(b["pm"]))
+        b["tmk"], l = l.strip().split(";", 1)
+        b["tmk"] = cleanup_value(float(b["tmk"]))
+        b["upm"], l = l.strip().split(";", 1)
+        b["upm"] = cleanup_value(float(b["upm"]))
+        b["txk"], l = l.strip().split(";", 1)
+        b["txk"] = cleanup_value(float(b["txk"]))
+        b["tnk"], l = l.strip().split(";", 1)
+        b["tnk"] = cleanup_value(float(b["tnk"]))
+        b["tgk"], l = l.strip().split(";", 1)
+        b["tgk"] = cleanup_value(float(b["tgk"]))
+
+        #print(b)
+        print(curr_station_id, b["date"].isoformat())
+
+        cur.execute("SELECT id FROM climate WHERE station = %s AND date = %s; ", [curr_station_id, b["date"].isoformat()])
+        if cur.rowcount == 0:
+            cur.execute("INSERT INTO climate (station, date, qn_3, fx, fm, qn_4, rsk, rskf, sdk, shk_tag, nm, vpm, pm, tmk, upm, txk, tnk, tgk) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);",
+            [curr_station_id, b["date"].isoformat(), b["qn_3"], b["fx"], b["fm"], b["qn_4"], b["rsk"], b["rskf"], b["sdk"], b["shk_tag"], b["nm"], b["vpm"], b["pm"], b["tmk"], b["upm"], b["txk"], b["tnk"], b["tgk"]])
+            print("imported")
+        else:
+            print("ignored")
+
+cur.execute("UPDATE stations SET dwd_last_update = %s WHERE id = %s;", [datetime.datetime.today().isoformat(), curr_station_id])
+conn.commit()
+
+
+cur.close()
+conn.close()
diff --git a/scraper/daily_import_stations.py b/scraper/daily_import_stations.py
new file mode 100755
index 0000000..4131f91
--- /dev/null
+++ b/scraper/daily_import_stations.py
@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+
+
+import requests
+import io
+import psycopg2
+from config.db import db as config_db
+
+r = requests.get('https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/daily/kl/recent/KL_Tageswerte_Beschreibung_Stationen.txt', stream=True)
+
+print(r)
+
+if r.status_code == 200:
+
+    f = r.iter_lines(decode_unicode=True)
+
+    buffer = []
+
+    is_first_line = True
+    is_second_line = True
+
+    for l in f:
+        if l:
+            if is_first_line:
+                is_first_line = False
+                continue
+            if is_second_line:
+                is_second_line = False
+                continue
+
+            l = str(l)
+
+            b = {}
+
+            print(l)
+
+            b["station_id"], l = l.strip().split(" ", 1)
+            b["station_id"] = str(b["station_id"])
+            b["date_from"], l = l.strip().split(" ", 1)
+            b["date_from"] = int(b["date_from"])
+            b["date_to"], l = l.strip().split(" ", 1)
+            b["date_to"] = int(b["date_to"])
+            b["sea_level"], l = l.strip().split(" ", 1)
+            b["sea_level"] = int(b["sea_level"])
+            b["lat"], l = l.strip().split(" ", 1)
+            b["lat"] = float(b["lat"])
+            b["lon"], l = l.strip().split(" ", 1)
+            b["lon"] = float(b["lon"])
+            b["name"], l = l.strip().split("  ", 1)
+            b["name"] = str(b["name"])
+            b["state"] = l.strip()
+            b["state"] = str(b["state"])
+
+            if b["date_to"] >= 20200000:
+                """
+                Im OpenData Programm sind auch Stationen enthalten, die nicht mehr betrieben werden. Wir importieren nur Stationen, die in diesem Jahr schon Werte geliefert haben.
+                """
+                print(b["station_id"])
+                print(b["date_from"])
+                print(b["date_to"])
+                print(b["sea_level"])
+                print(b["lat"])
+                print(b["lon"])
+                print(b["name"])
+                print(b["state"])
+                print()
+
+                buffer.append(b)
+            else:
+                print("ignore station " + str(b["station_id"]))
+                print()
+
+    print(buffer)
+
+    conn = psycopg2.connect(config_db['uri'])
+    cur = conn.cursor()
+
+    for b in buffer:
+        cur.execute("SELECT * FROM stations WHERE dwd_id LIKE %s;", [str(b["station_id"])])
+        if cur.rowcount == 0:
+            cur.execute("INSERT INTO stations (name, lat, lon, dwd_id, state, sea_level) VALUES (%s, %s, %s, %s, %s, %s);", [b["name"], b["lat"], b["lon"], b["station_id"], b["state"], b["sea_level"]])
+        else:
+            cur.execute("UPDATE stations SET name = %s, lat = %s, lon = %s, state = %s, sea_level = %s WHERE dwd_id LIKE %s", [b["name"], b["lat"], b["lon"], b["state"], b["sea_level"], b["station_id"]])
+
+        conn.commit()
+
+    cur.close()
+    conn.close()
diff --git a/sql/wetter.sql b/sql/wetter.sql
new file mode 100644
index 0000000..cd32788
--- /dev/null
+++ b/sql/wetter.sql
@@ -0,0 +1,163 @@
+--                                                                                          [161/267]
+-- PostgreSQL database dump
+--
+
+-- Dumped from database version 11.7 (Debian 11.7-0+deb10u1)
+-- Dumped by pg_dump version 11.7 (Debian 11.7-0+deb10u1)
+
+SET statement_timeout = 0;
+SET lock_timeout = 0;
+SET idle_in_transaction_session_timeout = 0;
+SET client_encoding = 'UTF8';
+SET standard_conforming_strings = on;
+SELECT pg_catalog.set_config('search_path', '', false);
+SET check_function_bodies = false;
+SET xmloption = content;
+SET client_min_messages = warning;
+SET row_security = off;
+
+SET default_tablespace = '';
+
+SET default_with_oids = false;
+
+--
+-- Name: climate; Type: TABLE; Schema: public; Owner: wetter
+--
+
+CREATE TABLE public.climate (
+    id bigint NOT NULL,
+    station bigint NOT NULL,
+    date date NOT NULL,
+    qn_3 integer,
+    fx real,
+    fm real,
+    qn_4 integer,
+    rsk real,
+    rskf integer,
+    sdk real,
+    shk_tag real,
+    nm real,
+    vpm real,
+    pm real,
+    tmk real,
+    upm real,
+    txk real,
+    tnk real,
+    tgk real,
+    dwd_last_update timestamp without time zone
+);
+
+
+ALTER TABLE public.climate OWNER TO wetter;
+
+--
+-- Name: climate_id_seq; Type: SEQUENCE; Schema: public; Owner: wetter
+--
+
+CREATE SEQUENCE public.climate_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+ALTER TABLE public.climate_id_seq OWNER TO wetter;
+
+--
+-- Name: climate_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: wetter
+--
+
+ALTER SEQUENCE public.climate_id_seq OWNED BY public.climate.id;
+
+
+--
+-- Name: stations; Type: TABLE; Schema: public; Owner: wetter
+--
+
+CREATE TABLE public.stations (
+    id bigint NOT NULL,
+    name character varying(200) NOT NULL,
+    lat numeric(10,8) NOT NULL,
+    lon numeric(11,8) NOT NULL,
+    dwd_id character varying(5),
+    dwd_last_update timestamp without time zone,
+    state character varying(200),
+    sea_level integer
+);
+
+
+ALTER TABLE public.stations OWNER TO wetter;
+
+--
+-- Name: stations_id_seq; Type: SEQUENCE; Schema: public; Owner: wetter
+--
+
+CREATE SEQUENCE public.stations_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+ALTER TABLE public.stations_id_seq OWNER TO wetter;
+
+--
+-- Name: stations_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: wetter
+--
+
+ALTER SEQUENCE public.stations_id_seq OWNED BY public.stations.id;
+
+
+--
+-- Name: climate id; Type: DEFAULT; Schema: public; Owner: wetter
+--
+
+ALTER TABLE ONLY public.climate ALTER COLUMN id SET DEFAULT nextval('public.climate_id_seq'::regclass
+);
+
+
+--
+-- Name: climate station; Type: DEFAULT; Schema: public; Owner: wetter
+--
+
+ALTER TABLE ONLY public.climate ALTER COLUMN station SET DEFAULT nextval('public.climate_station_seq'
+::regclass);
+
+
+--
+-- Name: stations id; Type: DEFAULT; Schema: public; Owner: wetter
+--
+
+ALTER TABLE ONLY public.stations ALTER COLUMN id SET DEFAULT nextval('public.stations_id_seq'::regcla
+ss);
+
+
+--
+-- Name: climate climate_pkey; Type: CONSTRAINT; Schema: public; Owner: wetter
+--
+
+ALTER TABLE ONLY public.climate
+    ADD CONSTRAINT climate_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: stations stations_pkey; Type: CONSTRAINT; Schema: public; Owner: wetter
+--
+
+ALTER TABLE ONLY public.stations
+    ADD CONSTRAINT stations_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: climate climate_station_fkey; Type: FK CONSTRAINT; Schema: public; Owner: wetter
+--
+
+ALTER TABLE ONLY public.climate
+    ADD CONSTRAINT climate_station_fkey FOREIGN KEY (station) REFERENCES public.stations(id);
+
+
+--
+-- PostgreSQL database dump complete
+--