values get stored into postgres now

README.md

@@ -15,7 +15,8 @@ Connected to that is:
 * BME280: Pressure Humidity Temperature Sensor Module.
 
 ## backend
-A [flask](https://pypi.org/project/Flask/) based application that takes the data from the ESP8266 for processing and storage.
+A [flask](https://pypi.org/project/Flask/) based application that takes the data from the ESP8266 for processing and storage.  
+[Postgres](https://www.postgresql.org/) handles the storage of the measurements. The data is split up into two different tables, one for aqi related data and one for the weather data.
 
 ## frontend
 A simple web site that gets AQI values from the backend to publish it nicely to see. Mostly build with HTML/CSS/JS and a little bit of PHP.

backend/flask/app/db_insert.py

@@ -0,0 +1,82 @@
+""" handles insert into postgres db """
+
+import psycopg2
+import traceback
+
+
+def db_connect(config, data):
+    """ handles the data insert """
+    # set config
+    db_host = config['db_host']
+    db_database = config['db_database']
+    db_user = config['db_user']
+    db_password = config['db_password']
+
+    # Connect to database
+    conn = psycopg2.connect(
+        host = db_host,
+        database = db_database,
+        user = db_user,
+        password = db_password
+    )
+
+    # Open a cursor to perform database operations
+    cur = conn.cursor()
+    
+    # insert
+    time_stamp = db_insert(cur, data)
+    
+    # finish
+    conn.commit()
+    cur.close()
+    conn.close()
+    # print
+    return time_stamp
+
+
+def db_insert(cur, json_dict):
+    """ make the db insert """
+    # read out data dict
+    try:
+        uptime = json_dict['uptime']
+        temperature = json_dict['temperature']
+        pressure = json_dict['pressure']
+        humidity = json_dict['humidity']
+        pm25 = json_dict['pm25']
+        pm10 = json_dict['pm10']
+        aqi_value = json_dict['aqi_value']
+        aqi_category = json_dict['aqi_category']
+        time_stamp = json_dict['timestamp']
+        epoch_time = json_dict['epoch_time']
+        weather_name = json_dict['weather_name']
+        weather_icon = json_dict['weather_icon']
+        wind_speed = json_dict['wind_speed']
+        wind_direction = json_dict['wind_direction']
+    except Exception:
+        print('json_dict parsing error')
+        print(traceback.format_exc())
+        time_stamp = "parse error"
+    try:
+        # insert aqi
+        cur.execute("INSERT INTO aqi \
+            (epoch_time, time_stamp, uptime, pm25, pm10, aqi_value, aqi_category) \
+            VALUES (%s, %s, %s, %s, %s, %s, %s)",
+            (epoch_time, time_stamp, uptime, pm25, pm10, aqi_value, aqi_category))
+    except Exception:
+        print('aqi INSERT error')
+        print(traceback.format_exc())
+        time_stamp = "aqi error"
+    try:
+        # insert weather
+        cur.execute("INSERT INTO weather \
+            (epoch_time, time_stamp, temperature, pressure, humidity, \
+            wind_speed, wind_direction, weather_name, weather_icon) \
+            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)", 
+            (epoch_time, time_stamp, temperature, pressure, humidity, 
+            wind_speed, wind_direction, weather_name, weather_icon)
+        )
+    except Exception:
+        print('weather INSERT error')
+        print(traceback.format_exc())
+        time_stamp = "weather error"
+    return time_stamp

backend/flask/app/views.py

@@ -1,4 +1,5 @@
 import configparser
+import json
 
 from flask_cors import CORS
 from flask import request
@@ -9,6 +10,7 @@ from apscheduler.schedulers.background import BackgroundScheduler
 from app import app
 from app import aqi_parser
 from app import weather
+from app import db_insert
 
 cors = CORS(app, resources={r"/": {"origins": "*"}})
 auth = HTTPBasicAuth()
@@ -26,12 +28,17 @@ def get_config():
     config["api_key"] = config_parser.get('openweathermap', "api_key")
     config["lat"] = config_parser.get('openweathermap', "lat")
     config["lon"] = config_parser.get('openweathermap', "lon")
+    # db
+    config["db_host"] = config_parser.get('postgres', "db_host")
+    config["db_database"] = config_parser.get('postgres', "db_database")
+    config["db_user"] = config_parser.get('postgres', "db_user")
+    config["db_password"] = config_parser.get('postgres', "db_password")
     return config
 
 
+# start up
 config = get_config()
-
-
+weather.handle_weather(config)
 # build username / pw dict for basic auth
 USER_DATA = {}
 USER_DATA[config['authUsername']] = config['authPassword']
@@ -58,7 +65,13 @@ def verify(username, password):
 def ingest():
     data = request.json
     if data:
-        data = aqi_parser.input_process(data)
+        # populate data dict
+        json_dict = aqi_parser.input_process(data)
+        # save to db
+        time_stamp = db_insert.db_connect(config, json_dict)
+        print(f'db insert done at {time_stamp}')
+        # save to webserver
+        data = json.dumps(json_dict)
         with open('dyn/air.json', 'w') as f:
             f.write(data)
         print(data)

backend/flask/config.sample

@@ -5,4 +5,10 @@ authPassword = password
 [openweathermap]
 api_key = xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 lat = 40.71
-lon = -73.99
+lon = -73.99
+
+[postgres]
+db_host = postgres
+db_database = aqi
+db_user = aqi
+db_password = aaaaaaaaaaaaaaaaa

backend/flask/requirements.txt

@@ -3,5 +3,6 @@ Flask
 flask-cors
 Flask-HTTPAuth
 numpy
+psycopg2-binary
 requests
 uWSGI