A $20-energy meter for your home

Log and Transfer Data

Data logging is becoming important and critical as we deals with data intensively. With ESP8266/ESP32 chips, transfering data wirelessly had become affordable and acessible. The simple option for a server is Raspberry Pi (RPi) ($35-$40).

Here are the few links to setup RPi for MQTT server:

The next step is to create a script to capture MQTT messages sent from ESP8266. The structure of main script is similar to this:

#! /usr/bin/python3
# Binh Nguyen, Mar 2021
# listen to MQTT server for logger, 

import time
import json
import os
import paho.mqtt.client as mqtt

import file_log

os.environ['TZ'] = 'Asia/Ho_Chi_Minh'

MQTT_PW = 'mqtt_pass'
MQTT_USER = 'mqtt_user'
MQTT_TOPIC = [('sensors/#',0),]

def takeTime():
    return time.strftime("%Y-%m-%d %H:%M:%S")

def on_connect(client, userdata, flags, rc):
    print("Connected with result code "+str(rc))
    return None

def on_message(client, userdata, msg):
    '''Main function
    INPUT: data client
    data is saved to a file

    payload = msg.payload.decode('UTF-8').lower()
    if msg.retain == 0:
            data = json.loads(payload)
            sensor = data['sensor']
            # if message does not have a timestamp
            if 'time' not in data.keys():
                data['time'] = takeTime()

            #  send to a log file
            file_log.to_log(data, json_=False)
        except Exception as e:
            print(f'Exception {e} - not json')          
        print("Retained MSG: ", payload)
    return None

def on_disconnect(client, userdata, rc):
    if rc !=0:
        print("Unexpected disconnection!")
    return None

# Program starts here
client = mqtt.Client(client_id='json-logger')
client.username_pw_set(username=MQTT_USER, password=MQTT_PW)
client.connect(BROKER_IP, 1883, 60)
client.on_connect = on_connect
client.on_message = on_message
client.on_disconnect = on_disconnect

We can put the listening script and logging script in one big file; however, factory the script is a better engineering. It allows clean and structural such adding other option for logging data.

Below are the script to actually log data to JSON or CSV format.

# Binh Nguyen, Dec 2020
# log data to file

import os
import csv
import time
import json

def host_folder(sensor, json_=False):
    """designate a folder to save each month"""

    this_month_folder = time.strftime('%b%Y')
    basedir = os.path.abspath(os.path.dirname(__file__))
    all_dirs = [d for d in os.listdir(basedir) if os.path.isdir(d)]
    if len(all_dirs) == 0 or this_month_folder not in all_dirs:
        print('created: {}'.format(this_month_folder))
    if json:
        return os.path.join(basedir, this_month_folder, f'{sensor}.txt')
    return os.path.join(basedir, this_month_folder, f'{sensor}.csv')

def to_log(data, json_):
    '''write a json stream to a CSV format'''

    sensor = data['sensor']
    fname = host_folder(sensor, json_)
    file_existed = os.path.isfile(fname)
    headers = list(data.keys())

    if json:
        fname = host_folder(data['sensor'], json_=json_)
        with open(host_folder(sensor), 'a') as f:
        with open(fname, 'a') as csvfile:
            writer = csv.DictWriter(
                csvfile, delimiter=',', lineterminator='\n',fieldnames=headers)
            if not file_existed:

    return None

if __name__ == '__main__':
    print('Testing area')