A $20-energy meter for your home

Log and Transfer Data

Data logging is becoming important and critical as we deals with data intensively. With ESP8266/ESP32 chips, transfering data wirelessly had become affordable and acessible. The simple option for a server is Raspberry Pi (RPi) ($35-$40).

Here are the few links to setup RPi for MQTT server:

The next step is to create a script to capture MQTT messages sent from ESP8266. The structure of main script is similar to this:

#! /usr/bin/python3
# Binh Nguyen, Mar 2021
# listen to MQTT server for logger, 

import time
import json
import os
import paho.mqtt.client as mqtt

import file_log

# MQTT
os.environ['TZ'] = 'Asia/Ho_Chi_Minh'
time.tzset()

BROKER_IP = '192.168.1.21'
MQTT_PW = 'mqtt_pass'
MQTT_USER = 'mqtt_user'
MQTT_TOPIC = [('sensors/#',0),]

def takeTime():
    return time.strftime("%Y-%m-%d %H:%M:%S")


def on_connect(client, userdata, flags, rc):
    client.subscribe(MQTT_TOPIC)
    print("Connected with result code "+str(rc))
    return None


def on_message(client, userdata, msg):
    '''Main function
    INPUT: data client
    data is saved to a file
    '''

    payload = msg.payload.decode('UTF-8').lower()
    if msg.retain == 0:
        try:
            data = json.loads(payload)
            sensor = data['sensor']
            
            # if message does not have a timestamp
            if 'time' not in data.keys():
                data['time'] = takeTime()

            #  send to a log file
            file_log.to_log(data, json_=False)
                
        except Exception as e:
            print(f'Exception {e} - not json')          
    else:
        print("Retained MSG: ", payload)
    return None

def on_disconnect(client, userdata, rc):
    if rc !=0:
        print("Unexpected disconnection!")
    else:
        print("Disconnecting")
    return None

# Program starts here
client = mqtt.Client(client_id='json-logger')
client.username_pw_set(username=MQTT_USER, password=MQTT_PW)
client.connect(BROKER_IP, 1883, 60)
client.on_connect = on_connect
client.on_message = on_message
client.on_disconnect = on_disconnect
time.sleep(0.1)
client.loop_forever()

We can put the listening script and logging script in one big file; however, factory the script is a better engineering. It allows clean and structural such adding other option for logging data.

Below are the script to actually log data to JSON or CSV format.

# Binh Nguyen, Dec 2020
# log data to file

import os
import csv
import time
import json

def host_folder(sensor, json_=False):
    """designate a folder to save each month"""

    this_month_folder = time.strftime('%b%Y')
    basedir = os.path.abspath(os.path.dirname(__file__))
    all_dirs = [d for d in os.listdir(basedir) if os.path.isdir(d)]
    if len(all_dirs) == 0 or this_month_folder not in all_dirs:
        os.makedirs(this_month_folder)
        print('created: {}'.format(this_month_folder))
    if json:
        return os.path.join(basedir, this_month_folder, f'{sensor}.txt')
    return os.path.join(basedir, this_month_folder, f'{sensor}.csv')

def to_log(data, json_):
    '''write a json stream to a CSV format'''

    sensor = data['sensor']
    fname = host_folder(sensor, json_)
    file_existed = os.path.isfile(fname)
    headers = list(data.keys())

    if json:
        fname = host_folder(data['sensor'], json_=json_)
        with open(host_folder(sensor), 'a') as f:
            f.write(json.dumps(data))
            f.write('\n')
    else:
        with open(fname, 'a') as csvfile:
            writer = csv.DictWriter(
                csvfile, delimiter=',', lineterminator='\n',fieldnames=headers)
            if not file_existed:
                writer.writeheader()
            writer.writerow(data)

    return None


if __name__ == '__main__':
    print('Testing area')