Stability and other improvements at the lake:

* Changed meta.json to meta.csv to avoid memory issues when parsing the
json once the file gets big.
* Power Chart now automatically refreshes and has a refresh button.
Includes caching so only new data is fetched and added.
* Increased max log files from 48 (~ 2 days) to 170 (~1 week).

Author: camlee

esp32-micropython/Makefile

@@ -1,7 +1,7 @@
 SERIAL_PORT=/dev/ttyS4
 BAUD=115200
 
-.PHONY: rshell upload upload_client upload_server rshell_repl repl static
+.PHONY: rshell upload upload_client upload_server rshell_repl repl static pull_logs backup_data upload_keep
 
 rshell:
 	rshell -p $(SERIAL_PORT)
@@ -12,6 +12,15 @@ upload_server:
 upload_client:
 	rshell -p $(SERIAL_PORT) rsync --mirror dist/ /pyboard/static/
 
+backup_data:
+	rshell -p $(SERIAL_PORT) rsync --mirror /pyboard/static/data/ dist/data/
+
+pull_logs:
+	rshell -p $(SERIAL_PORT) rsync --mirror /pyboard/static/data/ dist/logs/
+
+
+
+upload_keep: static pull_logs backup_data upload_client upload_server
 
 upload: static upload_client upload_server
 

esp32-micropython/client/common/DataProcessing.js

@@ -1,33 +1,39 @@
-export function processLogResponse(response_text, offset){
+export function processLogResponse(response_text, offset, min_time){
   let data = [];
   const records = response_text.split("\n");
   for (const record of records){
     let values = record.split(",");
+    if (values.length < 3){
+      continue;
+    }
     values[0] = Number(values[0]) + offset;
     values[1] = Number(values[1]);
     values[2] = Number(values[2]);
-    data.push(values);
+    if (min_time === undefined || min_time === null || values[0] > min_time){
+      data.push(values);
+    }
   }
   return data;
 }
 
 export function processDataMeta(meta){
   let logs = [];
-  for (const [log_number, log_meta] of Object.entries(meta["logs"])){
+  for (const record of meta.split("\n")){
+    let values = record.split(",");
+    if (record.length < 4){
+      continue;
+    }
+    let start_time_offset = Number(values[3]);
+    if (isNaN(start_time_offset)){
+      start_time_offset = null;
+    }
     logs.push({
-      number: log_number,
-      start_time_offset: log_meta.start_time_offset,
-      start_time: log_meta.start_time,
+      number: Number(values[0]),
+      active: Boolean(Number(values[1])),
+      start_time: Number(values[2]),
+      start_time_offset: start_time_offset,
     })
   }
-
-  // Skip data that doesn't have complete time information:
-  let original_log_file_count = logs.length;
-  logs = logs.filter(function(a){
-    return !(a.start_time_offset === null);
-  });
-  console.log(`Number of log files: total: ${original_log_file_count} valid: ${logs.length}`);
-
   return logs;
 }
 

esp32-micropython/client/components/App.svelte

@@ -52,14 +52,32 @@
     // Number between 0 and 1 to indicate progress.
   let getting_data_error = null;
 
+  let fetched_logs = [];
+
   async function* getData(){
     getting_data_progress = null;
-    let response = await fetch(API_URL + '/data/meta.json');
+    let response = await fetch(API_URL + '/data/meta.csv');
     if (!response.ok){
       throw new Error(response.statusText);
     }
-    const meta = await response.json();
+    const meta = await response.text();
     let logs = processDataMeta(meta);
+    let original_log_file_count = logs.length;
+
+    // Skip data that doesn't have complete time information:
+    logs = logs.filter(function(a){
+      return !(a.start_time_offset === null);
+    });
+
+    let valid_log_file_count = logs.length;
+
+    // Skip data that we already have:
+    logs = logs.filter(function(a){
+      return !fetched_logs.some(function(b){
+        return b.active == false && a.number == b.number && a.start_time_offset == b.start_time_offset && a.start_time == b.start_time;
+      })
+    });
+    console.log(`Number of log files: total: ${original_log_file_count} valid: ${valid_log_file_count} new: ${logs.length}`);
 
     // Load the newest data first as that's most likely to be viewed first:
     logs.sort(function(a, b) {
@@ -82,43 +100,92 @@
           if (new_progress > getting_data_progress){
             getting_data_progress = Math.round(new_progress * 100) / 100;
               // Rounding seems to fix a really weird problem where ther progress bar jumps
-              // around slightly when during chart updates. It also smoothes things out a
+              // around slightly when doing chart updates. It also smoothes things out a
               // bit and makes it look better overall.
           }
         });
 
+      // Seeing if we have downloaded part of this log file before and getting the time where we left off:
+      let previous_logs = fetched_logs.filter(function(a){
+        return a.active && a.number == log.number && a.start_time == log.start_time && a.start_time_offset == log.start_time_offset;
+      });
+      let last_processed_time = null;
+      if (previous_logs.length > 0){
+        let previous_log = previous_logs[previous_logs.length-1];
+        last_processed_time = previous_log.last_processed_time;
+      }
+
       // const response_text = await response.text();
-      yield await processLogResponse(response_text, log.start_time_offset);
+      let data = await processLogResponse(response_text, log.start_time_offset, last_processed_time);
+      if (data.length > 0){
+        log.last_processed_time = data[data.length-1][0];
+        fetched_logs.push(log);
+      }
+      console.log(`New Data points from ${log.number} since ${last_processed_time}: ${data.length}`);
+      yield data;
+    }
+  }
+
+  let historical_fetcher = null;
+  let historical_fetcher_setup = false;
+  let pull_historical_data_period_seconds = 300;
+
+  function setup_historical_fetcher(){
+    if (historical_fetcher_setup === false){
+      historical_fetcher_setup = true;
+      document.addEventListener("visibilitychange", function (){
+        if (historical_fetcher != null){
+          clearInterval(historical_fetcher);
+          historical_fetcher = null;
+        }
+        if (!document.hidden){
+          setup_historical_fetcher();
+        }
+      }, false);
     }
+
+    get_historical_data();
+    historical_fetcher = setInterval(function() {
+      get_historical_data();
+    }, pull_historical_data_period_seconds * 1000);
   }
 
+  let getting_data = false;
+
   async function get_historical_data(){
+    if (getting_data){
+      console.log("Still getting data from last time.");
+      return;
+    }
+    getting_data = true;
     try{
       for await (const data of getData()) {
         historical_data = [...historical_data, ...data];
       }
+      getting_data_error = null;
     } catch(e){
       getting_data_error = e.message;
       console.log(`Error getting data: ${getting_data_error}`);
     }
+    getting_data = false;
   }
 
   let websocket = null;
   let websocket_retry_period = 1;
   let close_websocket_after_tab_changed_for_seconds = 50;
-  let visibilitychange_setup = false;
-  let last_visibility_change_timeout = null;
+  let websocket_visibilitychange_setup = false;
+  let websocket_last_visibility_change_timeout = null;
 
   function setup_realtime_websocket(){
-    if (visibilitychange_setup === false) {
-      visibilitychange_setup = true;
+    if (websocket_visibilitychange_setup === false) {
+      websocket_visibilitychange_setup = true;
       document.addEventListener("visibilitychange", function (){
-        if (last_visibility_change_timeout != null){
-          clearTimeout(last_visibility_change_timeout);
-          last_visibility_change_timeout = null;
+        if (websocket_last_visibility_change_timeout != null){
+          clearTimeout(websocket_last_visibility_change_timeout);
+          websocket_last_visibility_change_timeout = null;
         }
         if (document.hidden){
-          last_visibility_change_timeout = setTimeout(function() {
+          websocket_last_visibility_change_timeout = setTimeout(function() {
             if (websocket !== null){
               websocket.close();
             }
@@ -188,7 +255,7 @@
     }
     loading = false;
     get_stats();
-    get_historical_data();
+    setup_historical_fetcher();
     setup_realtime_websocket();
 
   }
@@ -202,9 +269,18 @@
   <FullPageProgress action={loading_action} action_verb={loading_verb} error={loading_error}/>
 {:else}
   <h1>Power Meter</h1>
+
   <div class="mdc-layout-grid">
     <div class="mdc-layout-grid__inner">
-      <div class="mdc-layout-grid__cell--span-6">
+      <div class="mdc-layout-grid__cell--span-12">
+        <PowerUsage progress={getting_data_progress} data={historical_data} error={getting_data_error} refresh_data={get_historical_data}/>
+      </div>
+    </div>
+  </div>
+
+  <div class="mdc-layout-grid">
+    <div class="mdc-layout-grid__inner">
+      <div class="mdc-layout-grid__cell--span-12">
         <Realtime websocket={websocket} title="Realtime" y_label="Power [W]"
           series={[
             {
@@ -224,11 +300,9 @@
             },
             ]}/>
       </div>
-      <div class="mdc-layout-grid__cell--span-6">
-        <PowerUsage progress={getting_data_progress} data={historical_data} error={getting_data_error}/>
-      </div>
     </div>
   </div>
+
   <div class="mdc-layout-grid">
     <div class="mdc-layout-grid__inner">
       <div class="mdc-layout-grid__cell--span-6">

esp32-micropython/client/components/PowerUsage.svelte

@@ -2,16 +2,23 @@
   import { onMount } from 'svelte';
   import Chip, {Set, Icon, Text} from '@smui/chips';
   import LinearProgress from '@smui/linear-progress';
+  import IconButton from '@smui/icon-button';
+  import Refresh from "svelte-material-icons/Refresh.svelte";
 
   export let data = [];
   export let progress = false;
   export let error = null;
+  export let refresh_data = function(){};
 
   let time_choice = "Today";
 
   let canvas;
   let chart;
 
+  function round_for_graph(value){
+    return Math.round(value * 10) / 10;
+  }
+
   function processData(data, bucketize){
     let buckets = {};
     let panel_in = [];
@@ -41,8 +48,8 @@
 
     for (const key of Object.keys(buckets).sort()){
       let time = new Date(Number(key));
-      surplus.push({x: time, y: Math.round(buckets[key]["avail"] - buckets[key]["in"])}); // Stacked bar chart so only want to show the extra available.
-      let net_bat_val = Math.round(buckets[key]["in"] - buckets[key]["out"]);
+      surplus.push({x: time, y: round_for_graph(buckets[key]["avail"] - buckets[key]["in"])}); // Stacked bar chart so only want to show the extra available.
+      let net_bat_val = round_for_graph(buckets[key]["in"] - buckets[key]["out"]);
       let in_bat_val = 0;
       let out_bat_val = 0;
       if (net_bat_val >= 0){
@@ -52,8 +59,8 @@
       }
       in_bat.push({x: time, y: in_bat_val});
       out_bat.push({x: time, y: out_bat_val});
-      panel_in.push({x: time, y: Math.round(buckets[key]["in"]) - in_bat_val});
-      panel_usage.push({x: time, y: Math.round(-buckets[key]["out"]) - out_bat_val});
+      panel_in.push({x: time, y: round_for_graph(buckets[key]["in"]) - in_bat_val});
+      panel_usage.push({x: time, y: round_for_graph(-buckets[key]["out"]) - out_bat_val});
     }
 
     return [in_bat, out_bat, panel_in, panel_usage, surplus];
@@ -226,6 +233,16 @@
         max_time.setDate(min_time.getDate() + 7);
         return [min_time, max_time];
 
+      default:
+        min_time = new Date();
+        min_time.setHours(0);
+        min_time.setMinutes(0);
+        min_time.setSeconds(0);
+        min_time.setMilliseconds(0);
+
+        max_time = new Date(min_time);
+        max_time.setDate(min_time.getDate() + 1);
+        return [min_time, max_time];
     }
   }
 
@@ -248,10 +265,14 @@
   onMount(main);
 
 </script>
-<h2>{time_choice}{time_choice.endsWith("s") ? "'" : "'s"} Power Usage</h2>
-<Set chips={['Last 7 days', 'Last 3 days', 'Yesterday', 'Today']} let:chip choice bind:selected={time_choice}>
-  <Chip tabindex="0">{chip}</Chip>
-</Set>
+<h2>{(time_choice || "Today")}{(time_choice || "Today").endsWith("s") ? "'" : "'s"} Power Usage</h2>
+<div style="display:flex;">
+  <Set chips={['Last 7 days', 'Last 3 days', 'Yesterday', 'Today']} let:chip choice bind:selected={time_choice}>
+    <Chip tabindex="0">{chip}</Chip>
+  </Set>
+  <!-- <IconButton class="material-icons" on:click={refresh_data}>refresh</IconButton> -->
+  <IconButton style="margin-left:auto" on:click={refresh_data()}><Refresh/></IconButton>
+</div>
 <canvas bind:this={canvas}/>
 
 <LinearProgress

esp32-micropython/package-lock.json

@@ -21,6 +21,7 @@
     "parcel-plugin-svelte": "^4.0.6",
     "sass": "^1.26.7",
     "svelte": "^3.23.0",
+    "svelte-material-icons": "^1.0.3",
     "svelte-material-ui": "^1.0.0-beta.21"
   },
   "browserslist": [

esp32-micropython/package.json

@@ -8,5 +8,5 @@
     pass # Directory already exists
 
 def log_exception(msg):
-    with open("%s/%s" % (log_directory, "exception.log"), "a") as f:
+    with open("%s/%s" % (log_directory, "exception.txt"), "a") as f:
         f.write(msg)

esp32-micropython/server/logger.py

@@ -220,6 +220,7 @@ def ws_close_callback(ws):
 mws.StaticCacheByPath = [
     ("static/data/", 0),
     ("static/index.html", 0),
+    ("static/logs/", 0),
     ("static/", 2),
 ]
 mws.StaticHeaders = {"Access-Control-Allow-Origin": "*"}
@@ -286,8 +287,9 @@ def check_for_reboot():
     #     disp.fill_rect(0, 40, 128, 128, 0)
     #     disp.text("Connected: %s" % clients, 0, 40)
     #     disp.show()
+
     except Exception as e:
-        err = "Unhandled exception in main loop: %s" % e
+        err = "Unhandled exception in main loop: %s\n" % e
         print(err)
         log_exception(err)
         raise