Update to version 2.1.1

Bl4DEx · Bl4DEx · commit 82a8eeeff681 · 2023-06-25T22:42:39.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,10 @@
 # CHANGELOG.md
 
+## 2.1.1
+* Fixed a bug where --list did not list all entities with has_sum=1
+* Added *autocommit=True* for MySQL server connection
+* No longer round values but handle all values as they are stored in the database
+
 ## 2.1.0
 * Add support for a MySQL server using the package PyMySQL
 * Add support for entities that are no Riemann Sum Entities
diff --git a/HA_FixNegativeStatistics.py b/HA_FixNegativeStatistics.py
@@ -12,10 +12,11 @@
 import sys
 import shutil
 import sqlite3
+from decimal import Decimal, InvalidOperation
 from datetime import datetime
 
 __author__ = "Sebastian Hollas"
-__version__ = "2.1.0"
+__version__ = "2.1.1"
 
 ####################################################################################
 # USER INPUT REQUIRED !
@@ -35,27 +36,19 @@
 # Build Filepaths
 ENTITIES_FILE = os.path.join(HA_CONFIG_ROOT, "entities.list")
 RESTORE_STATE_PATH = os.path.join(HA_CONFIG_ROOT, ".storage", "core.restore_state")
-CONFIG_ENTRIES_PATH = os.path.join(HA_CONFIG_ROOT, ".storage", "core.config_entries")
-ENTITY_REGISTRY_PATH = os.path.join(HA_CONFIG_ROOT, ".storage", "core.entity_registry")
 
 if not os.path.isfile(RESTORE_STATE_PATH):
     sys.exit(f"File {RESTORE_STATE_PATH} does not exist! (Path to HomeAssistant config valid?)")
 
-if not os.path.isfile(CONFIG_ENTRIES_PATH):
-    sys.exit(f"File {CONFIG_ENTRIES_PATH} does not exist! (Path to HomeAssistant config valid?)")
-
-if not os.path.isfile(ENTITY_REGISTRY_PATH):
-    sys.exit(f"File {ENTITY_REGISTRY_PATH} does not exist! (Path to HomeAssistant config valid?)")
-
-
 # Open MySQL server connection if user provided DB_SERVER information
 if all(DB_SERVER.values()):
     import pymysql
     db = pymysql.connect(
         host=DB_SERVER["DB_HOST"],
         user=DB_SERVER["DB_USER"],
         password=DB_SERVER["DB_PASSWORD"],
-        database=DB_SERVER["DB_NAME"]
+        database=DB_SERVER["DB_NAME"],
+        autocommit=True
     )
 
 # Create connection to database file if no DB_SERVER information was provided
@@ -108,8 +101,11 @@ def main():
         with open(ENTITIES_FILE, "w") as file:
             # Get Entities that have a round option
             SqlExec("SELECT statistic_id FROM statistics_meta WHERE has_sum=1", ())
-            for entity_id in getEntitiesPrecision():
-                file.write(f"{entity_id}\n")
+            if not (result := cur.fetchall()):
+                sys.exit("There are no entities which can be fixed in the database (key 'sum' in table statistics_meta is not populated)")
+
+            for entity_id in result:
+                file.write(f"{entity_id[0]}\n")
 
         print(f"File '{ENTITIES_FILE}' created with entities that have the key 'sum'"
               f"\nPlease adjust to your needs and rerun the script with no arguments.")
@@ -119,8 +115,6 @@ def main():
 
 
 def fixDatabase(ENTITIES: list):
-    # Get Precision of Entities
-    EntityPrecision = getEntitiesPrecision()
 
     # Fix value for all metadata_ids
     for entity_id in ENTITIES:
@@ -147,44 +141,31 @@ def fixDatabase(ENTITIES: list):
         # Get metadata_id from SQL Query result
         metadata_id_statistics = result[0]
 
-        ################################################################################################################
-        # Get amount of decimals for Riemann Sum integral that the user configured
-        if entity_id not in EntityPrecision:
-            print(f"  [WARNING]: Entity seems not to be a Riemann Sum Entity! UNTESTED. USE WITH CAUTION!")
-            roundDigits = -1
-        else:
-            # Get Precision of Entity that user configured
-            roundDigits = EntityPrecision[entity_id]
-
         ################################################################################################################
         # FIX DATABASE
         print("\n========================================================================")
         print(f"{entity_id} | {metadata_id_states = } | {metadata_id_statistics = }")
 
         # Fix table "statistics"
-        lastValidSum = recalculateStatistics(metadata_id=metadata_id_statistics, key="sum", roundDigits=roundDigits)
-        lastValidState = recalculateStatistics(metadata_id=metadata_id_statistics, key="state", roundDigits=roundDigits)
+        lastValidSum = recalculateStatistics(metadata_id=metadata_id_statistics, key="sum")
+        lastValidState = recalculateStatistics(metadata_id=metadata_id_statistics, key="state")
 
         # Delete ShortTerm statistics and input one entry with current state
         fixShortTerm(metadata_id=metadata_id_statistics, lastValidSum=lastValidSum, lastValidState=lastValidState)
 
         # Fix table "states"
-        recalculateStates(metadata_id=metadata_id_states, roundDigits=roundDigits)
+        recalculateStates(metadata_id=metadata_id_states)
 
-        # Fix last valid state if entity seems to be a Riemann Sum Entity only
-        # OPEN: How to find out if entity is a Riemann Sum Entity?!
-        # Currently: If entity is in table statistics and has a "round" attribute, it is assumed to be a Riemann Sum Entity
-        if roundDigits != -1:
-            # Fix last valid state in HA to ensure a valid calculation with the next Riemann Sum calculation
-            fixLastValidState(entity_id=entity_id, lastValidState=lastValidState)
+        # Fix last valid state to current state
+        fixLastValidState(entity_id=entity_id, lastValidState=lastValidState)
 
     # Store database on disk
     print(f"\n{db.total_changes} changes made to database!")
     db.commit()
     db.close()
 
 
-def recalculateStatistics(metadata_id: int, key: str, roundDigits: int) -> str:
+def recalculateStatistics(metadata_id: int, key: str) -> str:
 
     print(f"  Fixing table statistics for key: {key}")
 
@@ -194,7 +175,7 @@ def recalculateStatistics(metadata_id: int, key: str, roundDigits: int) -> str:
 
     # Get first value from database; this is our starting point
     try:
-        current_value = float(result[0][1])
+        current_value = Decimal(str(result[0][1]))
     except ValueError:
         sys.exit(f"  [ERROR]: Cannot fix this entity because first entry in table 'statistics' for {key} is not a number! Sorry!")
 
@@ -204,33 +185,27 @@ def recalculateStatistics(metadata_id: int, key: str, roundDigits: int) -> str:
         # Get previous entry
         _, pre_value = result[index]
 
+        # Convert do decimal object
+        value = Decimal(str(value))
+        pre_value = Decimal(str(pre_value))
+
         if value < current_value:
             # Current value is out-dated
 
             if value >= pre_value:
                 # Recalculate new value with difference of previous entries
                 current_value += (value-pre_value)
 
-            if roundDigits != -1:
-                roundedValue = f"{current_value:.{roundDigits}f}"
-            else:
-                # Just copy because we don't round the value
-                roundedValue = current_value
-            print(f"    Updating {idx = }: {value = } -> {roundedValue = }")
-            SqlExec(f"UPDATE statistics SET {key}=? WHERE id=?", (roundedValue, idx))
+            print(f"    Updating {idx = }: {value} -> {current_value}")
+            SqlExec(f"UPDATE statistics SET {key}=? WHERE id=?", (float(current_value), idx))
 
             continue
 
         # Set current value as new value
         current_value = value
 
     # Return last value
-    if roundDigits != -1:
-        # Return rounded value
-        return f"{current_value:.{roundDigits}f}"
-    else:
-        # Return value as it is
-        return current_value
+    return str(current_value)
 
 
 def fixShortTerm(metadata_id: int, lastValidSum: str, lastValidState: str):
@@ -249,7 +224,7 @@ def fixShortTerm(metadata_id: int, lastValidSum: str, lastValidState: str):
             (lastValidState, lastValidSum, metadata_id, now_end.timestamp(), now_start.timestamp()))
 
 
-def recalculateStates(metadata_id: int, roundDigits: int):
+def recalculateStates(metadata_id: int):
     print(f"  Fixing table states")
 
     SqlExec("SELECT state_id,state,old_state_id,attributes_id FROM states WHERE metadata_id=? ORDER BY state_id",
@@ -258,10 +233,10 @@ def recalculateStates(metadata_id: int, roundDigits: int):
 
     # Get first value from database; this is our starting point
     try:
-        current_state = float(result[0][1])
+        current_state = Decimal(str(result[0][1]))
         attributes_id = result[0][3]
-    except ValueError:
-        sys.exit("  [ERROR]: Cannot fix this entity because first entry in table 'states' is not a number! Sorry!")
+    except InvalidOperation:
+        sys.exit(f"  [ERROR]: Cannot fix this entity because first entry in table 'states' is not a number! first entry: {result[0][3]}")
 
     # Loop over all entries starting with the second entry
     for index, (state_id, state, old_state_id, attr_id) in enumerate(result[1:]):
@@ -277,30 +252,20 @@ def recalculateStates(metadata_id: int, roundDigits: int):
 
         if state is None or not state.replace(".", "", 1).isdigit():
             # State is NULL or not numeric; update to current value
-            if roundDigits != -1:
-                roundedValue = f"{current_state:.{roundDigits}f}"
-            else:
-                # Just copy because we don't round the value
-                roundedValue = current_state
-            print(f"    Updating {state_id = }: {state = } -> {roundedValue}")
-            SqlExec("UPDATE states SET state=? WHERE state_id=?", (roundedValue, state_id))
+            print(f"    Updating {state_id = }: {state} -> {current_state}")
+            SqlExec("UPDATE states SET state=? WHERE state_id=?", (float(current_state), state_id))
             continue
 
-        state = float(state)
+        state = Decimal(str(state))
         if state < current_state:
             # Current value is out-dated
 
-            if pre_state and pre_state.replace(".", "", 1).isdigit() and state >= float(pre_state):
+            if pre_state and pre_state.replace(".", "", 1).isdigit() and state >= Decimal(str(pre_state)):
                 # Recalculate new value with difference of previous entries
-                current_state += (state - float(pre_state))
-
-            if roundDigits != -1:
-                roundedValue = f"{current_state:.{roundDigits}f}"
-            else:
-                # Just copy because we don't round the value
-                roundedValue = current_state
-            print(f"    Updating {state_id = }: {state = } -> {roundedValue}")
-            SqlExec("UPDATE states SET state=? WHERE state_id=?", (roundedValue, state_id))
+                current_state += (state - Decimal(str(pre_state)))
+
+            print(f"    Updating {state_id = }: {state} -> {current_state}")
+            SqlExec("UPDATE states SET state=? WHERE state_id=?", (float(current_state), state_id))
             continue
 
         # Set current value as new value
@@ -316,53 +281,26 @@ def fixLastValidState(entity_id: str, lastValidState: str):
     for state in restore_state["data"]:
 
         # Search for entity_id
-        if state["state"]["entity_id"] == entity_id:
-            # Modify state to new value
-            state["state"]["state"] = lastValidState
-            state["extra_data"]["native_value"]["decimal_str"] = lastValidState
-            state["extra_data"]["last_valid_state"] = lastValidState
-            break
-
-    # Write modified json
-    with open(RESTORE_STATE_PATH, "w") as file:
-        json.dump(restore_state, file, indent=2, ensure_ascii=False)
-
-
-def getEntitiesPrecision() -> dict[str: int]:
-    # Initialize return dictionary
-    returnDict = dict()
-
-    # Read file core.config_entries
-    with open(CONFIG_ENTRIES_PATH, "r") as file:
-        configEntries = json.load(file)
-
-    # Read file core.entity_registry
-    with open(ENTITY_REGISTRY_PATH, "r") as file:
-        configEntities = json.load(file)
+        if state["state"]["entity_id"] != entity_id:
+            continue
 
-    configIds = dict()
+        # Modify state to new value
+        if state["state"].get("state", ""):
+            state["state"]["state"] = lastValidState
 
-    # Find entry_ids which have the option/round attribute (these are most likely Riemann Sum Entities)
-    for configEntry in configEntries["data"]["entries"]:
-        number = configEntry["options"].get("round", -1)
-        if number == -1:
-            continue
+        if state["extra_data"]:
 
-        # Store precision value
-        configIds[configEntry["entry_id"]] = int(number)
+            if state["extra_data"].get("last_valid_state", ""):
+                state["extra_data"]["last_valid_state"] = lastValidState
 
-    # Find entity_id for all entry_ids
-    for configEntity in configEntities["data"]["entities"]:
-        if configEntity["config_entry_id"] not in configIds:
-            continue
+            if state["extra_data"].get("native_value", dict()).get("decimal_str", ""):
+                state["extra_data"]["native_value"]["decimal_str"] = lastValidState
 
-        entity_id = configEntity["entity_id"]
-        config_entry_id = configEntity["config_entry_id"]
-        # Store precision and entity_id
-        returnDict[entity_id] = configIds[config_entry_id]
+        break
 
-    # Return dict with format {entity_id: precision}
-    return returnDict
+    # Write modified json
+    with open(RESTORE_STATE_PATH, "w") as file:
+        json.dump(restore_state, file, indent=2, ensure_ascii=False)
 
 
 def SqlExec(SqlQuery: str, arguments: tuple):
