Per feed averages

There's still a lot of room to improve here. We really want to check
there is only one data point per feed per hour so there's a greater
emphasis on cardinality. The stats here are averaging and really only
divide by three without checking the specifics of what those three
mean.

src/itn_api/api.py

@@ -226,7 +226,7 @@ async def get_online_collectors() -> str:
     try:
         participants_count = app.state.connection.execute(
             """SELECT address, COUNT(*) AS total_count,
-            SUM(CASE WHEN datetime(date_time) >= datetime('now', '-1 day')
+            SUM(CASE WHEN datetime(date_time) >= datetime('now', '-24 hours')
             THEN 1 ELSE 0 END) AS count_24hr
             FROM data_points
             GROUP BY address ORDER BY total_count DESC;
@@ -235,24 +235,58 @@ async def get_online_collectors() -> str:
     except apsw.SQLError:
         return "zero collectors online"
 
+    try:
+        feed_count = app.state.connection.execute(
+            """SELECT distinct feed_id
+            from data_points
+            where datetime(date_time) >= datetime('now', '-48 hours');
+            """
+        )
+    except apsw.SQLError:
+        return "zero collectors online"
+
+    no_feeds = len(list(feed_count))
+
+    # FIXME: These can all be combined better, e.g. into a dataclass or
+    # somesuch. This is purely for expediency to have something up and
+    # running.
     participants_count_total = {}
     participants_count_24hr = {}
+    participant_count_24h_feed_average = {}
+    participant_count_1h_feed_average = {}
+    participant_count_1m_feed_average = {}
 
     for row in participants_count:
         address, total_count, count_24hr = row
         participants_count_total[address] = total_count
         participants_count_24hr[address] = count_24hr
+        try:
+            participant_count_24h_feed_average[address] = int(count_24hr / no_feeds) + 1
+            participant_count_1h_feed_average[address] = (
+                int(count_24hr / no_feeds / 24) + 1
+            )
+            participant_count_1m_feed_average[address] = round(
+                count_24hr / no_feeds / 24 / 60, 4
+            )
+        except ZeroDivisionError:
+            participant_count_24h_feed_average[address] = 0
+            participant_count_1h_feed_average[address] = 0
+            participant_count_1m_feed_average = 0
 
     htmx = htm_helpers.participants_count_table(
-        participants_count_total, participants_count_24hr
+        participants_count_total,
+        participants_count_24hr,
+        participant_count_24h_feed_average,
+        participant_count_1h_feed_average,
+        participant_count_1m_feed_average,
     )
     return htmx.strip()
 
 
 @app.get("/locations", tags=[TAG_HTMX], response_class=HTMLResponse)
 async def get_locations_hx():
     """Return countries participating in the ITN."""
-    locations = await reports.get_locations(app)
+    locations = await reports.get_locations_stake_key(app)
     return htm_helpers.locations_table(locations)
 
 

src/itn_api/htm_helpers.py

@@ -65,7 +65,13 @@ def aliases_to_html(alias_report: dict) -> str:
     return f"{head}\n{rows}\n{count_row}</table>\n"
 
 
-def participants_count_table(participants_count_total, participants_count_24hr):
+def participants_count_table(
+    participants_count_total,
+    participants_count_24hr,
+    participant_count_24h_feed_average,
+    participant_count_1h_feed_average,
+    participant_count_1m_feed_average,
+):
     """Return a table with active participant counts."""
 
     logging.info("formatting participants table")
@@ -79,17 +85,26 @@ def participants_count_table(participants_count_total, participants_count_24hr):
         <th>Stake Key</th>
         <th>Count (Total)</th>
         <th>Count (24hr)</th>
+        <th>Per feed (24hr)</th>
+        <th>Per feed (1hr)</th>
+        <th>Per feed (1min)</th>
     </tr>
     """.strip()
 
     rows = ""
     for stake_key, count in participants_count_total.items():
         count_24hr = participants_count_24hr.get(stake_key, 0)
+        average_24hr = participant_count_24h_feed_average.get(stake_key, 0)
+        average_1hr = participant_count_1h_feed_average.get(stake_key, 0)
+        average_min = participant_count_1m_feed_average.get(stake_key, 0)
         row = f"""
 <tr>
     <td>{stake_key}</td>
     <td nowrap>&nbsp;{humanize.intcomma(count)}&nbsp;</td>
     <td nowrap>&nbsp;{humanize.intcomma(count_24hr)}&nbsp;</td>
+    <td nowrap>&nbsp;{humanize.intcomma(average_24hr)}&nbsp;</td>
+    <td nowrap>&nbsp;{humanize.intcomma(average_1hr)}&nbsp;</td>
+    <td nowrap>&nbsp;{humanize.intcomma(average_min)}&nbsp;</td>
 </tr>
         """.strip()
 

src/itn_api/reports.py

@@ -310,6 +310,47 @@ async def get_date_ranges(app: FastAPI):
 async def get_locations(app: FastAPI) -> list:
     """Return locations from the database.
 
+    Select one of each kind example:
+
+      * https://stackoverflow.com/a/571487
+
+    """
+    try:
+        unique_raw_data = app.state.connection.execute(
+            "select min(node_id), raw_data from data_points group by node_id;"
+        )
+    except apsw.SQLError:
+        return "zero collectors online"
+
+    res = list(unique_raw_data)
+    countries = []
+    for item in res:
+        node = item[0]
+        message = json.loads(item[1])
+        try:
+            loc = message["message"]["identity"]["location"]
+            geo = loc.get("loc")
+            latitude, longitude = map(float, geo.split(","))
+
+            countries.append(
+                (
+                    {
+                        "latitude": latitude,
+                        "longitude": longitude,
+                        "region": loc.get("region"),
+                        "country": loc.get("country"),
+                    }
+                )
+            )
+        except KeyError as err:
+            logger.error("node: '%s' not reporting location (%s)", node, err)
+    return countries
+
+
+async def get_locations_stake_key(app: FastAPI) -> list:
+    """Return locations from the database with a users stake key. This
+    can eventually replace `get_locations()`.
+
     Select one of each kind example:
 
       * https://stackoverflow.com/a/571487
@@ -319,7 +360,7 @@ async def get_locations(app: FastAPI) -> list:
         unique_raw_data = app.state.connection.execute(
             """select node_id, raw_data, min(address), date_time
             from data_points
-            where datetime(date_time) >= datetime('now', '-48 hours')
+            where datetime(date_time) >= datetime('now', '-24 hours')
             group by address;
             """
         )