Merge pull request #137 from DSACMS/seperate-fetch-from-generation

Seperate Fetch Action From Generation of Reports and Graphs

Author: IsaacMilarky

.github/workflows/cicd.yml

@@ -19,9 +19,47 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
+  update-reports-and-graphs:
+    permissions: write-all
+    name: update reports and graphs
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      # update stats
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+      - name: cache pip
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - run: pip install -r requirements.txt
+      - run: ./gen_reports.sh
+        env:
+          GITHUB_TOKEN: ${{ secrets.METRICS_GITHUB_TOKEN }}
+          AUGUR_HOST: ${{ vars.AUGUR_HOST }}
+      - run: ./gen_graphs.sh
+        env:
+          GITHUB_TOKEN: ${{ secrets.METRICS_GITHUB_TOKEN }}
+          AUGUR_HOST: ${{ vars.AUGUR_HOST }}
+      - run: |
+          git config user.name 'GitHub Actions'
+          git config user.email '[email protected]'
+          git add -A
+          timestamp=$(date -u)
+          git commit -m "update reports and graphs: ${timestamp}" || exit 0
+      - name: Push to ${{ github.ref_name }}
+        uses: CasperWA/push-protected@v2
+        with:
+          token: ${{ secrets.METRICS_GITHUB_TOKEN }}
+          branch: ${{ github.ref_name }}
   deploy:
     runs-on: ubuntu-latest
     if: github.ref == 'refs/heads/main'
+    needs: update-reports-and-graphs
     defaults:
       run:
         working-directory: ./app
@@ -55,4 +93,5 @@ jobs:
           path: app/dist
 
       - name: Deploy to GitHub pages
-        uses: actions/deploy-pages@v2
+        uses: actions/deploy-pages@v2
+  

gen_graphs.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+pwd
+cd scripts
+python3 refresh_graphs.py

gen_reports.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+pwd
+cd scripts
+python3 refresh_reports.py

scripts/fetch_public_metrics.py

@@ -2,10 +2,58 @@
 Module to define methods that fetch data to store in the oss metric
 entity objects.
 """
+import os
 import json
 from metricsLib.metrics_definitions import SIMPLE_METRICS, ORG_METRICS, ADVANCED_METRICS
 from metricsLib.metrics_definitions import PERIODIC_METRICS, RESOURCE_METRICS
+from metricsLib.oss_metric_entities import GithubOrg, Repository
+from metricsLib.constants import PATH_TO_METADATA
 
+def parse_tracked_repos_file():
+    """
+    Function to parse projects_tracked.json
+
+    Returns:
+        Tuple of lists of strings that represent repos and orgs
+    """
+
+    # TODO: Create a read repos-to-include.txt
+    metadata_path = os.path.join(PATH_TO_METADATA, "projects_tracked.json")
+    with open(metadata_path, "r", encoding="utf-8") as file:
+        tracking_file = json.load(file)
+
+    # Track specific repositories e.g. ['dsacms.github.io']
+    repo_urls = tracking_file["Open Source Projects"]
+
+    # Get two lists of objects that will hold all the new metrics
+    return tracking_file["orgs"], repo_urls
+
+def parse_repos_and_orgs_into_objects(org_name_list, repo_name_list):
+    """
+    This function parses lists of strings into oss metric entities and
+    returns lists of corresponding oss metric entitiy objects.
+
+    Arguments:
+        org_name_list: list of logins for github orgs
+        repo_name_list: list of urls for git repositories with groups labeled
+
+    Returns:
+        Tuple of lists of oss metric entity objects
+    """
+    orgs = [GithubOrg(org) for org in org_name_list]
+
+    repos = []  # [Repository(repo_url) for repo_url in repo_name_list]
+
+    for owner, urls in repo_name_list.items():
+        print(owner)
+        # search for matching org
+        org_id = next(
+            (x.repo_group_id for x in orgs if x.login.lower() == owner.lower()), None)
+
+        # print(f"!!{org_id}")
+        for repo_url in urls:
+            repos.append(Repository(repo_url, org_id))
+    return orgs, repos
 
 def get_all_data(all_orgs, all_repos):
     """
@@ -95,8 +143,56 @@ def fetch_all_new_metric_data(all_orgs, all_repos):
         print(f"Fetching metrics for org {org.name} id #{org.repo_group_id}")
         for metric in ORG_METRICS:
             org.apply_metric_and_store_data(metric)
+            print(metric.name)
         add_info_to_org_from_list_of_repos(all_repos, org)
 
+def read_current_metric_data(repos,orgs):
+    """
+    Read current metrics and load previous metrics that 
+    were saved in .old files.
+
+    Arguments:
+        orgs: orgs to read data for.
+        repos: repos to read data for.
+    """
+
+    for org in orgs:
+
+        path = org.get_path_to_json_data()
+        #generate dict of previous and save it as {path}.old
+        #previous_metric_org_json = json.dumps(org.previous_metric_data, indent=4)
+
+        with open(f"{path}.old","r",encoding="utf-8") as file:
+            previous_metric_org_json = json.load(file)
+
+        #generate dict of current metric data.
+        org.previous_metric_data.update(previous_metric_org_json)
+
+
+        with open(path, "r", encoding="utf-8") as file:
+            #file.write(org_metric_data)
+            print(path)
+            current_metric_org_json = json.load(file)
+
+        org.metric_data.update(current_metric_org_json)
+
+    for repo in repos:
+        #previous_metric_repo_json = json.dumps(repo.previous_metric_data, indent=4)
+        path = repo.get_path_to_json_data()
+
+        with open(f"{path}.old","r",encoding="utf-8") as file:
+            #file.write(previous_metric_repo_json)
+            previous_metric_repo_json = json.load(file)
+
+        repo.previous_metric_data.update(previous_metric_repo_json)
+
+
+        with open(path, "r", encoding="utf-8") as file:
+            #file.write(repo_metric_data)
+            metric_repo_json = json.load(file)
+
+        repo.metric_data.update(metric_repo_json)
+
 
 def read_previous_metric_data(repos, orgs):
     """
@@ -116,7 +212,8 @@ def read_previous_metric_data(repos, orgs):
                 org.previous_metric_data.update(prev_data)
         except FileNotFoundError:
             print("Could not find previous data for records for org" +
-                  f"{org.login}")
+                  f"{org.login}")   
+
 
     for repo in repos:
         try:
@@ -131,20 +228,45 @@ def read_previous_metric_data(repos, orgs):
 def write_metric_data_json_to_file(orgs, repos):
     """
     Write all metric data to json files.
+    
+    Keep old metrics as a .old file.
 
     Arguments:
         orgs: orgs to write to file
         repos: repos to write to file
     """
 
     for org in orgs:
-        org_metric_data = json.dumps(org.metric_data, indent=4)
 
-        with open(org.get_path_to_json_data(), "w+", encoding="utf-8") as file:
+        path = org.get_path_to_json_data()
+        #generate dict of previous and save it as {path}.old
+        previous_metric_org_json = json.dumps(org.previous_metric_data, indent=4)
+
+        with open(f"{path}.old","w+",encoding="utf-8") as file:
+            file.write(previous_metric_org_json)
+
+        #generate dict of current metric data.
+        org_dict = org.previous_metric_data
+        org_dict.update(org.metric_data)
+        org_metric_data = json.dumps(org_dict, indent=4)
+
+        #print(org_metric_data)
+
+        with open(path, "w+", encoding="utf-8") as file:
             file.write(org_metric_data)
 
     for repo in repos:
-        repo_metric_data = json.dumps(repo.metric_data, indent=4)
+        path = repo.get_path_to_json_data()
+
+        previous_metric_repo_json = json.dumps(repo.previous_metric_data, indent=4)
+
+        with open(f"{path}.old","w+",encoding="utf-8") as file:
+            file.write(previous_metric_repo_json)
+
+        repo_dict = repo.previous_metric_data
+        repo_dict.update(repo.metric_data)
+        repo_metric_data = json.dumps(repo_dict, indent=4)
+
 
-        with open(repo.get_path_to_json_data(), "w+", encoding="utf-8") as file:
+        with open(path, "w+", encoding="utf-8") as file:
             file.write(repo_metric_data)

scripts/gen_reports.py

@@ -98,7 +98,7 @@ def get_heading_report_values(headings, oss_entity):
         #        +--------------+--------------------------------------+---+-------------------+
 
 
-        and_conditional = ((raw_diff>0) and (behavior == DesiredReportBehavior.VALUE_INCREASE.value))
+        and_conditional=((raw_diff>0) and (behavior == DesiredReportBehavior.VALUE_INCREASE.value))
         #Use a XOR by using the != operator
         if (raw_diff > 0) != (behavior == DesiredReportBehavior.VALUE_INCREASE.value):
             if raw_diff < 0:

scripts/metricsLib/oss_metric_entities.py

@@ -9,6 +9,7 @@
 import datetime
 import pathlib
 import requests
+from requests.exceptions import ReadTimeout
 from metricsLib.constants import PATH_TO_METRICS_DATA, PATH_TO_REPORTS_DATA, AUGUR_HOST
 from metricsLib.constants import TIMEOUT_IN_SECONDS, PATH_TO_GRAPHS_DATA
 
@@ -152,7 +153,7 @@ def apply_metric_and_store_data(self, metric, *args, **kwargs):
 
         try:
             self.store_metrics(metric.get_values(*args, **kwargs))
-        except TimeoutError as e:
+        except (TimeoutError, ReadTimeout) as e:
             print(f"Timeout for repo {self.name} with metric {metric.name}")
             print(f"Error: {e}")
         except ConnectionError as e:

scripts/refresh_graphs.py

@@ -0,0 +1,20 @@
+"""
+Script to run all graph generation from existing data.
+"""
+import os
+from fetch_public_metrics import parse_repos_and_orgs_into_objects
+from fetch_public_metrics import parse_tracked_repos_file, read_current_metric_data
+from gen_graphs import generate_all_graphs_for_repos, generate_all_graphs_for_orgs
+
+
+
+if __name__ == "__main__":
+    os.umask(0)
+
+    orgs_urls, repo_urls = parse_tracked_repos_file()
+
+    all_orgs, all_repos = parse_repos_and_orgs_into_objects(orgs_urls, repo_urls)
+
+    read_current_metric_data(all_repos,all_orgs)
+    generate_all_graphs_for_orgs(all_orgs)
+    generate_all_graphs_for_repos(all_repos)

scripts/refresh_metrics.py

@@ -2,60 +2,18 @@
 Script to run all metrics collection and update operations
 """
 import os
-import json
-from metricsLib.oss_metric_entities import GithubOrg, Repository
-from metricsLib.constants import PATH_TO_METADATA
-from fetch_public_metrics import get_all_data
-from gen_reports import generate_repo_report_files, generate_org_report_files
-from gen_graphs import generate_all_graphs_for_repos, generate_all_graphs_for_orgs
+from fetch_public_metrics import get_all_data, parse_repos_and_orgs_into_objects
+from fetch_public_metrics import parse_tracked_repos_file, read_previous_metric_data
 
 
-def parse_repos_and_orgs_into_objects(org_name_list, repo_name_list):
-    """
-    This function parses lists of strings into oss metric entities and
-    returns lists of corresponding oss metric entitiy objects.
-
-    Arguments:
-        org_name_list: list of logins for github orgs
-        repo_name_list: list of urls for git repositories with groups labeled
-
-    Returns:
-        Tuple of lists of oss metric entity objects
-    """
-    orgs = [GithubOrg(org) for org in org_name_list]
-
-    repos = []  # [Repository(repo_url) for repo_url in repo_name_list]
-
-    for owner, urls in repo_name_list.items():
-        print(owner)
-        # search for matching org
-        org_id = next(
-            (x.repo_group_id for x in orgs if x.login.lower() == owner.lower()), None)
-
-        # print(f"!!{org_id}")
-        for repo_url in urls:
-            repos.append(Repository(repo_url, org_id))
-    return orgs, repos
-
 
 if __name__ == "__main__":
     os.umask(0)
-    # TODO: Create a read repos-to-include.txt
-    metadata_path = os.path.join(PATH_TO_METADATA, "projects_tracked.json")
-    with open(metadata_path, "r", encoding="utf-8") as file:
-        tracking_file = json.load(file)
 
-    # Track specific repositories e.g. ['dsacms.github.io']
-    repo_urls = tracking_file["Open Source Projects"]
+    orgs_urls, repo_urls = parse_tracked_repos_file()
 
-    # Get two lists of objects that will hold all the new metrics
-    all_orgs, all_repos = parse_repos_and_orgs_into_objects(
-        tracking_file["orgs"], repo_urls)
+    all_orgs, all_repos = parse_repos_and_orgs_into_objects(orgs_urls, repo_urls)
 
     # Generate json data, report data, and graph data.
-    #read_previous_metric_data(all_repos,all_orgs)
+    read_previous_metric_data(all_repos,all_orgs)
     get_all_data(all_orgs, all_repos)
-    generate_repo_report_files(all_repos)
-    generate_org_report_files(all_orgs)
-    generate_all_graphs_for_repos(all_repos)
-    generate_all_graphs_for_orgs(all_orgs)

scripts/refresh_reports.py

@@ -0,0 +1,20 @@
+"""
+Script to run all report generation from existing data.
+"""
+import os
+from fetch_public_metrics import parse_repos_and_orgs_into_objects
+from fetch_public_metrics import parse_tracked_repos_file, read_current_metric_data
+from gen_reports import generate_repo_report_files, generate_org_report_files
+
+
+
+if __name__ == "__main__":
+    os.umask(0)
+
+    orgs_urls, repo_urls = parse_tracked_repos_file()
+
+    all_orgs, all_repos = parse_repos_and_orgs_into_objects(orgs_urls, repo_urls)
+
+    read_current_metric_data(all_repos,all_orgs)
+    generate_repo_report_files(all_repos)
+    generate_org_report_files(all_orgs)