From d76f0dbf00ccc0736bd76c86e664228578a11d88 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Wed, 13 Dec 2023 16:34:15 -0600 Subject: [PATCH] Conform to pylint Signed-off-by: Isaac Milarsky --- scripts/fetch_public_metrics.py | 13 +- scripts/gen_graphs.py | 24 ++-- scripts/gen_reports.py | 18 +-- scripts/metricsLib/constants.py | 6 +- scripts/metricsLib/metrics_data_structures.py | 69 +++++------ scripts/metricsLib/metrics_definitions.py | 116 +++++++++++------- scripts/metricsLib/oss_metric_entities.py | 71 ++++++----- scripts/refresh_metrics.py | 22 ++-- scripts/run_pylint.py | 2 +- scripts/tests/metrics_tests.py | 28 +++-- 10 files changed, 204 insertions(+), 165 deletions(-) diff --git a/scripts/fetch_public_metrics.py b/scripts/fetch_public_metrics.py index feb84babe5..7dd8b7d619 100755 --- a/scripts/fetch_public_metrics.py +++ b/scripts/fetch_public_metrics.py @@ -3,7 +3,8 @@ entity objects. """ import json -from metricsLib.metrics_definitions import SIMPLE_METRICS, ORG_METRICS, PERIODIC_METRICS, RESOURCE_METRICS +from metricsLib.metrics_definitions import SIMPLE_METRICS, ORG_METRICS +from metricsLib.metrics_definitions import PERIODIC_METRICS, RESOURCE_METRICS def get_all_data(all_orgs, all_repos): @@ -80,9 +81,9 @@ def fetch_all_new_metric_data(all_orgs, all_repos): for metric in PERIODIC_METRICS: repo.apply_metric_and_store_data(metric) - + for metric in RESOURCE_METRICS: - repo.apply_metric_and_store_data(metric,repo) + repo.apply_metric_and_store_data(metric, repo) # Capture all metric data for all Github orgs for org in all_orgs: @@ -109,7 +110,8 @@ def read_previous_metric_data(repos, orgs): prev_data = json.load(file) org.previous_metric_data.update(prev_data) except FileNotFoundError: - print(f"Could not find previous data for records for org {org.login}") + print("Could not find previous data for records for org" + + f"{org.login}") for repo in repos: try: @@ -117,7 +119,8 @@ def read_previous_metric_data(repos, orgs): prev_data = json.load(file) repo.previous_metric_data.update(prev_data) except FileNotFoundError: - print(f"Could not find previous data for records for repo {repo.name}") + print("Could not find previous data for records for repo" + + repo.name) def write_metric_data_json_to_file(orgs, repos): diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index a60ba012e7..a0ec68dce3 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -17,6 +17,7 @@ def generate_all_graphs_for_repos(all_repos): generate_solid_gauge_issue_graph(repo) generate_repo_sparklines(repo) + def generate_all_graphs_for_orgs(all_orgs): for org in all_orgs: print(f"Generating graphs for org {org.name}") @@ -69,7 +70,7 @@ def generate_repo_sparklines(repo): "margin": 10 } write_repo_chart_to_file( - repo, chart, "commit_sparklines", custom_func=chart.render_sparkline,custom_func_params=_kwargs_) + repo, chart, "commit_sparklines", custom_func=chart.render_sparkline, custom_func_params=_kwargs_) def generate_solid_gauge_issue_graph(oss_entity): @@ -80,14 +81,15 @@ def generate_solid_gauge_issue_graph(oss_entity): oss_entity: the OSSEntity to create a graph for. """ - issues_gauge = pygal.SolidGauge(inner_radius=0.70,legend_at_bottom=True) + issues_gauge = pygal.SolidGauge(inner_radius=0.70, legend_at_bottom=True) + def percent_formatter(x): return '{:0.2f}%'.format(x) issues_gauge.value_formatter = percent_formatter - - #Generate graph to measure percentage of issues that are open + + # Generate graph to measure percentage of issues that are open try: - #calculate portion of issues that are open. + # calculate portion of issues that are open. open_issue_percent = oss_entity.metric_data['open_issues_count'] / \ oss_entity.metric_data['issues_count'] except ZeroDivisionError: @@ -95,9 +97,8 @@ def percent_formatter(x): issues_gauge.add( 'Open Issues', [{'value': open_issue_percent * 100, 'max_value': 100}]) - try: - #calculate portion of pull requests that are open, merged, and closed + # calculate portion of pull requests that are open, merged, and closed open_pr_percent = oss_entity.metric_data['open_pull_requests_count'] / \ oss_entity.metric_data['pull_requests_count'] merged_pr_percent = oss_entity.metric_data['merged_pull_requests_count'] / \ @@ -109,14 +110,15 @@ def percent_formatter(x): merged_pr_percent = 0 closed_pr_percent = 0 - #Generate graph to measure portion of pull requests that are open + # Generate graph to measure portion of pull requests that are open issues_gauge.add('Open Pull Requests', [ {'value': open_pr_percent * 100, 'max_value': 100}]) - - #Generate graph to measure portion of pull requests that are merged or closed. + + # Generate graph to measure portion of pull requests that are merged or closed. issues_gauge.add( 'Closed and Merged Pull Requests', [ - {'label': "Merged Pull Requests",'value': merged_pr_percent * 100, 'max_value': 100}, + {'label': "Merged Pull Requests", + 'value': merged_pr_percent * 100, 'max_value': 100}, {'label': "Closed Pull Requests", 'value': closed_pr_percent * 100, 'max_value': 100}]) write_repo_chart_to_file(oss_entity, issues_gauge, "issue_gauge") diff --git a/scripts/gen_reports.py b/scripts/gen_reports.py index 58bb2f3bc0..57a30cf5c0 100644 --- a/scripts/gen_reports.py +++ b/scripts/gen_reports.py @@ -4,6 +4,7 @@ from datetime import date from metricsLib.constants import REPO_REPORT_TEMPLATE, ORG_REPORT_TEMPLATE + def calc_percent_difference(latest, prev): """ This function calculates the percent difference between @@ -28,7 +29,8 @@ def calc_percent_difference(latest, prev): return int(dec * 100) -def get_heading_report_values(headings,oss_entity): + +def get_heading_report_values(headings, oss_entity): report_values = {} for heading in headings: prev_record = oss_entity.metric_data[heading] @@ -57,12 +59,13 @@ def get_heading_report_values(headings,oss_entity): f"{heading}_diff_color": diff_color, f"{heading}_diff_percent_color": diff_color }) - + return report_values -def write_report_to_file(report_template,report_values,oss_entity): + +def write_report_to_file(report_template, report_values, oss_entity): raw_report = report_template.format(**report_values) - with open(oss_entity.get_path_to_report_data(),"w+", encoding="utf-8") as file: + with open(oss_entity.get_path_to_report_data(), "w+", encoding="utf-8") as file: file.write(raw_report) @@ -91,11 +94,11 @@ def generate_org_report_files(orgs): 'followers_count' ] - report_values.update(get_heading_report_values(org_metric_table_headings, org)) + report_values.update(get_heading_report_values( + org_metric_table_headings, org)) write_report_to_file(ORG_REPORT_TEMPLATE, report_values, org) - def generate_repo_report_files(repos): """ This function generates reports for each repo and writes them to file. @@ -127,6 +130,7 @@ def generate_repo_report_files(repos): 'watchers_count' ] - report_values.update(get_heading_report_values(metric_table_headings, repo)) + report_values.update(get_heading_report_values( + metric_table_headings, repo)) write_report_to_file(REPO_REPORT_TEMPLATE, report_values, repo) diff --git a/scripts/metricsLib/constants.py b/scripts/metricsLib/constants.py index 90b782dfb9..558f3572d0 100644 --- a/scripts/metricsLib/constants.py +++ b/scripts/metricsLib/constants.py @@ -26,6 +26,6 @@ template_path = os.path.join(PATH_TO_TEMPLATES, "repo_report_template.md") with open(template_path, "r", encoding="utf-8") as file: REPO_REPORT_TEMPLATE = file.read() - -with open(os.path.join(PATH_TO_TEMPLATES, "org_report_template.md"), "r",encoding="utf-8") as file: - ORG_REPORT_TEMPLATE = file.read() + +with open(os.path.join(PATH_TO_TEMPLATES, "org_report_template.md"), "r", encoding="utf-8") as file: + ORG_REPORT_TEMPLATE = file.read() diff --git a/scripts/metricsLib/metrics_data_structures.py b/scripts/metricsLib/metrics_data_structures.py index 04dfcd1d87..130cee7847 100644 --- a/scripts/metricsLib/metrics_data_structures.py +++ b/scripts/metricsLib/metrics_data_structures.py @@ -1,14 +1,13 @@ """ Module to define classes of metrics that gather data given parameters """ -import shutil import json from json.decoder import JSONDecodeError import datetime from functools import reduce import operator import requests -from metricsLib.constants import TIMEOUT_IN_SECONDS, GH_GQL_ENDPOINT, PATH_TO_GRAPHS_DATA +from metricsLib.constants import TIMEOUT_IN_SECONDS, GH_GQL_ENDPOINT # Simple metric that can be represented by a count or value. @@ -81,11 +80,10 @@ def hit_metric(self, params=None): "headers": self.headers, "timeout": TIMEOUT_IN_SECONDS } - response = requests.request(*_args_,**_kwargs_ ) + response = requests.request(*_args_, **_kwargs_) else: response = requests.request( self.method, self.url, params=request_params, timeout=TIMEOUT_IN_SECONDS) - try: response_json = json.loads(response.text) except JSONDecodeError: @@ -116,6 +114,7 @@ def get_values(self, params=None): return to_return + class ResourceMetric(BaseMetric): """ Class to define a metric that gets data from an endpoint that returns data @@ -133,11 +132,12 @@ class ResourceMetric(BaseMetric): ------- hit_metric(params={}): Fetch data from url using parameters - + get_values(repo,params={}): Fetch data and save it in desired path and format """ - def __init__(self, name, needed_params, url, fmt='png',token=None): + + def __init__(self, name, needed_params, url, fmt='png', token=None): super().__init__(name, needed_params, url, {}, token=token) self.format = fmt @@ -160,30 +160,28 @@ def hit_metric(self, params=None): "params": request_params, "headers": self.headers, "timeout": TIMEOUT_IN_SECONDS, - "stream" : True + "stream": True } - response = requests.request(*_args_,**_kwargs_) + response = requests.request(*_args_, **_kwargs_) else: response = requests.request( self.method, self.url, params=request_params, timeout=TIMEOUT_IN_SECONDS) - - #return response + # return response return response - def get_values(self,oss_entity, params=None): + def get_values(self, oss_entity, params=None): r = self.hit_metric(params=params) - path = oss_entity.get_path_to_resource_data(self.name,fmt=self.format) + path = oss_entity.get_path_to_resource_data(self.name, fmt=self.format) if r.status_code == 200: - with open(path,"wb+") as f: + with open(path, "wb+") as f: f.write(r.content) else: print(f"Status code: {r.status_code}") return {} - class GraphQLMetric(BaseMetric): """ Class to define a metric that gets data from a graphql endpoint. @@ -253,8 +251,8 @@ def get_values(self, params=None): raise requests.exceptions.InvalidJSONError( response_json['message']) - #print(f"Response_JSON: {response_json}") - #print(f"Return values: {self.return_values}") + # print(f"Response_JSON: {response_json}") + # print(f"Return values: {self.return_values}") for val, key_sequence in self.return_values.items(): # Extract the nested data and store it in a flat dict to return to the user to_return[val] = reduce( @@ -262,6 +260,7 @@ def get_values(self, params=None): return to_return + class SumMetric(BaseMetric): """ Class to define a metric that returns a returned list @@ -274,7 +273,8 @@ class SumMetric(BaseMetric): Fetch data from url using parameters, format and sum the data before returning it. """ - def __init__(self, name, needed_params, endpoint_url,return_val, token=None, method='GET'): + + def __init__(self, name, needed_params, endpoint_url, return_val, token=None, method='GET'): super().__init__(name, needed_params, endpoint_url, return_val, token=token, method=method) @@ -282,7 +282,6 @@ def get_values(self, params=None): return {self.return_values: len(self.hit_metric(params=params))} - class ListMetric(BaseMetric): """ Class to define a metric that returns a returned list @@ -295,6 +294,7 @@ class ListMetric(BaseMetric): Fetch data from url using parameters, format and sum the data before returning it. """ + def __init__(self, name, needed_params, endpoint_url, return_values, token=None, method='GET'): super().__init__(name, needed_params, endpoint_url, return_values, token=token, method=method) @@ -304,38 +304,33 @@ def get_values(self, params=None): to_return = {} - #print(f"URL: {self.url}") + # print(f"URL: {self.url}") for return_label, api_label in self.return_values.items(): - #Allow for multiple keys of each returned element to be stored. - #EX: storing the date and count of each time the amount of followers - #increased. + # Allow for multiple keys of each returned element to be stored. + # EX: storing the date and count of each time the amount of followers + # increased. try: list(api_label) - #initialize each label as an empty list + # initialize each label as an empty list to_return[return_label] = [] for item in metric_json: - #extract each key in returned json and add to sublist + # extract each key in returned json and add to sublist elem = [] for sub_label in api_label: elem.append(item[sub_label]) - - #Add up sublists and assign to return label key + # Add up sublists and assign to return label key to_return[return_label].append(elem) except TypeError: - #return_label key is assigned to list of extracted api_label value + # return_label key is assigned to list of extracted api_label value to_return[return_label] = [item[api_label] - for item in metric_json] - - + for item in metric_json] return to_return - - class RangeMetric(ListMetric): """ Class to define a metric that returns the sum of a returned list @@ -364,12 +359,12 @@ def get_values(self, params=None): Args: params: dict Dictionary of parameters to apply to endpoint. - + Returns: Dictionary containing the desired values in the requested mapping """ - return_dict = super().get_values(params=params)#self.hit_metric(params=params) + return_dict = super().get_values(params=params) # self.hit_metric(params=params) to_return = {} @@ -379,7 +374,6 @@ def get_values(self, params=None): return to_return - class CustomMetric(BaseMetric): """ Class to define a metric that is parsed in a custom way defined @@ -394,6 +388,7 @@ class CustomMetric(BaseMetric): Fetch data from url using parameters, format and sum the data before returning it. Using the custom parsing function passed in. """ + def __init__(self, name, needed_parameters, endpoint_url, func, token=None, method='GET'): super().__init__(name, needed_parameters, endpoint_url, None, token=token, method=method) @@ -415,7 +410,7 @@ def parse_commits_by_month(**kwargs): Args: kwargs: dict Keyword arguments used by the parsing function, - + Returns: Dictionary containing the desired values in the requested mapping """ @@ -423,7 +418,7 @@ def parse_commits_by_month(**kwargs): metric_json = kwargs['metric_json'] commits_by_month = {} - #print(metric_json) + # print(metric_json) # print(metric_json) for commit in metric_json: diff --git a/scripts/metricsLib/metrics_definitions.py b/scripts/metricsLib/metrics_definitions.py index b5e642cc64..731df5c43c 100644 --- a/scripts/metricsLib/metrics_definitions.py +++ b/scripts/metricsLib/metrics_definitions.py @@ -19,7 +19,7 @@ # Metrics gathered by org instead of by repo ORG_METRICS = [] -#Metrics that save a resource to a file +# Metrics that save a resource to a file RESOURCE_METRICS = [] REPO_GITHUB_GRAPHQL_QUERY = """ @@ -83,47 +83,64 @@ github_graphql_simple_counts_metric_map = { - "description": ["data", "repository", "description"], - "commits_count": ["data", "repository", "defaultBranchRef", "target", "history", "totalCount"], - "issues_count": ["data", "repository", "issues", "totalCount"], - "open_issues_count": ["data", "repository", "openIssues", "totalCount"], - "closed_issues_count": ["data", "repository", "closedIssues", "totalCount"], - "pull_requests_count": ["data", "repository", "pullRequests", "totalCount"], - "open_pull_requests_count": ["data", "repository", "openPullRequests", "totalCount"], - "merged_pull_requests_count": ["data", "repository", "mergedPullRequests", "totalCount"], - "closed_pull_requests_count": ["data", "repository", "closedPullRequests", "totalCount"], - "forks_count": ["data", "repository", "forkCount"], - "stargazers_count": ["data", "repository", "stargazerCount"], - "watchers_count": ["data", "repository", "watchers", "totalCount"] + "description": ["data", "repository", "description"], + "commits_count": ["data", "repository", "defaultBranchRef", "target", "history", "totalCount"], + "issues_count": ["data", "repository", "issues", "totalCount"], + "open_issues_count": ["data", "repository", "openIssues", "totalCount"], + "closed_issues_count": ["data", "repository", "closedIssues", "totalCount"], + "pull_requests_count": ["data", "repository", "pullRequests", "totalCount"], + "open_pull_requests_count": ["data", "repository", "openPullRequests", "totalCount"], + "merged_pull_requests_count": ["data", "repository", "mergedPullRequests", "totalCount"], + "closed_pull_requests_count": ["data", "repository", "closedPullRequests", "totalCount"], + "forks_count": ["data", "repository", "forkCount"], + "stargazers_count": ["data", "repository", "stargazerCount"], + "watchers_count": ["data", "repository", "watchers", "totalCount"] } SIMPLE_METRICS.append(GraphQLMetric("githubGraphqlSimpleCounts", ["repo", "owner"], - REPO_GITHUB_GRAPHQL_QUERY, github_graphql_simple_counts_metric_map, token=TOKEN)) + REPO_GITHUB_GRAPHQL_QUERY, + github_graphql_simple_counts_metric_map, token=TOKEN)) ORG_METRICS.append(ListMetric("topCommitters", ["repo_group_id"], - AUGUR_HOST + "/repo-groups/{repo_group_id}/top-committers", - {"top_committers": ["email", "commits"]})) - - - -PERIODIC_METRICS.append(ListMetric("newContributorsofCommitsWeekly", ["repo_id", "period", "begin_week", "end_date"], - AUGUR_HOST + "/repos/{repo_id}/pull-requests-merge-contributor-new?period={period}&begin_date={begin_week}&end_date={end_date}", - {"new_commit_contributors_by_day_over_last_month": ["commit_date", "count"]})) - -PERIODIC_METRICS.append(ListMetric("newContributorsofCommitsMonthly", ["repo_id", "period", "begin_month", "end_date"], - AUGUR_HOST + "/repos/{repo_id}/pull-requests-merge-contributor-new?period={period}&begin_date={begin_month}&end_date={end_date}", - {"new_commit_contributors_by_day_over_last_six_months": ["commit_date", "count"]})) - -PERIODIC_METRICS.append(ListMetric("issuesNewWeekly", ["repo_id", "period", "begin_week", "end_date"], - AUGUR_HOST + "/repos/{repo_id}/issues-new?period={period}&begin_date={begin_week}&end_date={end_date}", - {"new_issues_by_day_over_last_month": ["date", "issues"]})) - -PERIODIC_METRICS.append(ListMetric("issuesNewMonthly", ["repo_id", "period", "begin_month", "end_date"], - AUGUR_HOST + "/repos/{repo_id}/issues-new?period={period}&begin_date={begin_month}&end_date={end_date}", - {"new_issues_by_day_over_last_six_months": ["date", "issues"]})) - -#TODO: Ask Sean why this endpoint isn't working for any repo except for augur. might just be lack of data. -RESOURCE_METRICS.append(ResourceMetric("firstResponseForClosedPR", ["repo_id", "period", "begin_month", "end_date"], - AUGUR_HOST + "/pull_request_reports/PR_time_to_first_response/?repo_id={repo_id}&start_date={begin_month}&end_date={end_date}")) + AUGUR_HOST + + "/repo-groups/{repo_group_id}/top-committers", + {"top_committers": ["email", "commits"]})) + + +CONTRIBS_LABEL_LAST_MONTH = "new_commit_contributors_by_day_over_last_month" +PERIODIC_METRICS.append(ListMetric("newContributorsofCommitsWeekly", + ["repo_id", "period", "begin_week", "end_date"], + AUGUR_HOST + "/repos/{repo_id}" + + "/pull-requests-merge-contributor-new" + + "?period={period}&begin_date={begin_week}&end_date={end_date}", + { + CONTRIBS_LABEL_LAST_MONTH: ["commit_date", "count"] + })) + +sixMonthsParams = ["repo_id", "period", "begin_month", "end_date"] +LABEL = "new_commit_contributors_by_day_over_last_six_months" +PERIODIC_METRICS.append(ListMetric("newContributorsofCommitsMonthly", sixMonthsParams, + AUGUR_HOST + + "/repos/{repo_id}/pull-requests-merge-contributor-new" + + "?period={period}&begin_date={begin_month}&end_date={end_date}", + {LABEL: ["commit_date", "count"]})) + +PERIODIC_METRICS.append(ListMetric("issuesNewWeekly", ["repo_id","period","begin_week","end_date"], + AUGUR_HOST + + "/repos/{repo_id}/issues-new" + + "?period={period}&begin_date={begin_week}&end_date={end_date}", + {"new_issues_by_day_over_last_month": ["date", "issues"]})) + +PERIODIC_METRICS.append(ListMetric("issuesNewMonthly", sixMonthsParams, + AUGUR_HOST + + "/repos/{repo_id}/issues-new?" + + "period={period}&begin_date={begin_month}&end_date={end_date}", + {"new_issues_by_day_over_last_six_months": ["date", "issues"]})) + +# TODO: Ask Sean why this endpoint isn't working for any repo except for augur. +# might just be lack of data. +RESOURCE_METRICS.append(ResourceMetric("firstResponseForClosedPR", sixMonthsParams, + AUGUR_HOST + "/pull_request_reports/PR_time_to_first_response/" + + "?repo_id={repo_id}&start_date={begin_month}&end_date={end_date}")) ORG_GITHUB_GRAPHQL_QUERY = """ @@ -151,21 +168,26 @@ "is_verified": ["data", "organization", "isVerified"], "location": ["data", "organization", "location"], "twitter_username": ["data", "organization", "twitterUsername"], - "repo_count": ["data", "organization", "repositories", "totalCount"] + "repo_count": ["data","organization","repositories","totalCount"] }, token=TOKEN)) FOLLOWERS_ENDPOINT = "https://api.github.com/users/{org_login}/followers" ORG_METRICS.append( - SumMetric("orgFollowers", ["org_login"], FOLLOWERS_ENDPOINT, "followers_count",token=TOKEN) + SumMetric("orgFollowers", ["org_login"], + FOLLOWERS_ENDPOINT, "followers_count", token=TOKEN) ) -ORG_METRICS.append(ListMetric("issuesNewWeekly", ["repo_group_id", "period", "begin_week", "end_date"], - AUGUR_HOST + "/repo-groups/{repo_group_id}/issues-new?period={period}&begin_date={begin_week}&end_date={end_date}", - {"new_issues_by_day_over_last_month": ["date", "issues"]})) - -ORG_METRICS.append(ListMetric("issuesNewMonthly", ["repo_group_id", "period", "begin_month", "end_date"], - AUGUR_HOST + "/repo-groups/{repo_group_id}/issues-new?period={period}&begin_date={begin_month}&end_date={end_date}", - {"new_issues_by_day_over_last_six_months": ["date", "issues"]})) +ORG_METRICS.append(ListMetric("issueNewWeekly", ["repo_group_id","period","begin_week","end_date"], + AUGUR_HOST + + "/repo-groups/{repo_group_id}/issues-new" + + "?period={period}&begin_date={begin_week}&end_date={end_date}", + {"new_issues_by_day_over_last_month": ["date", "issues"]})) + +ORG_METRICS.append(ListMetric("issueNewMonthly",["repo_group_id","period","begin_month","end_date"], + AUGUR_HOST + + "/repo-groups/{repo_group_id}/issues-new" + + "?period={period}&begin_date={begin_month}&end_date={end_date}", + {"new_issues_by_day_over_last_six_months": ["date", "issues"]})) COMMITS_ENDPOINT = "https://api.github.com/repos/{owner}/{repo}/commits" SIMPLE_METRICS.append(CustomMetric("getCommitsByMonth", [ diff --git a/scripts/metricsLib/oss_metric_entities.py b/scripts/metricsLib/oss_metric_entities.py index bb6350e375..db3ef14d8b 100644 --- a/scripts/metricsLib/oss_metric_entities.py +++ b/scripts/metricsLib/oss_metric_entities.py @@ -6,12 +6,13 @@ import re import json import os +import datetime import pathlib import requests -import datetime from metricsLib.constants import PATH_TO_METRICS_DATA, PATH_TO_REPORTS_DATA, AUGUR_HOST from metricsLib.constants import TIMEOUT_IN_SECONDS, PATH_TO_GRAPHS_DATA + def get_repo_owner_and_name(repo_http_url): """ Gets the owner and repo from a url. @@ -39,13 +40,22 @@ def get_repo_owner_and_name(repo_http_url): return owner, repo + def get_timebox_timestamps(): + """ + Gets timeboxed timestamps for the time the + function was ran. + + Returns: + Dictionary of key timestamps and the desired period + for metrics + """ # Get timeboxed metrics today = datetime.date.today() week_ago = today - datetime.timedelta(weeks=4) month_ago = today - datetime.timedelta(weeks=24) - #Perpare params for weekly timebox + # Perpare params for weekly timebox periodic_params = { "period": "day", "end_date": today.strftime('%Y/%m/%d'), @@ -54,6 +64,8 @@ def get_timebox_timestamps(): } return periodic_params + + class OSSEntity: """ This serves as the base class to define an OSSEntity. An OSSEntity is an @@ -136,7 +148,7 @@ def apply_metric_and_store_data(self, metric, *args, **kwargs): """ params = self.get_parameters_for_metric(metric) - self.store_metrics(metric.get_values(params=params,*args,**kwargs)) + self.store_metrics(metric.get_values(params=params, *args, **kwargs)) class Repository(OSSEntity): @@ -177,11 +189,11 @@ class Repository(OSSEntity): and extension """ - def __init__(self, repo_git_url,owner_id): + def __init__(self, repo_git_url, owner_id): self.url = repo_git_url - #print(f"!!!!{self.url}") + # print(f"!!!!{self.url}") owner, repo_name = get_repo_owner_and_name(self.url) self.repo_owner = owner @@ -190,22 +202,22 @@ def __init__(self, repo_git_url,owner_id): endpoint = f"{AUGUR_HOST}/repos" else: endpoint = f"{AUGUR_HOST}/repo-groups/{owner_id}/repos" - super().__init__(repo_name,endpoint) + super().__init__(repo_name, endpoint) - response = requests.get( self.augur_util_endpoint, timeout=TIMEOUT_IN_SECONDS) response_json = json.loads(response.text) - #print(response_json) + # print(response_json) try: len(response_json) - repo_val = next(x for x in response_json if x['repo_name'].lower() == repo_name.lower()) + repo_val = next( + x for x in response_json if x['repo_name'].lower() == repo_name.lower()) - #print(f"!!!{repo_val}") - #for x in response_json: + # print(f"!!!{repo_val}") + # for x in response_json: # print(f"|{x['repo_name'].lower()}=={repo_name.lower()}|") - #print(repo_val) + # print(repo_val) self.repo_id = repo_val['repo_id'] if owner_id is not None: @@ -216,9 +228,7 @@ def __init__(self, repo_git_url,owner_id): self.repo_id = None self.repo_group_id = None - - - #print(f"BEGIN: {today.strftime('%Y/%m/%d')}") + # print(f"BEGIN: {today.strftime('%Y/%m/%d')}") # Prepare params self.needed_parameters = { "repo": self.name, @@ -277,7 +287,7 @@ def get_path_to_report_data(self): """ return self.get_path_to_data(PATH_TO_REPORTS_DATA, "md") - def get_path_to_resource_data(self,resource_name,fmt="png"): + def get_path_to_resource_data(self, resource_name, fmt="png"): """ Derive the path for resource data using svg parent path and extension @@ -301,7 +311,7 @@ def get_path_to_graph_data(self, graph_name): String path to data. """ - return self.get_path_to_resource_data(graph_name,fmt="svg") + return self.get_path_to_resource_data(graph_name, fmt="svg") class GithubOrg(OSSEntity): @@ -336,17 +346,18 @@ def __init__(self, organization_login): super().__init__(self.login, f"{AUGUR_HOST}/repo-groups") try: - response = requests.get(self.augur_util_endpoint,timeout=TIMEOUT_IN_SECONDS) + response = requests.get( + self.augur_util_endpoint, timeout=TIMEOUT_IN_SECONDS) response_dict = json.loads(response.text) except Exception: - print(f"It looks like Augur is down! Not able to get Augur data!") + print("It looks like Augur is down! Not able to get Augur data!") response_dict = {} try: print(self.login) # Get the item in the list that matches the login of the github org - group_id = next( - (item for item in response_dict if item["rg_name"].lower() == self.login.lower()), None) + gen = (item for item in response_dict if item["rg_name"].lower() == self.login.lower()) + group_id = next(gen, None) self.repo_group_id = group_id['repo_group_id'] @@ -368,7 +379,7 @@ def __init__(self, organization_login): self.previous_metric_data = {} - def get_path_to_data(self,super_parent_path,extension): + def get_path_to_data(self, super_parent_path, extension): """ Derive the path for data using parent path and extension @@ -382,7 +393,6 @@ def get_path_to_data(self,super_parent_path,extension): return org_path - def get_path_to_json_data(self): """ Derive the path for json data using json parent @@ -392,7 +402,7 @@ def get_path_to_json_data(self): String path to data. """ return self.get_path_to_data(PATH_TO_METRICS_DATA, "json") - + def get_path_to_report_data(self): """ Derive the path for report data using parent @@ -403,7 +413,7 @@ def get_path_to_report_data(self): """ return self.get_path_to_data(PATH_TO_REPORTS_DATA, "md") - def get_path_to_resource_data(self,resource_name,fmt="png"): + def get_path_to_resource_data(self, resource_name, fmt="png"): """ Derive the path for graph data using parent path and extension @@ -414,11 +424,12 @@ def get_path_to_resource_data(self,resource_name,fmt="png"): parent_path = os.path.join(PATH_TO_GRAPHS_DATA, f"{self.login}") pathlib.Path(parent_path).mkdir(parents=True, exist_ok=True) - org_path = os.path.join(parent_path, f"{self.login}_{resource_name}.{fmt}") + fname = f"{self.login}_{resource_name}.{fmt}" + org_path = os.path.join(parent_path, fname) return org_path - - def get_path_to_graph_data(self,chart_name): + + def get_path_to_graph_data(self, chart_name): """ Derive the path for graph data using parent path and extension @@ -426,7 +437,5 @@ def get_path_to_graph_data(self,chart_name): Returns: String path to data. """ - - return self.get_path_to_resource_data(chart_name,fmt="svg") - + return self.get_path_to_resource_data(chart_name, fmt="svg") diff --git a/scripts/refresh_metrics.py b/scripts/refresh_metrics.py index f15d3a09cb..7d75a2f468 100644 --- a/scripts/refresh_metrics.py +++ b/scripts/refresh_metrics.py @@ -3,7 +3,7 @@ """ import os import json -from metricsLib.oss_metric_entities import GithubOrg, Repository, get_repo_owner_and_name +from metricsLib.oss_metric_entities import GithubOrg, Repository from metricsLib.constants import PATH_TO_METADATA from fetch_public_metrics import get_all_data from gen_reports import generate_repo_report_files, generate_org_report_files @@ -24,16 +24,17 @@ def parse_repos_and_orgs_into_objects(org_name_list, repo_name_list): """ orgs = [GithubOrg(org) for org in org_name_list] - repos = []#[Repository(repo_url) for repo_url in repo_name_list] + repos = [] # [Repository(repo_url) for repo_url in repo_name_list] - for owner, repo_urls in repo_name_list.items(): + for owner, urls in repo_name_list.items(): - #search for matching org - org_id = next((x.repo_group_id for x in orgs if x.login.lower() == owner.lower()),None) + # search for matching org + org_id = next( + (x.repo_group_id for x in orgs if x.login.lower() == owner.lower()), None) - #print(f"!!{org_id}") - for repo_url in repo_urls: - repos.append(Repository(repo_url,org_id)) + # print(f"!!{org_id}") + for repo_url in urls: + repos.append(Repository(repo_url, org_id)) return orgs, repos @@ -44,12 +45,13 @@ def parse_repos_and_orgs_into_objects(org_name_list, repo_name_list): with open(metadata_path, "r", encoding="utf-8") as file: tracking_file = json.load(file) - repo_urls = tracking_file["Open Source Projects"] # Track specific repositories e.g. ['dsacms.github.io'] + # Track specific repositories e.g. ['dsacms.github.io'] + repo_urls = tracking_file["Open Source Projects"] # Get two lists of objects that will hold all the new metrics all_orgs, all_repos = parse_repos_and_orgs_into_objects( tracking_file["orgs"], repo_urls) - + # Generate json data, report data, and graph data. get_all_data(all_orgs, all_repos) generate_repo_report_files(all_repos) diff --git a/scripts/run_pylint.py b/scripts/run_pylint.py index 67a6704b5c..dfa1bcc9db 100644 --- a/scripts/run_pylint.py +++ b/scripts/run_pylint.py @@ -1,3 +1,3 @@ from tests.metrics_tests import run_pylint -run_pylint() \ No newline at end of file +run_pylint() diff --git a/scripts/tests/metrics_tests.py b/scripts/tests/metrics_tests.py index 25a951e9b6..e8c2d93f5c 100644 --- a/scripts/tests/metrics_tests.py +++ b/scripts/tests/metrics_tests.py @@ -1,30 +1,32 @@ -import pytest +""" +Module to describe data structures related to testing and code quality. +""" import json -from metricsLib import metrics_data_structures import subprocess def test_base_metric(): raise NotImplementedError + def run_pylint(): - #Run pylint command and parse json - pylint_out = subprocess.run(['pylint', '--output-format=json', "--init-hook=\"import sys; sys.path.append('scripts/')\"", "scripts/*"],stdout = subprocess.PIPE,check=False) + # Run pylint command and parse json + pylint_out = subprocess.run(['pylint', '--output-format=json', + "--init-hook=\"import sys; sys.path.append('scripts/')\"", "scripts/*"], stdout=subprocess.PIPE, check=False) pylint_list = json.loads(pylint_out.stdout) - - #Parse pylint format into GitHub Checks Annotation format + # Parse pylint format into GitHub Checks Annotation format for item in pylint_list: - #Rename error type to annotation_level + # Rename error type to annotation_level item['annotation_level'] = item.pop('type') - #translate pylint terms to github terms + # translate pylint terms to github terms if item['annotation_level'] == "convention": item['annotation_level'] = 'notice' elif item['annotation_level'] == "refactor": item['annotation_level'] = 'warning' - - #if end fields are empty, make them the same as the start + + # if end fields are empty, make them the same as the start item['end_line'] = item.pop('endLine') if item['end_line'] is None: item['end_line'] = item['line'] @@ -36,14 +38,14 @@ def run_pylint(): item['end_column'] = item['start_column'] item['title'] = item.pop('symbol') - #Don't include columns if annotation spans multiple lines + # Don't include columns if annotation spans multiple lines if item['start_line'] != item['end_line']: item.pop('start_column') item.pop('end_column') item.pop('module') item.pop('obj') item.pop('message-id') - + to_return = json.dumps(pylint_list, indent=2) - #Print max of 50 terms since thats the max for GitHub + # Print max of 50 terms since thats the max for GitHub print(to_return[:50])