From ded84703e939c37ad0def67587aa5fcebec4566d Mon Sep 17 00:00:00 2001
From: Mike Ounsworth <mike@ounsworth.ca>
Date: Sun, 21 Jul 2024 23:38:10 -0500
Subject: [PATCH 01/18] Refactoring for readability

---
 src/pqc_report_writer_common.py | 44 ++++++++++++++++-----------------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/src/pqc_report_writer_common.py b/src/pqc_report_writer_common.py
index c03bc4ab..b3803e57 100644
--- a/src/pqc_report_writer_common.py
+++ b/src/pqc_report_writer_common.py
@@ -36,7 +36,7 @@ def _parse_csv_file(
 ) -> Sequence[AlgorithmVerificationResult]:
     c = csv.DictReader(f)
 
-    avrs = []
+    algorithmVerificationResults = []
 
     for row in c:
         try:
@@ -50,7 +50,7 @@ def _parse_csv_file(
                 'test_result': row['test_result']
             }
 
-            avrs.append(AlgorithmVerificationResult(**d))
+            algorithmVerificationResults.append(AlgorithmVerificationResult(**d))
 
             if row['test_result'] != None and row['test_result'] != "":
                 e = {
@@ -68,12 +68,12 @@ def _parse_csv_file(
             print("Error reading "+ str(f.name))
             raise e
 
-    return avrs
+    return algorithmVerificationResults
 
-def _format_result_cell(avr) -> str:
+def _format_result_cell(algorithmVerificationResult) -> str:
     result_lines = []
 
-    r = getattr(avr, 'test_result')
+    r = getattr(algorithmVerificationResult, 'test_result')
 
     if r is None or r == "":
         display_result = '?'
@@ -146,7 +146,7 @@ def main():
     with open("oids.json", "w") as f:  
         json.dump(oids_json, f)
 
-    avrs = []
+    algorithmVerificationResults = []
 
     for file in args.files:
         m = _FILENAME_REGEX.match(os.path.basename(file))
@@ -159,30 +159,30 @@ def main():
             verifier = m['verifier']
 
             if m['extension'].casefold() == 'csv'.casefold():
-                avrs.extend(_parse_csv_file(generator, verifier, f, oid_name_mappings, args.include_all_oids))
+                algorithmVerificationResults.extend(_parse_csv_file(generator, verifier, f, oid_name_mappings, args.include_all_oids))
             else:
-                avrs.extend(_parse_json_file(generator, verifier, f))
+                algorithmVerificationResults.extend(_parse_json_file(generator, verifier, f))
 
     generators = set()
     verifiers = set()
-    for avr in avrs:
-        generators.add(avr.generator)
-        verifiers.add(avr.verifier)
+    for algorithmVerificationResult in algorithmVerificationResults:
+        generators.add(algorithmVerificationResult.generator)
+        verifiers.add(algorithmVerificationResult.verifier)
 
     generators = list(generators)
     generators.sort()
     verifiers = list (verifiers)
     verifiers.sort()
 
-    algorithms = list({avr.key_algorithm_oid for avr in avrs})
+    algorithms = list({algorithmVerificationResult.key_algorithm_oid for algorithmVerificationResult in algorithmVerificationResults})
     algorithms.sort()
 
     alg_oid_getter = operator.attrgetter('key_algorithm_oid')
-    avrs.sort(key=alg_oid_getter)
+    algorithmVerificationResults.sort(key=alg_oid_getter)
 
     avrs_by_alg = {k: [] for k in algorithms}
-    for avr in avrs:
-        avrs_by_alg[avr.key_algorithm_oid].append(avr)
+    for algorithmVerificationResult in algorithmVerificationResults:
+        avrs_by_alg[algorithmVerificationResult.key_algorithm_oid].append(algorithmVerificationResult)
 
     md_file = MdUtils(file_name=args.outfile, title=f'IETF PQC Hackathon {args.interop_type} Interoperability Results')
 
@@ -191,20 +191,20 @@ def main():
 
 
     md_file.new_header(level=1, title=f'Algorithms Submitted')
-
+    md_file.new_paragraph(text="✅ = passing on all verifiers\n◒ = passing on some verifiers\n○ = not passing any verifiers")
     _submittedAlgsList.sort()
 
     submittedAlgsCells = ['-'] + generators
     _sars.sort(key=alg_oid_getter)
     sars_by_alg = {k: [] for k in _submittedAlgsList}
-    for sar in _sars:
-        sars_by_alg[sar.key_algorithm_oid].append(sar)
+    for SubmittedAlgorithmResult in _sars:
+        sars_by_alg[SubmittedAlgorithmResult.key_algorithm_oid].append(SubmittedAlgorithmResult)
 
 
-    for alg_oid, sars in sars_by_alg.items():
+    for alg_oid, SubmittedAlgorithmResults in sars_by_alg.items():
         submittedAlgsCells.append(_get_alg_name_by_oid_str(oid_name_mappings, alg_oid))
         for generator in generators:
-            relevant_sars = [sar for sar in sars if sar.generator == generator ]
+            relevant_sars = [SubmittedAlgorithmResult for SubmittedAlgorithmResult in SubmittedAlgorithmResults if SubmittedAlgorithmResult.generator == generator ]
 
             if len(relevant_sars) > 1:
                 raise ValueError(f'Multiple results for {generator}')
@@ -219,7 +219,7 @@ def main():
 
 
 
-    for alg_oid, avrs in avrs_by_alg.items():
+    for alg_oid, algorithmVerificationResults in avrs_by_alg.items():
         alg_name = _get_alg_name_by_oid_str(oid_name_mappings, alg_oid)
 
         md_file.new_header(level=1, title=f'{alg_name} ({alg_oid})')
@@ -230,7 +230,7 @@ def main():
             cells.append(generator)
 
             for verifier in verifiers:
-                relevant_avrs = [avr for avr in avrs if avr.generator == generator and avr.verifier == verifier]
+                relevant_avrs = [algorithmVerificationResult for algorithmVerificationResult in algorithmVerificationResults if algorithmVerificationResult.generator == generator and algorithmVerificationResult.verifier == verifier]
 
                 if len(relevant_avrs) > 1:
                     raise ValueError(f'Multiple results for {alg_oid}: {generator}-{verifier}')

From 9d68c76217e29f826e59011cca00abfd7509ada2 Mon Sep 17 00:00:00 2001
From: Mike Ounsworth <mike@ounsworth.ca>
Date: Mon, 22 Jul 2024 00:09:01 -0500
Subject: [PATCH 02/18] Table shows if a generator passed all verifiers

---
 src/pqc_report_writer_common.py | 96 ++++++++++++++++++++++++---------
 1 file changed, 71 insertions(+), 25 deletions(-)

diff --git a/src/pqc_report_writer_common.py b/src/pqc_report_writer_common.py
index b3803e57..9cc48065 100644
--- a/src/pqc_report_writer_common.py
+++ b/src/pqc_report_writer_common.py
@@ -18,7 +18,7 @@ class SubmittedAlgorithmResult(NamedTuple):
     key_algorithm_oid: str
 
 # set to automatically not add duplicates
-_sars = []
+_submittedAlgorithmResults = []
 _submittedAlgsList = []
 
 class AlgorithmVerificationResult(NamedTuple):
@@ -27,16 +27,48 @@ class AlgorithmVerificationResult(NamedTuple):
     key_algorithm_oid: str
     test_result: Optional[bool]
 
+_algorithmVerificationResults = []
+
 def _parse_json_file(generator, verifier, f) -> Sequence[AlgorithmVerificationResult]:
     pass
 
 
+def passedAllVerifiers(generator, oid) -> int:
+"""
+-1: no verifiers
+0: did not pass any verifiers
+1: passed some verifiers
+2: passed all verifiers
+"""
+    passedOne = False
+    failedOne = False
+
+    relevant_avrs = [algorithmVerificationResult for algorithmVerificationResult in _algorithmVerificationResults if algorithmVerificationResult.generator == generator and algorithmVerificationResult.key_algorithm_oid == oid]
+
+    for algorithmVerificationResult in relevant_avrs:
+        if algorithmVerificationResult.test_result is None:
+            continue
+
+        if algorithmVerificationResult.test_result:
+            passedOne = True
+        else:
+            failedOne = True
+
+        if not passedOne and not failedOne:
+            return -1
+        elif not passedOne and failedOne:
+            return 0
+        elif passedone and failedOne:
+            return 1
+        elif passedOne and not failedOne:
+            return 2
+    
 def _parse_csv_file(
     generator, verifier, f, oid_name_mappings, include_all_oids
 ) -> Sequence[AlgorithmVerificationResult]:
     c = csv.DictReader(f)
 
-    algorithmVerificationResults = []
+    _algorithmVerificationResults = []
 
     for row in c:
         try:
@@ -50,7 +82,7 @@ def _parse_csv_file(
                 'test_result': row['test_result']
             }
 
-            algorithmVerificationResults.append(AlgorithmVerificationResult(**d))
+            _algorithmVerificationResults.append(AlgorithmVerificationResult(**d))
 
             if row['test_result'] != None and row['test_result'] != "":
                 e = {
@@ -59,8 +91,8 @@ def _parse_csv_file(
                 }
 
                 # The algorithms Tested table should only contain tests with a pass or fail result
-                if SubmittedAlgorithmResult(**e) not in _sars:
-                    _sars.append(SubmittedAlgorithmResult(**e))
+                if SubmittedAlgorithmResult(**e) not in _submittedAlgorithmResults:
+                    _submittedAlgorithmResults.append(SubmittedAlgorithmResult(**e))
 
                 if key_algorithm_oid not in _submittedAlgsList:
                     _submittedAlgsList.append( key_algorithm_oid )
@@ -68,7 +100,7 @@ def _parse_csv_file(
             print("Error reading "+ str(f.name))
             raise e
 
-    return algorithmVerificationResults
+    return _algorithmVerificationResults
 
 def _format_result_cell(algorithmVerificationResult) -> str:
     result_lines = []
@@ -145,8 +177,7 @@ def main():
 
     with open("oids.json", "w") as f:  
         json.dump(oids_json, f)
-
-    algorithmVerificationResults = []
+        
 
     for file in args.files:
         m = _FILENAME_REGEX.match(os.path.basename(file))
@@ -159,13 +190,13 @@ def main():
             verifier = m['verifier']
 
             if m['extension'].casefold() == 'csv'.casefold():
-                algorithmVerificationResults.extend(_parse_csv_file(generator, verifier, f, oid_name_mappings, args.include_all_oids))
+                _algorithmVerificationResults.extend(_parse_csv_file(generator, verifier, f, oid_name_mappings, args.include_all_oids))
             else:
-                algorithmVerificationResults.extend(_parse_json_file(generator, verifier, f))
+                _algorithmVerificationResults.extend(_parse_json_file(generator, verifier, f))
 
     generators = set()
     verifiers = set()
-    for algorithmVerificationResult in algorithmVerificationResults:
+    for algorithmVerificationResult in _algorithmVerificationResults:
         generators.add(algorithmVerificationResult.generator)
         verifiers.add(algorithmVerificationResult.verifier)
 
@@ -174,14 +205,14 @@ def main():
     verifiers = list (verifiers)
     verifiers.sort()
 
-    algorithms = list({algorithmVerificationResult.key_algorithm_oid for algorithmVerificationResult in algorithmVerificationResults})
+    algorithms = list({algorithmVerificationResult.key_algorithm_oid for algorithmVerificationResult in _algorithmVerificationResults})
     algorithms.sort()
 
     alg_oid_getter = operator.attrgetter('key_algorithm_oid')
-    algorithmVerificationResults.sort(key=alg_oid_getter)
+    _algorithmVerificationResults.sort(key=alg_oid_getter)
 
     avrs_by_alg = {k: [] for k in algorithms}
-    for algorithmVerificationResult in algorithmVerificationResults:
+    for algorithmVerificationResult in _algorithmVerificationResults:
         avrs_by_alg[algorithmVerificationResult.key_algorithm_oid].append(algorithmVerificationResult)
 
     md_file = MdUtils(file_name=args.outfile, title=f'IETF PQC Hackathon {args.interop_type} Interoperability Results')
@@ -195,31 +226,46 @@ def main():
     _submittedAlgsList.sort()
 
     submittedAlgsCells = ['-'] + generators
-    _sars.sort(key=alg_oid_getter)
+    _submittedAlgorithmResults.sort(key=alg_oid_getter)
     sars_by_alg = {k: [] for k in _submittedAlgsList}
-    for SubmittedAlgorithmResult in _sars:
+    for SubmittedAlgorithmResult in _submittedAlgorithmResults:
         sars_by_alg[SubmittedAlgorithmResult.key_algorithm_oid].append(SubmittedAlgorithmResult)
 
 
     for alg_oid, SubmittedAlgorithmResults in sars_by_alg.items():
         submittedAlgsCells.append(_get_alg_name_by_oid_str(oid_name_mappings, alg_oid))
         for generator in generators:
-            relevant_sars = [SubmittedAlgorithmResult for SubmittedAlgorithmResult in SubmittedAlgorithmResults if SubmittedAlgorithmResult.generator == generator ]
+            """
+            -1: no verifiers
+            0: did not pass any verifiers
+            1: passed some verifiers
+            2: passed all verifiers
+            """
+            no = passedAllVerifiers(generator, alg_oid)
+            if (no == -1):
+                submittedAlgsCells.append('')
+            elif (no == 0):
+                submittedAlgsCells.append('○')
+            elif (no == 1):
+                submittedAlgsCells.append('◒')
+            else:
+                submittedAlgsCells.append('✅')
+            # relevant_submittedAlgorithmResults = [SubmittedAlgorithmResult for SubmittedAlgorithmResult in SubmittedAlgorithmResults if SubmittedAlgorithmResult.generator == generator ]
 
-            if len(relevant_sars) > 1:
-                raise ValueError(f'Multiple results for {generator}')
+            # if len(relevant_submittedAlgorithmResults) > 1:
+            #     raise ValueError(f'Multiple results for {generator}')
 
-            if len(relevant_sars) == 1:
-                submittedAlgsCells.append('▣')
-            else:
-                submittedAlgsCells.append('')
+            # if len(relevant_submittedAlgorithmResults) == 1:
+            #     submittedAlgsCells.append('▣')
+            # else:
+            #     submittedAlgsCells.append('')
 
     
     md_file.new_table(columns=len(generators) + 1, rows=len(_submittedAlgsList) + 1, text=submittedAlgsCells, text_align='left')
 
 
 
-    for alg_oid, algorithmVerificationResults in avrs_by_alg.items():
+    for alg_oid, _algorithmVerificationResults in avrs_by_alg.items():
         alg_name = _get_alg_name_by_oid_str(oid_name_mappings, alg_oid)
 
         md_file.new_header(level=1, title=f'{alg_name} ({alg_oid})')
@@ -230,7 +276,7 @@ def main():
             cells.append(generator)
 
             for verifier in verifiers:
-                relevant_avrs = [algorithmVerificationResult for algorithmVerificationResult in algorithmVerificationResults if algorithmVerificationResult.generator == generator and algorithmVerificationResult.verifier == verifier]
+                relevant_avrs = [algorithmVerificationResult for algorithmVerificationResult in _algorithmVerificationResults if algorithmVerificationResult.generator == generator and algorithmVerificationResult.verifier == verifier]
 
                 if len(relevant_avrs) > 1:
                     raise ValueError(f'Multiple results for {alg_oid}: {generator}-{verifier}')

From 85bbd49a8774e2e4ef25ec82da3250bc701560f4 Mon Sep 17 00:00:00 2001
From: Mike Ounsworth <mike@ounsworth.ca>
Date: Mon, 22 Jul 2024 00:11:14 -0500
Subject: [PATCH 03/18] Table shows if a generator passed all verifiers

---
 src/pqc_report_writer_common.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/pqc_report_writer_common.py b/src/pqc_report_writer_common.py
index 9cc48065..a87a8a68 100644
--- a/src/pqc_report_writer_common.py
+++ b/src/pqc_report_writer_common.py
@@ -34,12 +34,12 @@ def _parse_json_file(generator, verifier, f) -> Sequence[AlgorithmVerificationRe
 
 
 def passedAllVerifiers(generator, oid) -> int:
-"""
--1: no verifiers
-0: did not pass any verifiers
-1: passed some verifiers
-2: passed all verifiers
-"""
+    """
+    -1: no verifiers
+    0: did not pass any verifiers
+    1: passed some verifiers
+    2: passed all verifiers
+    """
     passedOne = False
     failedOne = False
 

From 9ab0199165795bb212018ae22528f789cdfda6b1 Mon Sep 17 00:00:00 2001
From: Mike Ounsworth <mike@ounsworth.ca>
Date: Mon, 22 Jul 2024 00:21:10 -0500
Subject: [PATCH 04/18] Table shows if a generator passed all verifiers

---
 src/pqc_report_writer_common.py | 34 ++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/src/pqc_report_writer_common.py b/src/pqc_report_writer_common.py
index a87a8a68..36c7e781 100644
--- a/src/pqc_report_writer_common.py
+++ b/src/pqc_report_writer_common.py
@@ -27,13 +27,11 @@ class AlgorithmVerificationResult(NamedTuple):
     key_algorithm_oid: str
     test_result: Optional[bool]
 
-_algorithmVerificationResults = []
-
 def _parse_json_file(generator, verifier, f) -> Sequence[AlgorithmVerificationResult]:
     pass
 
 
-def passedAllVerifiers(generator, oid) -> int:
+def passedAllVerifiers(generator, oid, algorithmVerificationResults) -> int:
     """
     -1: no verifiers
     0: did not pass any verifiers
@@ -43,7 +41,7 @@ def passedAllVerifiers(generator, oid) -> int:
     passedOne = False
     failedOne = False
 
-    relevant_avrs = [algorithmVerificationResult for algorithmVerificationResult in _algorithmVerificationResults if algorithmVerificationResult.generator == generator and algorithmVerificationResult.key_algorithm_oid == oid]
+    relevant_avrs = [algorithmVerificationResult for algorithmVerificationResult in algorithmVerificationResults if algorithmVerificationResult.generator == generator and algorithmVerificationResult.key_algorithm_oid == oid]
 
     for algorithmVerificationResult in relevant_avrs:
         if algorithmVerificationResult.test_result is None:
@@ -68,7 +66,7 @@ def _parse_csv_file(
 ) -> Sequence[AlgorithmVerificationResult]:
     c = csv.DictReader(f)
 
-    _algorithmVerificationResults = []
+    algorithmVerificationResults = []
 
     for row in c:
         try:
@@ -82,7 +80,7 @@ def _parse_csv_file(
                 'test_result': row['test_result']
             }
 
-            _algorithmVerificationResults.append(AlgorithmVerificationResult(**d))
+            algorithmVerificationResults.append(AlgorithmVerificationResult(**d))
 
             if row['test_result'] != None and row['test_result'] != "":
                 e = {
@@ -100,7 +98,7 @@ def _parse_csv_file(
             print("Error reading "+ str(f.name))
             raise e
 
-    return _algorithmVerificationResults
+    return algorithmVerificationResults
 
 def _format_result_cell(algorithmVerificationResult) -> str:
     result_lines = []
@@ -177,7 +175,9 @@ def main():
 
     with open("oids.json", "w") as f:  
         json.dump(oids_json, f)
-        
+
+    
+    algorithmVerificationResults = []
 
     for file in args.files:
         m = _FILENAME_REGEX.match(os.path.basename(file))
@@ -190,13 +190,13 @@ def main():
             verifier = m['verifier']
 
             if m['extension'].casefold() == 'csv'.casefold():
-                _algorithmVerificationResults.extend(_parse_csv_file(generator, verifier, f, oid_name_mappings, args.include_all_oids))
+                algorithmVerificationResults.extend(_parse_csv_file(generator, verifier, f, oid_name_mappings, args.include_all_oids))
             else:
-                _algorithmVerificationResults.extend(_parse_json_file(generator, verifier, f))
+                algorithmVerificationResults.extend(_parse_json_file(generator, verifier, f))
 
     generators = set()
     verifiers = set()
-    for algorithmVerificationResult in _algorithmVerificationResults:
+    for algorithmVerificationResult in algorithmVerificationResults:
         generators.add(algorithmVerificationResult.generator)
         verifiers.add(algorithmVerificationResult.verifier)
 
@@ -205,14 +205,14 @@ def main():
     verifiers = list (verifiers)
     verifiers.sort()
 
-    algorithms = list({algorithmVerificationResult.key_algorithm_oid for algorithmVerificationResult in _algorithmVerificationResults})
+    algorithms = list({algorithmVerificationResult.key_algorithm_oid for algorithmVerificationResult in algorithmVerificationResults})
     algorithms.sort()
 
     alg_oid_getter = operator.attrgetter('key_algorithm_oid')
-    _algorithmVerificationResults.sort(key=alg_oid_getter)
+    algorithmVerificationResults.sort(key=alg_oid_getter)
 
     avrs_by_alg = {k: [] for k in algorithms}
-    for algorithmVerificationResult in _algorithmVerificationResults:
+    for algorithmVerificationResult in algorithmVerificationResults:
         avrs_by_alg[algorithmVerificationResult.key_algorithm_oid].append(algorithmVerificationResult)
 
     md_file = MdUtils(file_name=args.outfile, title=f'IETF PQC Hackathon {args.interop_type} Interoperability Results')
@@ -241,7 +241,7 @@ def main():
             1: passed some verifiers
             2: passed all verifiers
             """
-            no = passedAllVerifiers(generator, alg_oid)
+            no = passedAllVerifiers(generator, alg_oid, algorithmVerificationResults)
             if (no == -1):
                 submittedAlgsCells.append('')
             elif (no == 0):
@@ -265,7 +265,7 @@ def main():
 
 
 
-    for alg_oid, _algorithmVerificationResults in avrs_by_alg.items():
+    for alg_oid, algorithmVerificationResults in avrs_by_alg.items():
         alg_name = _get_alg_name_by_oid_str(oid_name_mappings, alg_oid)
 
         md_file.new_header(level=1, title=f'{alg_name} ({alg_oid})')
@@ -276,7 +276,7 @@ def main():
             cells.append(generator)
 
             for verifier in verifiers:
-                relevant_avrs = [algorithmVerificationResult for algorithmVerificationResult in _algorithmVerificationResults if algorithmVerificationResult.generator == generator and algorithmVerificationResult.verifier == verifier]
+                relevant_avrs = [algorithmVerificationResult for algorithmVerificationResult in algorithmVerificationResults if algorithmVerificationResult.generator == generator and algorithmVerificationResult.verifier == verifier]
 
                 if len(relevant_avrs) > 1:
                     raise ValueError(f'Multiple results for {alg_oid}: {generator}-{verifier}')

From 6be940029d16a9c8c24eccec2222c0b20e1e3e95 Mon Sep 17 00:00:00 2001
From: Mike Ounsworth <mike@ounsworth.ca>
Date: Mon, 22 Jul 2024 00:22:48 -0500
Subject: [PATCH 05/18] Table shows if a generator passed all verifiers

---
 src/pqc_report_writer_common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pqc_report_writer_common.py b/src/pqc_report_writer_common.py
index 36c7e781..c3dbabb4 100644
--- a/src/pqc_report_writer_common.py
+++ b/src/pqc_report_writer_common.py
@@ -56,7 +56,7 @@ def passedAllVerifiers(generator, oid, algorithmVerificationResults) -> int:
             return -1
         elif not passedOne and failedOne:
             return 0
-        elif passedone and failedOne:
+        elif passedOne and failedOne:
             return 1
         elif passedOne and not failedOne:
             return 2

From 4a3e1d971811911ecd54b67e9b0ee32d75b2f82d Mon Sep 17 00:00:00 2001
From: Mike Ounsworth <mike@ounsworth.ca>
Date: Mon, 22 Jul 2024 00:28:48 -0500
Subject: [PATCH 06/18] Table shows if a generator passed all verifiers

---
 src/pqc_report_writer_common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/pqc_report_writer_common.py b/src/pqc_report_writer_common.py
index c3dbabb4..5d9728b2 100644
--- a/src/pqc_report_writer_common.py
+++ b/src/pqc_report_writer_common.py
@@ -222,7 +222,7 @@ def main():
 
 
     md_file.new_header(level=1, title=f'Algorithms Submitted')
-    md_file.new_paragraph(text="✅ = passing on all verifiers\n◒ = passing on some verifiers\n○ = not passing any verifiers")
+    md_file.new_paragraph(text="✅ = passing all verifiers<br>◒ = passing some verifiers<br>⚪︎ = not passing any verifiers")
     _submittedAlgsList.sort()
 
     submittedAlgsCells = ['-'] + generators
@@ -245,7 +245,7 @@ def main():
             if (no == -1):
                 submittedAlgsCells.append('')
             elif (no == 0):
-                submittedAlgsCells.append('○')
+                submittedAlgsCells.append('⚪︎')
             elif (no == 1):
                 submittedAlgsCells.append('◒')
             else:

From d464730c799e8b65d310fb1bb4a9fc93771c485c Mon Sep 17 00:00:00 2001
From: Mike Ounsworth <mike@ounsworth.ca>
Date: Mon, 22 Jul 2024 00:35:48 -0500
Subject: [PATCH 07/18] Table shows if a generator passed all verifiers

---
 src/pqc_report_writer_common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pqc_report_writer_common.py b/src/pqc_report_writer_common.py
index 5d9728b2..3068a374 100644
--- a/src/pqc_report_writer_common.py
+++ b/src/pqc_report_writer_common.py
@@ -260,7 +260,7 @@ def main():
             # else:
             #     submittedAlgsCells.append('')
 
-    
+    print("DEBUG: cols: "+str(len(generators)+1)+"x rows: "+str(len(_submittedAlgsList)+1)+" =? "+len(submittedAlgsCells)
     md_file.new_table(columns=len(generators) + 1, rows=len(_submittedAlgsList) + 1, text=submittedAlgsCells, text_align='left')
 
 

From 8968d372cff7d1c3c41283574ca470348e2a5ac4 Mon Sep 17 00:00:00 2001
From: Mike Ounsworth <mike@ounsworth.ca>
Date: Mon, 22 Jul 2024 00:37:03 -0500
Subject: [PATCH 08/18] Table shows if a generator passed all verifiers

---
 src/pqc_report_writer_common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pqc_report_writer_common.py b/src/pqc_report_writer_common.py
index 3068a374..20550741 100644
--- a/src/pqc_report_writer_common.py
+++ b/src/pqc_report_writer_common.py
@@ -260,7 +260,7 @@ def main():
             # else:
             #     submittedAlgsCells.append('')
 
-    print("DEBUG: cols: "+str(len(generators)+1)+"x rows: "+str(len(_submittedAlgsList)+1)+" =? "+len(submittedAlgsCells)
+    print("DEBUG: cols: "+str(len(generators)+1)+"x rows: "+str(len(_submittedAlgsList)+1)+" =? "+len(submittedAlgsCells))
     md_file.new_table(columns=len(generators) + 1, rows=len(_submittedAlgsList) + 1, text=submittedAlgsCells, text_align='left')
 
 

From 42b6c37e23bb126ccdc20a497086b2c54ff2cb8a Mon Sep 17 00:00:00 2001
From: Mike Ounsworth <mike@ounsworth.ca>
Date: Mon, 22 Jul 2024 00:38:31 -0500
Subject: [PATCH 09/18] Table shows if a generator passed all verifiers

---
 src/pqc_report_writer_common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pqc_report_writer_common.py b/src/pqc_report_writer_common.py
index 20550741..e8293789 100644
--- a/src/pqc_report_writer_common.py
+++ b/src/pqc_report_writer_common.py
@@ -260,7 +260,7 @@ def main():
             # else:
             #     submittedAlgsCells.append('')
 
-    print("DEBUG: cols: "+str(len(generators)+1)+"x rows: "+str(len(_submittedAlgsList)+1)+" =? "+len(submittedAlgsCells))
+    print("DEBUG: cols: "+str(len(generators)+1)+"x rows: "+str(len(_submittedAlgsList)+1)+" =? "+str(len(submittedAlgsCells)))
     md_file.new_table(columns=len(generators) + 1, rows=len(_submittedAlgsList) + 1, text=submittedAlgsCells, text_align='left')
 
 

From 833449932be1fc1cc21dd3f06215daccbe0e26b6 Mon Sep 17 00:00:00 2001
From: Mike Ounsworth <mike@ounsworth.ca>
Date: Mon, 22 Jul 2024 00:41:15 -0500
Subject: [PATCH 10/18] Table shows if a generator passed all verifiers

---
 src/pqc_report_writer_common.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/pqc_report_writer_common.py b/src/pqc_report_writer_common.py
index e8293789..754a9bde 100644
--- a/src/pqc_report_writer_common.py
+++ b/src/pqc_report_writer_common.py
@@ -222,7 +222,7 @@ def main():
 
 
     md_file.new_header(level=1, title=f'Algorithms Submitted')
-    md_file.new_paragraph(text="✅ = passing all verifiers<br>◒ = passing some verifiers<br>⚪︎ = not passing any verifiers")
+    md_file.new_paragraph(text="✅ = passing all verifiers<br>◒ = passing some verifiers<br>⚪︎ = not passing any verifiers<br>")
     _submittedAlgsList.sort()
 
     submittedAlgsCells = ['-'] + generators
@@ -259,8 +259,7 @@ def main():
             #     submittedAlgsCells.append('▣')
             # else:
             #     submittedAlgsCells.append('')
-
-    print("DEBUG: cols: "+str(len(generators)+1)+"x rows: "+str(len(_submittedAlgsList)+1)+" =? "+str(len(submittedAlgsCells)))
+    
     md_file.new_table(columns=len(generators) + 1, rows=len(_submittedAlgsList) + 1, text=submittedAlgsCells, text_align='left')
 
 

From 11b733254037997cc7ca670e7dd50e437b3966f2 Mon Sep 17 00:00:00 2001
From: Mike Ounsworth <mike@ounsworth.ca>
Date: Mon, 22 Jul 2024 00:46:25 -0500
Subject: [PATCH 11/18] Table shows if a generator passed all verifiers

---
 src/pqc_report_writer_common.py | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/src/pqc_report_writer_common.py b/src/pqc_report_writer_common.py
index 754a9bde..89ca9048 100644
--- a/src/pqc_report_writer_common.py
+++ b/src/pqc_report_writer_common.py
@@ -245,21 +245,12 @@ def main():
             if (no == -1):
                 submittedAlgsCells.append('')
             elif (no == 0):
-                submittedAlgsCells.append('⚪︎')
+                submittedAlgsCells.append('a')
             elif (no == 1):
-                submittedAlgsCells.append('◒')
+                submittedAlgsCells.append('b')
             else:
                 submittedAlgsCells.append('✅')
-            # relevant_submittedAlgorithmResults = [SubmittedAlgorithmResult for SubmittedAlgorithmResult in SubmittedAlgorithmResults if SubmittedAlgorithmResult.generator == generator ]
 
-            # if len(relevant_submittedAlgorithmResults) > 1:
-            #     raise ValueError(f'Multiple results for {generator}')
-
-            # if len(relevant_submittedAlgorithmResults) == 1:
-            #     submittedAlgsCells.append('▣')
-            # else:
-            #     submittedAlgsCells.append('')
-    
     md_file.new_table(columns=len(generators) + 1, rows=len(_submittedAlgsList) + 1, text=submittedAlgsCells, text_align='left')
 
 

From 958e3d98d9c366b91644c65c260a864357226ea8 Mon Sep 17 00:00:00 2001
From: Mike Ounsworth <mike@ounsworth.ca>
Date: Mon, 22 Jul 2024 00:49:51 -0500
Subject: [PATCH 12/18]  Table shows if a generator passed all verifiers

---
 src/pqc_report_writer_common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/pqc_report_writer_common.py b/src/pqc_report_writer_common.py
index 89ca9048..544d65c0 100644
--- a/src/pqc_report_writer_common.py
+++ b/src/pqc_report_writer_common.py
@@ -245,9 +245,9 @@ def main():
             if (no == -1):
                 submittedAlgsCells.append('')
             elif (no == 0):
-                submittedAlgsCells.append('a')
+                submittedAlgsCells.append('⚪︎')
             elif (no == 1):
-                submittedAlgsCells.append('b')
+                submittedAlgsCells.append('◒')
             else:
                 submittedAlgsCells.append('✅')
 

From b99fd2be008b60ad8bc2cd4b0032d09c1f7dd600 Mon Sep 17 00:00:00 2001
From: Mike Ounsworth <mike@ounsworth.ca>
Date: Mon, 22 Jul 2024 00:52:14 -0500
Subject: [PATCH 13/18] Table shows if a generator passed all verifiers

---
 .github/workflows/artifact_validation.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/artifact_validation.yaml b/.github/workflows/artifact_validation.yaml
index 7e1944ba..008511c0 100644
--- a/.github/workflows/artifact_validation.yaml
+++ b/.github/workflows/artifact_validation.yaml
@@ -52,6 +52,8 @@ jobs:
         run: ./src/rebuild_results_certs_r3.sh
       - name: Copy output files
         run: mv ./docs/pqc_hackathon_results_certs_r3.html ./docs/pqc_hackathon_results_certs_r3_automated_tests.html ./docs/pqc_hackathon_results_cms_v1.html ./output/certs/oqs_certs.log ./docs/gh-pages
+      - name: "DEBUG: copy debug files"
+        run: mv ./docs/pqc_hackathon_results_certs_r3.md ./docs/pqc_hackathon_results_certs_r3_automated_tests.md ./docs/pqc_hackathon_results_cms_v1.md ./docs/gh-pages
       - name: Archive Compatibility Matrix For Download
         uses:  actions/upload-pages-artifact@v3
         with:

From eb2799e50fbe35eeb65ded770bfc87f48ae42ded Mon Sep 17 00:00:00 2001
From: Mike Ounsworth <mike@ounsworth.ca>
Date: Mon, 22 Jul 2024 00:56:11 -0500
Subject: [PATCH 14/18] Table shows if a generator passed all verifiers

---
 src/pqc_report_writer_common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/pqc_report_writer_common.py b/src/pqc_report_writer_common.py
index 544d65c0..b921cdbb 100644
--- a/src/pqc_report_writer_common.py
+++ b/src/pqc_report_writer_common.py
@@ -223,6 +223,7 @@ def main():
 
     md_file.new_header(level=1, title=f'Algorithms Submitted')
     md_file.new_paragraph(text="✅ = passing all verifiers<br>◒ = passing some verifiers<br>⚪︎ = not passing any verifiers<br>")
+    md_file.new_paragraph()
     _submittedAlgsList.sort()
 
     submittedAlgsCells = ['-'] + generators

From 2097b8dde1f283e2f440fedf8bcf1adc388c6527 Mon Sep 17 00:00:00 2001
From: Mike Ounsworth <mike@ounsworth.ca>
Date: Mon, 22 Jul 2024 00:58:23 -0500
Subject: [PATCH 15/18] Table shows if a generator passed all verifiers

---
 .github/workflows/artifact_validation.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/artifact_validation.yaml b/.github/workflows/artifact_validation.yaml
index 008511c0..7e1944ba 100644
--- a/.github/workflows/artifact_validation.yaml
+++ b/.github/workflows/artifact_validation.yaml
@@ -52,8 +52,6 @@ jobs:
         run: ./src/rebuild_results_certs_r3.sh
       - name: Copy output files
         run: mv ./docs/pqc_hackathon_results_certs_r3.html ./docs/pqc_hackathon_results_certs_r3_automated_tests.html ./docs/pqc_hackathon_results_cms_v1.html ./output/certs/oqs_certs.log ./docs/gh-pages
-      - name: "DEBUG: copy debug files"
-        run: mv ./docs/pqc_hackathon_results_certs_r3.md ./docs/pqc_hackathon_results_certs_r3_automated_tests.md ./docs/pqc_hackathon_results_cms_v1.md ./docs/gh-pages
       - name: Archive Compatibility Matrix For Download
         uses:  actions/upload-pages-artifact@v3
         with:

From 07e63fd56396cbfcb156a2d7d86cd868cfc667c5 Mon Sep 17 00:00:00 2001
From: Mike Ounsworth <mike@ounsworth.ca>
Date: Mon, 22 Jul 2024 01:00:25 -0500
Subject: [PATCH 16/18] Table shows if a generator passed all verifiers

---
 src/pqc_report_writer_common.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/pqc_report_writer_common.py b/src/pqc_report_writer_common.py
index b921cdbb..eecc117c 100644
--- a/src/pqc_report_writer_common.py
+++ b/src/pqc_report_writer_common.py
@@ -43,6 +43,8 @@ def passedAllVerifiers(generator, oid, algorithmVerificationResults) -> int:
 
     relevant_avrs = [algorithmVerificationResult for algorithmVerificationResult in algorithmVerificationResults if algorithmVerificationResult.generator == generator and algorithmVerificationResult.key_algorithm_oid == oid]
 
+    print("DEBUG: relevant_avrs for "+generator+", ("+oid+" is: "+str(relevant_avrs))
+    
     for algorithmVerificationResult in relevant_avrs:
         if algorithmVerificationResult.test_result is None:
             continue

From f03745408d4a366885b580c89f71f3deab9c4a91 Mon Sep 17 00:00:00 2001
From: Mike Ounsworth <mike@ounsworth.ca>
Date: Mon, 22 Jul 2024 01:06:32 -0500
Subject: [PATCH 17/18] Table shows if a generator passed all verifiers

---
 src/pqc_report_writer_common.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/pqc_report_writer_common.py b/src/pqc_report_writer_common.py
index eecc117c..76014171 100644
--- a/src/pqc_report_writer_common.py
+++ b/src/pqc_report_writer_common.py
@@ -43,13 +43,13 @@ def passedAllVerifiers(generator, oid, algorithmVerificationResults) -> int:
 
     relevant_avrs = [algorithmVerificationResult for algorithmVerificationResult in algorithmVerificationResults if algorithmVerificationResult.generator == generator and algorithmVerificationResult.key_algorithm_oid == oid]
 
-    print("DEBUG: relevant_avrs for "+generator+", ("+oid+" is: "+str(relevant_avrs))
+    print("DEBUG: relevant_avrs for "+generator+", ("+oid+") is: "+str(relevant_avrs))
     
     for algorithmVerificationResult in relevant_avrs:
-        if algorithmVerificationResult.test_result is None:
+        if algorithmVerificationResult.test_result is None or algorithmVerificationResult.test_result is '':
             continue
 
-        if algorithmVerificationResult.test_result:
+        if algorithmVerificationResult.test_result is 'Y':
             passedOne = True
         else:
             failedOne = True

From 4935858be59b2d503e7e5847126e7a82957bdba9 Mon Sep 17 00:00:00 2001
From: Mike Ounsworth <mike@ounsworth.ca>
Date: Mon, 22 Jul 2024 01:10:49 -0500
Subject: [PATCH 18/18] Table shows if a generator passed all verifiers

---
 src/pqc_report_writer_common.py | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/src/pqc_report_writer_common.py b/src/pqc_report_writer_common.py
index 76014171..73bb77ad 100644
--- a/src/pqc_report_writer_common.py
+++ b/src/pqc_report_writer_common.py
@@ -42,8 +42,6 @@ def passedAllVerifiers(generator, oid, algorithmVerificationResults) -> int:
     failedOne = False
 
     relevant_avrs = [algorithmVerificationResult for algorithmVerificationResult in algorithmVerificationResults if algorithmVerificationResult.generator == generator and algorithmVerificationResult.key_algorithm_oid == oid]
-
-    print("DEBUG: relevant_avrs for "+generator+", ("+oid+") is: "+str(relevant_avrs))
     
     for algorithmVerificationResult in relevant_avrs:
         if algorithmVerificationResult.test_result is None or algorithmVerificationResult.test_result is '':
@@ -54,14 +52,14 @@ def passedAllVerifiers(generator, oid, algorithmVerificationResults) -> int:
         else:
             failedOne = True
 
-        if not passedOne and not failedOne:
-            return -1
-        elif not passedOne and failedOne:
-            return 0
-        elif passedOne and failedOne:
-            return 1
-        elif passedOne and not failedOne:
-            return 2
+    if not passedOne and not failedOne:
+        return -1
+    elif not passedOne and failedOne:
+        return 0
+    elif passedOne and failedOne:
+        return 1
+    elif passedOne and not failedOne:
+        return 2
     
 def _parse_csv_file(
     generator, verifier, f, oid_name_mappings, include_all_oids