[Feature:Plagiarism] Improve run log readability (#74)

* Only print warning once

* add number of times it was truncated

* Add progress bars for most of pipeline

* Add compare_hashes progress bar

Author: williamjallen

bin/concatenate_all.py

@@ -8,7 +8,7 @@
 import os
 import sys
 import json
-import time
+import datetime
 import humanize
 import fnmatch
 import hashlib
@@ -186,10 +186,10 @@ def validate(config, args):
 
 
 def main():
-    start_time = time.time()
+    start_time = datetime.datetime.now()
     args = parse_args()
 
-    print("CONCATENATE ALL...", end="")
+    print("CONCATENATE ALL:", flush=True)
 
     config_path = os.path.join(args.basepath, "config.json")
     if not os.path.isfile(config_path):
@@ -291,9 +291,8 @@ def main():
     checkTotalSize(total_concat)
 
     # ==========================================================================
-    end_time = time.time()
-    print("done in " + "%.0f" % (end_time - start_time) + " seconds,",
-          humanize.naturalsize(total_concat) + " concatenated")
+    print("Concatenation done in", humanize.precisedelta(start_time, format="%1.f") + ",",
+          humanize.naturalsize(total_concat), "concatenated")
 
 
 if __name__ == "__main__":

bin/hash_all.py

@@ -8,9 +8,10 @@
 import argparse
 import os
 import json
-import time
 import hashlib
 from pathlib import Path
+import humanize
+import datetime
 
 
 def parse_args():
@@ -48,7 +49,7 @@ def hasher(lichen_config, lichen_run_config, my_tokenized_file, my_hashes_file):
 
 
 def main():
-    start_time = time.time()
+    start_time = datetime.datetime.now()
     args = parse_args()
 
     with open(Path(args.basepath, "config.json")) as lichen_run_config_file:
@@ -57,13 +58,26 @@ def main():
     with open(Path(__file__).resolve().parent / "lichen_config.json") as lichen_config_file:
         lichen_config = json.load(lichen_config_file)
 
-    print("HASH ALL...", end="")
+    print("HASH ALL:", flush="True")
+    print("[0%                      25%                     50%                     75%                     100%]\n[", end="", flush=True)  # noqa: E501
+
+    users_dir = os.path.join(args.basepath, "users")
+    if not os.path.isdir(users_dir):
+        raise SystemExit("ERROR: Unable to find users directory")
+
+    other_gradeables_dir = os.path.join(args.basepath, "other_gradeables")
+    if not os.path.isdir(other_gradeables_dir):
+        raise SystemExit("ERROR: Unable to find other gradeables directory")
+
+    total_users = len(os.listdir(users_dir))
+    for dir in os.listdir(other_gradeables_dir):
+        total_users += len(os.listdir(os.path.join(other_gradeables_dir, dir)))
+
+    users_hashed = 0
+    percent_progress = 0
 
     # ==========================================================================
     # walk the subdirectories of this gradeable
-    users_dir = Path(args.basepath, "users")
-    if not os.path.isdir(users_dir):
-        raise SystemExit("ERROR! Unable to find users directory")
 
     for user in sorted(os.listdir(users_dir)):
         user_dir = Path(users_dir, user)
@@ -79,13 +93,15 @@ def main():
             my_hashes_file = Path(my_dir, "hashes.txt")
             hasher(lichen_config, lichen_run_config, my_tokenized_file, my_hashes_file)
 
+        users_hashed += 1
+        if int((users_hashed / total_users) * 100) > percent_progress:
+            new_percent_progress = int((users_hashed / total_users) * 100)
+            print("|" * (new_percent_progress - percent_progress), end="", flush=True)
+            percent_progress = new_percent_progress
+
     # ==========================================================================
     # walk the subdirectories of the other gradeables
 
-    other_gradeables_dir = Path(args.basepath, "other_gradeables")
-    if not os.path.isdir(other_gradeables_dir):
-        raise SystemExit("ERROR! Unable to find other gradeables directory")
-
     for other_gradeable in sorted(os.listdir(other_gradeables_dir)):
         other_gradeable_dir = Path(other_gradeables_dir, other_gradeable)
         if not os.path.isdir(other_gradeable_dir):
@@ -105,15 +121,20 @@ def main():
                 other_hashes_file = Path(other_version_dir, "hashes.txt")
                 hasher(lichen_config, lichen_run_config, other_tokenized_file, other_hashes_file)
 
+            users_hashed += 1
+            if int((users_hashed / total_users) * 100) > percent_progress:
+                new_percent_progress = int((users_hashed / total_users) * 100)
+                print("|" * (new_percent_progress - percent_progress), end="", flush=True)
+                percent_progress = new_percent_progress
+
     # ==========================================================================
     # hash the provided code
     provided_code_tokenized = Path(args.basepath, "provided_code", "tokens.json")
     provided_code_hashed = Path(args.basepath, "provided_code", "hashes.txt")
     hasher(lichen_config, lichen_run_config, provided_code_tokenized, provided_code_hashed)
 
     # ==========================================================================
-    end_time = time.time()
-    print("done in " + "%.0f" % (end_time - start_time) + " seconds")
+    print("]\nHashing done in", humanize.precisedelta(start_time, format="%1.f"))
 
 
 if __name__ == "__main__":

bin/tokenize_all.py

@@ -6,7 +6,8 @@
 import argparse
 import os
 import json
-import time
+import humanize
+import datetime
 
 
 def parse_args():
@@ -43,19 +44,34 @@ def tokenize(lichen_config_data, my_concatenated_file, my_tokenized_file):
 
 
 def main():
-    start_time = time.time()
+    start_time = datetime.datetime.now()
     args = parse_args()
 
-    print("TOKENIZE ALL...", end="", flush=True)
+    print("TOKENIZE ALL:", flush=True)
+    print("[0%                      25%                     50%                     75%                     100%]\n[", end="", flush=True)  # noqa: E501
 
     with open(os.path.join(args.basepath, "config.json")) as lichen_config:
         lichen_config_data = json.load(lichen_config)
 
-    # ===========================================================================
-    # walk the subdirectories to tokenize this gradeable's submissions
     users_dir = os.path.join(args.basepath, "users")
     if not os.path.isdir(users_dir):
-        raise SystemExit("ERROR! Unable to find users directory")
+        raise SystemExit("ERROR: Unable to find users directory")
+
+    other_gradeables_dir = os.path.join(args.basepath, "other_gradeables")
+    if not os.path.isdir(other_gradeables_dir):
+        raise SystemExit("ERROR: Unable to find other gradeables directory")
+
+    # We'll make a rough estimate of the percentage of tokenization done by
+    # taking the percentage of users which have been tokenized thus far
+    total_users = len(os.listdir(users_dir))
+    for dir in os.listdir(other_gradeables_dir):
+        total_users += len(os.listdir(os.path.join(other_gradeables_dir, dir)))
+
+    users_tokenized = 0
+    percent_progress = 0
+
+    # ===========================================================================
+    # walk the subdirectories to tokenize this gradeable's submissions
 
     for user in sorted(os.listdir(users_dir)):
         user_dir = os.path.join(users_dir, user)
@@ -71,11 +87,14 @@ def main():
             my_tokenized_file = os.path.join(my_dir, "tokens.json")
             tokenize(lichen_config_data, my_concatenated_file, my_tokenized_file)
 
+        users_tokenized += 1
+        if int((users_tokenized / total_users) * 100) > percent_progress:
+            new_percent_progress = int((users_tokenized / total_users) * 100)
+            print("|" * (new_percent_progress - percent_progress), end="", flush=True)
+            percent_progress = new_percent_progress
+
     # ===========================================================================
     # tokenize the other other gradeables' submissions
-    other_gradeables_dir = os.path.join(args.basepath, "other_gradeables")
-    if not os.path.isdir(other_gradeables_dir):
-        raise SystemExit("ERROR! Unable to find other gradeables directory")
 
     for other_gradeable in sorted(os.listdir(other_gradeables_dir)):
         other_gradeable_dir = os.path.join(other_gradeables_dir, other_gradeable)
@@ -96,15 +115,20 @@ def main():
                 other_tokenized_file = os.path.join(other_version_dir, "tokens.json")
                 tokenize(lichen_config_data, other_concatenated_file, other_tokenized_file)
 
+            users_tokenized += 1
+            if int((users_tokenized / total_users) * 100) > percent_progress:
+                new_percent_progress = int((users_tokenized / total_users) * 100)
+                print("|" * (new_percent_progress - percent_progress), end="", flush=True)
+                percent_progress = new_percent_progress
+
     # ===========================================================================
     # tokenize the provided code
     provided_code_concat = os.path.join(args.basepath, "provided_code", "submission.concatenated")
     provided_code_tokenized = os.path.join(args.basepath, "provided_code", "tokens.json")
     tokenize(lichen_config_data, provided_code_concat, provided_code_tokenized)
 
     # ==========================================================================
-    end_time = time.time()
-    print("done in " + "%.0f" % (end_time - start_time) + " seconds")
+    print("]\nTokenization done in", humanize.precisedelta(start_time, format="%1.f"))
 
 
 if __name__ == "__main__":

compare_hashes/compare_hashes.cpp

@@ -99,7 +99,7 @@ bool ranking_sorter(const StudentRanking &a, const StudentRanking &b) {
 // MAIN
 
 int main(int argc, char* argv[]) {
-  std::cout << "COMPARE HASHES...";
+  std::cout << "COMPARE HASHES:" << std::endl;
   fflush(stdout);
   time_t overall_start, overall_end;
   time(&overall_start);
@@ -253,7 +253,7 @@ int main(int argc, char* argv[]) {
 
   time(&end);
   double diff = difftime(end, start);
-  std::cout << "finished loading in " << diff  << " seconds" << std::endl;
+  std::cout << "Finished loading in " << diff  << " seconds" << std::endl;
 
 
   // ===========================================================================
@@ -264,6 +264,9 @@ int main(int argc, char* argv[]) {
   int my_percent = 0;
   time(&start);
 
+  std::cout << "[0%                      25%                     50%                     75%                     100%]" << std::endl << "[";
+  fflush(stdout);
+
   // walk over every Submission
   for (std::vector<Submission*>::iterator submission_itr = all_submissions.begin();
        submission_itr != all_submissions.end(); ++submission_itr) {
@@ -588,14 +591,23 @@ int main(int argc, char* argv[]) {
     // print current progress
     my_counter++;
     if (int((my_counter / float(all_submissions.size())) * 100) > my_percent) {
-      my_percent = int((my_counter / float(all_submissions.size())) * 100);
-      std::cout << "Processing submissions: " << my_percent << "% complete" << std::endl;
+      int new_my_percent = int((my_counter / float(all_submissions.size())) * 100);
+      for (int i=0; i < new_my_percent - my_percent; i++) {
+        std::cout << "|";
+      }
+      fflush(stdout);
+      my_percent = new_my_percent;
     }
   }
 
+  // Finish printing any remaining portion of the progress bar
+  for (int i=0; i < 100 - my_percent; i++) {
+    std::cout << "|";
+  }
+
   time(&end);
   diff = difftime(end, start);
-  std::cout << "Finished processing submissions in " << diff << " seconds" << std::endl;
+  std::cout << "]" << std::endl << "Finished processing submissions in " << diff << " seconds" << std::endl;
 
   // Print out the list of users who had their matching positions array truncated
   if (matching_positions_truncations.size() > 0) {
@@ -606,6 +618,7 @@ int main(int argc, char* argv[]) {
     }
     std::cout << std::endl << "  - Try increasing the hash size or adding a regex to fix this problem." << std::endl;
   }
+  fflush(stdout);
 
   // ===========================================================================
   // Create a general summary of rankings of users by percentage match
@@ -645,5 +658,5 @@ int main(int argc, char* argv[]) {
 
   time(&overall_end);
   double overall_diff = difftime(overall_end, overall_start);
-  std::cout << "COMPARE HASHES done in " << overall_diff << " seconds" << std::endl;
+  std::cout << "Hash comparison done in " << overall_diff << " seconds" << std::endl;
 }