added packages

Author: kevinlu1248

.gitignore

@@ -1 +1,2 @@
-__pycache__
+__pycache__
+pyate

basic.py

@@ -12,8 +12,5 @@ def basic(technical_corpus, *args, **kwargs):
     return combo_basic(technical_corpus, weights=weights, *args, **kwargs)
 
 if __name__ == "__main__":
-    import pickle
-    pkl = pickle.load(open("../data/pmc_testing.pkl", "rb"))
-    print(len(pkl))
-    corpus = pkl
-    print(TermExtraction(pkl[0]).basic().sort_values(ascending=False).head(50))
+    pkl = "hello world"
+    print(TermExtraction(pkl).basic().sort_values(ascending=False).head(50))

combo_basic.py

@@ -5,27 +5,6 @@
 import numpy as np
 from term_extraction import TermExtraction, add_term_extraction_method
 
-start_ = 0
-tmp = 0
-# TOTAL_WORK = 27768
-# success = 27768
-# pbar = tqdm(total=27768)
-
-
-def start():
-    global start_
-    start_ = time.time()
-
-
-def end():
-    global start_
-    print(time.time() - start_)
-
-
-MAX_WORD_LENGTH = 6
-THRESHOLD = 0
-
-
 def helper_get_subsequences(s):
     sequence = s.split()
     if len(sequence) <= 2:
@@ -49,8 +28,6 @@ def combo_basic(
     weights=None,
 ):
 
-    # TODO
-
     if technical_counts is None:
         technical_counts = (
             TermExtraction(technical_corpus)
@@ -106,9 +83,10 @@ def score_of_children(candidate):
 
 
 if __name__ == "__main__":
-    import pickle
+    # import pickle
 
-    pkl = pickle.load(open("../data/pmc_testing.pkl", "rb"))
-    print(len(pkl))
-    corpus = pkl
-    print(TermExtraction(pkl[0]).combo_basic().sort_values(ascending=False).head(50))
+    # pkl = pickle.load(open("../data/pmc_testing.pkl", "rb"))
+    # print(len(pkl))
+    # corpus = pkl
+    pkl = "Hello I am a good extractor."
+    print(TermExtraction(pkl).combo_basic().sort_values(ascending=False).head(50))

term_extraction.py

@@ -6,8 +6,6 @@
 import math
 from tqdm import tqdm
 import pandas as pd
-
-# from pathos.multiprocessing import ProcessingPool as Pool
 from multiprocessing import Pool
 from spacy.matcher import Matcher
 from collections import defaultdict
@@ -105,7 +103,6 @@ def add_to_counter(matcher, doc, i, matches):
         return term_counter
 
     def count_terms_from_documents(self, seperate=False, verbose=False):
-
         if type(self.corpus) is str:
             term_counter = pd.Series(self.count_terms_from_document(self.corpus))
         elif type(self.corpus) is list or type(self.corpus) is pd.Series:
@@ -155,7 +152,6 @@ def error_callback(e):
             P.terminate()
             if verbose:
                 pbar.close()
-            # print(term_counter)
         else:
             raise TypeError()