preprocessing/helper.py at master · poyuwu/preprocessing · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
# coding=utf8
import re
from variation import Variation
from nltk.corpus import stopwords
import json

def is_ascii(s):
    return all(ord(c) < 128 for c in s)

def remove_year(vText):
    """ Remove Text with years like: (1999)
    Args:
      text: str
    Returns:
      without year String.
    """
    return re.sub(r"\(\d{4}\)", "", vText)  # maybe reduction


def remove_citation(vText):
    """ Remove Text with citation. [3], [10-12], [3, 4, 6], (1),  extra remove:(22%, n = 8/37)
    Args:
      text: str
    Returns:
      without citation String.
    """
    #vText = re.sub(r"\([^\)]*,[^\)]*\)", "", vText)  # (DMN; Raichle et al., 2001;)#TODO
    vText = re.sub(r"\[\d\{1,2}\]", "", vText)  # [1]
    vText = re.sub(r"\[\d{1,2}\s*-\s*\d{1,2}\]", "", vText)  # [10-12]
    vText = re.sub(r"\[[0-9]+(\ *,\ *[0-9]+\ *)*\]", "", vText)  # [1,3,4]
    vText = re.sub(r"\([0-9]+(\ *,\ *[0-9]+\ *)*\)", "", vText)  # (1,3,4)
    vText = re.sub(r"\(\d*\)", "", vText)  # (3)
    #vText = re.sub(r"\[\d\{1,2}\{,\d\{1,2}}]+","" , vText)
    return vText


def remove_http(vText):
    """ Remove specific string.
    Args:
      text: str
    Return:
      clear String
    """
    vText = re.sub(r'(https|http)?:\/\/(\w|\.|\/|\?|\=|\&|\%)*\b',
                   '', vText, flags=re.MULTILINE)  # remove url
    vText = re.sub(r"(e-?)?mail: ([\w+-]+[\w.+-]*@[a-zA-Z0-9-]+\.[a-zA-Z0-9-]+)",
                   '', vText, flags=re.IGNORECASE)  # remove email
    vText = re.sub(r"([\w+-]+[\w.+-]*@[a-zA-Z0-9-]+\.[a-zA-Z0-9-]+)",
                   '', vText, flags=re.MULTILINE)  # remove e-mail
    # vText = re.sub(r"[\(](supplementary|fig)\.?.*[\)]\.?", "" , vText,flags=re.IGNORECASE) #remove (fig) or (supplementary) #TODO
    vText = re.sub(r"fig\.", "" ,vText, flags=re.IGNORECASE)# simplest way to prevent sent tokenizer error
    return vText


def remove_stopwords(vText):
    """ Remove Stopwords in NLTK Stopwords List
    Args:
      text: str
    Returns:
        clear string
    """
    stopwords_list = stopwords.words('english')
    pattern = re.compile(r'\b(' + r'|'.join(stopwords_list) + r')\b')
    vText = pattern.sub("", vText)
    return vText


def preprocessing(text, gene, var):
    """ replace many amino to 1 amino.
        Returns:
          replace_text.
    """
    var = Variation(var)
    text = remove_year(text)
    text = remove_citation(text)
    text = remove_http(text)
    text = remove_stopwords(text)
    varalias = json.load(open("one2many.json"))
    # Handling Variation
    # re format: "^([A-Za-z])(\d+)([A-Za-z\*])", including *
    if var.type == "point":
        if var.end_amino == "*":
            alias_list = [] + ["%s%sX" %
                               (start_m, var.pos) for start_m in [var.start_amino] + varalias[var.start_amino.upper()]]
        elif var.end_amino == "":
            alias_list = ["%s%s" % (start_m, var.pos)
                          for start_m in varalias[var.start_amino.upper()]]
        else:
            alias_list = ["%s%s%s" % (start_m, var.pos, end_m) for start_m in varalias[var.start_amino.upper(
            )] for end_m in varalias[var.end_amino.upper()]]
        # replace many to 1
        text = re.sub("%s" % "|".join(alias_list),
                      var.var, text, flags=re.IGNORECASE)
    return text