Implemented FarasaDepParser

.gitignore

@@ -2,7 +2,7 @@
 __pycache__/
 *.py[cod]
 farasa/__pycache__/
-venv/
+.venv/
 
 # Installer logs
 pip-log.txt
@@ -15,5 +15,6 @@ farasa_bin_obselete/
 lib/
 build/
 dist/
+tmp/
 farasa/farasa_bin/
 farasa/__obselete.py

farasa/__base.py

@@ -7,6 +7,7 @@
 import tempfile
 import warnings
 import zipfile
+import gzip
 from pathlib import Path
 
 import requests
@@ -17,6 +18,7 @@ class FarasaBase:
     task = None
     __base_dir = Path(__file__).parent.absolute()
     __bin_dir = Path(f"{__base_dir}/farasa_bin")
+    __bin_dist_dir = Path(f"{__base_dir}/dist")
     __bin_lib_dir = Path(f"{__bin_dir}/lib")
 
     # shlex not compatible with Windows replace it with list()
@@ -29,7 +31,9 @@ class FarasaBase:
         "NER": __BASE_CMD + [str(__bin_dir / "FarasaNERJar.jar")],
         "POS": __BASE_CMD + [str(__bin_dir / "FarasaPOSJar.jar")],
         "diacritize": __BASE_CMD + [str(__bin_dir / "FarasaDiacritizeJar.jar")],
+        "depparse": __BASE_CMD + [str(__bin_dist_dir / "RBGParserWrapper.jar")],
     }
+    __APIs['depparse'].insert(1, '-Xmx4096m')
     __interactive = False
     __task_proc = None
     logger = None
@@ -83,9 +87,10 @@ def _check_toolkit_binaries(self):
         if (
             download
             or not Path(f"{self.__bin_lib_dir}/FarasaSegmenterJar.jar").is_file()
+            or not Path(f"{self.__bin_dist_dir}/RBGParserWrapper.jar").is_file()
         ):  # last check for binaries in farasa_bin/lib
             self.logger.info("some binaries are not existed.")
-            self._download_binaries()
+            # self._download_binaries()
 
     def _get_content_with_progressbar(self, request):
         totalsize = int(request.headers.get("content-length", 0))
@@ -109,20 +114,29 @@ def _get_content_with_progressbar(self, request):
 
     def _download_binaries(self):
         self.logger.info("downloading zipped binaries...")
-        try:
-            # change download url from github releases to qcri
-            # binaries_url = "https://github.com/MagedSaeed/farasapy/releases/download/toolkit-bins-released/farasa_bin.zip"
-            binaries_url = "https://farasa-api.qcri.org/farasapy/releases/download/toolkit-bins-released/farasa_bin.zip"
-            binaries_request = requests.get(binaries_url, stream=True, verify=False)
-            # show the progress bar while getting content
-            content_bytes = self._get_content_with_progressbar(binaries_request)
-            self.logger.debug("extracting...")
-            binzip = zipfile.ZipFile(io.BytesIO(content_bytes))
-            binzip.extractall(path=self.__base_dir)
-            self.logger.debug("toolkit binaries are downloaded and extracted.")
-        except Exception as e:
-            self.logger.error("an error occured")
-            self.logger.error(e)
+        # change download url from github releases to qcri
+        # binaries_url = "https://github.com/MagedSaeed/farasapy/releases/download/toolkit-bins-released/farasa_bin.zip"
+        binaries_urls = [
+            "https://github.com/ZOUHEIRBN/farasapy/releases/download/under-dev/farasa_bin.zip",
+            "https://github.com/ZOUHEIRBN/farasapy/releases/download/under-dev/FarasaDependencyJar.jar.tar.gz",
+        ]
+        for url in binaries_urls:
+            try:
+                binaries_request = requests.get(url, stream=True, verify=False)
+                # show the progress bar while getting content
+                content_bytes = self._get_content_with_progressbar(binaries_request)
+                self.logger.debug(f"extracting to {self.__base_dir}...")
+                if url.endswith('.zip'):
+                    binzip = zipfile.ZipFile(io.BytesIO(content_bytes))
+                    binzip.extractall(path=self.__base_dir)
+                elif url.endswith('.gz'):
+                    binzip = gzip.GzipFile(io.BytesIO(content_bytes))
+                    binzip.extractall(path=self.__base_dir)
+
+                self.logger.debug("toolkit binaries are downloaded and extracted.")
+            except Exception as e:
+                self.logger.error("an error occured")
+                self.logger.error(e)
 
     def __initialize_task_proc(self):
         self.__task_proc = subprocess.Popen(
@@ -174,8 +188,9 @@ def _run_task(self, btext):
             itmp.write(btext)
             # https://stackoverflow.com/questions/46004774/python-namedtemporaryfile-appears-empty-even-after-data-is-written
             itmp.flush()
+            cmd = self.__APIs[self.task] + ["-i", itmp.name, "-o", otmp.name]
             proc = subprocess.run(
-                self.__APIs[self.task] + ["-i", itmp.name, "-o", otmp.name],
+                cmd,
                 stdout=subprocess.DEVNULL,
                 stderr=subprocess.DEVNULL,
                 # this only compatiple with python>3.6

farasa/dependency.py

@@ -0,0 +1,46 @@
+from .__base import FarasaBase
+
+
+class FarasaDepParser(FarasaBase):
+    task = "depparse"
+
+    def parse(self, text):
+        return self._do_task(text=text)
+
+    def parse_segments(self, text):
+        parse_result = self._do_task(text=text).split('\n')
+        docs = []
+        result = []
+        for row in parse_result:
+            if row == '':
+                # docs.append(result)
+                # result = []
+                continue
+            i, word, lemma, pos, xpos, *morph, head_i, dep = row.split('\t')
+            i = int(i) - 1
+            head_i = int(head_i)
+            if dep == "---":
+                dep = "root"
+            else:
+                head_i -= 1
+
+            morph = f"dict({', '.join(morph)})"
+            # morph = eval(morph)
+            result.append({
+                "i": i,
+                # "head": head,
+                "dep": dep.lower(),
+                "pos": pos.upper(),
+                "lemma": lemma,
+                "morph": morph,
+                "text": word,
+                "head_i": head_i,
+            })
+
+        for i in range(len(result)):
+            h_i =  result[i]['head_i']
+            result[i]['head'] = result[h_i]['text']
+
+        docs.append(result)
+        return result
+

requirements.txt

@@ -2,3 +2,4 @@ black==19.10b0
 bleach==3.3.0
 pylint==2.5.2
 tqdm==4.46.0
+requests

tests.py

@@ -1,15 +1,19 @@
+from pprint import pprint
 from farasa.pos import FarasaPOSTagger
 from farasa.ner import FarasaNamedEntityRecognizer
 from farasa.diacratizer import FarasaDiacritizer
 from farasa.segmenter import FarasaSegmenter
 from farasa.stemmer import FarasaStemmer
+from farasa.dependency import FarasaDepParser
 
 
 # https://r12a.github.io/scripts/tutorial/summaries/arabic
 sample = """
 يُشار إلى أن اللغة العربية يتحدثها أكثر من 422 مليون نسمة ويتوزع متحدثوها في المنطقة المعروفة باسم الوطن العربي بالإضافة إلى العديد من المناطق الأخرى المجاورة مثل الأهواز وتركيا وتشاد والسنغال وإريتريا وغيرها.     وهي اللغة الرابعة من لغات منظمة الأمم المتحدة الرسمية الست منذ 99/9/1999. /
 """
-
+sample = """
+لا ينتهي الصراع بين الخير والشر
+"""
 
 """
 ---------------------
@@ -50,6 +54,12 @@
 print("sample diacritized:", diacritized)
 print("----------------------------------------------")
 
+depparser = FarasaDepParser()
+depparsed = depparser.parse_segments(sample)
+print("sample dependencies:")
+pprint(depparsed)
+print("----------------------------------------------")
+
 """
 ---------------------
 interactive mode
@@ -92,3 +102,9 @@
 diacritized_interactive = diacritizer_interactive.diacritize(sample)
 print("sample diacritized (interactive):", diacritized_interactive)
 print("----------------------------------------------")
+
+
+depparser_interactive = FarasaDepParser(interactive=True)
+depparsed_interactive = depparser_interactive.parse(sample)
+print("sample dependencies (interactive):", depparsed_interactive)
+print("----------------------------------------------")