diff --git a/.gitignore b/.gitignore index cd9be54..969cf26 100755 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ __pycache__/ *.py[cod] farasa/__pycache__/ -venv/ +.venv/ # Installer logs pip-log.txt @@ -15,5 +15,6 @@ farasa_bin_obselete/ lib/ build/ dist/ +tmp/ farasa/farasa_bin/ farasa/__obselete.py diff --git a/farasa/__base.py b/farasa/__base.py index 293ea48..3fd2ecb 100644 --- a/farasa/__base.py +++ b/farasa/__base.py @@ -7,6 +7,7 @@ import tempfile import warnings import zipfile +import gzip from pathlib import Path import requests @@ -17,6 +18,7 @@ class FarasaBase: task = None __base_dir = Path(__file__).parent.absolute() __bin_dir = Path(f"{__base_dir}/farasa_bin") + __bin_dist_dir = Path(f"{__base_dir}/dist") __bin_lib_dir = Path(f"{__bin_dir}/lib") # shlex not compatible with Windows replace it with list() @@ -29,7 +31,9 @@ class FarasaBase: "NER": __BASE_CMD + [str(__bin_dir / "FarasaNERJar.jar")], "POS": __BASE_CMD + [str(__bin_dir / "FarasaPOSJar.jar")], "diacritize": __BASE_CMD + [str(__bin_dir / "FarasaDiacritizeJar.jar")], + "depparse": __BASE_CMD + [str(__bin_dist_dir / "RBGParserWrapper.jar")], } + __APIs['depparse'].insert(1, '-Xmx4096m') __interactive = False __task_proc = None logger = None @@ -83,9 +87,10 @@ def _check_toolkit_binaries(self): if ( download or not Path(f"{self.__bin_lib_dir}/FarasaSegmenterJar.jar").is_file() + or not Path(f"{self.__bin_dist_dir}/RBGParserWrapper.jar").is_file() ): # last check for binaries in farasa_bin/lib self.logger.info("some binaries are not existed.") - self._download_binaries() + # self._download_binaries() def _get_content_with_progressbar(self, request): totalsize = int(request.headers.get("content-length", 0)) @@ -109,20 +114,29 @@ def _get_content_with_progressbar(self, request): def _download_binaries(self): self.logger.info("downloading zipped binaries...") - try: - # change download url from github releases to qcri - # binaries_url = "https://github.com/MagedSaeed/farasapy/releases/download/toolkit-bins-released/farasa_bin.zip" - binaries_url = "https://farasa-api.qcri.org/farasapy/releases/download/toolkit-bins-released/farasa_bin.zip" - binaries_request = requests.get(binaries_url, stream=True, verify=False) - # show the progress bar while getting content - content_bytes = self._get_content_with_progressbar(binaries_request) - self.logger.debug("extracting...") - binzip = zipfile.ZipFile(io.BytesIO(content_bytes)) - binzip.extractall(path=self.__base_dir) - self.logger.debug("toolkit binaries are downloaded and extracted.") - except Exception as e: - self.logger.error("an error occured") - self.logger.error(e) + # change download url from github releases to qcri + # binaries_url = "https://github.com/MagedSaeed/farasapy/releases/download/toolkit-bins-released/farasa_bin.zip" + binaries_urls = [ + "https://github.com/ZOUHEIRBN/farasapy/releases/download/under-dev/farasa_bin.zip", + "https://github.com/ZOUHEIRBN/farasapy/releases/download/under-dev/FarasaDependencyJar.jar.tar.gz", + ] + for url in binaries_urls: + try: + binaries_request = requests.get(url, stream=True, verify=False) + # show the progress bar while getting content + content_bytes = self._get_content_with_progressbar(binaries_request) + self.logger.debug(f"extracting to {self.__base_dir}...") + if url.endswith('.zip'): + binzip = zipfile.ZipFile(io.BytesIO(content_bytes)) + binzip.extractall(path=self.__base_dir) + elif url.endswith('.gz'): + binzip = gzip.GzipFile(io.BytesIO(content_bytes)) + binzip.extractall(path=self.__base_dir) + + self.logger.debug("toolkit binaries are downloaded and extracted.") + except Exception as e: + self.logger.error("an error occured") + self.logger.error(e) def __initialize_task_proc(self): self.__task_proc = subprocess.Popen( @@ -174,8 +188,9 @@ def _run_task(self, btext): itmp.write(btext) # https://stackoverflow.com/questions/46004774/python-namedtemporaryfile-appears-empty-even-after-data-is-written itmp.flush() + cmd = self.__APIs[self.task] + ["-i", itmp.name, "-o", otmp.name] proc = subprocess.run( - self.__APIs[self.task] + ["-i", itmp.name, "-o", otmp.name], + cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, # this only compatiple with python>3.6 diff --git a/farasa/dependency.py b/farasa/dependency.py new file mode 100644 index 0000000..216693c --- /dev/null +++ b/farasa/dependency.py @@ -0,0 +1,46 @@ +from .__base import FarasaBase + + +class FarasaDepParser(FarasaBase): + task = "depparse" + + def parse(self, text): + return self._do_task(text=text) + + def parse_segments(self, text): + parse_result = self._do_task(text=text).split('\n') + docs = [] + result = [] + for row in parse_result: + if row == '': + # docs.append(result) + # result = [] + continue + i, word, lemma, pos, xpos, *morph, head_i, dep = row.split('\t') + i = int(i) - 1 + head_i = int(head_i) + if dep == "---": + dep = "root" + else: + head_i -= 1 + + morph = f"dict({', '.join(morph)})" + # morph = eval(morph) + result.append({ + "i": i, + # "head": head, + "dep": dep.lower(), + "pos": pos.upper(), + "lemma": lemma, + "morph": morph, + "text": word, + "head_i": head_i, + }) + + for i in range(len(result)): + h_i = result[i]['head_i'] + result[i]['head'] = result[h_i]['text'] + + docs.append(result) + return result + diff --git a/requirements.txt b/requirements.txt index c8af2ea..8f01229 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ black==19.10b0 bleach==3.3.0 pylint==2.5.2 tqdm==4.46.0 +requests \ No newline at end of file diff --git a/tests.py b/tests.py index 9321cc3..0730eb8 100644 --- a/tests.py +++ b/tests.py @@ -1,15 +1,19 @@ +from pprint import pprint from farasa.pos import FarasaPOSTagger from farasa.ner import FarasaNamedEntityRecognizer from farasa.diacratizer import FarasaDiacritizer from farasa.segmenter import FarasaSegmenter from farasa.stemmer import FarasaStemmer +from farasa.dependency import FarasaDepParser # https://r12a.github.io/scripts/tutorial/summaries/arabic sample = """ يُشار إلى أن اللغة العربية يتحدثها أكثر من 422 مليون نسمة ويتوزع متحدثوها في المنطقة المعروفة باسم الوطن العربي بالإضافة إلى العديد من المناطق الأخرى المجاورة مثل الأهواز وتركيا وتشاد والسنغال وإريتريا وغيرها. وهي اللغة الرابعة من لغات منظمة الأمم المتحدة الرسمية الست منذ 99/9/1999. / """ - +sample = """ +لا ينتهي الصراع بين الخير والشر +""" """ --------------------- @@ -50,6 +54,12 @@ print("sample diacritized:", diacritized) print("----------------------------------------------") +depparser = FarasaDepParser() +depparsed = depparser.parse_segments(sample) +print("sample dependencies:") +pprint(depparsed) +print("----------------------------------------------") + """ --------------------- interactive mode @@ -92,3 +102,9 @@ diacritized_interactive = diacritizer_interactive.diacritize(sample) print("sample diacritized (interactive):", diacritized_interactive) print("----------------------------------------------") + + +depparser_interactive = FarasaDepParser(interactive=True) +depparsed_interactive = depparser_interactive.parse(sample) +print("sample dependencies (interactive):", depparsed_interactive) +print("----------------------------------------------")