Merge pull request #27 from PyThaiNLP/dev

update from origin

Author: bact

README-pypi.md

@@ -1,6 +1,6 @@
 ![PyThaiNLP Logo](https://avatars0.githubusercontent.com/u/32934255?s=200&v=4)
 
-# PyThaiNLP 2.0.1
+# PyThaiNLP 2.0.2
 
 [![Codacy Badge](https://api.codacy.com/project/badge/Grade/cb946260c87a4cc5905ca608704406f7)](https://www.codacy.com/app/pythainlp/pythainlp_2?utm_source=github.com&utm_medium=referral&utm_content=PyThaiNLP/pythainlp&utm_campaign=Badge_Grade)[![pypi](https://img.shields.io/pypi/v/pythainlp.svg)](https://pypi.python.org/pypi/pythainlp)
 [![Build Status](https://travis-ci.org/PyThaiNLP/pythainlp.svg?branch=develop)](https://travis-ci.org/PyThaiNLP/pythainlp)

bin/pythainlp

@@ -45,4 +45,4 @@ elif args.soundex!=None:
         args.engine="lk82"
     print(soundex(args.soundex, engine=args.engine))
 else:
-    print("PyThaiNLP 2.0")
+    print("PyThaiNLP 2.0.2")

conda.recipe/meta.yaml

@@ -1,4 +1,4 @@
-{% set version = "2.0.1" %}
+{% set version = "2.0.2" %}
 
 package:
   name: pythainlp

meta.yaml

@@ -1,4 +1,4 @@
-{% set version = "2.0.1" %}
+{% set version = "2.0.2" %}
 
 package:
   name: pythainlp

pythainlp/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-__version__ = "2.0.1"
+__version__ = "2.0.2"
 
 thai_consonants = "กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรลวศษสหฬอฮ"  # 44 chars
 thai_vowels = "ฤฦะ\u0e31าำ\u0e34\u0e35\u0e36\u0e37\u0e38\u0e39เแโใไ\u0e45\u0e47"  # 19

pythainlp/tokenize/__init__.py

@@ -122,17 +122,25 @@ def sent_tokenize(text: str, engine: str = "whitespace+newline") -> List[str]:
 def subword_tokenize(text: str, engine: str = "tcc") -> List[str]:
     """
     :param str text: text to be tokenized
-    :param str engine: choosing 'tcc' uses the Thai Character Cluster rule to segment words into the smallest unique units.
+    :param str engine: subword tokenizer 
+    :Parameters for engine:
+        * tcc (default) -  Thai Character Cluster (Theeramunkong et al. 2000)
+        * etcc - Enhanced Thai Character Cluster (Inrut et al. 2001) [In development] 
     :return: a list of tokenized strings.
     """
     if not text:
         return ""
 
     from .tcc import tcc
+    from .etcc import etcc
 
+    if engine == "tcc":
+        return tcc(text) 
+    elif engine == "etcc":
+        return etcc(text).split("/")
+    #default
     return tcc(text)
 
-
 def syllable_tokenize(text: str) -> List[str]:
     """
     :param str text: input string to be tokenized

pythainlp/tokenize/etcc.py

@@ -3,6 +3,8 @@
 โปรแกรม ETCC ใน Python
 พัฒนาโดย นาย วรรณพงษ์  ภัททิยไพบูลย์
 19 มิ.ย. 2560
+Reference: Inrut, Jeeragone, Patiroop Yuanghirun, Sarayut Paludkong, Supot Nitsuwat, and Para Limmaneepraserth. "Thai word segmentation using combination of forward and backward longest matching techniques." In International Symposium on Communications and Information Technology (ISCIT), pp. 37-40. 2001.
+
 
 วิธีใช้งาน
 etcc(คำ)

pythainlp/tokenize/tcc.py

@@ -1,8 +1,9 @@
 # -*- coding: utf-8 -*-
 """
 Separate Thai text into Thai Character Cluster (TCC).
-Based on "Character cluster based Thai information retrieval" (Theeramunkong et al. 2002)
-http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.59.2548
+Based on "Character cluster based Thai information retrieval" (Theeramunkong et al. 2000)
+https://dl.acm.org/citation.cfm?id=355225
+http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.59.2548 
 
 Credits:
 - TCC: Jakkrit TeCho

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.0.1
+current_version = 2.0.2
 commit = True
 tag = True
 

setup.py

@@ -34,7 +34,7 @@
 
 setup(
     name="pythainlp",
-    version="2.0.1",
+    version="2.0.2",
     description="Thai Natural Language Processing library",
     long_description=readme,
     long_description_content_type="text/markdown",