Merge pull request RasaHQ#1 from RasaHQ/master

lucidNTR · web-flow · commit dc1c337e6e52 · 2017-11-28T15:07:56.000+01:00
Update from upstream repo RasaHQ/rasa_nlu@master
diff --git a/README.md b/README.md
@@ -123,7 +123,7 @@ In general the process is rather simple:
 2. write your code, tests and documentation
 3. create a pull request describing your changes
 
-You pull request will be reviewed by a maintainer, who might get back to you about any necessary changes or questions.
+You pull request will be reviewed by a maintainer, who might get back to you about any necessary changes or questions. You will also be asked to sign the [Contributor License Agreement](https://cla-assistant.io/RasaHQ/rasa_nlu)
 
 # Advanced installation
 ### Advanced Python
diff --git a/docs/pipeline.rst b/docs/pipeline.rst
@@ -80,7 +80,7 @@ intent_featurizer_mitie
 ~~~~~~~~~~~~~~~~~~~~~~~
 
 :Short: MITIE intent featurizer
-:Outputs: nothing, used used as an input to intent classifiers that need intent features (e.g. ``intent_classifier_sklearn``)
+:Outputs: nothing, used as an input to intent classifiers that need intent features (e.g. ``intent_classifier_sklearn``)
 :Description:
     Creates feature for intent classification using the MITIE featurizer.
 
@@ -94,7 +94,7 @@ intent_featurizer_spacy
 ~~~~~~~~~~~~~~~~~~~~~~~
 
 :Short: spacy intent featurizer
-:Outputs: nothing, used used as an input to intent classifiers that need intent features (e.g. ``intent_classifier_sklearn``)
+:Outputs: nothing, used as an input to intent classifiers that need intent features (e.g. ``intent_classifier_sklearn``)
 :Description:
     Creates feature for intent classification using the spacy featurizer.
 
diff --git a/rasa_nlu/config.py b/rasa_nlu/config.py
@@ -33,6 +33,7 @@
     "max_number_of_ngrams": 7,
     "pipeline": [],
     "response_log": "logs",
+    "storage": None,
     "aws_endpoint_url": None,
     "duckling_dimensions": None,
     "duckling_http_url": None,
diff --git a/rasa_nlu/data_router.py b/rasa_nlu/data_router.py
@@ -139,7 +139,7 @@ def _create_emulator(self):
         elif mode.lower() == 'luis':
             from rasa_nlu.emulators.luis import LUISEmulator
             return LUISEmulator()
-        elif mode.lower() == 'api':
+        elif mode.lower() == 'dialogflow':
             from rasa_nlu.emulators.dialogflow import DialogflowEmulator
             return DialogflowEmulator()
         else:
diff --git a/rasa_nlu/extractors/duckling_extractor.py b/rasa_nlu/extractors/duckling_extractor.py
@@ -28,8 +28,19 @@
     from duckling import DucklingWrapper
 
 
+def extract_value(match):
+    if match["value"].get("type") == "interval":
+        value = {"to": match["value"].get("to", {}).get("value"),
+                 "from": match["value"].get("from", {}).get("value")}
+    else:
+        value = match["value"].get("value")
+
+    return value
+
+
 class DucklingExtractor(EntityExtractor):
-    """Adds entity normalization by analyzing found entities and transforming them into regular formats."""
+    """Adds entity normalization by analyzing found entities and
+    transforming them into regular formats."""
 
     name = "ner_duckling"
 
@@ -38,26 +49,34 @@ class DucklingExtractor(EntityExtractor):
     @staticmethod
     def available_dimensions():
         from duckling.dim import Dim
-        return [m[1] for m in getmembers(Dim) if not m[0].startswith("__") and not m[0].endswith("__")]
+        return [m[1]
+                for m in getmembers(Dim)
+                if not m[0].startswith("__") and not m[0].endswith("__")]
 
     def __init__(self, duckling, dimensions=None):
         # type: (DucklingWrapper, Optional[List[Text]]) -> None
 
-        self.dimensions = dimensions if dimensions is not None else self.available_dimensions()
+        super(DucklingExtractor, self).__init__()
         self.duckling = duckling
 
+        if dimensions is not None:
+            self.dimensions = dimensions
+        else:
+            self.dimensions = self.available_dimensions()
+
     @classmethod
     def required_packages(cls):
         # type: () -> List[Text]
         return ["duckling"]
 
     @classmethod
-    def _create_duckling_wrapper(cls, language):
+    def create_duckling_wrapper(cls, language):
         from duckling import DucklingWrapper
 
         try:
-            return DucklingWrapper(language=language)  # languages in duckling are eg "de$core"
-        except ValueError as e:     # pragma: no cover
+            # languages in duckling are eg "de$core"
+            return DucklingWrapper(language=language)
+        except ValueError as e:  # pragma: no cover
             raise Exception("Duckling error. {}".format(e))
 
     @classmethod
@@ -66,12 +85,17 @@ def create(cls, config):
 
         dims = config["duckling_dimensions"]
         if dims:
-            unknown_dimensions = [dim for dim in dims if dim not in cls.available_dimensions()]
+            unknown_dimensions = [dim
+                                  for dim in dims
+                                  if dim not in cls.available_dimensions()]
             if len(unknown_dimensions) > 0:
-                raise ValueError("Invalid duckling dimension. Got '{}'. Allowed: {}".format(
-                    ", ".join(unknown_dimensions), ", ".join(cls.available_dimensions())))
+                raise ValueError(
+                        "Invalid duckling dimension. Got '{}'. Allowed: {}"
+                        "".format(", ".join(unknown_dimensions),
+                                  ", ".join(cls.available_dimensions())))
 
-        return DucklingExtractor(cls._create_duckling_wrapper(config["language"]), dims)
+        wrapper = cls.create_duckling_wrapper(config["language"])
+        return DucklingExtractor(wrapper, dims)
 
     @classmethod
     def cache_key(cls, model_metadata):
@@ -82,55 +106,72 @@ def cache_key(cls, model_metadata):
     def process(self, message, **kwargs):
         # type: (Message, **Any) -> None
 
+        if self.duckling is None:
+            return
+
         extracted = []
-        if self.duckling is not None:
-            ref_time = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S+00:00')
-            if message.time is not None:
-                # check if time given is valid
-                try:
-                    ref_time = datetime.datetime\
-                        .utcfromtimestamp(int(message.time)/1000.0)\
-                        .strftime('%Y-%m-%dT%H:%M:%S+00:00')
-                    logging.debug(
-                            "Passing reference time {} to duckling".format(ref_time))
-                except Exception as e:
-                    logging.warning(
-                            "Could not parse timestamp {}. "
-                            "Instead current UTC time {} will be passed to duckling".format(message.time, ref_time))
-
-            matches = self.duckling.parse(message.text, reference_time=ref_time)
-            relevant_matches = [match for match in matches if match["dim"] in self.dimensions]
-            for match in relevant_matches:
-                entity = {"start": match["start"],
-                          "end": match["end"],
-                          "text": match["text"],
-                          "value": match["value"]["value"],
-                          "additional_info": match["value"],
-                          "entity": match["dim"]}
-
-                extracted.append(entity)
+
+        current_time = datetime.datetime.utcnow()
+        ref_time = current_time.strftime('%Y-%m-%dT%H:%M:%S+00:00')
+        if message.time is not None:
+            # check if time given is valid
+            try:
+                ref_time = datetime.datetime \
+                    .utcfromtimestamp(int(message.time) / 1000.0) \
+                    .strftime('%Y-%m-%dT%H:%M:%S+00:00')
+                logging.debug("Passing reference time {} "
+                              "to duckling".format(ref_time))
+            except Exception as e:
+                logging.warning("Could not parse timestamp {}. Instead "
+                                "current UTC time {} will be passed to "
+                                "duckling. Error: {}"
+                                "".format(message.time, ref_time, e))
+
+        matches = self.duckling.parse(message.text, reference_time=ref_time)
+        relevant_matches = [match
+                            for match in matches
+                            if match["dim"] in self.dimensions]
+
+        for match in relevant_matches:
+            value = extract_value(match)
+            entity = {"start": match["start"],
+                      "end": match["end"],
+                      "text": match["text"],
+                      "value": value,
+                      "additional_info": match["value"],
+                      "entity": match["dim"]}
+
+            extracted.append(entity)
 
         extracted = self.add_extractor_name(extracted)
-        message.set("entities", message.get("entities", []) + extracted, add_to_output=True)
+        message.set("entities", message.get("entities", []) + extracted,
+                    add_to_output=True)
 
     def persist(self, model_dir):
         # type: (Text) -> Dict[Text, Any]
 
-        file_name = self.name+".json"
+        file_name = self.name + ".json"
         full_name = os.path.join(model_dir, file_name)
         with io.open(full_name, 'w') as f:
             f.write(str(json.dumps({"dimensions": self.dimensions})))
         return {"ner_duckling_persisted": file_name}
 
     @classmethod
-    def load(cls, model_dir, model_metadata, cached_component, **kwargs):
-        # type: (Text, Metadata, Optional[DucklingExtractor], **Any) -> DucklingExtractor
-
-        persisted = os.path.join(model_dir, model_metadata.get("ner_duckling_persisted"))
+    def load(cls,
+             model_dir=None,  # type: Text
+             model_metadata=None,  # type: Metadata
+             cached_component=None,  # type:Optional[DucklingExtractor]
+             **kwargs  # type: **Any
+             ):
+        # type: (...) -> DucklingExtractor
+
+        persisted = os.path.join(model_dir,
+                                 model_metadata.get("ner_duckling_persisted"))
         if cached_component:
             duckling = cached_component.duckling
         else:
-            duckling = cls._create_duckling_wrapper(model_metadata.get("language"))
+            language = model_metadata.get("language")
+            duckling = cls.create_duckling_wrapper(language)
 
         if os.path.isfile(persisted):
             with io.open(persisted, encoding='utf-8') as f:
diff --git a/rasa_nlu/extractors/duckling_http_extractor.py b/rasa_nlu/extractors/duckling_http_extractor.py
@@ -19,6 +19,7 @@
 from rasa_nlu.extractors import EntityExtractor
 from rasa_nlu.model import Metadata
 from rasa_nlu.training_data import Message
+from rasa_nlu.extractors.duckling_extractor import extract_value
 
 logger = logging.getLogger(__name__)
 
@@ -92,11 +93,12 @@ def process(self, message, **kwargs):
             matches = self._duckling_parse(message.text)
             relevant_matches = self._filter_irrelevant_matches(matches)
             for match in relevant_matches:
+                value = extract_value(match)
                 entity = {
                     "start": match["start"],
                     "end": match["end"],
                     "text": match["body"],
-                    "value": match["value"]["value"],
+                    "value": value,
                     "additional_info": match["value"],
                     "entity": match["dim"]}
 
diff --git a/rasa_nlu/server.py b/rasa_nlu/server.py
@@ -6,6 +6,7 @@
 import argparse
 import logging
 import os
+import six
 from functools import wraps
 
 import simplejson
@@ -74,7 +75,10 @@ def decorated(*args, **kwargs):
                 request.setResponseCode(403)
                 return 'forbidden'
 
-        return f(*args, **kwargs)
+        if request.method.decode('utf-8', 'strict') == 'OPTIONS':
+            return ''  # if this is an options call we skip running `f`
+        else:
+            return f(*args, **kwargs)
 
     return decorated
 
@@ -86,8 +90,10 @@ def requires_auth(f):
     def decorated(*args, **kwargs):
         self = args[0]
         request = args[1]
-        token = str(request.args.get('token', [''])[0])
-
+        if six.PY3:
+            token = request.args.get(b'token', [b''])[0].decode("utf8")
+        else:
+            token = str(request.args.get('token', [''])[0])
         if self.data_router.token is None or token == self.data_router.token:
             return f(*args, **kwargs)
         request.setResponseCode(401)
@@ -116,13 +122,13 @@ def __init__(self, config, component_builder=None, testing=False):
     def _create_data_router(self, config, component_builder):
         return DataRouter(config, component_builder)
 
-    @app.route("/", methods=['GET'])
+    @app.route("/", methods=['GET', 'OPTIONS'])
     @check_cors
     def hello(self, request):
         """Main Rasa route to check if the server is online"""
         return "hello from Rasa NLU: " + __version__
 
-    @app.route("/parse", methods=['GET', 'POST'])
+    @app.route("/parse", methods=['GET', 'POST', 'OPTIONS'])
     @requires_auth
     @check_cors
     @inlineCallbacks
@@ -158,7 +164,7 @@ def parse_get(self, request):
                 logger.exception(e)
                 returnValue(simplejson.dumps({"error": "{}".format(e)}))
 
-    @app.route("/version", methods=['GET'])
+    @app.route("/version", methods=['GET', 'OPTIONS'])
     @requires_auth
     @check_cors
     def version(self, request):
@@ -167,7 +173,7 @@ def version(self, request):
         request.setHeader('Content-Type', 'application/json')
         return simplejson.dumps({'version': __version__})
 
-    @app.route("/config", methods=['GET'])
+    @app.route("/config", methods=['GET', 'OPTIONS'])
     @requires_auth
     @check_cors
     def rasaconfig(self, request):
@@ -176,14 +182,14 @@ def rasaconfig(self, request):
         request.setHeader('Content-Type', 'application/json')
         return simplejson.dumps(self.config.as_dict())
 
-    @app.route("/status", methods=['GET'])
+    @app.route("/status", methods=['GET', 'OPTIONS'])
     @requires_auth
     @check_cors
     def status(self, request):
         request.setHeader('Content-Type', 'application/json')
         return simplejson.dumps(self.data_router.get_status())
 
-    @app.route("/train", methods=['POST'])
+    @app.route("/train", methods=['POST', 'OPTIONS'])
     @requires_auth
     @check_cors
     @inlineCallbacks
diff --git a/rasa_nlu/training_data.py b/rasa_nlu/training_data.py
@@ -167,6 +167,9 @@ def validate(self):
         for intent, group in groupby(examples, lambda e: e.get("intent")):
             size = len(list(group))
             different_intents.append(intent)
+            if intent == "":
+                warnings.warn("Found empty intent, please check your training data."
+                              "This may result in wrong intent predictions.")
             if size < self.MIN_EXAMPLES_PER_INTENT:
                 template = "Intent '{}' has only {} training examples! minimum is {}, training may fail."
                 warnings.warn(template.format(intent, size, self.MIN_EXAMPLES_PER_INTENT))
diff --git a/sample_configs/config_defaults.json b/sample_configs/config_defaults.json
@@ -7,6 +7,7 @@
   "max_training_processes": 1,
   "path": "projects",
   "response_log": "logs",
+  "storage": null,
   "config": "config.json",
   "log_level": "INFO",
   "port": 5000,
