From ac72ca527166e968f602377167b99f93d4a60976 Mon Sep 17 00:00:00 2001 From: Marcos Cardoso Date: Fri, 29 Jul 2016 12:50:51 -0300 Subject: [PATCH 1/7] Feature(Sort Fields): Add special mapping for sorting fields See https://www.elastic.co/guide/en/elasticsearch/guide/1.x/multi-fields.html --- .gitignore | 3 +++ django_elasticsearch/managers.py | 12 ++++++++++++ django_elasticsearch/models.py | 1 + readme.md | 4 ++++ 4 files changed, 20 insertions(+) diff --git a/.gitignore b/.gitignore index b5a6589..5cef73c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ *.pyc *~ .tox +.idea +*.egg-info +dist diff --git a/django_elasticsearch/managers.py b/django_elasticsearch/managers.py index edf2718..d72d7d9 100644 --- a/django_elasticsearch/managers.py +++ b/django_elasticsearch/managers.py @@ -253,6 +253,8 @@ def make_mapping(self): """ mappings = {} + sort_fields = self.model.Elasticsearch.sort_fields + for field_name in self.get_fields(): try: field = self.model._meta.get_field(field_name) @@ -274,6 +276,16 @@ def make_mapping(self): mapping.update(self.model.Elasticsearch.mappings[field_name]) except (AttributeError, KeyError, TypeError): pass + + if sort_fields is not None and field_name in sort_fields: + if 'type' in mapping and mapping.get('type') == 'string': + mapping['fields'] = { + 'raw': { + 'type': 'string', + 'index': 'not_analyzed' + } + } + mappings[field_name] = mapping # add a completion mapping for every auto completable field diff --git a/django_elasticsearch/models.py b/django_elasticsearch/models.py index 39ab19f..dc327b8 100644 --- a/django_elasticsearch/models.py +++ b/django_elasticsearch/models.py @@ -31,6 +31,7 @@ class Elasticsearch: mapping = None serializer_class = EsJsonSerializer fields = None + sort_fields = None facets_limit = 10 facets_fields = None # http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-suggesters-term.html diff --git a/readme.md b/readme.md index 55b19ba..227c577 100644 --- a/readme.md +++ b/readme.md @@ -154,6 +154,10 @@ Each EsIndexable model receive an Elasticsearch class that contains its options Defaults to None The fields on which to activate auto-completion (needs a specific mapping). +* **sort_fields** + Defaults to None + A list of fields that will receive a specific mapping for sorting purposes. (See [this](https://www.elastic.co/guide/en/elasticsearch/guide/1.x/multi-fields.html)) + API === From fc0b6753dab4718be72d26b0b881a224769e0277 Mon Sep 17 00:00:00 2001 From: Marcos Cardoso Date: Fri, 29 Jul 2016 13:03:59 -0300 Subject: [PATCH 2/7] Fix(Testings Queryset): Test was broken --- django_elasticsearch/tests/test_qs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_elasticsearch/tests/test_qs.py b/django_elasticsearch/tests/test_qs.py index 60f8c12..62e4945 100644 --- a/django_elasticsearch/tests/test_qs.py +++ b/django_elasticsearch/tests/test_qs.py @@ -113,7 +113,7 @@ def test_suggestions(self): u'last_name': [ {u'length': 5, u'offset': 0, - u'options': [{u'freq': 3, + u'options': [{u'freq': 6, u'score': 0.8, u'text': u'smith'}], u'text': u'smath'}]} From 07b4ca226eb270a3a72e264356279ddf259bba75 Mon Sep 17 00:00:00 2001 From: Marcos Cardoso Date: Wed, 17 Aug 2016 15:03:19 -0300 Subject: [PATCH 3/7] Fixed tox.ini django19 env --- test_project/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_project/tox.ini b/test_project/tox.ini index 547e86b..9d8ed4f 100644 --- a/test_project/tox.ini +++ b/test_project/tox.ini @@ -15,7 +15,7 @@ deps = django16: django>=1.6, <1.7 django17: django>=1.7, <1.8 django18: django>=1.8, <1.9 - django19: django>=1.9, <2.0 + django19: django>=1.9, <1.10 django{14,16,17}: djangorestframework>=2.4, <3.0 django{18,19}: djangorestframework>3.0, <3.2 -r../requirements.txt From 2e4d5577ee35ca12f02f3729efbc39c08512fd9c Mon Sep 17 00:00:00 2001 From: Marcos Cardoso Date: Wed, 17 Aug 2016 15:59:37 -0300 Subject: [PATCH 4/7] Added fullmapping support --- .cache/v/cache/lastfailed | 3 ++ django_elasticsearch/managers.py | 16 ++++---- django_elasticsearch/models.py | 1 + django_elasticsearch/tests/test_indexable.py | 39 +++++++++++++------- django_elasticsearch/tests/test_qs.py | 8 ++-- 5 files changed, 42 insertions(+), 25 deletions(-) create mode 100644 .cache/v/cache/lastfailed diff --git a/.cache/v/cache/lastfailed b/.cache/v/cache/lastfailed new file mode 100644 index 0000000..d96086c --- /dev/null +++ b/.cache/v/cache/lastfailed @@ -0,0 +1,3 @@ +{ + "django_elasticsearch/tests/test_indexable.py": true +} \ No newline at end of file diff --git a/django_elasticsearch/managers.py b/django_elasticsearch/managers.py index d72d7d9..7e1b164 100644 --- a/django_elasticsearch/managers.py +++ b/django_elasticsearch/managers.py @@ -65,7 +65,7 @@ def __init__(self, k): self.model = k self.serializer = None - self._mapping = None + self._full_mapping = None def get_index(self): return self.model.Elasticsearch.index @@ -300,14 +300,14 @@ def make_mapping(self): } } - def get_mapping(self): - if self._mapping is None: - # TODO: could be done once for every index/doc_type ? - full_mapping = es_client.indices.get_mapping(index=self.index, - doc_type=self.doc_type) - self._mapping = full_mapping[self.index]['mappings'][self.doc_type]['properties'] + def get_full_mapping(self): + if self._full_mapping is None: + self._full_mapping = es_client.indices.get_mapping(index=self.index, doc_type=self.doc_type) + + return self._full_mapping - return self._mapping + def get_mapping(self): + return self.get_full_mapping()[self.index]['mappings'][self.doc_type]['properties'] def get_settings(self): """ diff --git a/django_elasticsearch/models.py b/django_elasticsearch/models.py index dc327b8..e2a2606 100644 --- a/django_elasticsearch/models.py +++ b/django_elasticsearch/models.py @@ -32,6 +32,7 @@ class Elasticsearch: serializer_class = EsJsonSerializer fields = None sort_fields = None + parent_model = None facets_limit = 10 facets_fields = None # http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-suggesters-term.html diff --git a/django_elasticsearch/tests/test_indexable.py b/django_elasticsearch/tests/test_indexable.py index 1d0d9b2..bca63c2 100644 --- a/django_elasticsearch/tests/test_indexable.py +++ b/django_elasticsearch/tests/test_indexable.py @@ -1,16 +1,13 @@ # -*- coding: utf-8 -*- -from elasticsearch import NotFoundError - +from django import get_version from django.test import TestCase from django.test.utils import override_settings +from elasticsearch import NotFoundError +from test_app.models import TestModel, Test2Model from django_elasticsearch.managers import es_client from django_elasticsearch.tests.utils import withattrs -from test_app.models import TestModel - -from django import get_version - class EsIndexableTestCase(TestCase): def setUp(self): @@ -95,8 +92,8 @@ def test_fuzziness(self): "default": "test_analyzer", "analyzer": { "test_analyzer": { - "type": "custom", - "tokenizer": "standard", + "type": "custom", + "tokenizer": "standard", } } } @@ -131,7 +128,7 @@ def test_auto_completion(self): @withattrs(TestModel.Elasticsearch, 'fields', ['username', 'date_joined']) def test_get_mapping(self): - TestModel.es._mapping = None + TestModel.es._full_mapping = None TestModel.es.flush() TestModel.es.do_update() @@ -140,7 +137,23 @@ def test_get_mapping(self): # Reset the eventual cache on the Model mapping mapping = TestModel.es.get_mapping() - TestModel.es._mapping = None + TestModel.es._full_mapping = None + self.assertEqual(expected, mapping) + + @withattrs(TestModel.Elasticsearch, 'fields', ['username', 'date_joined']) + def test_get_full_mapping(self): + TestModel.es._full_mapping = None + TestModel.es.flush() + TestModel.es.do_update() + + expected = {u'django-test': {u'mappings': {u'test-doc-type': {u'properties': { + u'date_joined': {u'format': u'dateOptionalTime', u'type': u'date'}, + u'username': {u'index': u'not_analyzed', u'type': u'string'} + }}}}} + + # Reset the eventual cache on the Model mapping + mapping = TestModel.es.get_full_mapping() + TestModel.es._full_mapping = None self.assertEqual(expected, mapping) def test_get_settings(self): @@ -171,8 +184,8 @@ def test_diff(self): expected = { u'first_name': { - 'es': u'woot', - 'db': u'pouet' + 'es': u'woot', + 'db': u'pouet' } } @@ -209,7 +222,7 @@ def setUp(self): post_save.connect(es_save_callback) post_delete.connect(es_delete_callback) post_migrate.connect(es_syncdb_callback) - + if int(get_version()[2]) >= 6: sender = app else: diff --git a/django_elasticsearch/tests/test_qs.py b/django_elasticsearch/tests/test_qs.py index 62e4945..08bd3cb 100644 --- a/django_elasticsearch/tests/test_qs.py +++ b/django_elasticsearch/tests/test_qs.py @@ -220,7 +220,7 @@ def test_isnull_lookup(self): @withattrs(TestModel.Elasticsearch, 'fields', ['id', 'date_joined_exp']) def test_sub_object_lookup(self): TestModel.es._fields = None - TestModel.es._mapping = None + TestModel.es._full_mapping = None TestModel.es.flush() # update the mapping time.sleep(2) @@ -232,14 +232,14 @@ def test_sub_object_lookup(self): self.assertEqual(qs.count(), 4) def test_nested_filter(self): - TestModel.es._mapping = None + TestModel.es._full_mapping = None qs = TestModel.es.filter(groups=self.group) self.assertEqual(qs.count(), 1) @withattrs(TestModel.Elasticsearch, 'fields', ['id', 'date_joined_exp']) def test_filter_date_range(self): TestModel.es._fields = None - TestModel.es._mapping = None + TestModel.es._full_mapping = None TestModel.es.flush() # update the mapping time.sleep(2) @@ -303,7 +303,7 @@ def test_chain_filter_exclude(self): @withattrs(TestModel.Elasticsearch, 'mappings', {}) def test_contains(self): TestModel.es._fields = None - TestModel.es._mapping = None + TestModel.es._full_mapping = None TestModel.es.flush() # update the mapping, username is now analyzed time.sleep(2) # TODO: flushing is not immediate, find a better way contents = TestModel.es.filter(username__contains='woot').deserialize() From 3a5e1e2e0d49479ddec5fa14bbeae4d5bd43903e Mon Sep 17 00:00:00 2001 From: Marcos Cardoso Date: Wed, 17 Aug 2016 18:43:57 -0300 Subject: [PATCH 5/7] Added parent mapping --- .cache/v/cache/lastfailed | 3 --- django_elasticsearch/managers.py | 12 ++++++++- django_elasticsearch/tests/test_indexable.py | 28 ++++++++++++++++++++ 3 files changed, 39 insertions(+), 4 deletions(-) delete mode 100644 .cache/v/cache/lastfailed diff --git a/.cache/v/cache/lastfailed b/.cache/v/cache/lastfailed deleted file mode 100644 index d96086c..0000000 --- a/.cache/v/cache/lastfailed +++ /dev/null @@ -1,3 +0,0 @@ -{ - "django_elasticsearch/tests/test_indexable.py": true -} \ No newline at end of file diff --git a/django_elasticsearch/managers.py b/django_elasticsearch/managers.py index 7e1b164..785ab15 100644 --- a/django_elasticsearch/managers.py +++ b/django_elasticsearch/managers.py @@ -15,6 +15,7 @@ from django_elasticsearch.query import EsQueryset from django_elasticsearch.client import es_client + # Note: we use long/double because different db backends # could store different sizes of numerics ? # Note: everything else is mapped to a string @@ -294,12 +295,21 @@ def make_mapping(self): complete_name = "{0}_complete".format(field_name) mappings[complete_name] = {"type": "completion"} - return { + es_mapping = { self.doc_type: { "properties": mappings } } + parent = self.model.Elasticsearch.parent_model + if parent: + parent.es.create_index() + es_mapping[self.doc_type]['_parent'] = { + 'type': parent.Elasticsearch.doc_type + } + + return es_mapping + def get_full_mapping(self): if self._full_mapping is None: self._full_mapping = es_client.indices.get_mapping(index=self.index, doc_type=self.doc_type) diff --git a/django_elasticsearch/tests/test_indexable.py b/django_elasticsearch/tests/test_indexable.py index bca63c2..b165907 100644 --- a/django_elasticsearch/tests/test_indexable.py +++ b/django_elasticsearch/tests/test_indexable.py @@ -156,6 +156,34 @@ def test_get_full_mapping(self): TestModel.es._full_mapping = None self.assertEqual(expected, mapping) + @withattrs(TestModel.Elasticsearch, 'fields', ['username', 'date_joined']) + @withattrs(Test2Model.Elasticsearch, 'doc_type', 'test-2-doc-type') + @withattrs(Test2Model.Elasticsearch, 'fields', ['text', 'email']) + @withattrs(Test2Model.Elasticsearch, 'parent_model', TestModel) + def test_get_parent_mapping(self): + self.maxDiff = None + Test2Model.es._full_mapping = None + Test2Model.es.flush() + Test2Model.es.do_update() + + expected = {u'django-test': {u'mappings': {u'test-2-doc-type': { + u'properties': { + u'text': {u'type': u'string'}, + u'email': {u'type': u'string'} + }, + u'_routing': { + u'required': True + }, + u'_parent': { + u'type': u'test-doc-type' + } + }}}} + + # Reset the eventual cache on the Model mapping + mapping = Test2Model.es.get_full_mapping() + Test2Model.es._full_mapping = None + self.assertEqual(expected, mapping) + def test_get_settings(self): # Note i don't really know what's in there so i just check # it doesn't crash and deserialize well. From b04c5d596398cab6f005f081d06ea51b39446233 Mon Sep 17 00:00:00 2001 From: Marcos Cardoso Date: Wed, 17 Aug 2016 18:48:33 -0300 Subject: [PATCH 6/7] Bump version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index aa8eebe..ee6e267 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="django-elasticsearch", - version="0.5", + version="0.5.2", description="Simple wrapper around py-elasticsearch to index/search a django Model.", author="Robin Tissot", url="https://github.com/liberation/django_elasticsearch", From bb4325b68a1ac1104c05d22551797a3988275b3c Mon Sep 17 00:00:00 2001 From: Marcos Cardoso Date: Wed, 17 Aug 2016 20:15:35 -0300 Subject: [PATCH 7/7] Fixed indexing parent model --- django_elasticsearch/managers.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/django_elasticsearch/managers.py b/django_elasticsearch/managers.py index 785ab15..c0dfe14 100644 --- a/django_elasticsearch/managers.py +++ b/django_elasticsearch/managers.py @@ -14,6 +14,7 @@ from django_elasticsearch.query import EsQueryset from django_elasticsearch.client import es_client +import inspect # Note: we use long/double because different db backends @@ -122,11 +123,26 @@ def deserialize(self, source): @needs_instance def do_index(self): - body = self.serialize() - es_client.index(index=self.index, - doc_type=self.doc_type, - id=self.instance.id, - body=body) + kwargs = { + 'index': self.index, + 'doc_type': self.doc_type, + 'id': self.instance.id, + 'body': self.serialize() + } + + parent = self.model.Elasticsearch.parent_model + if parent: + parent.es.create_index() + parent_instance = None + for member in inspect.getmembers(self.instance): + value = member[1] + if isinstance(value, parent): + parent_instance = value + break + parent_instance.es.do_index() + kwargs.update({'parent': parent_instance.id}) + + es_client.index(**kwargs) @needs_instance def delete(self): @@ -305,7 +321,7 @@ def make_mapping(self): if parent: parent.es.create_index() es_mapping[self.doc_type]['_parent'] = { - 'type': parent.Elasticsearch.doc_type + 'type': parent.es.doc_type } return es_mapping @@ -354,7 +370,6 @@ def create_index(self, ignore=True): body = {} if hasattr(settings, 'ELASTICSEARCH_SETTINGS'): body['settings'] = settings.ELASTICSEARCH_SETTINGS - es_client.indices.create(self.index, body=body, ignore=ignore and 400)