diff --git a/.travis.yml b/.travis.yml
index dff47e6a50..653a650f89 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,7 +10,7 @@ services:
python:
- '2.7'
before_install:
- - psql -c 'create database onadata_test;' -U postgres
+ - ./script/database/install_postgis onadata_test
- ./script/install_ubuntu
env:
- TESTFOLDER=onadata/apps/main
@@ -20,12 +20,12 @@ install:
- python manage.py syncdb --noinput --settings=onadata.settings.travis_test
- python manage.py migrate --noinput --settings=onadata.settings.travis_test
# make ~/tmp dir writable for npm downloads
- - sudo chmod a+w $HOME/tmp
- - npm install
- - bower install --force-latest
+ - if [[ "$TESTFOLDER" == onadata/apps/api* ]] ; then sudo chmod a+w $HOME/tmp ; fi
+ - if [[ "$TESTFOLDER" == onadata/apps/api* ]] ; then npm install ; fi
+ - if [[ "$TESTFOLDER" == onadata/apps/api* ]] ; then bower install --force-latest ; fi
script:
- python manage.py test $TESTFOLDER --noinput --settings=onadata.settings.travis_test
- - karma start onadata/libs/javascript/tests/karma.conf.js
+ - if [[ "$TESTFOLDER" == onadata/apps/api* ]] ; then karma start onadata/libs/javascript/tests/karma.conf.js ; fi
notifications:
email:
- tech@ona.io
diff --git a/onadata/apps/odk_logger/management/commands/set_xform_surveys_with_geopoints.py b/onadata/apps/odk_logger/management/commands/set_xform_surveys_with_geopoints.py
index 2b356868df..24a6fa2ac8 100644
--- a/onadata/apps/odk_logger/management/commands/set_xform_surveys_with_geopoints.py
+++ b/onadata/apps/odk_logger/management/commands/set_xform_surveys_with_geopoints.py
@@ -4,8 +4,7 @@
from django.core.management.base import BaseCommand
from django.utils.translation import ugettext_lazy
-from onadata.apps.odk_logger.models import XForm
-from onadata.apps.odk_viewer.models.parsed_instance import ParsedInstance
+from onadata.apps.odk_logger.models.xform import XForm
from onadata.libs.utils.model_tools import queryset_iterator
@@ -17,8 +16,7 @@ def handle(self, *args, **kwargs):
total = xforms.count()
count = 0
for xform in queryset_iterator(XForm.objects.all()):
- has_geo = ParsedInstance.objects.filter(
- instance__xform=xform, lat__isnull=False).count() > 0
+ has_geo = xform.geocoded_submission_count() > 0
try:
xform.instances_with_geopoints = has_geo
xform.save()
diff --git a/onadata/apps/odk_logger/migrations/0040_auto__add_field_instance_geom.py b/onadata/apps/odk_logger/migrations/0040_auto__add_field_instance_geom.py
new file mode 100644
index 0000000000..b4bc14989f
--- /dev/null
+++ b/onadata/apps/odk_logger/migrations/0040_auto__add_field_instance_geom.py
@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+ def forwards(self, orm):
+ # Adding field 'Instance.geom'
+ db.add_column(u'odk_logger_instance', 'geom',
+ self.gf('django.contrib.gis.db.models.fields.GeometryCollectionField')(null=True),
+ keep_default=False)
+
+
+ def backwards(self, orm):
+ # Deleting field 'Instance.geom'
+ db.delete_column(u'odk_logger_instance', 'geom')
+
+
+ models = {
+ u'auth.group': {
+ 'Meta': {'object_name': 'Group'},
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+ 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+ },
+ u'auth.permission': {
+ 'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
+ 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+ },
+ u'auth.user': {
+ 'Meta': {'object_name': 'User'},
+ 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+ 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+ 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+ 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+ 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+ 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+ 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+ 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+ },
+ u'contenttypes.contenttype': {
+ 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+ 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+ },
+ 'odk_logger.attachment': {
+ 'Meta': {'object_name': 'Attachment'},
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'instance': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'attachments'", 'to': "orm['odk_logger.Instance']"}),
+ 'media_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}),
+ 'mimetype': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'})
+ },
+ 'odk_logger.instance': {
+ 'Meta': {'object_name': 'Instance'},
+ 'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+ 'date_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+ 'deleted_at': ('django.db.models.fields.DateTimeField', [], {'default': 'None', 'null': 'True'}),
+ 'geom': ('django.contrib.gis.db.models.fields.GeometryCollectionField', [], {'null': 'True'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_deleted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'json': ('jsonfield.fields.JSONField', [], {'default': '{}'}),
+ 'status': ('django.db.models.fields.CharField', [], {'default': "u'submitted_via_web'", 'max_length': '20'}),
+ 'survey_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['odk_logger.SurveyType']"}),
+ 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'surveys'", 'null': 'True', 'to': u"orm['auth.User']"}),
+ 'uuid': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '249'}),
+ 'xform': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'surveys'", 'null': 'True', 'to': "orm['odk_logger.XForm']"}),
+ 'xml': ('django.db.models.fields.TextField', [], {})
+ },
+ 'odk_logger.instancehistory': {
+ 'Meta': {'object_name': 'InstanceHistory'},
+ 'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+ 'date_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'uuid': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '249'}),
+ 'xform_instance': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'submission_history'", 'to': "orm['odk_logger.Instance']"}),
+ 'xml': ('django.db.models.fields.TextField', [], {})
+ },
+ 'odk_logger.note': {
+ 'Meta': {'object_name': 'Note'},
+ 'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+ 'date_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'instance': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'notes'", 'to': "orm['odk_logger.Instance']"}),
+ 'note': ('django.db.models.fields.TextField', [], {})
+ },
+ 'odk_logger.surveytype': {
+ 'Meta': {'object_name': 'SurveyType'},
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'slug': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'})
+ },
+ 'odk_logger.xform': {
+ 'Meta': {'ordering': "('id_string',)", 'unique_together': "(('user', 'id_string'), ('user', 'sms_id_string'))", 'object_name': 'XForm'},
+ 'allows_sms': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'bamboo_dataset': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '60'}),
+ 'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+ 'date_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+ 'description': ('django.db.models.fields.TextField', [], {'default': "u''", 'null': 'True'}),
+ 'downloadable': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'encrypted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'has_start_time': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'id_string': ('django.db.models.fields.SlugField', [], {'max_length': '100'}),
+ 'is_crowd_form': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'json': ('django.db.models.fields.TextField', [], {'default': "u''"}),
+ 'last_submission_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+ 'num_of_submissions': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+ 'shared': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'shared_data': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'sms_id_string': ('django.db.models.fields.SlugField', [], {'default': "''", 'max_length': '100'}),
+ 'surveys_with_geopoints': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+ 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'xforms'", 'null': 'True', 'to': u"orm['auth.User']"}),
+ 'uuid': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '32'}),
+ 'xls': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True'}),
+ 'xml': ('django.db.models.fields.TextField', [], {})
+ },
+ 'odk_logger.ziggyinstance': {
+ 'Meta': {'object_name': 'ZiggyInstance'},
+ 'client_version': ('django.db.models.fields.BigIntegerField', [], {'default': 'None', 'null': 'True'}),
+ 'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+ 'date_deleted': ('django.db.models.fields.DateTimeField', [], {'default': 'None', 'null': 'True'}),
+ 'date_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+ 'entity_id': ('django.db.models.fields.CharField', [], {'max_length': '249'}),
+ 'form_instance': ('django.db.models.fields.TextField', [], {}),
+ 'form_version': ('django.db.models.fields.CharField', [], {'default': "u'1.0'", 'max_length': '10'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'instance_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '249'}),
+ 'reporter': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'ziggys'", 'to': u"orm['auth.User']"}),
+ 'server_version': ('django.db.models.fields.BigIntegerField', [], {}),
+ 'xform': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'ziggy_submissions'", 'null': 'True', 'to': "orm['odk_logger.XForm']"})
+ },
+ u'taggit.tag': {
+ 'Meta': {'object_name': 'Tag'},
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'}),
+ 'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '100'})
+ },
+ u'taggit.taggeditem': {
+ 'Meta': {'object_name': 'TaggedItem'},
+ 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'taggit_taggeditem_tagged_items'", 'to': u"orm['contenttypes.ContentType']"}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'object_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}),
+ 'tag': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'taggit_taggeditem_items'", 'to': u"orm['taggit.Tag']"})
+ }
+ }
+
+ complete_apps = ['odk_logger']
\ No newline at end of file
diff --git a/onadata/apps/odk_logger/migrations/0041_convert_lng_lat_to_points.py b/onadata/apps/odk_logger/migrations/0041_convert_lng_lat_to_points.py
new file mode 100644
index 0000000000..6cd0e5fd91
--- /dev/null
+++ b/onadata/apps/odk_logger/migrations/0041_convert_lng_lat_to_points.py
@@ -0,0 +1,157 @@
+# -*- coding: utf-8 -*-
+from south.v2 import DataMigration
+
+from onadata.apps.odk_logger.models.instance import Instance
+from onadata.libs.utils.model_tools import queryset_iterator
+
+
+class Migration(DataMigration):
+
+ def forwards(self, orm):
+ "Parse all instance to add geoms."
+ for obj in queryset_iterator(orm.Instance.objects.all()):
+ instance = Instance.objects.get(pk=obj.pk)
+ instance.save(force=True)
+
+ def backwards(self, orm):
+ "Write your backwards methods here."
+
+ models = {
+ u'auth.group': {
+ 'Meta': {'object_name': 'Group'},
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+ 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+ },
+ u'auth.permission': {
+ 'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
+ 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+ },
+ u'auth.user': {
+ 'Meta': {'object_name': 'User'},
+ 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+ 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+ 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+ 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+ 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+ 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+ 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+ 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+ },
+ u'contenttypes.contenttype': {
+ 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+ 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+ },
+ 'odk_logger.attachment': {
+ 'Meta': {'object_name': 'Attachment'},
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'instance': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'attachments'", 'to': "orm['odk_logger.Instance']"}),
+ 'media_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}),
+ 'mimetype': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'})
+ },
+ 'odk_logger.instance': {
+ 'Meta': {'object_name': 'Instance'},
+ 'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+ 'date_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+ 'deleted_at': ('django.db.models.fields.DateTimeField', [], {'default': 'None', 'null': 'True'}),
+ 'geom': ('django.contrib.gis.db.models.fields.GeometryCollectionField', [], {'null': 'True'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'json': ('jsonfield.fields.JSONField', [], {'default': '{}'}),
+ 'status': ('django.db.models.fields.CharField', [], {'default': "u'submitted_via_web'", 'max_length': '20'}),
+ 'survey_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['odk_logger.SurveyType']"}),
+ 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'surveys'", 'null': 'True', 'to': u"orm['auth.User']"}),
+ 'uuid': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '249'}),
+ 'xform': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'surveys'", 'null': 'True', 'to': "orm['odk_logger.XForm']"}),
+ 'xml': ('django.db.models.fields.TextField', [], {})
+ },
+ 'odk_logger.instancehistory': {
+ 'Meta': {'object_name': 'InstanceHistory'},
+ 'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+ 'date_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'uuid': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '249'}),
+ 'xform_instance': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'submission_history'", 'to': "orm['odk_logger.Instance']"}),
+ 'xml': ('django.db.models.fields.TextField', [], {})
+ },
+ 'odk_logger.note': {
+ 'Meta': {'object_name': 'Note'},
+ 'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+ 'date_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'instance': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'notes'", 'to': "orm['odk_logger.Instance']"}),
+ 'note': ('django.db.models.fields.TextField', [], {})
+ },
+ 'odk_logger.surveytype': {
+ 'Meta': {'object_name': 'SurveyType'},
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'slug': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'})
+ },
+ 'odk_logger.xform': {
+ 'Meta': {'ordering': "('id_string',)", 'unique_together': "(('user', 'id_string'), ('user', 'sms_id_string'))", 'object_name': 'XForm'},
+ 'allows_sms': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'bamboo_dataset': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '60'}),
+ 'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+ 'date_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+ 'description': ('django.db.models.fields.TextField', [], {'default': "u''", 'null': 'True'}),
+ 'downloadable': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'encrypted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'has_start_time': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'id_string': ('django.db.models.fields.SlugField', [], {'max_length': '100'}),
+ 'is_crowd_form': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'json': ('django.db.models.fields.TextField', [], {'default': "u''"}),
+ 'last_submission_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+ 'num_of_submissions': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+ 'shared': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'shared_data': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'sms_id_string': ('django.db.models.fields.SlugField', [], {'default': "''", 'max_length': '100'}),
+ 'surveys_with_geopoints': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+ 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'xforms'", 'null': 'True', 'to': u"orm['auth.User']"}),
+ 'uuid': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '32'}),
+ 'xls': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True'}),
+ 'xml': ('django.db.models.fields.TextField', [], {})
+ },
+ 'odk_logger.ziggyinstance': {
+ 'Meta': {'object_name': 'ZiggyInstance'},
+ 'client_version': ('django.db.models.fields.BigIntegerField', [], {'default': 'None', 'null': 'True'}),
+ 'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+ 'date_deleted': ('django.db.models.fields.DateTimeField', [], {'default': 'None', 'null': 'True'}),
+ 'date_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+ 'entity_id': ('django.db.models.fields.CharField', [], {'max_length': '249'}),
+ 'form_instance': ('django.db.models.fields.TextField', [], {}),
+ 'form_version': ('django.db.models.fields.CharField', [], {'default': "u'1.0'", 'max_length': '10'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'instance_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '249'}),
+ 'reporter': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'ziggys'", 'to': u"orm['auth.User']"}),
+ 'server_version': ('django.db.models.fields.BigIntegerField', [], {}),
+ 'xform': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'ziggy_submissions'", 'null': 'True', 'to': "orm['odk_logger.XForm']"})
+ },
+ u'taggit.tag': {
+ 'Meta': {'object_name': 'Tag'},
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'}),
+ 'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '100'})
+ },
+ u'taggit.taggeditem': {
+ 'Meta': {'object_name': 'TaggedItem'},
+ 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'taggit_taggeditem_tagged_items'", 'to': u"orm['contenttypes.ContentType']"}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'object_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}),
+ 'tag': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'taggit_taggeditem_items'", 'to': u"orm['taggit.Tag']"})
+ }
+ }
+
+ complete_apps = ['odk_logger']
+ symmetrical = True
diff --git a/onadata/apps/odk_logger/models/instance.py b/onadata/apps/odk_logger/models/instance.py
index a2120ff6d6..8f2d5d918c 100644
--- a/onadata/apps/odk_logger/models/instance.py
+++ b/onadata/apps/odk_logger/models/instance.py
@@ -1,7 +1,10 @@
-from django.db import models
+from datetime import datetime
+
+from django.contrib.gis.db import models
from django.db.models.signals import post_save
from django.db.models.signals import post_delete
from django.contrib.auth.models import User
+from django.contrib.gis.geos import GeometryCollection, Point
from django.utils import timezone
from django.utils.translation import ugettext as _
from jsonfield import JSONField
@@ -11,8 +14,9 @@
from onadata.apps.odk_logger.models.xform import XForm
from onadata.apps.odk_logger.xform_instance_parser import XFormInstanceParser,\
clean_and_parse_xml, get_uuid_from_xml
-from onadata.libs.utils.common_tags import MONGO_STRFTIME, SUBMISSION_TIME,\
- XFORM_ID_STRING
+from onadata.libs.utils.common_tags import ATTACHMENTS, BAMBOO_DATASET_ID,\
+ DELETEDAT, GEOLOCATION, ID, MONGO_STRFTIME, NOTES, SUBMISSION_TIME, TAGS,\
+ UUID, XFORM_ID_STRING
from onadata.libs.utils.model_tools import set_uuid
@@ -37,6 +41,49 @@ def submission_time():
return timezone.now()
+def update_xform_submission_count(sender, instance, created, **kwargs):
+ if created:
+ xform = XForm.objects.select_related().select_for_update()\
+ .get(pk=instance.xform.pk)
+ if xform.num_of_submissions == -1:
+ xform.num_of_submissions = 0
+ xform.num_of_submissions += 1
+ xform.last_submission_time = instance.date_created
+ xform.save()
+ profile_qs = User.profile.get_query_set()
+ try:
+ profile = profile_qs.select_for_update()\
+ .get(pk=xform.user.profile.pk)
+ except profile_qs.model.DoesNotExist:
+ pass
+ else:
+ profile.num_of_submissions += 1
+ profile.save()
+
+
+def update_xform_submission_count_delete(sender, instance, **kwargs):
+ try:
+ xform = XForm.objects.select_for_update().get(pk=instance.xform.pk)
+ except XForm.DoesNotExist:
+ pass
+ else:
+ xform.num_of_submissions -= 1
+ if xform.num_of_submissions < 0:
+ xform.num_of_submissions = 0
+ xform.save()
+ profile_qs = User.profile.get_query_set()
+ try:
+ profile = profile_qs.select_for_update()\
+ .get(pk=xform.user.profile.pk)
+ except profile_qs.model.DoesNotExist:
+ pass
+ else:
+ profile.num_of_submissions -= 1
+ if profile.num_of_submissions < 0:
+ profile.num_of_submissions = 0
+ profile.save()
+
+
class Instance(models.Model):
json = JSONField(default={}, null=False)
xml = models.TextField()
@@ -55,30 +102,60 @@ class Instance(models.Model):
# ODK keeps track of three statuses for an instance:
# incomplete, submitted, complete
- # we will add a fourth status: submitted_via_web
+ # we add a fourth status: submitted_via_web
status = models.CharField(max_length=20,
default=u'submitted_via_web')
uuid = models.CharField(max_length=249, default=u'')
+ # store an geographic objects associated with this instance
+ geom = models.GeometryCollectionField(null=True)
+ objects = models.GeoManager()
+
tags = TaggableManager()
class Meta:
app_label = 'odk_logger'
- def _set_xform(self, id_string):
- self.xform = XForm.objects.get(id_string=id_string, user=self.user)
+ @classmethod
+ def set_deleted_at(cls, instance_id, deleted_at=timezone.now()):
+ try:
+ instance = cls.objects.get(id=instance_id)
+ except cls.DoesNotExist:
+ pass
+ else:
+ instance.set_deleted(deleted_at)
- def get_root_node_name(self):
- self._set_parser()
- return self._parser.get_root_node_name()
+ def _check_active(self, kwargs):
+ """Check that form is active and raise exception if not.
- def get_root_node(self):
- self._set_parser()
- return self._parser.get_root_node()
+ :param kwargs: The kwargs passed to save, if it contains a 'force' key
+ ignore restrictions on saving.
+ """
+ force = kwargs.get('force')
- def get(self, abbreviated_xpath):
- self._set_parser()
- return self._parser.get(abbreviated_xpath)
+ if force:
+ del kwargs['force']
+
+ if not force and self.xform and not self.xform.downloadable:
+ raise FormInactiveError()
+
+ return kwargs
+
+ def _set_geom(self):
+ data_dictionary = self.xform.data_dictionary()
+ geo_xpaths = data_dictionary.geopoint_xpaths()
+ doc = self.get_dict()
+ points = []
+
+ if len(geo_xpaths):
+ for xpath in geo_xpaths:
+ geometry = [float(s) for s in doc.get(xpath, u'').split()]
+
+ if len(geometry):
+ lat, lng = geometry[0:2]
+ points.append(Point(lng, lat))
+
+ self.geom = GeometryCollection(points)
def _set_json(self):
doc = self.get_dict()
@@ -87,10 +164,19 @@ def _set_json(self):
now = submission_time()
self.date_created = now
+ point = self.point
+ if point:
+ doc[GEOLOCATION] = [point.y, point.x]
+
doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME)
doc[XFORM_ID_STRING] = self._parser.get_xform_id_string()
self.json = doc
+ def _set_parser(self):
+ if not hasattr(self, "_parser"):
+ self._parser = XFormInstanceParser(
+ self.xml, self.xform.data_dictionary())
+
def _set_survey_type(self):
self.survey_type, created = \
SurveyType.objects.get_or_create(slug=self.get_root_node_name())
@@ -102,21 +188,12 @@ def _set_uuid(self):
self.uuid = uuid
set_uuid(self)
- def save(self, *args, **kwargs):
- self._set_xform(get_id_string_from_xml_str(self.xml))
-
- if self.xform and not self.xform.downloadable:
- raise FormInactiveError()
-
- self._set_json()
- self._set_survey_type()
- self._set_uuid()
- super(Instance, self).save(*args, **kwargs)
+ def _set_xform(self, id_string):
+ self.xform = XForm.objects.get(id_string=id_string, user=self.user)
- def _set_parser(self):
- if not hasattr(self, "_parser"):
- self._parser = XFormInstanceParser(
- self.xml, self.xform.data_dictionary())
+ def get(self, abbreviated_xpath):
+ self._set_parser()
+ return self._parser.get(abbreviated_xpath)
def get_dict(self, force_new=False, flat=True):
"""Return a python object representation of this instance's XML."""
@@ -125,66 +202,67 @@ def get_dict(self, force_new=False, flat=True):
return self._parser.get_flat_dict_with_attributes() if flat else\
self._parser.to_dict()
+ def get_full_dict(self):
+ # TODO should we store all of these in the JSON no matter what?
+ d = self.json
+ data = {
+ UUID: self.uuid,
+ ID: self.id,
+ BAMBOO_DATASET_ID: self.xform.bamboo_dataset,
+ self.USERFORM_ID: u'%s_%s' % (
+ self.user.username,
+ self.xform.id_string),
+ ATTACHMENTS: [a.media_file.name for a in
+ self.attachments.all()],
+ self.STATUS: self.status,
+ TAGS: list(self.tags.names()),
+ NOTES: self.get_notes()
+ }
+
+ if isinstance(self.instance.deleted_at, datetime):
+ data[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME)
+
+ d.update(data)
+
+ return d
+
+ def get_notes(self):
+ return [note['note'] for note in self.notes.values('note')]
+
+ def get_root_node(self):
+ self._set_parser()
+ return self._parser.get_root_node()
+
+ def get_root_node_name(self):
+ self._set_parser()
+ return self._parser.get_root_node_name()
+
+ @property
+ def point(self):
+ gc = self.geom
+
+ if gc and len(gc):
+ return gc[0]
+
+ def save(self, *args, **kwargs):
+ kwargs = self._check_active(kwargs)
+
+ self._set_xform(get_id_string_from_xml_str(self.xml))
+ self._set_geom()
+ self._set_json()
+ self._set_survey_type()
+ self._set_uuid()
+ super(Instance, self).save(*args, **kwargs)
+
def set_deleted(self, deleted_at=timezone.now()):
self.deleted_at = deleted_at
self.save()
self.parsed_instance.save()
- @classmethod
- def set_deleted_at(cls, instance_id, deleted_at=timezone.now()):
- try:
- instance = cls.objects.get(id=instance_id)
- except cls.DoesNotExist:
- pass
- else:
- instance.set_deleted(deleted_at)
-
-
-def update_xform_submission_count(sender, instance, created, **kwargs):
- if created:
- xform = XForm.objects.select_related().select_for_update()\
- .get(pk=instance.xform.pk)
- if xform.num_of_submissions == -1:
- xform.num_of_submissions = 0
- xform.num_of_submissions += 1
- xform.last_submission_time = instance.date_created
- xform.save()
- profile_qs = User.profile.get_query_set()
- try:
- profile = profile_qs.select_for_update()\
- .get(pk=xform.user.profile.pk)
- except profile_qs.model.DoesNotExist:
- pass
- else:
- profile.num_of_submissions += 1
- profile.save()
post_save.connect(update_xform_submission_count, sender=Instance,
dispatch_uid='update_xform_submission_count')
-
-def update_xform_submission_count_delete(sender, instance, **kwargs):
- try:
- xform = XForm.objects.select_for_update().get(pk=instance.xform.pk)
- except XForm.DoesNotExist:
- pass
- else:
- xform.num_of_submissions -= 1
- if xform.num_of_submissions < 0:
- xform.num_of_submissions = 0
- xform.save()
- profile_qs = User.profile.get_query_set()
- try:
- profile = profile_qs.select_for_update()\
- .get(pk=xform.user.profile.pk)
- except profile_qs.model.DoesNotExist:
- pass
- else:
- profile.num_of_submissions -= 1
- if profile.num_of_submissions < 0:
- profile.num_of_submissions = 0
- profile.save()
-
post_delete.connect(update_xform_submission_count_delete, sender=Instance,
dispatch_uid='update_xform_submission_count_delete')
diff --git a/onadata/apps/odk_logger/models/xform.py b/onadata/apps/odk_logger/models/xform.py
index d44bb529aa..4158eddd92 100644
--- a/onadata/apps/odk_logger/models/xform.py
+++ b/onadata/apps/odk_logger/models/xform.py
@@ -172,13 +172,9 @@ def submission_count(self):
submission_count.short_description = ugettext_lazy("Submission Count")
def geocoded_submission_count(self):
- from onadata.apps.odk_viewer.models.parsed_instance import\
- ParsedInstance
- return ParsedInstance.objects.filter(
- instance__in=self.instances.filter(deleted_at__isnull=True),
- lat__isnull=False).count()
- geocoded_submission_count.short_description = \
- ugettext_lazy("Geocoded Submission Count")
+ """Number of geocoded submissions."""
+ return self.instances.filter(deleted_at__isnull=True,
+ geom__isnull=False).count()
def time_of_last_submission(self):
if self.last_submission_time is None and self.num_of_submissions > 0:
diff --git a/onadata/apps/odk_logger/tests/test_import_tools.py b/onadata/apps/odk_logger/tests/test_importing_database.py
similarity index 100%
rename from onadata/apps/odk_logger/tests/test_import_tools.py
rename to onadata/apps/odk_logger/tests/test_importing_database.py
diff --git a/onadata/apps/odk_viewer/models/data_dictionary.py b/onadata/apps/odk_viewer/models/data_dictionary.py
index 56a6e6a1c3..5889cf0b4e 100644
--- a/onadata/apps/odk_viewer/models/data_dictionary.py
+++ b/onadata/apps/odk_viewer/models/data_dictionary.py
@@ -9,10 +9,9 @@
from pyxform.xform2json import create_survey_element_from_xml
from xml.dom import Node
-from onadata.apps.odk_logger.models import XForm
+from onadata.apps.odk_logger.models.xform import XForm
from onadata.apps.odk_logger.xform_instance_parser import clean_and_parse_xml
-from onadata.apps.odk_viewer.models.parsed_instance import _encode_for_mongo,\
- ParsedInstance
+from onadata.apps.odk_viewer.models.parsed_instance import _encode_for_mongo
from onadata.libs.utils.common_tags import UUID, SUBMISSION_TIME, TAGS, NOTES
from onadata.libs.utils.export_tools import question_types_to_exclude,\
DictOrganizer
@@ -186,14 +185,22 @@ def get_mongo_field_names_dict(self):
survey_elements = property(get_survey_elements)
- def xpath_of_first_geopoint(self):
+ def geopoint_xpaths(self):
+ geo_xpaths = []
+
for e in self.get_survey_elements():
if e.bind.get(u'type') == u'geopoint':
- return e.get_abbreviated_xpath()
+ geo_xpaths.append(e.get_abbreviated_xpath())
+
+ return geo_xpaths
+
+ def xpath_of_first_geopoint(self):
+ geo_xpaths = self.geopoint_xpaths()
+
+ return len(geo_xpaths) > 0 and geo_xpaths[0]
def has_instances_with_geopoints(self):
- return ParsedInstance.objects.filter(
- instance__xform=self, lat__isnull=False).count() > 0
+ return self.geocoded_submission_count() > 0
def xpaths(self, prefix='', survey_element=None, result=None,
repeat_iterations=4):
diff --git a/onadata/apps/odk_viewer/pandas_mongo_bridge.py b/onadata/apps/odk_viewer/pandas_mongo_bridge.py
index 202c6346d3..e962647011 100644
--- a/onadata/apps/odk_viewer/pandas_mongo_bridge.py
+++ b/onadata/apps/odk_viewer/pandas_mongo_bridge.py
@@ -9,10 +9,10 @@
from pyxform.section import Section, RepeatingSection
from pyxform.question import Question
-from onadata.apps.odk_viewer.models.data_dictionary import ParsedInstance,\
- DataDictionary
-from onadata.libs.utils.common_tags import ID, XFORM_ID_STRING, STATUS, \
- ATTACHMENTS, GEOLOCATION, UUID, SUBMISSION_TIME, NA_REP, \
+from onadata.apps.odk_viewer.models.data_dictionary import DataDictionary
+from onadata.apps.odk_viewer.models.parsed_instance import ParsedInstance
+from onadata.libs.utils.common_tags import ID, XFORM_ID_STRING, STATUS,\
+ ATTACHMENTS, GEOLOCATION, UUID, SUBMISSION_TIME, NA_REP,\
BAMBOO_DATASET_ID, DELETEDAT, TAGS, NOTES
from onadata.libs.utils.export_tools import question_types_to_exclude
diff --git a/onadata/apps/odk_viewer/tests/test_kml_export.py b/onadata/apps/odk_viewer/tests/test_kml_export.py
index dc390a076b..7327d4eb9f 100644
--- a/onadata/apps/odk_viewer/tests/test_kml_export.py
+++ b/onadata/apps/odk_viewer/tests/test_kml_export.py
@@ -3,7 +3,7 @@
from django.core.urlresolvers import reverse
from onadata.apps.main.tests.test_base import TestBase
-from onadata.apps.odk_viewer.models.parsed_instance import ParsedInstance
+from onadata.apps.odk_logger.models.instance import Instance
from onadata.apps.odk_viewer.views import kml_export
@@ -31,13 +31,13 @@ def test_kml_export(self):
kml_export,
kwargs={'username': self.user.username, 'id_string': id_string})
response = self.client.get(url)
- pis = ParsedInstance.objects.filter(
- instance__user=self.user, instance__xform__id_string=id_string,
- lat__isnull=False, lng__isnull=False).order_by('id')
+ instances = Instance.objects.filter(
+ user=self.user, xform__id_string=id_string, geom__isnull=False
+ ).order_by('id')
- self.assertEqual(pis.count(), 2)
+ self.assertEqual(instances.count(), 2)
- first, second = [str(i.pk) for i in pis]
+ first, second = [str(i.pk) for i in instances]
with open(os.path.join(self.fixtures, 'export.kml')) as f:
expected_content = f.read()
diff --git a/onadata/libs/utils/bamboo.py b/onadata/libs/utils/bamboo.py
index 7633d5163e..c80e9226d3 100644
--- a/onadata/libs/utils/bamboo.py
+++ b/onadata/libs/utils/bamboo.py
@@ -5,7 +5,7 @@
from pybamboo.connection import Connection
from pybamboo.exceptions import ErrorParsingBambooData
-from onadata.apps.odk_viewer.models.parsed_instance import ParsedInstance
+from onadata.apps.odk_logger.models.instance import Instance
from onadata.apps.odk_viewer.pandas_mongo_bridge import CSVDataFrameBuilder,\
NoRecordsFoundError
from onadata.apps.restservice.models import RestService
@@ -89,10 +89,10 @@ def get_csv_data_manual(xform,
# when form has only one submission, CSVDFB is empty.
# we still want to create the BB ds with row 1
# so we extract is and CSV it.
- pifilter = ParsedInstance.objects.filter(instance__xform=xform) \
- .order_by('-instance__date_modified')
+ instances = Instance.objects.filter(xform=xform).order_by(
+ '-date_modified')
- if pifilter.count() == 0:
+ if instances.count() == 0:
raise NoRecordsFoundError
else:
# we should only do it for count == 1 but eh.
@@ -100,9 +100,9 @@ def get_csv_data_manual(xform,
csv_buf = getbuff()
if only_last:
- pifilter = [pifilter[0]]
+ instances = instances[0:1]
- rows = [pi.to_dict_for_mongo() for pi in pifilter]
+ rows = [instance.get_full_dict() for instance in instances]
if headers_to_use is None:
headers_to_use = [key for key in rows[0].keys()
diff --git a/onadata/libs/utils/export_tools.py b/onadata/libs/utils/export_tools.py
index 6ca0581461..e21d6495d0 100644
--- a/onadata/libs/utils/export_tools.py
+++ b/onadata/libs/utils/export_tools.py
@@ -18,13 +18,15 @@
from pyxform.section import Section, RepeatingSection
from savReaderWriter import SavWriter
-from onadata.apps.odk_logger.models import XForm, Attachment
-from onadata.apps.odk_viewer.models.parsed_instance import _is_invalid_for_mongo,\
- _encode_for_mongo, dict_for_mongo, _decode_from_mongo
-from onadata.libs.utils.viewer_tools import create_attachments_zipfile, image_urls
-from onadata.libs.utils.common_tags import ID, XFORM_ID_STRING, STATUS, ATTACHMENTS,\
- GEOLOCATION, BAMBOO_DATASET_ID, DELETEDAT, USERFORM_ID, INDEX,\
- PARENT_INDEX, PARENT_TABLE_NAME, SUBMISSION_TIME, UUID, TAGS, NOTES
+from onadata.apps.odk_logger.models import Attachment, Instance, XForm
+from onadata.apps.odk_viewer.models.parsed_instance import\
+ _is_invalid_for_mongo, _encode_for_mongo, dict_for_mongo,\
+ _decode_from_mongo
+from onadata.libs.utils.viewer_tools import create_attachments_zipfile,\
+ image_urls
+from onadata.libs.utils.common_tags import ID, XFORM_ID_STRING, STATUS,\
+ ATTACHMENTS, GEOLOCATION, BAMBOO_DATASET_ID, DELETEDAT, USERFORM_ID,\
+ INDEX, PARENT_INDEX, PARENT_TABLE_NAME, SUBMISSION_TIME, UUID, TAGS, NOTES
# this is Mongo Collection where we will store the parsed submissions
@@ -184,7 +186,8 @@ def format_field_title(cls, abbreviated_xpath, field_delimiter):
def set_survey(self, survey):
# TODO resolve circular import
- from onadata.apps.odk_viewer.models.data_dictionary import DataDictionary
+ from onadata.apps.odk_viewer.models.data_dictionary import\
+ DataDictionary
def build_sections(
current_section, survey_element, sections, select_multiples,
@@ -550,7 +553,8 @@ def write_row(data, work_sheet, fields, work_sheet_titles):
def to_flat_csv_export(
self, path, data, username, id_string, filter_query):
# TODO resolve circular import
- from onadata.apps.odk_viewer.pandas_mongo_bridge import CSVDataFrameBuilder
+ from onadata.apps.odk_viewer.pandas_mongo_bridge import\
+ CSVDataFrameBuilder
csv_builder = CSVDataFrameBuilder(
username, id_string, filter_query, self.GROUP_DELIMITER,
@@ -871,11 +875,10 @@ def generate_kml_export(
def kml_export_data(id_string, user):
# TODO resolve circular import
from onadata.apps.odk_viewer.models.data_dictionary import DataDictionary
- from onadata.apps.odk_viewer.models.parsed_instance import ParsedInstance
dd = DataDictionary.objects.get(id_string=id_string, user=user)
- pis = ParsedInstance.objects.filter(
- instance__user=user, instance__xform__id_string=id_string,
- lat__isnull=False, lng__isnull=False).order_by('id')
+ instances = Instance.objects.filter(
+ user=user, xform__id_string=id_string, geom__isnull=False
+ ).order_by('id')
data_for_template = []
labels = {}
@@ -886,11 +889,11 @@ def cached_get_labels(xpath):
labels[xpath] = dd.get_label(xpath)
return labels[xpath]
- for pi in pis:
+ for instance in instances:
# read the survey instances
- data_for_display = pi.to_dict()
+ data_for_display = instance.get_dict()
xpaths = data_for_display.keys()
- xpaths.sort(cmp=pi.data_dictionary.get_xpath_cmp())
+ xpaths.sort(cmp=instance.xform.data_dictionary().get_xpath_cmp())
label_value_pairs = [
(cached_get_labels(xpath),
data_for_display[xpath]) for xpath in xpaths
@@ -898,13 +901,14 @@ def cached_get_labels(xpath):
table_rows = []
for key, value in label_value_pairs:
table_rows.append('| %s | %s |
' % (key, value))
- img_urls = image_urls(pi.instance)
+ img_urls = image_urls(instance)
img_url = img_urls[0] if img_urls else ""
+ point = instance.point
data_for_template.append({
'name': id_string,
- 'id': pi.id,
- 'lat': pi.lat,
- 'lng': pi.lng,
+ 'id': instance.id,
+ 'lat': point.y,
+ 'lng': point.x,
'image_urls': img_urls,
'table': '
%s'
diff --git a/onadata/settings/common.py b/onadata/settings/common.py
index 6998499ecd..4105b4121a 100644
--- a/onadata/settings/common.py
+++ b/onadata/settings/common.py
@@ -178,6 +178,7 @@
'django.contrib.humanize',
'django.contrib.admin',
'django.contrib.admindocs',
+ 'django.contrib.gis',
'registration',
'south',
'django_nose',
diff --git a/onadata/settings/default_settings.py b/onadata/settings/default_settings.py
index 1f438469b5..e863f1f39f 100644
--- a/onadata/settings/default_settings.py
+++ b/onadata/settings/default_settings.py
@@ -10,12 +10,9 @@
from staging_example import * # nopep8
# # # now override the settings which came from staging # # # #
-
-# choose a different database...
-# sqlite
DATABASES = {
'default': {
- 'ENGINE': 'django.db.backends.postgresql_psycopg2',
+ 'ENGINE': 'django.contrib.gis.db.backends.postgis',
'NAME': 'onadata',
'USER': 'onadata',
'PASSWORD': '',
diff --git a/onadata/settings/example_gis.py b/onadata/settings/example_gis.py
deleted file mode 100644
index 6b92a1fe29..0000000000
--- a/onadata/settings/example_gis.py
+++ /dev/null
@@ -1,52 +0,0 @@
-from common import * # nopep8
-
-DEBUG = True
-TEMPLATE_DEBUG = DEBUG
-TEMPLATE_STRING_IF_INVALID = '' # '***Invalid Template String***'
-
-DATABASES = {
- 'default': {
- 'ENGINE': 'django.db.backends.postgresql_psycopg2',
- 'NAME': 'formhub_dev',
- 'USER': 'formhub_dev',
- 'PASSWORD': '12345678',
- },
- 'gis': {
- 'ENGINE': 'django.contrib.gis.db.backends.postgis',
- 'NAME': 'phis',
- 'USER': 'nomadstaff',
- 'PASSWORD': 'nopolio',
- 'HOST': 'localhost',
- 'OPTIONS': {
- 'autocommit': True,
- }
- }
-}
-
-TOUCHFORMS_URL = 'http://localhost:9000/'
-
-SECRET_KEY = 'mlfs33^s1l4xf6a36$0#srgcpj%dd*sisfo6HOktYXB9y'
-
-TESTING_MODE = False
-if len(sys.argv) >= 2 and (sys.argv[1] == "test" or sys.argv[1] == "test_all"):
- # This trick works only when we run tests from the command line.
- TESTING_MODE = True
-
-if TESTING_MODE:
- MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'test_media/')
- subprocess.call(["rm", "-r", MEDIA_ROOT])
- MONGO_DATABASE['NAME'] = "formhub_test"
- # need to have CELERY_ALWAYS_EAGER True and BROKER_BACKEND as memory
- # to run tasks immediately while testing
- CELERY_ALWAYS_EAGER = True
- BROKER_BACKEND = 'memory'
- ENKETO_API_TOKEN = 'abc'
-else:
- MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'media/')
-
-if PRINT_EXCEPTION and DEBUG:
- MIDDLEWARE_CLASSES += ('utils.middleware.ExceptionLoggingMiddleware',)
-
-# Clear out the test database
-if TESTING_MODE:
- MONGO_DB.instances.drop()
diff --git a/onadata/settings/travis_test.py b/onadata/settings/travis_test.py
index 909538c257..5d837b6830 100644
--- a/onadata/settings/travis_test.py
+++ b/onadata/settings/travis_test.py
@@ -4,7 +4,7 @@
DATABASES = {
'default': {
- 'ENGINE': 'django.db.backends.postgresql_psycopg2',
+ 'ENGINE': 'django.contrib.gis.db.backends.postgis',
'NAME': 'onadata_test',
'USER': 'postgres',
'PASSWORD': '',
diff --git a/requirements/common.pip b/requirements/common.pip
index cb0dbd0d54..7797406d23 100644
--- a/requirements/common.pip
+++ b/requirements/common.pip
@@ -33,7 +33,6 @@ pandas==0.12.0
requests==2.0.1
elaphe==0.5.6
-
# sms support
dict2xml==1.1
diff --git a/script/database/install_postgis b/script/database/install_postgis
new file mode 100755
index 0000000000..299906c288
--- /dev/null
+++ b/script/database/install_postgis
@@ -0,0 +1,4 @@
+#!/bin/bash
+psql -c "CREATE DATABASE $1;" -U postgres
+psql -c "CREATE EXTENSION IF NOT EXISTS postgis;" -d $1 -U postgres
+psql -c "CREATE EXTENSION IF NOT EXISTS postgis_topology;" -d $1 -U postgres
diff --git a/script/database/load_backup b/script/database/load_backup
new file mode 100755
index 0000000000..92820c0bcd
--- /dev/null
+++ b/script/database/load_backup
@@ -0,0 +1,3 @@
+scp ona:~/onadata.sql.tar.gz
+tar -xzvf onadata.sql.tar.gz
+psql -U onadata -h 127.0.0.1 onadata -f backups/postgres/onadata.sql
diff --git a/script/install_arch b/script/install_arch
new file mode 100755
index 0000000000..21a219a9d5
--- /dev/null
+++ b/script/install_arch
@@ -0,0 +1,2 @@
+# install PostGIS
+sudo pacman -S postgis
diff --git a/script/install_ubuntu b/script/install_ubuntu
index f3e378c05c..8c73c3a93b 100755
--- a/script/install_ubuntu
+++ b/script/install_ubuntu
@@ -1,8 +1,16 @@
# add node repo
sudo add-apt-repository -y ppa:chris-lea/node.js
+# add postgres repo, instructions:
+# http://trac.osgeo.org/postgis/wiki/UsersWikiPostGIS21UbuntuPGSQL93Apt
+sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main" >> /etc/apt/sources.list'
+wget --quiet -O - http://apt.postgresql.org/pub/repos/apt/ACCC4CF8.asc | sudo apt-key add -
sudo apt-get update -qqy
sudo apt-get install -qq gfortran libatlas-base-dev libjpeg-dev python-numpy zlib1g-dev python-software-properties
-sudo apt-get install -qqy nodejs
+# install Geospatial libraries for GeoDjango
+sudo apt-get install -qq binutils libproj-dev gdal-bin Postgresql-9.3-postgis
sudo ln -s /usr/lib/`uname -i`-linux-gnu/libjpeg.so ~/virtualenv/python2.7/lib/
sudo ln -s /usr/lib/`uname -i`-linux-gnu/libz.so ~/virtualenv/python2.7/lib/
-sudo npm install -g --silent bower karma grunt-cli
+if [[ "$TESTFOLDER" == onadata/apps/api* ]]; then
+ sudo apt-get install -qqy nodejs
+ sudo npm install --silent -g bower karma grunt-cli
+fi