From f29fa0f67ee806f81630a73879a57b41a50a882b Mon Sep 17 00:00:00 2001 From: pld Date: Tue, 14 Jan 2014 21:43:45 -0500 Subject: [PATCH 01/31] PLD: add geodjango and use postgis db engine --- onadata/settings/common.py | 1 + onadata/settings/default_settings.py | 5 +-- onadata/settings/example_gis.py | 52 ---------------------------- onadata/settings/travis_test.py | 2 +- 4 files changed, 3 insertions(+), 57 deletions(-) delete mode 100644 onadata/settings/example_gis.py diff --git a/onadata/settings/common.py b/onadata/settings/common.py index 6998499ecd..4105b4121a 100644 --- a/onadata/settings/common.py +++ b/onadata/settings/common.py @@ -178,6 +178,7 @@ 'django.contrib.humanize', 'django.contrib.admin', 'django.contrib.admindocs', + 'django.contrib.gis', 'registration', 'south', 'django_nose', diff --git a/onadata/settings/default_settings.py b/onadata/settings/default_settings.py index 1f438469b5..e863f1f39f 100644 --- a/onadata/settings/default_settings.py +++ b/onadata/settings/default_settings.py @@ -10,12 +10,9 @@ from staging_example import * # nopep8 # # # now override the settings which came from staging # # # # - -# choose a different database... -# sqlite DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'ENGINE': 'django.contrib.gis.db.backends.postgis', 'NAME': 'onadata', 'USER': 'onadata', 'PASSWORD': '', diff --git a/onadata/settings/example_gis.py b/onadata/settings/example_gis.py deleted file mode 100644 index 6b92a1fe29..0000000000 --- a/onadata/settings/example_gis.py +++ /dev/null @@ -1,52 +0,0 @@ -from common import * # nopep8 - -DEBUG = True -TEMPLATE_DEBUG = DEBUG -TEMPLATE_STRING_IF_INVALID = '' # '***Invalid Template String***' - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'NAME': 'formhub_dev', - 'USER': 'formhub_dev', - 'PASSWORD': '12345678', - }, - 'gis': { - 'ENGINE': 'django.contrib.gis.db.backends.postgis', - 'NAME': 'phis', - 'USER': 'nomadstaff', - 'PASSWORD': 'nopolio', - 'HOST': 'localhost', - 'OPTIONS': { - 'autocommit': True, - } - } -} - -TOUCHFORMS_URL = 'http://localhost:9000/' - -SECRET_KEY = 'mlfs33^s1l4xf6a36$0#srgcpj%dd*sisfo6HOktYXB9y' - -TESTING_MODE = False -if len(sys.argv) >= 2 and (sys.argv[1] == "test" or sys.argv[1] == "test_all"): - # This trick works only when we run tests from the command line. - TESTING_MODE = True - -if TESTING_MODE: - MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'test_media/') - subprocess.call(["rm", "-r", MEDIA_ROOT]) - MONGO_DATABASE['NAME'] = "formhub_test" - # need to have CELERY_ALWAYS_EAGER True and BROKER_BACKEND as memory - # to run tasks immediately while testing - CELERY_ALWAYS_EAGER = True - BROKER_BACKEND = 'memory' - ENKETO_API_TOKEN = 'abc' -else: - MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'media/') - -if PRINT_EXCEPTION and DEBUG: - MIDDLEWARE_CLASSES += ('utils.middleware.ExceptionLoggingMiddleware',) - -# Clear out the test database -if TESTING_MODE: - MONGO_DB.instances.drop() diff --git a/onadata/settings/travis_test.py b/onadata/settings/travis_test.py index 909538c257..5d837b6830 100644 --- a/onadata/settings/travis_test.py +++ b/onadata/settings/travis_test.py @@ -4,7 +4,7 @@ DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'ENGINE': 'django.contrib.gis.db.backends.postgis', 'NAME': 'onadata_test', 'USER': 'postgres', 'PASSWORD': '', From a7eb3503aba3b9845f92328e56b49e8bc68e5d46 Mon Sep 17 00:00:00 2001 From: pld Date: Tue, 14 Jan 2014 21:46:36 -0500 Subject: [PATCH 02/31] PLD: add geom field for #110 --- .../0038_auto__add_field_instance_geom.py | 160 ++++++++++++++++++ onadata/apps/odk_logger/models/instance.py | 8 +- 2 files changed, 166 insertions(+), 2 deletions(-) create mode 100644 onadata/apps/odk_logger/migrations/0038_auto__add_field_instance_geom.py diff --git a/onadata/apps/odk_logger/migrations/0038_auto__add_field_instance_geom.py b/onadata/apps/odk_logger/migrations/0038_auto__add_field_instance_geom.py new file mode 100644 index 0000000000..b4bc14989f --- /dev/null +++ b/onadata/apps/odk_logger/migrations/0038_auto__add_field_instance_geom.py @@ -0,0 +1,160 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding field 'Instance.geom' + db.add_column(u'odk_logger_instance', 'geom', + self.gf('django.contrib.gis.db.models.fields.GeometryCollectionField')(null=True), + keep_default=False) + + + def backwards(self, orm): + # Deleting field 'Instance.geom' + db.delete_column(u'odk_logger_instance', 'geom') + + + models = { + u'auth.group': { + 'Meta': {'object_name': 'Group'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + u'auth.permission': { + 'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + u'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + u'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'odk_logger.attachment': { + 'Meta': {'object_name': 'Attachment'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'attachments'", 'to': "orm['odk_logger.Instance']"}), + 'media_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}), + 'mimetype': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}) + }, + 'odk_logger.instance': { + 'Meta': {'object_name': 'Instance'}, + 'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'date_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'deleted_at': ('django.db.models.fields.DateTimeField', [], {'default': 'None', 'null': 'True'}), + 'geom': ('django.contrib.gis.db.models.fields.GeometryCollectionField', [], {'null': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_deleted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'json': ('jsonfield.fields.JSONField', [], {'default': '{}'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "u'submitted_via_web'", 'max_length': '20'}), + 'survey_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['odk_logger.SurveyType']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'surveys'", 'null': 'True', 'to': u"orm['auth.User']"}), + 'uuid': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '249'}), + 'xform': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'surveys'", 'null': 'True', 'to': "orm['odk_logger.XForm']"}), + 'xml': ('django.db.models.fields.TextField', [], {}) + }, + 'odk_logger.instancehistory': { + 'Meta': {'object_name': 'InstanceHistory'}, + 'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'date_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'uuid': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '249'}), + 'xform_instance': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'submission_history'", 'to': "orm['odk_logger.Instance']"}), + 'xml': ('django.db.models.fields.TextField', [], {}) + }, + 'odk_logger.note': { + 'Meta': {'object_name': 'Note'}, + 'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'date_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'notes'", 'to': "orm['odk_logger.Instance']"}), + 'note': ('django.db.models.fields.TextField', [], {}) + }, + 'odk_logger.surveytype': { + 'Meta': {'object_name': 'SurveyType'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'slug': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'}) + }, + 'odk_logger.xform': { + 'Meta': {'ordering': "('id_string',)", 'unique_together': "(('user', 'id_string'), ('user', 'sms_id_string'))", 'object_name': 'XForm'}, + 'allows_sms': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'bamboo_dataset': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '60'}), + 'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'date_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'description': ('django.db.models.fields.TextField', [], {'default': "u''", 'null': 'True'}), + 'downloadable': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'encrypted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'has_start_time': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'id_string': ('django.db.models.fields.SlugField', [], {'max_length': '100'}), + 'is_crowd_form': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'json': ('django.db.models.fields.TextField', [], {'default': "u''"}), + 'last_submission_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'num_of_submissions': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'shared': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'shared_data': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'sms_id_string': ('django.db.models.fields.SlugField', [], {'default': "''", 'max_length': '100'}), + 'surveys_with_geopoints': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'xforms'", 'null': 'True', 'to': u"orm['auth.User']"}), + 'uuid': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '32'}), + 'xls': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True'}), + 'xml': ('django.db.models.fields.TextField', [], {}) + }, + 'odk_logger.ziggyinstance': { + 'Meta': {'object_name': 'ZiggyInstance'}, + 'client_version': ('django.db.models.fields.BigIntegerField', [], {'default': 'None', 'null': 'True'}), + 'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'date_deleted': ('django.db.models.fields.DateTimeField', [], {'default': 'None', 'null': 'True'}), + 'date_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'entity_id': ('django.db.models.fields.CharField', [], {'max_length': '249'}), + 'form_instance': ('django.db.models.fields.TextField', [], {}), + 'form_version': ('django.db.models.fields.CharField', [], {'default': "u'1.0'", 'max_length': '10'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '249'}), + 'reporter': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'ziggys'", 'to': u"orm['auth.User']"}), + 'server_version': ('django.db.models.fields.BigIntegerField', [], {}), + 'xform': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'ziggy_submissions'", 'null': 'True', 'to': "orm['odk_logger.XForm']"}) + }, + u'taggit.tag': { + 'Meta': {'object_name': 'Tag'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'}), + 'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '100'}) + }, + u'taggit.taggeditem': { + 'Meta': {'object_name': 'TaggedItem'}, + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'taggit_taggeditem_tagged_items'", 'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}), + 'tag': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'taggit_taggeditem_items'", 'to': u"orm['taggit.Tag']"}) + } + } + + complete_apps = ['odk_logger'] \ No newline at end of file diff --git a/onadata/apps/odk_logger/models/instance.py b/onadata/apps/odk_logger/models/instance.py index a2120ff6d6..a2ac87cbc8 100644 --- a/onadata/apps/odk_logger/models/instance.py +++ b/onadata/apps/odk_logger/models/instance.py @@ -1,4 +1,4 @@ -from django.db import models +from django.contrib.gis.db import models from django.db.models.signals import post_save from django.db.models.signals import post_delete from django.contrib.auth.models import User @@ -55,11 +55,15 @@ class Instance(models.Model): # ODK keeps track of three statuses for an instance: # incomplete, submitted, complete - # we will add a fourth status: submitted_via_web + # we add a fourth status: submitted_via_web status = models.CharField(max_length=20, default=u'submitted_via_web') uuid = models.CharField(max_length=249, default=u'') + # store an geographic objects associated with this instance + geom = models.GeometryCollectionField(null=True) + objects = models.GeoManager() + tags = TaggableManager() class Meta: From 6599ab99bc044d87876c09d842933f4a45d1a922 Mon Sep 17 00:00:00 2001 From: pld Date: Tue, 14 Jan 2014 21:53:20 -0500 Subject: [PATCH 03/31] PLD: add geospatial libs, move to install file, #110 --- script/install_ubuntu | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/script/install_ubuntu b/script/install_ubuntu index f3e378c05c..512f172389 100755 --- a/script/install_ubuntu +++ b/script/install_ubuntu @@ -1,8 +1,19 @@ # add node repo sudo add-apt-repository -y ppa:chris-lea/node.js +<<<<<<< HEAD sudo apt-get update -qqy sudo apt-get install -qq gfortran libatlas-base-dev libjpeg-dev python-numpy zlib1g-dev python-software-properties sudo apt-get install -qqy nodejs sudo ln -s /usr/lib/`uname -i`-linux-gnu/libjpeg.so ~/virtualenv/python2.7/lib/ sudo ln -s /usr/lib/`uname -i`-linux-gnu/libz.so ~/virtualenv/python2.7/lib/ sudo npm install -g --silent bower karma grunt-cli +======= +sudo apt-get update -y +sudo apt-get install -qq gfortran libatlas-base-dev libjpeg-dev python-numpy zlib1g-dev python-software-properties +sudo apt-get install -qqy nodejs +# install Geospatial libraries for GeoDjango +sudo apt-get install -qq binutils libproj-dev gdal-bin +sudo ln -s /usr/lib/`uname -i`-linux-gnu/libjpeg.so ~/virtualenv/python2.7/lib/ +sudo ln -s /usr/lib/`uname -i`-linux-gnu/libz.so ~/virtualenv/python2.7/lib/ +sudo npm install -g bower karma grunt-cli +>>>>>>> PLD: add geospatial libs, move to install file, #111 From 7c01efc6258f8f218728122afa98b7f20d8a0a95 Mon Sep 17 00:00:00 2001 From: pld Date: Wed, 15 Jan 2014 00:44:58 -0500 Subject: [PATCH 04/31] PLD: quieter install --- script/install_ubuntu | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/script/install_ubuntu b/script/install_ubuntu index 512f172389..61cd5db700 100755 --- a/script/install_ubuntu +++ b/script/install_ubuntu @@ -1,19 +1,10 @@ # add node repo sudo add-apt-repository -y ppa:chris-lea/node.js -<<<<<<< HEAD sudo apt-get update -qqy sudo apt-get install -qq gfortran libatlas-base-dev libjpeg-dev python-numpy zlib1g-dev python-software-properties sudo apt-get install -qqy nodejs -sudo ln -s /usr/lib/`uname -i`-linux-gnu/libjpeg.so ~/virtualenv/python2.7/lib/ -sudo ln -s /usr/lib/`uname -i`-linux-gnu/libz.so ~/virtualenv/python2.7/lib/ -sudo npm install -g --silent bower karma grunt-cli -======= -sudo apt-get update -y -sudo apt-get install -qq gfortran libatlas-base-dev libjpeg-dev python-numpy zlib1g-dev python-software-properties -sudo apt-get install -qqy nodejs # install Geospatial libraries for GeoDjango sudo apt-get install -qq binutils libproj-dev gdal-bin sudo ln -s /usr/lib/`uname -i`-linux-gnu/libjpeg.so ~/virtualenv/python2.7/lib/ sudo ln -s /usr/lib/`uname -i`-linux-gnu/libz.so ~/virtualenv/python2.7/lib/ -sudo npm install -g bower karma grunt-cli ->>>>>>> PLD: add geospatial libs, move to install file, #111 +sudo npm install -g --silent bower karma grunt-cli From fdbc844884936ff3e8366dc75eeb9ef012f42e4f Mon Sep 17 00:00:00 2001 From: pld Date: Wed, 15 Jan 2014 01:18:37 -0500 Subject: [PATCH 05/31] PLD: install db and create extension for postgis #110 --- .travis.yml | 2 +- script/install_database | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100755 script/install_database diff --git a/.travis.yml b/.travis.yml index dff47e6a50..913fd6dd19 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ services: python: - '2.7' before_install: - - psql -c 'create database onadata_test;' -U postgres + - ./script/install_database onadata_test - ./script/install_ubuntu env: - TESTFOLDER=onadata/apps/main diff --git a/script/install_database b/script/install_database new file mode 100755 index 0000000000..3d21159f38 --- /dev/null +++ b/script/install_database @@ -0,0 +1,3 @@ +#!/bin/bash +psql -c "CREATE DATABASE $1;" -U postgres +psql -c "CREATE EXTENSION IF NOT EXISTS postgis;" -U postgres From dae2ebb96bc5bf9e9005ce6d7da54ca625fe86a1 Mon Sep 17 00:00:00 2001 From: pld Date: Wed, 15 Jan 2014 09:18:43 -0500 Subject: [PATCH 06/31] PLD: add postgis install to ubuntu build per instructions for 9.3 --- script/install_database | 3 ++- script/install_ubuntu | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/script/install_database b/script/install_database index 3d21159f38..299906c288 100755 --- a/script/install_database +++ b/script/install_database @@ -1,3 +1,4 @@ #!/bin/bash psql -c "CREATE DATABASE $1;" -U postgres -psql -c "CREATE EXTENSION IF NOT EXISTS postgis;" -U postgres +psql -c "CREATE EXTENSION IF NOT EXISTS postgis;" -d $1 -U postgres +psql -c "CREATE EXTENSION IF NOT EXISTS postgis_topology;" -d $1 -U postgres diff --git a/script/install_ubuntu b/script/install_ubuntu index 61cd5db700..e132b278a8 100755 --- a/script/install_ubuntu +++ b/script/install_ubuntu @@ -1,10 +1,14 @@ # add node repo sudo add-apt-repository -y ppa:chris-lea/node.js +# add postgres repo, instructions: +# http://trac.osgeo.org/postgis/wiki/UsersWikiPostGIS21UbuntuPGSQL93Apt +sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main" >> /etc/apt/sources.list' +wget --quiet -O - http://apt.postgresql.org/pub/repos/apt/ACCC4CF8.asc | sudo apt-key add - sudo apt-get update -qqy sudo apt-get install -qq gfortran libatlas-base-dev libjpeg-dev python-numpy zlib1g-dev python-software-properties sudo apt-get install -qqy nodejs # install Geospatial libraries for GeoDjango -sudo apt-get install -qq binutils libproj-dev gdal-bin +sudo apt-get install -qq binutils libproj-dev gdal-bin Postgresql-9.3-postgis sudo ln -s /usr/lib/`uname -i`-linux-gnu/libjpeg.so ~/virtualenv/python2.7/lib/ sudo ln -s /usr/lib/`uname -i`-linux-gnu/libz.so ~/virtualenv/python2.7/lib/ -sudo npm install -g --silent bower karma grunt-cli +sudo npm install --silent -g bower karma grunt-cli From 201e1505fc7a13dc25065803b437382c23e2e57f Mon Sep 17 00:00:00 2001 From: pld Date: Wed, 15 Jan 2014 11:28:24 -0500 Subject: [PATCH 07/31] PLD: add set geom helper on instance save #110 --- onadata/apps/odk_logger/models/instance.py | 39 ++++++++++++++++------ 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/onadata/apps/odk_logger/models/instance.py b/onadata/apps/odk_logger/models/instance.py index a2ac87cbc8..cc52c7351e 100644 --- a/onadata/apps/odk_logger/models/instance.py +++ b/onadata/apps/odk_logger/models/instance.py @@ -2,6 +2,7 @@ from django.db.models.signals import post_save from django.db.models.signals import post_delete from django.contrib.auth.models import User +from django.contrib.gis.geos import GeometryCollection, Point from django.utils import timezone from django.utils.translation import ugettext as _ from jsonfield import JSONField @@ -69,6 +70,32 @@ class Instance(models.Model): class Meta: app_label = 'odk_logger' + def _set_geom(self): + data_dictionary = self.xform.data_dictionary() + geo_xpaths = data_dictionary.geopoint_xpaths() + doc = self.get_dict() + points = [] + + if len(geo_xpaths): + for xpath in geo_xpaths: + # TODO store the precision somewhere + lat, lng, alt, precision = [float(s) for s in + doc.get(xpath, u'').split()] + points.append(Point(lat, lng, alt)) + + self.geom = GeometryCollection(points) + + def _set_json(self): + doc = self.get_dict() + + if not self.date_created: + now = submission_time() + self.date_created = now + + doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME) + doc[XFORM_ID_STRING] = self._parser.get_xform_id_string() + self.json = doc + def _set_xform(self, id_string): self.xform = XForm.objects.get(id_string=id_string, user=self.user) @@ -84,17 +111,6 @@ def get(self, abbreviated_xpath): self._set_parser() return self._parser.get(abbreviated_xpath) - def _set_json(self): - doc = self.get_dict() - - if not self.date_created: - now = submission_time() - self.date_created = now - - doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME) - doc[XFORM_ID_STRING] = self._parser.get_xform_id_string() - self.json = doc - def _set_survey_type(self): self.survey_type, created = \ SurveyType.objects.get_or_create(slug=self.get_root_node_name()) @@ -113,6 +129,7 @@ def save(self, *args, **kwargs): raise FormInactiveError() self._set_json() + self._set_geom() self._set_survey_type() self._set_uuid() super(Instance, self).save(*args, **kwargs) From 47a0457e6199f241c701fffaca6dc27b90e5f91a Mon Sep 17 00:00:00 2001 From: pld Date: Wed, 15 Jan 2014 11:28:39 -0500 Subject: [PATCH 08/31] PLD: method for get all geopoints #110 --- onadata/apps/odk_viewer/models/data_dictionary.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/onadata/apps/odk_viewer/models/data_dictionary.py b/onadata/apps/odk_viewer/models/data_dictionary.py index 56a6e6a1c3..f3ec1b4201 100644 --- a/onadata/apps/odk_viewer/models/data_dictionary.py +++ b/onadata/apps/odk_viewer/models/data_dictionary.py @@ -186,10 +186,19 @@ def get_mongo_field_names_dict(self): survey_elements = property(get_survey_elements) - def xpath_of_first_geopoint(self): + def geopoint_xpaths(self): + geo_xpaths = [] + for e in self.get_survey_elements(): if e.bind.get(u'type') == u'geopoint': - return e.get_abbreviated_xpath() + geo_xpaths.append(e.get_abbreviated_xpath()) + + return geo_xpaths + + def xpath_of_first_geopoint(self): + geo_xpaths = self.geopoint_xpaths() + + return len(geo_xpaths) > 0 and geo_xpaths[0] def has_instances_with_geopoints(self): return ParsedInstance.objects.filter( From 625e45bb31971dd7ac55c827040babb6ed8eb3b8 Mon Sep 17 00:00:00 2001 From: pld Date: Wed, 15 Jan 2014 11:29:07 -0500 Subject: [PATCH 09/31] PLD: migration to resave and add geoms #110 --- .../0039_convert_lng_lat_to_points.py | 158 ++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 onadata/apps/odk_logger/migrations/0039_convert_lng_lat_to_points.py diff --git a/onadata/apps/odk_logger/migrations/0039_convert_lng_lat_to_points.py b/onadata/apps/odk_logger/migrations/0039_convert_lng_lat_to_points.py new file mode 100644 index 0000000000..4f5a1ffc00 --- /dev/null +++ b/onadata/apps/odk_logger/migrations/0039_convert_lng_lat_to_points.py @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- +from south.v2 import DataMigration + +from onadata.apps.odk_logger.models.instance import Instance +from onadata.libs.utils.model_tools import queryset_iterator + + +class Migration(DataMigration): + + def forwards(self, orm): + "Parse all instance to add geoms." + for obj in queryset_iterator(orm.Instance.objects.all()): + instance = Instance.objects.get(pd=obj.pk) + instance.save() + + def backwards(self, orm): + "Write your backwards methods here." + + models = { + u'auth.group': { + 'Meta': {'object_name': 'Group'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + u'auth.permission': { + 'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + u'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + u'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'odk_logger.attachment': { + 'Meta': {'object_name': 'Attachment'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'attachments'", 'to': "orm['odk_logger.Instance']"}), + 'media_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}), + 'mimetype': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}) + }, + 'odk_logger.instance': { + 'Meta': {'object_name': 'Instance'}, + 'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'date_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'deleted_at': ('django.db.models.fields.DateTimeField', [], {'default': 'None', 'null': 'True'}), + 'geom': ('django.contrib.gis.db.models.fields.GeometryCollectionField', [], {'null': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_deleted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'json': ('jsonfield.fields.JSONField', [], {'default': '{}'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "u'submitted_via_web'", 'max_length': '20'}), + 'survey_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['odk_logger.SurveyType']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'surveys'", 'null': 'True', 'to': u"orm['auth.User']"}), + 'uuid': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '249'}), + 'xform': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'surveys'", 'null': 'True', 'to': "orm['odk_logger.XForm']"}), + 'xml': ('django.db.models.fields.TextField', [], {}) + }, + 'odk_logger.instancehistory': { + 'Meta': {'object_name': 'InstanceHistory'}, + 'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'date_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'uuid': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '249'}), + 'xform_instance': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'submission_history'", 'to': "orm['odk_logger.Instance']"}), + 'xml': ('django.db.models.fields.TextField', [], {}) + }, + 'odk_logger.note': { + 'Meta': {'object_name': 'Note'}, + 'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'date_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'notes'", 'to': "orm['odk_logger.Instance']"}), + 'note': ('django.db.models.fields.TextField', [], {}) + }, + 'odk_logger.surveytype': { + 'Meta': {'object_name': 'SurveyType'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'slug': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'}) + }, + 'odk_logger.xform': { + 'Meta': {'ordering': "('id_string',)", 'unique_together': "(('user', 'id_string'), ('user', 'sms_id_string'))", 'object_name': 'XForm'}, + 'allows_sms': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'bamboo_dataset': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '60'}), + 'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'date_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'description': ('django.db.models.fields.TextField', [], {'default': "u''", 'null': 'True'}), + 'downloadable': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'encrypted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'has_start_time': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'id_string': ('django.db.models.fields.SlugField', [], {'max_length': '100'}), + 'is_crowd_form': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'json': ('django.db.models.fields.TextField', [], {'default': "u''"}), + 'last_submission_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'num_of_submissions': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'shared': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'shared_data': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'sms_id_string': ('django.db.models.fields.SlugField', [], {'default': "''", 'max_length': '100'}), + 'surveys_with_geopoints': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'xforms'", 'null': 'True', 'to': u"orm['auth.User']"}), + 'uuid': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '32'}), + 'xls': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True'}), + 'xml': ('django.db.models.fields.TextField', [], {}) + }, + 'odk_logger.ziggyinstance': { + 'Meta': {'object_name': 'ZiggyInstance'}, + 'client_version': ('django.db.models.fields.BigIntegerField', [], {'default': 'None', 'null': 'True'}), + 'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'date_deleted': ('django.db.models.fields.DateTimeField', [], {'default': 'None', 'null': 'True'}), + 'date_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'entity_id': ('django.db.models.fields.CharField', [], {'max_length': '249'}), + 'form_instance': ('django.db.models.fields.TextField', [], {}), + 'form_version': ('django.db.models.fields.CharField', [], {'default': "u'1.0'", 'max_length': '10'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '249'}), + 'reporter': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'ziggys'", 'to': u"orm['auth.User']"}), + 'server_version': ('django.db.models.fields.BigIntegerField', [], {}), + 'xform': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'ziggy_submissions'", 'null': 'True', 'to': "orm['odk_logger.XForm']"}) + }, + u'taggit.tag': { + 'Meta': {'object_name': 'Tag'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'}), + 'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '100'}) + }, + u'taggit.taggeditem': { + 'Meta': {'object_name': 'TaggedItem'}, + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'taggit_taggeditem_tagged_items'", 'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}), + 'tag': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'taggit_taggeditem_items'", 'to': u"orm['taggit.Tag']"}) + } + } + + complete_apps = ['odk_logger'] + symmetrical = True From 3994593a9ce5480c6f28aeb560e3cf384f9e796e Mon Sep 17 00:00:00 2001 From: pld Date: Fri, 17 Jan 2014 19:28:14 -0500 Subject: [PATCH 10/31] PLD: do not store the z dimension, not supported out-of-the-box --- onadata/apps/odk_logger/models/instance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onadata/apps/odk_logger/models/instance.py b/onadata/apps/odk_logger/models/instance.py index cc52c7351e..516987c0e0 100644 --- a/onadata/apps/odk_logger/models/instance.py +++ b/onadata/apps/odk_logger/models/instance.py @@ -78,10 +78,10 @@ def _set_geom(self): if len(geo_xpaths): for xpath in geo_xpaths: - # TODO store the precision somewhere + # TODO store the altitude and precision lat, lng, alt, precision = [float(s) for s in doc.get(xpath, u'').split()] - points.append(Point(lat, lng, alt)) + points.append(Point(lat, lng)) self.geom = GeometryCollection(points) From f698176519abd76b6f3e39ceda246448d8c222ec Mon Sep 17 00:00:00 2001 From: pld Date: Sat, 18 Jan 2014 17:42:01 -0500 Subject: [PATCH 11/31] PLD organize and add geolocation lat, lng from geom to json --- onadata/apps/odk_logger/models/instance.py | 169 +++++++++++---------- requirements/common.pip | 1 - 2 files changed, 90 insertions(+), 80 deletions(-) diff --git a/onadata/apps/odk_logger/models/instance.py b/onadata/apps/odk_logger/models/instance.py index 516987c0e0..57c73d2d8c 100644 --- a/onadata/apps/odk_logger/models/instance.py +++ b/onadata/apps/odk_logger/models/instance.py @@ -12,8 +12,8 @@ from onadata.apps.odk_logger.models.xform import XForm from onadata.apps.odk_logger.xform_instance_parser import XFormInstanceParser,\ clean_and_parse_xml, get_uuid_from_xml -from onadata.libs.utils.common_tags import MONGO_STRFTIME, SUBMISSION_TIME,\ - XFORM_ID_STRING +from onadata.libs.utils.common_tags import GEOLOCATION, MONGO_STRFTIME,\ + SUBMISSION_TIME, XFORM_ID_STRING from onadata.libs.utils.model_tools import set_uuid @@ -38,6 +38,49 @@ def submission_time(): return timezone.now() +def update_xform_submission_count(sender, instance, created, **kwargs): + if created: + xform = XForm.objects.select_related().select_for_update()\ + .get(pk=instance.xform.pk) + if xform.num_of_submissions == -1: + xform.num_of_submissions = 0 + xform.num_of_submissions += 1 + xform.last_submission_time = instance.date_created + xform.save() + profile_qs = User.profile.get_query_set() + try: + profile = profile_qs.select_for_update()\ + .get(pk=xform.user.profile.pk) + except profile_qs.model.DoesNotExist: + pass + else: + profile.num_of_submissions += 1 + profile.save() + + +def update_xform_submission_count_delete(sender, instance, **kwargs): + try: + xform = XForm.objects.select_for_update().get(pk=instance.xform.pk) + except XForm.DoesNotExist: + pass + else: + xform.num_of_submissions -= 1 + if xform.num_of_submissions < 0: + xform.num_of_submissions = 0 + xform.save() + profile_qs = User.profile.get_query_set() + try: + profile = profile_qs.select_for_update()\ + .get(pk=xform.user.profile.pk) + except profile_qs.model.DoesNotExist: + pass + else: + profile.num_of_submissions -= 1 + if profile.num_of_submissions < 0: + profile.num_of_submissions = 0 + profile.save() + + class Instance(models.Model): json = JSONField(default={}, null=False) xml = models.TextField() @@ -70,6 +113,15 @@ class Instance(models.Model): class Meta: app_label = 'odk_logger' + @classmethod + def set_deleted_at(cls, instance_id, deleted_at=timezone.now()): + try: + instance = cls.objects.get(id=instance_id) + except cls.DoesNotExist: + pass + else: + instance.set_deleted(deleted_at) + def _set_geom(self): data_dictionary = self.xform.data_dictionary() geo_xpaths = data_dictionary.geopoint_xpaths() @@ -92,35 +144,57 @@ def _set_json(self): now = submission_time() self.date_created = now + point = self.point() + if point: + doc[GEOLOCATION] = [point.y, point.x] + doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME) doc[XFORM_ID_STRING] = self._parser.get_xform_id_string() self.json = doc + def _set_parser(self): + if not hasattr(self, "_parser"): + self._parser = XFormInstanceParser( + self.xml, self.xform.data_dictionary()) + + def _set_survey_type(self): + self.survey_type, created = \ + SurveyType.objects.get_or_create(slug=self.get_root_node_name()) + + def _set_uuid(self): + if self.xml and not self.uuid: + uuid = get_uuid_from_xml(self.xml) + if uuid is not None: + self.uuid = uuid + set_uuid(self) + def _set_xform(self, id_string): self.xform = XForm.objects.get(id_string=id_string, user=self.user) - def get_root_node_name(self): + def get(self, abbreviated_xpath): self._set_parser() - return self._parser.get_root_node_name() + return self._parser.get(abbreviated_xpath) + + def get_dict(self, force_new=False, flat=True): + """Return a python object representation of this instance's XML.""" + self._set_parser() + + return self._parser.get_flat_dict_with_attributes() if flat else\ + self._parser.to_dict() def get_root_node(self): self._set_parser() return self._parser.get_root_node() - def get(self, abbreviated_xpath): + def get_root_node_name(self): self._set_parser() - return self._parser.get(abbreviated_xpath) + return self._parser.get_root_node_name() - def _set_survey_type(self): - self.survey_type, created = \ - SurveyType.objects.get_or_create(slug=self.get_root_node_name()) + def point(self): + gc = self.geom - def _set_uuid(self): - if self.xml and not self.uuid: - uuid = get_uuid_from_xml(self.xml) - if uuid is not None: - self.uuid = uuid - set_uuid(self) + if gc and len(gc): + return gc[0] def save(self, *args, **kwargs): self._set_xform(get_id_string_from_xml_str(self.xml)) @@ -128,84 +202,21 @@ def save(self, *args, **kwargs): if self.xform and not self.xform.downloadable: raise FormInactiveError() - self._set_json() self._set_geom() + self._set_json() self._set_survey_type() self._set_uuid() super(Instance, self).save(*args, **kwargs) - def _set_parser(self): - if not hasattr(self, "_parser"): - self._parser = XFormInstanceParser( - self.xml, self.xform.data_dictionary()) - - def get_dict(self, force_new=False, flat=True): - """Return a python object representation of this instance's XML.""" - self._set_parser() - - return self._parser.get_flat_dict_with_attributes() if flat else\ - self._parser.to_dict() - def set_deleted(self, deleted_at=timezone.now()): self.deleted_at = deleted_at self.save() self.parsed_instance.save() - @classmethod - def set_deleted_at(cls, instance_id, deleted_at=timezone.now()): - try: - instance = cls.objects.get(id=instance_id) - except cls.DoesNotExist: - pass - else: - instance.set_deleted(deleted_at) - - -def update_xform_submission_count(sender, instance, created, **kwargs): - if created: - xform = XForm.objects.select_related().select_for_update()\ - .get(pk=instance.xform.pk) - if xform.num_of_submissions == -1: - xform.num_of_submissions = 0 - xform.num_of_submissions += 1 - xform.last_submission_time = instance.date_created - xform.save() - profile_qs = User.profile.get_query_set() - try: - profile = profile_qs.select_for_update()\ - .get(pk=xform.user.profile.pk) - except profile_qs.model.DoesNotExist: - pass - else: - profile.num_of_submissions += 1 - profile.save() post_save.connect(update_xform_submission_count, sender=Instance, dispatch_uid='update_xform_submission_count') - -def update_xform_submission_count_delete(sender, instance, **kwargs): - try: - xform = XForm.objects.select_for_update().get(pk=instance.xform.pk) - except XForm.DoesNotExist: - pass - else: - xform.num_of_submissions -= 1 - if xform.num_of_submissions < 0: - xform.num_of_submissions = 0 - xform.save() - profile_qs = User.profile.get_query_set() - try: - profile = profile_qs.select_for_update()\ - .get(pk=xform.user.profile.pk) - except profile_qs.model.DoesNotExist: - pass - else: - profile.num_of_submissions -= 1 - if profile.num_of_submissions < 0: - profile.num_of_submissions = 0 - profile.save() - post_delete.connect(update_xform_submission_count_delete, sender=Instance, dispatch_uid='update_xform_submission_count_delete') diff --git a/requirements/common.pip b/requirements/common.pip index cb0dbd0d54..7797406d23 100644 --- a/requirements/common.pip +++ b/requirements/common.pip @@ -33,7 +33,6 @@ pandas==0.12.0 requests==2.0.1 elaphe==0.5.6 - # sms support dict2xml==1.1 From 0db9b94d5cedc9cfcb882a4345ed2095b34b57cb Mon Sep 17 00:00:00 2001 From: pld Date: Sat, 18 Jan 2014 17:42:10 -0500 Subject: [PATCH 12/31] PLD: start of arch install script --- script/install_arch | 2 ++ 1 file changed, 2 insertions(+) create mode 100755 script/install_arch diff --git a/script/install_arch b/script/install_arch new file mode 100755 index 0000000000..21a219a9d5 --- /dev/null +++ b/script/install_arch @@ -0,0 +1,2 @@ +# install PostGIS +sudo pacman -S postgis From c65f84aab3c74ec1fd8188eb71c608cffd5a3e85 Mon Sep 17 00:00:00 2001 From: pld Date: Sat, 18 Jan 2014 22:25:45 -0500 Subject: [PATCH 13/31] PLD: build KML directly from instance --- onadata/libs/utils/export_tools.py | 44 ++++++++++++++++-------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/onadata/libs/utils/export_tools.py b/onadata/libs/utils/export_tools.py index 6ca0581461..e21d6495d0 100644 --- a/onadata/libs/utils/export_tools.py +++ b/onadata/libs/utils/export_tools.py @@ -18,13 +18,15 @@ from pyxform.section import Section, RepeatingSection from savReaderWriter import SavWriter -from onadata.apps.odk_logger.models import XForm, Attachment -from onadata.apps.odk_viewer.models.parsed_instance import _is_invalid_for_mongo,\ - _encode_for_mongo, dict_for_mongo, _decode_from_mongo -from onadata.libs.utils.viewer_tools import create_attachments_zipfile, image_urls -from onadata.libs.utils.common_tags import ID, XFORM_ID_STRING, STATUS, ATTACHMENTS,\ - GEOLOCATION, BAMBOO_DATASET_ID, DELETEDAT, USERFORM_ID, INDEX,\ - PARENT_INDEX, PARENT_TABLE_NAME, SUBMISSION_TIME, UUID, TAGS, NOTES +from onadata.apps.odk_logger.models import Attachment, Instance, XForm +from onadata.apps.odk_viewer.models.parsed_instance import\ + _is_invalid_for_mongo, _encode_for_mongo, dict_for_mongo,\ + _decode_from_mongo +from onadata.libs.utils.viewer_tools import create_attachments_zipfile,\ + image_urls +from onadata.libs.utils.common_tags import ID, XFORM_ID_STRING, STATUS,\ + ATTACHMENTS, GEOLOCATION, BAMBOO_DATASET_ID, DELETEDAT, USERFORM_ID,\ + INDEX, PARENT_INDEX, PARENT_TABLE_NAME, SUBMISSION_TIME, UUID, TAGS, NOTES # this is Mongo Collection where we will store the parsed submissions @@ -184,7 +186,8 @@ def format_field_title(cls, abbreviated_xpath, field_delimiter): def set_survey(self, survey): # TODO resolve circular import - from onadata.apps.odk_viewer.models.data_dictionary import DataDictionary + from onadata.apps.odk_viewer.models.data_dictionary import\ + DataDictionary def build_sections( current_section, survey_element, sections, select_multiples, @@ -550,7 +553,8 @@ def write_row(data, work_sheet, fields, work_sheet_titles): def to_flat_csv_export( self, path, data, username, id_string, filter_query): # TODO resolve circular import - from onadata.apps.odk_viewer.pandas_mongo_bridge import CSVDataFrameBuilder + from onadata.apps.odk_viewer.pandas_mongo_bridge import\ + CSVDataFrameBuilder csv_builder = CSVDataFrameBuilder( username, id_string, filter_query, self.GROUP_DELIMITER, @@ -871,11 +875,10 @@ def generate_kml_export( def kml_export_data(id_string, user): # TODO resolve circular import from onadata.apps.odk_viewer.models.data_dictionary import DataDictionary - from onadata.apps.odk_viewer.models.parsed_instance import ParsedInstance dd = DataDictionary.objects.get(id_string=id_string, user=user) - pis = ParsedInstance.objects.filter( - instance__user=user, instance__xform__id_string=id_string, - lat__isnull=False, lng__isnull=False).order_by('id') + instances = Instance.objects.filter( + user=user, xform__id_string=id_string, geom__isnull=False + ).order_by('id') data_for_template = [] labels = {} @@ -886,11 +889,11 @@ def cached_get_labels(xpath): labels[xpath] = dd.get_label(xpath) return labels[xpath] - for pi in pis: + for instance in instances: # read the survey instances - data_for_display = pi.to_dict() + data_for_display = instance.get_dict() xpaths = data_for_display.keys() - xpaths.sort(cmp=pi.data_dictionary.get_xpath_cmp()) + xpaths.sort(cmp=instance.xform.data_dictionary().get_xpath_cmp()) label_value_pairs = [ (cached_get_labels(xpath), data_for_display[xpath]) for xpath in xpaths @@ -898,13 +901,14 @@ def cached_get_labels(xpath): table_rows = [] for key, value in label_value_pairs: table_rows.append('%s%s' % (key, value)) - img_urls = image_urls(pi.instance) + img_urls = image_urls(instance) img_url = img_urls[0] if img_urls else "" + point = instance.point data_for_template.append({ 'name': id_string, - 'id': pi.id, - 'lat': pi.lat, - 'lng': pi.lng, + 'id': instance.id, + 'lat': point.y, + 'lng': point.x, 'image_urls': img_urls, 'table': '%s' From b4e12886ee5950d825723ed3d7816e710789c2e4 Mon Sep 17 00:00:00 2001 From: pld Date: Sat, 18 Jan 2014 22:30:24 -0500 Subject: [PATCH 14/31] PLD: check geometries exists before attempt to extract data --- onadata/apps/odk_logger/models/instance.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/onadata/apps/odk_logger/models/instance.py b/onadata/apps/odk_logger/models/instance.py index 57c73d2d8c..00bf0bc0c6 100644 --- a/onadata/apps/odk_logger/models/instance.py +++ b/onadata/apps/odk_logger/models/instance.py @@ -131,9 +131,11 @@ def _set_geom(self): if len(geo_xpaths): for xpath in geo_xpaths: # TODO store the altitude and precision - lat, lng, alt, precision = [float(s) for s in - doc.get(xpath, u'').split()] - points.append(Point(lat, lng)) + geometry = [float(s) for s in doc.get(xpath, u'').split()] + + if len(geometry): + lat, lng, alt, precision = geometry + points.append(Point(lat, lng)) self.geom = GeometryCollection(points) From 6f0e45c5a3a97fb81e7e31af58f66b037eab441c Mon Sep 17 00:00:00 2001 From: pld Date: Sun, 19 Jan 2014 09:06:54 -0500 Subject: [PATCH 15/31] PLD: change name to match class --- .../tests/{test_import_tools.py => test_importing_database.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename onadata/apps/odk_logger/tests/{test_import_tools.py => test_importing_database.py} (100%) diff --git a/onadata/apps/odk_logger/tests/test_import_tools.py b/onadata/apps/odk_logger/tests/test_importing_database.py similarity index 100% rename from onadata/apps/odk_logger/tests/test_import_tools.py rename to onadata/apps/odk_logger/tests/test_importing_database.py From 8777c4e2ac27105841b83c515585c8d467343c29 Mon Sep 17 00:00:00 2001 From: pld Date: Sun, 19 Jan 2014 10:15:15 -0500 Subject: [PATCH 16/31] PLD: use instance in KML export --- onadata/apps/odk_viewer/tests/test_kml_export.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/onadata/apps/odk_viewer/tests/test_kml_export.py b/onadata/apps/odk_viewer/tests/test_kml_export.py index dc390a076b..7327d4eb9f 100644 --- a/onadata/apps/odk_viewer/tests/test_kml_export.py +++ b/onadata/apps/odk_viewer/tests/test_kml_export.py @@ -3,7 +3,7 @@ from django.core.urlresolvers import reverse from onadata.apps.main.tests.test_base import TestBase -from onadata.apps.odk_viewer.models.parsed_instance import ParsedInstance +from onadata.apps.odk_logger.models.instance import Instance from onadata.apps.odk_viewer.views import kml_export @@ -31,13 +31,13 @@ def test_kml_export(self): kml_export, kwargs={'username': self.user.username, 'id_string': id_string}) response = self.client.get(url) - pis = ParsedInstance.objects.filter( - instance__user=self.user, instance__xform__id_string=id_string, - lat__isnull=False, lng__isnull=False).order_by('id') + instances = Instance.objects.filter( + user=self.user, xform__id_string=id_string, geom__isnull=False + ).order_by('id') - self.assertEqual(pis.count(), 2) + self.assertEqual(instances.count(), 2) - first, second = [str(i.pk) for i in pis] + first, second = [str(i.pk) for i in instances] with open(os.path.join(self.fixtures, 'export.kml')) as f: expected_content = f.read() From 9a600d2a82914e1b8b29009bbc1815bc231f3dbc Mon Sep 17 00:00:00 2001 From: pld Date: Sun, 19 Jan 2014 10:38:02 -0500 Subject: [PATCH 17/31] PLD: add get_full_dict method to instance that returns all previous mongo dict entries --- onadata/apps/odk_logger/models/instance.py | 34 ++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/onadata/apps/odk_logger/models/instance.py b/onadata/apps/odk_logger/models/instance.py index 00bf0bc0c6..e5447a2df9 100644 --- a/onadata/apps/odk_logger/models/instance.py +++ b/onadata/apps/odk_logger/models/instance.py @@ -1,3 +1,5 @@ +from datetime import datetime + from django.contrib.gis.db import models from django.db.models.signals import post_save from django.db.models.signals import post_delete @@ -12,8 +14,9 @@ from onadata.apps.odk_logger.models.xform import XForm from onadata.apps.odk_logger.xform_instance_parser import XFormInstanceParser,\ clean_and_parse_xml, get_uuid_from_xml -from onadata.libs.utils.common_tags import GEOLOCATION, MONGO_STRFTIME,\ - SUBMISSION_TIME, XFORM_ID_STRING +from onadata.libs.utils.common_tags import ATTACHMENTS, BAMBOO_DATASET_ID,\ + DELETEDAT, GEOLOCATION, ID, MONGO_STRFTIME, NOTES, SUBMISSION_TIME, TAGS,\ + UUID, XFORM_ID_STRING from onadata.libs.utils.model_tools import set_uuid @@ -184,6 +187,33 @@ def get_dict(self, force_new=False, flat=True): return self._parser.get_flat_dict_with_attributes() if flat else\ self._parser.to_dict() + def get_full_dict(self): + # TODO should we store all of these in the JSON no matter what? + d = self.json + data = { + UUID: self.uuid, + ID: self.id, + BAMBOO_DATASET_ID: self.xform.bamboo_dataset, + self.USERFORM_ID: u'%s_%s' % ( + self.user.username, + self.xform.id_string), + ATTACHMENTS: [a.media_file.name for a in + self.attachments.all()], + self.STATUS: self.status, + TAGS: list(self.tags.names()), + NOTES: self.get_notes() + } + + if isinstance(self.instance.deleted_at, datetime): + data[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME) + + d.update(data) + + return d + + def get_notes(self): + return [note['note'] for note in self.notes.values('note')] + def get_root_node(self): self._set_parser() return self._parser.get_root_node() From 2e2453a71f35065edefb2ec25808f4c03808f472 Mon Sep 17 00:00:00 2001 From: pld Date: Sun, 19 Jan 2014 10:38:12 -0500 Subject: [PATCH 18/31] PLD: use instance over parsed instance when building csv for bamboo --- onadata/libs/utils/bamboo.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/onadata/libs/utils/bamboo.py b/onadata/libs/utils/bamboo.py index 7633d5163e..c80e9226d3 100644 --- a/onadata/libs/utils/bamboo.py +++ b/onadata/libs/utils/bamboo.py @@ -5,7 +5,7 @@ from pybamboo.connection import Connection from pybamboo.exceptions import ErrorParsingBambooData -from onadata.apps.odk_viewer.models.parsed_instance import ParsedInstance +from onadata.apps.odk_logger.models.instance import Instance from onadata.apps.odk_viewer.pandas_mongo_bridge import CSVDataFrameBuilder,\ NoRecordsFoundError from onadata.apps.restservice.models import RestService @@ -89,10 +89,10 @@ def get_csv_data_manual(xform, # when form has only one submission, CSVDFB is empty. # we still want to create the BB ds with row 1 # so we extract is and CSV it. - pifilter = ParsedInstance.objects.filter(instance__xform=xform) \ - .order_by('-instance__date_modified') + instances = Instance.objects.filter(xform=xform).order_by( + '-date_modified') - if pifilter.count() == 0: + if instances.count() == 0: raise NoRecordsFoundError else: # we should only do it for count == 1 but eh. @@ -100,9 +100,9 @@ def get_csv_data_manual(xform, csv_buf = getbuff() if only_last: - pifilter = [pifilter[0]] + instances = instances[0:1] - rows = [pi.to_dict_for_mongo() for pi in pifilter] + rows = [instance.get_full_dict() for instance in instances] if headers_to_use is None: headers_to_use = [key for key in rows[0].keys() From 24095159f37588cfabf216fbc9866ad58437aa06 Mon Sep 17 00:00:00 2001 From: pld Date: Sun, 19 Jan 2014 11:01:22 -0500 Subject: [PATCH 19/31] PLD: make point a property, fix lng is x --- onadata/apps/odk_logger/models/instance.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/onadata/apps/odk_logger/models/instance.py b/onadata/apps/odk_logger/models/instance.py index e5447a2df9..21c2aa9ca6 100644 --- a/onadata/apps/odk_logger/models/instance.py +++ b/onadata/apps/odk_logger/models/instance.py @@ -138,7 +138,7 @@ def _set_geom(self): if len(geometry): lat, lng, alt, precision = geometry - points.append(Point(lat, lng)) + points.append(Point(lng, lat)) self.geom = GeometryCollection(points) @@ -149,7 +149,7 @@ def _set_json(self): now = submission_time() self.date_created = now - point = self.point() + point = self.point if point: doc[GEOLOCATION] = [point.y, point.x] @@ -222,6 +222,7 @@ def get_root_node_name(self): self._set_parser() return self._parser.get_root_node_name() + @property def point(self): gc = self.geom From 1ef68d60892a13888b7094ab1efd0a1e1d088dcd Mon Sep 17 00:00:00 2001 From: pld Date: Sun, 19 Jan 2014 16:36:13 -0500 Subject: [PATCH 20/31] PLD: use instance to check for geopoints --- onadata/apps/odk_viewer/models/data_dictionary.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/onadata/apps/odk_viewer/models/data_dictionary.py b/onadata/apps/odk_viewer/models/data_dictionary.py index f3ec1b4201..60e69d385e 100644 --- a/onadata/apps/odk_viewer/models/data_dictionary.py +++ b/onadata/apps/odk_viewer/models/data_dictionary.py @@ -9,10 +9,10 @@ from pyxform.xform2json import create_survey_element_from_xml from xml.dom import Node -from onadata.apps.odk_logger.models import XForm +from onadata.apps.odk_logger.models.instance import Instance +from onadata.apps.odk_logger.models.xform import XForm from onadata.apps.odk_logger.xform_instance_parser import clean_and_parse_xml -from onadata.apps.odk_viewer.models.parsed_instance import _encode_for_mongo,\ - ParsedInstance +from onadata.apps.odk_viewer.models.parsed_instance import _encode_for_mongo from onadata.libs.utils.common_tags import UUID, SUBMISSION_TIME, TAGS, NOTES from onadata.libs.utils.export_tools import question_types_to_exclude,\ DictOrganizer @@ -201,8 +201,8 @@ def xpath_of_first_geopoint(self): return len(geo_xpaths) > 0 and geo_xpaths[0] def has_instances_with_geopoints(self): - return ParsedInstance.objects.filter( - instance__xform=self, lat__isnull=False).count() > 0 + return Instance.objects.filter( + xform=self, geom__isnull=False).count() > 0 def xpaths(self, prefix='', survey_element=None, result=None, repeat_iterations=4): From d994518933c793f732d51caf5cf74e513f8caa48 Mon Sep 17 00:00:00 2001 From: pld Date: Sun, 19 Jan 2014 16:42:16 -0500 Subject: [PATCH 21/31] PLD: geocoded submission count from instances --- onadata/apps/odk_logger/models/xform.py | 12 +++++------- onadata/apps/odk_viewer/models/data_dictionary.py | 4 +--- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/onadata/apps/odk_logger/models/xform.py b/onadata/apps/odk_logger/models/xform.py index d44bb529aa..9373c54575 100644 --- a/onadata/apps/odk_logger/models/xform.py +++ b/onadata/apps/odk_logger/models/xform.py @@ -13,6 +13,7 @@ from guardian.shortcuts import assign_perm, get_perms_for_model from taggit.managers import TaggableManager +from onadata.apps.odk_logger.models.instance import Instance from onadata.apps.odk_logger.xform_instance_parser import XLSFormError from onadata.apps.stats.tasks import stat_log @@ -172,13 +173,10 @@ def submission_count(self): submission_count.short_description = ugettext_lazy("Submission Count") def geocoded_submission_count(self): - from onadata.apps.odk_viewer.models.parsed_instance import\ - ParsedInstance - return ParsedInstance.objects.filter( - instance__in=self.instances.filter(deleted_at__isnull=True), - lat__isnull=False).count() - geocoded_submission_count.short_description = \ - ugettext_lazy("Geocoded Submission Count") + """Number of geocoded submissions.""" + return Instance.objects.filter( + xform=self, deleted_at__isnull=True, + geom__isnull=False).count() def time_of_last_submission(self): if self.last_submission_time is None and self.num_of_submissions > 0: diff --git a/onadata/apps/odk_viewer/models/data_dictionary.py b/onadata/apps/odk_viewer/models/data_dictionary.py index 60e69d385e..5889cf0b4e 100644 --- a/onadata/apps/odk_viewer/models/data_dictionary.py +++ b/onadata/apps/odk_viewer/models/data_dictionary.py @@ -9,7 +9,6 @@ from pyxform.xform2json import create_survey_element_from_xml from xml.dom import Node -from onadata.apps.odk_logger.models.instance import Instance from onadata.apps.odk_logger.models.xform import XForm from onadata.apps.odk_logger.xform_instance_parser import clean_and_parse_xml from onadata.apps.odk_viewer.models.parsed_instance import _encode_for_mongo @@ -201,8 +200,7 @@ def xpath_of_first_geopoint(self): return len(geo_xpaths) > 0 and geo_xpaths[0] def has_instances_with_geopoints(self): - return Instance.objects.filter( - xform=self, geom__isnull=False).count() > 0 + return self.geocoded_submission_count() > 0 def xpaths(self, prefix='', survey_element=None, result=None, repeat_iterations=4): From 8bc4e1051dbae6875377036899fa223b103ef39e Mon Sep 17 00:00:00 2001 From: pld Date: Sun, 19 Jan 2014 16:49:11 -0500 Subject: [PATCH 22/31] PLD: use xform geocoded submission count --- .../management/commands/set_xform_surveys_with_geopoints.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/onadata/apps/odk_logger/management/commands/set_xform_surveys_with_geopoints.py b/onadata/apps/odk_logger/management/commands/set_xform_surveys_with_geopoints.py index 2b356868df..24a6fa2ac8 100644 --- a/onadata/apps/odk_logger/management/commands/set_xform_surveys_with_geopoints.py +++ b/onadata/apps/odk_logger/management/commands/set_xform_surveys_with_geopoints.py @@ -4,8 +4,7 @@ from django.core.management.base import BaseCommand from django.utils.translation import ugettext_lazy -from onadata.apps.odk_logger.models import XForm -from onadata.apps.odk_viewer.models.parsed_instance import ParsedInstance +from onadata.apps.odk_logger.models.xform import XForm from onadata.libs.utils.model_tools import queryset_iterator @@ -17,8 +16,7 @@ def handle(self, *args, **kwargs): total = xforms.count() count = 0 for xform in queryset_iterator(XForm.objects.all()): - has_geo = ParsedInstance.objects.filter( - instance__xform=xform, lat__isnull=False).count() > 0 + has_geo = xform.geocoded_submission_count() > 0 try: xform.instances_with_geopoints = has_geo xform.save() From 94ce26bd5c98b2bf49cd79303abbc30f7355cb38 Mon Sep 17 00:00:00 2001 From: pld Date: Sun, 19 Jan 2014 17:56:39 -0500 Subject: [PATCH 23/31] PLD: use instances from relation --- onadata/apps/odk_logger/models/xform.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/onadata/apps/odk_logger/models/xform.py b/onadata/apps/odk_logger/models/xform.py index 9373c54575..11a8423ba9 100644 --- a/onadata/apps/odk_logger/models/xform.py +++ b/onadata/apps/odk_logger/models/xform.py @@ -13,7 +13,6 @@ from guardian.shortcuts import assign_perm, get_perms_for_model from taggit.managers import TaggableManager -from onadata.apps.odk_logger.models.instance import Instance from onadata.apps.odk_logger.xform_instance_parser import XLSFormError from onadata.apps.stats.tasks import stat_log @@ -174,9 +173,8 @@ def submission_count(self): def geocoded_submission_count(self): """Number of geocoded submissions.""" - return Instance.objects.filter( - xform=self, deleted_at__isnull=True, - geom__isnull=False).count() + return self.surveys.filter(deleted_at__isnull=True, + geom__isnull=False).count() def time_of_last_submission(self): if self.last_submission_time is None and self.num_of_submissions > 0: From dc4e97d73dd3bbe728a6eb3ea1b16e138e2e9181 Mon Sep 17 00:00:00 2001 From: pld Date: Sun, 19 Jan 2014 18:32:19 -0500 Subject: [PATCH 24/31] PLD: correct model imports --- onadata/apps/odk_viewer/pandas_mongo_bridge.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/onadata/apps/odk_viewer/pandas_mongo_bridge.py b/onadata/apps/odk_viewer/pandas_mongo_bridge.py index 202c6346d3..e962647011 100644 --- a/onadata/apps/odk_viewer/pandas_mongo_bridge.py +++ b/onadata/apps/odk_viewer/pandas_mongo_bridge.py @@ -9,10 +9,10 @@ from pyxform.section import Section, RepeatingSection from pyxform.question import Question -from onadata.apps.odk_viewer.models.data_dictionary import ParsedInstance,\ - DataDictionary -from onadata.libs.utils.common_tags import ID, XFORM_ID_STRING, STATUS, \ - ATTACHMENTS, GEOLOCATION, UUID, SUBMISSION_TIME, NA_REP, \ +from onadata.apps.odk_viewer.models.data_dictionary import DataDictionary +from onadata.apps.odk_viewer.models.parsed_instance import ParsedInstance +from onadata.libs.utils.common_tags import ID, XFORM_ID_STRING, STATUS,\ + ATTACHMENTS, GEOLOCATION, UUID, SUBMISSION_TIME, NA_REP,\ BAMBOO_DATASET_ID, DELETEDAT, TAGS, NOTES from onadata.libs.utils.export_tools import question_types_to_exclude From 793b9cc4f18805cd629a9818cbed6f83692a1d5e Mon Sep 17 00:00:00 2001 From: pld Date: Mon, 20 Jan 2014 07:41:46 -0500 Subject: [PATCH 25/31] PLD: order migrations --- ...eld_instance_geom.py => 0039_auto__add_field_instance_geom.py} | 0 ...ert_lng_lat_to_points.py => 0040_convert_lng_lat_to_points.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename onadata/apps/odk_logger/migrations/{0038_auto__add_field_instance_geom.py => 0039_auto__add_field_instance_geom.py} (100%) rename onadata/apps/odk_logger/migrations/{0039_convert_lng_lat_to_points.py => 0040_convert_lng_lat_to_points.py} (100%) diff --git a/onadata/apps/odk_logger/migrations/0038_auto__add_field_instance_geom.py b/onadata/apps/odk_logger/migrations/0039_auto__add_field_instance_geom.py similarity index 100% rename from onadata/apps/odk_logger/migrations/0038_auto__add_field_instance_geom.py rename to onadata/apps/odk_logger/migrations/0039_auto__add_field_instance_geom.py diff --git a/onadata/apps/odk_logger/migrations/0039_convert_lng_lat_to_points.py b/onadata/apps/odk_logger/migrations/0040_convert_lng_lat_to_points.py similarity index 100% rename from onadata/apps/odk_logger/migrations/0039_convert_lng_lat_to_points.py rename to onadata/apps/odk_logger/migrations/0040_convert_lng_lat_to_points.py From 2f20bed5c5514459a7e66092cf4b85b5a124b075 Mon Sep 17 00:00:00 2001 From: pld Date: Mon, 20 Jan 2014 17:55:48 -0500 Subject: [PATCH 26/31] PLD: correct migration sequence numbers --- ...eld_instance_geom.py => 0040_auto__add_field_instance_geom.py} | 0 ...ert_lng_lat_to_points.py => 0041_convert_lng_lat_to_points.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename onadata/apps/odk_logger/migrations/{0039_auto__add_field_instance_geom.py => 0040_auto__add_field_instance_geom.py} (100%) rename onadata/apps/odk_logger/migrations/{0040_convert_lng_lat_to_points.py => 0041_convert_lng_lat_to_points.py} (100%) diff --git a/onadata/apps/odk_logger/migrations/0039_auto__add_field_instance_geom.py b/onadata/apps/odk_logger/migrations/0040_auto__add_field_instance_geom.py similarity index 100% rename from onadata/apps/odk_logger/migrations/0039_auto__add_field_instance_geom.py rename to onadata/apps/odk_logger/migrations/0040_auto__add_field_instance_geom.py diff --git a/onadata/apps/odk_logger/migrations/0040_convert_lng_lat_to_points.py b/onadata/apps/odk_logger/migrations/0041_convert_lng_lat_to_points.py similarity index 100% rename from onadata/apps/odk_logger/migrations/0040_convert_lng_lat_to_points.py rename to onadata/apps/odk_logger/migrations/0041_convert_lng_lat_to_points.py From 932eb7e0a797beebfe0539228dc98d7c6f3efd77 Mon Sep 17 00:00:00 2001 From: pld Date: Tue, 21 Jan 2014 08:55:14 -0500 Subject: [PATCH 27/31] PLD: instances replaces surveys --- onadata/apps/odk_logger/models/xform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onadata/apps/odk_logger/models/xform.py b/onadata/apps/odk_logger/models/xform.py index 11a8423ba9..4158eddd92 100644 --- a/onadata/apps/odk_logger/models/xform.py +++ b/onadata/apps/odk_logger/models/xform.py @@ -173,8 +173,8 @@ def submission_count(self): def geocoded_submission_count(self): """Number of geocoded submissions.""" - return self.surveys.filter(deleted_at__isnull=True, - geom__isnull=False).count() + return self.instances.filter(deleted_at__isnull=True, + geom__isnull=False).count() def time_of_last_submission(self): if self.last_submission_time is None and self.num_of_submissions > 0: From 5de89c2d6e5d926947888c9a05dd61bbec727785 Mon Sep 17 00:00:00 2001 From: pld Date: Tue, 21 Jan 2014 16:08:01 -0500 Subject: [PATCH 28/31] PLD: create db scripts folder and add load backup script --- .travis.yml | 2 +- script/{install_database => database/install_postgis} | 0 script/database/load_backup | 3 +++ 3 files changed, 4 insertions(+), 1 deletion(-) rename script/{install_database => database/install_postgis} (100%) create mode 100755 script/database/load_backup diff --git a/.travis.yml b/.travis.yml index 913fd6dd19..6401ab9e48 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ services: python: - '2.7' before_install: - - ./script/install_database onadata_test + - ./script/database/install_postgis onadata_test - ./script/install_ubuntu env: - TESTFOLDER=onadata/apps/main diff --git a/script/install_database b/script/database/install_postgis similarity index 100% rename from script/install_database rename to script/database/install_postgis diff --git a/script/database/load_backup b/script/database/load_backup new file mode 100755 index 0000000000..92820c0bcd --- /dev/null +++ b/script/database/load_backup @@ -0,0 +1,3 @@ +scp ona:~/onadata.sql.tar.gz +tar -xzvf onadata.sql.tar.gz +psql -U onadata -h 127.0.0.1 onadata -f backups/postgres/onadata.sql From 0a238c5c3b9cbcebc83ecc4abeb642d184a5411e Mon Sep 17 00:00:00 2001 From: pld Date: Tue, 21 Jan 2014 21:57:51 -0500 Subject: [PATCH 29/31] PLD: add force option to instance save and use in migration --- .../0041_convert_lng_lat_to_points.py | 5 ++--- onadata/apps/odk_logger/models/instance.py | 22 +++++++++++++++---- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/onadata/apps/odk_logger/migrations/0041_convert_lng_lat_to_points.py b/onadata/apps/odk_logger/migrations/0041_convert_lng_lat_to_points.py index 4f5a1ffc00..6cd0e5fd91 100644 --- a/onadata/apps/odk_logger/migrations/0041_convert_lng_lat_to_points.py +++ b/onadata/apps/odk_logger/migrations/0041_convert_lng_lat_to_points.py @@ -10,8 +10,8 @@ class Migration(DataMigration): def forwards(self, orm): "Parse all instance to add geoms." for obj in queryset_iterator(orm.Instance.objects.all()): - instance = Instance.objects.get(pd=obj.pk) - instance.save() + instance = Instance.objects.get(pk=obj.pk) + instance.save(force=True) def backwards(self, orm): "Write your backwards methods here." @@ -67,7 +67,6 @@ def backwards(self, orm): 'deleted_at': ('django.db.models.fields.DateTimeField', [], {'default': 'None', 'null': 'True'}), 'geom': ('django.contrib.gis.db.models.fields.GeometryCollectionField', [], {'null': 'True'}), u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'is_deleted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), 'json': ('jsonfield.fields.JSONField', [], {'default': '{}'}), 'status': ('django.db.models.fields.CharField', [], {'default': "u'submitted_via_web'", 'max_length': '20'}), 'survey_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['odk_logger.SurveyType']"}), diff --git a/onadata/apps/odk_logger/models/instance.py b/onadata/apps/odk_logger/models/instance.py index 21c2aa9ca6..c1ab9b0387 100644 --- a/onadata/apps/odk_logger/models/instance.py +++ b/onadata/apps/odk_logger/models/instance.py @@ -125,6 +125,22 @@ def set_deleted_at(cls, instance_id, deleted_at=timezone.now()): else: instance.set_deleted(deleted_at) + def _check_active(self, kwargs): + """Check that form is active and raise exception if not. + + :param kwargs: The kwargs passed to save, if it contains a 'force' key + ignore restrictions on saving. + """ + force = kwargs.get('force') + + if force: + del kwargs['force'] + + if not force and self.xform and not self.xform.downloadable: + raise FormInactiveError() + + return kwargs + def _set_geom(self): data_dictionary = self.xform.data_dictionary() geo_xpaths = data_dictionary.geopoint_xpaths() @@ -230,11 +246,9 @@ def point(self): return gc[0] def save(self, *args, **kwargs): - self._set_xform(get_id_string_from_xml_str(self.xml)) - - if self.xform and not self.xform.downloadable: - raise FormInactiveError() + kwargs = self._check_active(kwargs) + self._set_xform(get_id_string_from_xml_str(self.xml)) self._set_geom() self._set_json() self._set_survey_type() From f59a21f7e9144c017021d1e533890651df838cd5 Mon Sep 17 00:00:00 2001 From: pld Date: Tue, 21 Jan 2014 22:11:35 -0500 Subject: [PATCH 30/31] PLD: only install js test deps and run on 1 build --- .travis.yml | 8 ++++---- script/install_ubuntu | 6 ++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6401ab9e48..653a650f89 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,12 +20,12 @@ install: - python manage.py syncdb --noinput --settings=onadata.settings.travis_test - python manage.py migrate --noinput --settings=onadata.settings.travis_test # make ~/tmp dir writable for npm downloads - - sudo chmod a+w $HOME/tmp - - npm install - - bower install --force-latest + - if [[ "$TESTFOLDER" == onadata/apps/api* ]] ; then sudo chmod a+w $HOME/tmp ; fi + - if [[ "$TESTFOLDER" == onadata/apps/api* ]] ; then npm install ; fi + - if [[ "$TESTFOLDER" == onadata/apps/api* ]] ; then bower install --force-latest ; fi script: - python manage.py test $TESTFOLDER --noinput --settings=onadata.settings.travis_test - - karma start onadata/libs/javascript/tests/karma.conf.js + - if [[ "$TESTFOLDER" == onadata/apps/api* ]] ; then karma start onadata/libs/javascript/tests/karma.conf.js ; fi notifications: email: - tech@ona.io diff --git a/script/install_ubuntu b/script/install_ubuntu index e132b278a8..8c73c3a93b 100755 --- a/script/install_ubuntu +++ b/script/install_ubuntu @@ -6,9 +6,11 @@ sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main wget --quiet -O - http://apt.postgresql.org/pub/repos/apt/ACCC4CF8.asc | sudo apt-key add - sudo apt-get update -qqy sudo apt-get install -qq gfortran libatlas-base-dev libjpeg-dev python-numpy zlib1g-dev python-software-properties -sudo apt-get install -qqy nodejs # install Geospatial libraries for GeoDjango sudo apt-get install -qq binutils libproj-dev gdal-bin Postgresql-9.3-postgis sudo ln -s /usr/lib/`uname -i`-linux-gnu/libjpeg.so ~/virtualenv/python2.7/lib/ sudo ln -s /usr/lib/`uname -i`-linux-gnu/libz.so ~/virtualenv/python2.7/lib/ -sudo npm install --silent -g bower karma grunt-cli +if [[ "$TESTFOLDER" == onadata/apps/api* ]]; then + sudo apt-get install -qqy nodejs + sudo npm install --silent -g bower karma grunt-cli +fi From 856f0570079e9efa83411932c9afbce38dd8e462 Mon Sep 17 00:00:00 2001 From: pld Date: Wed, 22 Jan 2014 09:22:17 -0500 Subject: [PATCH 31/31] PLD: only require 2 values in geometry for lat and lng --- onadata/apps/odk_logger/models/instance.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/onadata/apps/odk_logger/models/instance.py b/onadata/apps/odk_logger/models/instance.py index c1ab9b0387..8f2d5d918c 100644 --- a/onadata/apps/odk_logger/models/instance.py +++ b/onadata/apps/odk_logger/models/instance.py @@ -149,11 +149,10 @@ def _set_geom(self): if len(geo_xpaths): for xpath in geo_xpaths: - # TODO store the altitude and precision geometry = [float(s) for s in doc.get(xpath, u'').split()] if len(geometry): - lat, lng, alt, precision = geometry + lat, lng = geometry[0:2] points.append(Point(lng, lat)) self.geom = GeometryCollection(points)