bug fixes, exporting csv files

Author: iniju

FileWr.py

@@ -0,0 +1,36 @@
+from __future__ import with_statement
+
+import logging
+from google.appengine.api import files
+from google.appengine.ext import db
+from google.appengine.ext import blobstore
+
+class FileWr(db.Model):
+	bkey = blobstore.BlobReferenceProperty(default=None)
+	def append(self, filename, data):
+		if self.bkey:
+			blob_reader = blobstore.BlobReader(self.bkey)
+		newblob = files.blobstore.create(mime_type='text/csv', _blobinfo_uploaded_filename=filename)
+		with files.open(newblob, 'a') as f:
+			if self.bkey:
+				buf = blob_reader.read(1000000)
+				while buf:
+					f.write(buf)
+					buf = blob_reader.read(1000000)
+					logging.info("rewritting %d" % len(buf))
+			f.write(data)
+			logging.info("writing %d" % len(data))
+		files.finalize(newblob)
+		if self.bkey:
+			self.bkey.delete()
+		blobkey = files.blobstore.get_blob_key(newblob)
+		self.bkey = blobkey
+		self.put()
+	def clear(self):
+		self.bkey.delete()
+		self.bkey = None
+		self.put()
+	def get(self):
+		blob_reader = blobstore.BlobReader(self.bkey)
+		return blob_reader.read()
+

admin_ops.py

@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU General Public License along with ankidroid-triage.
 # If not, see http://www.gnu.org/licenses/.
 # #####
+from __future__ import with_statement
 
 import os, sys, logging, re, hashlib, time
 from datetime import datetime
@@ -28,47 +29,35 @@
 settings._target = None
 
 from google.appengine.ext import webapp
+from google.appengine.ext import db
+from google.appengine.ext import blobstore
 from google.appengine.ext.webapp.util import run_wsgi_app
 from google.appengine.ext.webapp import template
 from google.appengine.api.urlfetch import fetch
 from google.appengine.api.urlfetch import Error
 from google.appengine.api import taskqueue
 from google.appengine.api import memcache
+from google.appengine.api import files
+from google.appengine.ext.webapp import blobstore_handlers
 
 from receive_ankicrashes import AppVersion
 from receive_ankicrashes import Feedback
 from receive_ankicrashes import CrashReport
-from receive_ankicrashes import HospitalizedReport
 from receive_ankicrashes import Bug
 from BeautifulSoup import BeautifulSoup
 from Cnt import Cnt
+from FileWr import FileWr
 from Lst import Lst
 
-# Remove the standard version of Django
-#for k in [k for k in sys.modules if k.startswith('django')]:
-#	del sys.modules[k]
 webapp.template.register_template_library('templatetags.basic_math')
 
-class AdminOpsBackendManager(webapp.RequestHandler):
+class BlobSetter(webapp.RequestHandler):
 	def get(self):
-		vq = AppVersion.all()
-		vq.order("-activeFrom")
-		vo = vq.fetch(2000)
-		vn = [v.name for v in vo]
-		vn.insert(0, 'all')
-		Lst.set("all_version_names_list", vn)
-
-		outputstr = repr(Lst.get("all_version_names_list"))
-		self.response.out.write(outputstr)
-
-	def post(self):
-		params = {'op': self.request.get('op'), 'cmd': self.request.get('cmd')}
-		#taskqueue.add(queue_name='admin-ops-queue', url='/backend/testing', params=params, method='GET', target='admin-ops-handler')
-		for i in range(1, 98):
-			taskqueue.add(queue_name='admin-ops-queue', url='/backend/testing', params=params, method='GET')#, target='admin-ops-handler')
-
-		#time.sleep(5)
-		#self.redirect('/admin_ops/backend')
+		fwe = FileWr.get_by_key_name("export_csv")
+		fwc = FileWr(key_name="crash_export_csv")
+		fwc.bkey = fwe.bkey
+		fwc.put()
+		self.response.out.write("OK")
 
 class ShowCrashBody(webapp.RequestHandler):
 	def get(self):
@@ -82,135 +71,9 @@ def get(self):
 		path = os.path.join(os.path.dirname(__file__), 'templates/admin_ops_show.html')
 		self.response.out.write(template.render(path, template_values))
 
-###
-# This class is used for bulk ad hoc operations in the DB, eg reparsing signatures, rebuilding the
-# bugs table or adding new properties to the entities.
-#
-# Due to limitations of time and memory in the AppEngine, we can't process all the entities in a
-# big loop, we have to work in batches. Each page loaded is a batch, following Next we can process
-# the next one, until all are done.
-##########
-class RebuildVersions(webapp.RequestHandler):
-	batch_size = 400
-	def get(self):
-		batch = RebuildVersions.batch_size
-		crashes_query = CrashReport.all()
-		crashes = []
-		page = int(self.request.get('page', 0))
-		if page == 0:
-			# Reset appVersion crashCount & lastIncident
-			versions_query = AppVersion.all()
-			versions = versions_query.fetch(2000)
-			for v in versions:
-				v.crashCount = 0
-				v.lastIncident = datetime(2000,1,1)
-				v.put()
-
-		total_results = crashes_query.count(1000000)
-		logging.info("Admin ops - total_results: ", total_results)
-		last_page = max((total_results - 1) // batch, 0)
-		if page > last_page:
-			page = last_page
-		crashes = crashes_query.fetch(batch, page * batch)
-		versionCounts = {}
-		versionLastIncidents = {}
-		for cr in crashes:
-			vname = cr.versionName.strip()
-			if cr.versionName <> vname:
-				cr.versionName = vname
-				cr.put()
-			if vname in versionCounts:
-				versionCounts[vname] = versionCounts[vname] + 1
-			else:
-				versionCounts[vname] = 1
-
-			if cr.versionName in versionLastIncidents:
-				if versionLastIncidents[vname] < cr.crashTime:
-					versionLastIncidents[vname] = cr.crashTime
-			else:
-				versionLastIncidents[vname] = cr.crashTime
-
-		for vname in versionCounts:
-			versions_query = AppVersion.all()
-			versions_query.filter('name =', vname)
-			versions = versions_query.fetch(1)
-			if versions:
-				version = versions[0]
-				version.crashCount = version.crashCount + versionCounts[vname]
-				if version.lastIncident < versionLastIncidents[vname]:
-					version.lastIncident = versionLastIncidents[vname]
-				version.put()
-			else:
-				logging.info("missing version: " + vname)
-
-		template_values = {
-				'values': versionCounts,
-				'page': page,
-				'last_page': last_page,
-				'page_size': batch,
-				'op_link': 'rebuild_versions',
-				'column_key': 'Version',
-				'column_value': 'Count',
-				'page_results': len(crashes),
-				'total_results': total_results}
-		path = os.path.join(os.path.dirname(__file__), 'templates/admin_ops.html')
-		self.response.out.write(template.render(path, template_values))
-
-class RebuildBugs(webapp.RequestHandler):
-	def get(self):
-		batch = RebuildBugs.batch_size
-		crashes_query = CrashReport.all()
-		crashes = []
-		page = int(self.request.get('page', 0))
-		if page == 0:
-			# Remove Bugs
-			bugs_query = Bug.all()
-			bugs = bugs_query.fetch(2000)
-			for b in bugs:
-				b.delete()
-
-		total_results = crashes_query.count(1000000)
-		last_page = max((total_results - 1) // batch, 0)
-		if page > last_page:
-			page = last_page
-		logging.info("Admin ops - total_results: ", str(total_results) + ", page: " + str(page) + "/" + str(last_page))
-		crashes = crashes_query.fetch(batch, page * batch)
-		valueSet = {}
-		valueSet["unlinked"] = 0
-		# Main ops loop
-		for cr in crashes:
-			cr.bugKey = None
-			cr.crashSignature = CrashReport.getCrashSignature(cr.report)
-			cr.put()
-			if cr.crashSignature == '\n':
-				logging.warning("Can't get signature for CrashReport: " + str(cr.key().id()))
-				valueSet["unlinked"] = valueSet["unlinked"] + 1
-			else:
-				cr.signHash = hashlib.sha1(cr.crashSignature).hexdigest()
-				cr.linkToBug()
-				bugId = str(cr.bugKey.key().id())
-				if bugId in valueSet:
-					valueSet[bugId] = valueSet[bugId] + 1
-				else:
-					valueSet[bugId] = 1
-		template_values = {
-				'values': valueSet,
-				'page': page,
-				'last_page': last_page,
-				'page_size': batch,
-				'op_link': 'rebuild_bugs',
-				'column_key': 'BugId',
-				'column_value': 'Count',
-				'page_results': len(crashes),
-				'total_results': total_results}
-		path = os.path.join(os.path.dirname(__file__), 'templates/admin_ops.html')
-		self.response.out.write(template.render(path, template_values))
-
 application = webapp.WSGIApplication(
 		[(r'^/admin_show.*$', ShowCrashBody),
-		(r'^/admin_ops/rebuild_versions$', RebuildVersions),
-		#(r'^/admin_ops/rebuild_bugs$', RebuildBugs)],
-		(r'^/admin_ops/backend.*$', AdminOpsBackendManager)],
+		(r'^/admin_ops/bset.*$', BlobSetter)],
 		debug=True)
 
 def main():

app.yaml

@@ -15,17 +15,13 @@ handlers:
 - url: /css
   static_dir: css
 
-#- url: /_ah/mail/ankidroid_crash_reports@ankidroid-triage\.appspotmail\.com.*
-#  script: receive_ankicrashes.py
-#  login: admin
-
 - url: /admin_.*
   script: admin_ops.py
   login: admin
 
-- url: /backend/testing
+- url: /backend/append_export
   script: backends.py
-#login: admin
+  login: admin
 
 - url: /cron_updater.*
   script: cron_updater.py
@@ -44,10 +40,6 @@ handlers:
 - url: /.*
   script: report_ankicrashes.py
 
-
-inbound_services:
-- mail
-
 builtins:
 - datastore_admin: on