From db7e6e954b1c5b76e5222b592e7ebefc266198e9 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 20 Jun 2019 18:27:21 -0400 Subject: [PATCH 1/2] fixes for Django v2.2 --- README.rst | 11 +++- profiler/__init__.py | 13 ++--- profiler/instrument.py | 9 ++-- profiler/middleware.py | 75 ++++++++++++++++----------- profiler/templates/profiler/base.html | 20 +++---- profiler/urls.py | 22 +++++--- profiler/views.py | 31 +++++------ 7 files changed, 104 insertions(+), 77 deletions(-) diff --git a/README.rst b/README.rst index c4d494b..edbcfff 100644 --- a/README.rst +++ b/README.rst @@ -9,8 +9,8 @@ Installation ------------ 1. Run `pip install django-live-profiler` 2. Add `'profiler'` app to `INSTALLED_APPS` -3. Add `'profiler.middleware.ProfilerMiddleware'` to `MIDDLEWARE_CLASSES` -4. Optionally add `'profiler.middleware.StatProfMiddleware'` to `MIDDLEWARE_CLASSES` to enable Python code statistical profiling (using statprof_). WARNING: this is an experimental feature, beware of possible incorrect output. +3. Add `'profiler.middleware.ProfilerMiddleware'` to `MIDDLEWARE` +4. Optionally add `'profiler.middleware.StatProfMiddleware'` to `MIDDLEWARE` to enable Python code statistical profiling (using statprof_). WARNING: this is an experimental feature, beware of possible incorrect output. 5. Add `url(r'^profiler/', include('profiler.urls'))` to your urlconf .. _statprof: https://github.com/bos/statprof.py @@ -23,5 +23,12 @@ In order to start gathering data you need to start the aggregation server:: $ aggregated --host 127.0.0.1 --port 5556 +Note, you must run Django with threading disabled in order for statprof to work! + + $ ./manage runserver --noreload --nothreading + +You may experience issues with staticfiles loading in chrome when `--nothreading` is passed. + +This is because chrome opens two initial connections, which blocks when Django is only able to respond to one of them. To fix this, you must serve staticfiles via separate staticfile server, such as nginx with a reverse_proxy to your Django runserver. Visit http://yoursite.com/profiler/ for results. diff --git a/profiler/__init__.py b/profiler/__init__.py index 33f96ff..c233dc8 100644 --- a/profiler/__init__.py +++ b/profiler/__init__.py @@ -1,9 +1,10 @@ -import threading +current_view = None -_local = threading.local() - -def _set_current_view(view): - _local.current_view = view +def _set_current_view(view_name): + global current_view + assert view_name is not None + current_view = view_name def _get_current_view(): - return getattr(_local, 'current_view', None) + global current_view + return current_view diff --git a/profiler/instrument.py b/profiler/instrument.py index 36a8791..5c12acc 100644 --- a/profiler/instrument.py +++ b/profiler/instrument.py @@ -3,12 +3,14 @@ from django.db.models.sql.compiler import SQLCompiler from django.db.models.sql.datastructures import EmptyResultSet from django.db.models.sql.constants import MULTI +from django.db.models.query import QuerySet from django.db import connection from aggregate.client import get_client from profiler import _get_current_view + def execute_sql(self, *args, **kwargs): client = get_client() if client is None: @@ -19,7 +21,7 @@ def execute_sql(self, *args, **kwargs): raise EmptyResultSet except EmptyResultSet: if kwargs.get('result_type', MULTI) == MULTI: - return iter([]) + return QuerySet.none() else: return start = datetime.now() @@ -27,9 +29,9 @@ def execute_sql(self, *args, **kwargs): return self.__execute_sql(*args, **kwargs) finally: d = (datetime.now() - start) - client.insert({'query' : q, 'view' : _get_current_view(), 'type' : 'sql'}, + client.insert({'query' : q, 'view' : _get_current_view(), 'type' : 'sql'}, {'time' : 0.0 + d.seconds * 1000 + d.microseconds/1000, 'count' : 1}) - + INSTRUMENTED = False @@ -38,3 +40,4 @@ def execute_sql(self, *args, **kwargs): SQLCompiler.__execute_sql = SQLCompiler.execute_sql SQLCompiler.execute_sql = execute_sql INSTRUMENTED = True + diff --git a/profiler/middleware.py b/profiler/middleware.py index 7e8f01a..617292d 100644 --- a/profiler/middleware.py +++ b/profiler/middleware.py @@ -1,60 +1,73 @@ -from datetime import datetime import inspect import statprof -from django.db import connection -from django.core.cache import cache from django.conf import settings - +from django.urls import resolve from aggregate.client import get_client from profiler import _set_current_view -class ProfilerMiddleware(object): - def process_view(self, request, view_func, view_args, view_kwargs): - if inspect.ismethod(view_func): - view_name = view_func.im_class.__module__+ '.' + view_func.im_class.__name__ + view_func.__name__ +def ProfilerMiddleware(get_response): + def middleware(request): + if request.path.startswith('/profiler'): + return get_response(request) + + view = resolve(request.path).func + if inspect.ismethod(view): + view_name = view.__class__.__module__+ '.' + view.__class__.__name__ else: - view_name = view_func.__module__ + '.' + view_func.__name__ + view_name = view.__module__ + '.' + view.__name__ _set_current_view(view_name) - - def process_response(self, request, response): - _set_current_view(None) - return response + return get_response(request) + + return middleware + -class StatProfMiddleware(object): +def StatProfMiddleware(get_response): + def middleware(request): + if request.path.startswith('/profiler'): + return get_response(request) - def process_request(self, request): + # print(f'[i] Starting sampling on {request.path}..') statprof.reset(getattr(settings, 'LIVEPROFILER_STATPROF_FREQUENCY', 100)) statprof.start() - def process_response(self, request, response): + response = get_response(request) + statprof.stop() - client = get_client() total_samples = statprof.state.sample_count if total_samples == 0: return response secs_per_sample = statprof.state.accumulated_time / total_samples - client.insert_all([( - {'file' : c.key.filename, - 'lineno' : c.key.lineno, - 'function' : c.key.name, - 'type' : 'python'}, - {'self_nsamples' : c.self_sample_count, - 'cum_nsamples' : c.cum_sample_count, - 'tot_nsamples' : total_samples, - 'cum_time' : c.cum_sample_count * secs_per_sample, - 'self_time' : c.self_sample_count * secs_per_sample - }) - for c in statprof.CallData.all_calls.itervalues()]) - - + # print('[i] Getting ZQM client...') + client = get_client() + client.insert_all([ + ( + { + 'file': c.key.filename, + 'lineno': c.key.lineno, + 'function': c.key.name, + 'type': 'python', + }, + { + 'self_nsamples': c.self_sample_count, + 'cum_nsamples': c.cum_sample_count, + 'tot_nsamples': total_samples, + 'cum_time': c.cum_sample_count * secs_per_sample, + 'self_time': c.self_sample_count * secs_per_sample, + } + ) + for c in statprof.CallData.all_calls.values() + ]) + # print(f'[i] Saved {statprof.state.sample_count} samples for {request.path}.') return response + + return middleware diff --git a/profiler/templates/profiler/base.html b/profiler/templates/profiler/base.html index 953bcee..6dee746 100644 --- a/profiler/templates/profiler/base.html +++ b/profiler/templates/profiler/base.html @@ -1,3 +1,5 @@ +{% load static %} + @@ -8,12 +10,12 @@ - - + + - - + + @@ -29,10 +31,10 @@
@@ -44,7 +46,7 @@ {% endblock %} - + diff --git a/profiler/urls.py b/profiler/urls.py index 07f3b7f..c723dc4 100644 --- a/profiler/urls.py +++ b/profiler/urls.py @@ -1,10 +1,16 @@ -from django.conf.urls.defaults import * +from django.urls import path -urlpatterns = patterns( - 'profiler.views', - url(r'^$', 'global_stats', name='profiler_global_stats'), - url(r'^by_view/$', 'stats_by_view', name='profiler_stats_by_view'), - url(r'^code/$', 'python_stats', name='profiler_python_stats'), - url(r'^reset/$', 'reset', name='profiler_reset'), - ) +from .views import ( + global_stats, + stats_by_view, + python_stats, + reset, +) + +urlpatterns = [ + path('', global_stats, name='profiler_global_stats'), + path('by_view/', stats_by_view, name='profiler_stats_by_view'), + path('code/', python_stats, name='profiler_python_stats'), + path('reset/', reset, name='profiler_reset'), +] diff --git a/profiler/views.py b/profiler/views.py index 150f3ba..005a66b 100644 --- a/profiler/views.py +++ b/profiler/views.py @@ -1,10 +1,9 @@ -from django.http import HttpResponse, HttpResponseRedirect -from django.shortcuts import render_to_response -from django.template.context import RequestContext -from django.core.cache import cache +import json + +from django.http import HttpResponseRedirect +from django.shortcuts import render from django.contrib.auth.decorators import user_passes_test -from django.core.urlresolvers import reverse -from django.utils import simplejson +from django.urls import reverse from aggregate.client import get_client @@ -13,9 +12,8 @@ def global_stats(request): stats = get_client().select(group_by=['query'], where={'type':'sql'}) for s in stats: s['average_time'] = s['time'] / s['count'] - return render_to_response('profiler/index.html', - {'queries' : stats}, - context_instance=RequestContext(request)) + return render(request, 'profiler/index.html', + {'queries' : stats}) @user_passes_test(lambda u:u.is_superuser) def stats_by_view(request): @@ -40,10 +38,9 @@ def stats_by_view(request): for r in stats: r['normtime'] = (0.0+r['average_time'])/maxtime - return render_to_response('profiler/by_view.html', + return render(request, 'profiler/by_view.html', {'queries' : grouped, - 'stats' :simplejson.dumps(stats)}, - context_instance=RequestContext(request)) + 'stats' :json.dumps(stats)}) @user_passes_test(lambda u:u.is_superuser) def reset(request): @@ -51,15 +48,13 @@ def reset(request): if request.method == 'POST': get_client().clear() return HttpResponseRedirect(next) - return render_to_response('profiler/reset.html', - {'next' : next}, - context_instance=RequestContext(request)) + return render(request, 'profiler/reset.html', + {'next' : next}) @user_passes_test(lambda u:u.is_superuser) def python_stats(request): stats = get_client().select(group_by=['file','lineno'], where={'type':'python'}) - return render_to_response('profiler/code.html', - {'stats' : stats}, - context_instance=RequestContext(request)) + return render(request, 'profiler/code.html', + {'stats' : stats}) From 0f34bef4f9ac8784a8c23005dc86d69a5ac2848f Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 20 Jun 2019 18:32:55 -0400 Subject: [PATCH 2/2] add superuser required note to readme --- README.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.rst b/README.rst index edbcfff..c00f827 100644 --- a/README.rst +++ b/README.rst @@ -32,3 +32,9 @@ You may experience issues with staticfiles loading in chrome when `--nothreading This is because chrome opens two initial connections, which blocks when Django is only able to respond to one of them. To fix this, you must serve staticfiles via separate staticfile server, such as nginx with a reverse_proxy to your Django runserver. Visit http://yoursite.com/profiler/ for results. + + +Note: you must be logged in as a superuser to view the profiler page. +You can create a superuser account with: + + $ ./manage.py createsuperuser