diff --git a/project/tests/factories.py b/project/tests/factories.py index b5005c7a..bb40cb9d 100644 --- a/project/tests/factories.py +++ b/project/tests/factories.py @@ -1,19 +1,21 @@ # -*- coding: utf-8 -*- import factory import factory.fuzzy - -from silk.models import Request, Response, SQLQuery - +from silk.models import Request, Response, SQLQuery, SQLQueryDetails HTTP_METHODS = ['GET', 'POST', 'PUT', 'PATCH', 'HEAD', 'OPTIONS'] STATUS_CODES = [200, 201, 300, 301, 302, 401, 403, 404] - -class SQLQueryFactory(factory.django.DjangoModelFactory): - +class SQLQueryDetailsFactory(factory.django.DjangoModelFactory): query = factory.Sequence(lambda num: u'SELECT foo FROM bar WHERE foo=%s' % num) traceback = factory.Sequence(lambda num: u'Traceback #%s' % num) + class Meta: + model = SQLQueryDetails + +class SQLQueryFactory(factory.django.DjangoModelFactory): + details = factory.SubFactory(SQLQueryDetailsFactory) + class Meta: model = SQLQuery diff --git a/project/tests/test_collector.py b/project/tests/test_collector.py index f94a9727..090c5832 100644 --- a/project/tests/test_collector.py +++ b/project/tests/test_collector.py @@ -1,5 +1,6 @@ from django.test import TestCase from silk.collector import DataCollector +from silk.models import SQLQuery class TestCollector(TestCase): @@ -18,3 +19,17 @@ def test_clear(self): self.test_query_registration() DataCollector().clear() self.assertFalse(DataCollector().queries) + + def test_finalise(self): + """ + Just call the example app to try a full registration/finalise run + """ + self.client.get("/example_app/") + assert SQLQuery.objects.count() == 1 + query = SQLQuery.objects.get() + assert ( + query.details.query + == 'SELECT "example_app_blind"."id", "example_app_blind"."photo", "example_app_blind"."name", "example_app_blind"."child_safe" FROM "example_app_blind"' + ) + assert query.details.traceback != "" + assert query.details.analysis == "2 0 0 SCAN example_app_blind" diff --git a/project/tests/test_lib/mock_suite.py b/project/tests/test_lib/mock_suite.py index 7db952c5..d3419413 100644 --- a/project/tests/test_lib/mock_suite.py +++ b/project/tests/test_lib/mock_suite.py @@ -9,7 +9,7 @@ from django.utils import timezone from silk import models -from silk.models import SQLQuery, Profile +from silk.models import Profile, SQLQuery, SQLQueryDetails class MockSuite(object): @@ -93,7 +93,12 @@ def mock_sql_queries(self, request=None, profile=None, n=1, as_dict=False): if as_dict: queries.append(d) else: + details_kwargs = { + key: d.pop(key) + for key in ['query', 'traceback'] + } query = SQLQuery.objects.create(**d) + SQLQueryDetails.objects.create(query_obj=query, **details_kwargs) queries.append(query) if profile: if as_dict: diff --git a/project/tests/test_models.py b/project/tests/test_models.py index 8d4bafa8..09678ec8 100644 --- a/project/tests/test_models.py +++ b/project/tests/test_models.py @@ -290,7 +290,7 @@ def test_query_manager_instance(self): def test_traceback_ln_only(self): - self.obj.traceback = """Traceback (most recent call last): + self.obj.details.traceback = """Traceback (most recent call last): File "/home/user/some_script.py", line 10, in some_func pass File "/usr/lib/python2.7/bdb.py", line 20, in trace_dispatch @@ -308,7 +308,7 @@ def test_traceback_ln_only(self): def test_formatted_query_if_no_query(self): - self.obj.query = "" + self.obj.details.query = "" self.obj.formatted_query def test_formatted_query_if_has_a_query(self): @@ -320,7 +320,7 @@ def test_formatted_query_if_has_a_query(self): ON Book.isbn = Book_author.isbn GROUP BY Book.title;""" - self.obj.query = query + self.obj.details.query = query self.obj.formatted_query def test_num_joins_if_no_joins_in_query(self): @@ -330,7 +330,7 @@ def test_num_joins_if_no_joins_in_query(self): FROM Book GROUP BY Book.title;""" - self.obj.query = query + self.obj.details.query = query self.assertEqual(self.obj.num_joins, 0) @@ -344,14 +344,14 @@ def test_num_joins_if_joins_in_query(self): JOIN option_address_type oat ON o.id = oat.option_id WHERE a.country_id = 1 AND at.id <> oat.type_id;""" - self.obj.query = query + self.obj.details.query = query self.assertEqual(self.obj.num_joins, 4) def test_num_joins_if_no_joins_in_query_but_this_word_searched(self): query = """SELECT Book.title FROM Book WHERE Book.title=`Join the dark side, Luke!`;""" - self.obj.query = query + self.obj.details.query = query self.assertEqual(self.obj.num_joins, 0) def test_num_joins_if_multiple_statement_in_query(self): @@ -361,52 +361,52 @@ def test_num_joins_if_multiple_statement_in_query(self): INNER JOIN joined ON Book.joiner = joined.joiner WHERE Book.joiner='Join i am'""" - self.obj.query = query + self.obj.details.query = query self.assertEqual(self.obj.num_joins, 2) def test_tables_involved_if_no_query(self): - self.obj.query = '' + self.obj.details.query = '' self.assertEqual(self.obj.tables_involved, []) def test_tables_involved_if_query_has_only_a_from_token(self): query = """SELECT * FROM Book;""" - self.obj.query = query + self.obj.details.query = query self.assertEqual(self.obj.tables_involved, ['Book;']) def test_tables_involved_if_query_has_a_join_token(self): query = """SELECT p.id FROM Person p JOIN Address a ON p.Id = a.Person_ID;""" - self.obj.query = query + self.obj.details.query = query self.assertEqual(self.obj.tables_involved, ['Person', 'Address']) def test_tables_involved_if_query_has_an_as_token(self): query = 'SELECT Book.title AS Title FROM Book GROUP BY Book.title;' - self.obj.query = query + self.obj.details.query = query self.assertEqual(self.obj.tables_involved, ['Title', 'Book']) # FIXME bug, not a feature def test_tables_involved_check_with_fake_a_from_token(self): query = """SELECT * FROM Book WHERE Book.title=`EVIL FROM WITHIN`;""" - self.obj.query = query + self.obj.details.query = query self.assertEqual(self.obj.tables_involved, ['Book', 'WITHIN`;']) # FIXME bug, not a feature def test_tables_involved_check_with_fake_a_join_token(self): query = """SELECT * FROM Book WHERE Book.title=`Luke, join the dark side!`;""" - self.obj.query = query + self.obj.details.query = query self.assertEqual(self.obj.tables_involved, ['Book', 'the']) # FIXME bug, not a feature def test_tables_involved_check_with_fake_an_as_token(self): query = """SELECT * FROM Book WHERE Book.title=`AS SOON AS POSIABLE`;""" - self.obj.query = query + self.obj.details.query = query self.assertEqual(self.obj.tables_involved, ['Book', 'POSIABLE`;']) def test_tables_involved_if_query_has_subquery(self): @@ -418,14 +418,14 @@ def test_tables_involved_if_query_has_subquery(self): WHERE RealTableY.Col11>14 ) AS B INNER JOIN A ON A.ForeignKeyY=B.ID;''' - self.obj.query = query + self.obj.details.query = query self.assertEqual(self.obj.tables_involved, ['ID', 'RealTableZ', 'RealTableY', 'B', 'A']) # FIXME bug, not a feature def test_tables_involved_if_query_has_django_aliase_on_column_names(self): query = 'SELECT foo AS bar FROM some_table;' - self.obj.query = query + self.obj.details.query = query self.assertEqual(self.obj.tables_involved, ['bar', 'some_table;']) def test_save_if_no_end_and_start_time(self): diff --git a/silk/collector.py b/silk/collector.py index 84997cd7..dc01cfe6 100644 --- a/silk/collector.py +++ b/silk/collector.py @@ -152,14 +152,25 @@ def finalise(self): self.request.save() sql_queries = [] + sql_details = [] for identifier, query in self.queries.items(): query['identifier'] = identifier + details_kwargs = { + key: query.pop(key, "") + for key in ['query', 'traceback', 'analysis'] + } sql_query = models.SQLQuery(**query) + sql_detail = models.SQLQueryDetails(query_obj=sql_query, **details_kwargs) sql_queries += [sql_query] + sql_details += [sql_detail] models.SQLQuery.objects.bulk_create(sql_queries) - sql_queries = models.SQLQuery.objects.filter(request=self.request) - for sql_query in sql_queries.all(): + sql_queries = list(models.SQLQuery.objects.filter(request=self.request)) + query_by_identifer = {item.identifier: item for item in sql_queries} + for sql_detail in sql_details: + sql_detail.query_obj = query_by_identifer[sql_detail.query_obj.identifier] + models.SQLQueryDetails.objects.bulk_create(sql_details) + for sql_query in sql_queries: query = self.queries.get(sql_query.identifier) if query: query['model'] = sql_query diff --git a/silk/migrations/0009_sqlquerydetails.py b/silk/migrations/0009_sqlquerydetails.py new file mode 100644 index 00000000..a3f43355 --- /dev/null +++ b/silk/migrations/0009_sqlquerydetails.py @@ -0,0 +1,41 @@ +# Generated by Django 2.2.24 on 2021-06-24 09:54 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("silk", "0008_sqlquery_analysis"), + ] + + operations = [ + migrations.CreateModel( + name="SQLQueryDetails", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("query", models.TextField()), + ("traceback", models.TextField()), + ("analysis", models.TextField()), + ( + "query_obj", + models.OneToOneField( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="details", + to="silk.SQLQuery", + ), + ), + ], + ), + ] diff --git a/silk/migrations/0010_fill_query_details.py b/silk/migrations/0010_fill_query_details.py new file mode 100644 index 00000000..fb18057e --- /dev/null +++ b/silk/migrations/0010_fill_query_details.py @@ -0,0 +1,38 @@ +# Generated by Django 2.2.24 on 2021-06-24 09:54 + +from django.db import migrations, reset_queries + +BATCH_SIZE = 100 + + +def fill_query_details(apps, schema_editor): + SQLQueryDetails = apps.get_model("silk", "SQLQueryDetails") + SQLQuery = apps.get_model("silk", "SQLQuery") + queries = SQLQuery.objects.all().order_by("pk") + buffer = [] + for query in queries.iterator(BATCH_SIZE): + buffer.append( + SQLQueryDetails( + query_obj=query, + query=query.query, + traceback=query.traceback, + analysis=query.analysis or "", + ) + ) + if len(buffer) == BATCH_SIZE: + SQLQueryDetails.objects.bulk_create(buffer, BATCH_SIZE) + reset_queries() # query log leads to massive memory issue + buffer = [] + + +def backwards(apps, schema_editor): + pass + + +class Migration(migrations.Migration): + + dependencies = [ + ("silk", "0009_sqlquerydetails"), + ] + + operations = [migrations.RunPython(fill_query_details, backwards)] diff --git a/silk/migrations/0011_auto_20210624_1450.py b/silk/migrations/0011_auto_20210624_1450.py new file mode 100644 index 00000000..6fd24ebe --- /dev/null +++ b/silk/migrations/0011_auto_20210624_1450.py @@ -0,0 +1,25 @@ +# Generated by Django 2.2.24 on 2021-06-24 12:50 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("silk", "0010_fill_query_details"), + ] + + operations = [ + migrations.RemoveField( + model_name="sqlquery", + name="analysis", + ), + migrations.RemoveField( + model_name="sqlquery", + name="query", + ), + migrations.RemoveField( + model_name="sqlquery", + name="traceback", + ), + ] diff --git a/silk/models.py b/silk/models.py index 38d47244..7fefd327 100644 --- a/silk/models.py +++ b/silk/models.py @@ -231,7 +231,6 @@ def bulk_create(self, *args, **kwargs): class SQLQuery(models.Model): - query = TextField() start_time = DateTimeField(null=True, blank=True, default=timezone.now) end_time = DateTimeField(null=True, blank=True) time_taken = FloatField(blank=True, null=True) @@ -240,22 +239,20 @@ class SQLQuery(models.Model): Request, related_name='queries', null=True, blank=True, db_index=True, on_delete=models.CASCADE, ) - traceback = TextField() - analysis = TextField(null=True, blank=True) objects = SQLQueryManager() # TODO docstring @property def traceback_ln_only(self): - return '\n'.join(self.traceback.split('\n')[::2]) + return '\n'.join(self.details.traceback.split('\n')[::2]) @property def formatted_query(self): - return sqlparse.format(self.query, reindent=True, keyword_case='upper') + return sqlparse.format(self.details.query, reindent=True, keyword_case='upper') @property def num_joins(self): - parsed_query = sqlparse.parse(self.query) + parsed_query = sqlparse.parse(self.details.query) count = 0 for statement in parsed_query: count += sum(map(lambda t: t.match(sqlparse.tokens.Keyword, r'\.*join\.*', regex=True), statement.flatten())) @@ -269,7 +266,7 @@ def tables_involved(self): TODO: Can probably parse the SQL using sqlparse etc and pull out table info that way? """ - components = [x.strip() for x in self.query.split()] + components = [x.strip() for x in self.details.query.split()] tables = [] for idx, component in enumerate(components): @@ -309,6 +306,13 @@ def delete(self, *args, **kwargs): super(SQLQuery, self).delete(*args, **kwargs) +class SQLQueryDetails(models.Model): + query_obj = OneToOneField(SQLQuery, blank=True, null=True, on_delete=models.CASCADE, related_name='details') + query = TextField() + traceback = TextField() + analysis = TextField() + + class BaseProfile(models.Model): name = CharField(max_length=300, blank=True, default='') start_time = DateTimeField(default=timezone.now) diff --git a/silk/views/sql_detail.py b/silk/views/sql_detail.py index a682eaf2..63a24823 100644 --- a/silk/views/sql_detail.py +++ b/silk/views/sql_detail.py @@ -39,12 +39,12 @@ def get(self, request, *_, **kwargs): sql_id = kwargs.get('sql_id', None) request_id = kwargs.get('request_id', None) profile_id = kwargs.get('profile_id', None) - sql_query = SQLQuery.objects.get(pk=sql_id) + sql_query = SQLQuery.objects.select_related('details').get(pk=sql_id) pos = int(request.GET.get('pos', 0)) file_path = request.GET.get('file_path', '') line_num = int(request.GET.get('line_num', 0)) tb = sql_query.traceback_ln_only - analysis = sql_query.analysis + analysis = sql_query.details.analysis str, files = self._urlify(tb) if file_path and file_path not in files: raise PermissionDenied