From 2593bf5509e9f415de114103087136099ce1f3aa Mon Sep 17 00:00:00 2001 From: Pi Delport Date: Tue, 26 Sep 2017 14:26:15 +0200 Subject: [PATCH 01/10] (Factor out key function for duplicate queries) --- debug_toolbar/panels/sql/panel.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/debug_toolbar/panels/sql/panel.py b/debug_toolbar/panels/sql/panel.py index 96eee8128..e28026f8c 100644 --- a/debug_toolbar/panels/sql/panel.py +++ b/debug_toolbar/panels/sql/panel.py @@ -142,6 +142,11 @@ def generate_stats(self, request, response): colors = contrasting_color_generator() trace_colors = defaultdict(lambda: next(colors)) query_duplicates = defaultdict(lambda: defaultdict(int)) + + # The key used to determine duplicate queries. + def duplicate_key(query): + return query['raw_sql'] + if self._queries: width_ratio_tally = 0 factor = int(256.0 / (len(self._databases) * 2.5)) @@ -164,7 +169,7 @@ def generate_stats(self, request, response): trans_id = None i = 0 for alias, query in self._queries: - query_duplicates[alias][query["raw_sql"]] += 1 + query_duplicates[alias][duplicate_key(query)] += 1 trans_id = query.get('trans_id') last_trans_id = trans_ids.get(alias) @@ -223,7 +228,7 @@ def generate_stats(self, request, response): for alias, query in self._queries: try: - duplicates_count, color = query_duplicates[alias][query["raw_sql"]] + duplicates_count, color = query_duplicates[alias][duplicate_key(query)] query["duplicate_count"] = duplicates_count query["duplicate_color"] = color except KeyError: From cb9546d68af1e2a66b6132a30bec92cbaa63ef76 Mon Sep 17 00:00:00 2001 From: Pi Delport Date: Tue, 26 Sep 2017 14:28:00 +0200 Subject: [PATCH 02/10] Consider the query params when detecting duplicate queries --- debug_toolbar/panels/sql/panel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug_toolbar/panels/sql/panel.py b/debug_toolbar/panels/sql/panel.py index e28026f8c..4e2619547 100644 --- a/debug_toolbar/panels/sql/panel.py +++ b/debug_toolbar/panels/sql/panel.py @@ -145,7 +145,7 @@ def generate_stats(self, request, response): # The key used to determine duplicate queries. def duplicate_key(query): - return query['raw_sql'] + return (query['raw_sql'], query['params']) if self._queries: width_ratio_tally = 0 From a0cc2b592b43c1283d5a668ee1ecf5a06f1185cc Mon Sep 17 00:00:00 2001 From: Pi Delport Date: Wed, 29 Aug 2018 13:10:07 +0200 Subject: [PATCH 03/10] SQL query recording: Track the raw_params too The duplicate query detection needs this, because `params` is not always JSON-serialisable. --- debug_toolbar/panels/sql/tracking.py | 1 + 1 file changed, 1 insertion(+) diff --git a/debug_toolbar/panels/sql/tracking.py b/debug_toolbar/panels/sql/tracking.py index 7d07e444e..4f8466d46 100644 --- a/debug_toolbar/panels/sql/tracking.py +++ b/debug_toolbar/panels/sql/tracking.py @@ -134,6 +134,7 @@ def _record(self, method, sql, params): 'duration': duration, 'raw_sql': sql, 'params': _params, + 'raw_params': params, 'stacktrace': stacktrace, 'start_time': start_time, 'stop_time': stop_time, From 9e38b6af868679ca4412242bc63a1800c635d1be Mon Sep 17 00:00:00 2001 From: Pi Delport Date: Wed, 29 Aug 2018 13:18:02 +0200 Subject: [PATCH 04/10] (Rename variable to avoid shadowing) --- debug_toolbar/panels/sql/panel.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/debug_toolbar/panels/sql/panel.py b/debug_toolbar/panels/sql/panel.py index 4e2619547..b36262641 100644 --- a/debug_toolbar/panels/sql/panel.py +++ b/debug_toolbar/panels/sql/panel.py @@ -217,7 +217,7 @@ def duplicate_key(query): # Queries are duplicates only if there's as least 2 of them. # Also, to hide queries, we need to give all the duplicate groups an id query_colors = contrasting_color_generator() - query_duplicates = { + query_duplicates_colors = { alias: { query: (duplicate_count, next(query_colors)) for query, duplicate_count in queries.items() @@ -228,7 +228,7 @@ def duplicate_key(query): for alias, query in self._queries: try: - duplicates_count, color = query_duplicates[alias][duplicate_key(query)] + duplicates_count, color = query_duplicates_colors[alias][duplicate_key(query)] query["duplicate_count"] = duplicates_count query["duplicate_color"] = color except KeyError: @@ -236,7 +236,9 @@ def duplicate_key(query): for alias, alias_info in self._databases.items(): try: - alias_info["duplicate_count"] = sum(e[0] for e in query_duplicates[alias].values()) + alias_info["duplicate_count"] = sum( + e[0] for e in query_duplicates_colors[alias].values() + ) except KeyError: pass From 054d55fc627616ec7991a465e3cb194d52f11668 Mon Sep 17 00:00:00 2001 From: Pi Delport Date: Wed, 29 Aug 2018 13:23:49 +0200 Subject: [PATCH 05/10] SQLPanel: Track duplicated queries both with and without params --- debug_toolbar/panels/sql/panel.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/debug_toolbar/panels/sql/panel.py b/debug_toolbar/panels/sql/panel.py index b36262641..2d6e9042c 100644 --- a/debug_toolbar/panels/sql/panel.py +++ b/debug_toolbar/panels/sql/panel.py @@ -142,10 +142,14 @@ def generate_stats(self, request, response): colors = contrasting_color_generator() trace_colors = defaultdict(lambda: next(colors)) query_duplicates = defaultdict(lambda: defaultdict(int)) + query_params_duplicates = defaultdict(lambda: defaultdict(int)) - # The key used to determine duplicate queries. + # The keys used to determine duplicate queries. def duplicate_key(query): - return (query['raw_sql'], query['params']) + return query['raw_sql'] + + def duplicate_params_key(query): + return (query['raw_sql'], query['raw_params']) if self._queries: width_ratio_tally = 0 @@ -170,6 +174,7 @@ def duplicate_key(query): i = 0 for alias, query in self._queries: query_duplicates[alias][duplicate_key(query)] += 1 + query_params_duplicates[alias][duplicate_params_key(query)] += 1 trans_id = query.get('trans_id') last_trans_id = trans_ids.get(alias) @@ -225,12 +230,23 @@ def duplicate_key(query): } for alias, queries in query_duplicates.items() } + query_params_duplicates_colors = { + alias: { + query: (duplicate_count, next(query_colors)) + for query, duplicate_count in queries.items() + if duplicate_count >= 2 + } + for alias, queries in query_params_duplicates.items() + } for alias, query in self._queries: try: - duplicates_count, color = query_duplicates_colors[alias][duplicate_key(query)] - query["duplicate_count"] = duplicates_count - query["duplicate_color"] = color + (query["duplicate_count"], query["duplicate_color"]) = ( + query_duplicates_colors[alias][duplicate_key(query)] + ) + (query["duplicate_params_count"], query["duplicate_params_color"]) = ( + query_params_duplicates_colors[alias][duplicate_params_key(query)] + ) except KeyError: pass @@ -239,6 +255,9 @@ def duplicate_key(query): alias_info["duplicate_count"] = sum( e[0] for e in query_duplicates_colors[alias].values() ) + alias_info["duplicate_params_count"] = sum( + e[0] for e in query_params_duplicates_colors[alias].values() + ) except KeyError: pass From 4653452355a8079f43727a50c7a6928ca3b7ebb6 Mon Sep 17 00:00:00 2001 From: Pi Delport Date: Wed, 29 Aug 2018 13:39:58 +0200 Subject: [PATCH 06/10] SQL panel template: Include counts of duplicates with params too --- debug_toolbar/templates/debug_toolbar/panels/sql.html | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/debug_toolbar/templates/debug_toolbar/panels/sql.html b/debug_toolbar/templates/debug_toolbar/panels/sql.html index ccede1664..61211e01b 100644 --- a/debug_toolbar/templates/debug_toolbar/panels/sql.html +++ b/debug_toolbar/templates/debug_toolbar/panels/sql.html @@ -7,6 +7,9 @@ {{ info.time_spent|floatformat:"2" }} ms ({% blocktrans count info.num_queries as num %}{{ num }} query{% plural %}{{ num }} queries{% endblocktrans %} {% if info.duplicate_count %} {% blocktrans with dupes=info.duplicate_count %}including {{ dupes }} duplicates{% endblocktrans %} + {% if info.duplicate_params_count %} + {% blocktrans with dupes=info.duplicate_params_count %}and {{ dupes }} duplicates with params{% endblocktrans %} + {% endif %} {% endif %}) {% endfor %} @@ -41,6 +44,12 @@ {% blocktrans with dupes=query.duplicate_count %}Duplicated {{ dupes }} times.{% endblocktrans %} {% endif %} + {% if query.duplicate_params_count %} + +   + {% blocktrans with dupes=query.duplicate_params_count %}Duplicated {{ dupes }} times with params.{% endblocktrans %} + + {% endif %}
{{ query.width_ratio }}%
From 3f57676bd9a867407b8003804b62ace1185bdd78 Mon Sep 17 00:00:00 2001 From: Pi Delport Date: Wed, 29 Aug 2018 15:55:18 +0200 Subject: [PATCH 07/10] Convert raw_params lists to hashable tuples --- debug_toolbar/panels/sql/panel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug_toolbar/panels/sql/panel.py b/debug_toolbar/panels/sql/panel.py index 2d6e9042c..8e842efff 100644 --- a/debug_toolbar/panels/sql/panel.py +++ b/debug_toolbar/panels/sql/panel.py @@ -149,7 +149,7 @@ def duplicate_key(query): return query['raw_sql'] def duplicate_params_key(query): - return (query['raw_sql'], query['raw_params']) + return (query['raw_sql'], tuple(query['raw_params'])) if self._queries: width_ratio_tally = 0 From 46e8a33284e8e452fd67251d3f2af6e3b901593f Mon Sep 17 00:00:00 2001 From: Pi Delport Date: Wed, 29 Aug 2018 14:35:00 +0200 Subject: [PATCH 08/10] Wording change: duplicate -> similar, for queries with differing params --- debug_toolbar/panels/sql/panel.py | 26 +++++++++---------- .../templates/debug_toolbar/panels/sql.html | 10 +++---- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/debug_toolbar/panels/sql/panel.py b/debug_toolbar/panels/sql/panel.py index 8e842efff..6ca5ee0b1 100644 --- a/debug_toolbar/panels/sql/panel.py +++ b/debug_toolbar/panels/sql/panel.py @@ -141,11 +141,11 @@ def disable_instrumentation(self): def generate_stats(self, request, response): colors = contrasting_color_generator() trace_colors = defaultdict(lambda: next(colors)) - query_duplicates = defaultdict(lambda: defaultdict(int)) + query_similar = defaultdict(lambda: defaultdict(int)) query_params_duplicates = defaultdict(lambda: defaultdict(int)) - # The keys used to determine duplicate queries. - def duplicate_key(query): + # The keys used to determine similar and duplicate queries. + def similar_key(query): return query['raw_sql'] def duplicate_params_key(query): @@ -173,7 +173,7 @@ def duplicate_params_key(query): trans_id = None i = 0 for alias, query in self._queries: - query_duplicates[alias][duplicate_key(query)] += 1 + query_similar[alias][similar_key(query)] += 1 query_params_duplicates[alias][duplicate_params_key(query)] += 1 trans_id = query.get('trans_id') @@ -222,13 +222,13 @@ def duplicate_params_key(query): # Queries are duplicates only if there's as least 2 of them. # Also, to hide queries, we need to give all the duplicate groups an id query_colors = contrasting_color_generator() - query_duplicates_colors = { + query_similar_colors = { alias: { - query: (duplicate_count, next(query_colors)) - for query, duplicate_count in queries.items() - if duplicate_count >= 2 + query: (similar_count, next(query_colors)) + for query, similar_count in queries.items() + if similar_count >= 2 } - for alias, queries in query_duplicates.items() + for alias, queries in query_similar.items() } query_params_duplicates_colors = { alias: { @@ -241,8 +241,8 @@ def duplicate_params_key(query): for alias, query in self._queries: try: - (query["duplicate_count"], query["duplicate_color"]) = ( - query_duplicates_colors[alias][duplicate_key(query)] + (query["similar_count"], query["similar_color"]) = ( + query_similar_colors[alias][similar_key(query)] ) (query["duplicate_params_count"], query["duplicate_params_color"]) = ( query_params_duplicates_colors[alias][duplicate_params_key(query)] @@ -252,8 +252,8 @@ def duplicate_params_key(query): for alias, alias_info in self._databases.items(): try: - alias_info["duplicate_count"] = sum( - e[0] for e in query_duplicates_colors[alias].values() + alias_info["similar_count"] = sum( + e[0] for e in query_similar_colors[alias].values() ) alias_info["duplicate_params_count"] = sum( e[0] for e in query_params_duplicates_colors[alias].values() diff --git a/debug_toolbar/templates/debug_toolbar/panels/sql.html b/debug_toolbar/templates/debug_toolbar/panels/sql.html index 61211e01b..9388e458b 100644 --- a/debug_toolbar/templates/debug_toolbar/panels/sql.html +++ b/debug_toolbar/templates/debug_toolbar/panels/sql.html @@ -5,8 +5,8 @@
  •   {{ alias }} {{ info.time_spent|floatformat:"2" }} ms ({% blocktrans count info.num_queries as num %}{{ num }} query{% plural %}{{ num }} queries{% endblocktrans %} - {% if info.duplicate_count %} - {% blocktrans with dupes=info.duplicate_count %}including {{ dupes }} duplicates{% endblocktrans %} + {% if info.similar_count %} + {% blocktrans with count=info.similar_count %}including {{ count }} similar{% endblocktrans %} {% if info.duplicate_params_count %} {% blocktrans with dupes=info.duplicate_params_count %}and {{ dupes }} duplicates with params{% endblocktrans %} {% endif %} @@ -38,10 +38,10 @@
    {{ query.sql|safe }}
    - {% if query.duplicate_count %} + {% if query.similar_count %} -   - {% blocktrans with dupes=query.duplicate_count %}Duplicated {{ dupes }} times.{% endblocktrans %} +   + {% blocktrans with count=query.similar_count %}{{ count }} similar queries.{% endblocktrans %} {% endif %} {% if query.duplicate_params_count %} From ddb5fc97b71e540c777f6fcab3fadf6cbf016825 Mon Sep 17 00:00:00 2001 From: Pi Delport Date: Wed, 29 Aug 2018 15:56:15 +0200 Subject: [PATCH 09/10] Wording: "duplicate with params" -> duplicate, for identical queries --- debug_toolbar/panels/sql/panel.py | 20 +++++++++---------- .../templates/debug_toolbar/panels/sql.html | 10 +++++----- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/debug_toolbar/panels/sql/panel.py b/debug_toolbar/panels/sql/panel.py index 6ca5ee0b1..d586acedf 100644 --- a/debug_toolbar/panels/sql/panel.py +++ b/debug_toolbar/panels/sql/panel.py @@ -142,13 +142,13 @@ def generate_stats(self, request, response): colors = contrasting_color_generator() trace_colors = defaultdict(lambda: next(colors)) query_similar = defaultdict(lambda: defaultdict(int)) - query_params_duplicates = defaultdict(lambda: defaultdict(int)) + query_duplicates = defaultdict(lambda: defaultdict(int)) # The keys used to determine similar and duplicate queries. def similar_key(query): return query['raw_sql'] - def duplicate_params_key(query): + def duplicate_key(query): return (query['raw_sql'], tuple(query['raw_params'])) if self._queries: @@ -174,7 +174,7 @@ def duplicate_params_key(query): i = 0 for alias, query in self._queries: query_similar[alias][similar_key(query)] += 1 - query_params_duplicates[alias][duplicate_params_key(query)] += 1 + query_duplicates[alias][duplicate_key(query)] += 1 trans_id = query.get('trans_id') last_trans_id = trans_ids.get(alias) @@ -219,7 +219,7 @@ def duplicate_params_key(query): if trans_id: self._queries[(i - 1)][1]['ends_trans'] = True - # Queries are duplicates only if there's as least 2 of them. + # Queries are similar / duplicates only if there's as least 2 of them. # Also, to hide queries, we need to give all the duplicate groups an id query_colors = contrasting_color_generator() query_similar_colors = { @@ -230,13 +230,13 @@ def duplicate_params_key(query): } for alias, queries in query_similar.items() } - query_params_duplicates_colors = { + query_duplicates_colors = { alias: { query: (duplicate_count, next(query_colors)) for query, duplicate_count in queries.items() if duplicate_count >= 2 } - for alias, queries in query_params_duplicates.items() + for alias, queries in query_duplicates.items() } for alias, query in self._queries: @@ -244,8 +244,8 @@ def duplicate_params_key(query): (query["similar_count"], query["similar_color"]) = ( query_similar_colors[alias][similar_key(query)] ) - (query["duplicate_params_count"], query["duplicate_params_color"]) = ( - query_params_duplicates_colors[alias][duplicate_params_key(query)] + (query["duplicate_count"], query["duplicate_color"]) = ( + query_duplicates_colors[alias][duplicate_key(query)] ) except KeyError: pass @@ -255,8 +255,8 @@ def duplicate_params_key(query): alias_info["similar_count"] = sum( e[0] for e in query_similar_colors[alias].values() ) - alias_info["duplicate_params_count"] = sum( - e[0] for e in query_params_duplicates_colors[alias].values() + alias_info["duplicate_count"] = sum( + e[0] for e in query_duplicates_colors[alias].values() ) except KeyError: pass diff --git a/debug_toolbar/templates/debug_toolbar/panels/sql.html b/debug_toolbar/templates/debug_toolbar/panels/sql.html index 9388e458b..1dbe53440 100644 --- a/debug_toolbar/templates/debug_toolbar/panels/sql.html +++ b/debug_toolbar/templates/debug_toolbar/panels/sql.html @@ -7,8 +7,8 @@ {{ info.time_spent|floatformat:"2" }} ms ({% blocktrans count info.num_queries as num %}{{ num }} query{% plural %}{{ num }} queries{% endblocktrans %} {% if info.similar_count %} {% blocktrans with count=info.similar_count %}including {{ count }} similar{% endblocktrans %} - {% if info.duplicate_params_count %} - {% blocktrans with dupes=info.duplicate_params_count %}and {{ dupes }} duplicates with params{% endblocktrans %} + {% if info.duplicate_count %} + {% blocktrans with dupes=info.duplicate_count %}and {{ dupes }} duplicates{% endblocktrans %} {% endif %} {% endif %})
    • @@ -44,10 +44,10 @@ {% blocktrans with count=query.similar_count %}{{ count }} similar queries.{% endblocktrans %} {% endif %} - {% if query.duplicate_params_count %} + {% if query.duplicate_count %} -   - {% blocktrans with dupes=query.duplicate_params_count %}Duplicated {{ dupes }} times with params.{% endblocktrans %} +   + {% blocktrans with dupes=query.duplicate_count %}Duplicated {{ dupes }} times.{% endblocktrans %} {% endif %} From 50267363f4fa1565e4d18ed3aa8cbacfd84f7a88 Mon Sep 17 00:00:00 2001 From: Pi Delport Date: Fri, 31 Aug 2018 14:43:50 +0200 Subject: [PATCH 10/10] SQL panel: Add explanatory tooltip text to similar / duplicate query counts --- debug_toolbar/templates/debug_toolbar/panels/sql.html | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/debug_toolbar/templates/debug_toolbar/panels/sql.html b/debug_toolbar/templates/debug_toolbar/panels/sql.html index 1dbe53440..c8e93f0a6 100644 --- a/debug_toolbar/templates/debug_toolbar/panels/sql.html +++ b/debug_toolbar/templates/debug_toolbar/panels/sql.html @@ -6,9 +6,13 @@   {{ alias }} {{ info.time_spent|floatformat:"2" }} ms ({% blocktrans count info.num_queries as num %}{{ num }} query{% plural %}{{ num }} queries{% endblocktrans %} {% if info.similar_count %} - {% blocktrans with count=info.similar_count %}including {{ count }} similar{% endblocktrans %} + {% blocktrans with count=info.similar_count trimmed %} + including {{ count }} similar + {% endblocktrans %} {% if info.duplicate_count %} - {% blocktrans with dupes=info.duplicate_count %}and {{ dupes }} duplicates{% endblocktrans %} + {% blocktrans with dupes=info.duplicate_count trimmed %} + and {{ dupes }} duplicates + {% endblocktrans %} {% endif %} {% endif %})