From af6976596d43264f5947decbf2bacdc45b85fb66 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 11 Apr 2012 13:48:08 +0200 Subject: [PATCH 01/10] - MAX_SHOW_ALL_ALLOWED was removed in favor or ChangeList.list_max_show_all since Django changeset #16725 (backport from https://github.com/amccloud/django-haystack/commit/4c5a6bd55a013be764da57f1b27e153247f1d501, for django 1.4) --- haystack/admin.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/haystack/admin.py b/haystack/admin.py index 3390a9dd0..0e295f369 100644 --- a/haystack/admin.py +++ b/haystack/admin.py @@ -1,6 +1,5 @@ from django.contrib.admin.options import ModelAdmin -from django.contrib.admin.views.main import (ChangeList, MAX_SHOW_ALL_ALLOWED, - SEARCH_VAR) +from django.contrib.admin.views.main import ChangeList, SEARCH_VAR from django.core.exceptions import PermissionDenied, ImproperlyConfigured from django.core.paginator import Paginator, InvalidPage from django.shortcuts import render_to_response @@ -22,6 +21,10 @@ def wraps(request, *args, **kwargs): csrf_protect_m = method_decorator(csrf_protect) +try: + from django.contrib.admin.views.main import MAX_SHOW_ALL_ALLOWED +except ImportError: + pass class SearchChangeList(ChangeList): def get_results(self, request): @@ -36,7 +39,8 @@ def get_results(self, request): result_count = paginator.count full_result_count = SearchQuerySet().models(self.model).all().count() - can_show_all = result_count <= MAX_SHOW_ALL_ALLOWED + can_show_all = result_count <= getattr(self, 'list_max_show_all', MAX_SHOW_ALL_ALLOWED) + multi_page = result_count > self.list_per_page # Get the list of objects to display on this page. From d161049e887eab67d4f86fa04aca9f4582e7e7da Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Thu, 12 Apr 2012 13:31:26 +0200 Subject: [PATCH 02/10] - if in DEBUG mode, now I record the stacktrace too - added a debugtoolbar panel for haystack --- haystack/backends/__init__.py | 3 +- haystack/utils/debugtoolbar.py | 59 ++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 haystack/utils/debugtoolbar.py diff --git a/haystack/backends/__init__.py b/haystack/backends/__init__.py index ac2a34100..65c9499d5 100644 --- a/haystack/backends/__init__.py +++ b/haystack/backends/__init__.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import inspect from copy import deepcopy from time import time from django.conf import settings @@ -22,7 +23,6 @@ # A means to inspect all search queries that have run in the last request. queries = [] - # Per-request, reset the ghetto query log. # Probably not extraordinarily thread-safe but should only matter when # DEBUG = True. @@ -55,6 +55,7 @@ def wrapper(obj, query_string, *args, **kwargs): 'additional_args': args, 'additional_kwargs': kwargs, 'time': "%.3f" % (stop - start), + 'stacktrace': inspect.stack() }) return wrapper diff --git a/haystack/utils/debugtoolbar.py b/haystack/utils/debugtoolbar.py new file mode 100644 index 000000000..487520290 --- /dev/null +++ b/haystack/utils/debugtoolbar.py @@ -0,0 +1,59 @@ +import django +from django.utils.translation import ugettext_lazy as _ +from django.utils.html import escape +from django.utils.safestring import mark_safe +from debug_toolbar.panels import DebugPanel +from debug_toolbar.utils import tidy_stacktrace + +import haystack.backends + +class HaystackDebugPanel(DebugPanel): + """ + Panel that displays the Haystack queries. + """ + name = 'Haystack' + template = 'debug_toolbar/panels/sql.html' + has_content = True + + def __init__(self, *args, **kwargs): + super(HaystackDebugPanel, self).__init__(*args, **kwargs) + self._queries = haystack.backends.queries + + def nav_title(self): + return _('Haystack queries') + + def nav_subtitle(self): + return "%s queries" % len(self._queries) + + def url(self): + return '' + + def title(self): + return 'Haystack Queries' + + def _transform_row(self, row): + data = { + 'sql': unicode(row['query_string']) + unicode(row['additional_args']) + unicode(row['additional_kwargs']), + 'duration': row['time'], + 'width_ratio_relative': 0, + 'start_offset': 0 + } + stacktrace = [] + if 'stacktrace' in row: + row['stacktrace'] = reversed(tidy_stacktrace(row['stacktrace'])) + for frame in row['stacktrace']: + params = map(escape, frame[0].rsplit('/', 1) + list(frame[1:])) + try: + stacktrace.append(u'{0}/{1} in {3}({2})\n {4}'.format(*params)) + except IndexError: + # This frame doesn't have the expected format, so skip it and move on to the next one + continue + data['stacktrace'] = mark_safe('\n'.join(stacktrace)) + return data + + def process_response(self, request, response): + self.record_stats({ + 'databases': [], + 'queries': [self._transform_row(q) for q in self._queries], + 'duration': 0, + }) From 194d3509e6b81f69ab1a0b03faf69d5f2a34ab87 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 8 Aug 2012 10:40:29 +0200 Subject: [PATCH 03/10] - add facet_sort parameter --- haystack/backends/__init__.py | 12 ++++++++++-- haystack/backends/solr_backend.py | 12 ++++++++++-- haystack/query.py | 6 ++++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/haystack/backends/__init__.py b/haystack/backends/__init__.py index 65c9499d5..6d97cc50e 100644 --- a/haystack/backends/__init__.py +++ b/haystack/backends/__init__.py @@ -289,6 +289,7 @@ def __init__(self, site=None, backend=None): self.facet_mincount = None self.facet_limit = None self.facet_prefix = None + self.facet_sort = None self.query_facets = [] self.narrow_queries = set() self._raw_query = None @@ -367,6 +368,9 @@ def build_params(self, spelling_query=None): if self.facet_prefix: kwargs['facet_prefix'] = self.facet_prefix + + if self.facet_sort: + kwargs['facet_sort'] = self.facet_sort if self.narrow_queries: kwargs['narrow_queries'] = self.narrow_queries @@ -605,14 +609,17 @@ def add_order_by(self, field): """Orders the search result by a field.""" self.order_by.append(field) - def set_facet_mincount(self,mincount): + def set_facet_mincount(self, mincount): self.facet_mincount = mincount def set_facet_limit(self, limit): self.facet_limit = limit - def set_facet_prefix(self,prefix): + def set_facet_prefix(self, prefix): self.facet_prefix = prefix + + def set_facet_sort(self, sort): + self.facet_sort = sort def clear_order_by(self): """ @@ -760,6 +767,7 @@ def _clone(self, klass=None): clone.facet_mincount = self.facet_mincount clone.facet_limit = self.facet_limit clone.facet_prefix = self.facet_prefix + clone.facet_sort = self.facet_sort clone.narrow_queries = self.narrow_queries.copy() clone.start_offset = self.start_offset clone.end_offset = self.end_offset diff --git a/haystack/backends/solr_backend.py b/haystack/backends/solr_backend.py index 75ddc18fb..45ae040e8 100644 --- a/haystack/backends/solr_backend.py +++ b/haystack/backends/solr_backend.py @@ -99,8 +99,9 @@ def clear(self, models=[], commit=True): @log_query def search(self, query_string, sort_by=None, start_offset=0, end_offset=None, fields='', highlight=False, facets=None, date_facets=None, query_facets=None, - narrow_queries=None, spelling_query=None, facet_mincount=None, facet_limit=None, facet_prefix=None, - limit_to_registered_models=None, result_class=None, **kwargs): + narrow_queries=None, spelling_query=None, facet_mincount=None, facet_limit=None, + facet_prefix=None, facet_sort=None, limit_to_registered_models=None, + result_class=None, **kwargs): if len(query_string) == 0: return { 'results': [], @@ -150,6 +151,10 @@ def search(self, query_string, sort_by=None, start_offset=0, end_offset=None, if facet_prefix is not None: kwargs['facet'] = 'on' kwargs['facet.prefix'] = facet_prefix + + if facet_sort is not None: + kwargs['facet'] = 'on' + kwargs['facet.sort'] = facet_sort if date_facets is not None: kwargs['facet'] = 'on' @@ -475,6 +480,9 @@ def run(self, spelling_query=None): if self.facet_prefix: kwargs['facet_prefix'] = self.facet_prefix + + if self.facet_sort: + kwargs['facet_sort'] = self.facet_sort if self.query_facets: kwargs['query_facets'] = self.query_facets diff --git a/haystack/query.py b/haystack/query.py index 52e10d1be..f9860c5a1 100644 --- a/haystack/query.py +++ b/haystack/query.py @@ -356,6 +356,12 @@ def facet_prefix(self,prefix): clone = self._clone() clone.query.set_facet_prefix(prefix) return clone + + def facet_sort(self,sort): + """Sets order of facet result""" + clone = self._clone() + clone.query.set_facet_sort(sort) + return clone def narrow(self, query): """Pushes existing facet choices into the search.""" From e6bcb34608db3ea3e79a18b03e9b7037b93c0965 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 10 Sep 2012 15:24:14 +0200 Subject: [PATCH 04/10] - add _facet_counts to cloned fields --- haystack/backends/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/haystack/backends/__init__.py b/haystack/backends/__init__.py index 6d97cc50e..771b53686 100644 --- a/haystack/backends/__init__.py +++ b/haystack/backends/__init__.py @@ -775,4 +775,5 @@ def _clone(self, klass=None): clone.result_class = self.result_class clone._raw_query = self._raw_query clone._raw_query_params = self._raw_query_params + clone._facet_counts = self._facet_counts return clone From 5f5b7d13fb4df520d7acf26270169951a399e490 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Fri, 17 May 2013 14:49:24 +0200 Subject: [PATCH 05/10] - added support for master-slave solr --- haystack/backends/solr_backend.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/haystack/backends/solr_backend.py b/haystack/backends/solr_backend.py index 45ae040e8..a0927877e 100644 --- a/haystack/backends/solr_backend.py +++ b/haystack/backends/solr_backend.py @@ -41,11 +41,12 @@ class SearchBackend(BaseSearchBackend): def __init__(self, site=None): super(SearchBackend, self).__init__(site) - if not hasattr(settings, 'HAYSTACK_SOLR_URL'): - raise ImproperlyConfigured('You must specify a HAYSTACK_SOLR_URL in your settings.') + if not hasattr(settings, 'HAYSTACK_SOLR_URLS'): + raise ImproperlyConfigured('You must specify a HAYSTACK_SOLR_URLS in your settings.') timeout = getattr(settings, 'HAYSTACK_SOLR_TIMEOUT', 10) - self.conn = Solr(settings.HAYSTACK_SOLR_URL, timeout=timeout) + self.conn = Solr(settings.HAYSTACK_SOLR_URLS['MASTER'], timeout=timeout) + self.conn_slave = Solr(settings.HAYSTACK_SOLR_URLS['SLAVE'], timeout=timeout) self.log = logging.getLogger('haystack') def update(self, index, iterable, commit=True): @@ -162,8 +163,8 @@ def search(self, query_string, sort_by=None, start_offset=0, end_offset=None, kwargs['facet.date.other'] = 'none' for key, value in date_facets.items(): - kwargs["f.%s.facet.date.start" % key] = self.conn._from_python(value.get('start_date')) - kwargs["f.%s.facet.date.end" % key] = self.conn._from_python(value.get('end_date')) + kwargs["f.%s.facet.date.start" % key] = self.conn_slave._from_python(value.get('start_date')) + kwargs["f.%s.facet.date.end" % key] = self.conn_slave._from_python(value.get('end_date')) gap_by_string = value.get('gap_by').upper() gap_string = "%d%s" % (value.get('gap_amount'), gap_by_string) @@ -194,7 +195,7 @@ def search(self, query_string, sort_by=None, start_offset=0, end_offset=None, kwargs['fq'] = list(narrow_queries) try: - raw_results = self.conn.search(query_string, **kwargs) + raw_results = self.conn_slave.search(query_string, **kwargs) except (IOError, SolrError), e: self.log.error("Failed to query Solr using '%s': %s", query_string, e) raw_results = EmptyResults() @@ -247,7 +248,7 @@ def more_like_this(self, model_instance, additional_query_string=None, query = "%s:%s" % (ID, get_identifier(model_instance)) try: - raw_results = self.conn.more_like_this(query, field_name, **params) + raw_results = self.conn_slave.more_like_this(query, field_name, **params) except (IOError, SolrError), e: self.log.error("Failed to fetch More Like This from Solr for document '%s': %s", query, e) raw_results = EmptyResults() @@ -303,7 +304,7 @@ def _process_results(self, raw_results, highlight=False, result_class=None): if string_key in index.fields and hasattr(index.fields[string_key], 'convert'): additional_fields[string_key] = index.fields[string_key].convert(value) else: - additional_fields[string_key] = self.conn._to_python(value) + additional_fields[string_key] = self.conn_slave._to_python(value) for name in [DJANGO_CT, DJANGO_ID, 'score']: if name in additional_fields: @@ -403,7 +404,7 @@ def build_query_fragment(self, field, filter_type, value): if not isinstance(value, (list, tuple)): # Convert whatever we find to what pysolr wants. - value = self.backend.conn._from_python(value) + value = self.backend.conn_slave._from_python(value) # Check to see if it's a phrase for an exact match. if ' ' in value: @@ -429,7 +430,7 @@ def build_query_fragment(self, field, filter_type, value): in_options = [] for possible_value in value: - in_options.append('%s:"%s"' % (index_fieldname, self.backend.conn._from_python(possible_value))) + in_options.append('%s:"%s"' % (index_fieldname, self.backend.conn_slave._from_python(possible_value))) result = "(%s)" % " OR ".join(in_options) elif filter_type == 'range': From 76b3c81b9755499a8d07e8dcafc051337f5d5a33 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Fri, 17 May 2013 18:02:51 +0200 Subject: [PATCH 06/10] - enhanced support to solr local params (on narrowing and faceting operations) --- haystack/backends/solr_backend.py | 49 +++++++++++++++++++++++++++++-- haystack/query.py | 8 ++--- 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/haystack/backends/solr_backend.py b/haystack/backends/solr_backend.py index a0927877e..2a3f6980e 100644 --- a/haystack/backends/solr_backend.py +++ b/haystack/backends/solr_backend.py @@ -466,9 +466,33 @@ def run(self, spelling_query=None): if self.highlight: kwargs['highlight'] = self.highlight - + if self.facets: - kwargs['facets'] = list(self.facets) + facets = [] + for facet, key, ex in list(self.facets): + if not facet: + continue + + f_mask = '' + tok = [] + if key or ex: + f_mask += '{!' + if key: + f_mask += 'key=%s' + tok.append(key) + if ex: + if key: + f_mask += ' ' + f_mask += 'ex=%s' + tok.append(ex) + if key or ex: + f_mask += '}' + f_mask += facet + if len(tok) > 0: + facet = f_mask % tuple(tok) + facets.append(facet) + + kwargs['facets'] = facets if self.date_facets: kwargs['date_facets'] = self.date_facets @@ -489,7 +513,13 @@ def run(self, spelling_query=None): kwargs['query_facets'] = self.query_facets if self.narrow_queries: - kwargs['narrow_queries'] = self.narrow_queries + narrow_queries = [] + for query, tag in list(self.narrow_queries): + if query and tag: + query = '{!tag=%s}' % tag + query + narrow_queries.append(query) + + kwargs['narrow_queries'] = narrow_queries if spelling_query: kwargs['spelling_query'] = spelling_query @@ -517,3 +547,16 @@ def run_mlt(self): results = self.backend.more_like_this(self._mlt_instance, additional_query_string, **kwargs) self._results = results.get('results', []) self._hit_count = results.get('hits', 0) + + def add_field_facet(self, field, key=None, ex=[]): + """Adds a regular facet on a field.""" + facet_field = self.backend.site.get_facet_field_name(field) + self.facets.add((facet_field, key, ','.join(ex))) + + def add_narrow_query(self, query, tag=None): + """ + Narrows a search to a subset of all documents per the query. + + Generally used in conjunction with faceting. + """ + self.narrow_queries.add((query, tag)) diff --git a/haystack/query.py b/haystack/query.py index f9860c5a1..d9a258a87 100644 --- a/haystack/query.py +++ b/haystack/query.py @@ -321,10 +321,10 @@ def boost(self, term, boost): clone.query.add_boost(term, boost) return clone - def facet(self, field): + def facet(self, field, **kwargs): """Adds faceting to a query for the provided field.""" clone = self._clone() - clone.query.add_field_facet(field) + clone.query.add_field_facet(field, **kwargs) return clone def date_facet(self, field, start_date, end_date, gap_by, gap_amount=1): @@ -363,10 +363,10 @@ def facet_sort(self,sort): clone.query.set_facet_sort(sort) return clone - def narrow(self, query): + def narrow(self, query, **kwargs): """Pushes existing facet choices into the search.""" clone = self._clone() - clone.query.add_narrow_query(query) + clone.query.add_narrow_query(query, **kwargs) return clone def raw_search(self, query_string, **kwargs): From a6218c8e4e77987f09ccb662bffeb155e293a89b Mon Sep 17 00:00:00 2001 From: Alessandro Nalato Date: Tue, 29 Oct 2013 15:15:40 +0100 Subject: [PATCH 07/10] Added support for Dismax using solr_backend Inspired by: https://github.com/dedsm/django-haystack/commit/c761d18c121c0b4f07f005a072ce7a3b13b783b5 --- haystack/backends/__init__.py | 12 +++++++++++- haystack/backends/solr_backend.py | 22 +++++++++++++++++++++- haystack/query.py | 6 ++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/haystack/backends/__init__.py b/haystack/backends/__init__.py index 771b53686..71879c30d 100644 --- a/haystack/backends/__init__.py +++ b/haystack/backends/__init__.py @@ -281,6 +281,7 @@ def __init__(self, site=None, backend=None): self.order_by = [] self.models = set() self.boost = {} + self.dismax = {} self.start_offset = 0 self.end_offset = None self.highlight = False @@ -668,7 +669,15 @@ def raw_search(self, query_string, **kwargs): """ self._raw_query = query_string self._raw_query_params = kwargs - + + def add_dismax(self, **kwargs): + """ + Allows backends with support for "dismax" to perform enhanced user query parsing + """ + + self.dismax = dict([(p, v) for p, v in kwargs.iteritems() + if p in getattr(self.backend, 'DISMAX_PARAMETERS', [])]) + def more_like_this(self, model_instance): """ Allows backends with support for "More Like This" to return results @@ -760,6 +769,7 @@ def _clone(self, klass=None): clone.order_by = self.order_by[:] clone.models = self.models.copy() clone.boost = self.boost.copy() + clone.dismax = self.dismax.copy() clone.highlight = self.highlight clone.facets = self.facets.copy() clone.date_facets = self.date_facets.copy() diff --git a/haystack/backends/solr_backend.py b/haystack/backends/solr_backend.py index 2a3f6980e..15e3d2e2c 100644 --- a/haystack/backends/solr_backend.py +++ b/haystack/backends/solr_backend.py @@ -38,6 +38,20 @@ class SearchBackend(BaseSearchBackend): '[', ']', '^', '"', '~', '*', '?', ':', ) + DISMAX_PARAMETERS = ( + 'defType', + 'qf', + 'q.alt', + 'mm', + 'pf', + 'ps', + 'qs', + 'tie', + 'bq', + 'bf', + 'boost', + ) + def __init__(self, site=None): super(SearchBackend, self).__init__(site) @@ -101,7 +115,7 @@ def clear(self, models=[], commit=True): def search(self, query_string, sort_by=None, start_offset=0, end_offset=None, fields='', highlight=False, facets=None, date_facets=None, query_facets=None, narrow_queries=None, spelling_query=None, facet_mincount=None, facet_limit=None, - facet_prefix=None, facet_sort=None, limit_to_registered_models=None, + facet_prefix=None, facet_sort=None, dismax=None, limit_to_registered_models=None, result_class=None, **kwargs): if len(query_string) == 0: return { @@ -193,6 +207,9 @@ def search(self, query_string, sort_by=None, start_offset=0, end_offset=None, if narrow_queries is not None: kwargs['fq'] = list(narrow_queries) + + if dismax is not None: + kwargs.update(dismax) try: raw_results = self.conn_slave.search(query_string, **kwargs) @@ -521,6 +538,9 @@ def run(self, spelling_query=None): kwargs['narrow_queries'] = narrow_queries + if self.dismax: + kwargs['dismax'] = self.dismax + if spelling_query: kwargs['spelling_query'] = spelling_query diff --git a/haystack/query.py b/haystack/query.py index d9a258a87..37642ee3c 100644 --- a/haystack/query.py +++ b/haystack/query.py @@ -375,6 +375,12 @@ def raw_search(self, query_string, **kwargs): clone.query.raw_search(query_string, **kwargs) return clone + def dismax(self, **kwargs): + """Sets dismax parameters for capable backends""" + clone = self._clone() + clone.query.add_dismax(**kwargs) + return clone + def load_all(self): """Efficiently populates the objects in the search results.""" clone = self._clone() From 6220ea1019bc9f4f22d945992e1c5544dbcaaf69 Mon Sep 17 00:00:00 2001 From: alenalato Date: Thu, 18 Aug 2016 13:34:00 +0200 Subject: [PATCH 08/10] Added per-field facet limit and Solr 4 pivot facets Additionally, local param "tag" now supports multiple values (as stated in Solr docs) --- haystack/backends/__init__.py | 31 ++++++- haystack/backends/solr_backend.py | 142 ++++++++++++++++++++++++------ haystack/query.py | 24 ++++- 3 files changed, 164 insertions(+), 33 deletions(-) diff --git a/haystack/backends/__init__.py b/haystack/backends/__init__.py index 71879c30d..d4cfd2bec 100644 --- a/haystack/backends/__init__.py +++ b/haystack/backends/__init__.py @@ -123,7 +123,7 @@ def clear(self, models=[], commit=True): @log_query def search(self, query_string, sort_by=None, start_offset=0, end_offset=None, fields='', highlight=False, facets=None, date_facets=None, query_facets=None, - narrow_queries=None, spelling_query=None, + pivot_facets=None, narrow_queries=None, spelling_query=None, limit_to_registered_models=None, result_class=None, **kwargs): """ Takes a query to search on and returns dictionary. @@ -289,9 +289,12 @@ def __init__(self, site=None, backend=None): self.date_facets = {} self.facet_mincount = None self.facet_limit = None + self.facet_field_limit = {} self.facet_prefix = None self.facet_sort = None self.query_facets = [] + self.pivot_facets = set() + self.facet_pivot_mincount = None self.narrow_queries = set() self._raw_query = None self._raw_query_params = {} @@ -360,18 +363,27 @@ def build_params(self, spelling_query=None): if self.query_facets: kwargs['query_facets'] = self.query_facets + + if self.pivot_facets: + kwargs['pivot_facets'] = list(self.pivot_facets) if self.facet_mincount: kwargs['facet_mincount'] = self.facet_mincount if self.facet_limit: kwargs['facet_limit'] = self.facet_limit + + if self.facet_field_limit: + kwargs['facet_field_limit'] = self.facet_field_limit if self.facet_prefix: kwargs['facet_prefix'] = self.facet_prefix if self.facet_sort: kwargs['facet_sort'] = self.facet_sort + + if self.facet_pivot_mincount: + kwargs['facet_pivot_mincount'] = self.facet_pivot_mincount if self.narrow_queries: kwargs['narrow_queries'] = self.narrow_queries @@ -615,12 +627,19 @@ def set_facet_mincount(self, mincount): def set_facet_limit(self, limit): self.facet_limit = limit + + def set_facet_field_limit(self, field, limit): + facet_field = self.backend.site.get_facet_field_name(field) + self.facet_field_limit[facet_field] = limit def set_facet_prefix(self, prefix): self.facet_prefix = prefix def set_facet_sort(self, sort): self.facet_sort = sort + + def set_facet_pivot_mincount(self, mincount): + self.facet_pivot_mincount = mincount def clear_order_by(self): """ @@ -710,6 +729,13 @@ def add_date_facet(self, field, start_date, end_date, gap_by, gap_amount=1): def add_query_facet(self, field, query): """Adds a query facet on a field.""" self.query_facets.append((self.backend.site.get_facet_field_name(field), query)) + + def add_pivot_facet(self, *args): + """Adds pivot faceting to a query for the provided fields.""" + facet_fields = [] + for field in args: + facet_fields.append(self.backend.site.get_facet_field_name(field)) + self.pivot_facets.add((','.join(facet_fields))) def add_narrow_query(self, query): """ @@ -774,10 +800,13 @@ def _clone(self, klass=None): clone.facets = self.facets.copy() clone.date_facets = self.date_facets.copy() clone.query_facets = self.query_facets[:] + clone.pivot_facets = self.pivot_facets.copy() clone.facet_mincount = self.facet_mincount clone.facet_limit = self.facet_limit + clone.facet_field_limit = self.facet_field_limit.copy() clone.facet_prefix = self.facet_prefix clone.facet_sort = self.facet_sort + clone.facet_pivot_mincount = self.facet_pivot_mincount clone.narrow_queries = self.narrow_queries.copy() clone.start_offset = self.start_offset clone.end_offset = self.end_offset diff --git a/haystack/backends/solr_backend.py b/haystack/backends/solr_backend.py index 15e3d2e2c..96f9e2c61 100644 --- a/haystack/backends/solr_backend.py +++ b/haystack/backends/solr_backend.py @@ -114,9 +114,10 @@ def clear(self, models=[], commit=True): @log_query def search(self, query_string, sort_by=None, start_offset=0, end_offset=None, fields='', highlight=False, facets=None, date_facets=None, query_facets=None, - narrow_queries=None, spelling_query=None, facet_mincount=None, facet_limit=None, - facet_prefix=None, facet_sort=None, dismax=None, limit_to_registered_models=None, - result_class=None, **kwargs): + pivot_facets=None, narrow_queries=None, spelling_query=None, facet_mincount=None, facet_limit=None, + facet_field_limit=None, facet_prefix=None, facet_sort=None, facet_pivot_mincount=None, dismax=None, + limit_to_registered_models=None, result_class=None, **kwargs): + if len(query_string) == 0: return { 'results': [], @@ -162,6 +163,11 @@ def search(self, query_string, sort_by=None, start_offset=0, end_offset=None, if facet_limit is not None: kwargs['facet'] = 'on' kwargs['facet.limit'] = facet_limit + + if facet_field_limit is not None: + kwargs['facet'] = 'on' + for f, limit in facet_field_limit.iteritems(): + kwargs['f.%s.facet.limit' % f] = limit if facet_prefix is not None: kwargs['facet'] = 'on' @@ -190,6 +196,14 @@ def search(self, query_string, sort_by=None, start_offset=0, end_offset=None, if query_facets is not None: kwargs['facet'] = 'on' kwargs['facet.query'] = ["%s:%s" % (field, value) for field, value in query_facets] + + if pivot_facets is not None: + kwargs['facet'] = 'on' + kwargs['facet.pivot'] = pivot_facets + + if facet_pivot_mincount is not None: + kwargs['facet'] = 'on' + kwargs['facet.pivot_mincount'] = facet_pivot_mincount if limit_to_registered_models is None: limit_to_registered_models = getattr(settings, 'HAYSTACK_LIMIT_TO_REGISTERED_MODELS', True) @@ -285,12 +299,13 @@ def _process_results(self, raw_results, highlight=False, result_class=None): if result_class is None: result_class = SearchResult - + if hasattr(raw_results, 'facets'): facets = { 'fields': raw_results.facets.get('facet_fields', {}), 'dates': raw_results.facets.get('facet_dates', {}), 'queries': raw_results.facets.get('facet_queries', {}), + 'pivots': raw_results.facets.get('facet_pivot', {}), } for key in ['fields']: @@ -298,7 +313,22 @@ def _process_results(self, raw_results, highlight=False, result_class=None): # Convert to a two-tuple, as Solr's json format returns a list of # pairs. facets[key][facet_field] = zip(facets[key][facet_field][::2], facets[key][facet_field][1::2]) - + + def _process_pivot(pivot): + facet = [] + for p in pivot: + if not 'pivot' in p: + facet.append((str(p['value']), p['count'], ())) + else: + facet.append((str(p['value']), p['count'], tuple(_process_pivot(p['pivot'])))) + + return facet + + for key in ['pivots']: + for facet_pivot in facets[key]: + # Convert to a three-tuple, with pairs + nested pivot + facets[key][facet_pivot] = _process_pivot(facets[key][facet_pivot]) + if getattr(settings, 'HAYSTACK_INCLUDE_SPELLING', False) is True: if hasattr(raw_results, 'spellcheck'): if len(raw_results.spellcheck.get('suggestions', [])): @@ -459,6 +489,27 @@ def build_query_fragment(self, field, filter_type, value): return result + def build_facet_field(self, facet, key, ex): + f_mask = '' + tok = [] + if key or ex: + f_mask += '{!' + if key: + f_mask += 'key=%s' + tok.append(key) + if ex: + if key: + f_mask += ' ' + f_mask += 'ex=%s' + tok.append(ex) + if key or ex: + f_mask += '}' + f_mask += facet + if len(tok) > 0: + facet = f_mask % tuple(tok) + + return facet + def run(self, spelling_query=None): """Builds and executes the query. Returns a list of search results.""" final_query = self.build_query() @@ -487,28 +538,8 @@ def run(self, spelling_query=None): if self.facets: facets = [] for facet, key, ex in list(self.facets): - if not facet: - continue - - f_mask = '' - tok = [] - if key or ex: - f_mask += '{!' - if key: - f_mask += 'key=%s' - tok.append(key) - if ex: - if key: - f_mask += ' ' - f_mask += 'ex=%s' - tok.append(ex) - if key or ex: - f_mask += '}' - f_mask += facet - if len(tok) > 0: - facet = f_mask % tuple(tok) - facets.append(facet) - + if facet: + facets.append(self.build_facet_field(facet, key, ex)) kwargs['facets'] = facets if self.date_facets: @@ -520,14 +551,27 @@ def run(self, spelling_query=None): if self.facet_limit: kwargs['facet_limit'] = self.facet_limit + if self.facet_field_limit: + kwargs['facet_field_limit'] = self.facet_field_limit + if self.facet_prefix: kwargs['facet_prefix'] = self.facet_prefix if self.facet_sort: kwargs['facet_sort'] = self.facet_sort + + if self.facet_pivot_mincount: + kwargs['facet_mincount'] = self.facet_mincount if self.query_facets: kwargs['query_facets'] = self.query_facets + + if self.pivot_facets: + facets = [] + for facet, key, ex in list(self.pivot_facets): + if facet: + facets.append(self.build_facet_field(facet, key, ex)) + kwargs['pivot_facets'] = facets if self.narrow_queries: narrow_queries = [] @@ -573,10 +617,50 @@ def add_field_facet(self, field, key=None, ex=[]): facet_field = self.backend.site.get_facet_field_name(field) self.facets.add((facet_field, key, ','.join(ex))) - def add_narrow_query(self, query, tag=None): + def add_narrow_query(self, query, tag=[]): """ Narrows a search to a subset of all documents per the query. Generally used in conjunction with faceting. """ - self.narrow_queries.add((query, tag)) + self.narrow_queries.add((query, ','.join(tag))) + + def add_pivot_facet(self, *args, **kwargs): + """Adds pivot faceting to a query for the provided fields.""" + key = kwargs.get('key', None) + ex = kwargs.get('ex', []) + facet_fields = [] + for field in args: + facet_fields.append(self.backend.site.get_facet_field_name(field)) + self.pivot_facets.add((','.join(facet_fields), key, ','.join(ex))) + + def post_process_facets(self, results): + # Handle renaming the facet fields. Undecorate and all that. + revised_facets = {} + field_data = self.backend.site.all_searchfields() + + for facet_type, field_details in results.get('facets', {}).items(): + temp_facets = {} + + for field, field_facets in field_details.items(): + fieldname = [] + + if facet_type in ['pivots']: + field = field.split(',') + else: + field = [field] + + for f in field: + if f in field_data and hasattr(field_data[f], 'get_facet_for_name'): + fieldname.append(field_data[f].get_facet_for_name()) + + if len(fieldname) == 0: + fieldname = field[0] + else: + fieldname = ','.join(fieldname) + + temp_facets[fieldname] = field_facets + + revised_facets[facet_type] = temp_facets + + return revised_facets diff --git a/haystack/query.py b/haystack/query.py index 37642ee3c..dc03a57d9 100644 --- a/haystack/query.py +++ b/haystack/query.py @@ -338,18 +338,30 @@ def query_facet(self, field, query): clone = self._clone() clone.query.add_query_facet(field, query) return clone + + def pivot_facet(self, *args, **kwargs): + """Adds pivot faceting to a query for the provided fields.""" + clone = self._clone() + clone.query.add_pivot_facet(*args, **kwargs) + return clone - def facet_mincount(self,mincount): + def facet_mincount(self, mincount): """Sets mincount for facet result""" clone = self._clone() clone.query.set_facet_mincount(mincount) return clone - def facet_limit(self,limit): + def facet_limit(self, limit): """Sets limit for facet result""" clone = self._clone() clone.query.set_facet_limit(limit) return clone + + def facet_field_limit(self, field, limit): + """Sets limit for facet result on given field""" + clone = self._clone() + clone.query.set_facet_field_limit(field, limit) + return clone def facet_prefix(self,prefix): """Sets prefix for facet result""" @@ -357,12 +369,18 @@ def facet_prefix(self,prefix): clone.query.set_facet_prefix(prefix) return clone - def facet_sort(self,sort): + def facet_sort(self, sort): """Sets order of facet result""" clone = self._clone() clone.query.set_facet_sort(sort) return clone + def facet_pivot_mincount(self, mincount): + """Sets mincount for pivot facet result""" + clone = self._clone() + clone.query.set_facet_mincount(mincount) + return clone + def narrow(self, query, **kwargs): """Pushes existing facet choices into the search.""" clone = self._clone() From 36b60593f624dbcb138e06a6ec41b08ce5bab90c Mon Sep 17 00:00:00 2001 From: alenalato Date: Thu, 18 Aug 2016 15:09:35 +0200 Subject: [PATCH 09/10] BugFix, unicode support in pivot facet values --- haystack/backends/solr_backend.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/haystack/backends/solr_backend.py b/haystack/backends/solr_backend.py index 96f9e2c61..b8c179586 100644 --- a/haystack/backends/solr_backend.py +++ b/haystack/backends/solr_backend.py @@ -318,9 +318,9 @@ def _process_pivot(pivot): facet = [] for p in pivot: if not 'pivot' in p: - facet.append((str(p['value']), p['count'], ())) + facet.append((unicode(p['value']), p['count'], ())) else: - facet.append((str(p['value']), p['count'], tuple(_process_pivot(p['pivot'])))) + facet.append((unicode(p['value']), p['count'], tuple(_process_pivot(p['pivot'])))) return facet From 5c344faba6f44199929122ffd3bd87c48c05b0a8 Mon Sep 17 00:00:00 2001 From: alenalato Date: Fri, 19 Aug 2016 16:16:23 +0200 Subject: [PATCH 10/10] BugFix, incomplete facet.pivot_mincount support --- haystack/backends/solr_backend.py | 2 +- haystack/query.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/haystack/backends/solr_backend.py b/haystack/backends/solr_backend.py index b8c179586..ce4eb9be8 100644 --- a/haystack/backends/solr_backend.py +++ b/haystack/backends/solr_backend.py @@ -561,7 +561,7 @@ def run(self, spelling_query=None): kwargs['facet_sort'] = self.facet_sort if self.facet_pivot_mincount: - kwargs['facet_mincount'] = self.facet_mincount + kwargs['facet_pivot_mincount'] = self.facet_pivot_mincount if self.query_facets: kwargs['query_facets'] = self.query_facets diff --git a/haystack/query.py b/haystack/query.py index dc03a57d9..400574b50 100644 --- a/haystack/query.py +++ b/haystack/query.py @@ -378,7 +378,7 @@ def facet_sort(self, sort): def facet_pivot_mincount(self, mincount): """Sets mincount for pivot facet result""" clone = self._clone() - clone.query.set_facet_mincount(mincount) + clone.query.set_facet_pivot_mincount(mincount) return clone def narrow(self, query, **kwargs):