Add Python 2/3 dual-compatibility for Python 3.6 integration

search/api.py

@@ -1,4 +1,6 @@
 """ search business logic implementations """
+from __future__ import absolute_import, division, print_function, unicode_literals
+
 import logging
 from datetime import datetime
 import dateutil.parser
@@ -204,10 +206,10 @@ def process_range_data(results):
             new_start_terms['future']
 
             for key, value in start_terms.items():
-                if not isinstance(key, (str, unicode, bytes, bytearray)):
+                if not isinstance(key, six.string_types + (bytes, bytearray)):
                     continue
                 key = dateutil.parser.parse(key, ignoretz=True)
-                
+
                 new_key = 'current'
                 if key > now:
                     new_key = 'future'
@@ -225,10 +227,10 @@ def process_range_data(results):
             end_term = course.get('data', {}).get('end', None)
             now = datetime.utcnow()
             # start property always has value(not None)
-            if not isinstance(start_term, (str, unicode, bytes, bytearray)):
+            if not isinstance(start_term, six.string_types + (bytes, bytearray)):
                 continue
             if start_term and dateutil.parser.parse(start_term, ignoretz=True) <= now:
-                if not isinstance(end_term, (str, unicode, bytes, bytearray)):
+                if not isinstance(end_term, six.string_types + (bytes, bytearray)):
                     continue
                 if end_term and dateutil.parser.parse(end_term, ignoretz=True) <= now:
                     status_terms['past'] += 1

search/elastic.py

@@ -1,4 +1,6 @@
 """ Elastic Search implementation for courseware search index """
+from __future__ import absolute_import, division, print_function, unicode_literals
+
 import copy
 import logging
 import six
@@ -322,7 +324,7 @@ def search_mixed_discovery(engine, scoped_queries, sort, size, from_,
         if 'QueryParsingException' in message:
             log.exception("Malformed mixed search query: %s", message)
             raise QueryParseError('Malformed search query.')
-        log.exception("error while mixed search - %s", ex.message)
+        log.exception("error while mixed search - %s", six.text_type(ex))
         raise
 
     return _translate_hits(es_response, None)
@@ -373,7 +375,7 @@ def log_indexing_error(cls, indexing_errors):
         """ Logs indexing errors and raises a general ElasticSearch Exception"""
         indexing_errors_log = []
         for indexing_error in indexing_errors:
-            indexing_errors_log.append(indexing_error.message)
+            indexing_errors_log.append(six.text_type(indexing_error))
         raise exceptions.ElasticsearchException(', '.join(indexing_errors_log))
 
     def _get_mappings(self, doc_type):
@@ -582,7 +584,7 @@ def index(self, doc_type, sources, **kwargs):
         # Broad exception handler to protect around bulk call
         except Exception as ex:
             # log information and re-raise
-            log.exception("error while indexing - %s", ex.message)
+            log.exception("error while indexing - %s", six.text_type(ex))
             raise
 
     def displace_index_to_alias(self, new_index_name, alias_name, expired_index_name=None):
@@ -629,7 +631,7 @@ def displace_index_to_alias(self, new_index_name, alias_name, expired_index_name
             return existing_indexs
 
         except Exception as e:
-            log.exception('error while displacing alias - %s'.format(e.message))
+            log.exception('error while displacing alias - %s', six.text_type(e))
             raise
 
     def remove_by_index_name(self, index_name, retry_times=3):
@@ -641,7 +643,7 @@ def remove_by_index_name(self, index_name, retry_times=3):
                 return
 
             except Exception as e:
-                log.exception('error while deleting index - %s'.format(e.message))
+                log.exception('error while deleting index - %s', six.text_type(e))
 
     def remove(self, doc_type, doc_ids, **kwargs):
         """ Implements call to remove the documents from the index """
@@ -840,7 +842,7 @@ def search(self,
                 raise QueryParseError('Malformed search query.')
             else:
                 # log information and re-raise
-                log.exception("error while searching index - %s", ex.message)
+                log.exception("error while searching index - %s", six.text_type(ex))
                 raise
 
         return _translate_hits(es_response, total_keys)

search/result_processor.py

@@ -1,4 +1,6 @@
 """ overridable result processor object to allow additional properties to be exposed """
+from __future__ import absolute_import, division, print_function, unicode_literals
+
 import inspect
 from itertools import chain
 import json
@@ -7,6 +9,8 @@
 import shlex
 import textwrap
 
+import six
+
 from django.conf import settings
 from django.core.serializers.json import DjangoJSONEncoder
 
@@ -43,8 +47,8 @@ def __init__(self, dictionary, match_phrase):
     @staticmethod
     def strings_in_dictionary(dictionary):
         """ Used by default implementation for finding excerpt """
-        strings = [value for value in dictionary.itervalues() if not isinstance(value, dict)]
-        for child_dict in [dv for dv in dictionary.itervalues() if isinstance(dv, dict)]:
+        strings = [value for value in six.itervalues(dictionary) if not isinstance(value, dict)]
+        for child_dict in [dv for dv in six.itervalues(dictionary) if isinstance(dv, dict)]:
             strings.extend(SearchResultProcessor.strings_in_dictionary(child_dict))
         return strings
 
@@ -117,7 +121,7 @@ def process_result(cls, dictionary, match_phrase, user):
         # protect around any problems introduced by subclasses within their properties
         except Exception as ex:  # pylint: disable=broad-except
             log.exception("error processing properties for %s - %s: will remove from results",
-                          json.dumps(dictionary, cls=DjangoJSONEncoder), ex.message)
+                          json.dumps(dictionary, cls=DjangoJSONEncoder), six.text_type(ex))
             return None
         return dictionary
 
@@ -130,10 +134,14 @@ def excerpt(self):
             return None
 
         match_phrases = [self._match_phrase]
-        separate_phrases = [
-            phrase.decode('utf-8')
-            for phrase in shlex.split(self._match_phrase.encode('utf-8'))
-        ]
+        if six.PY2:
+            # shlex.split does not handle unicode on Python 2, so round-trip via utf-8 bytes.
+            separate_phrases = [
+                phrase.decode('utf-8')
+                for phrase in shlex.split(self._match_phrase.encode('utf-8'))
+            ]
+        else:
+            separate_phrases = shlex.split(self._match_phrase)
         if len(separate_phrases) > 1:
             match_phrases.extend(separate_phrases)
         else:

search/tests/mock_search_engine.py

@@ -1,10 +1,14 @@
 """ Implementation of search interface to be used for tests where ElasticSearch is unavailable """
+from __future__ import absolute_import, division, print_function, unicode_literals
+
 import copy
 from datetime import datetime
 import json
 import os
 import pytz
 
+import six
+
 from django.conf import settings
 from django.core.serializers.json import DjangoJSONEncoder
 
@@ -37,7 +41,7 @@ def _find_field(doc, field_name):
     if not isinstance(doc, dict):
         raise ValueError('Parameter `doc` should be a python dict object')
 
-    if not isinstance(field_name, basestring):
+    if not isinstance(field_name, six.string_types):
         raise ValueError('Parameter `field_name` should be a string')
 
     immediate_field, remaining_path = field_name.split('.', 1) if '.' in field_name else (field_name, None)
@@ -67,7 +71,7 @@ def value_matches(doc, field_name, field_value):
 
         # if we have a string that we are trying to process as a date object
         if isinstance(field_value, (DateRange, datetime)):
-            if isinstance(compare_value, basestring):
+            if isinstance(compare_value, six.string_types):
                 compare_value = json_date_to_datetime(compare_value)
 
             field_has_tz_info = False
@@ -96,7 +100,7 @@ def value_matches(doc, field_name, field_value):
             return any((item == compare_value for item in field_value))
 
         elif _is_iterable(compare_value) and _is_iterable(field_value):
-            return any((unicode(item) in field_value for item in compare_value))
+            return any((six.text_type(item) in field_value for item in compare_value))
 
         return compare_value == field_value
 
@@ -110,8 +114,8 @@ def value_matches(doc, field_name, field_value):
 def _process_query_string(documents_to_search, query_string):
     """ keep the documents that contain at least one of the search strings provided """
     def _encode_string(string):
-        """Encode a Unicode string in the same way as the Elasticsearch search engine."""
-        return string.encode('utf-8').translate(None, RESERVED_CHARACTERS)
+        """Strip the characters the Elasticsearch engine treats as reserved."""
+        return u''.join(char for char in string if char not in RESERVED_CHARACTERS)
 
     def has_string(dictionary_object, search_string):
         """ search for string in dictionary items, look down into nested dictionaries """

search/tests/test_views.py

@@ -1,6 +1,9 @@
 """ High-level view tests"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+
 from datetime import datetime
 import ddt
+import six
 
 from django.core.urlresolvers import Resolver404, resolve
 from django.test import TestCase
@@ -48,7 +51,7 @@ def assert_search_initiated_event(self, search_term, size, page):
         """Ensures an search initiated event was emitted"""
         initiated_search_call = self.mock_tracker.emit.mock_calls[0]  # pylint: disable=maybe-no-member
         expected_result = call('edx.course.search.initiated', {
-            "search_term": unicode(search_term),
+            "search_term": six.text_type(search_term),
             "page_size": size,
             "page_number": page,
         })
@@ -58,7 +61,7 @@ def assert_results_returned_event(self, search_term, size, page, total):
         """Ensures an results returned event was emitted"""
         returned_results_call = self.mock_tracker.emit.mock_calls[1]  # pylint: disable=maybe-no-member
         expected_result = call('edx.course.search.results_displayed', {
-            "search_term": unicode(search_term),
+            "search_term": six.text_type(search_term),
             "page_size": size,
             "page_number": page,
             "results_count": total,

search/utils.py

@@ -1,6 +1,14 @@
 """ Utility classes to support others """
+from __future__ import absolute_import, division, print_function, unicode_literals
+
 import importlib
-import collections
+
+import six
+
+try:
+    from collections.abc import Iterable
+except ImportError:  # Python 2
+    from collections import Iterable
 
 
 def _load_class(class_path, default):
@@ -20,7 +28,7 @@ def _load_class(class_path, default):
 
 def _is_iterable(item):
     """ Checks if an item is iterable (list, tuple, generator), but not string """
-    return isinstance(item, collections.Iterable) and not isinstance(item, basestring)
+    return isinstance(item, Iterable) and not isinstance(item, six.string_types)
 
 
 class ValueRange(object):

setup.py

@@ -19,6 +19,7 @@
         'Operating System :: OS Independent',
         'Programming Language :: Python',
         'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3.6',
         'Framework :: Django',
         'Framework :: Django :: 1.8',
         'Framework :: Django :: 1.9',
@@ -28,6 +29,7 @@
     packages=['search', 'search.tests'],
     install_requires=[
         "django >= 1.8, < 2.0",
-        "elasticsearch>=1.0.0,<2.0.0"
+        "elasticsearch>=1.0.0,<2.0.0",
+        "six"
     ]
 )

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py{27}-django{18,19,110,111}, quality
+envlist = py{27,36}-django{18,19,110,111}, quality
 
 [testenv]
 setenv =