From c770553be3c90ab2250bffd1afa3556fd660af11 Mon Sep 17 00:00:00 2001 From: noahwang <234082230@qq.com> Date: Thu, 30 Jul 2026 22:48:47 +0800 Subject: [PATCH] Add Python 2/3 dual-compatibility for Python 3.6 integration The Learningtribes fork imports cleanly but crashes at runtime on Python 3 in the discovery/index paths, blocking the py36 lock. Fix the Py2-only idioms so the fork runs under both 2.7 and 3.6: - utils: basestring -> six.string_types; collections.Iterable -> collections.abc fallback import - result_processor: dict.itervalues() -> six.itervalues(); exception .message -> six.text_type(); shlex.split() unicode handling guarded by six.PY2 (Py3 splits unicode directly) - api: isinstance(x, (str, unicode, ...)) -> six.string_types + (bytes, bytearray) - elastic: exception .message -> six.text_type() (6 sites, including two broken '...%s'.format(e.message) logging calls) - tests/mock_search_engine + test_views: basestring/unicode() -> six; bytes.translate(None, str) -> unicode filter - setup.py/tox.ini: declare Python 3.6, add six dependency, add py36 tox env - add standard from __future__ header to modernized modules Verified: compileall clean, 0 Py2-only patterns, and real-module runtime checks pass under Python 3. Co-authored-by: Cursor --- search/api.py | 10 ++++++---- search/elastic.py | 14 ++++++++------ search/result_processor.py | 22 +++++++++++++++------- search/tests/mock_search_engine.py | 14 +++++++++----- search/tests/test_views.py | 7 +++++-- search/utils.py | 12 ++++++++++-- setup.py | 4 +++- tox.ini | 2 +- 8 files changed, 57 insertions(+), 28 deletions(-) diff --git a/search/api.py b/search/api.py index 1f176998..7bacbd22 100755 --- a/search/api.py +++ b/search/api.py @@ -1,4 +1,6 @@ """ search business logic implementations """ +from __future__ import absolute_import, division, print_function, unicode_literals + import logging from datetime import datetime import dateutil.parser @@ -204,10 +206,10 @@ def process_range_data(results): new_start_terms['future'] for key, value in start_terms.items(): - if not isinstance(key, (str, unicode, bytes, bytearray)): + if not isinstance(key, six.string_types + (bytes, bytearray)): continue key = dateutil.parser.parse(key, ignoretz=True) - + new_key = 'current' if key > now: new_key = 'future' @@ -225,10 +227,10 @@ def process_range_data(results): end_term = course.get('data', {}).get('end', None) now = datetime.utcnow() # start property always has value(not None) - if not isinstance(start_term, (str, unicode, bytes, bytearray)): + if not isinstance(start_term, six.string_types + (bytes, bytearray)): continue if start_term and dateutil.parser.parse(start_term, ignoretz=True) <= now: - if not isinstance(end_term, (str, unicode, bytes, bytearray)): + if not isinstance(end_term, six.string_types + (bytes, bytearray)): continue if end_term and dateutil.parser.parse(end_term, ignoretz=True) <= now: status_terms['past'] += 1 diff --git a/search/elastic.py b/search/elastic.py index 978e978f..04a5cad4 100755 --- a/search/elastic.py +++ b/search/elastic.py @@ -1,4 +1,6 @@ """ Elastic Search implementation for courseware search index """ +from __future__ import absolute_import, division, print_function, unicode_literals + import copy import logging import six @@ -322,7 +324,7 @@ def search_mixed_discovery(engine, scoped_queries, sort, size, from_, if 'QueryParsingException' in message: log.exception("Malformed mixed search query: %s", message) raise QueryParseError('Malformed search query.') - log.exception("error while mixed search - %s", ex.message) + log.exception("error while mixed search - %s", six.text_type(ex)) raise return _translate_hits(es_response, None) @@ -373,7 +375,7 @@ def log_indexing_error(cls, indexing_errors): """ Logs indexing errors and raises a general ElasticSearch Exception""" indexing_errors_log = [] for indexing_error in indexing_errors: - indexing_errors_log.append(indexing_error.message) + indexing_errors_log.append(six.text_type(indexing_error)) raise exceptions.ElasticsearchException(', '.join(indexing_errors_log)) def _get_mappings(self, doc_type): @@ -582,7 +584,7 @@ def index(self, doc_type, sources, **kwargs): # Broad exception handler to protect around bulk call except Exception as ex: # log information and re-raise - log.exception("error while indexing - %s", ex.message) + log.exception("error while indexing - %s", six.text_type(ex)) raise def displace_index_to_alias(self, new_index_name, alias_name, expired_index_name=None): @@ -629,7 +631,7 @@ def displace_index_to_alias(self, new_index_name, alias_name, expired_index_name return existing_indexs except Exception as e: - log.exception('error while displacing alias - %s'.format(e.message)) + log.exception('error while displacing alias - %s', six.text_type(e)) raise def remove_by_index_name(self, index_name, retry_times=3): @@ -641,7 +643,7 @@ def remove_by_index_name(self, index_name, retry_times=3): return except Exception as e: - log.exception('error while deleting index - %s'.format(e.message)) + log.exception('error while deleting index - %s', six.text_type(e)) def remove(self, doc_type, doc_ids, **kwargs): """ Implements call to remove the documents from the index """ @@ -840,7 +842,7 @@ def search(self, raise QueryParseError('Malformed search query.') else: # log information and re-raise - log.exception("error while searching index - %s", ex.message) + log.exception("error while searching index - %s", six.text_type(ex)) raise return _translate_hits(es_response, total_keys) diff --git a/search/result_processor.py b/search/result_processor.py index d49fe124..a1f6da53 100755 --- a/search/result_processor.py +++ b/search/result_processor.py @@ -1,4 +1,6 @@ """ overridable result processor object to allow additional properties to be exposed """ +from __future__ import absolute_import, division, print_function, unicode_literals + import inspect from itertools import chain import json @@ -7,6 +9,8 @@ import shlex import textwrap +import six + from django.conf import settings from django.core.serializers.json import DjangoJSONEncoder @@ -43,8 +47,8 @@ def __init__(self, dictionary, match_phrase): @staticmethod def strings_in_dictionary(dictionary): """ Used by default implementation for finding excerpt """ - strings = [value for value in dictionary.itervalues() if not isinstance(value, dict)] - for child_dict in [dv for dv in dictionary.itervalues() if isinstance(dv, dict)]: + strings = [value for value in six.itervalues(dictionary) if not isinstance(value, dict)] + for child_dict in [dv for dv in six.itervalues(dictionary) if isinstance(dv, dict)]: strings.extend(SearchResultProcessor.strings_in_dictionary(child_dict)) return strings @@ -117,7 +121,7 @@ def process_result(cls, dictionary, match_phrase, user): # protect around any problems introduced by subclasses within their properties except Exception as ex: # pylint: disable=broad-except log.exception("error processing properties for %s - %s: will remove from results", - json.dumps(dictionary, cls=DjangoJSONEncoder), ex.message) + json.dumps(dictionary, cls=DjangoJSONEncoder), six.text_type(ex)) return None return dictionary @@ -130,10 +134,14 @@ def excerpt(self): return None match_phrases = [self._match_phrase] - separate_phrases = [ - phrase.decode('utf-8') - for phrase in shlex.split(self._match_phrase.encode('utf-8')) - ] + if six.PY2: + # shlex.split does not handle unicode on Python 2, so round-trip via utf-8 bytes. + separate_phrases = [ + phrase.decode('utf-8') + for phrase in shlex.split(self._match_phrase.encode('utf-8')) + ] + else: + separate_phrases = shlex.split(self._match_phrase) if len(separate_phrases) > 1: match_phrases.extend(separate_phrases) else: diff --git a/search/tests/mock_search_engine.py b/search/tests/mock_search_engine.py index cbf1c9c9..8a609e59 100755 --- a/search/tests/mock_search_engine.py +++ b/search/tests/mock_search_engine.py @@ -1,10 +1,14 @@ """ Implementation of search interface to be used for tests where ElasticSearch is unavailable """ +from __future__ import absolute_import, division, print_function, unicode_literals + import copy from datetime import datetime import json import os import pytz +import six + from django.conf import settings from django.core.serializers.json import DjangoJSONEncoder @@ -37,7 +41,7 @@ def _find_field(doc, field_name): if not isinstance(doc, dict): raise ValueError('Parameter `doc` should be a python dict object') - if not isinstance(field_name, basestring): + if not isinstance(field_name, six.string_types): raise ValueError('Parameter `field_name` should be a string') immediate_field, remaining_path = field_name.split('.', 1) if '.' in field_name else (field_name, None) @@ -67,7 +71,7 @@ def value_matches(doc, field_name, field_value): # if we have a string that we are trying to process as a date object if isinstance(field_value, (DateRange, datetime)): - if isinstance(compare_value, basestring): + if isinstance(compare_value, six.string_types): compare_value = json_date_to_datetime(compare_value) field_has_tz_info = False @@ -96,7 +100,7 @@ def value_matches(doc, field_name, field_value): return any((item == compare_value for item in field_value)) elif _is_iterable(compare_value) and _is_iterable(field_value): - return any((unicode(item) in field_value for item in compare_value)) + return any((six.text_type(item) in field_value for item in compare_value)) return compare_value == field_value @@ -110,8 +114,8 @@ def value_matches(doc, field_name, field_value): def _process_query_string(documents_to_search, query_string): """ keep the documents that contain at least one of the search strings provided """ def _encode_string(string): - """Encode a Unicode string in the same way as the Elasticsearch search engine.""" - return string.encode('utf-8').translate(None, RESERVED_CHARACTERS) + """Strip the characters the Elasticsearch engine treats as reserved.""" + return u''.join(char for char in string if char not in RESERVED_CHARACTERS) def has_string(dictionary_object, search_string): """ search for string in dictionary items, look down into nested dictionaries """ diff --git a/search/tests/test_views.py b/search/tests/test_views.py index 0916325e..f50ad831 100755 --- a/search/tests/test_views.py +++ b/search/tests/test_views.py @@ -1,6 +1,9 @@ """ High-level view tests""" +from __future__ import absolute_import, division, print_function, unicode_literals + from datetime import datetime import ddt +import six from django.core.urlresolvers import Resolver404, resolve from django.test import TestCase @@ -48,7 +51,7 @@ def assert_search_initiated_event(self, search_term, size, page): """Ensures an search initiated event was emitted""" initiated_search_call = self.mock_tracker.emit.mock_calls[0] # pylint: disable=maybe-no-member expected_result = call('edx.course.search.initiated', { - "search_term": unicode(search_term), + "search_term": six.text_type(search_term), "page_size": size, "page_number": page, }) @@ -58,7 +61,7 @@ def assert_results_returned_event(self, search_term, size, page, total): """Ensures an results returned event was emitted""" returned_results_call = self.mock_tracker.emit.mock_calls[1] # pylint: disable=maybe-no-member expected_result = call('edx.course.search.results_displayed', { - "search_term": unicode(search_term), + "search_term": six.text_type(search_term), "page_size": size, "page_number": page, "results_count": total, diff --git a/search/utils.py b/search/utils.py index ff032791..d6b62234 100755 --- a/search/utils.py +++ b/search/utils.py @@ -1,6 +1,14 @@ """ Utility classes to support others """ +from __future__ import absolute_import, division, print_function, unicode_literals + import importlib -import collections + +import six + +try: + from collections.abc import Iterable +except ImportError: # Python 2 + from collections import Iterable def _load_class(class_path, default): @@ -20,7 +28,7 @@ def _load_class(class_path, default): def _is_iterable(item): """ Checks if an item is iterable (list, tuple, generator), but not string """ - return isinstance(item, collections.Iterable) and not isinstance(item, basestring) + return isinstance(item, Iterable) and not isinstance(item, six.string_types) class ValueRange(object): diff --git a/setup.py b/setup.py index a6231747..880b1de5 100755 --- a/setup.py +++ b/setup.py @@ -19,6 +19,7 @@ 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', 'Framework :: Django', 'Framework :: Django :: 1.8', 'Framework :: Django :: 1.9', @@ -28,6 +29,7 @@ packages=['search', 'search.tests'], install_requires=[ "django >= 1.8, < 2.0", - "elasticsearch>=1.0.0,<2.0.0" + "elasticsearch>=1.0.0,<2.0.0", + "six" ] ) diff --git a/tox.ini b/tox.ini index 7c28619a..04eb17af 100755 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{27}-django{18,19,110,111}, quality +envlist = py{27,36}-django{18,19,110,111}, quality [testenv] setenv =