diff --git a/mopidy_dirble/actor.py b/mopidy_dirble/actor.py index 9f11c6a..8aa8b4b 100644 --- a/mopidy_dirble/actor.py +++ b/mopidy_dirble/actor.py @@ -29,54 +29,75 @@ class DirbleLibrary(backend.LibraryProvider): # TODO: add countries when there is a lookup for countries with stations def browse(self, uri): - result = [] - variant, identifier = translator.parse_uri(uri) + user_countries = [] + geographic = [] + categories = [] + tracks = [] + + variant, identifier, page = translator.parse_uri(uri) + + limit = 20 + offset = (page or 0) * limit + next_offset = None + next_name = None if variant == 'root': for category in self.backend.dirble.categories(): - result.append(translator.category_to_ref(category)) + categories.append(translator.category_to_ref(category)) for continent in self.backend.dirble.continents(): - result.append(translator.continent_to_ref(continent)) + geographic.append(translator.continent_to_ref(continent)) elif variant == 'category' and identifier: - for category in self.backend.dirble.subcategories(identifier): - result.append(translator.category_to_ref(category)) - for station in self.backend.dirble.stations(category=identifier): - result.append(translator.station_to_ref(station)) + if not page: + for category in self.backend.dirble.subcategories(identifier): + categories.append(translator.category_to_ref(category)) + next_name = self.backend.dirble.category(identifier)['title'] + stations, next_offset = self.backend.dirble.stations( + category=identifier, offset=offset, limit=limit) + for station in stations: + tracks.append(translator.station_to_ref(station)) elif variant == 'continent' and identifier: for country in self.backend.dirble.countries(continent=identifier): - result.append(translator.country_to_ref(country)) + geographic.append(translator.country_to_ref(country)) elif variant == 'country' and identifier: - for station in self.backend.dirble.stations(country=identifier): - result.append( + next_name = self.backend.dirble.country(identifier)['name'] + stations, next_offset = self.backend.dirble.stations( + country=identifier, offset=offset, limit=limit) + for station in stations: + tracks.append( translator.station_to_ref(station, show_country=False)) else: logger.debug('Unknown URI: %s', uri) return [] - result.sort(key=lambda ref: ref.name) - - # Handle this case after the general ones as we want the user defined - # countries be the first entries, and retain their config sort order. if variant == 'root': - user_countries = [] for country_code in self.backend.countries: country = self.backend.dirble.country(country_code) if country: user_countries.append(translator.country_to_ref(country)) else: logger.debug('Unknown country: %s', country_code) - result = user_countries + result + + categories.sort(key=lambda ref: ref.name) + geographic.sort(key=lambda ref: ref.name) + + result = user_countries + geographic + categories + tracks if not result: logger.debug('Did not find any browse results for: %s', uri) + if next_offset: + next_page = int(next_offset / limit) + next_uri = translator.unparse_uri(variant, identifier, next_page) + next_name += ' page %d' % (next_page + 1) + result.append(Ref.directory(uri=next_uri, name=next_name)) + return result def refresh(self, uri=None): self.backend.dirble.flush() def lookup(self, uri): - variant, identifier = translator.parse_uri(uri) + variant, identifier, _ = translator.parse_uri(uri) if variant != 'station': return [] station = self.backend.dirble.station(identifier) @@ -88,29 +109,20 @@ def search(self, query=None, uris=None, exact=False): if not query.get('any'): return None - categories = set() - countries = [] - + filters = {} for uri in uris or []: - variant, identifier = translator.parse_uri(uri) + variant, identifier, _ = translator.parse_uri(uri) if variant == 'country': - countries.append(identifier.lower()) + filters['country'] = identifier elif variant == 'continent': - countries.extend(self.backend.dirble.countries(identifier)) + pass elif variant == 'category': - pending = [self.backend.dirble.category(identifier)] - while pending: - c = pending.pop(0) - categories.add(c['id']) - pending.extend(c['children']) + filters['category'] = identifier tracks = [] - for station in self.backend.dirble.search(' '.join(query['any'])): - if countries and station['country'].lower() not in countries: - continue - station_categories = {c['id'] for c in station['categories']} - if categories and not station_categories.intersection(categories): - continue + query = ' '.join(query['any']) + stations, _ = self.backend.dirble.search(query, limit=20, **filters) + for station in stations: tracks.append(translator.station_to_track(station)) return SearchResult(tracks=tracks) @@ -120,7 +132,7 @@ def get_images(self, uris): for uri in uris: result[uri] = [] - variant, identifier = translator.parse_uri(uri) + variant, identifier, _ = translator.parse_uri(uri) if variant != 'station' or not identifier: continue @@ -138,7 +150,7 @@ def get_images(self, uris): class DirblePlayback(backend.PlaybackProvider): def translate_uri(self, uri): - variant, identifier = translator.parse_uri(uri) + variant, identifier, _ = translator.parse_uri(uri) if variant != 'station': return None diff --git a/mopidy_dirble/client.py b/mopidy_dirble/client.py index 6574021..a77099c 100644 --- a/mopidy_dirble/client.py +++ b/mopidy_dirble/client.py @@ -1,12 +1,13 @@ from __future__ import unicode_literals import logging +import math +import os.path import time -import urllib from mopidy import __version__ as mopidy_version -from requests import Session, exceptions +from requests import Request, Session, exceptions from requests.adapters import HTTPAdapter from mopidy_dirble import __version__ as dirble_version @@ -18,6 +19,22 @@ def _normalize_keys(data): return {k.lower(): v for k, v in data.items()} +def _repaginate(fetch, offset, limit, page_size): + result = [] + page = int(math.floor(offset / float(page_size))) + discard = offset - page * page_size + + while len(result) < limit + 1: + fetched = fetch(page) + result.extend(fetched[discard:]) + if len(fetched) < page_size: + break + page += 1 + discard = 0 + + return result[:limit], offset + limit if len(result) > limit else None + + class Dirble(object): """Light wrapper for Dirble API lookup. @@ -42,7 +59,7 @@ def __init__(self, api_key, timeout): self._backoff_max = 60 self._backoff = 1 - self._base_uri = 'http://api.dirble.com/v2/' + self._base_uri = 'https://api.dirble.com/v2/' self._session = Session() self._session.params = {'token': api_key} @@ -75,18 +92,21 @@ def subcategories(self, identifier): category = self.category(identifier) return (category or {}).get('children', []) - def stations(self, category=None, country=None): + def stations(self, category=None, country=None, offset=0, limit=20): if category and not country: path = 'category/%s/stations' % category elif country and not category: - path = 'countries/%s/stations?all=1' % country.lower() + path = 'countries/%s/stations' % country.lower() else: return [] - stations = self._fetch(path, []) + def fetch(page): + return self._fetch(path, [], {'page': page, 'per_page': 30}) + + stations, next_offset = _repaginate(fetch, offset, limit, 30) for station in stations: self._stations.setdefault(station['id'], station) - return stations + return stations, next_offset def station(self, identifier): identifier = int(identifier) # Ensure we are consistent for cache key. @@ -114,38 +134,49 @@ def country(self, country_code): self._countries[c['country_code'].lower()] = c return self._countries.get(country_code.lower()) - def search(self, query): - quoted_query = urllib.quote(query.encode('utf-8')) - stations = self._fetch('search/%s' % quoted_query, []) + def search(self, query, category=None, country=None, offset=0, limit=20): + params = {'query': query, 'per_page': 30} + if category is not None: + params['category'] = category + if country is not None: + params['country'] = country.upper() + + def fetch(page): + params['page'] = page + return self._fetch('search', [], params, 'POST') + + stations, next_offset = _repaginate(fetch, offset, limit, 30) for station in stations: self._stations.setdefault(station['id'], station) - return stations + return stations, next_offset - def _fetch(self, path, default): + def _fetch(self, path, default, params=None, method='GET'): # Give up right away if we know the token is bad. if self._invalid_token: return default - uri = self._base_uri + path + request = Request( + method, os.path.join(self._base_uri, path), params=params) + prepared = self._session.prepare_request(request) # Try and serve request from our cache. - if uri in self._cache: - logger.debug('Cache hit: %s', uri) - return self._cache[uri] + if prepared.url in self._cache: + logger.debug('Cache hit: %s', prepared.url) + return self._cache[prepared.url] # Check if we should back of sending queries. if time.time() < self._backoff_until: - logger.debug('Back off fallback used: %s', uri) + logger.debug('Back off fallback used: %s', prepared.url) return default try: - logger.debug('Fetching: %s', uri) - resp = self._session.get(uri, timeout=self._timeout) + logger.debug('Fetching: %s', prepared.url) + resp = self._session.send(prepared, timeout=self._timeout) # Get succeeded, convert JSON, normalize and return. if resp.status_code == 200: data = resp.json(object_hook=_normalize_keys) - self._cache[uri] = data + self._cache[prepared.url] = data self._backoff = 1 return data diff --git a/mopidy_dirble/translator.py b/mopidy_dirble/translator.py index fd3aae9..481740f 100644 --- a/mopidy_dirble/translator.py +++ b/mopidy_dirble/translator.py @@ -1,26 +1,59 @@ from __future__ import unicode_literals -import re +import collections from mopidy.models import Ref, Track -def unparse_uri(variant, identifier): - return b'dirble:%s:%s' % (variant, identifier) +DirbleURI = collections.namedtuple( + 'DirbleURI', ['variant', 'identifier', 'page']) + + +def unparse_uri(variant, identifier, page=None): + uri = b'dirble:%s:%s' % (variant, identifier) + if page is not None: + uri += b':%s' % page + return uri def parse_uri(uri): - result = re.findall(r'^dirble:([a-z]+)(?::(\d+|[a-z]{2}))?$', uri) - if result: - return result[0] - return None, None + parts = uri.split(':') + none = DirbleURI(None, None, None) + + if tuple(parts) == ('dirble', 'root'): + return DirbleURI(parts[1], None, None) + + if len(parts) not in (3, 4): + return none + + if parts[0] != 'dirble': + return none + + if parts[1] in ('station', 'category', 'continent'): + if not parts[2].isdigit(): + return none + elif parts[1] in ('country'): + if len(parts[2]) != 2 or not parts[2].isalpha(): + return none + else: + return none + + offset = None + if len(parts) == 4: + if parts[1] not in ('category', 'country'): + return none + if not parts[3].isdigit(): + return none + offset = int(parts[3]) + + return DirbleURI(parts[1], parts[2], offset) def station_to_ref(station, show_country=True): name = station.get('name').strip() # TODO: fallback to streams URI? - if show_country: + if show_country and 'country' in station: # TODO: make this a setting so users can set '$name [$country]' etc? - name = '%s [%s]' % (name, station.get('country', '??')) + name = '%s [%s]' % (name, station['country']) uri = unparse_uri('station', station['id']) return Ref.track(uri=uri, name=name)