diff --git a/pylookup.py b/pylookup.py index 655483f..fe8e34c 100755 --- a/pylookup.py +++ b/pylookup.py @@ -16,6 +16,7 @@ import os import sys import re + try: import cPickle as pickle except: @@ -26,37 +27,42 @@ from contextlib import closing if sys.version_info[0] == 3: - import html.parser as htmllib - import urllib.parse as urlparse + import html.parser as htmllib + import urllib.parse as urlparse import urllib.request as urllib else: import htmllib, urllib, urlparse VERBOSE = False FORMATS = { - "Emacs" : "{entry}\t({desc})\t[{book}];{url}", - "Terminal" : "{entry}\t({desc})\t[{book}]\n{url}" - } + "Emacs": "{entry}\t({desc})\t[{book}];{url}", + "Terminal": "{entry}\t({desc})\t[{book}]\n{url}", +} + def build_book(s, num): """ Build book identifier from `s`, with `num` links. """ - for matcher, replacement in (("library", "lib"), - ("c-api", "api"), - ("reference", "ref"), - ("", "etc")): + for matcher, replacement in ( + ("library", "lib"), + ("c-api", "api"), + ("reference", "ref"), + ("", "etc"), + ): if matcher in s: return replacement if num == 1 else "%s/%d" % (replacement, num) + def trim(s): """ Add any globle filtering rules here """ - s = s.replace( "Python Enhancement Proposals!", "") - s = s.replace( "PEP ", "PEP-") + s = s.replace("Python Enhancement Proposals!", "") + s = s.replace("PEP ", "PEP-") return s + class Element(object): def __init__(self, entry, desc, book, url): self.book = book @@ -65,8 +71,9 @@ def __init__(self, entry, desc, book, url): self.entry = entry def __format__(self, format_spec): - return format_spec.format(entry=self.entry, desc=self.desc, - book=self.book, url=self.url) + return format_spec.format( + entry=self.entry, desc=self.desc, book=self.book, url=self.url + ) def match_insensitive(self, key): """ @@ -120,63 +127,64 @@ def get_matcher(insensitive=True, desc=True): return getattr(Element, "match{0}{1}".format(_in_entry, _sensitive)) -class IndexProcessor( htmllib.HTMLParser ): +class IndexProcessor(htmllib.HTMLParser): """ Extract the index links from a Python HTML documentation index. """ - def __init__( self, writer, dirn): - htmllib.HTMLParser.__init__( self, formatter.NullFormatter() ) + def __init__(self, writer, dirn): + htmllib.HTMLParser.__init__(self, formatter.NullFormatter()) - self.writer = writer - self.dirn = dirn - self.entry = "" - self.desc = "" + self.writer = writer + self.dirn = dirn + self.entry = "" + self.desc = "" self.list_entry = False - self.one_entry = False - self.num_of_a = 0 - self.desc_cnt = 0 + self.one_entry = False + self.num_of_a = 0 + self.desc_cnt = 0 - def start_dd( self, att ): + def start_dd(self, att): self.list_entry = True - def end_dd( self ): + def end_dd(self): self.list_entry = False - def start_li( self, att ): + def start_li(self, att): self.one_entry = True self.num_of_a = 0 def start_a(self, att): if self.one_entry: - self.url = join(self.dirn, dict(att)['href']) + self.url = join(self.dirn, dict(att)["href"]) self.save_bgn() def end_a(self): global VERBOSE if self.one_entry: - if self.num_of_a == 0 : + if self.num_of_a == 0: self.desc = self.save_end() if VERBOSE: self.desc_cnt += 1 if self.desc_cnt % 100 == 0: - sys.stdout.write("%04d %s\r" \ - % (self.desc_cnt, self.desc.ljust(80))) + sys.stdout.write( + "%04d %s\r" % (self.desc_cnt, self.desc.ljust(80)) + ) # extract first element # ex) __and__() (in module operator) - if not self.list_entry : - self.entry = re.sub( "\([^)]+\)", "", self.desc ) + if not self.list_entry: + self.entry = re.sub("\([^)]+\)", "", self.desc) # clean up PEP self.entry = trim(self.entry) - match = re.search( "\([^)]+\)", self.desc ) - if match : + match = re.search("\([^)]+\)", self.desc) + if match: self.desc = match.group(0) - self.desc = trim(re.sub( "[()]", "", self.desc )) + self.desc = trim(re.sub("[()]", "", self.desc)) self.num_of_a += 1 book = build_book(self.url, self.num_of_a) @@ -184,6 +192,7 @@ def end_a(self): self.writer(e) + def update(db, urls, append=False): """Update database with entries from urls. @@ -200,14 +209,14 @@ def update(db, urls, append=False): if not parsed.scheme or parsed.scheme == "file": dst = abspath(expanduser(parsed.path)) if not os.path.exists(dst): - print("Error: %s doesn't exist" % dst) + print ("Error: %s doesn't exist" % dst) exit(1) url = "file://%s" % dst else: url = parsed.geturl() potential_urls = [] - if url.endswith('.html'): + if url.endswith(".html"): potential_urls.append(url) else: # guess index URLs @@ -258,10 +267,11 @@ def lookup(db, key, format_spec, out=sys.stdout, insensitive=True, desc=True): while True: e = pickle.load(f) if matcher(e, key): - out.write('%s\n' % format(e, format_spec)) + out.write("%s\n" % format(e, format_spec)) except EOFError: pass + def cache(db, out=sys.stdout): """Print unique entries from db to out. @@ -274,46 +284,79 @@ def cache(db, out=sys.stdout): while True: e = pickle.load(f) k = e.entry - k = re.sub( "\([^)]*\)", "", k ) - k = re.sub( "\[[^]]*\]", "", k ) + k = re.sub("\([^)]*\)", "", k) + k = re.sub("\[[^]]*\]", "", k) keys.add(k) except EOFError: pass for k in keys: - out.write('%s\n' % k) + out.write("%s\n" % k) + if __name__ == "__main__": import optparse - parser = optparse.OptionParser( __doc__.strip() ) - parser.add_option( "-d", "--db", - help="database name", - dest="db", default="pylookup.db" ) - parser.add_option( "-l", "--lookup", - help="keyword to search", - dest="key" ) - parser.add_option( "-u", "--update", - help="update url or path", - action="append", type="str", dest="url" ) - parser.add_option( "-c", "--cache" , - help="extract keywords, internally used", - action="store_true", default=False, dest="cache") - parser.add_option( "-a", "--append", - help="append to the db from multiple sources", - action="store_true", default=False, dest="append") - parser.add_option( "-f", "--format", - help="type of output formatting, valid: Emacs, Terminal", - choices=["Emacs", "Terminal"], - default="Terminal", dest="format") - parser.add_option( "-i", "--insensitive", default=1, choices=['0', '1'], - help="SEARCH OPTION: insensitive search " - "(valid: 0, 1; default: %default)") - parser.add_option( "-s", "--desc", default=1, choices=['0', '1'], - help="SEARCH OPTION: include description field " - "(valid: 0, 1; default: %default)") - parser.add_option("-v", "--verbose", - help="verbose", action="store_true", - dest="verbose", default=False) - ( opts, args ) = parser.parse_args() + + parser = optparse.OptionParser(__doc__.strip()) + parser.add_option( + "-d", "--db", help="database name", dest="db", default="pylookup.db" + ) + parser.add_option("-l", "--lookup", help="keyword to search", dest="key") + parser.add_option( + "-u", + "--update", + help="update url or path", + action="append", + type="str", + dest="url", + ) + parser.add_option( + "-c", + "--cache", + help="extract keywords, internally used", + action="store_true", + default=False, + dest="cache", + ) + parser.add_option( + "-a", + "--append", + help="append to the db from multiple sources", + action="store_true", + default=False, + dest="append", + ) + parser.add_option( + "-f", + "--format", + help="type of output formatting, valid: Emacs, Terminal", + choices=["Emacs", "Terminal"], + default="Terminal", + dest="format", + ) + parser.add_option( + "-i", + "--insensitive", + default=1, + choices=["0", "1"], + help="SEARCH OPTION: insensitive search " "(valid: 0, 1; default: %default)", + ) + parser.add_option( + "-s", + "--desc", + default=1, + choices=["0", "1"], + help="SEARCH OPTION: include description field " + "(valid: 0, 1; default: %default)", + ) + parser.add_option( + "-v", + "--verbose", + help="verbose", + action="store_true", + dest="verbose", + default=False, + ) + (opts, args) = parser.parse_args() VERBOSE = opts.verbose if opts.url: @@ -321,5 +364,10 @@ def cache(db, out=sys.stdout): if opts.cache: cache(opts.db) if opts.key: - lookup(opts.db, opts.key, FORMATS[opts.format], - insensitive=int(opts.insensitive), desc=int(opts.desc)) + lookup( + opts.db, + opts.key, + FORMATS[opts.format], + insensitive=int(opts.insensitive), + desc=int(opts.desc), + )