Skip to content

IndexProcessor uses <ul><li> to identify index #36

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
oberix wants to merge 2 commits into
base: master
Choose a base branch
from
Open

IndexProcessor uses <ul><li> to identify index #36

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
72c690c
IndexProcessor uses <ul><li> to identify index
oberix Jan 26, 2018
f8a02b0
Merge remote-tracking branch 'tsgates/master'
oberix Nov 20, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
194 changes: 121 additions & 73 deletions pylookup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import os
import sys
import re

try:
import cPickle as pickle
except:
Expand All @@ -26,37 +27,42 @@
from contextlib import closing

if sys.version_info[0] == 3:
import html.parser as htmllib
import urllib.parse as urlparse
import html.parser as htmllib
import urllib.parse as urlparse
import urllib.request as urllib
else:
import htmllib, urllib, urlparse

VERBOSE = False
FORMATS = {
"Emacs" : "{entry}\t({desc})\t[{book}];{url}",
"Terminal" : "{entry}\t({desc})\t[{book}]\n{url}"
}
"Emacs": "{entry}\t({desc})\t[{book}];{url}",
"Terminal": "{entry}\t({desc})\t[{book}]\n{url}",
}


def build_book(s, num):
"""
Build book identifier from `s`, with `num` links.
"""
for matcher, replacement in (("library", "lib"),
("c-api", "api"),
("reference", "ref"),
("", "etc")):
for matcher, replacement in (
("library", "lib"),
("c-api", "api"),
("reference", "ref"),
("", "etc"),
):
if matcher in s:
return replacement if num == 1 else "%s/%d" % (replacement, num)


def trim(s):
"""
Add any globle filtering rules here
"""
s = s.replace( "Python Enhancement Proposals!", "")
s = s.replace( "PEP ", "PEP-")
s = s.replace("Python Enhancement Proposals!", "")
s = s.replace("PEP ", "PEP-")
return s


class Element(object):
def __init__(self, entry, desc, book, url):
self.book = book
Expand All @@ -65,8 +71,9 @@ def __init__(self, entry, desc, book, url):
self.entry = entry

def __format__(self, format_spec):
return format_spec.format(entry=self.entry, desc=self.desc,
book=self.book, url=self.url)
return format_spec.format(
entry=self.entry, desc=self.desc, book=self.book, url=self.url
)

def match_insensitive(self, key):
"""
Expand Down Expand Up @@ -120,70 +127,72 @@ def get_matcher(insensitive=True, desc=True):
return getattr(Element, "match{0}{1}".format(_in_entry, _sensitive))


class IndexProcessor( htmllib.HTMLParser ):
class IndexProcessor(htmllib.HTMLParser):
"""
Extract the index links from a Python HTML documentation index.
"""

def __init__( self, writer, dirn):
htmllib.HTMLParser.__init__( self, formatter.NullFormatter() )
def __init__(self, writer, dirn):
htmllib.HTMLParser.__init__(self, formatter.NullFormatter())

self.writer = writer
self.dirn = dirn
self.entry = ""
self.desc = ""
self.writer = writer
self.dirn = dirn
self.entry = ""
self.desc = ""
self.list_entry = False
self.one_entry = False
self.num_of_a = 0
self.desc_cnt = 0
self.one_entry = False
self.num_of_a = 0
self.desc_cnt = 0

def start_dd( self, att ):
def start_dd(self, att):
self.list_entry = True

def end_dd( self ):
def end_dd(self):
self.list_entry = False

def start_li( self, att ):
def start_li(self, att):
self.one_entry = True
self.num_of_a = 0

def start_a(self, att):
if self.one_entry:
self.url = join(self.dirn, dict(att)['href'])
self.url = join(self.dirn, dict(att)["href"])
self.save_bgn()

def end_a(self):
global VERBOSE
if self.one_entry:
if self.num_of_a == 0 :
if self.num_of_a == 0:
self.desc = self.save_end()

if VERBOSE:
self.desc_cnt += 1
if self.desc_cnt % 100 == 0:
sys.stdout.write("%04d %s\r" \
% (self.desc_cnt, self.desc.ljust(80)))
sys.stdout.write(
"%04d %s\r" % (self.desc_cnt, self.desc.ljust(80))
)

# extract first element
# ex) __and__() (in module operator)
if not self.list_entry :
self.entry = re.sub( "\([^)]+\)", "", self.desc )
if not self.list_entry:
self.entry = re.sub("\([^)]+\)", "", self.desc)

# clean up PEP
self.entry = trim(self.entry)

match = re.search( "\([^)]+\)", self.desc )
if match :
match = re.search("\([^)]+\)", self.desc)
if match:
self.desc = match.group(0)

self.desc = trim(re.sub( "[()]", "", self.desc ))
self.desc = trim(re.sub("[()]", "", self.desc))

self.num_of_a += 1
book = build_book(self.url, self.num_of_a)
e = Element(self.entry, self.desc, book, self.url)

self.writer(e)


def update(db, urls, append=False):
"""Update database with entries from urls.

Expand All @@ -200,14 +209,14 @@ def update(db, urls, append=False):
if not parsed.scheme or parsed.scheme == "file":
dst = abspath(expanduser(parsed.path))
if not os.path.exists(dst):
print("Error: %s doesn't exist" % dst)
print ("Error: %s doesn't exist" % dst)
exit(1)
url = "file://%s" % dst
else:
url = parsed.geturl()

potential_urls = []
if url.endswith('.html'):
if url.endswith(".html"):
potential_urls.append(url)
else:
# guess index URLs
Expand Down Expand Up @@ -258,10 +267,11 @@ def lookup(db, key, format_spec, out=sys.stdout, insensitive=True, desc=True):
while True:
e = pickle.load(f)
if matcher(e, key):
out.write('%s\n' % format(e, format_spec))
out.write("%s\n" % format(e, format_spec))
except EOFError:
pass


def cache(db, out=sys.stdout):
"""Print unique entries from db to out.

Expand All @@ -274,52 +284,90 @@ def cache(db, out=sys.stdout):
while True:
e = pickle.load(f)
k = e.entry
k = re.sub( "\([^)]*\)", "", k )
k = re.sub( "\[[^]]*\]", "", k )
k = re.sub("\([^)]*\)", "", k)
k = re.sub("\[[^]]*\]", "", k)
keys.add(k)
except EOFError:
pass
for k in keys:
out.write('%s\n' % k)
out.write("%s\n" % k)


if __name__ == "__main__":
import optparse
parser = optparse.OptionParser( __doc__.strip() )
parser.add_option( "-d", "--db",
help="database name",
dest="db", default="pylookup.db" )
parser.add_option( "-l", "--lookup",
help="keyword to search",
dest="key" )
parser.add_option( "-u", "--update",
help="update url or path",
action="append", type="str", dest="url" )
parser.add_option( "-c", "--cache" ,
help="extract keywords, internally used",
action="store_true", default=False, dest="cache")
parser.add_option( "-a", "--append",
help="append to the db from multiple sources",
action="store_true", default=False, dest="append")
parser.add_option( "-f", "--format",
help="type of output formatting, valid: Emacs, Terminal",
choices=["Emacs", "Terminal"],
default="Terminal", dest="format")
parser.add_option( "-i", "--insensitive", default=1, choices=['0', '1'],
help="SEARCH OPTION: insensitive search "
"(valid: 0, 1; default: %default)")
parser.add_option( "-s", "--desc", default=1, choices=['0', '1'],
help="SEARCH OPTION: include description field "
"(valid: 0, 1; default: %default)")
parser.add_option("-v", "--verbose",
help="verbose", action="store_true",
dest="verbose", default=False)
( opts, args ) = parser.parse_args()

parser = optparse.OptionParser(__doc__.strip())
parser.add_option(
"-d", "--db", help="database name", dest="db", default="pylookup.db"
)
parser.add_option("-l", "--lookup", help="keyword to search", dest="key")
parser.add_option(
"-u",
"--update",
help="update url or path",
action="append",
type="str",
dest="url",
)
parser.add_option(
"-c",
"--cache",
help="extract keywords, internally used",
action="store_true",
default=False,
dest="cache",
)
parser.add_option(
"-a",
"--append",
help="append to the db from multiple sources",
action="store_true",
default=False,
dest="append",
)
parser.add_option(
"-f",
"--format",
help="type of output formatting, valid: Emacs, Terminal",
choices=["Emacs", "Terminal"],
default="Terminal",
dest="format",
)
parser.add_option(
"-i",
"--insensitive",
default=1,
choices=["0", "1"],
help="SEARCH OPTION: insensitive search " "(valid: 0, 1; default: %default)",
)
parser.add_option(
"-s",
"--desc",
default=1,
choices=["0", "1"],
help="SEARCH OPTION: include description field "
"(valid: 0, 1; default: %default)",
)
parser.add_option(
"-v",
"--verbose",
help="verbose",
action="store_true",
dest="verbose",
default=False,
)
(opts, args) = parser.parse_args()

VERBOSE = opts.verbose
if opts.url:
update(opts.db, opts.url, opts.append)
if opts.cache:
cache(opts.db)
if opts.key:
lookup(opts.db, opts.key, FORMATS[opts.format],
insensitive=int(opts.insensitive), desc=int(opts.desc))
lookup(
opts.db,
opts.key,
FORMATS[opts.format],
insensitive=int(opts.insensitive),
desc=int(opts.desc),
)