diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 51c5744..d7ad319 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest] - python-version: [3.7, 3.8, 3.9] + python-version: [3.8, 3.9] steps: - uses: actions/checkout@v3 diff --git a/Makefile b/Makefile index eefab82..26c8ded 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,3 @@ isort: isort --profile black wikimapper/ tests/ format: black isort - -html: - cd docs && make html \ No newline at end of file diff --git a/README.rst b/README.rst index 8a31420..5bb1691 100644 --- a/README.rst +++ b/README.rst @@ -159,7 +159,9 @@ Create your own index While some indices are precomupted, it is sometimes useful to create your own. The following section describes the steps need. Regarding creation speed: The index creation code works, but is not optimized. It takes around 10 minutes on my Notebook (T480s) -to create it for English Wikipedia if the data is already downloaded. +to create it for English Wikipedia if the data is already downloaded. With the English +dump of October 2026 (19.3 million pages), it takes around 8 minutes on an ASUS ROG +Zephyrus G14 (GA403UV), most of which is spent on resolving the redirects. **1. Download the data** @@ -223,7 +225,7 @@ line. url2id Map a Wikipedia URL to a Wikidata ID. id2titles Map a Wikidata ID to one or more Wikipedia titles. - optional arguments: + options: -h, --help show this help message and exit --version show program's version number and exit @@ -238,7 +240,7 @@ created and activated by :: - virtualenv venv --python=python3 --no-site-packages + virtualenv venv --python=python3 source venv/bin/activate pip install -e ".[test, dev, doc]" @@ -248,21 +250,16 @@ The tests can be run in the current environment by invoking make test -or in a clean environment via - -:: - - tox - FAQ --- How does the parsing of the dump work? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -`jamesmishra `__ has noticed that -SQL dumps from Wikipedia almost look like CSV. He provides some basic functions -to parse insert statements into tuples. We then use the Wikipedia SQL page +The SQL dumps consist of ``INSERT`` statements. The rows of these statements are +matched with regular expressions, which works for dumps that put all rows on one +line (older dumps) as well as for dumps that put every row on its own line (dumps +since 2025). We then use the Wikipedia SQL page dump to get the mapping between title and internal id, page props to get the Wikidata ID for a title and then the redirect dump in order to fill titles that are only redirects and do not have an entry in the page props table. diff --git a/setup.py b/setup.py index 946daef..a56124b 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ HOMEPAGE = "https://github.com/jcklie/wikimapper" EMAIL = "git@mrklie.com" AUTHOR = "Jan-Christoph Klie" -REQUIRES_PYTHON = ">=3.5.0" +REQUIRES_PYTHON = ">=3.6" install_requires=[] @@ -135,7 +135,6 @@ def run(self): "Intended Audience :: Science/Research", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Topic :: Internet :: WWW/HTTP :: Dynamic Content :: Wiki", diff --git a/tests/test_download.py b/tests/test_download.py index 4936ec4..b818479 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -1,6 +1,16 @@ +import http.server +import io import os +import socket +import socketserver +import threading +import urllib.request +import pytest + +import wikimapper.download from wikimapper import download_wikidumps +from wikimapper.download import USER_AGENT def test_download(tmpdir): @@ -19,3 +29,86 @@ def test_download(tmpdir): for e in files: statinfo = os.stat(os.path.join(path, e)) assert statinfo.st_size > 0, "[{0}] should be a non-empty file".format(e) + + +class _FakeResponse(io.BytesIO): + def __init__(self, content: bytes): + super().__init__(content) + self.headers = {"Content-Length": str(len(content))} + + +@pytest.mark.parametrize( + "mirror, expected_url", + [ + ("https://dumps.wikimedia.org/", "https://dumps.wikimedia.org/barwiki/latest/"), + ("https://dumps.wikimedia.org", "https://dumps.wikimedia.org/barwiki/latest/"), + ( + "https://mirror.org/wikimedia/dumps", + "https://mirror.org/wikimedia/dumps/barwiki/latest/", + ), + ( + "https://mirror.org/wikimedia/dumps/", + "https://mirror.org/wikimedia/dumps/barwiki/latest/", + ), + ], +) +def test_download_request(tmpdir, monkeypatch, mirror: str, expected_url: str): + requests = [] + + def urlopen(request, timeout): + requests.append(request) + return _FakeResponse(b"content") + + monkeypatch.setattr(urllib.request, "urlopen", urlopen) + + download_wikidumps("barwiki-latest", tmpdir.strpath, mirror) + + assert [r.full_url for r in requests] == [ + expected_url + "barwiki-latest-page.sql.gz", + expected_url + "barwiki-latest-page_props.sql.gz", + expected_url + "barwiki-latest-redirect.sql.gz", + ] + # Wikimedia blocks the default `Python-urllib` user agent + assert all(r.get_header("User-agent") == USER_AGENT for r in requests) + assert sorted(os.listdir(tmpdir.strpath)) == [ + "barwiki-latest-page.sql.gz", + "barwiki-latest-page_props.sql.gz", + "barwiki-latest-redirect.sql.gz", + ] + + +class _StalledServer(socketserver.ThreadingMixIn, http.server.HTTPServer): + daemon_threads = True + + +def test_download_times_out_on_stalled_server(tmpdir, monkeypatch): + released = threading.Event() + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + # Accept the request, then stop sending data, like the hang seen with Wikimedia + self.send_response(200) + self.send_header("Content-Length", "1000") + self.end_headers() + self.wfile.flush() + released.wait(30) + + def log_message(self, *args): + pass + + server = _StalledServer(("127.0.0.1", 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + monkeypatch.setattr(wikimapper.download, "_TIMEOUT", 1) + try: + with pytest.raises(socket.timeout): + download_wikidumps( + "barwiki-latest", tmpdir.strpath, "http://127.0.0.1:{0}".format(server.server_port) + ) + finally: + released.set() + server.shutdown() + server.server_close() + + # The incomplete download is not saved as a dump + assert os.listdir(tmpdir.strpath) == [] + diff --git a/tests/test_mapper.py b/tests/test_mapper.py index 10ceece..077ed41 100644 --- a/tests/test_mapper.py +++ b/tests/test_mapper.py @@ -143,7 +143,7 @@ def test_wikipedia_id_to_title(bavarian_wiki_mapper, wikipedia_id: int, expected ("xxxxxxxxxx", None), ] ) -def test_wikipedia_id_to_title(bavarian_wiki_mapper, title: str, expected: int): +def test_title_to_wikipedia_id(bavarian_wiki_mapper, title: str, expected: int): mapper = bavarian_wiki_mapper wikipedia_id = mapper.title_to_wikipedia_id(title) diff --git a/tests/test_processor.py b/tests/test_processor.py index aaa5274..e7a8d35 100644 --- a/tests/test_processor.py +++ b/tests/test_processor.py @@ -1,7 +1,10 @@ +import gzip import os import sqlite3 -from wikimapper import create_index +import pytest + +from wikimapper import WikiMapper, create_index def test_create_index(tmpdir, bavarian_wiki_dump): @@ -19,3 +22,106 @@ def test_create_index(tmpdir, bavarian_wiki_dump): results = c.fetchall() assert len(results) > 0 + + +# Rows as they appear in the dumps, columns according to the current MediaWiki schema +PAGE_ROWS = [ + "(1,0,'Brezn',0,0,0.1,'20240125183010','20240125183006',1,10,'wikitext',NULL)", + "(2,0,'Brezel',1,0,0.2,'20240125183010','20240125183006',2,10,'wikitext',NULL)", + "(3,1,'Brezn',0,0,0.3,'20240125183010','20240125183006',3,10,'wikitext',NULL)", + "(4,0,'D\\'_boarische_Woocha',0,0,0.4,'20240125183010','20240125183006',4,10,'wikitext',NULL)", + "(5,0,'Weißwurscht_(Bayern)',0,0,0.5,'20240125183010','20240125183006',5,10,'wikitext',NULL)", + "(7,0,'Brezen',1,0,0.7,'20240125183010','20240125183006',7,10,'wikitext',NULL)", +] +PAGE_PROPS_ROWS = [ + "(1,'page_image_free','Brezel.jpg',NULL)", + "(1,'wikibase_item','Q160525',NULL)", + "(3,'wikibase_item','Q999',NULL)", + "(4,'wikibase_item','Q20616808',NULL)", + "(5,'displaytitle','(1,\\'wikibase_item\\',\\'Q1\\')',NULL)", + "(5,'wikibase_item','Q1386052',NULL)", +] +REDIRECT_ROWS = [ + "(2,0,'Brezn','','')", + "(7,0,'Brezn','','')", + "(8,4,'Brezn','','')", +] + + +def _write_dump(path: str, table: str, rows, one_row_per_line: bool): + header = "-- MySQL dump\n\nCREATE TABLE `{0}` (\n `id` int(8)\n);\n".format(table) + insert = "INSERT INTO `{0}` VALUES".format(table) + if one_row_per_line: + # Layout used by dumps since 2025: one row per line + body = insert + "\n" + ",\n".join(rows) + ";\n" + else: + # Older layout: all rows on the same line as the INSERT statement, split over two statements + middle = len(rows) // 2 + body = "".join( + insert + " " + ",".join(part) + ";\n" for part in [rows[:middle], rows[middle:]] if part + ) + with gzip.open(path, "wt", encoding="utf-8") as f: + f.write(header + body + "UNLOCK TABLES;\n") + + +def _write_dumps(path: str, dumpname: str, one_row_per_line: bool): + for table, rows in [ + ("page", PAGE_ROWS), + ("page_props", PAGE_PROPS_ROWS), + ("redirect", REDIRECT_ROWS), + ]: + _write_dump( + os.path.join(path, "{0}-{1}.sql.gz".format(dumpname, table)), + table, + rows, + one_row_per_line, + ) + + +@pytest.mark.parametrize( + "one_row_per_line", [True, False], ids=["one row per line", "many rows per line"] +) +def test_create_index_parses_dump_layouts(tmpdir, one_row_per_line: bool): + path = tmpdir.strpath + _write_dumps(path, "testwiki-latest", one_row_per_line) + + path_to_db = create_index("testwiki-latest", path, os.path.join(path, "index.db")) + + with sqlite3.connect(path_to_db) as conn: + rows = conn.execute("SELECT * FROM mapping ORDER BY wikipedia_id").fetchall() + + assert rows == [ + (1, "Brezn", "Q160525"), + (2, "Brezel", "Q160525"), # Redirect gets the id of its target + (4, "D'_boarische_Woocha", "Q20616808"), # Escaped quote + (5, "Weißwurscht_(Bayern)", "Q1386052"), # Not confused by the row in the displaytitle + (7, "Brezen", "Q160525"), + ] + + mapper = WikiMapper(path_to_db) + assert set(mapper.id_to_titles("Q160525")) == {"Brezn", "Brezel", "Brezen"} + + +def test_create_index_fails_for_empty_dump(tmpdir): + path = tmpdir.strpath + _write_dumps(path, "testwiki-latest", True) + _write_dump(os.path.join(path, "testwiki-latest-page.sql.gz"), "page", [], True) + + with pytest.raises(ValueError): + create_index("testwiki-latest", path, os.path.join(path, "index.db")) + + +def test_create_index_fails_for_missing_dump(tmpdir): + path = tmpdir.strpath + _write_dumps(path, "testwiki-latest", True) + os.remove(os.path.join(path, "testwiki-latest-redirect.sql.gz")) + path_to_db = os.path.join(path, "index.db") + with open(path_to_db, "w") as f: + f.write("existing index") + + with pytest.raises(FileNotFoundError): + create_index("testwiki-latest", path, path_to_db) + + # An existing index is not deleted if the dump is incomplete + with open(path_to_db) as f: + assert f.read() == "existing index" diff --git a/wikimapper/download.py b/wikimapper/download.py index 28766fb..bca99e6 100644 --- a/wikimapper/download.py +++ b/wikimapper/download.py @@ -2,11 +2,22 @@ import os import shutil import sys -import urllib.parse +import tempfile +import urllib.error import urllib.request +from wikimapper.__version__ import __version__ + _logger = logging.getLogger(__name__) +# Wikimedia rejects requests with generic user agents like `Python-urllib/3.x` (HTTP 403), see +# https://meta.wikimedia.org/wiki/User-Agent_policy +USER_AGENT = "wikimapper/{0} (https://github.com/jcklie/wikimapper)".format(__version__) + +# Seconds to wait for the server to connect or to send the next block. Without it, a server that +# accepts the request but stops sending data makes the download wait forever. +_TIMEOUT = 60 + def _report_hook(count: int, block_size: int, total_size: int): percent = min(int(count * block_size * 100 / total_size), 100) @@ -21,7 +32,51 @@ def _download_file(url: str, target: str, overwrite: bool): _logger.info("Downloading [%s] to [%s]", url, target) - local_name, _ = urllib.request.urlretrieve(url, reporthook=_report_hook) + # Same as `urllib.request.urlretrieve`, which cannot set the user agent of a single request. + # Set the user agent on this request only, instead of installing a global opener, which would + # also change the requests of every other code in the same process + request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + # Open the connection; raises `HTTPError` for responses like 403 or 404. The timeout applies to + # every single network operation, not to the whole download, so large dumps are not affected. + with urllib.request.urlopen(request, timeout=_TIMEOUT) as response: + # Download into a temporary file first, so `target` only appears once the download is + # complete; `delete=False` keeps the file after closing so that it can be moved + with tempfile.NamedTemporaryFile(delete=False) as f: + # Remember the path of the temporary file for the move (or the error) below + local_name = f.name + # Read in blocks of 8 KiB, the same block size `urlretrieve` uses + block_size = 8192 + # Expected size in bytes from the server, or -1 if it did not send one + total_size = int(response.headers.get("Content-Length", -1)) + # Number of bytes received so far + read = 0 + # Number of blocks received so far, passed to the progress hook + count = 0 + # Report 0% before the first block arrives, as `urlretrieve` does + _report_hook(count, block_size, total_size) + # Copy the response into the temporary file block by block + while True: + # Read the next block; an empty result means the server has sent everything + block = response.read(block_size) + if not block: + break + # Append the block to the temporary file + f.write(block) + # Count the bytes that were actually received; the last block may be shorter + read += len(block) + # Count the block and update the progress shown on the console + count += 1 + _report_hook(count, block_size, total_size) + + # The connection ended before all announced bytes arrived: fail instead of keeping a + # truncated dump. The incomplete temporary file is never moved to `target`. + if read < total_size: + raise urllib.error.ContentTooShortError( + "retrieval incomplete: got only %i out of %i bytes" % (read, total_size), + # Same content as `urlretrieve` attaches: the temporary file and the response headers + (local_name, response.headers), + ) + sys.stdout.write("\r") shutil.move(local_name, target) @@ -44,7 +99,8 @@ def download_wikidumps( wiki_name, date = dumpname.split("-") - url = urllib.parse.urljoin(mirror, wiki_name) + "/" + date + "/" + # Not `urljoin`, as that drops the last path segment of mirrors like `https://host/dumps` + url = mirror.rstrip("/") + "/" + wiki_name + "/" + date + "/" pages_dump = dumpname + "-page.sql.gz" page_props_dump = dumpname + "-page_props.sql.gz" diff --git a/wikimapper/processor.py b/wikimapper/processor.py index 7c8661c..824c5c9 100644 --- a/wikimapper/processor.py +++ b/wikimapper/processor.py @@ -1,99 +1,239 @@ -""" Uses a Wikipedia dump to construct a mapping between Wikidata and Wikipedia. - Credit: Uses parts of https://github.com/jamesmishra/mysqldump-to-csv +r""" Uses a Wikipedia dump to construct a mapping between Wikidata and Wikipedia. + Credit: Originally used parts of https://github.com/jamesmishra/mysqldump-to-csv + +Goal +---- +Build an SQLite database (the "index") with a single table:: + + mapping(wikipedia_id int PRIMARY KEY, wikipedia_title text, wikidata_id text) + +It holds one row for every page in the main namespace of one Wikipedia (articles and redirects). +Example with real pages of the Bavarian Wikipedia (barwiki):: + + wikipedia_id | wikipedia_title | wikidata_id + -------------+-----------------+------------ + 1997 | Brezn | Q160525 the article about pretzels + 24100 | Brezel | Q160525 a redirect to Brezn, it gets the id of Brezn + +`WikiMapper` (mapper.py) answers all its questions with lookups in this table, e.g. +`title_to_id("Brezel")` returns "Q160525" and `id_to_titles("Q160525")` returns all titles of +the item, including its redirects. Two database indexes make these lookups fast: one on +`wikipedia_title` (unique) and one on `wikidata_id`. + +Input +----- +Three dumps from https://dumps.wikimedia.org/// (see download.py):: + + dump contains used for + --page.sql.gz every page: id, namespace, title wikipedia_id, wikipedia_title + --page_props.sql.gz properties, e.g. the Wikidata id wikidata_id + --redirect.sql.gz the target of each redirect wikidata_id of redirects + +Each one is a gzip-compressed MySQL dump: a `CREATE TABLE` statement, followed by `INSERT` +statements that hold the rows. Real rows from barwiki, shortened (`...`):: + + page: (1997,0,'Brezn',0,0,0.745707325237,'20260130182758',...,'wikitext',NULL) + (5740,1,'Brezn',0,0,0.533009202776,...) the talk page of Brezn (namespace 1) + (24100,0,'Brezel',1,1,0.993641585491,...) page_is_redirect = 1 + page_props: (1997,'page_image_free','Pretzel_01-14.jpg',NULL) + (1997,'wikibase_item','Q160525',NULL) the Wikidata id of page 1997 + redirect: (24100,0,'Brezn','','') page 24100 redirects to Brezn + +Strings are enclosed in `'`, and special characters inside them are escaped with a backslash: +the title D'_boarische_Woocha is written as 'D\'_boarische_Woocha'. + +The rows of an `INSERT` statement come in one of two layouts, and both are supported:: + + older dumps: INSERT INTO `page` VALUES (1,0,'Main_Page',...),(2,2,'Purodha',...),...; + dumps since 2025: INSERT INTO `page` VALUES + (1,0,'Main_Page',...), + (2,2,'Purodha',...), + ...; + +Steps +----- +1. `page`: insert id and title of every page in namespace 0 (articles and redirects). The other + namespaces hold talk pages, user pages, templates, ..., which have no place in the mapping. +2. `page_props`: set `wikidata_id` of every page that has a `wikibase_item` property. +3. `redirect`: redirects usually have no Wikidata id of their own, so they get the one of the page + they point to (Brezel gets Q160525 from Brezn). +4. Create the index on `wikidata_id`. + +For the English Wikipedia of October 2026 this results in 19,315,587 rows, 7,590,958 of them get +their Wikidata id in step 2 and most of the others in step 3 (19,280,223 rows have one in the end). """ -import csv -import ctypes as ct import gzip import logging import os +import re import sqlite3 +from typing import Iterator, Optional, Tuple _logger = logging.getLogger(__name__) - -def _is_insert(line): - """ - Returns true if the line begins a SQL insert statement. +# ------------------------------------------------------------------------------------------------ +# Row patterns +# +# The dumps are not parsed as SQL. Instead, regular expressions find the rows that are needed and +# extract only the leading columns that are needed. `re.findall` returns one tuple of the captured +# groups `( ... )` per row, scanning the line from left to right. This works the same way no matter +# whether a line holds one row (new layout) or thousands of rows (old layout). +# +# A match can never start inside of a string: inside a string, every quote is escaped (`\'`), but +# all patterns below need an unescaped quote right after the leading numbers. For example, the +# property value "(1,'wikibase_item','Q1')" is stored as '(1,\'wikibase_item\',\'Q1\')', which +# does not match `_PAGE_PROPS_ROW`. +# +# The patterns only match the rows that are needed (namespace 0, Wikidata ids), so the regex +# engine skips all other rows without handing them to Python, and they match strings in long runs +# (see `_STRING`). Both together reduce the time for parsing the English dumps from 193 s (matching +# every row and filtering in Python) to 55 s. +# ------------------------------------------------------------------------------------------------ + +# A quoted MySQL string, e.g. 'D\'_boarische_Woocha'; the group captures the content without the +# enclosing quotes, still escaped (`_unescape` removes the backslashes): +# +# ' the opening quote +# ( start of the captured content +# [^'\\]* a run of characters that are neither a quote nor a backslash +# (?:\\.[^'\\]*)* any number of: an escape (a backslash and the character after it, e.g. \') +# followed by the next run +# ) end of the captured content +# ' the closing quote: the first quote that is not escaped +# +# Matching whole runs at a time is much faster than deciding for every single character, as the +# equivalent `'((?:[^'\\]|\\.)*)'` would. +_STRING = r"'([^'\\]*(?:\\.[^'\\]*)*)'" + +# A row of the page table: (page_id, page_namespace, page_title, ...) +# https://www.mediawiki.org/wiki/Manual:Page_table +# +# \( the opening parenthesis of the row +# (\d+) page_id, captured +# ,0, page_namespace must be exactly 0, the main namespace (articles and their redirects), +# see https://www.mediawiki.org/wiki/Manual:Namespace. Rows of other namespaces, e.g. +# (5740,1,'Brezn',...), do not match and are skipped by the regex engine itself. +# _STRING page_title, captured. The remaining columns of the row are not needed. +# +# (1997,0,'Brezn',0,0,0.745707325237,...) -> ("1997", "Brezn") +_PAGE_ROW = re.compile(r"\((\d+),0," + _STRING) + +# A row of the page properties table: (pp_page, pp_propname, pp_value, pp_sortkey) +# https://www.mediawiki.org/wiki/Manual:Page_props_table +# +# \( the opening parenthesis of the row +# (\d+) pp_page, the page id, captured +# ,'wikibase_item', pp_propname must be exactly wikibase_item, the property that holds the +# Wikidata id. Other properties (page images, display titles, ...) do not +# match. They make up most of the table. +# _STRING pp_value, the Wikidata id, captured +# +# (1997,'wikibase_item','Q160525',NULL) -> ("1997", "Q160525") +_PAGE_PROPS_ROW = re.compile(r"\((\d+),'wikibase_item'," + _STRING) + +# A row of the redirect table: (rd_from, rd_namespace, rd_title, rd_interwiki, rd_fragment) +# https://www.mediawiki.org/wiki/Manual:Redirect_table +# +# \( the opening parenthesis of the row +# (\d+) rd_from, the page id of the redirect itself, captured +# ,0, rd_namespace, the namespace of the TARGET, must be 0: only redirects to articles +# _STRING rd_title, the title of the target page, captured +# +# Note that rd_namespace is the namespace of the target, not of the redirect. Redirects that are +# not in namespace 0 themselves are not in the mapping table, so updating them changes nothing. +# +# (24100,0,'Brezn','','') -> ("24100", "Brezn") +_REDIRECT_ROW = re.compile(r"\((\d+),0," + _STRING) + +# A backslash and the character it escapes, e.g. \' or \\ +_ESCAPE = re.compile(r"\\(.)") + + +def _unescape(value: str) -> str: + """Removes the MySQL escapes from a string captured by `_STRING`. + + Every backslash is dropped and the character after it is kept: `D\\'_boarische_Woocha` becomes + `D'_boarische_Woocha` and `C:\\\\` becomes `C:\\`. Titles and Wikidata ids almost never contain + an escape, so the (slower) substitution only runs if there is a backslash at all. """ - return line.startswith("INSERT INTO") + return _ESCAPE.sub(r"\1", value) if "\\" in value else value -def _get_values(line): - """ - Returns the portion of an INSERT statement containing values - """ - return line[line.find("` VALUES ") + 9 :] +def _data_lines(path: str, errors: str) -> Iterator[str]: + """Yields the lines of `path` that contain rows of an INSERT statement. + + Older dumps put all rows of an INSERT statement on the same line as `INSERT INTO ... VALUES`, + newer ones (2025 onwards) put every row on its own line. Both layouts are supported. + The dump is decompressed and read line by line, so even the dumps of the English Wikipedia + (several GB uncompressed) never have to fit into memory at once. + + Args: + path (str): Path to a `.sql.gz` dump. + errors (str): How invalid UTF-8 is handled, see `_parse_pages` and `_parse_wikidata_ids`. -def _values_sanity_check(values): """ - Ensures that values from the INSERT statement meet basic checks. + # "rt": decompress and decode to text. newline="\n": a line ends only at "\n", exactly as the + # dump writes it, without any translation of line endings. + with gzip.open(path, "rt", encoding="utf-8", errors=errors, newline="\n") as f: + for line in f: + # Rows are either on the `INSERT INTO ... VALUES ...` line itself (old layout) or on + # the lines after it, each starting with "(" (new layout). All other lines, like + # comments and the `CREATE TABLE` statement, are skipped. + if line.startswith("(") or line.startswith("INSERT INTO"): + yield line + + +def _parse_pages(path: str) -> Iterator[Tuple[int, str]]: + """Yields `(wikipedia_id, wikipedia_title)` of every page in namespace 0 of the page dump. + + The order of the values matches the parameters of the `INSERT` in `create_index`. + Example: (1997,0,'Brezn',...) -> (1997, "Brezn") """ - assert values - assert values[0] == "(" - # Assertions have not been raised - return True + # Titles are always valid UTF-8 in the page dump (all 66 million rows of the English dump of + # October 2026 are), so an invalid byte means the file is damaged: fail instead of silently + # creating wrong titles. + for line in _data_lines(path, errors="strict"): + for page_id, title in _PAGE_ROW.findall(line): + yield int(page_id), _unescape(title) -def _parse_values(values): +def _parse_wikidata_ids(path: str) -> Iterator[Tuple[str, int]]: + """Yields `(wikidata_id, wikipedia_id)` of every `wikibase_item` row of the page props dump. + + The order of the values matches the parameters of the `UPDATE` in `create_index`. + Example: (1997,'wikibase_item','Q160525',NULL) -> ("Q160525", 1997) """ - Given a file handle and the raw values from a MySQL INSERT - statement, write the equivalent CSV to the file + # The page properties contain binary values that are not valid UTF-8 (e.g. compressed + # template data, 74,333 lines in the English dump of October 2026). Invalid bytes are dropped, + # which does not affect the Wikidata ids: they are plain ASCII like Q160525. + for line in _data_lines(path, errors="ignore"): + for page_id, value in _PAGE_PROPS_ROW.findall(line): + yield _unescape(value), int(page_id) + + +def _parse_redirects(path: str) -> Iterator[Tuple[str, int]]: + """Yields `(target_title, redirect_id)` of every redirect to namespace 0 of the redirect dump. + + The order of the values matches the parameters of the redirect `UPDATE` in `create_index`. + Example: (24100,0,'Brezn','','') -> ("Brezn", 24100) """ - latest_row = [] - - reader = csv.reader( - [values], delimiter=",", doublequote=False, escapechar="\\", quotechar="'", strict=True - ) - - for reader_row in reader: - for column in reader_row: - # If our current string is empty... - if len(column) == 0 or column == "NULL": - latest_row.append(chr(0)) - continue - # If our string starts with an open paren - if column[0] == "(": - # Assume that this column does not begin - # a new row. - new_row = False - # If we've been filling out a row - if len(latest_row) > 0: - # Check if the previous entry ended in - # a close paren. If so, the row we've - # been filling out has been COMPLETED - # as: - # 1) the previous entry ended in a ) - # 2) the current entry starts with a ( - if latest_row[-1][-1] == ")": - # Remove the close paren. - latest_row[-1] = latest_row[-1][:-1] - new_row = True - # If we've found a new row, write it out - # and begin our new one - if new_row: - yield latest_row - latest_row = [] - # If we're beginning a new row, eliminate the - # opening parentheses. - if len(latest_row) == 0: - column = column[1:] - # Add our column to the row we're working on. - latest_row.append(column) - # At the end of an INSERT statement, we'll - # have the semicolon. - # Make sure to remove the semicolon and - # the close paren. - if latest_row[-1][-2:] == ");": - latest_row[-1] = latest_row[-1][:-2] - yield latest_row - - -def create_index(dumpname: str, path_to_dumps: str, path_to_db: str = None) -> str: + # Invalid UTF-8 is ignored here as well, as it has been done since + # https://github.com/jcklie/wikimapper/issues/12 + for line in _data_lines(path, errors="ignore"): + for source_id, target_title in _REDIRECT_ROW.findall(line): + yield _unescape(target_title), int(source_id) + + +def create_index(dumpname: str, path_to_dumps: str, path_to_db: Optional[str] = None) -> str: """Creates an index mapping Wikipedia page titles to Wikidata IDs and vice versa. This requires a previously downloaded dump `dumpname` in `path_to_dumps`. + The index is the SQLite table `mapping(wikipedia_id, wikipedia_title, wikidata_id)` described + at the top of this module. It is built in four steps: pages, Wikidata ids, redirects, indexes. + An existing index at `path_to_db` is replaced. + Args: dumpname(str): Name of the Wikipedia SQL dump that should be used for creating an index. path_to_dumps(str): Folder in which the dump has been downloaded to. @@ -102,20 +242,27 @@ def create_index(dumpname: str, path_to_dumps: str, path_to_db: str = None) -> s Returns: str: The path to the created database. + Raises: + FileNotFoundError: If one of the three dumps is missing. The existing index is kept. + ValueError: If not a single page could be parsed, e.g. because the dump format changed. + """ if path_to_db is None: path_to_db = "index_{0}.db".format(dumpname) _logger.info("Creating index for [%s] in [%s]", dumpname, path_to_db) - wiki_name, date = dumpname.split("-") - + # The three dumps as named by download.py, e.g. data/barwiki-latest-page.sql.gz pages_dump = os.path.join(path_to_dumps, dumpname + "-page.sql.gz") page_props_dump = os.path.join(path_to_dumps, dumpname + "-page_props.sql.gz") redirects_dump = os.path.join(path_to_dumps, dumpname + "-redirect.sql.gz") - # https://stackoverflow.com/a/54517228 - csv.field_size_limit(int(ct.c_ulong(-1).value // 2)) + # Check all dumps before touching the existing index. Otherwise a missing redirect dump would + # only be noticed after the old index has been deleted and half of the new one has been built, + # which leaves an index without redirects behind. + for dump in [pages_dump, page_props_dump, redirects_dump]: + if not os.path.isfile(dump): + raise FileNotFoundError("Dump [{0}] does not exist".format(dump)) # (Re)Create the database file try: @@ -123,8 +270,14 @@ def create_index(dumpname: str, path_to_dumps: str, path_to_db: str = None) -> s except FileNotFoundError: pass + # With an isolation level, sqlite3 starts a transaction before the first change and every + # `with conn:` block below commits its changes at the end (or rolls them back on an error). + # Writing millions of rows in one transaction is far faster than committing every single row. + # EXCLUSIVE locks the database for other connections while it is written. conn = sqlite3.connect(path_to_db, isolation_level="EXCLUSIVE") + # wikipedia_id is the PRIMARY KEY: page ids are unique, and the updates below look rows up by + # it. wikidata_id stays NULL until a Wikidata id is known for the page. with conn: conn.execute( """CREATE TABLE mapping ( @@ -133,98 +286,74 @@ def create_index(dumpname: str, path_to_dumps: str, path_to_db: str = None) -> s wikidata_id text)""" ) - c = conn.cursor() - - # Parse the Wikipedia page dump; extract page id and page title from the sql - # https://www.mediawiki.org/wiki/Manual:Page_table + # Step 1: Parse the Wikipedia page dump; extract page id and page title from the sql. + # `executemany` takes the rows directly from the generator, one after another, so the parsed + # rows are never all in memory. `rowcount` is the number of inserted rows. _logger.info("Parsing pages dump") - with gzip.open(pages_dump, "rt", encoding="utf-8", newline="\n") as f: - for line in f: - # Look for an INSERT statement and parse it. - if not _is_insert(line): - continue - - values = _get_values(line) - - for v in _parse_values(values): - # Filter the namespace; only use real articles - # https://www.mediawiki.org/wiki/Manual:Namespace - if v[1] == "0": - c.execute( - "INSERT INTO mapping (wikipedia_id, wikipedia_title) VALUES (?, ?)", - (v[0], v[2]), - ) - - conn.commit() + with conn: + count = conn.executemany( + "INSERT INTO mapping (wikipedia_id, wikipedia_title) VALUES (?, ?)", + _parse_pages(pages_dump), + ).rowcount + _logger.info("Inserted [%d] pages", count) + + # Every Wikipedia has pages, so no rows at all means that the rows were not recognized, e.g. + # because the dump format changed. Fail loudly instead of writing an empty index. + if count == 0: + conn.close() + raise ValueError( + "No pages could be parsed from [{0}], the dump format may have changed".format( + pages_dump + ) + ) # We create this index here as all titles have been inserted now. # Doing it earlier would recreate the index on every title insert. + # The redirect step below looks up every redirect target by its title, which needs this index, + # and it is used by `WikiMapper.title_to_id`. UNIQUE: a title exists only once per namespace. _logger.info("Creating database index on 'wikipedia_title'") - conn.execute("""CREATE UNIQUE INDEX idx_wikipedia_title ON mapping(wikipedia_title);""") - conn.commit() + with conn: + conn.execute("""CREATE UNIQUE INDEX idx_wikipedia_title ON mapping(wikipedia_title);""") - # Parse the Wikipedia page property dump; extract page id and Wikidata id from the sql - # https://www.mediawiki.org/wiki/Manual:Page_props_table/en + # Step 2: Parse the Wikipedia page property dump; extract page id and Wikidata id from the sql. + # The page props of other namespaces (categories, templates, ...) match no row of the mapping + # table, so they change nothing; `rowcount` only counts the rows that were actually updated. _logger.info("Parsing page properties dump") - with gzip.open(page_props_dump, "r") as f: - for line in f: - line = line.decode("utf-8", "ignore") - - # Look for an INSERT statement and parse it. - if not _is_insert(line): - continue - - values = _get_values(line) - for v in _parse_values(values): - # The page property table contains many properties, we only care about the Wikidata id - if v[1] == "wikibase_item": - c.execute( - "UPDATE mapping SET wikidata_id = ? WHERE wikipedia_id = ?", (v[2], v[0]) - ) - conn.commit() - - # Parse the Wikipedia redirect dump; fill in missing Wikidata ids - # https://www.mediawiki.org/wiki/Manual:Redirect_table + with conn: + count = conn.executemany( + "UPDATE mapping SET wikidata_id = ? WHERE wikipedia_id = ?", + _parse_wikidata_ids(page_props_dump), + ).rowcount + _logger.info("Mapped [%d] pages to Wikidata", count) + + # Step 3: Parse the Wikipedia redirect dump; give redirects the Wikidata id of their target. + # For every redirect (target_title, redirect_id), e.g. ("Brezn", 24100): + # - the subquery looks up the Wikidata id of the target page by its title (via the title + # index), e.g. Q160525 for Brezn; + # - COALESCE takes that id, or keeps the current one of the redirect if the subquery returns + # NULL, i.e. the target does not exist in namespace 0 or has no Wikidata id itself; + # - WHERE selects the redirect itself. Redirects that are not in the mapping table (not in + # namespace 0) match no row and are skipped. _logger.info("Parsing redirects dump") - with gzip.open(redirects_dump, "r") as f: - for line in f: - line = line.decode("utf-8", "ignore") - # Look for an INSERT statement and parse it. - if not _is_insert(line): - continue - - values = _get_values(line) - - for v in _parse_values(values): - source_wikipedia_id = v[0] - target_title = v[2] - namespace = v[1] - - # We only care about main articles - if namespace != "0": - continue - - c.execute( - "SELECT wikidata_id FROM mapping WHERE wikipedia_title = ?", (target_title,) - ) - result = c.fetchone() - - if result is None or result[0] is None: - continue - - wikidata_id = result[0] - c.execute( - "UPDATE mapping SET wikidata_id = ? WHERE wikipedia_id = ?", - (wikidata_id, source_wikipedia_id), - ) - - conn.commit() + with conn: + conn.executemany( + """UPDATE mapping SET wikidata_id = COALESCE( + (SELECT target.wikidata_id FROM mapping AS target WHERE target.wikipedia_title = ?), + wikidata_id + ) WHERE wikipedia_id = ?""", + _parse_redirects(redirects_dump), + ) + # Step 4: The index for `WikiMapper.id_to_titles` and `id_to_wikipedia_ids`. Created after all + # Wikidata ids are set, which is faster than updating the index for every single change. _logger.info("Creating database index on 'wikidata_id'") - conn.execute("""CREATE INDEX idx_wikidata_id ON mapping(wikidata_id);""") - conn.commit() + with conn: + conn.execute("""CREATE INDEX idx_wikidata_id ON mapping(wikidata_id);""") + + # Summary for the log: count(wikidata_id) counts the rows whose Wikidata id is not NULL + total, mapped = conn.execute("SELECT count(*), count(wikidata_id) FROM mapping").fetchone() + _logger.info("Index contains [%d] pages, [%d] of them mapped to Wikidata", total, mapped) - c.close() conn.close() return path_to_db