Support the current Wikimedia dumps: fix downloads (HTTP 403) and parse both the old and the new SQL dump format

.github/workflows/run_tests.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
-        python-version: [3.7, 3.8, 3.9]
+        python-version: [3.8, 3.9]
 
     steps:
     - uses: actions/checkout@v3

Makefile

@@ -9,6 +9,3 @@ isort:
 	isort --profile black wikimapper/ tests/
 
 format: black isort
-
-html:
-	cd docs && make html

README.rst

@@ -159,7 +159,9 @@ Create your own index
 While some indices are precomupted, it is sometimes useful to create your own. The
 following section describes the steps need. Regarding creation speed: The index creation
 code works, but is not optimized. It takes around 10 minutes on my Notebook (T480s)
-to create it for English Wikipedia if the data is already downloaded.
+to create it for English Wikipedia if the data is already downloaded. With the English
+dump of October 2026 (19.3 million pages), it takes around 8 minutes on an ASUS ROG
+Zephyrus G14 (GA403UV), most of which is spent on resolving the redirects.
 
 **1. Download the data**
 
@@ -223,7 +225,7 @@ line.
         url2id              Map a Wikipedia URL to a Wikidata ID.
         id2titles           Map a Wikidata ID to one or more Wikipedia titles.
 
-    optional arguments:
+    options:
       -h, --help            show this help message and exit
       --version             show program's version number and exit
 
@@ -238,7 +240,7 @@ created and activated by
 
 ::
 
-    virtualenv venv --python=python3 --no-site-packages
+    virtualenv venv --python=python3
     source venv/bin/activate
     pip install -e ".[test, dev, doc]"
 
@@ -248,21 +250,16 @@ The tests can be run in the current environment by invoking
 
     make test
 
-or in a clean environment via
-
-::
-
-    tox
-
 FAQ
 ---
 
 How does the parsing of the dump work?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-`jamesmishra <https://github.com/jamesmishra/mysqldump-to-csv>`__ has noticed that
-SQL dumps from Wikipedia almost look like CSV. He provides some basic functions
-to parse insert statements into tuples. We then use the Wikipedia SQL page
+The SQL dumps consist of ``INSERT`` statements. The rows of these statements are
+matched with regular expressions, which works for dumps that put all rows on one
+line (older dumps) as well as for dumps that put every row on its own line (dumps
+since 2025). We then use the Wikipedia SQL page
 dump to get the mapping between title and internal id, page props to get
 the Wikidata ID for a title and then the redirect dump in order to fill
 titles that are only redirects and do not have an entry in the page props table.

setup.py

@@ -17,7 +17,7 @@
 HOMEPAGE = "https://github.com/jcklie/wikimapper"
 EMAIL = "git@mrklie.com"
 AUTHOR = "Jan-Christoph Klie"
-REQUIRES_PYTHON = ">=3.5.0"
+REQUIRES_PYTHON = ">=3.6"
 
 install_requires=[]
 
@@ -135,7 +135,6 @@ def run(self):
         "Intended Audience :: Science/Research",
         "License :: OSI Approved :: Apache Software License",
         "Programming Language :: Python :: 3 :: Only",
-        "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Topic :: Internet :: WWW/HTTP :: Dynamic Content :: Wiki",

tests/test_download.py

@@ -1,6 +1,16 @@
+import http.server
+import io
 import os
+import socket
+import socketserver
+import threading
+import urllib.request
 
+import pytest
+
+import wikimapper.download
 from wikimapper import download_wikidumps
+from wikimapper.download import USER_AGENT
 
 
 def test_download(tmpdir):
@@ -19,3 +29,86 @@ def test_download(tmpdir):
     for e in files:
         statinfo = os.stat(os.path.join(path, e))
         assert statinfo.st_size > 0, "[{0}] should be a non-empty file".format(e)
+
+
+class _FakeResponse(io.BytesIO):
+    def __init__(self, content: bytes):
+        super().__init__(content)
+        self.headers = {"Content-Length": str(len(content))}
+
+
+@pytest.mark.parametrize(
+    "mirror, expected_url",
+    [
+        ("https://dumps.wikimedia.org/", "https://dumps.wikimedia.org/barwiki/latest/"),
+        ("https://dumps.wikimedia.org", "https://dumps.wikimedia.org/barwiki/latest/"),
+        (
+            "https://mirror.org/wikimedia/dumps",
+            "https://mirror.org/wikimedia/dumps/barwiki/latest/",
+        ),
+        (
+            "https://mirror.org/wikimedia/dumps/",
+            "https://mirror.org/wikimedia/dumps/barwiki/latest/",
+        ),
+    ],
+)
+def test_download_request(tmpdir, monkeypatch, mirror: str, expected_url: str):
+    requests = []
+
+    def urlopen(request, timeout):
+        requests.append(request)
+        return _FakeResponse(b"content")
+
+    monkeypatch.setattr(urllib.request, "urlopen", urlopen)
+
+    download_wikidumps("barwiki-latest", tmpdir.strpath, mirror)
+
+    assert [r.full_url for r in requests] == [
+        expected_url + "barwiki-latest-page.sql.gz",
+        expected_url + "barwiki-latest-page_props.sql.gz",
+        expected_url + "barwiki-latest-redirect.sql.gz",
+    ]
+    # Wikimedia blocks the default `Python-urllib` user agent
+    assert all(r.get_header("User-agent") == USER_AGENT for r in requests)
+    assert sorted(os.listdir(tmpdir.strpath)) == [
+        "barwiki-latest-page.sql.gz",
+        "barwiki-latest-page_props.sql.gz",
+        "barwiki-latest-redirect.sql.gz",
+    ]
+
+
+class _StalledServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
+    daemon_threads = True
+
+
+def test_download_times_out_on_stalled_server(tmpdir, monkeypatch):
+    released = threading.Event()
+
+    class Handler(http.server.BaseHTTPRequestHandler):
+        def do_GET(self):
+            # Accept the request, then stop sending data, like the hang seen with Wikimedia
+            self.send_response(200)
+            self.send_header("Content-Length", "1000")
+            self.end_headers()
+            self.wfile.flush()
+            released.wait(30)
+
+        def log_message(self, *args):
+            pass
+
+    server = _StalledServer(("127.0.0.1", 0), Handler)
+    threading.Thread(target=server.serve_forever, daemon=True).start()
+    monkeypatch.setattr(wikimapper.download, "_TIMEOUT", 1)
+    try:
+        with pytest.raises(socket.timeout):
+            download_wikidumps(
+                "barwiki-latest", tmpdir.strpath, "http://127.0.0.1:{0}".format(server.server_port)
+            )
+    finally:
+        released.set()
+        server.shutdown()
+        server.server_close()
+
+    # The incomplete download is not saved as a dump
+    assert os.listdir(tmpdir.strpath) == []
+

tests/test_mapper.py

@@ -143,7 +143,7 @@ def test_wikipedia_id_to_title(bavarian_wiki_mapper, wikipedia_id: int, expected
         ("xxxxxxxxxx", None),
     ]
 )
-def test_wikipedia_id_to_title(bavarian_wiki_mapper, title: str, expected: int):
+def test_title_to_wikipedia_id(bavarian_wiki_mapper, title: str, expected: int):
     mapper = bavarian_wiki_mapper
 
     wikipedia_id = mapper.title_to_wikipedia_id(title)

tests/test_processor.py

@@ -1,7 +1,10 @@
+import gzip
 import os
 import sqlite3
 
-from wikimapper import create_index
+import pytest
+
+from wikimapper import WikiMapper, create_index
 
 
 def test_create_index(tmpdir, bavarian_wiki_dump):
@@ -19,3 +22,106 @@ def test_create_index(tmpdir, bavarian_wiki_dump):
         results = c.fetchall()
 
     assert len(results) > 0
+
+
+# Rows as they appear in the dumps, columns according to the current MediaWiki schema
+PAGE_ROWS = [
+    "(1,0,'Brezn',0,0,0.1,'20240125183010','20240125183006',1,10,'wikitext',NULL)",
+    "(2,0,'Brezel',1,0,0.2,'20240125183010','20240125183006',2,10,'wikitext',NULL)",
+    "(3,1,'Brezn',0,0,0.3,'20240125183010','20240125183006',3,10,'wikitext',NULL)",
+    "(4,0,'D\\'_boarische_Woocha',0,0,0.4,'20240125183010','20240125183006',4,10,'wikitext',NULL)",
+    "(5,0,'Weißwurscht_(Bayern)',0,0,0.5,'20240125183010','20240125183006',5,10,'wikitext',NULL)",
+    "(7,0,'Brezen',1,0,0.7,'20240125183010','20240125183006',7,10,'wikitext',NULL)",
+]
+PAGE_PROPS_ROWS = [
+    "(1,'page_image_free','Brezel.jpg',NULL)",
+    "(1,'wikibase_item','Q160525',NULL)",
+    "(3,'wikibase_item','Q999',NULL)",
+    "(4,'wikibase_item','Q20616808',NULL)",
+    "(5,'displaytitle','(1,\\'wikibase_item\\',\\'Q1\\')',NULL)",
+    "(5,'wikibase_item','Q1386052',NULL)",
+]
+REDIRECT_ROWS = [
+    "(2,0,'Brezn','','')",
+    "(7,0,'Brezn','','')",
+    "(8,4,'Brezn','','')",
+]
+
+
+def _write_dump(path: str, table: str, rows, one_row_per_line: bool):
+    header = "-- MySQL dump\n\nCREATE TABLE `{0}` (\n  `id` int(8)\n);\n".format(table)
+    insert = "INSERT INTO `{0}` VALUES".format(table)
+    if one_row_per_line:
+        # Layout used by dumps since 2025: one row per line
+        body = insert + "\n" + ",\n".join(rows) + ";\n"
+    else:
+        # Older layout: all rows on the same line as the INSERT statement, split over two statements
+        middle = len(rows) // 2
+        body = "".join(
+            insert + " " + ",".join(part) + ";\n" for part in [rows[:middle], rows[middle:]] if part
+        )
+    with gzip.open(path, "wt", encoding="utf-8") as f:
+        f.write(header + body + "UNLOCK TABLES;\n")
+
+
+def _write_dumps(path: str, dumpname: str, one_row_per_line: bool):
+    for table, rows in [
+        ("page", PAGE_ROWS),
+        ("page_props", PAGE_PROPS_ROWS),
+        ("redirect", REDIRECT_ROWS),
+    ]:
+        _write_dump(
+            os.path.join(path, "{0}-{1}.sql.gz".format(dumpname, table)),
+            table,
+            rows,
+            one_row_per_line,
+        )
+
+
+@pytest.mark.parametrize(
+    "one_row_per_line", [True, False], ids=["one row per line", "many rows per line"]
+)
+def test_create_index_parses_dump_layouts(tmpdir, one_row_per_line: bool):
+    path = tmpdir.strpath
+    _write_dumps(path, "testwiki-latest", one_row_per_line)
+
+    path_to_db = create_index("testwiki-latest", path, os.path.join(path, "index.db"))
+
+    with sqlite3.connect(path_to_db) as conn:
+        rows = conn.execute("SELECT * FROM mapping ORDER BY wikipedia_id").fetchall()
+
+    assert rows == [
+        (1, "Brezn", "Q160525"),
+        (2, "Brezel", "Q160525"),  # Redirect gets the id of its target
+        (4, "D'_boarische_Woocha", "Q20616808"),  # Escaped quote
+        (5, "Weißwurscht_(Bayern)", "Q1386052"),  # Not confused by the row in the displaytitle
+        (7, "Brezen", "Q160525"),
+    ]
+
+    mapper = WikiMapper(path_to_db)
+    assert set(mapper.id_to_titles("Q160525")) == {"Brezn", "Brezel", "Brezen"}
+
+
+def test_create_index_fails_for_empty_dump(tmpdir):
+    path = tmpdir.strpath
+    _write_dumps(path, "testwiki-latest", True)
+    _write_dump(os.path.join(path, "testwiki-latest-page.sql.gz"), "page", [], True)
+
+    with pytest.raises(ValueError):
+        create_index("testwiki-latest", path, os.path.join(path, "index.db"))
+
+
+def test_create_index_fails_for_missing_dump(tmpdir):
+    path = tmpdir.strpath
+    _write_dumps(path, "testwiki-latest", True)
+    os.remove(os.path.join(path, "testwiki-latest-redirect.sql.gz"))
+    path_to_db = os.path.join(path, "index.db")
+    with open(path_to_db, "w") as f:
+        f.write("existing index")
+
+    with pytest.raises(FileNotFoundError):
+        create_index("testwiki-latest", path, path_to_db)
+
+    # An existing index is not deleted if the dump is incomplete
+    with open(path_to_db) as f:
+        assert f.read() == "existing index"