diff --git a/CHANGELOG.md b/CHANGELOG.md index 2deb94b..a825f4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes are listed here. Format based on [Keep a Changelog](https://keepachangelog.com/). +## Unreleased + +### Added +- `GovSource` (`gov`): central-government tenders from mr.gov.il, read from the portal's HTML search. It is part of + the default sources. +- Tests use a fixed clock, so open/closed results no longer depend on the date the suite runs. +- `HttpClient.get_text` for sources that serve HTML. + ## 0.1.0 (2026-11-05) ### Added diff --git a/README.md b/README.md index 6ae9e31..6d27265 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,9 @@ Israeli public tenders (מכרזים) in Python. One `Tender` object, whichever site it came from. -Sources today: Israel Land Authority land tenders and Beer Sheva municipal tenders. +Sources today: Israel Land Authority land tenders, Beer Sheva municipal tenders, and central-government tenders +from mr.gov.il. The first search reads about 25 pages from mr.gov.il, so it takes around 20 seconds. After that the +responses are cached for an hour. > Unofficial client. It can lag or miss tenders, and parsing can break when a source changes. Confirm deadlines on > the publisher's page before you act. Not affiliated with the Israel Land Authority or any municipality. diff --git a/docs/images/pr-gov-check.png b/docs/images/pr-gov-check.png new file mode 100644 index 0000000..4f2d158 Binary files /dev/null and b/docs/images/pr-gov-check.png differ diff --git a/docs/images/pr-gov-cli.png b/docs/images/pr-gov-cli.png new file mode 100644 index 0000000..edb080c Binary files /dev/null and b/docs/images/pr-gov-cli.png differ diff --git a/docs/images/pr-gov-tests.png b/docs/images/pr-gov-tests.png new file mode 100644 index 0000000..518e6f8 Binary files /dev/null and b/docs/images/pr-gov-tests.png differ diff --git a/docs/model.md b/docs/model.md index 707faab..db6bedf 100644 --- a/docs/model.md +++ b/docs/model.md @@ -48,6 +48,12 @@ Keys are the members of `michraz.RmiExtra`, a string enum: `tender.extra[RmiExtr | `khal_yaad_rashi` | `KhalYaadRashi`, kept as is. Observed values are 1, 2 and null, so it is a code and not a bid amount. | | `mekuvan` | `Mekuvan`, kept as is. Its meaning is unconfirmed. | +### Government `extra` + +Keys are the members of `michraz.GovExtra`: `type` (for example `מכרז פומבי`), `status` (the portal's status, for +example `עודכן`) and `raw`. `source_id` is the portal's publication number, `number` is its procedure number, and +`documents` is empty because detail pages are not fetched. + ## `Document` | Field | Type | Meaning | diff --git a/docs/sources.md b/docs/sources.md index c9bfca2..61c4bfc 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -75,6 +75,25 @@ correct and not confirmed. - 402 rows have a winner, 51 have no files. `קבצים` is HTML that is not always wrapped in `

`, and some links encode the URL as `http://...`. It is read with `html.parser`. +## Central government, `gov` + +- Portal: the Government Procurement Administration's tender site, `https://mr.gov.il/ilgstorefront/he/search/`. + It has no JSON API. The search page is server-rendered HTML, 20 results per page, `page` starts at 0. +- Attribution: `Government Procurement Administration (מינהל הרכש הממשלתי), mr.gov.il`. The license of the data was + not checked. +- Query: `q=:updateDate:archive:false:status:`, `s=TENDER`. The portal reports 8 835 tenders in total, 7 654 + of them with status `חלף מועד הגשה` (submission date passed). michraz reads only the statuses `פורסם`, `עודכן` and + `חדש`. +- Verified on 2026-11-05: those three statuses returned 123, 256 and 40 tenders. Reading all of them gave 419 + tenders and 0 parse errors in about 21 seconds. 183 had a deadline still ahead and 14 had no deadline. +- Fields read: title, type (`מכרז פומבי`), publisher, publication number (used as `source_id`), procedure number + (used as `number`), publication date, update date, deadline (`dd/mm/yyyy HH:MM`) and status. +- Not verified: statuses `בעדכון` (32) and `עתידי` (15) are not read, because it is unclear whether they accept + offers. Detail pages and attached documents are not fetched, so `documents` is empty. +- `tests/fixtures/gov_search.html` is five real result blocks copied from a live page. The embedded images were + removed and the count sentence was edited to say 5 so that it matches the excerpt. +- The site has no `robots.txt` (the request returns an empty body). + ## Robots and terms Checked 2026-11-04 from the development machine with the michraz User-Agent. diff --git a/src/michraz/__init__.py b/src/michraz/__init__.py index 7ea4903..40f835c 100644 --- a/src/michraz/__init__.py +++ b/src/michraz/__init__.py @@ -12,12 +12,23 @@ from michraz.errors import MichrazError, SourceFormatError, SourceUnavailableError from michraz.export import to_csv, to_dataframe, to_dicts from michraz.models import Document, Tender -from michraz.sources import BeerShevaExtra, BeerShevaSource, RmiExtra, RmiSource, Source, SourceName +from michraz.sources import ( + BeerShevaExtra, + BeerShevaSource, + GovExtra, + GovSource, + RmiExtra, + RmiSource, + Source, + SourceName, +) __all__ = [ "BeerShevaExtra", "BeerShevaSource", "Document", + "GovExtra", + "GovSource", "Michraz", "MichrazError", "RmiExtra", diff --git a/src/michraz/_http.py b/src/michraz/_http.py index cdfd3c8..55d5442 100644 --- a/src/michraz/_http.py +++ b/src/michraz/_http.py @@ -135,7 +135,7 @@ class HttpClient(Closeable): min_interval: Minimum seconds between requests to the same host. Must be at least 0.5. client: An ``httpx.Client`` to use. By default one is created and owned by this object. clock: Monotonic clock in seconds. Replace it in tests. - sleep: Sleep function. Replace it in tests so they do not wait. + sleep: Sleep function. Defaults to ``time.sleep``, looked up when the client is created. wall_clock: Wall-clock time in seconds since the epoch, used for cache age. Raises: @@ -150,14 +150,14 @@ def __init__( min_interval: float = DEFAULT_MIN_INTERVAL, client: httpx.Client | None = None, clock: Callable[[], float] = time.monotonic, - sleep: Callable[[float], None] = time.sleep, + sleep: Callable[[float], None] | None = None, wall_clock: Callable[[], float] = time.time, ) -> None: if min_interval < MIN_INTERVAL_FLOOR: raise ValueError(f"min_interval must be at least {MIN_INTERVAL_FLOOR} seconds") self.min_interval = min_interval self._clock = clock - self._sleep = sleep + self._sleep = sleep or time.sleep self._wall_clock = wall_clock self._owns_client = client is None self._client = client or httpx.Client( @@ -207,16 +207,32 @@ def cached_at( def _full_url(url: str, params: Mapping[str, Any] | None) -> str: return str(httpx.URL(url, params=params)) if params else url + def get_text(self, source: str, url: str, params: Mapping[str, Any] | None = None) -> str: + """Fetches ``url`` with GET and returns the body as text. Cached and rate-limited like JSON. + + Raises: + SourceUnavailableError: If the host cannot be reached or answers with an error status. + """ + text: str = self._fetch(source, HttpMethod.GET, self._full_url(url, params), None, lambda r: r.text) + return text + def _json(self, source: str, method: HttpMethod, url: str, payload: Any) -> Any: + def decode(response: httpx.Response) -> Any: + try: + return response.json() + except ValueError as exc: + raise SourceFormatError(source, f"response is not JSON: {_truncate(response.text)!r}") from exc + + return self._fetch(source, method, url, payload, decode) + + def _fetch( + self, source: str, method: HttpMethod, url: str, payload: Any, decode: Callable[[httpx.Response], Any] + ) -> Any: key = _request_key(method, url, payload) if self._cache and (hit := self._cache.fresh(key)) is not None: logger.debug("cache hit for %s", url) return hit[1] - response = self._send(source, method, url, payload) - try: - data = response.json() - except ValueError as exc: - raise SourceFormatError(source, f"response is not JSON: {_truncate(response.text)!r}") from exc + data = decode(self._send(source, method, url, payload)) if self._cache: self._cache.write(key, data) return data diff --git a/src/michraz/client.py b/src/michraz/client.py index d26fc15..05f7a56 100644 --- a/src/michraz/client.py +++ b/src/michraz/client.py @@ -16,7 +16,7 @@ ) from michraz.filters import TenderFilter, sort_by_deadline from michraz.models import Tender -from michraz.sources import BaseSource, BeerShevaSource, RmiSource, Source +from michraz.sources import BaseSource, BeerShevaSource, GovSource, RmiSource, Source logger = logging.getLogger(__name__) @@ -30,7 +30,7 @@ class Michraz(Closeable): tenders = m.search(keyword="גינון", open_only=True) Args: - sources: Sources to search. Defaults to RMI and Beer Sheva. + sources: Sources to search. Defaults to RMI, Beer Sheva and the central government. cache_dir: Directory for cached responses. Defaults to ``~/.cache/michraz``. None disables the cache. cache_ttl: How long a cached response stays valid. min_interval: Minimum seconds between requests to the same host. Must be at least 0.5. @@ -51,7 +51,9 @@ def __init__( min_interval: float = DEFAULT_MIN_INTERVAL, ) -> None: self._http = HttpClient(cache_dir=cache_dir, cache_ttl=cache_ttl, min_interval=min_interval) - self.sources: tuple[Source, ...] = tuple(sources) if sources is not None else (RmiSource(), BeerShevaSource()) + self.sources: tuple[Source, ...] = ( + tuple(sources) if sources is not None else (RmiSource(), BeerShevaSource(), GovSource()) + ) names = [s.name for s in self.sources] if len(set(names)) != len(names): raise ValueError(f"source names must be unique, got {names}") diff --git a/src/michraz/sources/__init__.py b/src/michraz/sources/__init__.py index afed88e..24f234a 100644 --- a/src/michraz/sources/__init__.py +++ b/src/michraz/sources/__init__.py @@ -2,6 +2,17 @@ from michraz.sources.base import BaseSource, Source, SourceName from michraz.sources.beersheva import BeerShevaExtra, BeerShevaSource +from michraz.sources.gov import GovExtra, GovSource from michraz.sources.rmi import RmiExtra, RmiSource -__all__ = ["BaseSource", "BeerShevaExtra", "BeerShevaSource", "RmiExtra", "RmiSource", "Source", "SourceName"] +__all__ = [ + "BaseSource", + "BeerShevaExtra", + "BeerShevaSource", + "GovExtra", + "GovSource", + "RmiExtra", + "RmiSource", + "Source", + "SourceName", +] diff --git a/src/michraz/sources/base.py b/src/michraz/sources/base.py index d423c7e..1a00d85 100644 --- a/src/michraz/sources/base.py +++ b/src/michraz/sources/base.py @@ -22,6 +22,7 @@ class SourceName(StrEnum): RMI = "rmi" BEERSHEVA = "beersheva" + GOV = "gov" @runtime_checkable diff --git a/src/michraz/sources/gov.py b/src/michraz/sources/gov.py new file mode 100644 index 0000000..29d302a --- /dev/null +++ b/src/michraz/sources/gov.py @@ -0,0 +1,219 @@ +"""Central government tenders from the Government Procurement Administration portal (mr.gov.il). + +The portal has no JSON API. Its search page is server-rendered HTML, so this source parses it. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterator +from datetime import datetime +from html.parser import HTMLParser +from typing import Any + +from michraz import _dates +from michraz._enums import StrEnum +from michraz.errors import SourceFormatError +from michraz.models import RAW_EXTRA_KEY, Tender +from michraz.sources.base import BaseSource, SourceName + +SEARCH_URL = "https://mr.gov.il/ilgstorefront/he/search/" +TENDER_URL = "https://mr.gov.il/ilgstorefront/he/p/{}" +ATTRIBUTION = "Government Procurement Administration (מינהל הרכש הממשלתי), mr.gov.il" +DATE_FORMAT = "%d/%m/%Y" +DATETIME_FORMAT = "%d/%m/%Y %H:%M" +LABEL_SUFFIX = ":" +SEPARATOR = "|" + +_RESULT_CLASS = "result-container" +_TITLE_CLASS = "search-results-content-head" +_TYPE_CLASS = "search-result-sub-head" +_TENDER_LINK = re.compile(r"^/ilgstorefront/he/p/(\d+)$") +_TOTAL = re.compile(r"נמצאו ([\d,]+) תוצאות") + + +class _Status(StrEnum): + """Status facet values of the portal that mean a tender is still collecting offers.""" + + PUBLISHED = "פורסם" + UPDATED = "עודכן" + NEW = "חדש" + + +class _Label(StrEnum): + """Labels printed next to each value in a search result.""" + + PUBLISHER = "שם המפרסם" + PUBLICATION_NUMBER = "מס' פרסום" + STATUS = "סטטוס" + PROCEDURE_NUMBER = "מס׳ הליך" + PUBLISHED = "תאריך פרסום" + UPDATED = "תאריך עדכון" + DEADLINE = "מועד אחרון להגשה" + + +class GovExtra(StrEnum): + """Keys of ``Tender.extra`` for government tenders.""" + + TYPE = "type" + STATUS = "status" + RAW = RAW_EXTRA_KEY + + +class _ResultParser(HTMLParser): + """Splits a search page into one dict per result: ``id``, ``title``, ``type`` and the labelled values.""" + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.results: list[dict[str, Any]] = [] + self._capture: str | None = None + self._buffer: list[str] = [] + self._spans: list[str] = [] + + def _current(self) -> dict[str, Any] | None: + return self.results[-1] if self.results else None + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + values = dict(attrs) + classes = (values.get("class") or "").split() + if tag == "div" and _RESULT_CLASS in classes: + self._finish() + self.results.append({"id": None, "title": None, "type": None, "fields": {}}) + elif (result := self._current()) is None: + return + elif tag == "a" and result["id"] is None and (m := _TENDER_LINK.match(values.get("href") or "")): + result["id"] = m.group(1) + elif tag == "h2" and _TITLE_CLASS in classes: + self._start("title") + elif tag == "h3" and _TYPE_CLASS in classes: + self._start("type") + elif tag == "span": + self._start("span") + + def _start(self, target: str) -> None: + self._capture, self._buffer = target, [] + + def handle_data(self, data: str) -> None: + if self._capture: + self._buffer.append(data) + + def handle_endtag(self, tag: str) -> None: + if self._capture is None or tag not in {"h2", "h3", "span"}: + return + text = " ".join("".join(self._buffer).split()) + if self._capture == "span": + if text: + self._spans.append(text) + elif (result := self._current()) is not None: + result[self._capture] = text or None + self._capture = None + + def _finish(self) -> None: + result = self._current() + if result is None: + return + spans, fields = self._spans, result["fields"] + for position, text in enumerate(spans): + following = spans[position + 1] if position + 1 < len(spans) else None + if text.endswith(LABEL_SUFFIX) and following and not following.endswith(LABEL_SUFFIX): + if following != SEPARATOR: + fields[text.removesuffix(LABEL_SUFFIX).strip()] = following + self._spans = [] + + def close(self) -> None: + super().close() + self._finish() + + +def parse_page(page: str) -> tuple[list[dict[str, Any]], int]: + """Returns the results on one search page and the total result count the portal reports. + + Raises: + ValueError: If the page has no result count, so it is not a search page. + """ + match = _TOTAL.search(page) + if match is None: + raise ValueError("no result count on the page") + parser = _ResultParser() + parser.feed(page) + parser.close() + return parser.results, int(match.group(1).replace(",", "")) + + +class GovSource(BaseSource): + """Tenders of ministries and other central-government bodies, from mr.gov.il. + + Only tenders whose portal status says they are still taking offers are read (published, updated, new). + The portal returns 20 results per page, so a full read takes a few dozen requests. They are cached. + + Args: + **settings: ``cache_dir``, ``cache_ttl`` and ``min_interval``, as for :class:`BaseSource`. + """ + + name = SourceName.GOV.value + description = "Central government tenders (mr.gov.il)" + attribution = ATTRIBUTION + + @staticmethod + def _params(status: _Status, page: int) -> dict[str, Any]: + return {"q": f":updateDate:archive:false:status:{status}", "text": "", "s": "TENDER", "page": page} + + def last_fetched(self) -> datetime | None: + return None + + def fetch(self) -> Iterator[Tender]: + """Yields every open tender, skipping records that fail to parse. + + Raises: + SourceUnavailableError: If mr.gov.il cannot be reached. + SourceFormatError: If a page is not a portal search page. + """ + self.errors = [] + http = self._client() + seen: set[str] = set() + for status in _Status: + page, read = 0, 0 + while True: + text = http.get_text(self.name, SEARCH_URL, self._params(status, page)) + try: + results, total = parse_page(text) + except ValueError as exc: + raise SourceFormatError(self.name, f"not a portal search page ({exc})") from exc + for result in results: + try: + tender = self.parse(result) + except (KeyError, ValueError) as exc: + self._skip(result.get("id"), exc) + continue + if tender.source_id not in seen: + seen.add(tender.source_id) + yield tender + read += len(results) + page += 1 + if not results or read >= total: + break + + def parse(self, result: dict[str, Any]) -> Tender: + """Converts one parsed search result to a :class:`Tender`. + + Raises: + KeyError: If a required value is missing. + ValueError: If a date does not match the portal format. + """ + source_id, title = result["id"], result["title"] + if not source_id or not title: + raise KeyError("id or title") + fields: dict[str, str] = result["fields"] + return Tender( + source=self.name, + source_id=source_id, + title=title, + number=fields.get(_Label.PROCEDURE_NUMBER), + publisher=fields.get(_Label.PUBLISHER), + published_at=_dates.parse_local(fields.get(_Label.PUBLISHED), DATE_FORMAT), + updated_at=_dates.parse_local(fields.get(_Label.UPDATED), DATE_FORMAT), + deadline=_dates.parse_local(fields.get(_Label.DEADLINE), DATETIME_FORMAT), + url=TENDER_URL.format(source_id), + attribution=self.attribution, + extra={GovExtra.TYPE: result["type"], GovExtra.STATUS: fields.get(_Label.STATUS), GovExtra.RAW: result}, + ) diff --git a/tests/conftest.py b/tests/conftest.py index 6dc9653..e978a92 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,13 +1,28 @@ import socket from collections.abc import Callable +from datetime import datetime from typing import Any import pytest +from michraz._dates import JERUSALEM from michraz._http import HttpClient from tests.helpers import FakeClock, load_fixture +@pytest.fixture(autouse=True) +def _pinned_clock(monkeypatch: pytest.MonkeyPatch) -> None: + """Fixes "now" to the day the fixtures were recorded, so open/closed results do not drift.""" + monkeypatch.setattr("michraz._dates.now", lambda: datetime(2026, 11, 4, 12, 0, tzinfo=JERUSALEM)) + + +@pytest.fixture(autouse=True) +def _no_real_sleep(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> None: + """Skips the rate-limit wait of clients built inside a test. Live tests keep the real wait.""" + if not request.node.get_closest_marker("live"): + monkeypatch.setattr("time.sleep", lambda seconds: None) + + @pytest.fixture(autouse=True) def _no_real_network(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> None: """Fails any test that tries to open a real socket, unless it is marked ``live``.""" diff --git a/tests/fixtures/gov_search.html b/tests/fixtures/gov_search.html new file mode 100644 index 0000000..4d29660 --- /dev/null +++ b/tests/fixtures/gov_search.html @@ -0,0 +1,343 @@ +

נמצאו 5 תוצאות עבור ערך במכרזים

+
+
+
+ +
+ +
+

מכרז פומבי

+

מכרז פומבי לחלוקי מנתחים לחדרי ניתוח עבור אספקה סטרילית השכרת ערכות חלוקי מנתחים "בליסינג"

 + +
+ + שם המפרסם: משרד הבריאות - המרכז הרפואי ע"ש חיים שיבא, תל השומר
+ + מס' פרסום: 4000623708 +  |  + סטטוס: עודכן +  |  + +
+ מס׳ הליך: 10003782 +
+  |  + +
+ תאריך פרסום: 03/11/2026
+
+ +
+
+ תאריך עדכון: 05/11/2026 +
+ +  |  + +
+ מועד אחרון להגשה: 28/12/2026 11:00
+
+
+ +
+ +
+
+
+
+ + מכרזים +

מכרז פומבי

+
+
+
+ + + +
+
+
+ +
+ +
+

מכרז פומבי

+

אספקת שירותי דפוס עבור אגף בכיר קשרי חוץ ודוברות ומעונות המשרד

 + +
+ + שם המפרסם: משרד הרווחה
+ + מס' פרסום: 4000621860 +  |  + סטטוס: עודכן +  |  + +
+ מס׳ הליך: 734/2026 +
+  |  + +
+ תאריך פרסום: 14/09/2026
+
+ +
+
+ תאריך עדכון: 04/11/2026 +
+ +  |  + +
+ מועד אחרון להגשה: 09/12/2026 12:00
+
+
+ +
+ +
+
+
+
+ + מכרזים +

מכרז פומבי

+
+
+
+ + + +
+
+
+ +
+ +
+

מכרז פומבי

+

לרכש סל פריטים משקי חד"פ

 + +
+ + שם המפרסם: משרד הבריאות - המרכז הרפואי תל אביב ע"ש סוראסקי, איכילוב
+ + מס' פרסום: 4000623437 +  |  + סטטוס: עודכן +  |  + +
+ מס׳ הליך: 265647 +
+  |  + +
+ תאריך פרסום: 28/10/2026
+
+ +
+
+ תאריך עדכון: 04/11/2026 +
+ +  |  + +
+ מועד אחרון להגשה: 09/11/2026 12:00
+
+
+ +
+ +
+
+
+
+ + מכרזים +

מכרז פומבי

+
+
+
+ + + +
+
+
+ +
+ +
+

מכרז פומבי

+

מכרז 11/2026 +לרישוי, מתן שירותי תמיכה ותחזוקה למערך ה-SIEM +

 + +
+ + שם המפרסם: משרד הפנים
+ + מס' פרסום: 4000623757 +  |  + סטטוס: פורסם +  |  + +
+ מס׳ הליך: 11/2026 +
+  |  + +
+ תאריך פרסום: 04/11/2026
+
+ +
+
+ תאריך עדכון: 04/11/2026 +
+ +  |  + +
+ מועד אחרון להגשה: 15/12/2026 14:00
+
+
+ +
+ +
+
+
+
+ + מכרזים +

מכרז פומבי

+
+
+
+ + + +
+
+
+ +
+ +
+

מכרז פומבי

+

להתקנת תשתיות לגז טבעי +ולהסבת מערכות בית החולים +במרכז הרפואי זיו

 + +
+ + שם המפרסם: משרד הבריאות - המרכז הרפואי זיו
+ + מס' פרסום: 4000623755 +  |  + סטטוס: פורסם +  |  + +
+ מס׳ הליך: 15/2026 +
+  |  + +
+ תאריך פרסום: 04/11/2026
+
+ +
+
+ תאריך עדכון: 04/11/2026 +
+ +  |  + +
+ מועד אחרון להגשה: 20/12/2026 14:00
+
+
+ +
+ +
+
+
+
+ + מכרזים +

מכרז פומבי

+
+
+
+ + diff --git a/tests/helpers.py b/tests/helpers.py index 7e11cc4..28d197a 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -41,3 +41,6 @@ def make_tender(**overrides: Any) -> Tender: fields: dict[str, Any] = {"source": "test", "source_id": "1", "title": "title", "attribution": "credit"} fields.update(overrides) return Tender(**fields) + + +GOV_EMPTY_PAGE = "

נמצאו 0 תוצאות עבור ערך במכרזים

" diff --git a/tests/live/test_live_smoke.py b/tests/live/test_live_smoke.py index 7e1c81c..6788ef9 100644 --- a/tests/live/test_live_smoke.py +++ b/tests/live/test_live_smoke.py @@ -6,7 +6,7 @@ import pytest -from michraz import BeerShevaSource, RmiSource +from michraz import BeerShevaSource, GovSource, RmiSource pytestmark = pytest.mark.live @@ -33,3 +33,11 @@ def test_beersheva_shape(): tender = next(iter(source.fetch())) assert tender.title and tender.source_id and tender.url assert {"שם מכרז", "מספר מכרז", "תאריך אחרון להגשה", "זוכה במכרז", "קבצים", "URL"} <= set(tender.extra["raw"]) + + +def test_gov_first_page_shape(): + with GovSource(cache_dir=None) as source: + tender = next(iter(source.fetch())) + assert tender.title and tender.source_id and tender.publisher + assert tender.url.startswith("https://mr.gov.il/") + assert not source.errors diff --git a/tests/test_cli.py b/tests/test_cli.py index 5c957fd..fdfd94f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -6,14 +6,16 @@ from michraz import __version__ from michraz.cli import main from michraz.sources.beersheva import API_URL +from michraz.sources.gov import SEARCH_URL as GOV_URL from michraz.sources.rmi import SEARCH_URL -from tests.helpers import ckan +from tests.helpers import GOV_EMPTY_PAGE, ckan @pytest.fixture def served(respx_mock, rmi_records, bs_records): respx_mock.post(SEARCH_URL).respond(json=rmi_records) respx_mock.get(API_URL).respond(json=ckan(bs_records)) + respx_mock.get(GOV_URL).respond(text=GOV_EMPTY_PAGE) def run(capsys, *argv): diff --git a/tests/test_client.py b/tests/test_client.py index 8a8dd7a..5c64ee2 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -5,8 +5,9 @@ from michraz import Michraz, RmiSource, SourceUnavailableError, Tender from michraz.sources.beersheva import API_URL +from michraz.sources.gov import SEARCH_URL as GOV_URL from michraz.sources.rmi import SEARCH_URL -from tests.helpers import at, ckan, make_tender +from tests.helpers import GOV_EMPTY_PAGE, at, ckan, make_tender class FakeSource: @@ -26,6 +27,7 @@ def fetch(self) -> Iterator[Tender]: def served(respx_mock, rmi_records, bs_records): respx_mock.post(SEARCH_URL).respond(json=rmi_records) respx_mock.get(API_URL).respond(json=ckan(bs_records)) + respx_mock.get(GOV_URL).respond(text=GOV_EMPTY_PAGE) def test_searches_every_default_source_sorted_by_deadline(served): @@ -114,7 +116,7 @@ def test_second_search_uses_the_cache(respx_mock, rmi_records, tmp_path): def test_repr_lists_source_names(): with Michraz(cache_dir=None) as m: - assert repr(m) == "Michraz(sources=['rmi', 'beersheva'])" + assert repr(m) == "Michraz(sources=['rmi', 'beersheva', 'gov'])" def test_base_source_is_abstract(): diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 2650c5f..3fbe2e9 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -4,20 +4,23 @@ import json import subprocess import sys +from datetime import date import pytest from michraz import Michraz, to_csv, to_dicts from michraz.cli import main from michraz.sources.beersheva import API_URL +from michraz.sources.gov import SEARCH_URL as GOV_URL from michraz.sources.rmi import SEARCH_URL -from tests.helpers import ckan +from tests.helpers import FIXTURES, GOV_EMPTY_PAGE, ckan @pytest.fixture def services(respx_mock, rmi_records, bs_records): rmi = respx_mock.post(SEARCH_URL).respond(json=rmi_records) beersheva = respx_mock.get(API_URL).respond(json=ckan(bs_records)) + respx_mock.get(GOV_URL).respond(text=GOV_EMPTY_PAGE) return rmi, beersheva @@ -58,6 +61,7 @@ def test_cli_end_to_end_with_a_warm_cache(services, tmp_path, monkeypatch, capsy def test_one_bad_record_does_not_kill_the_batch(respx_mock, rmi_records, bs_records): respx_mock.post(SEARCH_URL).respond(json=[*rmi_records, {"MichrazID": 1, "SgiraDate": "garbage"}]) respx_mock.get(API_URL).respond(json=ckan([*bs_records, {"שם מכרז": "missing columns"}])) + respx_mock.get(GOV_URL).respond(text=GOV_EMPTY_PAGE) with Michraz(cache_dir=None) as m: tenders = m.search() assert len(tenders) == len(rmi_records) + len(bs_records) @@ -67,3 +71,44 @@ def test_one_bad_record_does_not_kill_the_batch(respx_mock, rmi_records, bs_reco def test_module_entry_point_runs(): done = subprocess.run([sys.executable, "-m", "michraz", "--version"], capture_output=True, text=True, check=True) assert done.stdout.startswith("michraz ") + + +@pytest.fixture +def all_services(respx_mock, rmi_records, bs_records): + respx_mock.post(SEARCH_URL).respond(json=rmi_records) + respx_mock.get(API_URL).respond(json=ckan(bs_records)) + respx_mock.get(GOV_URL).respond(text=(FIXTURES / "gov_search.html").read_text(encoding="utf-8")) + + +def test_every_default_source_contributes_and_exports(all_services, tmp_path): + with Michraz(cache_dir=None) as m: + tenders = m.search() + assert m.last_errors == [] + assert {t.source for t in tenders} == {"rmi", "beersheva", "gov"} + + gov = [t for t in tenders if t.source == "gov"] + assert len(gov) == 5 + assert all(t.url.startswith("https://mr.gov.il/") and t.publisher and t.deadline for t in gov) + + path = tmp_path / "all.csv" + to_csv(tenders, path) + with open(path, encoding="utf-8-sig", newline="") as handle: + rows = list(csv.DictReader(handle)) + assert {r["source"] for r in rows} == {"rmi", "beersheva", "gov"} + assert all("mr.gov.il" in r["attribution"] for r in rows if r["source"] == "gov") + + +def test_gov_keyword_and_window_filters(all_services): + with Michraz(cache_dir=None) as m: + hits = m.search(keyword="מנתחים", source="gov") + window = m.search(source="gov", closing_after=date(2026, 12, 1), closing_before=date(2026, 12, 31)) + assert [t.source_id for t in hits] == ["4000623708"] + assert window and all(date(2026, 12, 1) <= t.deadline.date() < date(2026, 12, 31) for t in window) + + +def test_cli_prints_government_tenders_as_json(all_services, tmp_path, monkeypatch, capsys): + monkeypatch.setattr("michraz.cli.DEFAULT_CACHE_DIR", tmp_path / "cache") + assert main(["search", "--source", "gov", "--json"]) == 0 + out = json.loads(capsys.readouterr().out) + assert len(out) == 5 and {t["source"] for t in out} == {"gov"} + assert all("mr.gov.il" in t["attribution"] for t in out) diff --git a/tests/test_enums.py b/tests/test_enums.py index 58c8adf..5840eb3 100644 --- a/tests/test_enums.py +++ b/tests/test_enums.py @@ -9,8 +9,9 @@ from michraz.export import CSV_COLUMNS, CsvColumn from michraz.models import RAW_EXTRA_KEY from michraz.sources.beersheva import API_URL +from michraz.sources.gov import SEARCH_URL as GOV_URL from michraz.sources.rmi import SEARCH_URL -from tests.helpers import ckan +from tests.helpers import GOV_EMPTY_PAGE, ckan class Sample(StrEnum): @@ -26,7 +27,7 @@ def test_str_enum_behaves_like_its_value(): def test_source_names_match_the_sources_and_the_cli(): assert RmiSource.name == SourceName.RMI and BeerShevaSource.name == SourceName.BEERSHEVA - assert [s.value for s in SourceName] == ["rmi", "beersheva"] + assert [s.value for s in SourceName] == ["rmi", "beersheva", "gov"] assert RmiSource.name == "rmi" and type(RmiSource.name) is str # plain str, so custom sources need no enum @@ -69,6 +70,7 @@ def test_exit_codes_and_commands(): def test_enum_keys_work_end_to_end_through_the_cli(respx_mock, rmi_records, bs_records, capsys): respx_mock.post(SEARCH_URL).respond(json=rmi_records) respx_mock.get(API_URL).respond(json=ckan(bs_records)) + respx_mock.get(GOV_URL).respond(text=GOV_EMPTY_PAGE) for name in SourceName: assert main([Command.SEARCH, "--no-cache", "--source", name]) == ExitCode.OK assert "rmi" in capsys.readouterr().out diff --git a/tests/test_gov.py b/tests/test_gov.py new file mode 100644 index 0000000..b09c4ac --- /dev/null +++ b/tests/test_gov.py @@ -0,0 +1,106 @@ +from datetime import timedelta +from urllib.parse import parse_qs + +import pytest + +from michraz import GovSource, SourceFormatError, SourceUnavailableError +from michraz.sources.gov import SEARCH_URL, GovExtra, parse_page +from tests.helpers import FIXTURES + + +@pytest.fixture +def page() -> str: + return (FIXTURES / "gov_search.html").read_text(encoding="utf-8") + + +@pytest.fixture +def source(http): + src = GovSource() + src.bind(http) + return src + + +@pytest.fixture +def by_id(respx_mock, source, page): + respx_mock.get(SEARCH_URL).respond(text=page) + return {t.source_id: t for t in source.fetch()} + + +def test_parse_page_reads_every_result_and_the_total(page): + results, total = parse_page(page) + assert total == 5 + assert [r["id"] for r in results][:2] == ["4000623708", "4000621860"] + assert results[0]["type"] == "מכרז פומבי" + + +def test_parse_page_rejects_a_page_without_a_count(): + with pytest.raises(ValueError, match="result count"): + parse_page("maintenance") + + +def test_basic_fields(by_id): + t = by_id["4000623708"] + assert (t.source, t.number) == ("gov", "10003782") + assert t.publisher == 'משרד הבריאות - המרכז הרפואי ע"ש חיים שיבא, תל השומר' + assert t.title.startswith("מכרז פומבי לחלוקי מנתחים") + assert t.url == "https://mr.gov.il/ilgstorefront/he/p/4000623708" + assert "mr.gov.il" in t.attribution + assert t.extra[GovExtra.TYPE] == "מכרז פומבי" + assert t.extra["status"] == "עודכן" + + +def test_dates_are_jerusalem_local_time(by_id): + t = by_id["4000623708"] + assert (t.deadline.month, t.deadline.day, t.deadline.hour) == (12, 28, 11) + assert t.deadline.utcoffset() == timedelta(hours=2) + assert (t.published_at.day, t.updated_at.day) == (3, 5) + + +def test_a_tender_listed_under_two_statuses_is_returned_once(by_id): + assert len(by_id) == 5 + + +def test_requests_each_open_status_with_the_archive_hidden(respx_mock, source, page): + route = respx_mock.get(SEARCH_URL).respond(text=page) + list(source.fetch()) + queries = [parse_qs(call.request.url.query.decode())["q"][0] for call in route.calls] + assert [q.rsplit(":", 1)[1] for q in queries] == ["פורסם", "עודכן", "חדש"] + assert all(q.startswith(":updateDate:archive:false:status:") for q in queries) + + +def test_reads_pages_until_the_total_is_reached(respx_mock, source, page): + first = page.replace("נמצאו 5 תוצאות", "נמצאו 10 תוצאות") + route = respx_mock.get(SEARCH_URL).respond(text=first) + list(source.fetch()) + pages = [parse_qs(call.request.url.query.decode())["page"][0] for call in route.calls] + assert pages[:3] == ["0", "1", "0"] + + +def test_a_record_without_a_title_is_skipped_and_reported(respx_mock, source, page): + broken = page.replace("search-results-content-head", "search-results-other-class", 1) + respx_mock.get(SEARCH_URL).respond(text=broken) + tenders = list(source.fetch()) + assert len(tenders) == 4 + assert len(set(source.errors)) == 1 and "4000623708" in source.errors[0] + + +def test_a_missing_deadline_gives_none(source): + tender = source.parse({"id": "1", "title": "t", "type": None, "fields": {}}) + assert tender.deadline is None and not tender.is_open() + + +def test_a_bad_date_raises(source): + with pytest.raises(ValueError): + source.parse({"id": "1", "title": "t", "type": None, "fields": {"מועד אחרון להגשה": "soon"}}) + + +def test_non_search_page_raises_format_error(respx_mock, source): + respx_mock.get(SEARCH_URL).respond(text="captcha") + with pytest.raises(SourceFormatError): + list(source.fetch()) + + +def test_server_error_raises_unavailable(respx_mock, source): + respx_mock.get(SEARCH_URL).respond(status_code=503) + with pytest.raises(SourceUnavailableError): + list(source.fetch())