diff --git a/CHANGELOG.md b/CHANGELOG.md index c857bed..714f00e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ All notable changes are listed here. Format based on [Keep a Changelog](https://keepachangelog.com/). -## Unreleased +## 0.2.0 (2026-11-05) ### Added - `BidsSource` and one small class per city for 12 municipalities that share the `/bids/` page layout (Akko, Arad, @@ -24,6 +24,18 @@ All notable changes are listed here. Format based on [Keep a Changelog](https:// - `Michraz.search` no longer fails when only some sources are down. Their errors go to `last_errors`, and it raises only if every searched source failed. +### Fixed +- Kiryat Motzkin: rows about the tender committee (`ועדת מכרזים`, meetings and protocols) are no longer returned as + tenders. + +### Upgrading from 0.1.0 +- A default `Michraz()` search now reads 15 sources instead of 2. The first search takes about 20 seconds, mostly + because mr.gov.il is read page by page. Responses are cached for an hour, so later searches are fast. Use + `source="rmi"` (or any other name) to search one source. +- Every `Tender` has a `kind`, and `to_dicts` and the CSV include it. `Michraz.search` hides staff vacancies by + default. Pass `kind=None` to get everything. +- `RmiSource.fetch()` returns a lazy iterator like the other sources. Use `search()` for a list. + ## 0.1.0 (2026-11-05) ### Added diff --git a/docs/images/pr-release-checks.png b/docs/images/pr-release-checks.png new file mode 100644 index 0000000..18080b2 Binary files /dev/null and b/docs/images/pr-release-checks.png differ diff --git a/docs/images/pr-release-cli.png b/docs/images/pr-release-cli.png new file mode 100644 index 0000000..7a6229d Binary files /dev/null and b/docs/images/pr-release-cli.png differ diff --git a/docs/sources.md b/docs/sources.md index 0286a5f..76f17f5 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -106,7 +106,7 @@ correct and not confirmed. `סגירת המכרז`). `source_id` is the `id` in the link, or the tender number when a row has no such link (Dimona). - Dates have no hour, so a deadline is read as 23:59:59 of that day. This is an assumption: the real cut-off hour is in the tender documents, not on the page. -- Rows without a closing date are kept with `deadline=None`. Kiryat Motzkin lists 26 of 29 rows that way. +- Rows without a closing date are kept with `deadline=None`. Kiryat Motzkin listed 26 of 29 that way before its committee rows were filtered out. - Not covered: the 25 other sites that answered the same path with HTML but no `bids` table (for example Herzliya, Raanana, Holon, Rehovot, Ashkelon, Lod, Modiin, Karmiel, Tiberias, Afula) use a different layout. Jerusalem, Tel Aviv, Haifa, Petah Tikva, Rishon LeZion and Ashdod were not reachable the same way. @@ -114,8 +114,8 @@ correct and not confirmed. A row is a vacancy (`kind=vacancy`) when its table caption or category says `דרושים`, `כוח אדם`, `כח אדם`, `משאבי אנוש` or `מכרזי כ"א`, or its title says `דרושים`, `דרוש/ה` or `מכרז חיצוני/פנימי`. On 2026-11-05 this marked 28 rows as vacancies. It is a heuristic: a vacancy under an unexpected caption is still listed as a tender. -- Kiryat Motzkin lists 29 rows titled `ועדת מכרזים` (tender committee pages). They are not tenders and are not - filtered out. +- Kiryat Motzkin lists rows about its tender committee (category or title `ועדת מכרזים`: meetings and protocols, 24 of + 29 rows on 2026-11-05). They are not tenders and are left out. - Azor uses the same platform but its `robots.txt` disallows every path, so it is not included. - The other 12 sites returned an HTML page (not a rules file) for `/robots.txt`, so they state no rules. - `tests/fixtures/bids_.html` holds the real table of each city, cut to its header and the first three rows diff --git a/pyproject.toml b/pyproject.toml index 7aed2ca..993de58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "michraz" -version = "0.1.0" +version = "0.2.0" description = "One clean interface to Israeli public tenders." readme = "README.md" requires-python = ">=3.10" diff --git a/src/michraz/sources/bids.py b/src/michraz/sources/bids.py index d8f43f9..2c8961b 100644 --- a/src/michraz/sources/bids.py +++ b/src/michraz/sources/bids.py @@ -30,6 +30,7 @@ _PUBLISHED_HEADER = re.compile(r"(מועד|תאריך) (פרסום|פתיחת)|התחלת") _DEADLINE_HEADER = re.compile(r"אחרון להגשה|סגירת") _VACANCY_SECTION = re.compile(r"דרושים|כוח אדם|כח אדם|משאבי אנוש|מכרזי כ\"א") +_COMMITTEE = re.compile(r"^ועדת מכרזים") _VACANCY_TITLE = re.compile(r"דרושים|דרוש/ה|מכרז (חיצוני|פנימי|פנים)") @@ -213,6 +214,9 @@ def __init__(self, **settings: Any) -> None: def records(self) -> Iterator[tuple[Table, list[Cell]]]: """Yields ``(table, row)`` for every row of every table that has a tender name column. + Rows about the tender committee itself (``ועדת מכרזים``, its meetings and protocols) are not tenders and + are left out. + Raises: SourceUnavailableError: If the site cannot be reached. SourceFormatError: If the page has no bids table. @@ -222,8 +226,12 @@ def records(self) -> Iterator[tuple[Table, list[Cell]]]: if not usable: raise SourceFormatError(self.name, "no bids table with a tender name column on the page") for table in usable: + name, category = _column(table.headers, _Header.NAME), _column(table.headers, _Header.CATEGORY) for row in table.rows: - yield table, row + if not ( + _COMMITTEE.search(_cell_text(row, name) or "") or _COMMITTEE.search(_cell_text(row, category) or "") + ): + yield table, row def record_id(self, record: tuple[Table, list[Cell]]) -> object: _, row = record diff --git a/tests/test_bids.py b/tests/test_bids.py index 26ef170..c1dde3c 100644 --- a/tests/test_bids.py +++ b/tests/test_bids.py @@ -164,3 +164,15 @@ def test_a_vacancy_is_recognized_by_its_title_when_the_table_has_no_caption(): def test_the_caption_is_read_even_when_the_site_hides_it(): tables = parse_tables('
דרושים
x
') assert tables[0].caption == "דרושים" + + +def test_tender_committee_pages_are_not_tenders(respx_mock, http): + body = ( + '' + '' + '' + '
שם המכרזקטגוריהתאריך סגירת המכרז
ועדת מכרזים 26\\1ועדת מכרזים
פרוטוקולועדת מכרזים
מכרז 26\\6 מתקני פרסוםמכרזים09/12/2026
' + ) + source, tenders = fetch(respx_mock, http, "kiryat-motzkin", body) + assert [t.source_id for t in tenders] == ["3"] + assert source.errors == []