diff --git a/CHANGELOG.md b/CHANGELOG.md index a825f4f..c857bed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,25 @@ All notable changes are listed here. Format based on [Keep a Changelog](https:// ## Unreleased ### Added +- `BidsSource` and one small class per city for 12 municipalities that share the `/bids/` page layout (Akko, Arad, + Beit Shemesh, Dimona, Kfar Saba, Kiryat Motzkin, Nesher, Netivot, Nof HaGalil, Savyon, Tamra, Yavne). They are + part of the default sources. +- `michraz.sources.BUILT_IN` and `default_sources()`: one registry of every built-in source. `SourceName` lists all + of them, and the CLI `--source` choices and the default search both follow the registry. - `GovSource` (`gov`): central-government tenders from mr.gov.il, read from the portal's HTML search. It is part of the default sources. - Tests use a fixed clock, so open/closed results no longer depend on the date the suite runs. - `HttpClient.get_text` for sources that serve HTML. +### Changed +- `BaseSource` is now a template. A source implements `records()` and `parse(record)`, and the shared `fetch()` + skips and reports bad records and drops repeated tenders. `RmiSource.fetch()` is now lazy like the others. +- `Tender.kind` (`TenderKind.TENDER` or `VACANCY`) and a `kind` filter on `Michraz.search` and the CLI + (`--kind`). Staff vacancies that some municipal sites list beside their tenders are hidden by default. Pass + `kind=None` for everything. The CSV has a new `kind` column. +- `Michraz.search` no longer fails when only some sources are down. Their errors go to `last_errors`, and it raises + only if every searched source failed. + ## 0.1.0 (2026-11-05) ### Added diff --git a/Makefile b/Makefile index c5ef408..fb3cb9f 100644 --- a/Makefile +++ b/Makefile @@ -39,5 +39,6 @@ clean: ## Remove build and cache files rm -rf dist build *.egg-info src/*.egg-info .pytest_cache .mypy_cache .ruff_cache .coverage find . -name __pycache__ -not -path './.venv/*' -prune -exec rm -rf {} + -diagrams: ## Re-render docs/images from docs/diagrams (needs Graphviz and the Inter font) - for f in docs/diagrams/*.dot; do dot -Tpng -Gdpi=200 $$f -o docs/images/$$(basename $$f .dot).png; done +diagrams: ## Re-render docs/images from docs/diagrams (needs draw.io desktop and Graphviz) + for f in docs/diagrams/*.drawio; do drawio -x -f png -s 2 -b 16 --no-sandbox -o docs/images/$$(basename $$f .drawio).png $$f; done + dot -Tpng -Gdpi=200 docs/diagrams/cli-output.dot -o docs/images/cli-output.png diff --git a/README.md b/README.md index 6d27265..a3719cb 100644 --- a/README.md +++ b/README.md @@ -7,14 +7,17 @@ Israeli public tenders (מכרזים) in Python. One `Tender` object, whichever site it came from. -Sources today: Israel Land Authority land tenders, Beer Sheva municipal tenders, and central-government tenders -from mr.gov.il. The first search reads about 25 pages from mr.gov.il, so it takes around 20 seconds. After that the +Sources today: Israel Land Authority land tenders, central-government tenders (mr.gov.il), Beer Sheva, and 12 more +municipalities (Akko, Arad, Beit Shemesh, Dimona, Kfar Saba, Kiryat Motzkin, Nesher, Netivot, Nof HaGalil, Savyon, +Tamra, Yavne). The first search takes around 20 seconds because mr.gov.il is read page by page. After that the responses are cached for an hour. > Unofficial client. It can lag or miss tenders, and parsing can break when a source changes. Confirm deadlines on > the publisher's page before you act. Not affiliated with the Israel Land Authority or any municipality. -![What michraz does](https://raw.githubusercontent.com/Zikithezikit/michraz/main/docs/images/overview.png) +![michraz architecture: how one search reaches the data and comes back](https://raw.githubusercontent.com/Zikithezikit/michraz/main/docs/images/architecture.png) + +[Edit in draw.io](https://app.diagrams.net/#Uhttps%3A%2F%2Fraw.githubusercontent.com%2FZikithezikit%2Fmichraz%2Fmain%2Fdocs%2Fdiagrams%2Farchitecture.drawio) ## Install @@ -40,6 +43,9 @@ to_csv(tenders, "tenders.csv") Results are sorted by deadline. Filter by `source`, `closing_before` and `closing_after` (dates or aware datetimes) too. Each tender has an `attribution` string. Keep it when you republish the data. +Some city sites list staff vacancies next to their tenders. They are hidden by default. Pass `kind=None` to see both, +or `--kind all` on the command line. + Or from the terminal: ```bash @@ -47,24 +53,49 @@ michraz search --open --source rmi michraz search --keyword גינון --csv gardening.csv ``` -![Terminal output](https://raw.githubusercontent.com/Zikithezikit/michraz/main/docs/images/cli.png) +![michraz search in the terminal](https://raw.githubusercontent.com/Zikithezikit/michraz/main/docs/images/cli-output.png) -A record that fails to parse is skipped and listed in `m.last_errors`. An unreachable source raises -`SourceUnavailableError`. More scripts are in [examples](https://github.com/Zikithezikit/michraz/tree/main/examples). +A record that fails to parse is skipped and listed in `m.last_errors`. If one source is down, the others still +answer and its error is listed there too. `SourceUnavailableError` is raised only when every searched source fails. + +More scripts are in [examples](https://github.com/Zikithezikit/michraz/tree/main/examples). ## Add a source -Write a class with `name`, `description`, `attribution`, `errors` and a `fetch()` that yields `Tender`s, then pass -it to `Michraz([MySource()])`. +Subclass `BaseSource`. Say how to read the records and how to turn one into a `Tender`. Skipping bad records, +reporting them in `errors` and dropping repeats is shared. + +```python +from michraz.sources import BaseSource + + +class MySource(BaseSource[dict]): + name = "my-city" + description = "My city's tenders" + attribution = "My City, my-city.example" -![Classes](https://raw.githubusercontent.com/Zikithezikit/michraz/main/docs/images/classes.png) + def records(self): + yield from self._client().get_json(self.name, "https://my-city.example/api/tenders") + + def parse(self, record): + return Tender(source=self.name, source_id=record["id"], title=record["title"], attribution=self.attribution) +``` + +Then `Michraz([MySource()])`. A city that uses the shared `/bids/` page needs only a name, a publisher and a domain +(see `michraz/sources/municipal.py`). + +![Class diagram: sources and the Tender model](https://raw.githubusercontent.com/Zikithezikit/michraz/main/docs/images/class-diagram.png) + +[Edit in draw.io](https://app.diagrams.net/#Uhttps%3A%2F%2Fraw.githubusercontent.com%2FZikithezikit%2Fmichraz%2Fmain%2Fdocs%2Fdiagrams%2Fclass-diagram.drawio) Fields are documented in [docs/model.md](https://github.com/Zikithezikit/michraz/blob/main/docs/model.md), and how each source was checked in [docs/sources.md](https://github.com/Zikithezikit/michraz/blob/main/docs/sources.md). ## Polite by default -![Request flow](https://raw.githubusercontent.com/Zikithezikit/michraz/main/docs/images/request_flow.png) +![Flowchart: what happens during one search](https://raw.githubusercontent.com/Zikithezikit/michraz/main/docs/images/search-flowchart.png) + +[Edit in draw.io](https://app.diagrams.net/#Uhttps%3A%2F%2Fraw.githubusercontent.com%2FZikithezikit%2Fmichraz%2Fmain%2Fdocs%2Fdiagrams%2Fsearch-flowchart.drawio) Honest User-Agent, one request per second per host, one-hour disk cache, at most two retries. Tune it with `Michraz(cache_dir=None, cache_ttl=..., min_interval=...)`. @@ -79,9 +110,9 @@ Honest User-Agent, one request per second per host, one-hour disk cache, at most ```bash make venv && make check +make diagrams # re-export the PNGs from docs/diagrams (needs draw.io desktop and Graphviz) ``` -Tests run against recorded responses and never touch the network. See -[CONTRIBUTING.md](https://github.com/Zikithezikit/michraz/blob/main/CONTRIBUTING.md). +See [CONTRIBUTING.md](https://github.com/Zikithezikit/michraz/blob/main/CONTRIBUTING.md). MIT licensed. The data belongs to its publishers and carries their licenses. diff --git a/docs/diagrams/architecture.drawio b/docs/diagrams/architecture.drawio new file mode 100644 index 0000000..2d3b465 --- /dev/null +++ b/docs/diagrams/architecture.drawio @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/docs/diagrams/class-diagram.drawio b/docs/diagrams/class-diagram.drawio new file mode 100644 index 0000000..8a065dc --- /dev/null +++ b/docs/diagrams/class-diagram.drawio @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/diagrams/classes.dot b/docs/diagrams/classes.dot deleted file mode 100644 index 81308c2..0000000 --- a/docs/diagrams/classes.dot +++ /dev/null @@ -1,61 +0,0 @@ -digraph classes { - rankdir=TB; bgcolor="white"; pad=0.35; nodesep=0.55; ranksep=0.75; splines=true; - node [fontname="Inter", fontsize=11, shape=plain, fontcolor="#1e293b"]; - edge [fontname="Inter", fontsize=10, color="#64748b", fontcolor="#64748b", arrowsize=0.8, penwidth=1.2]; - - Michraz [label=< - - -
- -
- -
Michraz
sources: tuple[Source, ...]
last_errors: list[str]
search(keyword, open_only, source,
closing_before, closing_after) -> list[Tender]
close()
>]; - - Source [label=< - - -
- -
- -
<<protocol>>
Source
name: str
description: str
attribution: str
errors: list[str]
fetch() -> Iterator[Tender]
>]; - - RmiSource [label=< - - -
- -
RmiSource
search(settlement_code, neighborhood,
tender_number, region, ...) -> list[Tender]
>]; - - BeerShevaSource [label=< - - -
- -
BeerShevaSource
page_size: int
>]; - - Tender [label=< - - -
- -
- -
Tender (frozen)
source: str
source_id: str
title: str
number: str | None
deadline: datetime | None
winner: str | None
url: str | None
documents: tuple[Document, ...]
attribution: str
extra: Mapping[str, Any]
key: tuple[str, str]
is_open(now=None) -> bool
>]; - - Document [label=< - - -
- -
Document (frozen)
title: str
url: str
>]; - - Michraz -> Source [arrowhead=vee, headlabel="1..*", labeldistance=2.6, labelangle=35, label=" searches "]; - RmiSource -> Source [arrowhead=empty, style=dashed]; - BeerShevaSource -> Source [arrowhead=empty, style=dashed]; - Source -> Tender [arrowhead=vee, style=dashed, label=" yields "]; - Tender -> Document [dir=both, arrowtail=diamond, arrowhead=none, headlabel="0..*", taillabel="1", labeldistance=2.4, labelangle=-40]; - { rank=same; RmiSource; BeerShevaSource; } - { rank=same; Tender; Document; } -} diff --git a/docs/diagrams/cli.dot b/docs/diagrams/cli-output.dot similarity index 100% rename from docs/diagrams/cli.dot rename to docs/diagrams/cli-output.dot diff --git a/docs/diagrams/overview.dot b/docs/diagrams/overview.dot deleted file mode 100644 index 3388e95..0000000 --- a/docs/diagrams/overview.dot +++ /dev/null @@ -1,39 +0,0 @@ -digraph overview { - rankdir=LR; bgcolor="white"; pad=0.35; nodesep=0.3; ranksep=0.7; newrank=true; splines=true; - node [fontname="Inter", fontsize=12, shape=box, style="rounded,filled", fillcolor="white", - color="#cbd5e1", fontcolor="#1e293b", margin="0.22,0.13", penwidth=1.2]; - edge [fontname="Inter", fontsize=10, color="#94a3b8", fontcolor="#64748b", arrowsize=0.7, penwidth=1.2]; - - you [label=<You
Python or terminal>, fillcolor="#f1f5f9"]; - - subgraph cluster_core { - label=<michraz>; labeljust=l; fontname="Inter"; fontsize=12; fontcolor="#1d4ed8"; - style="rounded,filled"; fillcolor="#eff6ff"; color="#bfdbfe"; margin=16; - api [label=<Michraz.search()
filter, sort, collect errors>, color="#93c5fd"]; - http [label=<Polite HTTP client
rate limit, disk cache, retries>, color="#93c5fd"]; - tenders [label=<list of Tender
one model for every source>, color="#93c5fd"]; - } - - subgraph cluster_feeds { - label=<Public feeds>; labeljust=l; fontname="Inter"; fontsize=12; fontcolor="#047857"; - style="rounded,filled"; fillcolor="#ecfdf5"; color="#a7f3d0"; margin=16; - rmi [label=<Israel Land Authority
land tenders, live>, color="#6ee7b7"]; - bs [label=<Beer Sheva
municipal tenders, weekly>, color="#6ee7b7"]; - own [label=<Your source
one small class>, color="#6ee7b7", style="rounded,filled,dashed"]; - } - - subgraph cluster_out { - label=<Output>; labeljust=l; fontname="Inter"; fontsize=12; fontcolor="#b45309"; - style="rounded,filled"; fillcolor="#fffbeb"; color="#fde68a"; margin=16; - table [label="Table in the terminal", color="#fcd34d"]; - csv [label=(opens in Excel)>, color="#fcd34d"]; - json [label="JSON", color="#fcd34d"]; - df [label="pandas DataFrame", color="#fcd34d"]; - } - - you -> api; - api -> http [label=" asks"]; - http -> rmi; http -> bs; http -> own [style=dashed]; - api -> tenders [label=" returns"]; - tenders -> table; tenders -> csv; tenders -> json; tenders -> df; -} diff --git a/docs/diagrams/request_flow.dot b/docs/diagrams/request_flow.dot deleted file mode 100644 index d9a46b4..0000000 --- a/docs/diagrams/request_flow.dot +++ /dev/null @@ -1,42 +0,0 @@ -digraph request_flow { - rankdir=TB; bgcolor="white"; pad=0.35; nodesep=0.45; ranksep=0.42; - node [fontname="Inter", fontsize=11, shape=box, style="rounded,filled", fillcolor="white", color="#cbd5e1", - fontcolor="#1e293b", margin="0.2,0.11", penwidth=1.2]; - edge [fontname="Inter", fontsize=10, color="#94a3b8", fontcolor="#64748b", arrowsize=0.7, penwidth=1.2]; - - start [shape=circle, label="", width=0.2, style=filled, fillcolor="#1e293b", color="#1e293b"]; - search [label=<Michraz.search()
for each source>, fillcolor="#eff6ff", color="#93c5fd"]; - cached [label="Fresh response\nin the cache?", shape=diamond, style="filled", fillcolor="#f8fafc", margin="0.05,0.05"]; - hit [label="Use the cached JSON"]; - wait [label=per-host rate limit
at least 1 s apart>]; - send [label=honest User-Agent>]; - status [label="HTTP status?", shape=diamond, style="filled", fillcolor="#f8fafc", margin="0.05,0.05"]; - retry [label=honor Retry-After, at most 2 retries>, fillcolor="#fffbeb", color="#fcd34d"]; - fail [label=SourceUnavailableError>, fillcolor="#fef2f2", color="#fca5a5"]; - store [label="Store the JSON in the cache"]; - parse [label="Parse each record", fillcolor="#ecfdf5", color="#6ee7b7"]; - valid [label="Record valid?", shape=diamond, style="filled", fillcolor="#f8fafc", margin="0.05,0.05"]; - skip [label=add to last_errors>, fillcolor="#fffbeb", color="#fcd34d"]; - tender [label=Tender>, fillcolor="#ecfdf5", color="#6ee7b7"]; - filter [label="Filter and sort by deadline", fillcolor="#eff6ff", color="#93c5fd"]; - done [label=list[Tender]>, fillcolor="#eff6ff", color="#93c5fd"]; - end [shape=doublecircle, label="", width=0.2, style=filled, fillcolor="#1e293b", color="#1e293b"]; - - start -> search -> cached; - cached -> hit [label=" yes"]; - cached -> wait [label=" no"]; - wait -> send -> status; - status -> store [label=" 2xx"]; - status -> retry [label=" 429 or 5xx"]; - status -> fail [label=" other 4xx,\n redirect"]; - retry -> wait; - hit -> parse; - store -> parse; - parse -> valid; - valid -> tender [label=" yes"]; - valid -> skip [label=" no"]; - skip -> parse [style=dashed, label=" next record"]; - tender -> filter -> done -> end; - { rank=same; cached; hit; } - { rank=same; status; retry; } -} diff --git a/docs/diagrams/search-flowchart.drawio b/docs/diagrams/search-flowchart.drawio new file mode 100644 index 0000000..1fd6c27 --- /dev/null +++ b/docs/diagrams/search-flowchart.drawio @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/images/architecture.png b/docs/images/architecture.png new file mode 100644 index 0000000..2d375b3 Binary files /dev/null and b/docs/images/architecture.png differ diff --git a/docs/images/class-diagram.png b/docs/images/class-diagram.png new file mode 100644 index 0000000..a21fd68 Binary files /dev/null and b/docs/images/class-diagram.png differ diff --git a/docs/images/classes.png b/docs/images/classes.png deleted file mode 100644 index 9aa86aa..0000000 Binary files a/docs/images/classes.png and /dev/null differ diff --git a/docs/images/cli.png b/docs/images/cli-output.png similarity index 100% rename from docs/images/cli.png rename to docs/images/cli-output.png diff --git a/docs/images/overview.png b/docs/images/overview.png deleted file mode 100644 index bd98a09..0000000 Binary files a/docs/images/overview.png and /dev/null differ diff --git a/docs/images/pr-all-sources.png b/docs/images/pr-all-sources.png new file mode 100644 index 0000000..bfa5da7 Binary files /dev/null and b/docs/images/pr-all-sources.png differ diff --git a/docs/images/pr-bids-cli.png b/docs/images/pr-bids-cli.png new file mode 100644 index 0000000..a087b33 Binary files /dev/null and b/docs/images/pr-bids-cli.png differ diff --git a/docs/images/pr-bids-tests.png b/docs/images/pr-bids-tests.png new file mode 100644 index 0000000..e3a1acc Binary files /dev/null and b/docs/images/pr-bids-tests.png differ diff --git a/docs/images/request_flow.png b/docs/images/request_flow.png deleted file mode 100644 index be534c9..0000000 Binary files a/docs/images/request_flow.png and /dev/null differ diff --git a/docs/images/search-flowchart.png b/docs/images/search-flowchart.png new file mode 100644 index 0000000..5d722e4 Binary files /dev/null and b/docs/images/search-flowchart.png differ diff --git a/docs/model.md b/docs/model.md index db6bedf..b0a4003 100644 --- a/docs/model.md +++ b/docs/model.md @@ -18,6 +18,7 @@ safe to share. | `updated_at` | `datetime \| None` | not available | `תאריך עדכון` | | `winner` | `str \| None` | always None | `זוכה במכרז`, None when blank | | `url` | `str \| None` | `https://apps.land.gov.il/MichrazimSite/#/michraz/{id}` | `URL` | +| `kind` | `TenderKind` | always `tender` | always `tender` | | `documents` | `tuple[Document, ...]` | empty (details endpoint not used yet) | links parsed from `קבצים` | | `attribution` | `str` | credit line, always present | credit line, always present | | `extra` | `Mapping[str, Any]` | see below (`RmiExtra`) | `payment_url`, `raw` (`BeerShevaExtra`) | @@ -29,6 +30,7 @@ Rules that hold for every source: normalized. - `extra["raw"]` is the untouched source record. - Equality and hashing ignore `extra`. +- `kind` is `vacancy` only for staff vacancies that municipal bids sources list next to their tenders. `Tender.key` is `(source, source_id)`. `Tender.is_open(now=None)` is true when there is no winner and the deadline is strictly after `now`. A tender with no known deadline counts as closed. @@ -54,6 +56,12 @@ Keys are the members of `michraz.GovExtra`: `type` (for example `מכרז פומ example `עודכן`) and `raw`. `source_id` is the portal's publication number, `number` is its procedure number, and `documents` is empty because detail pages are not fetched. +### Municipal `extra` + +Keys are the members of `michraz.BidsExtra`: `type` (for example `פומבי`), `category` (when the site has such a +column) and `raw` (every column of the row by its header). `source` is the municipality's subdomain, for example +`kfar-saba`. Dates carry no hour on these sites, so `deadline` is the end of the closing day. + ## `Document` | Field | Type | Meaning | diff --git a/docs/sources.md b/docs/sources.md index 61c4bfc..0286a5f 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -94,6 +94,33 @@ correct and not confirmed. removed and the count sentence was edited to say 5 so that it matches the excerpt. - The site has no `robots.txt` (the request returns an empty body). +## Municipalities on the shared bids page, `akko`, `arad`, `betshemesh`, `dimona`, `kfar-saba`, `kiryat-motzkin`, `nesher`, `netivot`, `nof-hagalil`, `savyon`, `tamra`, `yavne` + +- Platform: many municipal websites share one page layout and publish current tenders at `/bids/` (the path is + `/he/bids/` for Akko and `/ar/bids/` for Tamra) as HTML tables with the class `bids`. Detail pages are + `/bids/?id=`. The archive is at `?archive=1` and is not read. +- Verified on 2026-11-05: each of the 12 sites returned a parsable page. The parser read 3 to 29 rows per site with + 0 parse errors. +- The column layout differs per city, so columns are found by header text: tender name, number, type, category, + start (`תאריך פרסום`, `תאריך פתיחת המכרז`, `התחלת המכרז`) and deadline (`מועד אחרון להגשה`, `תאריך סגירת המכרז`, + `סגירת המכרז`). `source_id` is the `id` in the link, or the tender number when a row has no such link (Dimona). +- Dates have no hour, so a deadline is read as 23:59:59 of that day. This is an assumption: the real cut-off hour + is in the tender documents, not on the page. +- Rows without a closing date are kept with `deadline=None`. Kiryat Motzkin lists 26 of 29 rows that way. +- Not covered: the 25 other sites that answered the same path with HTML but no `bids` table (for example Herzliya, + Raanana, Holon, Rehovot, Ashkelon, Lod, Modiin, Karmiel, Tiberias, Afula) use a different layout. Jerusalem, + Tel Aviv, Haifa, Petah Tikva, Rishon LeZion and Ashdod were not reachable the same way. +- Several sites mix staff vacancies (job ads) into the same tables: Beit Shemesh, Dimona, Kfar Saba and Nof HaGalil. + A row is a vacancy (`kind=vacancy`) when its table caption or category says `דרושים`, `כוח אדם`, `כח אדם`, + `משאבי אנוש` or `מכרזי כ"א`, or its title says `דרושים`, `דרוש/ה` or `מכרז חיצוני/פנימי`. On 2026-11-05 this + marked 28 rows as vacancies. It is a heuristic: a vacancy under an unexpected caption is still listed as a tender. +- Kiryat Motzkin lists 29 rows titled `ועדת מכרזים` (tender committee pages). They are not tenders and are not + filtered out. +- Azor uses the same platform but its `robots.txt` disallows every path, so it is not included. +- The other 12 sites returned an HTML page (not a rules file) for `/robots.txt`, so they state no rules. +- `tests/fixtures/bids_.html` holds the real table of each city, cut to its header and the first three rows + of the first table that has rows. + ## Robots and terms Checked 2026-11-04 from the development machine with the michraz User-Agent. diff --git a/examples/custom_source.py b/examples/custom_source.py index 03e5c13..9eea3fc 100644 --- a/examples/custom_source.py +++ b/examples/custom_source.py @@ -1,30 +1,30 @@ -"""Add your own source. Any class with these members works; no network is involved in this example.""" +"""Add your own source: subclass BaseSource, read records, turn each into a Tender. No network in this example.""" from collections.abc import Iterator from datetime import datetime from zoneinfo import ZoneInfo from michraz import Michraz, Tender +from michraz.sources import BaseSource JERUSALEM = ZoneInfo("Asia/Jerusalem") -class DemoCitySource: +class DemoCitySource(BaseSource[dict[str, str]]): name = "demo-city" description = "A made-up municipality" attribution = "Demo City, example data" - def __init__(self) -> None: - self.errors: list[str] = [] # message for every record you skip + def records(self) -> Iterator[dict[str, str]]: + yield {"id": "2026-1", "title": "מכרז לדוגמה", "deadline": "2999-01-01T12:00"} + yield {"id": "2026-2", "deadline": "2999-01-01T12:00"} # no title: skipped and listed in errors - def fetch(self) -> Iterator[Tender]: - yield Tender( + def parse(self, record: dict[str, str]) -> Tender: + return Tender( source=self.name, - source_id="2026-1", - title="מכרז לדוגמה", - number="1/2026", - publisher="עיריית דמו", - deadline=datetime(2999, 1, 1, 12, 0, tzinfo=JERUSALEM), + source_id=record["id"], + title=record["title"], + deadline=datetime.fromisoformat(record["deadline"]).replace(tzinfo=JERUSALEM), attribution=self.attribution, ) @@ -32,4 +32,5 @@ def fetch(self) -> Iterator[Tender]: if __name__ == "__main__": with Michraz([DemoCitySource()], cache_dir=None) as m: for tender in m.search(open_only=True): - print(tender.number, tender.title, tender.deadline.isoformat()) + print(tender.source_id, tender.title, tender.deadline.isoformat()) + print(m.last_errors) diff --git a/src/michraz/__init__.py b/src/michraz/__init__.py index 40f835c..3eb8c47 100644 --- a/src/michraz/__init__.py +++ b/src/michraz/__init__.py @@ -11,21 +11,26 @@ from michraz.client import Michraz from michraz.errors import MichrazError, SourceFormatError, SourceUnavailableError from michraz.export import to_csv, to_dataframe, to_dicts -from michraz.models import Document, Tender +from michraz.models import Document, Tender, TenderKind from michraz.sources import ( BeerShevaExtra, BeerShevaSource, + BidsExtra, + BidsSource, GovExtra, GovSource, RmiExtra, RmiSource, Source, SourceName, + default_sources, ) __all__ = [ "BeerShevaExtra", "BeerShevaSource", + "BidsExtra", + "BidsSource", "Document", "GovExtra", "GovSource", @@ -38,7 +43,9 @@ "SourceFormatError", "SourceUnavailableError", "Tender", + "TenderKind", "__version__", + "default_sources", "to_csv", "to_dataframe", "to_dicts", diff --git a/src/michraz/cli.py b/src/michraz/cli.py index 77d41d9..7e0a754 100644 --- a/src/michraz/cli.py +++ b/src/michraz/cli.py @@ -17,8 +17,8 @@ from michraz.client import Michraz from michraz.errors import MichrazError from michraz.export import to_csv, to_dicts -from michraz.models import Tender -from michraz.sources import BaseSource, SourceName +from michraz.models import Tender, TenderKind +from michraz.sources import BUILT_IN, BaseSource PROGRAM = "michraz" DATE_FORMAT = "%Y-%m-%d %H:%M" @@ -28,6 +28,9 @@ TITLE_HEADER = "TITLE" +ALL_KINDS = "all" + + class Command(StrEnum): """Subcommands of the ``michraz`` program.""" @@ -59,7 +62,13 @@ def _build_parser() -> argparse.ArgumentParser: search = commands.add_parser(Command.SEARCH.value, parents=[common], help="search tenders") search.add_argument("--keyword", "-k", help="text to look for in the title or number") search.add_argument("--open", action="store_true", help="only tenders that can still receive submissions") - search.add_argument("--source", choices=[name.value for name in SourceName], help="search a single source") + search.add_argument("--source", choices=[source.name for source in BUILT_IN], help="search a single source") + search.add_argument( + "--kind", + choices=[*(kind.value for kind in TenderKind), ALL_KINDS], + default=TenderKind.TENDER.value, + help="tenders (default), staff vacancies that some municipalities list beside them, or all", + ) search.add_argument("--csv", metavar="FILE", help="also write the results to a CSV file (UTF-8 with BOM)") search.add_argument("--json", action="store_true", help="print JSON instead of a table") @@ -77,7 +86,7 @@ def _clip(text: str, width: int) -> str: return text if len(text) <= width else text[: width - 1] + ELLIPSIS -def _print_table(tenders: Sequence[Tender]) -> None: +def _print_table(tenders: Sequence[Tender], kind: str) -> None: # Hebrew is printed in logical order and the terminal lays it out. The title goes last so the # fixed-width columns stay aligned whatever the text direction. header = TABLE_HEADER @@ -100,20 +109,22 @@ def line(cells: Sequence[str], title: str) -> str: print(line(header, TITLE_HEADER)) for cells, tender in zip(rows, tenders, strict=True): print(line(cells, _clip(tender.title, title_width))) - print(f"\n{len(rows)} tender{'' if len(rows) == 1 else 's'}") + singular, plural = ("vacancy", "vacancies") if kind == TenderKind.VACANCY else ("tender", "tenders") + print(f"\n{len(rows)} {singular if len(rows) == 1 else plural}") for credit in sorted({t.attribution for t in tenders}): print(f"Data: {credit}") def _search(args: argparse.Namespace, m: Michraz) -> int: - tenders = m.search(keyword=args.keyword, open_only=args.open, source=args.source) + kind = None if args.kind == ALL_KINDS else TenderKind(args.kind) + tenders = m.search(keyword=args.keyword, open_only=args.open, source=args.source, kind=kind) if args.csv: count = to_csv(tenders, args.csv) print(f"wrote {count} tenders to {args.csv}", file=sys.stderr) if args.json: print(json.dumps(to_dicts(tenders), ensure_ascii=False, indent=2)) else: - _print_table(tenders) + _print_table(tenders, args.kind) for message in m.last_errors: print(f"warning: {message}", file=sys.stderr) return ExitCode.OK diff --git a/src/michraz/client.py b/src/michraz/client.py index 05f7a56..c30f1a8 100644 --- a/src/michraz/client.py +++ b/src/michraz/client.py @@ -14,9 +14,10 @@ DEFAULT_MIN_INTERVAL, HttpClient, ) +from michraz.errors import MichrazError from michraz.filters import TenderFilter, sort_by_deadline -from michraz.models import Tender -from michraz.sources import BaseSource, BeerShevaSource, GovSource, RmiSource, Source +from michraz.models import Tender, TenderKind +from michraz.sources import BaseSource, Source, default_sources logger = logging.getLogger(__name__) @@ -30,7 +31,8 @@ class Michraz(Closeable): tenders = m.search(keyword="גינון", open_only=True) Args: - sources: Sources to search. Defaults to RMI, Beer Sheva and the central government. + sources: Sources to search. Defaults to RMI, Beer Sheva, the central government and the municipalities listed in + :data:`michraz.sources.BUILT_IN`. cache_dir: Directory for cached responses. Defaults to ``~/.cache/michraz``. None disables the cache. cache_ttl: How long a cached response stays valid. min_interval: Minimum seconds between requests to the same host. Must be at least 0.5. @@ -51,9 +53,7 @@ def __init__( min_interval: float = DEFAULT_MIN_INTERVAL, ) -> None: self._http = HttpClient(cache_dir=cache_dir, cache_ttl=cache_ttl, min_interval=min_interval) - self.sources: tuple[Source, ...] = ( - tuple(sources) if sources is not None else (RmiSource(), BeerShevaSource(), GovSource()) - ) + self.sources: tuple[Source, ...] = tuple(sources) if sources is not None else default_sources() names = [s.name for s in self.sources] if len(set(names)) != len(names): raise ValueError(f"source names must be unique, got {names}") @@ -69,6 +69,7 @@ def search( source: str | None = None, closing_before: datetime | date | None = None, closing_after: datetime | date | None = None, + kind: TenderKind | None = TenderKind.TENDER, ) -> list[Tender]: """Fetches tenders from the sources and filters them. @@ -79,23 +80,40 @@ def search( closing_before: Keep tenders whose deadline is before this moment. A date means the start of that day in Asia/Jerusalem. closing_after: Keep tenders whose deadline is at or after this moment. + kind: Keep only this kind of listing. Defaults to tenders, which hides the staff vacancies that some + municipal sites list next to them. Pass ``TenderKind.VACANCY`` for only those, or None for both. Returns: Matching tenders sorted by deadline, earliest first. Tenders without a deadline come last. Raises: ValueError: If ``source`` names no known source or a datetime bound is naive. - SourceUnavailableError: If a source cannot be reached. - SourceFormatError: If a source answers in an unexpected shape. + SourceUnavailableError: If every searched source cannot be reached. + SourceFormatError: If every searched source answers in an unexpected shape. When only some + sources fail, their errors are listed in :attr:`last_errors` and the rest are returned. """ criteria = TenderFilter( - keyword=keyword, open_only=open_only, closing_before=closing_before, closing_after=closing_after + keyword=keyword, + open_only=open_only, + closing_before=closing_before, + closing_after=closing_after, + kind=kind, ) self.last_errors = [] tenders: list[Tender] = [] - for src in self._select(source): - tenders.extend(src.fetch()) + chosen = self._select(source) + failures: list[MichrazError] = [] + for src in chosen: + try: + tenders.extend(list(src.fetch())) + except MichrazError as exc: + logger.warning("%s failed: %s", src.name, exc) + failures.append(exc) + self.last_errors.append(str(exc)) + continue self.last_errors.extend(src.errors) + if failures and len(failures) == len(chosen): + raise failures[0] return sort_by_deadline(criteria.apply(tenders)) def _select(self, name: str | None) -> tuple[Source, ...]: diff --git a/src/michraz/export.py b/src/michraz/export.py index 8a2a92e..be4809c 100644 --- a/src/michraz/export.py +++ b/src/michraz/export.py @@ -21,6 +21,7 @@ class CsvColumn(StrEnum): SOURCE = "source" SOURCE_ID = "source_id" + KIND = "kind" NUMBER = "number" TITLE = "title" PUBLISHER = "publisher" diff --git a/src/michraz/filters.py b/src/michraz/filters.py index 38bacb6..be79ee5 100644 --- a/src/michraz/filters.py +++ b/src/michraz/filters.py @@ -9,7 +9,7 @@ from typing import Final from michraz import _dates -from michraz.models import Tender +from michraz.models import Tender, TenderKind NORMALIZATION_FORM: Final = "NFKC" @@ -51,6 +51,7 @@ class TenderFilter: closing_before: Keep tenders whose deadline is strictly before this moment. A plain date means 00:00 at the start of that day in Asia/Jerusalem. closing_after: Keep tenders whose deadline is at or after this moment. Dates are read as above. + kind: Keep only tenders of this kind. None keeps every kind. now: Moment used for ``open_only``. Defaults to the current time on each call. Raises: @@ -61,6 +62,7 @@ class TenderFilter: open_only: bool = False closing_before: datetime | date | None = None closing_after: datetime | date | None = None + kind: TenderKind | None = None now: datetime | None = None def __post_init__(self) -> None: @@ -71,6 +73,8 @@ def __post_init__(self) -> None: def matches(self, tender: Tender) -> bool: """Tells whether ``tender`` meets every condition.""" + if self.kind is not None and tender.kind != self.kind: + return False if self.keyword and not matches_keyword(tender, self.keyword): return False if self.open_only and not tender.is_open(self.now): @@ -98,11 +102,17 @@ def apply_filters( open_only: bool = False, closing_before: datetime | date | None = None, closing_after: datetime | date | None = None, + kind: TenderKind | None = None, now: datetime | None = None, ) -> list[Tender]: """Filters tenders. Shorthand for ``TenderFilter(...).apply(tenders)``; see :class:`TenderFilter`.""" return TenderFilter( - keyword=keyword, open_only=open_only, closing_before=closing_before, closing_after=closing_after, now=now + keyword=keyword, + open_only=open_only, + closing_before=closing_before, + closing_after=closing_after, + kind=kind, + now=now, ).apply(tenders) diff --git a/src/michraz/models.py b/src/michraz/models.py index 6ce1a0e..5268705 100644 --- a/src/michraz/models.py +++ b/src/michraz/models.py @@ -8,6 +8,15 @@ from typing import Any from michraz import _dates +from michraz._enums import StrEnum + + +class TenderKind(StrEnum): + """What a listing is. Some municipal sites list staff vacancies next to their tenders.""" + + TENDER = "tender" + VACANCY = "vacancy" + RAW_EXTRA_KEY = "raw" """Key under which every source stores the untouched record in ``Tender.extra``.""" @@ -46,6 +55,7 @@ class Tender: winner: Name of the winner. None while the tender is still open. url: Page of the tender on the publisher's site. documents: Attached files. + kind: Whether this is a tender or a staff vacancy. Only some municipal sources can tell. attribution: Credit line required by the data license. Always keep it with the data. extra: Source-specific fields. The untouched record is under ``extra[RAW_EXTRA_KEY]`` (``"raw"``). """ @@ -63,6 +73,7 @@ class Tender: winner: str | None = None url: str | None = None documents: tuple[Document, ...] = () + kind: TenderKind = TenderKind.TENDER extra: Mapping[str, Any] = field(default_factory=dict, compare=False, hash=False, repr=False) @property diff --git a/src/michraz/sources/__init__.py b/src/michraz/sources/__init__.py index 24f234a..eee2a5a 100644 --- a/src/michraz/sources/__init__.py +++ b/src/michraz/sources/__init__.py @@ -2,17 +2,23 @@ from michraz.sources.base import BaseSource, Source, SourceName from michraz.sources.beersheva import BeerShevaExtra, BeerShevaSource +from michraz.sources.bids import BidsExtra, BidsSource from michraz.sources.gov import GovExtra, GovSource +from michraz.sources.registry import BUILT_IN, default_sources from michraz.sources.rmi import RmiExtra, RmiSource __all__ = [ + "BUILT_IN", "BaseSource", "BeerShevaExtra", "BeerShevaSource", + "BidsExtra", + "BidsSource", "GovExtra", "GovSource", "RmiExtra", "RmiSource", "Source", "SourceName", + "default_sources", ] diff --git a/src/michraz/sources/base.py b/src/michraz/sources/base.py index 1a00d85..1d38551 100644 --- a/src/michraz/sources/base.py +++ b/src/michraz/sources/base.py @@ -5,9 +5,9 @@ import logging import os from abc import ABC, abstractmethod -from collections.abc import Iterator +from collections.abc import Iterable, Iterator from datetime import datetime, timedelta -from typing import Protocol, runtime_checkable +from typing import ClassVar, Generic, Protocol, TypeVar, runtime_checkable from michraz._closeable import Closeable from michraz._enums import StrEnum @@ -16,13 +16,30 @@ logger = logging.getLogger(__name__) +RecordT = TypeVar("RecordT") + +PARSE_ERRORS = (KeyError, TypeError, ValueError, AttributeError) +"""What :meth:`BaseSource.parse` may raise for a record it cannot read. Such a record is skipped.""" + class SourceName(StrEnum): - """Names of the built-in sources, as they appear in ``Tender.source`` and ``--source``.""" + """Names of the built-in sources, as they appear in ``Tender.source`` and ``--source``, in search order.""" RMI = "rmi" BEERSHEVA = "beersheva" GOV = "gov" + AKKO = "akko" + ARAD = "arad" + BETSHEMESH = "betshemesh" + DIMONA = "dimona" + KFAR_SABA = "kfar-saba" + KIRYAT_MOTZKIN = "kiryat-motzkin" + NESHER = "nesher" + NETIVOT = "netivot" + NOF_HAGALIL = "nof-hagalil" + SAVYON = "savyon" + TAMRA = "tamra" + YAVNE = "yavne" @runtime_checkable @@ -38,9 +55,15 @@ class Source(Protocol): errors: Records skipped during the last :meth:`fetch`, one message each. """ - name: str - description: str - attribution: str + @property + def name(self) -> str: ... + + @property + def description(self) -> str: ... + + @property + def attribution(self) -> str: ... + errors: list[str] def fetch(self) -> Iterator[Tender]: @@ -48,11 +71,13 @@ def fetch(self) -> Iterator[Tender]: ... -class BaseSource(Closeable, ABC): - """Shared plumbing for sources that read JSON over HTTP. +class BaseSource(Closeable, ABC, Generic[RecordT]): + """Template for a source: subclasses say how to read records and how to turn one into a tender. - A subclass sets ``name``, ``description`` and ``attribution`` and implements ``fetch``. Standalone use - works without any setup: the source builds its own polite client on first use. Inside + :meth:`fetch` is the same for every source. It reads :meth:`records`, converts each one with :meth:`parse`, + skips and reports records that cannot be parsed, and drops repeated tenders. A subclass sets the class + attributes ``name``, ``description`` and ``attribution`` and implements :meth:`records` and :meth:`parse`. + Standalone use works without any setup: the source builds its own polite client on first use. Inside :class:`michraz.Michraz` the facade's client is shared instead. Args: @@ -61,9 +86,9 @@ class BaseSource(Closeable, ABC): min_interval: Minimum seconds between requests to the same host. """ - name: str - description: str - attribution: str + name: ClassVar[str] + description: ClassVar[str] + attribution: ClassVar[str] def __init__( self, @@ -98,8 +123,38 @@ def _skip(self, record_id: object, exc: Exception) -> None: logger.warning(message) @abstractmethod + def records(self) -> Iterator[RecordT]: + """Reads the upstream data and yields its records one by one. + + Raises: + SourceUnavailableError: If the service cannot be reached. + SourceFormatError: If the response does not have the expected shape. + """ + + @abstractmethod + def parse(self, record: RecordT) -> Tender: + """Converts one record to a :class:`Tender`. May raise any of :data:`PARSE_ERRORS`.""" + + def record_id(self, record: RecordT) -> object: + """Identifies a record in the message reported when it is skipped.""" + return None + def fetch(self) -> Iterator[Tender]: """Yields every tender the source offers. A bad record is skipped and reported in ``errors``.""" + return self._tenders(self.records()) + + def _tenders(self, records: Iterable[RecordT]) -> Iterator[Tender]: + self.errors = [] + seen: set[tuple[str, str]] = set() + for record in records: + try: + tender = self.parse(record) + except PARSE_ERRORS as exc: + self._skip(self.record_id(record), exc) + continue + if tender.key not in seen: + seen.add(tender.key) + yield tender def last_fetched(self) -> datetime | None: """Returns when this source last fetched data into the cache, or None if it never did.""" diff --git a/src/michraz/sources/beersheva.py b/src/michraz/sources/beersheva.py index 2a79e62..6dff16d 100644 --- a/src/michraz/sources/beersheva.py +++ b/src/michraz/sources/beersheva.py @@ -111,7 +111,7 @@ def _item_id(url: str | None) -> str | None: return ids[0] if ids else None -class BeerShevaSource(BaseSource): +class BeerShevaSource(BaseSource[dict[str, Any]]): """Tenders of the Beer Sheva municipality (dataset ``tender-br7``, refreshed weekly). ``source_id`` is the municipality's own list-item id taken from the tender page URL. The tender number is @@ -138,28 +138,26 @@ def _params(self, offset: int) -> dict[str, Any]: def last_fetched(self) -> datetime | None: return self._client().cached_at(HttpMethod.GET, API_URL, self._params(0)) - def fetch(self) -> Iterator[Tender]: - """Yields every tender in the dataset. + def records(self) -> Iterator[dict[str, Any]]: + """Yields every row of the dataset, page by page. Raises: SourceUnavailableError: If data.gov.il cannot be reached. SourceFormatError: If a page is not a CKAN datastore response. """ - self.errors = [] http = self._client() offset = 0 while True: payload = http.get_json(self.name, API_URL, self._params(offset)) records, total = self._unwrap(payload) - for record in records: - try: - yield self.parse(record) - except (KeyError, TypeError, ValueError, AttributeError) as exc: - self._skip(record.get(_Column.ROW_ID) if isinstance(record, dict) else None, exc) + yield from records offset += len(records) if not records or offset >= total: return + def record_id(self, record: dict[str, Any]) -> object: + return record.get(_Column.ROW_ID) if isinstance(record, dict) else None + def _unwrap(self, payload: Any) -> tuple[list[Any], int]: try: if payload[_Ckan.SUCCESS] is not True: diff --git a/src/michraz/sources/bids.py b/src/michraz/sources/bids.py new file mode 100644 index 0000000..d8f43f9 --- /dev/null +++ b/src/michraz/sources/bids.py @@ -0,0 +1,270 @@ +"""Municipal tenders from the shared "bids" page that many Israeli municipal websites use. + +Every site that runs the platform publishes its current tenders at ``https://www./bids/`` as HTML tables. +The tables are laid out differently from city to city, so columns are found by their header text. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterator +from dataclasses import dataclass +from datetime import datetime, time +from html.parser import HTMLParser +from typing import Any, ClassVar +from urllib.parse import parse_qs, urljoin, urlparse + +from michraz import _dates +from michraz._enums import StrEnum +from michraz.errors import SourceFormatError +from michraz.models import RAW_EXTRA_KEY, Tender, TenderKind +from michraz.sources.base import BaseSource + +BIDS_PATH = "/bids/" +DATE_FORMAT = "%d/%m/%Y" +ITEM_ID_PARAMETER = "id" +TABLE_CLASS = "bids" +_HIDDEN_CLASSES = frozenset({"only-in-mobile", "mobile-only", "sr-only"}) +_VOID_TAGS = frozenset({"input", "br", "img", "hr", "meta", "link"}) +_DATE = re.compile(r"\d{2}/\d{2}/\d{4}") +_PUBLISHED_HEADER = re.compile(r"(מועד|תאריך) (פרסום|פתיחת)|התחלת") +_DEADLINE_HEADER = re.compile(r"אחרון להגשה|סגירת") +_VACANCY_SECTION = re.compile(r"דרושים|כוח אדם|כח אדם|משאבי אנוש|מכרזי כ\"א") +_VACANCY_TITLE = re.compile(r"דרושים|דרוש/ה|מכרז (חיצוני|פנימי|פנים)") + + +class BidsExtra(StrEnum): + """Keys of ``Tender.extra`` for tenders read from the bids page.""" + + TYPE = "type" + CATEGORY = "category" + RAW = RAW_EXTRA_KEY + + +class _Header(StrEnum): + """Header texts that name a column. Other headers are kept only in ``extra[RAW]``.""" + + NAME = "שם המכרז" + NUMBER = "מספר המכרז" + TYPE = "סוג המכרז" + CATEGORY = "קטגוריה" + + +@dataclass(slots=True) +class Table: + """One bids table: its caption, its header texts and its rows.""" + + caption: str + headers: list[str] + rows: list[list[Cell]] + + +@dataclass(slots=True) +class Cell: + text: str = "" + href: str | None = None + + +class _TableParser(HTMLParser): + """Collects every ```` as a :class:`Table`.""" + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.tables: list[Table] = [] + self._caption = "" + self._in_caption = False + self._in_table = False + self._headers: list[str] = [] + self._rows: list[list[Cell]] = [] + self._row: list[Cell] = [] + self._cell: Cell | None = None + self._is_header = False + self._hidden_tag: str | None = None + self._hidden_depth = 0 + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + values = dict(attrs) + classes = set((values.get("class") or "").split()) + if self._hidden_tag == tag: + self._hidden_depth += 1 + elif self._hidden_tag is None and tag not in _VOID_TAGS and classes & _HIDDEN_CLASSES: + self._hidden_tag, self._hidden_depth = tag, 1 + if tag == "table" and TABLE_CLASS in classes: + self._in_table, self._headers, self._rows, self._caption = True, [], [], "" + elif not self._in_table: + return + elif tag == "caption": + self._in_caption = True + elif tag == "tr": + self._row = [] + elif tag in {"td", "th"}: + self._cell, self._is_header = Cell(), tag == "th" + elif tag == "a" and self._cell is not None and self._cell.href is None: + self._cell.href = values.get("href") + + def handle_data(self, data: str) -> None: + if self._in_caption: + self._caption += data + elif self._cell is not None and self._hidden_tag is None: + self._cell.text += data + + def handle_endtag(self, tag: str) -> None: + if self._hidden_tag == tag: + self._hidden_depth -= 1 + if self._hidden_depth == 0: + self._hidden_tag = None + if not self._in_table: + return + if tag == "caption": + self._in_caption = False + elif tag in {"td", "th"} and self._cell is not None: + self._cell.text = " ".join(self._cell.text.split()) + self._row.append(self._cell) + self._cell = None + elif tag == "tr": + if self._is_header and not self._headers: + self._headers = [cell.text for cell in self._row] + elif self._row: + self._rows.append(self._row) + self._row, self._is_header = [], False + elif tag == "table": + self.tables.append(Table(" ".join(self._caption.split()), self._headers, self._rows)) + self._in_table = False + + +def parse_tables(page: str) -> list[Table]: + """Returns every bids table on the page.""" + parser = _TableParser() + parser.feed(page) + parser.close() + return parser.tables + + +def _column(headers: list[str], wanted: str | re.Pattern[str]) -> int | None: + for index, header in enumerate(headers): + matched = wanted.search(header) if isinstance(wanted, re.Pattern) else header == wanted + if matched: + return index + return None + + +def _cell_text(row: list[Cell], index: int | None) -> str | None: + if index is None or index >= len(row): + return None + return row[index].text or None + + +def _date(text: str | None, end_of_day: bool) -> datetime | None: + match = _DATE.search(text or "") + if match is None: + return None + moment = _dates.parse_local(match.group(), DATE_FORMAT) + assert moment is not None + return ( + datetime.combine(moment.date(), time.max.replace(microsecond=0), tzinfo=moment.tzinfo) if end_of_day else moment + ) + + +def _kind(caption: str, category: str | None, title: str) -> TenderKind: + """Staff vacancies are recognized by the section they are listed in, or by wording in the title.""" + if _VACANCY_SECTION.search(caption) or _VACANCY_SECTION.search(category or "") or _VACANCY_TITLE.search(title): + return TenderKind.VACANCY + return TenderKind.TENDER + + +class BidsSource(BaseSource[tuple[Table, list[Cell]]]): + """Base class for a municipality that runs the shared bids page. + + A subclass names the municipality and where its page lives, and nothing else:: + + class KfarSabaSource(BidsSource): + name = SourceName.KFAR_SABA + publisher = "עיריית כפר סבא" + domain = "www.kfar-saba.muni.il" + + ``description``, ``attribution`` and ``url`` are derived from these. A date-only deadline is read as the end + of that day, because the page gives no hour. Staff vacancies that the site lists beside its tenders get + ``kind=TenderKind.VACANCY``. + + Attributes: + publisher: Name of the municipality in Hebrew. + domain: Host of the municipal site. + path: Path of the bids page. Some sites put a language in front of it. + url: Full address of the bids page. + """ + + publisher: ClassVar[str] + domain: ClassVar[str] + path: ClassVar[str] = BIDS_PATH + url: ClassVar[str] + + def __init_subclass__(cls, **kwargs: Any) -> None: + super().__init_subclass__(**kwargs) + if "domain" in cls.__dict__: + cls.description = f"{cls.publisher} tenders ({cls.domain})" + cls.attribution = f"{cls.publisher}, {cls.domain}" + cls.url = f"https://{cls.domain}{cls.path}" + + def __init__(self, **settings: Any) -> None: + if not hasattr(type(self), "url"): + raise TypeError("BidsSource is a base class: subclass it and set name, publisher and domain") + super().__init__(**settings) + + def records(self) -> Iterator[tuple[Table, list[Cell]]]: + """Yields ``(table, row)`` for every row of every table that has a tender name column. + + Raises: + SourceUnavailableError: If the site cannot be reached. + SourceFormatError: If the page has no bids table. + """ + tables = parse_tables(self._client().get_text(self.name, self.url)) + usable = [table for table in tables if _column(table.headers, _Header.NAME) is not None] + if not usable: + raise SourceFormatError(self.name, "no bids table with a tender name column on the page") + for table in usable: + for row in table.rows: + yield table, row + + def record_id(self, record: tuple[Table, list[Cell]]) -> object: + _, row = record + return row[0].text[:60] if row else None + + def parse(self, record: tuple[Table, list[Cell]]) -> Tender: + """Converts one table row to a :class:`Tender`. + + Raises: + ValueError: If the row has no title or no identifier. + """ + table, row = record + headers = table.headers + title = _cell_text(row, _column(headers, _Header.NAME)) + if not title: + raise ValueError("row has no title") + number = _cell_text(row, _column(headers, _Header.NUMBER)) + link = next((cell.href for cell in row if cell.href), None) + name_index = _column(headers, _Header.NAME) + name_link = row[name_index].href if name_index is not None and name_index < len(row) else None + href = name_link or link + url = urljoin(f"https://{self.domain}/", href) if href else None + ids = parse_qs(urlparse(url or "").query).get(ITEM_ID_PARAMETER) + source_id = ids[0] if ids else number + if not source_id: + raise ValueError("row has neither an item id nor a tender number") + raw = {header: cell.text for header, cell in zip(headers, row, strict=False)} + return Tender( + source=self.name, + source_id=source_id, + title=title, + number=number, + publisher=self.publisher, + published_at=_date(_cell_text(row, _column(headers, _PUBLISHED_HEADER)), end_of_day=False), + deadline=_date(_cell_text(row, _column(headers, _DEADLINE_HEADER)), end_of_day=True), + url=url, + kind=_kind(table.caption, _cell_text(row, _column(headers, _Header.CATEGORY)), title), + attribution=self.attribution, + extra={ + BidsExtra.TYPE: _cell_text(row, _column(headers, _Header.TYPE)), + BidsExtra.CATEGORY: _cell_text(row, _column(headers, _Header.CATEGORY)), + BidsExtra.RAW: raw, + }, + ) diff --git a/src/michraz/sources/gov.py b/src/michraz/sources/gov.py index 29d302a..8e7d2c3 100644 --- a/src/michraz/sources/gov.py +++ b/src/michraz/sources/gov.py @@ -7,7 +7,6 @@ import re from collections.abc import Iterator -from datetime import datetime from html.parser import HTMLParser from typing import Any @@ -140,7 +139,7 @@ def parse_page(page: str) -> tuple[list[dict[str, Any]], int]: return parser.results, int(match.group(1).replace(",", "")) -class GovSource(BaseSource): +class GovSource(BaseSource[dict[str, Any]]): """Tenders of ministries and other central-government bodies, from mr.gov.il. Only tenders whose portal status says they are still taking offers are read (published, updated, new). @@ -158,19 +157,14 @@ class GovSource(BaseSource): def _params(status: _Status, page: int) -> dict[str, Any]: return {"q": f":updateDate:archive:false:status:{status}", "text": "", "s": "TENDER", "page": page} - def last_fetched(self) -> datetime | None: - return None - - def fetch(self) -> Iterator[Tender]: - """Yields every open tender, skipping records that fail to parse. + def records(self) -> Iterator[dict[str, Any]]: + """Yields every result of every open status, page by page. Raises: SourceUnavailableError: If mr.gov.il cannot be reached. SourceFormatError: If a page is not a portal search page. """ - self.errors = [] http = self._client() - seen: set[str] = set() for status in _Status: page, read = 0, 0 while True: @@ -179,31 +173,26 @@ def fetch(self) -> Iterator[Tender]: results, total = parse_page(text) except ValueError as exc: raise SourceFormatError(self.name, f"not a portal search page ({exc})") from exc - for result in results: - try: - tender = self.parse(result) - except (KeyError, ValueError) as exc: - self._skip(result.get("id"), exc) - continue - if tender.source_id not in seen: - seen.add(tender.source_id) - yield tender + yield from results read += len(results) page += 1 if not results or read >= total: break - def parse(self, result: dict[str, Any]) -> Tender: + def record_id(self, record: dict[str, Any]) -> object: + return record.get("id") + + def parse(self, record: dict[str, Any]) -> Tender: """Converts one parsed search result to a :class:`Tender`. Raises: KeyError: If a required value is missing. ValueError: If a date does not match the portal format. """ - source_id, title = result["id"], result["title"] + source_id, title = record["id"], record["title"] if not source_id or not title: raise KeyError("id or title") - fields: dict[str, str] = result["fields"] + fields: dict[str, str] = record["fields"] return Tender( source=self.name, source_id=source_id, @@ -215,5 +204,5 @@ def parse(self, result: dict[str, Any]) -> Tender: deadline=_dates.parse_local(fields.get(_Label.DEADLINE), DATETIME_FORMAT), url=TENDER_URL.format(source_id), attribution=self.attribution, - extra={GovExtra.TYPE: result["type"], GovExtra.STATUS: fields.get(_Label.STATUS), GovExtra.RAW: result}, + extra={GovExtra.TYPE: record["type"], GovExtra.STATUS: fields.get(_Label.STATUS), GovExtra.RAW: record}, ) diff --git a/src/michraz/sources/municipal.py b/src/michraz/sources/municipal.py new file mode 100644 index 0000000..85ea767 --- /dev/null +++ b/src/michraz/sources/municipal.py @@ -0,0 +1,104 @@ +"""Municipalities that run the shared bids page. Each one is a short subclass of :class:`BidsSource`.""" + +from __future__ import annotations + +from michraz.sources.base import SourceName +from michraz.sources.bids import BidsSource + + +class AkkoSource(BidsSource): + """Tenders of עיריית עכו.""" + + name = SourceName.AKKO.value + publisher = "עיריית עכו" + domain = "www.akko.muni.il" + path = "/he/bids/" + + +class AradSource(BidsSource): + """Tenders of עיריית ערד.""" + + name = SourceName.ARAD.value + publisher = "עיריית ערד" + domain = "www.arad.muni.il" + + +class BetShemeshSource(BidsSource): + """Tenders of עיריית בית שמש.""" + + name = SourceName.BETSHEMESH.value + publisher = "עיריית בית שמש" + domain = "www.betshemesh.muni.il" + + +class DimonaSource(BidsSource): + """Tenders of עיריית דימונה.""" + + name = SourceName.DIMONA.value + publisher = "עיריית דימונה" + domain = "www.dimona.muni.il" + + +class KfarSabaSource(BidsSource): + """Tenders of עיריית כפר סבא.""" + + name = SourceName.KFAR_SABA.value + publisher = "עיריית כפר סבא" + domain = "www.kfar-saba.muni.il" + + +class KiryatMotzkinSource(BidsSource): + """Tenders of עיריית קריית מוצקין.""" + + name = SourceName.KIRYAT_MOTZKIN.value + publisher = "עיריית קריית מוצקין" + domain = "www.kiryat-motzkin.muni.il" + + +class NesherSource(BidsSource): + """Tenders of עיריית נשר.""" + + name = SourceName.NESHER.value + publisher = "עיריית נשר" + domain = "www.nesher.muni.il" + + +class NetivotSource(BidsSource): + """Tenders of עיריית נתיבות.""" + + name = SourceName.NETIVOT.value + publisher = "עיריית נתיבות" + domain = "www.netivot.muni.il" + + +class NofHaGalilSource(BidsSource): + """Tenders of עיריית נוף הגליל.""" + + name = SourceName.NOF_HAGALIL.value + publisher = "עיריית נוף הגליל" + domain = "www.nof-hagalil.muni.il" + + +class SavyonSource(BidsSource): + """Tenders of המועצה המקומית סביון.""" + + name = SourceName.SAVYON.value + publisher = "המועצה המקומית סביון" + domain = "www.savyon.muni.il" + + +class TamraSource(BidsSource): + """Tenders of עיריית טמרה.""" + + name = SourceName.TAMRA.value + publisher = "עיריית טמרה" + domain = "www.tamra.muni.il" + path = "/ar/bids/" + + +class YavneSource(BidsSource): + """Tenders of עיריית יבנה.""" + + name = SourceName.YAVNE.value + publisher = "עיריית יבנה" + domain = "www.yavne.muni.il" diff --git a/src/michraz/sources/registry.py b/src/michraz/sources/registry.py new file mode 100644 index 0000000..7e9e9b9 --- /dev/null +++ b/src/michraz/sources/registry.py @@ -0,0 +1,52 @@ +"""The built-in sources, in search order.""" + +from __future__ import annotations + +from typing import Any + +from michraz.sources.base import BaseSource +from michraz.sources.beersheva import BeerShevaSource +from michraz.sources.gov import GovSource +from michraz.sources.municipal import ( + AkkoSource, + AradSource, + BetShemeshSource, + DimonaSource, + KfarSabaSource, + KiryatMotzkinSource, + NesherSource, + NetivotSource, + NofHaGalilSource, + SavyonSource, + TamraSource, + YavneSource, +) +from michraz.sources.rmi import RmiSource + +BUILT_IN: tuple[type[BaseSource[Any]], ...] = ( + RmiSource, + BeerShevaSource, + GovSource, + AkkoSource, + AradSource, + BetShemeshSource, + DimonaSource, + KfarSabaSource, + KiryatMotzkinSource, + NesherSource, + NetivotSource, + NofHaGalilSource, + SavyonSource, + TamraSource, + YavneSource, +) +"""Every built-in source class. Its order matches :class:`SourceName`.""" + + +def default_sources(**settings: Any) -> tuple[BaseSource[Any], ...]: + """Returns one instance of every built-in source. + + Args: + **settings: ``cache_dir``, ``cache_ttl`` and ``min_interval``, passed to each source. + """ + return tuple(source(**settings) for source in BUILT_IN) diff --git a/src/michraz/sources/rmi.py b/src/michraz/sources/rmi.py index 42c59a4..da13d11 100644 --- a/src/michraz/sources/rmi.py +++ b/src/michraz/sources/rmi.py @@ -86,7 +86,7 @@ def _title(settlement: str | None, neighborhood: str | None, number: str | None, return TITLE_SEPARATOR.join(parts) if parts else (number or source_id) -class RmiSource(BaseSource): +class RmiSource(BaseSource[dict[str, Any]]): """Open land tenders of the Israel Land Authority. The API returns every match in one response, so there is no paging. Only ``settlement_code`` and @@ -117,9 +117,12 @@ def _body(active_only: bool, settlement_code: int | None, neighborhood: str | No def last_fetched(self) -> datetime | None: return self._client().cached_at(HttpMethod.POST, SEARCH_URL, payload=self._body(True, None, None)) - def fetch(self) -> Iterator[Tender]: - """Yields all currently open tenders. Same as ``iter(search())``.""" - return iter(self.search()) + def records(self) -> Iterator[dict[str, Any]]: + """Yields the records of all currently open tenders.""" + return self._read(active_only=True) + + def record_id(self, record: dict[str, Any]) -> object: + return record.get(_Field.ID) if isinstance(record, dict) else None def search( self, @@ -157,7 +160,35 @@ def search( SourceUnavailableError: If the service cannot be reached. SourceFormatError: If the response is not a list of records. """ - self.errors = [] + return list( + self._tenders( + self._read( + active_only=active_only, + settlement_code=settlement_code, + neighborhood=neighborhood, + tender_number=tender_number, + region=region, + tender_type=tender_type, + purpose=purpose, + status=status, + raw_filters=raw_filters, + ) + ) + ) + + def _read( + self, + *, + active_only: bool, + settlement_code: int | None = None, + neighborhood: str | None = None, + tender_number: str | None = None, + region: int | None = None, + tender_type: int | None = None, + purpose: int | None = None, + status: int | None = None, + raw_filters: dict[str, Any] | None = None, + ) -> Iterator[dict[str, Any]]: body = self._body(active_only, settlement_code, neighborhood) body.update(raw_filters or {}) payload = self._client().post_json(self.name, SEARCH_URL, body) @@ -170,15 +201,10 @@ def search( _Field.PURPOSE: purpose, _Field.STATUS: status, } - tenders = [] for record in payload: - try: - if any(v is not None and record.get(k) != v for k, v in wanted.items()): - continue - tenders.append(self.parse(record)) - except (KeyError, TypeError, ValueError, AttributeError) as exc: - self._skip(record.get(_Field.ID) if isinstance(record, dict) else None, exc) - return tenders + if isinstance(record, dict) and any(v is not None and record.get(k) != v for k, v in wanted.items()): + continue + yield record def parse(self, record: dict[str, Any]) -> Tender: """Converts one search result to a :class:`Tender`. diff --git a/tests/fixtures/bids_akko.html b/tests/fixtures/bids_akko.html new file mode 100644 index 0000000..93a4ad3 --- /dev/null +++ b/tests/fixtures/bids_akko.html @@ -0,0 +1,24 @@ +
+ + + + + + + + + + + + + + + + + + + + + + +
שם המכרזתאריך פתיחת המכרזתאריך סגירת המכרז
שם המכרז: מכרז פומבי 36-2026 הפעלה וביצוע עבודות נוספות בבית חולים גריאטרי ובית אבות בעיר עכותאריך פתיחת המכרז: 04/11/2026תאריך סגירת המכרז: 28/12/2026
שם המכרז: קול קורא להגשת הצעות לביצוע ציורים אומנותיים על קיר, ארונות חשמל ותקשורת ומיגוניות ברחבי העירתאריך פתיחת המכרז: 19/10/2026תאריך סגירת המכרז: 05/11/2026
שם המכרז: מכרז פומבי  מס' 27/2026  השכרת שטחים  במתחם ברחוב הרצוג - הארכהתאריך פתיחת המכרז: 05/10/2026תאריך סגירת המכרז: 29/11/2026
diff --git a/tests/fixtures/bids_arad.html b/tests/fixtures/bids_arad.html new file mode 100644 index 0000000..ce68dff --- /dev/null +++ b/tests/fixtures/bids_arad.html @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
שם המכרז סוג המכרז תאריך פתיחת המכרז תאריך סגירת המכרז רכישת המכרז
+ + הודעה על כוונה להתקשר עם ספק יחיד עפ"י תקנה 3(4) לתקנות העיריות (מכרזים), התשמ"ח – 1987 + פומבי01/11/202629/11/2026 +
+ + מכרז פומבי מס' 27/2026 : ביצוע עבודות פירוק, הובלה, אחסנה, תיקון, עבודות תחזוקה והרכבה של מתקני הצללה ברחבי העיר ערד + פומבי16/10/202612/11/2026 + רכישה +
diff --git a/tests/fixtures/bids_betshemesh.html b/tests/fixtures/bids_betshemesh.html new file mode 100644 index 0000000..482209b --- /dev/null +++ b/tests/fixtures/bids_betshemesh.html @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + +
מכרזי כ"א
שם המכרזסוג המכרזהתחלת המכרזסגירת המכרז
שם המכרז: מכרז חיצוני 164-26 תומך/ת עו''ססוג המכרז: חיצוניהתחלת המכרז: 01/11/2026סגירת המכרז: 18/11/2026
שם המכרז: מכרז חיצוני 163-26 אדריכל/ית הרשות וסגן/ית מהנדס העירסוג המכרז: חיצוניהתחלת המכרז: 22/10/2026סגירת המכרז: 10/11/2026
שם המכרז: מכרז חיצוני 157-26 פסיכולוג/ית חינוכי/ת - הארכת הגשהסוג המכרז: חיצוניהתחלת המכרז: 05/10/2026סגירת המכרז: 05/11/2026
diff --git a/tests/fixtures/bids_dimona.html b/tests/fixtures/bids_dimona.html new file mode 100644 index 0000000..2bf5db1 --- /dev/null +++ b/tests/fixtures/bids_dimona.html @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
כח אדם
מספר המכרזמועד פרסוםשם המכרזנוסח פרסוםמסמכי המכרזמועד אחרון להגשההבהרות
68/2603/11/2026מנכ"ל החברה לפיתוח דימונה בע"מ + להורדה לחצו כאן + + 17/11/2026
63/2601/11/2026מנהל מחלקת הבריאות ברשות + להורדה לחצו כאן + + 15/11/2026
71/2601/11/2026מתאם הסעות לאגף הרווחה + להורדה לחצו כאן + + 15/11/2026
diff --git a/tests/fixtures/bids_kfar-saba.html b/tests/fixtures/bids_kfar-saba.html new file mode 100644 index 0000000..43afe6d --- /dev/null +++ b/tests/fixtures/bids_kfar-saba.html @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
מכרזי משאבי אנוש
שם המכרזסוג המכרזתאריך פרסום המכרזמועד אחרון להגשה
מכרז פומבי 139/26 - מנהל/ת מועדון גימלאים כיסופיםפומבי28/10/202612/11/2026
מכרז פומבי 140/26 מנהל/ת אולם ספורט טהרלבפומבי28/10/202612/11/2026
מכרז פומבי 144/26 מנהל/ת תחום תכנון במינהלת התחדשות עירוניתפומבי28/10/202612/11/2026
+ + + + + + + + + + + + + + + + + + + + + + + + +
קולות קוראים ו-RFI
שם המכרזסוג המכרזתאריך פרסום המכרזמועד אחרון להגשה
קול קורא לקבלת אות הוקרה עירוני לעסקים בעיר כפר סבא, המעסיקים אנשים עם מוגבלות20/10/202615/11/2026
הזמנה להגשת בקשה להיכלל במאגר יועצים של עיריית כפר-סבאפומבי
diff --git a/tests/fixtures/bids_kiryat-motzkin.html b/tests/fixtures/bids_kiryat-motzkin.html new file mode 100644 index 0000000..d62304d --- /dev/null +++ b/tests/fixtures/bids_kiryat-motzkin.html @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
שם המכרזקטגוריהסוג המכרזתאריך פתיחת המכרזתאריך סגירת המכרז
שם המכרז: מכרז 26\6 לתפעול אספקה והתקנה של מתקני פרסום חוצות ומתן זכות שיווק ופרסום על מתקנים אלו בעיר קריית מוצקיןקטגוריה: מכרזיםסוג המכרז: פומביתאריך פתיחת המכרז: 01/11/2026תאריך סגירת המכרז: 09/12/2026
שם המכרז: מכרז לביצוע עבודות אחזקה שוטפת של שטחי גינון ציבוריים בתחומי העיר קריית מוצקיןקטגוריה: מכרזיםסוג המכרז: פומביתאריך פתיחת המכרז: 28/10/2026תאריך סגירת המכרז: 09/12/2026
שם המכרז: מכרז פומבי מס' 11/26 לאספקה ותחזוקת מערכות מידע לניהול הרשות לרבות גבייה, גזברות, רכש, חינוך, רישוי עסקים, רווחה, הנדסה, משאבי אנוש, שכר וכו'קטגוריה: מכרזיםסוג המכרז: פומביתאריך פתיחת המכרז: 22/10/2026תאריך סגירת המכרז: 03/12/2026
diff --git a/tests/fixtures/bids_nesher.html b/tests/fixtures/bids_nesher.html new file mode 100644 index 0000000..ca5404a --- /dev/null +++ b/tests/fixtures/bids_nesher.html @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + +
מכרזים והצעות - עיריית נשר
שם המכרזסוג המכרזהתחלת המכרזסגירת המכרז
שם המכרז: מכרז פומבי מס' 15/26 למתן שירותי ביטוח רכוש וחבויות לעיריית נשרסוג המכרז: פומביהתחלת המכרז: 02/11/2026סגירת המכרז: 23/11/2026
diff --git a/tests/fixtures/bids_netivot.html b/tests/fixtures/bids_netivot.html new file mode 100644 index 0000000..6cf6334 --- /dev/null +++ b/tests/fixtures/bids_netivot.html @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
שם המכרזקטגוריהסוג המכרזתאריך פתיחת המכרזתאריך סגירת המכרז
שם המכרז: קול קורא לקבלת הצעות להשכרת דירת מגורים עבור עיריית נתיבות (עבור מיזם אנובנו)קטגוריה: קולות קוראיםסוג המכרז: חיצוניתאריך פתיחת המכרז: 03/11/2026תאריך סגירת המכרז: 24/11/2026
שם המכרז: מכרז פומבי מס' 47/2026 לתכנון, אספקה והקמה של מתקני משחק, כושר ופנאי בשכונת רמות יורם ופארק יום הולדת במתחם האגם בעיר נתיבותקטגוריה: כלליסוג המכרז: פומביתאריך פתיחת המכרז: 02/11/2026תאריך סגירת המכרז: 26/11/2026
שם המכרז: מכרז פומבי מס' 22/2026 למתן שירותי אחזקת מזרקות ברחבי העיר נתיבותקטגוריה: כלליסוג המכרז: פומביתאריך פתיחת המכרז: 16/09/2026תאריך סגירת המכרז: 11/11/2026
diff --git a/tests/fixtures/bids_nof-hagalil.html b/tests/fixtures/bids_nof-hagalil.html new file mode 100644 index 0000000..c96da82 --- /dev/null +++ b/tests/fixtures/bids_nof-hagalil.html @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + +
מכרזי קבלנים, ספקים ונותני שירותים
שם המכרזסוג המכרזהתחלת המכרזסגירת המכרז
הודעה על כוונה להתקשר עם ספק יחיד במסגרת קול קורא של משרד התרבות והספורט  "אוצרות תרבות"פומבי04/11/2026
הודעה על כוונה להתקשר עם ספק יחיד "כנפיים של קרמבו"פומבי04/11/2026
הודעה על כוונה להתקשר עם ספק יחיד עם עמותת המפעל הלאומי לפתוח חברתי (ע"ר) מספר עמותה 580231546 ספק יחיד להפעלת פרויקט "מחשב לכל ילד" לשנת 2026פומבי19/10/2026
diff --git a/tests/fixtures/bids_savyon.html b/tests/fixtures/bids_savyon.html new file mode 100644 index 0000000..6236668 --- /dev/null +++ b/tests/fixtures/bids_savyon.html @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
שם המכרזקטגוריהסוג המכרזתאריך פתיחת המכרזתאריך סגירת המכרז
שם המכרז: מכרז פומבי מס' 43/2026 למתן שירותי תחזוקה מלאה, שוטפת ומונעת לרבות חלפים לחמשת מחסומי הכניסה והיציאה הקיימים ואופציה לרכישת מחסומים חדשיםקטגוריה: כלליסוג המכרז: פומביתאריך פתיחת המכרז: 20/10/2026תאריך סגירת המכרז: 09/11/2026
שם המכרז: מכרז פומבי מס' 40/2026 לביצוע עבודות גיזום עצים וצמחיה ברחבי המועצה המקומית סביון קטגוריה: כלליסוג המכרז: פומביתאריך פתיחת המכרז: 13/10/2026תאריך סגירת המכרז: 12/11/2026
שם המכרז: קול קורא מס' 41/2026 למתן שירותי צילומי אוויר והפקת אורתופוטו, שכבות מיפוי ומודל תלת מימדקטגוריה: קולות קוראיםסוג המכרז: פומביתאריך פתיחת המכרז: 06/10/2026תאריך סגירת המכרז: 05/11/2026
diff --git a/tests/fixtures/bids_tamra.html b/tests/fixtures/bids_tamra.html new file mode 100644 index 0000000..b183e20 --- /dev/null +++ b/tests/fixtures/bids_tamra.html @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
مناقصات
שם המכרזסוג המכרזתאריך פתיחת המכרזתאריך סגירת המכרזפרוטוקול
+ + مناقصة عامة رقم 38/2026 الخاصة بإزالة وتركيب عشب صناعي في ملعب كرة قدم مصغر في مدينة طمرة + 29/10/202619/11/2026 +
+ + مناقصة عامة رقم 39/2026 الخاصة بإزالة وتركيب عشب صناعي في ملعب كرة قدم مصغر بجانب مدرسه الخوارزمي مدينة طمرة + 29/10/202619/11/2026 +
+ + المناقصة العامة رقم 36/2026 لتنفيذ أعمال تأهيل الطريق رقم 93 – حي معاذ بن جبل في مدينة طمرة + 19/10/202605/11/2026 +
diff --git a/tests/fixtures/bids_yavne.html b/tests/fixtures/bids_yavne.html new file mode 100644 index 0000000..6ae7ef5 --- /dev/null +++ b/tests/fixtures/bids_yavne.html @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
שם המכרז סוג המכרז תאריך פתיחת המכרז תאריך סגירת המכרז
+ + מכרז 40-2026- ניהול תיקי השקעות עודפי מזומנים + חיצוני03/11/202603/12/2026
+ + הודעה על כוונה להתקשר עם ספק יחיד-אקווה מערכות ניהול תשתיות בע"מ + חיצוני03/11/202623/11/2026
+ + מכרז 12/2026- קבלת שירותים משפטיים חיצוניים לעירייה        + חיצוני14/10/202608/11/2026
diff --git a/tests/helpers.py b/tests/helpers.py index 28d197a..55fd3f3 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -5,6 +5,7 @@ from michraz import Tender from michraz._dates import JERUSALEM +from michraz.sources import BUILT_IN, BidsSource FIXTURES = Path(__file__).parent / "fixtures" @@ -44,3 +45,14 @@ def make_tender(**overrides: Any) -> Tender: GOV_EMPTY_PAGE = "

נמצאו 0 תוצאות עבור ערך במכרזים

" + + +CITIES: tuple[type[BidsSource], ...] = tuple(c for c in BUILT_IN if issubclass(c, BidsSource)) + +BIDS_EMPTY_PAGE = '
שם המכרזמועד אחרון להגשה
' + + +def mock_municipalities(respx_mock: Any) -> None: + """Answers every municipal bids page with an empty table.""" + for city in CITIES: + respx_mock.get(city.url).respond(text=BIDS_EMPTY_PAGE) diff --git a/tests/live/test_live_smoke.py b/tests/live/test_live_smoke.py index 6788ef9..2e07df2 100644 --- a/tests/live/test_live_smoke.py +++ b/tests/live/test_live_smoke.py @@ -6,7 +6,8 @@ import pytest -from michraz import BeerShevaSource, GovSource, RmiSource +from michraz import BeerShevaSource, BidsSource, GovSource, RmiSource +from michraz.sources import BUILT_IN pytestmark = pytest.mark.live @@ -41,3 +42,12 @@ def test_gov_first_page_shape(): assert tender.title and tender.source_id and tender.publisher assert tender.url.startswith("https://mr.gov.il/") assert not source.errors + + +@pytest.mark.parametrize("city", [c for c in BUILT_IN if issubclass(c, BidsSource)], ids=lambda c: c.name) +def test_municipal_bids_page_shape(city: type[BidsSource]): + with city(cache_dir=None) as source: + tenders = list(source.fetch()) + assert tenders, f"{source.name} lists no tenders" + assert not source.errors + assert all(t.title and t.source_id and t.url for t in tenders) diff --git a/tests/test_base.py b/tests/test_base.py new file mode 100644 index 0000000..e0bfcd7 --- /dev/null +++ b/tests/test_base.py @@ -0,0 +1,72 @@ +from collections.abc import Iterator + +import pytest + +from michraz import Tender +from michraz.sources import BUILT_IN, BaseSource, Source, SourceName, default_sources +from tests.helpers import make_tender + + +class ListSource(BaseSource[dict[str, str]]): + name = "list" + description = "records from a list" + attribution = "test" + + def __init__(self, records: list[dict[str, str]]) -> None: + super().__init__(cache_dir=None) + self._records = records + + def records(self) -> Iterator[dict[str, str]]: + return iter(self._records) + + def record_id(self, record: dict[str, str]) -> object: + return record.get("id") + + def parse(self, record: dict[str, str]) -> Tender: + return make_tender(source=self.name, source_id=record["id"], title=record["title"]) + + +def test_fetch_parses_every_record(): + source = ListSource([{"id": "1", "title": "a"}, {"id": "2", "title": "b"}]) + assert [t.source_id for t in source.fetch()] == ["1", "2"] + assert source.errors == [] + + +def test_fetch_skips_and_reports_records_that_cannot_be_parsed(): + source = ListSource([{"id": "1", "title": "a"}, {"id": "2"}, {"id": "3", "title": "c"}]) + assert [t.source_id for t in source.fetch()] == ["1", "3"] + assert len(source.errors) == 1 and "'2'" in source.errors[0] and "KeyError" in source.errors[0] + + +def test_fetch_drops_repeated_tenders(): + source = ListSource([{"id": "1", "title": "a"}, {"id": "1", "title": "again"}]) + assert [t.title for t in source.fetch()] == ["a"] + + +def test_errors_start_empty_on_every_fetch(): + source = ListSource([{"id": "2"}]) + list(source.fetch()) + source._records = [{"id": "1", "title": "a"}] + list(source.fetch()) + assert source.errors == [] + + +def test_a_source_without_records_or_parse_cannot_be_created(): + class Incomplete(BaseSource[str]): + name = "x" + + with pytest.raises(TypeError): + Incomplete() # type: ignore[abstract] + + +@pytest.mark.parametrize("cls", BUILT_IN, ids=lambda c: c.name) +def test_every_built_in_source_follows_the_same_contract(cls): + assert issubclass(cls, BaseSource) + assert cls.name in {n.value for n in SourceName} and type(cls.name) is str + assert cls.description and cls.attribution + source = cls(cache_dir=None) + assert isinstance(source, Source) and source.errors == [] + + +def test_default_sources_follow_the_registry_order(): + assert [type(s) for s in default_sources(cache_dir=None)] == list(BUILT_IN) diff --git a/tests/test_bids.py b/tests/test_bids.py new file mode 100644 index 0000000..26ef170 --- /dev/null +++ b/tests/test_bids.py @@ -0,0 +1,166 @@ +from datetime import timedelta + +import pytest + +from michraz import BidsSource, SourceFormatError, SourceUnavailableError, TenderKind, default_sources +from michraz.sources import BUILT_IN, BidsExtra, SourceName +from michraz.sources.bids import Cell, Table, parse_tables +from tests.helpers import CITIES as CITY_CLASSES +from tests.helpers import FIXTURES, at + +CITIES = {c.name: c for c in CITY_CLASSES} + + +def page(key: str) -> str: + return (FIXTURES / f"bids_{key}.html").read_text(encoding="utf-8") + + +def fetch(respx_mock, http, key: str, body: str | None = None): + city = CITIES[key] + respx_mock.get(city.url).respond(text=body if body is not None else page(key)) + source = city() + source.bind(http) + return source, list(source.fetch()) + + +@pytest.mark.parametrize("key", sorted(CITIES)) +def test_every_layout_parses_without_errors(respx_mock, http, key): + source, tenders = fetch(respx_mock, http, key) + assert tenders and not source.errors + assert all(t.title and t.source_id and t.publisher == CITIES[key].publisher for t in tenders) + assert all(t.source == key and CITIES[key].domain in t.attribution for t in tenders) + + +def test_basic_fields_and_end_of_day_deadline(respx_mock, http): + _, tenders = fetch(respx_mock, http, "kfar-saba") + t = next(t for t in tenders if t.source_id == "663") + assert t.title == "מכרז פומבי 139/26 - מנהל/ת מועדון גימלאים כיסופים" + assert t.url == "https://www.kfar-saba.muni.il/bids/?id=663" + assert t.extra[BidsExtra.TYPE] == "פומבי" + assert (t.published_at.day, t.published_at.hour) == (28, 0) + assert (t.deadline.month, t.deadline.day, t.deadline.hour, t.deadline.minute) == (11, 12, 23, 59) + assert t.deadline.utcoffset() == timedelta(hours=2) + + +def test_a_tender_is_open_through_its_last_day(respx_mock, http): + _, tenders = fetch(respx_mock, http, "kfar-saba") + t = next(t for t in tenders if t.source_id == "663") + assert t.is_open(at(2026, 11, 12, 18, 0)) + assert not t.is_open(at(2026, 11, 13, 0, 1)) + + +@pytest.mark.parametrize("key", ["akko", "kiryat-motzkin", "betshemesh"]) +def test_labels_meant_for_mobile_screens_are_not_part_of_the_text(respx_mock, http, key): + _, tenders = fetch(respx_mock, http, key) + assert not any(t.title.startswith("שם המכרז") for t in tenders) + assert all(t.deadline is not None for t in tenders) + + +def test_a_void_element_with_a_hiding_class_does_not_hide_the_page(respx_mock, http): + _, tenders = fetch(respx_mock, http, "nof-hagalil") + assert tenders and all(t.title for t in tenders) + + +def test_category_column_is_kept(respx_mock, http): + _, tenders = fetch(respx_mock, http, "kiryat-motzkin") + assert {t.extra[BidsExtra.CATEGORY] for t in tenders} == {"מכרזים"} + + +def test_rows_without_an_item_id_use_the_tender_number(respx_mock, http): + _, tenders = fetch(respx_mock, http, "dimona") + t = tenders[0] + assert t.source_id == t.number == "68/26" + assert t.url.endswith(".pdf") + + +def test_missing_deadline_is_none(respx_mock, http): + _, tenders = fetch(respx_mock, http, "nof-hagalil") + assert all(t.deadline is None for t in tenders) and not any(t.is_open() for t in tenders) + + +def test_language_prefixed_pages_are_requested_where_they_live(): + assert CITIES["akko"].url == "https://www.akko.muni.il/he/bids/" + assert CITIES["tamra"].url == "https://www.tamra.muni.il/ar/bids/" + + +def test_a_row_without_a_title_is_skipped_and_reported(respx_mock, http): + body = page("kfar-saba").replace("מכרז פומבי 139/26 - מנהל/ת מועדון גימלאים כיסופים", "") + source, tenders = fetch(respx_mock, http, "kfar-saba", body) + assert "663" not in {t.source_id for t in tenders} + assert len(source.errors) == 1 + + +def test_a_tender_in_two_tables_is_returned_once(respx_mock, http): + body = page("yavne") + page("yavne") + _, tenders = fetch(respx_mock, http, "yavne", body) + assert len(tenders) == len({t.source_id for t in tenders}) + + +def test_page_without_a_bids_table_raises_format_error(respx_mock, http): + city = CITIES["arad"] + respx_mock.get(city.url).respond(text="moved") + source = city() + source.bind(http) + with pytest.raises(SourceFormatError): + list(source.fetch()) + + +def test_server_error_raises_unavailable(respx_mock, http): + city = CITIES["arad"] + respx_mock.get(city.url).respond(status_code=503) + source = city() + source.bind(http) + with pytest.raises(SourceUnavailableError): + list(source.fetch()) + + +def test_parse_tables_ignores_tables_that_are_not_bids_tables(): + tables = parse_tables("
x
y
") + assert [(t.headers, t.rows) for t in tables] == [(["y"], [])] + + +def test_registry_is_consistent(): + names = [c.name for c in BUILT_IN] + assert names == [n.value for n in SourceName] + assert len(names) == len(set(names)) + for city in CITY_CLASSES: + assert city.domain == f"www.{city.name}.muni.il" and city.publisher and city.path.endswith("/bids/") + assert city.url == f"https://{city.domain}{city.path}" + assert city.publisher in city.attribution and city.domain in city.description + assert "azor" not in names # its robots.txt disallows every path + assert [s.name for s in default_sources()] == names + + +def test_the_base_class_cannot_be_used_directly(): + with pytest.raises(TypeError): + BidsSource() + + +@pytest.mark.parametrize( + ("key", "source_id", "kind"), + [ + ("kfar-saba", "663", TenderKind.VACANCY), # table captioned "human resources tenders" + ("kfar-saba", "662", TenderKind.TENDER), # table captioned "calls for proposals" + ("dimona", "68/26", TenderKind.VACANCY), # table captioned "manpower" + ("betshemesh", "3312", TenderKind.VACANCY), # table captioned "HR tenders" + ("yavne", "177", TenderKind.TENDER), # external investment-management tender + ("nof-hagalil", "1923", TenderKind.TENDER), + ], +) +def test_staff_vacancies_are_told_apart_from_tenders(respx_mock, http, key, source_id, kind): + _, tenders = fetch(respx_mock, http, key) + assert next(t for t in tenders if t.source_id == source_id).kind == kind + + +def test_a_vacancy_is_recognized_by_its_title_when_the_table_has_no_caption(): + source = CITIES["savyon"]() + table = Table(caption="", headers=["שם המכרז"], rows=[]) + job = source.parse((table, [Cell("מכרז חיצוני 12/26 מנהל/ת מחלקה", "?id=1")])) + ad = source.parse((table, [Cell("מודעת דרושים D1-26 סייעת", "?id=2")])) + work = source.parse((table, [Cell("מכרז פומבי 12/26 עבודות גינון", "?id=3")])) + assert (job.kind, ad.kind, work.kind) == (TenderKind.VACANCY, TenderKind.VACANCY, TenderKind.TENDER) + + +def test_the_caption_is_read_even_when_the_site_hides_it(): + tables = parse_tables('
דרושים
x
') + assert tables[0].caption == "דרושים" diff --git a/tests/test_cli.py b/tests/test_cli.py index fdfd94f..79289e2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -8,7 +8,7 @@ from michraz.sources.beersheva import API_URL from michraz.sources.gov import SEARCH_URL as GOV_URL from michraz.sources.rmi import SEARCH_URL -from tests.helpers import GOV_EMPTY_PAGE, ckan +from tests.helpers import GOV_EMPTY_PAGE, ckan, mock_municipalities @pytest.fixture @@ -16,6 +16,7 @@ def served(respx_mock, rmi_records, bs_records): respx_mock.post(SEARCH_URL).respond(json=rmi_records) respx_mock.get(API_URL).respond(json=ckan(bs_records)) respx_mock.get(GOV_URL).respond(text=GOV_EMPTY_PAGE) + mock_municipalities(respx_mock) def run(capsys, *argv): diff --git a/tests/test_client.py b/tests/test_client.py index 5c64ee2..cdee88b 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -3,11 +3,11 @@ import pytest -from michraz import Michraz, RmiSource, SourceUnavailableError, Tender +from michraz import BeerShevaSource, Michraz, RmiSource, SourceUnavailableError, Tender from michraz.sources.beersheva import API_URL from michraz.sources.gov import SEARCH_URL as GOV_URL from michraz.sources.rmi import SEARCH_URL -from tests.helpers import GOV_EMPTY_PAGE, at, ckan, make_tender +from tests.helpers import GOV_EMPTY_PAGE, at, ckan, make_tender, mock_municipalities class FakeSource: @@ -28,6 +28,7 @@ def served(respx_mock, rmi_records, bs_records): respx_mock.post(SEARCH_URL).respond(json=rmi_records) respx_mock.get(API_URL).respond(json=ckan(bs_records)) respx_mock.get(GOV_URL).respond(text=GOV_EMPTY_PAGE) + mock_municipalities(respx_mock) def test_searches_every_default_source_sorted_by_deadline(served): @@ -100,9 +101,28 @@ def test_sources_share_the_facade_client(served): assert rmi._http is m._http -def test_a_source_that_is_down_raises(respx_mock): +def test_a_source_that_is_down_raises_when_it_is_the_only_one(respx_mock): respx_mock.post(SEARCH_URL).respond(404) - with Michraz(cache_dir=None) as m, pytest.raises(SourceUnavailableError): + with Michraz([RmiSource()], cache_dir=None) as m, pytest.raises(SourceUnavailableError): + m.search() + + +def test_one_source_down_does_not_hide_the_others(respx_mock, bs_records): + respx_mock.post(SEARCH_URL).respond(404) + respx_mock.get(API_URL).respond(json=ckan(bs_records)) + with Michraz([RmiSource(), BeerShevaSource()], cache_dir=None) as m: + tenders = m.search() + assert {t.source for t in tenders} == {"beersheva"} + assert len(m.last_errors) == 1 and m.last_errors[0].startswith("rmi: HTTP 404") + + +def test_all_sources_down_raises_the_first_error(respx_mock): + respx_mock.post(SEARCH_URL).respond(404) + respx_mock.get(API_URL).respond(503) + with ( + Michraz([RmiSource(), BeerShevaSource()], cache_dir=None) as m, + pytest.raises(SourceUnavailableError, match="rmi"), + ): m.search() @@ -116,7 +136,7 @@ def test_second_search_uses_the_cache(respx_mock, rmi_records, tmp_path): def test_repr_lists_source_names(): with Michraz(cache_dir=None) as m: - assert repr(m) == "Michraz(sources=['rmi', 'beersheva', 'gov'])" + assert repr(m).startswith("Michraz(sources=['rmi', 'beersheva', 'gov', 'akko'") def test_base_source_is_abstract(): diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 3fbe2e9..bd883b7 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -13,7 +13,7 @@ from michraz.sources.beersheva import API_URL from michraz.sources.gov import SEARCH_URL as GOV_URL from michraz.sources.rmi import SEARCH_URL -from tests.helpers import FIXTURES, GOV_EMPTY_PAGE, ckan +from tests.helpers import CITIES, FIXTURES, GOV_EMPTY_PAGE, ckan, mock_municipalities @pytest.fixture @@ -21,6 +21,7 @@ def services(respx_mock, rmi_records, bs_records): rmi = respx_mock.post(SEARCH_URL).respond(json=rmi_records) beersheva = respx_mock.get(API_URL).respond(json=ckan(bs_records)) respx_mock.get(GOV_URL).respond(text=GOV_EMPTY_PAGE) + mock_municipalities(respx_mock) return rmi, beersheva @@ -62,6 +63,7 @@ def test_one_bad_record_does_not_kill_the_batch(respx_mock, rmi_records, bs_reco respx_mock.post(SEARCH_URL).respond(json=[*rmi_records, {"MichrazID": 1, "SgiraDate": "garbage"}]) respx_mock.get(API_URL).respond(json=ckan([*bs_records, {"שם מכרז": "missing columns"}])) respx_mock.get(GOV_URL).respond(text=GOV_EMPTY_PAGE) + mock_municipalities(respx_mock) with Michraz(cache_dir=None) as m: tenders = m.search() assert len(tenders) == len(rmi_records) + len(bs_records) @@ -78,6 +80,7 @@ def all_services(respx_mock, rmi_records, bs_records): respx_mock.post(SEARCH_URL).respond(json=rmi_records) respx_mock.get(API_URL).respond(json=ckan(bs_records)) respx_mock.get(GOV_URL).respond(text=(FIXTURES / "gov_search.html").read_text(encoding="utf-8")) + mock_municipalities(respx_mock) def test_every_default_source_contributes_and_exports(all_services, tmp_path): @@ -112,3 +115,62 @@ def test_cli_prints_government_tenders_as_json(all_services, tmp_path, monkeypat out = json.loads(capsys.readouterr().out) assert len(out) == 5 and {t["source"] for t in out} == {"gov"} assert all("mr.gov.il" in t["attribution"] for t in out) + + +@pytest.fixture +def every_service(respx_mock, rmi_records, bs_records): + respx_mock.post(SEARCH_URL).respond(json=rmi_records) + respx_mock.get(API_URL).respond(json=ckan(bs_records)) + respx_mock.get(GOV_URL).respond(text=(FIXTURES / "gov_search.html").read_text(encoding="utf-8")) + for city in CITIES: + body = (FIXTURES / f"bids_{city.name}.html").read_text(encoding="utf-8") + respx_mock.get(city.url).respond(text=body) + + +def test_every_default_source_feeds_one_search_and_one_csv(every_service, tmp_path): + with Michraz(cache_dir=None) as m: + tenders = m.search(kind=None) + assert m.last_errors == [] + assert {t.source for t in tenders} == {"rmi", "beersheva", "gov", *(c.name for c in CITIES)} + + path = tmp_path / "everything.csv" + to_csv(tenders, path) + with open(path, encoding="utf-8-sig", newline="") as handle: + rows = list(csv.DictReader(handle)) + assert len(rows) == len(tenders) and all(r["attribution"] for r in rows) + + +def test_one_municipal_site_down_leaves_the_rest_of_the_search_intact(every_service, respx_mock): + respx_mock.get(CITIES[0].url).respond(status_code=503) + with Michraz(cache_dir=None) as m: + tenders = m.search() + assert len(m.last_errors) == 1 and m.last_errors[0].startswith(f"{CITIES[0].name}:") + assert CITIES[0].name not in {t.source for t in tenders} + assert {"rmi", "gov", "kfar-saba"} <= {t.source for t in tenders} + + +def test_cli_searches_one_municipality(every_service, tmp_path, monkeypatch, capsys): + monkeypatch.setattr("michraz.cli.DEFAULT_CACHE_DIR", tmp_path / "cache") + assert main(["search", "--source", "kfar-saba", "--keyword", "מועדון", "--kind", "all"]) == 0 + out = capsys.readouterr().out + assert "kfar-saba" in out and "עיריית כפר סבא" in out and "1 tender" in out + + +def test_vacancies_are_hidden_by_default_and_shown_on_request(every_service): + from michraz import TenderKind + + with Michraz(cache_dir=None) as m: + default = m.search(source="kfar-saba") + vacancies = m.search(source="kfar-saba", kind=TenderKind.VACANCY) + both = m.search(source="kfar-saba", kind=None) + assert {t.kind for t in default} == {TenderKind.TENDER} + assert {t.kind for t in vacancies} == {TenderKind.VACANCY} and vacancies + assert len(both) == len(default) + len(vacancies) + + +def test_cli_kind_option(every_service, tmp_path, monkeypatch, capsys): + monkeypatch.setattr("michraz.cli.DEFAULT_CACHE_DIR", tmp_path / "cache") + assert main(["search", "--source", "kfar-saba", "--keyword", "מועדון"]) == 0 + assert "0 tenders" in capsys.readouterr().out + assert main(["search", "--source", "kfar-saba", "--keyword", "מועדון", "--kind", "vacancy"]) == 0 + assert "1 vacancy" in capsys.readouterr().out diff --git a/tests/test_enums.py b/tests/test_enums.py index 5840eb3..7c223c4 100644 --- a/tests/test_enums.py +++ b/tests/test_enums.py @@ -8,10 +8,11 @@ from michraz.cli import Command, ExitCode, main from michraz.export import CSV_COLUMNS, CsvColumn from michraz.models import RAW_EXTRA_KEY +from michraz.sources import BUILT_IN from michraz.sources.beersheva import API_URL from michraz.sources.gov import SEARCH_URL as GOV_URL from michraz.sources.rmi import SEARCH_URL -from tests.helpers import GOV_EMPTY_PAGE, ckan +from tests.helpers import GOV_EMPTY_PAGE, ckan, mock_municipalities class Sample(StrEnum): @@ -27,7 +28,7 @@ def test_str_enum_behaves_like_its_value(): def test_source_names_match_the_sources_and_the_cli(): assert RmiSource.name == SourceName.RMI and BeerShevaSource.name == SourceName.BEERSHEVA - assert [s.value for s in SourceName] == ["rmi", "beersheva", "gov"] + assert [s.value for s in SourceName] == [source.name for source in BUILT_IN] assert RmiSource.name == "rmi" and type(RmiSource.name) is str # plain str, so custom sources need no enum @@ -53,7 +54,7 @@ def test_csv_columns_are_tender_attributes_in_a_fixed_order(): fields = {f.name for f in dataclasses.fields(Tender)} assert {c.value for c in CsvColumn} <= fields assert CSV_COLUMNS == ( - "source", "source_id", "number", "title", "publisher", "deadline", "winner", "url", "attribution" + "source", "source_id", "kind", "number", "title", "publisher", "deadline", "winner", "url", "attribution" ) # fmt: skip @@ -71,6 +72,7 @@ def test_enum_keys_work_end_to_end_through_the_cli(respx_mock, rmi_records, bs_r respx_mock.post(SEARCH_URL).respond(json=rmi_records) respx_mock.get(API_URL).respond(json=ckan(bs_records)) respx_mock.get(GOV_URL).respond(text=GOV_EMPTY_PAGE) + mock_municipalities(respx_mock) for name in SourceName: assert main([Command.SEARCH, "--no-cache", "--source", name]) == ExitCode.OK assert "rmi" in capsys.readouterr().out diff --git a/tests/test_filters.py b/tests/test_filters.py index 67a31fa..7fca1c4 100644 --- a/tests/test_filters.py +++ b/tests/test_filters.py @@ -105,3 +105,13 @@ def test_tender_filter_rejects_naive_bounds_on_creation(): TenderFilter(closing_before=datetime(2026, 12, 1)) with pytest.raises(ValueError, match="timezone-aware"): TenderFilter(now=datetime(2026, 12, 1)) + + +def test_kind_filter_keeps_only_that_kind(): + from michraz import TenderKind + + tender = make_tender(source_id="1") + vacancy = make_tender(source_id="2", kind=TenderKind.VACANCY) + assert apply_filters([tender, vacancy], kind=TenderKind.TENDER) == [tender] + assert apply_filters([tender, vacancy], kind=TenderKind.VACANCY) == [vacancy] + assert apply_filters([tender, vacancy]) == [tender, vacancy] diff --git a/tests/test_rmi.py b/tests/test_rmi.py index 5b3eda3..b03978c 100644 --- a/tests/test_rmi.py +++ b/tests/test_rmi.py @@ -77,8 +77,8 @@ def test_khal_yaad_rashi_is_exposed_raw_not_as_a_bid(respx_mock, source, rmi_rec def test_unknown_settlement_falls_back_to_neighborhood_then_number(respx_mock, source, rmi_records): - odd = dict(rmi_records[0], KodYeshuv=999999) - bare = dict(rmi_records[0], KodYeshuv=999999, Shchuna=None) + odd = dict(rmi_records[0], MichrazID=9001, KodYeshuv=999999) + bare = dict(rmi_records[0], MichrazID=9002, KodYeshuv=999999, Shchuna=None) serve(respx_mock, [odd, bare]) a, b = source.search() assert a.title == "גבעת טל" and a.extra["settlement"] is None @@ -172,5 +172,5 @@ def test_last_fetched_reads_the_cache(respx_mock, rmi_records, tmp_path): serve(respx_mock, rmi_records) with RmiSource(cache_dir=tmp_path) as src: assert src.last_fetched() is None - src.fetch() + list(src.fetch()) assert src.last_fetched() is not None