diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7bbcc8..383f0e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,6 +25,21 @@ jobs: - name: Config and protocol tests run: python3 tests/test_config.py + - name: Authentication state machine test + run: python3 tests/test_auth.py + + - name: Request transport and retry test + run: python3 tests/test_transport.py + + - name: Channel listing and paging test + run: python3 tests/test_listing.py + + - name: Programme guide test + run: python3 tests/test_epg.py + + - name: Scheduled refresh test + run: python3 tests/test_schedule.py + - name: Mock portal integration test run: python3 tests/test_mock_portal.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d294ef0..fb470b9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -40,6 +40,13 @@ jobs: # entry does not happen. A changelog nobody is obliged to write is a # changelog that stops being written, and the reader of a release page # deserves better than a list of commit subjects. + # + # Only the part of the section above the "" marker is + # published, though: a release page is read by someone deciding whether to + # upgrade, who wants a line per subject, while the prose below the marker + # is the reasoning behind each one and belongs to whoever goes looking for + # it in the file. A section without the marker is published whole, so an + # entry written before this split still releases. - name: Take the release notes from CHANGELOG.md run: | python3 - "${GITHUB_REF_NAME#v}" <<'PY' > release-notes.md @@ -53,7 +60,10 @@ jobs: ) if section is None or not section.group(1).strip(): sys.exit(f"CHANGELOG.md has no entry for {version}") - print(section.group(1).strip()) + summary = section.group(1).split("")[0].strip() + if not summary: + sys.exit(f"the {version} entry is all detail and no summary") + print(summary) PY - name: Install test dependencies diff --git a/CHANGELOG.md b/CHANGELOG.md index 3935371..8775743 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,194 @@ # Changelog +## 0.9.3 + +**After upgrading, restart Dispatcharr, then press Test portals and Re-fetch +all.** The restart is what puts every worker on the new code, since plugins are +loaded once per process. Sync on its own would then report every portal as +unchanged and fetch nothing — it compares your settings against what was last +published, and none of them changed, only the code did. Test writes nothing and +is where a portal that now wants credentials will say so. + +- **Portals can now serve their programme guide.** `epg=1` on a portal line + writes an XMLTV file and registers it under **EPGs**; off unless asked for, + because a guide is by far the largest thing a sync downloads. +- **Distalker can be put on a schedule.** *Refresh every (hours)* re-fetches + each portal on that interval — channels, and the guide where `epg=1` is set — + without anyone pressing Sync. Left at `0`, everything stays manual. +- **Channels on portals that answer with a resolved link now play**, at sync + time and at tune time both, and stop duplicating themselves on every sync. +- **The portal now decides which authentication it gets**, so an expired or + blocked account says so in the provider's own words instead of arriving as an + empty channel list. +- **Portals that will not list their channels in one request now sync**, a page + at a time, and a sync survives a portal having a bad minute. +- **The channel is named in the request as well as in the command**, for the + providers that could not find it in the command alone. Reported and first + fixed by [@shayward](https://github.com/shayward). +- Channel logos and archive flags are read as the portal sends them. + + + +**Refreshing by itself** + +- **Distalker can now be put on a schedule.** Set *Refresh every (hours)* and + each portal is re-fetched on that interval — channels, and the guide where + `epg=1` is set — without anyone pressing Sync. Left at `0`, nothing changes: + everything stays manual, as before. +- The clock is Dispatcharr's, not ours. A plugin cannot register a Celery task + a stock install can consume, so the setting drives the refresh interval of + the M3U accounts this plugin owns, and the event Dispatcharr emits at the end + of each refresh is what calls back to re-fetch the portal. +- Refreshing an M3U account that is not one of ours does nothing, and a + scheduled run cannot follow another within thirty minutes — the second is the + echo of the first, and answering it would loop. In practice it does more than + that: twelve accounts sharing one interval all fire at the same second, and + eleven of those are turned away. +- A sync started by hand no longer comes back as a scheduled one. Every synced + portal asks Dispatcharr to re-read its playlist, and that re-read emits the + event the schedule listens for — so adding a single portal could set off a + re-fetch of every other one. +- A scheduled sync runs inside the task that woke it rather than in a thread of + its own. The thread was right for the button, which answers a browser and + cannot block; it was wrong here, where the worker hosting it is reaped when + the pool scales down — taking the sync with it, silently. +- **A scheduled sync now appears in the log.** It always ran; it just wrote + everything it did to a logger that prints from the process serving the + panel and not from the one running the schedule. Channel counts, guide + sizes and failures alike went nowhere, which made a refresh that worked + indistinguishable from one that never fired. +- The portal whose own refresh woke the sync is asked a second time, a minute + later, to re-read its playlist. Its first request is refused: the sync runs + inside that account's refresh task, which holds the account's lock until we + are done. Left as it was, that one portal would be downloaded on every cycle + and read on the next — a day late on a daily schedule, and a different + portal each time, since they race to be the one that wakes us. + +**Guide** + +- **Distalker can now fetch a portal's programme guide.** Add `epg=1` to a + portal line and the next sync writes an XMLTV file and registers it under + **EPGs** as `Distalker: ` — the same arrangement as the M3U account + it already creates for the channels. Matching is automatic: the `tvg-id` the + playlist has always carried is what Dispatcharr joins on. +- Off unless asked for, because it is by a wide margin the largest thing a sync + downloads — a portal answers for its whole line-up at once, so a + 13,000-channel provider means roughly 100 MB for a single day. `epg_hours=48` + raises the default 24; a guide past 200 MB is abandoned with a message saying + to lower it. +- A guide that fails never fails the sync around it. You keep the channel list. +- Channels the portal has no programmes for are left out rather than written as + empty entries, and removing `epg=1` deactivates the EPG source instead of + deleting it. +- **Only whole-grid guides are read.** A portal that serves programmes one + channel at a time is reported as having no guide — which, measured across + twelve portals, is close to the truth: that route covered none of the + channels anyone had configured, at the price of one request per channel. +- A portal with no guide says so plainly and suggests removing `epg=1`, rather + than reporting the same message as a portal that answered with something + unrecognisable — which is now reported as exactly that, and asks to be. + +**Playing** + +- **Channels a portal answers about with a resolved link now play.** Portals + are meant to name a channel with a marker and turn that into a link when + asked; some answer with the link itself, and then cannot read it back — one + spliced part of the URL into its own stream parameter, another returned that + parameter empty. Either way the channel was unplayable. Such a command is now + rebuilt into the marker the portal expects. +- It also stops those channels duplicating. The links carry a token that + changes on every request, and Dispatcharr identifies a stream partly by its + URL — so every sync invented a new stream and stranded the previous one. One + portal was producing 647 duplicates an hour. +- A command that already looks like a marker is left untouched, which is every + channel on every other portal tested — their identities do not move, and no + duplicate is created by the fix itself. The sync log says how many were + rewritten. +- **The same portals are also caught at tune time.** The rewrite above needs + the portal to number its channels; a listing that does not still hands back a + resolved link, as does every portal synced before this release. One family of + them answers by gluing its own base in front of whatever it was given, so a + command that was already a URL comes back with `/user/pass/` twice and gets + 401 — while the command itself plays. That reply is now recognised by its + shape and the command played instead, with a line on the channel's log + saying so. Nothing is asked of the provider to decide it: a check request + would take a connection slot from the tune that is about to need it. +- **The channel is now named in the request as well as in the command.** Some + providers cannot find it in the command they are handed back and answer about + no channel at all — the link arrives with its stream number left empty, and + plays nothing. Both shapes a command takes here carry that number, and it is + sent alongside, so those portals resolve the channel that was asked for. + Reported and first fixed by [@shayward](https://github.com/shayward). +- Sent only when there is one to send, so a portal that never needed it sees + the request it has always seen, and the number can only ever be one the + portal itself wrote. + +**Connecting** + +- **The portal now decides which authentication it gets.** Distalker reads the + `status` its profile request comes back with and does what it asks: nothing + further when the session is already good, or `do_auth` followed by a second + profile call when the portal says it wants credentials. It used to pick the + flow itself from whether a username and password happened to be configured, + which was wrong in both directions — it ran a device-ID step at portals that + wanted a password, and had no way to tell a refused account from an empty + channel list. +- A portal that refuses the account now says so in the provider's own words — + "subscription expired", "blocked" — instead of failing later as a channel + list that came back empty and a suggestion to check the MAC address. +- A portal that wants credentials and has none on its line now fails the sync + with that as the message, rather than appearing to work. +- Expired sessions are recognised from the plain-text `Authorization failed.` + some portals answer with, and from HTTP 401/403, instead of being reported as + a portal talking nonsense. +- The box's profile now carries the full identity every other Stalker client + sends, `signature` included — which had been a documented setting that no + request ever contained, so setting it configured nothing. +- Portals with no profile endpoint at all keep working on the MAC alone, with a + warning. That tolerance is deliberate: most portals this plugin meets are not + Ministra and answer with less than it would. +- The `device_id_auth` line key is gone. Nothing needs to be changed: it was + never something to write on a portal line, only a value the plugin derived + for itself, and lines are re-read on every sync. +- The MAC and the session token now travel in the query string as well as in + the cookie and the `Authorization` header. Portals read one form or the + other, and sending both costs nothing. +- **A portal reached at the wrong path now finds itself.** Ministra answers on + both `…/c/portal.php` and `…/server/load.php`, and installs differ in which + they expose; being handed the one your provider does not serve used to mean a + 404 and no suggestion. The other path is now tried once, and the log says + which one worked so you can put it on the portal line and stop paying for the + failed request. Only a 404 or a reply that is not JSON earns the second + attempt — a portal that is merely down answers the same way on both. + +**Syncing** + +- **Portals that will not list their channels in one request now sync.** Some + cap `get_all_channels`, some never implemented it; either way the portal was + unusable, since the empty answer was reported as a probable wrong MAC. The + line-up is now collected a page at a time instead when that happens — slower + by a long way on a big bouquet, and the only way those portals work at all. + The sync log says when it has fallen back and how far along it is. +- A portal that refuses the session is not paged as a second attempt, and an + empty listing from both routes still reports the original "check the MAC + address" message, which remains the likelier explanation. +- Channel logos survive two shapes that used to come out broken: a logo served + from another scheme than `http(s)` was treated as a filename and glued behind + the portal's logo path, and an inline `data:` image got the same treatment. + The first is now left alone, the second dropped — Dispatcharr keeps this in a + URL field, where a base64 payload does not belong. +- **A sync survives a portal having a bad minute.** Requests made while + syncing are attempted up to three times, one then two then four seconds + apart, where a single dropped connection or gateway error used to cost the + whole line-up until the next scheduled run. Only failures that another + attempt could fix are repeated: a refused login, a blocked account or a + missing endpoint still fails immediately. +- Nothing is retried at tune time, deliberately. A source that is not answering + has to fail fast enough for Dispatcharr to move to the next one, which is the + same reason `-reconnect` is not in the default ffmpeg arguments. "Test + portals" does not retry either — it answers a click, and three attempts at + the portal timeout outlast the browser waiting for it. + ## 0.9.2 **Playing** diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 181f568..ab98fa9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,6 +33,11 @@ script with its own `__main__` block: ```bash python3 tests/test_config.py # portal-line parsing, STB defaults, pseudo-URLs +python3 tests/test_auth.py # which authentication a portal asks for +python3 tests/test_transport.py # what a request carries, and when it is retried +python3 tests/test_listing.py # reading a portal's channel list +python3 tests/test_epg.py # the guide, and the tvg-id both halves share +python3 tests/test_schedule.py # being woken by Dispatcharr's own clock python3 tests/test_registry.py # surviving the settings panel python3 tests/test_manifest.py # plugin.json vs plugin.py, and run()'s plumbing python3 tests/test_fallback.py # non-portal sources on a Distalker channel @@ -91,6 +96,17 @@ which the workflow gates on two things — the tag must match `plugin.json`, and `CHANGELOG.md` must have a section for that version, which becomes the release body. A changelog nobody is obliged to write is one that stops being written. +That section is written in two parts, separated by a `` line: + +- Above it, what upgrading gets you — a line per subject, plus anything that has + to be done by hand after installing. This is what the release page shows, and + it is read by someone deciding whether to upgrade at all. +- Below it, why each of those exists: the portal behaviour that forced it, what + was tried first, what is deliberately not done. Nobody reads that on a release + page, and everybody wants it six months later when the same thing breaks. + +A section with no marker is published whole, so nothing has to be back-filled. + Keep changes to `.github/workflows/` in their own commit: pushing one needs a token carrying the `workflow` scope, which not every contributor's will have, and splitting it is what makes the rest shippable when it does not. diff --git a/README.md b/README.md index b991d7c..076b5b9 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,11 @@ URL: | `http://host/…/load.php` | unchanged — explicit endpoints are preserved | | `http://host/c/other.php` | `http://host/c/portal.php` | +If that path turns out not to be where the portal answers, Distalker tries the +other one Ministra uses — `…/c/portal.php` and `…/server/load.php` are swapped +for each other — and logs which one worked. Putting the working one on the +portal line saves a failed request on every sync. + Anything unusual goes in trailing `key=value` pairs, separated by spaces or further `|` characters, quoted where a value contains spaces (`password="two words"`): @@ -168,6 +173,8 @@ further `|` characters, quoted where a value contains spaces | `username` | Portal login, if your provider issued one | — | | `password` | Portal password | — | | `max_streams` | Concurrent connections allowed for this MAC | `1` | +| `epg` | `epg=1` fetches this portal's programme guide — see below | off | +| `epg_hours` | How much guide to ask for, in hours | `24` | | *STB keys* | `model`, `serial`, `device_id`, `device_id2`, `signature`, `timezone` — see below | MAG254 | > **`max_streams` cannot be detected.** Portals do not tell the box what the @@ -193,9 +200,9 @@ http://portal.example.com/c/ | 00:1A:79:AA:BB:CC | model=MAG322 timezone=Europe/ | --- | --- | --- | | `model` | `MAG254` | Sent as the `X-User-Agent` device model. Also `MAG250`, `MAG322`, … | | `serial` | `0000000000000` | Portal cookie `sn`. | -| `device_id` | 64 × `f` | Sent during device-ID authentication. | +| `device_id` | 64 × `f` | Sent with the box's profile, and with credentials. | | `device_id2` | same as `device_id` | Most boxes carry the same value in both slots. | -| `signature` | 64 × `f` | Accepted, but **currently unused**: no request Distalker makes includes it. | +| `signature` | 64 × `f` | Sent with the box's profile. | | `timezone` | `UTC` | Portal cookie `timezone`, e.g. `Europe/Paris`. | > **These are the untested part of this plugin.** Every portal it has run @@ -204,11 +211,20 @@ http://portal.example.com/c/ | 00:1A:79:AA:BB:CC | model=MAG322 timezone=Europe/ > gets it wrong, that is a bug worth reporting with the values your provider > gave you. -Which authentication runs is decided by one thing — whether you supplied -credentials. With a username *and* password: handshake, then `do_auth`. Without: -handshake, then a device-ID step. A failed device-ID step is **not fatal**, as -plenty of portals authorise on the MAC alone; if the session really is -unauthorised, the channel fetch says so straight after, and far more clearly. +**The portal decides which authentication runs**, not the settings. Distalker +shakes hands, presents the box's profile, and does what the answer asks for: +nothing more if the portal is satisfied, or `do_auth` with your username and +password if it says it wants them. If it refuses the account outright, the +message you get is the provider's own — "subscription expired", "blocked" — +rather than a guess made here. + +So credentials on a portal line are there for the portals that ask; a portal +that never asks ignores them. Two consequences worth knowing: + +- A portal that asks for credentials **and has none on its line** now fails the + sync, saying so. It used to fetch an empty channel list and blame the MAC. +- A portal that has no profile endpoint at all — some do not — still works, with + a warning in the log, on the strength of the MAC alone. ### Other settings @@ -218,7 +234,7 @@ You should not need any of these. | --- | --- | --- | | ffmpeg arguments | a plain remux, plus the MAG headers and `-rw_timeout` | Placeholders `{url}`, `{ua}`, `{referer}`, `{headers}`. Must write MPEG-TS to `pipe:1`. **Do not add `-reconnect`** — it retries a link that has already expired, and stops Dispatcharr failing over to the channel's other sources. | | Fallback stream profile | `ffmpeg` | Plays the *other* sources on a Distalker channel — see below. | -| Portal request timeout | `60` s | Every portal request, sync and tune alike. Raise it if a busy portal times out assembling its channel list. | +| Portal request timeout | `60` s | Every portal request, sync and tune alike. Raise it if a busy portal times out assembling its channel list. A sync retries twice on top of this — 1 s then 2 s apart — so raising it far also lengthens the worst case of a failing sync. | | Auto-assign stream profile | on | Gives a channel the Distalker profile as it gains a portal stream, after each M3U refresh, and once more after any channel fails to start. | ### Channels that mix a portal with another provider @@ -264,8 +280,9 @@ button you should need. ### Sync only fetches what changed A line-up is one request per portal that a busy provider can take minutes to -assemble, so re-downloading portals that did not change is time spent for -nothing: +assemble — and on portals that refuse to list everything at once, hundreds of +requests instead, collected page by page. Either way, re-downloading portals +that did not change is time spent for nothing: | Your line | What Sync does | | --- | --- | @@ -308,11 +325,72 @@ dates natively. If the portal reports the account as blocked, the line says so in capitals: a blocked account otherwise looks exactly like an empty channel list. +### Refreshing on a schedule + +Set **Refresh every (hours)** and Distalker re-fetches every portal on that +interval — channels, and the guide on lines carrying `epg=1`. At `0`, the +default, nothing happens on its own and Sync stays a button. + +There is no timer inside the plugin. The setting is written to the refresh +interval of the M3U accounts it owns; Dispatcharr schedules those itself, and +the event it emits after each one is what wakes Distalker to re-fetch the +portal behind it. Two consequences worth knowing: + +- **Below one hour is refused.** A scheduled run blocks the next for thirty + minutes, because a sync ends by asking Dispatcharr to re-read the playlist it + just wrote — and that re-read emits the same event. The cooldown is what + stops the pair going round for ever. +- Refreshing an M3U account by hand also triggers it, if the account is one of + Distalker's and the last scheduled run was over half an hour ago. + +### Programme guide + +Add `epg=1` to a portal line and the next sync also fetches its guide, writes +an XMLTV file, and registers it under **EPGs** as `Distalker: ` — the +same arrangement as the M3U account it creates for the channels. Dispatcharr +then matches programmes to channels on the `tvg-id` the playlist already +carries, so nothing needs mapping by hand. + +``` +Living room | http://portal.example/c/ | 00:1A:79:AA:BB:CC | epg=1 +Living room | http://portal.example/c/ | 00:1A:79:AA:BB:CC | epg=1 epg_hours=48 +``` + +**It is off by default because it is expensive.** A portal answers `get_epg_info` +for its *entire* line-up in one response: on a 13,000-channel portal, a single +day is on the order of 100 MB, and the period multiplies that directly. Start at +the default 24 hours and raise it only if your portal copes. Beyond 200 MB the +download is abandoned with a message telling you to lower `epg_hours`. + +**Not every portal has one.** A provider can carry thousands of channels and no +programmes for any of them; it answers with an empty guide and the log says so, +suggesting you drop `epg=1` from that line. Some serve a guide one channel at a +time instead of as a whole grid — Distalker does not use that, because it costs +one request per channel and, where it was measured, covered almost none of the +channels anyone had actually configured. + +A guide that fails — too large, refused, in a shape this does not recognise — is +logged and skipped. It never fails the sync around it, so you keep the channel +list either way. + +Channels the portal has no programmes for are left out of the file rather than +written empty, and turning `epg=1` off again deactivates the EPG source without +deleting it. + ## Limitations - **Live TV only.** No VOD, no series. -- **No EPG yet.** Generated `tvg-id`s are stable (`.`), so EPG - can be added later without disturbing existing streams. +- **Programme guides are supported only where a portal serves a whole grid.** + `get_epg_info` answers for the entire line-up in one request, and that is the + only shape Distalker reads. Portals that instead serve a guide one channel at + a time are not covered: it costs one request per channel, and where it was + measured across twelve portals it covered none of the channels that had + actually been configured. Those portals report having no guide, which for + practical purposes is what it amounts to. +- **Scheduling is borrowed, not built.** A plugin cannot register a Celery task + a stock Dispatcharr can run, so *Refresh every (hours)* drives the M3U + accounts' own refresh interval and answers the event that follows. It works, + and it is why the interval cannot usefully go below an hour. - **Credentials are stored unencrypted**, in the Dispatcharr database and on disk — see [What it writes, and where](#what-it-writes-and-where). - **No session keep-alive.** A cached token is reused and re-issued on demand. @@ -338,7 +416,10 @@ Resolver output otherwise appears in the channel's log, prefixed `[distalker]`: | `cannot reach Redis (…); reading the mirrored portal instead` | Informational — playback carried on from the copy on disk. | | `cached session rejected` | Normal. The token expired and is being renewed. | | `create_link returned an empty command` | The portal refused the channel — often a connection limit or an expired subscription. | -| `portal returned an empty channel list` | Wrong MAC address or portal URL. | +| `N channel(s) answered with a resolved link rather than a marker, and were rewritten to one` | Informational. That provider hands out playable links where a channel marker belongs; Distalker rebuilds the marker, without which those channels neither play nor keep a stable identity. | +| `the portal answered with its own base in front of a command that was already a link` | Informational. Same provider habit as the row above, seen at tune time: it built the link by repeating its own `/user/pass/` path, which answers 401. Distalker plays the channel's own command instead, which is what that path resolves to. | +| `portal returned an empty channel list` | Wrong MAC address or portal URL. Reported only after paging the line-up was tried too and also came back empty. | +| `the portal would not list its channels in one request … collecting them a page at a time` | Informational. This portal caps or lacks `get_all_channels`, so the sync is reading its line-up page by page. Expect it to take minutes on a large bouquet. | | ffmpeg: `Server returned 5XX Server Error reply` | The portal issued a link but refused to serve it. Probe it (below) — usually a connection limit. | **Nothing plays and you see 503 / "max connections".** Every viewer, preview, diff --git a/plugin.json b/plugin.json index d819ef5..27325d0 100644 --- a/plugin.json +++ b/plugin.json @@ -1,6 +1,6 @@ { "name": "Distalker", - "version": "0.9.2", + "version": "0.9.3", "description": "Stalker/MAG portal support for Dispatcharr. Syncs portal channels into a native M3U account and resolves short-lived stream links at tune time -- no extra containers, no extra ports.", "author": "PiloUnk", "license": "AGPL-3.0-only", @@ -19,7 +19,7 @@ "type": "text", "default": "", "placeholder": "http://portal.example.com:8080/c/ | 00:1A:79:AA:BB:CC", - "help_text": "Portal URL | MAC address, one line each. The name is taken from the host, so put one in front only if you want a different label -- or if two portals share a host, which the sync will then ask you to do. Trailing key=value pairs cover the rest: username, password, max_streams, model, serial, device_id, device_id2, timezone. A line starting with '#' is ignored, which is how you suspend a portal without losing its channels. Credentials are stored unencrypted and are visible in this box." + "help_text": "Portal URL | MAC address, one line each. The name is taken from the host, so put one in front only if you want a different label -- or if two portals share a host, which the sync will then ask you to do. Trailing key=value pairs cover the rest: username, password, max_streams, model, serial, device_id, device_id2, timezone, and epg=1 to fetch this portal's programme guide (epg_hours=48 for more than a day -- a guide is by far the largest thing a sync downloads, which is why it is off unless asked for). A line starting with '#' is ignored, which is how you suspend a portal without losing its channels. Credentials are stored unencrypted and are visible in this box." }, { "id": "status", @@ -64,6 +64,13 @@ "type": "boolean", "default": true, "help_text": "Give a channel the Distalker profile as soon as it gains a portal stream, again after every M3U refresh, and once more if a channel ever fails to start -- which repairs any channel created while the plugin was disabled or upgrading. Dispatcharr resolves stream profiles per channel, so a channel without it will not tune its portal source. A channel that also holds sources from other providers gets the profile too -- those sources then play through the fallback profile above instead of Dispatcharr's direct relay." + }, + { + "id": "refresh_hours", + "label": "Refresh every (hours)", + "type": "number", + "default": 0, + "help_text": "0 keeps everything manual: portals are only fetched when you press Sync. Any other value asks Dispatcharr to refresh this plugin's M3U accounts on that schedule, and each refresh re-fetches its portal -- channels and, where 'epg=1' is set, the programme guide. Minimum 1 hour." } ], "actions": [ @@ -86,7 +93,10 @@ "label": "Re-fetch everything", "description": "Download every portal's channel list again, changed or not. Sync only fetches what you edited, so this is how you pick up a line-up that changed at the provider. Slow, and it spends one connection per portal.", "button_label": "Re-fetch all", - "button_variant": "light" + "button_variant": "light", + "events": [ + "m3u_refresh" + ] }, { "id": "apply_profile", diff --git a/plugin.py b/plugin.py index 49ef3de..e0e430a 100644 --- a/plugin.py +++ b/plugin.py @@ -33,11 +33,13 @@ save_registry, ) from .stalker_api import ( + DEFAULT_EPG_HOURS, DEFAULT_FFMPEG_ARGS, DEFAULT_TIMEOUT, STB_KEYS, PortalConfig, PortalError, + claim_auto_sync, forget_portal, format_portal_line, is_superseded_ffmpeg_args, @@ -50,11 +52,15 @@ ) from .sync import ( + ACCOUNT_PREFIX, + TRIGGER_REPARSE_DELAY, announce, + apply_refresh_interval, apply_stream_profile, install_stream_profile, portal_status, publish_fallback, + request_reparse_later, sync_all, test_portal, ) @@ -70,6 +76,13 @@ "stb_timezone": "timezone", } +# How long a scheduled sync blocks the next one. It has to outlast the sync it +# guards, because that sync ends by asking Dispatcharr to re-read the playlist +# it wrote -- which emits the event that started it. Thirty minutes is chosen +# against a large portal with a guide, and is why the schedule cannot usefully +# be finer than an hour. +AUTO_SYNC_COOLDOWN = 1800 + _MANIFEST_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "plugin.json") with open(_MANIFEST_PATH, "r", encoding="utf-8") as _handle: @@ -206,6 +219,13 @@ def _migrate_legacy_globals(self, settings: Dict[str, Any], logger) -> Dict[str, extras.get("password", ""), int(extras.get("max_streams", 1) or 1), stb, + # Carried through explicitly. This rewrite keeps only + # what it is handed, so anything left out here is + # silently deleted from the user's line. + extras.get("epg", "").strip().lower() + in ("1", "true", "yes", "on"), + int(extras.get("epg_hours", DEFAULT_EPG_HOURS) + or DEFAULT_EPG_HOURS), ) ) changed += 1 @@ -484,9 +504,16 @@ def _action_test_portals(self, params, settings, logger) -> Dict[str, Any]: # must not cost a download of everything the user already has. LINEUP_KEYS = ( "url", "mac", "username", "password", "device_id", "device_id2", - "serial_number", "model", "timezone", "device_id_auth", + "serial_number", "model", "timezone", "signature", ) + # Everything a change to which means the portal has to be asked again. + # The line-up keys, plus the guide: turning 'epg=1' on changes nothing + # about the channels, so without this the plan would call the portal + # unchanged, fetch nothing, and leave the user pressing Sync at a setting + # that appears to do nothing. + FETCH_KEYS = LINEUP_KEYS + ("epg", "epg_hours") + def _plan(self, portals: List[PortalConfig]) -> Dict[str, Any]: """Sort the configured portals into what needs the network and what does not. @@ -505,7 +532,7 @@ def _plan(self, portals: List[PortalConfig]) -> Dict[str, Any]: previous = load_portal(cfg.slug) if cfg.slug in published else None if previous is None: plan["new"].append(cfg) - elif any(getattr(cfg, key) != getattr(previous, key) for key in self.LINEUP_KEYS): + elif any(getattr(cfg, key) != getattr(previous, key) for key in self.FETCH_KEYS): plan["changed"].append(cfg) else: plan["unchanged"].append(cfg) @@ -546,20 +573,131 @@ def _action_sync_now(self, params, settings, logger) -> Dict[str, Any]: return {"status": "ok", "message": "sync started: " + self._describe(plan)} def _action_resync_all(self, params, settings, logger) -> Dict[str, Any]: - """Re-fetch every portal, changed or not.""" + """Re-fetch every portal, changed or not. + + A button, and also the plugin's clock. It subscribes to ``m3u_refresh`` + because that is the only periodic thing a plugin can be woken by: + Dispatcharr schedules a refresh per M3U account, and the event it ends + with reaches here -- see :meth:`_scheduled_run_wanted` for the three + things that have to be true before an event is allowed to act on it. + + A full re-fetch rather than a planned one, deliberately. The plan + compares the portal list against what is published, and on a schedule + nothing in the list has changed -- so a planned run would find nothing + to do every single time, which is the opposite of a refresh. + """ + if params.get("event"): + reason = self._scheduled_run_wanted(params, settings, logger) + if reason: + return {"status": "ok", "message": reason, "changed": False} + + # Here, not in a thread. The event arrives inside a Celery task, + # where a daemon thread dies with the worker the moment the pool + # scales down -- see tasks.run_sync_here. Running it where we + # stand also means the report is the real one rather than a + # promise that something has started. + result = tasks.run_sync_here(full=True, logger=logger) + if result is None: + # Said out loud, because this is the one outcome that used to + # leave no trace at all: an event run that changes nothing is + # not recorded in the panel either, so a schedule refused by + # the lock looked exactly like a schedule that never fired. + logger.info( + "distalker: the scheduled sync stood down -- another one " + "holds the lock" + ) + return {"status": "ok", "message": self._already_running(), + "changed": False} + + self._reparse_the_account_that_woke_us(params, logger) + return {"status": "ok", "message": result["message"], "changed": True} + portals = self._portals(settings) if not tasks.run_sync_in_background(full=True): - return {"status": "ok", "message": self._already_running()} + return {"status": "ok", "message": self._already_running(), "changed": False} return { "status": "ok", + "changed": True, "message": ( f"re-fetching all {len(portals)} portal(s) in the background. Refresh " "the Plugins page to read the result in Last action." ), } + def _reparse_the_account_that_woke_us(self, params, logger) -> None: + """Ask again for the one playlist the sync could not get re-read. + + Here rather than in the sync, because it is the event path that knows + which account is the awkward one, and because by now the sync has + returned -- there is nothing left of our work for the lock to outlast. + See sync.request_reparse_later for what the lock is and why it refuses. + """ + account = str((params.get("payload") or {}).get("account_name") or "") + try: + if request_reparse_later(account): + logger.info( + "distalker: asking again in %ds for '%s' to be re-read -- " + "its own refresh is what woke us, so it held the lock while " + "we worked", + TRIGGER_REPARSE_DELAY, + account, + ) + except Exception: + # Never worth failing a completed sync over: the playlist is + # written either way, and the next cycle reads it. + logger.debug( + "distalker: could not re-ask for '%s'", account, exc_info=True + ) + + def _scheduled_run_wanted(self, params, settings, logger) -> str: + """Why this event should be ignored, or '' to let it through. + + Three guards, each covering something different: + + * the setting. ``refresh_hours`` at 0 means the user never asked to be + synced on a schedule, and a plugin that starts contacting portals by + itself after an upgrade is not a good surprise. + * whose account it was. ``m3u_refresh`` fires for *every* M3U account on + the install, so without this a user refreshing an unrelated playlist + would set off a full round of portal logins. + * the cooldown. A scheduled sync ends by asking Dispatcharr to re-read + the playlist it just wrote, and that re-read emits this same event -- + so the ring has to be broken somewhere, and this is where. + """ + if self._refresh_hours(settings) <= 0: + return "scheduled refresh is off" + + payload = params.get("payload") or {} + account = str(payload.get("account_name") or "") + if not account.startswith(ACCOUNT_PREFIX): + return f"'{account}' is not one of ours" + + # Long enough to outlast the sync it is guarding, which on a large + # portal with a guide is minutes rather than seconds. + if not claim_auto_sync(ttl=AUTO_SYNC_COOLDOWN): + return "a scheduled sync ran too recently" + + logger.info( + "distalker: answering the scheduled refresh of '%s'", account + ) + return "" + + @staticmethod + def _refresh_hours(settings: Dict[str, Any]) -> int: + """The schedule in hours, or 0. Never less than an hour when set. + + Below that the cooldown that stops the refresh loop would be longer than + the interval, so every other run would be swallowed -- a schedule that + silently does half of what it says. + """ + try: + hours = int(settings.get("refresh_hours") or 0) + except (TypeError, ValueError): + return 0 + return max(1, hours) if hours > 0 else 0 + @staticmethod def _already_running() -> str: """Turn down a second press without being blunt about it. @@ -590,13 +728,21 @@ def _describe(plan: Dict[str, Any]) -> str: parts.append(f"leaving {len(plan['unchanged'])} untouched") return "; ".join(parts) + ". Refresh the Plugins page for the result." - def run_sync_now(self, full: bool = False) -> Dict[str, Any]: + def run_sync_now(self, full: bool = False, logger=None) -> Dict[str, Any]: """The sync itself, off the request thread. Entry point for the thread. Loads its own settings, because a background thread has no request context and no panel state to be handed. + + The caller may hand its logger over, and the scheduled path does. + Ours -- ``_dispatcharr_plugin_distalker.plugin`` -- reaches the + container log from a uWSGI worker and not from a Celery one, so a + scheduled sync wrote its whole run to nowhere: no 'synced' line, no + channel counts, nothing to tell a working refresh from one that never + happened. Dispatcharr's own logger, the one handed to every action, + prints from both. """ - logger = logging.getLogger(__name__) + logger = logger or logging.getLogger(__name__) settings = self._settings_with_defaults() settings = self._reconcile_registry(settings, logger) settings = self._migrate_legacy_globals(settings, logger) @@ -632,8 +778,13 @@ def _sync_portals(self, settings: Dict[str, Any], logger, full: bool = False) -> for cfg in portals: save_portal(cfg) + # Before the fetch, and regardless of it: the schedule is a global + # setting, so no portal ever looks "changed" because of it and the + # plan would leave every account untouched. + apply_refresh_interval(self._refresh_hours(settings), logger) + targets = portals if full else plan["new"] + plan["changed"] - outcome = sync_all(targets, logger) + outcome = sync_all(targets, logger, refresh_hours=self._refresh_hours(settings)) # Portals the user deleted stop resolving. Their M3U account and # channels stay: deleting those is a decision, not a side effect. @@ -725,6 +876,11 @@ def _report(portals, plan, outcome, full, logger) -> str: entry += f", expires {state['expires']:%d %b %Y}" if just and just.get("blocked"): entry += " -- THE PORTAL REPORTS THIS ACCOUNT AS BLOCKED" + # Dispatcharr's own guide toast names the source by its numeric id + # and nothing else (apps/epg/utils.py, send_epg_update), so this is + # the only place the portal's name and its guide appear together. + if just and just.get("epg"): + entry += f", guide for {just['epg']['channels']} of them" if just: entry += " (just fetched)" lines.append(entry) @@ -774,6 +930,12 @@ def _republish(self, settings: Dict[str, Any], logger) -> int: except Exception: logger.exception("distalker: could not republish the fallback profile") + # Here rather than only in _sync_portals, because the path that most + # needs it never gets there: changing the schedule changes no portal, + # so Sync finds nothing to fetch, returns early -- and returns through + # this function. Idempotent, and it writes only when the value differs. + apply_refresh_interval(self._refresh_hours(settings), logger) + return published def _action_apply_profile(self, params, settings, logger) -> Dict[str, Any]: diff --git a/resolver.py b/resolver.py index ce72312..e10407b 100644 --- a/resolver.py +++ b/resolver.py @@ -69,19 +69,33 @@ def resolve(slug: str, cmd: str) -> tuple[str, stalker_api.PortalConfig]: if cached: # Optimistic path: reuse the cached token and skip the handshake. + # + # Only an actual refusal is worth a second attempt. A portal that is + # unreachable or answering with rubbish will do the same during the + # handshake, and a tune that spends two round-trips discovering that is + # a tune Dispatcharr spends not failing over to the next source. try: link = portal.create_link(cmd) - return link, cfg - except PortalError as exc: + except stalker_api.PortalAuthError as exc: log(f"cached session rejected ({exc}); re-authenticating") stalker_api.clear_cached_token(slug, client) portal = stalker_api.Portal(cfg) + else: + for warning in portal.warnings: + log(f"{slug}: {warning}") + return link, cfg portal.login() + # Cached before the link is asked for, not after: the token is good either + # way, and a create_link that fails must not cost the next tune a second + # handshake to learn the same thing. + stalker_api.set_cached_token(slug, portal.token, ttl=3600, client=client) + link = portal.create_link(cmd) + # After the link, so that what create_link has to say about the portal's + # answer is reported too, and not only what login() found. for warning in portal.warnings: log(f"{slug}: {warning}") - stalker_api.set_cached_token(slug, portal.token, ttl=3600, client=client) - return portal.create_link(cmd), cfg + return link, cfg def build_ffmpeg_command(cfg: stalker_api.PortalConfig, url: str) -> list: diff --git a/stalker_api.py b/stalker_api.py index a719cad..abaa5b5 100644 --- a/stalker_api.py +++ b/stalker_api.py @@ -26,10 +26,11 @@ import shutil import sys import tempfile +import time from dataclasses import asdict, dataclass from datetime import datetime, timezone from typing import Any, Dict, List, Optional, Tuple -from urllib.parse import quote, urlparse +from urllib.parse import parse_qs, quote, urlparse import requests @@ -46,6 +47,69 @@ DEFAULT_SIGNATURE = "f" * 64 DEFAULT_TIMEZONE = "UTC" +# The rest of what a MAG box tells get_profile about itself. Fixed rather than +# configurable: these describe a firmware image, not an account, and a portal +# that cared would want them to agree with each other -- which they only do as +# the block libstalkerclient has been sending since 2015 (lib/libstalkerclient/ +# stb.c, `sc_stb_get_profile_defaults`). They describe a MAG250 image even when +# stb_type says MAG254; no portal has ever been seen to cross-check the two, +# and every Stalker client in the wild sends this same mismatch. +STB_VERSION = ( + "ImageDescription: 0.2.16-250; " + "ImageDate: 18 Mar 2013 19:56:53 GMT+0200; " + "PORTAL version: 4.9.9; " + "API Version: JS API version: 328; " + "STB API version: 134; " + "Player Engine version: 0x566" +) +STB_IMAGE_VERSION = 216 +STB_HW_VERSION = "1.7-BD-00" +STB_NUM_BANKS = 1 + +# What a portal answers with, in plain text and with no JSON around it, once +# the token it was given is no longer good. Matched exactly because it is a +# fixed string in Ministra rather than something a reseller writes. +AUTH_FAILED_BODY = "authorization failed." + +# Answers worth asking again for. Everything else is the portal having made up +# its mind: a 404 is not going to become a 200, and a 403 is the subject of +# PortalAuthError, which must never be retried -- repeating a rejected login is +# how a MAC gets itself banned. +# +# 500 is in the list and is the debatable one. Ministra returns it both for +# "busy right now" and for some permanent failures, so a third of these retries +# will be spent on something that cannot succeed. Three attempts is a small +# enough bill for covering the transient half. +RETRYABLE_STATUS = frozenset({429, 500, 502, 503, 504}) + +# Hours of guide asked for when a portal line says 'epg=1' without saying how +# much. pvr.stalker holds itself to the same figure, and its own comment says +# why: get_epg_info answers for the *entire* line-up in one response, so the +# period is a direct multiplier on a download that is already the largest thing +# this plugin ever makes. +DEFAULT_EPG_HOURS = 24 + +# Refuse a guide bigger than this rather than discover the limit by having the +# worker killed. A 13,000-channel portal answers with something like 100 MB for +# a day, and the parsed form of that costs several times more again -- so this +# is not a generous allowance, it is the point past which the sensible thing is +# to say so and keep the channel list, which matters more than the guide. +EPG_MAX_BYTES = 200 * 1024 * 1024 + +# Last-resort bound on the paginated channel listing, for a portal that keeps +# answering with something new and never says how much there is. Deliberately +# far above any real line-up: whenever the portal reports 'total_items' the +# computed page count wins long before this, and the two cheaper guards -- +# an empty page, a page that repeats one already read -- stop nearly everything +# else. This only catches a portal generating rubbish indefinitely. +ORDERED_LIST_PAGE_CAP = 1000 + +# Seconds before the 1st, 2nd and 3rd retry. Fixed rather than jittered: the +# calls that retry are made one portal at a time from a single process, so +# there is no herd to spread out, and a deterministic delay is one a test can +# assert on. +RETRY_BACKOFF = (1.0, 2.0, 4.0) + # Seconds to wait on any single portal request. Generous by HTTP standards # because get_all_channels is one request for the entire line-up, and a busy # portal can take minutes to assemble it. @@ -148,6 +212,26 @@ class PortalError(Exception): """Raised when the portal rejects us or answers with nonsense.""" +class PortalEndpointError(PortalError): + """Something answered, but it was not a Stalker API. + + A 404, or a body that is not JSON at all. Separated because it is the one + failure with a second thing worth trying: the same portal on its other + path -- see :func:`alternate_endpoint`. + """ + + +class PortalAuthError(PortalError): + """The portal understood us and refused the session. + + Separated from its parent because the two want opposite handling: a + transport failure is worth retrying, an account the portal has declined is + not, and only the second is worth repeating verbatim to the user -- the + portal's own wording ("blocked", "subscription expired") says more than + anything this plugin could infer. + """ + + # --------------------------------------------------------------------------- # Configuration # --------------------------------------------------------------------------- @@ -168,12 +252,13 @@ class PortalConfig: serial_number: str = DEFAULT_SERIAL model: str = DEFAULT_MODEL timezone: str = DEFAULT_TIMEZONE - # Accepted for parity with stalkerhek's profile settings. stalkerhek only - # ever sends it from its STB proxy mode (proxy/proxy.go); none of the - # handshake / auth / create_link requests this plugin makes include it. signature: str = DEFAULT_SIGNATURE - device_id_auth: bool = False max_streams: int = 1 + # Off unless the line says otherwise. One guide is a bigger download than + # everything else this plugin fetches put together, so it is asked for + # rather than assumed -- see DEFAULT_EPG_HOURS. + epg: bool = False + epg_hours: int = DEFAULT_EPG_HOURS ffmpeg_args: str = DEFAULT_FFMPEG_ARGS # Travels to Redis with the rest, so the resolver waits as long as the sync # does rather than giving up on a portal the sync copes with. @@ -217,6 +302,141 @@ def extract_link(raw: str) -> str: return "" +def as_int(value: Any, default: int = 0) -> int: + """An int from whatever the portal felt like sending. + + Numbers arrive as numbers on some portals and as strings on others, often + both within one response, so nothing that reads a count can assume either. + """ + try: + return int(str(value).strip()) + except (TypeError, ValueError): + return default + + +# What a Stalker channel's command normally looks like once the prefix is off: +# a marker naming the channel, which the portal resolves at create_link time. +# Ministra writes 'http://localhost/ch/_'; the host varies, the shape does +# not. +_CANONICAL_CMD = re.compile(r"/ch/(\d+)_?$") + + +def canonical_cmd(cmd: str, channel_id: str) -> str: + """The command as the portal expects to be handed it back. + + Portals are supposed to answer get_all_channels with a marker, and to turn + that marker into a playable link when asked. Some answer with the playable + link itself -- and then cannot read it back. Observed on one provider: + + create_link('ffmpeg http://host/play/live.php?...&stream=553690&...') + -> 'http://host/play/live.php?...&stream=-host:80/play/live.' + + It looks for the channel id inside what it is given, fails, and splices a + piece of the URL into the parameter. Another provider answers the same + request with the id left empty. Either way the link is unplayable, so a + channel synced during one of those replies simply does not work. + + It also breaks the channel's identity. Dispatcharr hashes a stream partly + on its URL, ours encodes this command, and these links carry a token that + changes on every request -- so each sync invented a new stream and left the + previous one behind. One portal produced 647 duplicates an hour. + + So a command that is not a marker is rebuilt into one. Narrow on purpose: + a command that already looks like a marker is returned untouched, which is + every channel on every other portal tested, and their identities do not + move. + """ + if not channel_id: + return cmd + link = extract_link(cmd) + # No URL at all is a shape this does not understand -- VOD commands look + # like 'auto /media/file.mpg' -- and guessing at it would be worse than + # leaving it to the portal. + if not link or _CANONICAL_CMD.search(link): + return cmd + return f"ffmpeg http://localhost/ch/{channel_id}_" + + +def undoubled_link(cmd: str, link: str) -> str: + """The link the portal meant, once its own base is unglued from it. + + The same illness as :func:`canonical_cmd`, caught at the other end. A + portal that answered the listing with a resolved link is handed one back at + create_link, and one family of them builds its reply by gluing its base in + front of whatever it was given -- so a command that was already a complete + URL comes back carrying that base twice:: + + create_link('http://portal.example:80/USER/PASS/1225691') + -> 'http://portal.example:80/USER/PASS/USER/PASS/1225691?play_token=...' + + That path answers 401. The command itself answers 302 and plays, so the + command is what is returned, and the reply is dropped with its token: the + token was minted for a path that does not exist. + + ``canonical_cmd`` heads this off at sync time and is the better cure, but + it only fires on rows carrying an id -- a listing without one still stores + the resolved link, as does every portal synced before it existed. + + Structural on purpose, and settled without asking the provider anything. A + probe request would answer the question outright, and cost a connection + slot at the exact moment the tune needs it: providers count those, and a + subscription with one line would spend it here and fail the playback it was + checking for. + + Narrow, because a false positive throws a good token away: same scheme and + host, and the reply's path has to be the command's path exactly, behind a + prefix that is itself a directory of it. Everything else is returned as it + came -- including the ordinary case of a portal answering the same path + with a token added, which is what a working one does. + """ + cmd_link = extract_link(cmd) + if not cmd_link or not link: + return link + + meant, answered = urlparse(cmd_link), urlparse(link) + if (meant.scheme, meant.netloc) != (answered.scheme, answered.netloc): + return link + if not meant.path or not answered.path.endswith(meant.path): + return link + + prefix = answered.path[: -len(meant.path)].rstrip("/") + if prefix and meant.path.startswith(prefix + "/"): + return cmd_link + return link + + +def stream_id(cmd: str) -> str: + """The channel's own number, read back out of the command, or "". + + Named in the create_link request alongside the command, because a portal + that cannot find the channel in the command answers about no channel at + all: one returns the link with its ``stream`` parameter left empty, which + plays nothing. Reported and first fixed by @shayward, whose providers need + it on every request. + + Read from the two shapes a command takes here. The marker is what a portal + is meant to send and what :func:`canonical_cmd` rebuilds, so it is the only + one left after a sync -- reading the query alone would leave exactly the + installs that need this without it. The query is what an unrewritten + command carries, which is every portal synced before that existed. + + Whatever comes back is the portal's own number for the channel, taken from + what the portal itself wrote: this cannot name it a channel it did not name + first. An id that is not there at all gives "", and nothing is sent -- + an empty parameter is a question no portal asked to be asked. + """ + link = extract_link(cmd) + if not link: + return "" + + marker = _CANONICAL_CMD.search(urlparse(link).path) + if marker: + return marker.group(1) + + found = parse_qs(urlparse(link).query).get("stream") or [""] + return found[0].strip() + + def slugify(value: str) -> str: """Reduce a display name to something safe for URLs, keys and filenames.""" slug = re.sub(r"[^a-z0-9]+", "-", value.strip().lower()).strip("-") @@ -408,6 +628,19 @@ def parse_portals(text: str) -> Tuple[List[PortalConfig], List[str]]: errors.append(f"line {lineno}: max_streams must be a number") continue + # 'epg=1', 'epg=true', 'epg=yes' all mean the same thing to someone + # typing it from memory, so all of them are accepted; anything else + # -- including 'epg=0' -- leaves it off. + epg = extras.get("epg", "").strip().lower() in ("1", "true", "yes", "on") + try: + epg_hours = int(extras.get("epg_hours", DEFAULT_EPG_HOURS)) + except ValueError: + errors.append(f"line {lineno}: epg_hours must be a number of hours") + continue + if epg_hours < 1: + errors.append(f"line {lineno}: epg_hours must be at least 1") + continue + def resolve(key: str, fallback: str) -> Tuple[str, bool]: """This portal's value, else the built-in default. @@ -443,12 +676,8 @@ def resolve(key: str, fallback: str) -> Tuple[str, bool]: timezone=timezone, signature=signature, max_streams=max_streams, - # Absence of credentials selects device-ID auth, exactly as - # stalkerhek does (`deviceIdAuth := pd.Username == "" && - # pd.Password == ""` in webui/profiles.go). The default ffff... - # IDs are what a MAC-only portal expects, so this fires even when - # the user supplied no IDs of their own. - device_id_auth=not (username and password), + epg=epg, + epg_hours=epg_hours, ) portals.append(cfg) @@ -470,6 +699,8 @@ def format_portal_line( password: str = "", max_streams: int = 1, stb: Optional[Dict[str, str]] = None, + epg: bool = False, + epg_hours: int = DEFAULT_EPG_HOURS, ) -> str: """Render one canonical line for the Portals setting. @@ -493,6 +724,12 @@ def format_portal_line( # seeing on lines they never wrote themselves. if int(max_streams) != 1: extras.append(f"max_streams={int(max_streams)}") + if epg: + extras.append("epg=1") + # Same rule as max_streams: only written when it says something the + # default does not already say. + if int(epg_hours) != DEFAULT_EPG_HOURS: + extras.append(f"epg_hours={int(epg_hours)}") for key in STB_KEYS: value = (stb or {}).get(key, "").strip() @@ -535,6 +772,41 @@ def normalize_portal_url(url: str) -> str: return parsed._replace(path=path).geturl() +def alternate_endpoint(url: str) -> str: + """The other place a Stalker API lives, or '' when there isn't one. + + Ministra answers on two paths and installs differ in which they expose: + ``/c/portal.php``, which is what :func:`normalize_portal_url` builds + and what most providers hand out, and ``/server/load.php``, which is + the older canonical one and the only one pvr.stalker has ever asked for. + A portal serving just one of them used to be unusable if the user had been + given the other, with a 404 and nothing to suggest. + + The mapping is pvr.stalker's, read backwards as well as forwards:: + + http://h/c/portal.php -> http://h/server/load.php + http://h/stalker_portal/c/portal.php -> http://h/stalker_portal/server/load.php + http://h/server/load.php -> http://h/c/portal.php + """ + parsed = urlparse(url) + path = parsed.path + directory, _, filename = path.rpartition("/") + filename = filename.lower() + + if filename == "portal.php": + base = directory[:-2] if directory.lower().endswith("/c") else directory + new_path = base + "/server/load.php" + elif filename == "load.php": + base = directory[:-7] if directory.lower().endswith("/server") else directory + new_path = base + "/c/portal.php" + else: + return "" + + if new_path == path: + return "" + return parsed._replace(path=new_path).geturl() + + # --------------------------------------------------------------------------- # Shared state: Redis, mirrored to disk # --------------------------------------------------------------------------- @@ -747,6 +1019,60 @@ def claim_sync_lock(token: str, ttl: int = 1800, client=None) -> Optional[bool]: return None +def _auto_sync_key() -> str: + return f"{REDIS_PREFIX}:auto_sync" + + +def claim_auto_sync(ttl: int = 1800, client=None) -> bool: + """Claim the right to start a sync from a scheduled refresh. + + Load-bearing rather than defensive. A scheduled sync ends by asking + Dispatcharr to re-read the playlist it just wrote, and that re-read emits + the very event that started it -- so without a cooldown the two would take + turns forever, each round costing a login and a full channel download on a + MAC most providers allow one connection for. + + The TTL is what breaks the ring: it has to outlast a sync, which on a large + portal with a guide is minutes, so the schedule cannot usefully be finer + than an hour either way. + + Returns True when the caller may proceed. Redis being unreachable answers + False -- the opposite of :func:`claim_sync_lock`, and deliberately: that one + protects a button the user just pressed and should not refuse on a cache + outage, this one protects against a loop that nobody asked for. + """ + client = _client_or_none(client) + if client is None: + return False + try: + return bool(client.set(_auto_sync_key(), "1", nx=True, ex=max(60, int(ttl)))) + except Exception: + return False + + +def hold_auto_sync(ttl: int = 1800, client=None) -> None: + """Mute the scheduled path for a while, without asking permission. + + Claimed by any sync that ends by asking Dispatcharr to re-read a playlist, + because that re-read emits the event the schedule listens for -- and a sync + the user started by hand would otherwise come back as an echo and set off a + full re-fetch of every portal. Observed: adding one portal, and the echo + arriving two seconds later. + + A plain write rather than :func:`claim_auto_sync`'s NX, since the point is + to push the window forward whether or not one is already open. The cost is + that a manual sync delays the next scheduled one, which is the right + trade -- everything was just fetched. + """ + client = _client_or_none(client) + if client is None: + return + try: + client.set(_auto_sync_key(), "1", ex=max(60, int(ttl))) + except Exception: + pass + + def release_sync_lock(token: str, client=None) -> None: """Release the lock, but only while it is still ours. @@ -902,6 +1228,21 @@ class ChannelEntry: logo: str = "" genre_id: str = "" number: str = "" + # Catch-up, as the portal advertises it. Read but not yet published: + # Dispatcharr does pick these up from an M3U -- it turns 'tv_archive' and + # 'tv_archive_duration' attributes into a stream's is_catchup and + # catchup_days -- but playing one back is Xtream-only, built from a + # server URL and credentials this plugin's sources do not have + # (apps/timeshift/helpers.py). Emitting them would light up a catch-up + # badge on channels whose catch-up cannot play, which is worse than not + # offering it. Captured here so the day that path stops being + # Xtream-shaped, the data is already arriving. + tv_archive: bool = False + tv_archive_duration: str = "" + # Whether the portal's own command had to be rebuilt into a marker -- see + # canonical_cmd. Counted rather than logged per channel, because on the + # portal that prompted it, 647 of them arrived at once. + cmd_rewritten: bool = False class Portal: @@ -912,15 +1253,45 @@ class Portal: and skip the handshake. """ - def __init__(self, cfg: PortalConfig, token: str = "", timeout: Optional[int] = None): + def __init__( + self, + cfg: PortalConfig, + token: str = "", + timeout: Optional[int] = None, + retries: int = 0, + ): self.cfg = cfg + # Where the API is asked, which starts as the configured URL and may be + # swapped once by login() for the portal's other endpoint. Kept apart + # from cfg.url on purpose: that one is still what logos are resolved + # against, and swapping the API path must not move them. + self.url = cfg.url self.token = token # The portal's own setting unless a caller insists, so every request # made about a portal honours what the user configured for it. self.timeout = timeout or getattr(cfg, "timeout", None) or DEFAULT_TIMEOUT + # Retrying is the caller's decision, not this class's, and it defaults + # to off because the caller that matters most must not have it. At tune + # time a portal that is not answering has to fail *now*: the resolver's + # only job on a bad source is to exit non-zero fast enough that + # Dispatcharr moves to the next one -- the same reasoning that keeps + # ffmpeg's -reconnect out of the default arguments. A sync has the + # opposite need, and asks for retries explicitly. + self.retries = max(0, int(retries)) self.session = requests.Session() # Non-fatal notes from login(), for the caller to surface. self.warnings: List[str] = [] + # Whether the portal said the token it handed back is already good for + # more than the handshake. Reported straight back to it in get_profile. + self.valid_token = False + # What get_profile answered during login(), kept so nothing has to ask + # twice: the expiry report and the blocked flag both read it. + self.profile: Dict[str, Any] = {} + # Which flow login() ended up taking, for the "Test portals" report. + # Worth saying out loud: it is the portal's choice, not the user's, so + # it is the one thing that tells them whether the credentials they + # typed in are being used at all. + self.auth_method = "handshake only" # -- plumbing --------------------------------------------------------- @@ -949,27 +1320,82 @@ def _headers(self, with_auth: bool = True) -> Dict[str, str]: headers["Authorization"] = "Bearer " + self.token return headers + def _common_params(self, with_auth: bool = True) -> str: + """Identity repeated in the query string, beside the cookie and header. + + Belt and braces, and cheap. The MAC travels in a cookie and the token in + an Authorization header because that is what a MAG box does and what + Ministra reads -- but plenty of what this plugin meets are not Ministra, + and open-tv authenticates against real portals using *only* these two + query parameters, with no cookie and no header at all. Sending both + forms covers portals that read either, and no portal has been seen to + mind the one it ignores. + """ + params = f"mac={quote(self.cfg.mac)}" + if with_auth and self.token: + params += f"&token={quote(self.token)}" + return params + + def _request(self, method: str, url: str, **kwargs) -> requests.Response: + """One portal request, repeated only while repeating it could help. + + Returns the response for the caller to interpret, including a final + failing one: deciding what an HTTP 404 means is :meth:`_get_json`'s job, + not this one's. Only exhausting the attempts without ever getting an + answer raises here. + """ + last_error = "" + for attempt in range(self.retries + 1): + if attempt: + time.sleep(RETRY_BACKOFF[min(attempt, len(RETRY_BACKOFF)) - 1]) + try: + resp = self.session.request( + method, url, timeout=self.timeout, **kwargs + ) + except requests.RequestException as exc: + last_error = f"request to portal failed: {exc}" + continue + if resp.status_code in RETRYABLE_STATUS and attempt < self.retries: + last_error = f"portal returned HTTP {resp.status_code}" + continue + return resp + + raise PortalError(last_error or "request to portal failed") + def _get_json(self, query: str, with_auth: bool = True) -> Any: - url = f"{self.cfg.url}?{query}" - try: - resp = self.session.get( - url, headers=self._headers(with_auth), timeout=self.timeout + url = f"{self.url}?{query}&{self._common_params(with_auth)}" + resp = self._request("GET", url, headers=self._headers(with_auth)) + + if resp.status_code in (401, 403): + raise PortalAuthError( + f"portal refused the session (HTTP {resp.status_code})" ) - except requests.RequestException as exc: - raise PortalError(f"request to portal failed: {exc}") from exc if resp.status_code < 200 or resp.status_code >= 300: snippet = (resp.text or "").strip()[:300] - raise PortalError( - f"portal returned HTTP {resp.status_code}" - + (f": {snippet}" if snippet else "") + message = f"portal returned HTTP {resp.status_code}" + ( + f": {snippet}" if snippet else "" ) + # A 404 is a web server saying nothing lives at this path -- which + # is a statement about the path, not about the portal, and login() + # has somewhere else to look. + if resp.status_code == 404: + raise PortalEndpointError(message) + raise PortalError(message) try: return resp.json() except ValueError: snippet = (resp.text or "").strip()[:300] - raise PortalError(f"portal returned non-JSON response: {snippet}") + # A dead session is answered in plain text with a 200 attached, so + # it arrives here rather than as an HTTP error. Saying so is what + # lets the resolver re-authenticate instead of failing the tune. + if snippet.lower() == AUTH_FAILED_BODY: + raise PortalAuthError("portal says the session is no longer authorised") + # Anything else that is not JSON is an HTML error page, a login + # form, or a landing page: something is listening, but it is not a + # Stalker API, so the other endpoint is worth a try. + raise PortalEndpointError(f"portal returned non-JSON response: {snippet}") # -- authentication --------------------------------------------------- @@ -982,12 +1408,22 @@ def handshake(self) -> str: js = data.get("js") if isinstance(data, dict) else None if isinstance(js, dict) and js.get("token"): self.token = str(js["token"]) + # 'not_valid' is the portal saying the token still has to be + # earned. get_profile is told the same thing back, which is how it + # knows whether it is being asked to validate or merely to report. + self.valid_token = str(js.get("not_valid") or "0") in ("0", "") if not self.token: raise PortalError("handshake did not yield a token") return self.token def authenticate(self) -> None: - """Associate credentials with the token (username/password portals).""" + """Associate credentials with the token. Run when the profile says 2. + + Sent as a POST, where every other call here is a GET: pvr.stalker puts + the login and password in the query string, and there is no reason for + this plugin to write a subscriber's password into a proxy's access log + when the portal accepts a form body just as happily. + """ form = { "type": "stb", "action": "do_auth", @@ -999,72 +1435,200 @@ def authenticate(self) -> None: } headers = self._headers() headers["Content-Type"] = "application/x-www-form-urlencoded" + url = f"{self.url}?{self._common_params()}" + resp = self._request("POST", url, data=form, headers=headers) + try: - resp = self.session.post( - self.cfg.url, data=form, headers=headers, timeout=self.timeout - ) payload = resp.json() - except requests.RequestException as exc: - raise PortalError(f"authentication request failed: {exc}") from exc except ValueError: raise PortalError("authentication returned a non-JSON response") if not payload.get("js"): - raise PortalError(payload.get("text") or "invalid credentials") - - def authenticate_with_device_ids(self) -> None: - """Second-step auth for portals keyed on device IDs rather than login.""" - data = self._get_json( - "type=stb&action=get_profile&JsHttpRequest=1-xml&hd=1" - f"&sn={self.cfg.serial_number}&stb_type={self.cfg.model}" - f"&device_id={self.cfg.device_id}&device_id2={self.cfg.device_id2}" - "&auth_second_step=1" + # The portal read the credentials and said no. Not retryable, and + # not the same failure as never having reached it. + raise PortalAuthError(payload.get("text") or "invalid credentials") + + def get_profile(self, auth_second_step: bool = False) -> Dict[str, Any]: + """Present the box to the portal and read back what it makes of it. + + This is the request that carries the whole STB identity, and the reply + is the portal stating what it wants next -- see :meth:`login`. The + field list is libstalkerclient's, unchanged: portals have been known to + reject a profile that arrives with fields missing, and the cost of + sending all of them is a longer query string. + """ + query = ( + "type=stb&action=get_profile&JsHttpRequest=1-xml" + f"&hd=1&num_banks={STB_NUM_BANKS}" + f"&image_version={STB_IMAGE_VERSION}&hw_version={quote(STB_HW_VERSION)}" + f"&ver={quote(STB_VERSION)}" + f"&stb_type={quote(self.cfg.model)}&sn={quote(self.cfg.serial_number)}" + f"&device_id={quote(self.cfg.device_id)}" + f"&device_id2={quote(self.cfg.device_id2)}" + f"&signature={quote(self.cfg.signature)}" + f"¬_valid_token={0 if self.valid_token else 1}" + f"&auth_second_step={1 if auth_second_step else 0}" ) + data = self._get_json(query) js = data.get("js") if isinstance(data, dict) else None - if not isinstance(js, dict) or not js.get("id"): - raise PortalError( - (data or {}).get("text") or "device ID authentication rejected" - ) + return js if isinstance(js, dict) else {} + + # What get_profile's 'status' means. The portal decides which authentication + # this account needs and says so here, rather than the client guessing from + # whether a password happens to be configured. + _PROFILE_OK = 0 + _PROFILE_NEEDS_AUTH = 2 def login(self) -> str: - """Handshake plus whichever auth flow this portal needs. - - Credentials win when present. Otherwise the device-ID step runs, which - is what stalkerhek does for MAC-only portals -- but its failure is not - fatal here: plenty of portals authorise purely on the MAC cookie and - either lack ``get_profile`` or answer it without a profile id. If the - session really is unauthorised, the very next call fails with a far - more useful message than "device ID authentication rejected". + """Handshake, then whichever authentication the portal asks for. + + The portal is the one that knows:: + + handshake -> token (+ 'not_valid': is it good for anything yet?) + get_profile + status 0 -> done + status 2 -> do_auth, then get_profile(auth_second_step=1) + anything -> refused; 'block_msg'/'msg' says why + + This replaces guessing the flow from whether credentials were typed in. + The old guess was wrong in both directions -- it ran a device-ID step + against portals that wanted a password, and it had no way to tell a + blocked account from an empty line-up. + + Two deliberate departures from that state machine, both of them + tolerance for portals that are not really Ministra: + + * a reply with no ``status`` at all counts as 0. Ministra always sends + one; the clones this plugin mostly meets often do not, and the + previous version happily served them. pvr.stalker treats the same + silence as a failure, which would break every one of those installs. + * ``get_profile`` failing to answer *at all* -- 404, a gateway error, + prose instead of JSON -- is a warning, not an error. Plenty of + portals authorise on the MAC cookie alone and never implement it. An + explicit refusal (:class:`PortalAuthError`) is still fatal, because + that is the portal answering rather than failing to. """ - self.handshake() + self._handshake_on_either_endpoint() - if self.cfg.username and self.cfg.password: - self.authenticate() - elif self.cfg.device_id_auth: - try: - self.authenticate_with_device_ids() - except PortalError as exc: - self.warnings.append( - f"device-ID authentication did not succeed ({exc}); " - "continuing with MAC-only authorisation" + try: + self.profile = self.get_profile() + except PortalAuthError: + raise + except PortalError as exc: + self.warnings.append( + f"the portal did not answer get_profile ({exc}); " + "continuing with MAC-only authorisation" + ) + return self.token + + status = self._profile_status(self.profile) + self.auth_method = "profile" + + if status == self._PROFILE_NEEDS_AUTH: + if not (self.cfg.username and self.cfg.password): + raise PortalAuthError( + self._profile_message(self.profile) + or "this portal wants a username and password; add " + "'username=... password=...' to its line" ) + self.authenticate() + self.profile = self.get_profile(auth_second_step=True) + status = self._profile_status(self.profile) + self.auth_method = "credentials" + + if status != self._PROFILE_OK: + raise PortalAuthError( + self._profile_message(self.profile) + or f"portal refused the session (status {status})" + ) return self.token + def _handshake_on_either_endpoint(self) -> None: + """Shake hands, trying the portal's other API path if this one is not it. + + The handshake is every session's first request, so a portal reached at + the wrong path fails here and nowhere later -- which makes this the one + place worth spending an extra round-trip on. + + Only a :class:`PortalEndpointError` earns that second try: a 404, or an + answer that is not JSON. A portal that is unreachable, unwell or + refusing the MAC would answer identically on both paths, and at tune + time a wasted round-trip is time Dispatcharr is not yet spending on the + next source. + + The swap lasts for this session only. Nothing is written back, so the + cost is one failed request per sync and per token expiry -- small, and + the warning tells the user how to stop paying it for good. + """ + try: + self.handshake() + return + except PortalEndpointError as exc: + alternate = alternate_endpoint(self.url) + if not alternate: + raise + first_failure = exc + + self.url = alternate + try: + self.handshake() + except PortalError: + # The other path is no better. Report the original failure: it is + # the one about the URL the user actually configured. + self.url = self.cfg.url + raise first_failure + + self.warnings.append( + f"the portal does not answer at {self.cfg.url} ({first_failure}), " + f"but does at {alternate}; put that on its portal line to save a " + "failed request on every sync" + ) + + @staticmethod + def _profile_status(profile: Dict[str, Any]) -> int: + """``status`` as an int. Absent, blank or unparseable all mean OK.""" + raw = profile.get("status") + if raw is None or raw == "": + return Portal._PROFILE_OK + try: + return int(raw) + except (TypeError, ValueError): + return Portal._PROFILE_OK + + @staticmethod + def _profile_message(profile: Dict[str, Any]) -> str: + """The portal's own explanation, if it gave one. + + ``block_msg`` first: when both are set it is the specific one, and it + is what the reseller wrote for exactly this situation. + """ + for key in ("block_msg", "msg"): + value = str(profile.get(key) or "").strip() + if value: + return value + return "" + def account_snapshot(self) -> Dict[str, Any]: """What the portal will say about the subscription itself. - Two calls, both made once per sync and never at tune time. Neither is - required for anything to work, so this returns what it managed to read - and never raises. + One call, made once per sync and never at tune time. Not required for + anything to work, so this returns what it managed to read and never + raises. ``get_main_info`` is where resellers put the expiry date: Ministra shows the ``phone`` field in the MAG interface, so that is the field they fill in with it -- observed on every portal tested, in the form - "August 18, 2027, 4:53 pm". ``get_profile`` carries ``blocked``, which - turns "nothing plays and I do not know why" into one line of the report. + "August 18, 2027, 4:53 pm". + + ``blocked`` comes from the profile :meth:`login` already read, rather + than from a second ``get_profile``. It is nearly always redundant now -- + a blocked account normally answers with a non-zero ``status``, which + login refuses outright -- but portals that set the flag and leave the + status at 0 exist, and for those it is still the only warning anyone + gets. - No connection limit is available from either: neither ``max_online`` nor + No connection limit is available anywhere: neither ``max_online`` nor an equivalent exists in the responses, and ``playback_limit`` is a portal-wide Ministra default (3 on unrelated providers, next to ``tv_playback_retry_limit`` = 3), not this account's allowance. Guessing @@ -1081,23 +1645,18 @@ def account_snapshot(self) -> Dict[str, Any]: except Exception: pass - try: - js = self._get_json( - "type=stb&action=get_profile&hd=1&JsHttpRequest=1-xml" - ).get("js") - if isinstance(js, dict): - snapshot["blocked"] = str(js.get("blocked") or "0") not in ("0", "") - except Exception: - pass + snapshot["blocked"] = str(self.profile.get("blocked") or "0") not in ("0", "") return snapshot - def watchdog(self) -> None: - """Keep-alive ping. Only needed by portals that drop idle sessions.""" - self._get_json( - "action=get_events&event_active_id=0&init=0&type=watchdog" - "&cur_play_type=1&JsHttpRequest=1-xml" - ) + # There is no watchdog here, and its absence is a decision rather than an + # omission. Stalker clients keep a session warm by calling get_events every + # 'timeslot' seconds, which needs something alive between requests to do the + # calling. This plugin has no such thing: the sync is a task that ends, and + # the resolver is a process that becomes ffmpeg. A ping method existed for + # two releases with no caller, which is worse than not having one -- it read + # as a feature. Sessions are re-established instead, which is what the token + # cache and the resolver's re-authentication are for. # -- content ---------------------------------------------------------- @@ -1130,24 +1689,245 @@ def get_all_channels(self) -> List[ChannelEntry]: channels: List[ChannelEntry] = [] for row in rows: - if not isinstance(row, dict): - continue - cmd = str(row.get("cmd") or "").strip() - name = str(row.get("name") or "").strip() - if not cmd or not name: - continue - channels.append( - ChannelEntry( - channel_id=str(row.get("id") or ""), - name=name, - cmd=cmd, - logo=str(row.get("logo") or ""), - genre_id=str(row.get("tv_genre_id") or ""), - number=str(row.get("number") or ""), - ) + channel = self._channel_from_row(row) + if channel is not None: + channels.append(channel) + return channels + + @staticmethod + def _channel_from_row(row: Any) -> Optional[ChannelEntry]: + """One row of a channel listing, or None when it is not usable. + + A row without a name or without a command is not a channel this plugin + can do anything with, and portals do emit them. + """ + if not isinstance(row, dict): + return None + cmd = str(row.get("cmd") or "").strip() + name = str(row.get("name") or "").strip() + if not cmd or not name: + return None + + channel_id = str(row.get("id") or "") + marker = canonical_cmd(cmd, channel_id) + + return ChannelEntry( + channel_id=channel_id, + name=name, + cmd=marker, + cmd_rewritten=marker != cmd, + logo=str(row.get("logo") or ""), + genre_id=str(row.get("tv_genre_id") or ""), + number=str(row.get("number") or ""), + # Portals write these as 1/0, and sometimes as "1"/"0". + tv_archive=str(row.get("enable_tv_archive") or "0") not in ("0", ""), + tv_archive_duration=str(row.get("tv_archive_duration") or ""), + ) + + def get_ordered_list(self, page: int) -> Dict[str, Any]: + """One page of the paginated listing, as the MAG interface browses it. + + ``genre=*`` is every genre, which is what the box asks for on its "All" + screen. pvr.stalker leaves it out -- its request builder drops any + optional parameter still equal to its own default, and its default is + ``*`` -- but sending it says the same thing to portals that have no + such default to fall back on. + """ + data = self._get_json( + "type=itv&action=get_ordered_list&JsHttpRequest=1-xml" + f"&genre=*&fav=0&sortby=number&p={int(page)}" + ) + js = data.get("js") if isinstance(data, dict) else None + return js if isinstance(js, dict) else {} + + def list_channels(self, progress=None) -> List[ChannelEntry]: + """Every channel, however this portal is willing to hand them over. + + ``get_all_channels`` first, because one request for the whole line-up is + what nearly every portal supports and is enormously cheaper. Portals + that cap it, or never implemented it, answer with an error or with + nothing -- and used to leave the portal unusable. For those, the listing + is collected a page at a time instead: hundreds of requests where there + was one, several minutes where there were seconds, and worth every bit + of it because the alternative is a portal that cannot be synced at all. + + ``progress`` is called with a line of English whenever there is + something worth saying, so a sync that has gone quiet for four minutes + can account for itself. Passed as a callback rather than a logger to + keep this module free of anything Django-shaped. + + A portal that refuses the session is not asked twice: paging would be + refused for exactly the same reason, and the second refusal is the one + that would get a MAC noticed. + """ + try: + return self.get_all_channels() + except PortalAuthError: + raise + except PortalError as exc: + single_shot_failure = exc + + if progress: + progress( + f"the portal would not list its channels in one request " + f"({single_shot_failure}); collecting them a page at a time" ) + + channels = self._paged_channels(progress) + if not channels: + # Nothing worked, so the first failure is the one worth reporting: + # it is the one whose message names the likely cause. + raise single_shot_failure + + if progress: + progress(f"collected {len(channels)} channels by paging") return channels + def _paged_channels(self, progress=None) -> List[ChannelEntry]: + """Walk get_ordered_list until one of three things says to stop. + + All three are needed, because each covers a portal the others do not: + + * the page count the portal itself implies, from ``total_items`` and + ``max_page_items`` on the first page. The bound pvr.stalker uses, and + the only one that stops at exactly the right place. + * a page with no rows. What a well-behaved portal does past the end, + and open-tv's only guard. + * a page whose rows have all been seen already. Portals that clamp ``p`` + to their last page answer forever otherwise, which is the case that + turns open-tv's loop into an infinite one. + """ + channels: List[ChannelEntry] = [] + seen = set() + max_pages = ORDERED_LIST_PAGE_CAP + page = 1 + + while page <= max_pages: + js = self.get_ordered_list(page) + rows = js.get("data") + if not isinstance(rows, list) or not rows: + break + + if page == 1: + implied = self._page_count(js) + if implied: + max_pages = min(max_pages, implied) + if progress: + progress(f"the portal reports {implied} pages to read") + + fresh = 0 + for row in rows: + channel = self._channel_from_row(row) + if channel is None: + continue + # The id when there is one, the command when there is not: two + # channels never share a command, and a portal that omits ids + # would otherwise collapse its whole line-up into one entry. + key = channel.channel_id or channel.cmd + if key in seen: + continue + seen.add(key) + channels.append(channel) + fresh += 1 + + if not fresh: + break + + if progress and page % 20 == 0: + progress(f"page {page}, {len(channels)} channels so far") + page += 1 + + return channels + + @staticmethod + def _page_count(js: Dict[str, Any]) -> int: + """How many pages the portal implies, or 0 when it does not say.""" + total = as_int(js.get("total_items")) + per_page = as_int(js.get("max_page_items")) + if total > 0 and per_page > 0: + return (total + per_page - 1) // per_page + return 0 + + def get_epg_info(self, hours: int, scratch_dir: Optional[str] = None) -> Dict[str, Any]: + """The whole line-up's guide, keyed by the portal's channel id. + + Deliberately not routed through :meth:`_get_json`, which is built for + replies that fit in a breath. This one does not: a portal with 13,000 + channels answers a single day with something in the order of 100 MB, + and ``_get_json`` would hold the encoded bytes and the decoded object + at the same time. + + So the reply is streamed to a scratch file first. That does not make + the parse cheaper -- the decoded structure is what it is -- but it does + two things worth the detour: nothing is decoded until the size is known, + so an absurd answer is refused instead of discovered by having the + worker killed; and requests never has to buffer the whole body. + + Returns ``{channel_id: [programme, ...]}``, empty when the portal has + no guide to give. Never returns None, so a caller can iterate it. + """ + query = ( + "type=itv&action=get_epg_info&JsHttpRequest=1-xml" + f"&period={int(hours)}" + ) + url = f"{self.url}?{query}&{self._common_params()}" + resp = self._request("GET", url, headers=self._headers(), stream=True) + + if resp.status_code in (401, 403): + raise PortalAuthError( + f"portal refused the guide (HTTP {resp.status_code})" + ) + if resp.status_code < 200 or resp.status_code >= 300: + raise PortalError(f"portal returned HTTP {resp.status_code} for the guide") + + with tempfile.TemporaryFile(dir=scratch_dir) as scratch: + size = 0 + for chunk in resp.iter_content(chunk_size=1024 * 256): + if not chunk: + continue + size += len(chunk) + if size > EPG_MAX_BYTES: + resp.close() + raise PortalError( + f"the guide is larger than {EPG_MAX_BYTES // (1024 * 1024)} MB " + f"and was abandoned; ask for fewer hours than {int(hours)} " + "with 'epg_hours=' on the portal line" + ) + scratch.write(chunk) + + scratch.seek(0) + try: + data = json.load(scratch) + except ValueError: + raise PortalError("the guide was not JSON") + + js = data.get("js") if isinstance(data, dict) else None + if not isinstance(js, dict): + raise PortalError( + f"the guide came back as {type(js).__name__} rather than an object" + ) + + rows = js.get("data") + if isinstance(rows, dict): + return rows + + # An empty list is how a portal says it has no full guide -- observed on + # several, always as `{"js": {"data": []}}`. Not an error: plenty of + # portals carry a channel list and no programmes for it, or serve a + # guide one channel at a time, which is a different action and not one + # this uses. + if rows is None or (isinstance(rows, list) and not rows): + return {} + + # Anything else is a shape nobody has met yet, and silently treating it + # as "no guide" is how it would stay unmet. A flat list of programmes, + # say, would be perfectly usable if someone knew it was arriving. + raise PortalError( + f"the guide arrived as {type(rows).__name__} with " + f"{len(rows) if hasattr(rows, '__len__') else '?'} entries, which " + "is not a shape this understands -- please report it" + ) + def create_link(self, cmd: str) -> str: """Ask the portal for a playable URL for ``cmd``. @@ -1155,28 +1935,69 @@ def create_link(self, cmd: str) -> str: ``"ffmpeg http://host/stream.m3u8?token=..."`` -- so :func:`extract_link` finds the playable part. These links are short-lived, which is why this is called at tune time rather than sync time. + + The channel is named twice where it can be: in the command, and in a + ``stream`` parameter beside it. Ministra reads the command and ignores + the rest; the portals that do not answer about no channel at all + without it -- see :func:`stream_id`. Only ever sent with a value, so a + portal that never asked for it sees the request it has always seen. """ - data = self._get_json( - f"action=create_link&type=itv&cmd={quote(cmd, safe='')}&JsHttpRequest=1-xml" - ) + query = f"action=create_link&type=itv&cmd={quote(cmd, safe='')}" + channel = stream_id(cmd) + if channel: + query += f"&stream={quote(channel, safe='')}" + + data = self._get_json(f"{query}&JsHttpRequest=1-xml") js = data.get("js") if isinstance(data, dict) else None + # Typed as auth failures, both of them, because that is overwhelmingly + # what they are: a portal whose token has expired usually answers + # create_link with a hollow success -- 'js' false, or a 'cmd' that is + # empty -- rather than with the plain-text refusal. The resolver only + # re-authenticates on this class, so mistyping these would leave the + # cached-token path unable to recover from the very thing it exists + # for. The cost of being wrong is one handshake. if not isinstance(js, dict): - raise PortalError("create_link returned no data (session may have expired)") + raise PortalAuthError( + "create_link returned no data (session may have expired)" + ) raw = str(js.get("cmd") or "").strip() if not raw: - raise PortalError("create_link returned an empty command") + raise PortalAuthError("create_link returned an empty command") link = extract_link(raw) if not link: raise PortalError(f"create_link returned an unusable command: {raw[:200]}") + + unglued = undoubled_link(cmd, link) + if unglued != link: + self.warnings.append( + f"the portal answered with its own base in front of a command " + f"that was already a link ({link[:160]}); that path does not " + "exist, so the command itself is what plays" + ) + return unglued return link def logo_url(self, logo: str) -> str: - """Absolute URL for a channel logo, or '' when there isn't one.""" + """Absolute URL for a channel logo, or '' when there isn't one. + + Two shapes beyond the obvious, both from pvr.stalker's + ``DetermineLogoURI``, both seen in the wild: + + * an inline ``data:`` image, which is dropped. It is a valid logo and a + useless one here -- Dispatcharr stores this in a URL field, and a + base64 payload has no business in an M3U attribute. + * any scheme at all, not just http(s). A portal serving its logos from + somewhere else says so with a scheme, and treating that as a filename + produced a URL that pointed nowhere. + """ + logo = (logo or "").strip() if not logo: return "" - if logo.startswith(("http://", "https://")): + if logo[:5].lower() == "data:": + return "" + if "://" in logo: return logo parsed = urlparse(self.cfg.url) base_dir = parsed.path.rsplit("/", 1)[0] or "" diff --git a/sync.py b/sync.py index 8e69ae4..c519cf6 100644 --- a/sync.py +++ b/sync.py @@ -12,15 +12,20 @@ import logging import os +import re import tempfile -from typing import Any, Dict, List, Optional +from datetime import datetime, timezone +from typing import Any, Dict, Iterator, List, Optional +from xml.sax.saxutils import escape, quoteattr from .stalker_api import ( ChannelEntry, Portal, PortalConfig, PortalError, + as_int, encode_pseudo_url, + hold_auto_sync, python_executable, save_fallback, save_portal, @@ -30,6 +35,16 @@ # generated playlists alongside user-uploaded ones and inside the data volume. M3U_DIR = "/data/uploads/m3us" +# The same idea for guides. Kept out of Dispatcharr's own 'cached_epg', which it +# fills with files named after a source id -- ours are named after a portal and +# are inputs to that machinery rather than products of it. +XMLTV_DIR = "/data/uploads/epgs" + +# How long a sync mutes the scheduled path after asking for a re-read. Matches +# plugin.AUTO_SYNC_COOLDOWN, and is defined here because this module may not +# import from plugin.py -- the dependency runs the other way. +AUTO_SYNC_COOLDOWN = 1800 + STREAM_PROFILE_NAME = "Distalker" ACCOUNT_PREFIX = "Distalker: " @@ -67,6 +82,10 @@ def build_m3u( ``group-title`` carries the portal's own genre, which is what Dispatcharr turns into Channel Groups -- so the existing M3U Accounts / Groups UI does all the filtering, and this plugin does none. + + Catch-up is deliberately not advertised here even though the portal tells + us about it and Dispatcharr would read it -- see ``ChannelEntry`` for why + the badge would be a promise nothing can keep. """ slug = portal.cfg.slug lines = ["#EXTM3U"] @@ -110,6 +129,162 @@ def write_m3u(slug: str, content: str) -> str: return path +# --------------------------------------------------------------------------- +# XMLTV generation +# --------------------------------------------------------------------------- +# +# Dispatcharr reads a guide with lxml's iterparse and takes five things from it +# (apps/epg/tasks.py): a channel's id, its first display-name and its icon; +# then each programme's channel, start, stop, title, desc and sub-title. +# Everything else in the XMLTV vocabulary is ignored, so none of it is written. + +# Characters XML 1.0 has no way to carry. Portals do send them -- a stray 0x03 +# inside a programme description is enough to make lxml's recovery drop the +# element around it, so they are removed rather than escaped. +_ILLEGAL_XML = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f]") + + +def _xml_text(value: Any) -> str: + """A string safe to place between XML tags.""" + return escape(_ILLEGAL_XML.sub("", str(value or ""))) + + +def _xmltv_time(value: Any) -> str: + """A Unix timestamp as XMLTV writes it, or '' if it is not one. + + ``YYYYMMDDHHMMSS +0000``: exactly the 20 characters Dispatcharr's + ``parse_xmltv_time`` expects, and always UTC, because a portal's epoch is + an instant and the local time it corresponds to is nobody's business here. + """ + seconds = as_int(value, -1) + if seconds <= 0: + return "" + try: + moment = datetime.fromtimestamp(seconds, timezone.utc) + except (OverflowError, OSError, ValueError): + return "" + return moment.strftime("%Y%m%d%H%M%S +0000") + + +def build_xmltv( + portal: Portal, + channels: List[ChannelEntry], + epg_data: Dict[str, Any], +) -> Iterator[str]: + """Yield an XMLTV document for the channels the portal has a guide for. + + A generator, and ``epg_data`` is emptied as it goes: the guide for a large + portal is the biggest structure this plugin ever holds, and building the + document as one string would mean holding it twice. The caller writes each + piece out and nothing accumulates. + + Only channels with at least one programme get an entry. A ```` + with nothing under it still becomes a row in Dispatcharr's EPG table, and + 13,000 rows that will never match a programme are not a guide, they are + thirteen thousand empty promises in the channel-to-EPG picker. + """ + slug = portal.cfg.slug + + def tvg_id(channel: ChannelEntry) -> str: + # The identifier already written into the playlist. The two must agree + # exactly or nothing binds -- see test_the_playlist_and_the_guide_agree. + return f"{slug}.{channel.channel_id}" if channel.channel_id else "" + + listed = [ + channel for channel in channels + if tvg_id(channel) and epg_data.get(channel.channel_id) + ] + + yield '\n' + yield '\n' + + for channel in listed: + yield f" \n" + yield f" {_xml_text(channel.name)}\n" + logo = portal.logo_url(channel.logo) + if logo: + yield f" \n" + yield " \n" + + for channel in listed: + # pop, not get: this is where the memory goes back. + programmes = epg_data.pop(channel.channel_id, None) or [] + channel_id = quoteattr(tvg_id(channel)) + for start, stop, title, description in _timeline(programmes): + yield ( + f" \n" + ) + yield f" {title}\n" + if description: + yield f" {description}\n" + yield " \n" + + yield "\n" + + +def _timeline(programmes: Any) -> Iterator[tuple]: + """One channel's programmes, in order and without overlaps. + + Portals do send overlapping entries -- the same show listed twice with two + start times and one end, which is what a guide that has been corrected in + place looks like from outside. XMLTV permits it and readers do not expect + it: Dispatcharr picks a programme for an instant by searching the ones that + span it (``_match_epg_program_by_timeslot``), so two candidates for the same + minute is an arbitrary answer to "what is on now". + + Earliest start wins, since it is the one covering the whole slot. Anything + beginning before the kept programme ends is dropped; touching exactly, which + is what back-to-back programmes do, is not an overlap and is kept. + """ + usable = [] + for programme in programmes if isinstance(programmes, list) else []: + if not isinstance(programme, dict): + continue + start = _xmltv_time(programme.get("start_timestamp")) + stop = _xmltv_time(programme.get("stop_timestamp")) + # A programme without both ends is not a programme. Dispatcharr would + # store it with a nonsense duration rather than reject it -- and the + # portal's own 'duration' field is no help: it arrives negative on + # programmes that plainly last two hours. + if not start or not stop or stop <= start: + continue + title = _xml_text(programme.get("name")) + if not title: + continue + usable.append((start, stop, title, _xml_text(programme.get("descr")))) + + # The timestamps sort correctly as strings: fixed width, most significant + # first, and all of them UTC. + usable.sort(key=lambda item: (item[0], item[1])) + + last_stop = "" + for entry in usable: + if entry[0] < last_stop: + continue + last_stop = entry[1] + yield entry + + +def write_xmltv(slug: str, chunks: Iterator[str]) -> str: + """Stream a guide to disk atomically, as :func:`write_m3u` does for a playlist.""" + os.makedirs(XMLTV_DIR, exist_ok=True) + path = os.path.join(XMLTV_DIR, f"distalker-{slug}.xml") + + fd, temp_path = tempfile.mkstemp(dir=XMLTV_DIR, prefix=f".distalker-{slug}-") + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + for chunk in chunks: + handle.write(chunk) + os.replace(temp_path, path) + except Exception: + if os.path.exists(temp_path): + os.unlink(temp_path) + raise + + return path + + # --------------------------------------------------------------------------- # Dispatcharr model wiring # --------------------------------------------------------------------------- @@ -124,7 +299,7 @@ def distalker_accounts(): ) -def upsert_account(cfg: PortalConfig, file_path: str): +def upsert_account(cfg: PortalConfig, file_path: str, refresh_hours: int = 0): """Create or update the M3U account backing one portal. Dispatcharr supports file-backed accounts as a first-class refresh source @@ -153,9 +328,14 @@ def upsert_account(cfg: PortalConfig, file_path: str): # Portals typically allow a single connection per MAC; exceeding it is the # fastest way to get the account blocked. account.max_streams = cfg.max_streams - # We regenerate the file ourselves before every refresh, so Dispatcharr - # scheduling its own re-parse of a stale file would only cause confusion. - account.refresh_interval = 0 + # Dispatcharr's own schedule, turned into ours. Left at 0 the account is + # never re-read on its own and every fetch is a button press -- which is + # what this plugin did for its whole life, because a plugin cannot register + # a Celery task of its own (see tasks.py). Set, Dispatcharr creates and + # runs the periodic task itself, and the m3u_refresh it ends with is what + # calls the plugin back to re-fetch the portal. The clock is borrowed + # rather than built. + account.refresh_interval = max(0, int(refresh_hours or 0)) properties = dict(account.custom_properties or {}) properties[MARKER_KEY] = {"slug": cfg.slug, "portal": cfg.url} @@ -168,6 +348,191 @@ def upsert_account(cfg: PortalConfig, file_path: str): return account, created +def distalker_epg_sources(): + """Every EPG source this plugin owns, found by marker rather than by name.""" + from apps.epg.models import EPGSource + + return EPGSource.objects.filter(custom_properties__has_key=MARKER_KEY) + + +def upsert_epg_source(cfg: PortalConfig, file_path: str): + """Create or update the EPG source backing one portal. + + The mirror image of :func:`upsert_account`, and for the same reason: a + source with a ``file_path`` and no ``url`` is a first-class case in + Dispatcharr (``apps/epg/tasks.py`` takes the local-file branch when + ``not source.url``), so the guide costs no HTTP endpoint either. + """ + from apps.epg.models import EPGSource + + source = distalker_epg_sources().filter( + **{f"custom_properties__{MARKER_KEY}__slug": cfg.slug} + ).first() + + created = False + if source is None: + source, created = EPGSource.objects.get_or_create( + name=ACCOUNT_PREFIX + cfg.name, + defaults={"source_type": "xmltv"}, + ) + + source.name = ACCOUNT_PREFIX + cfg.name + source.source_type = "xmltv" + source.file_path = file_path + # Blank, and that is what selects the local-file branch. A source with both + # would be downloaded from the URL and our file ignored. + source.url = None + source.is_active = True + # We rewrite the file ourselves before asking for a re-parse, so a schedule + # of Dispatcharr's own would only re-read a file that had not changed. + source.refresh_interval = 0 + + properties = dict(source.custom_properties or {}) + properties[MARKER_KEY] = {"slug": cfg.slug, "portal": cfg.url} + source.custom_properties = properties + + source.save() + return source, created + + +def deactivate_epg_source(cfg: PortalConfig) -> bool: + """Switch off the guide for a portal that no longer asks for one. + + Not deleted: the plugin does not remove things a user can see and may have + configured around -- the same restraint that leaves an M3U account standing + when its portal line goes away. Deactivating is enough to stop a guide that + is no longer refreshed from binding itself to channels. + """ + source = distalker_epg_sources().filter( + **{f"custom_properties__{MARKER_KEY}__slug": cfg.slug} + ).first() + if source is None or not source.is_active: + return False + + source.is_active = False + source.save(update_fields=["is_active"]) + return True + + +def request_reparse(account_id: int, refresh_hours: int) -> str: + """Ask Dispatcharr to read the playlist just written. Returns which task. + + **Exactly one task, never both.** ``refresh_single_m3u_account`` refreshes + the groups itself, so dispatching ``refresh_m3u_groups`` alongside it puts + the pair in a race for the same per-account lock -- and the loser reports + "Failed to refresh M3U groups" at the user, leaving the account in Pending + Setup. Which is worth stating in a function of its own, because the two + calls read as complementary and are not. + + Off a schedule, only the groups are refreshed: that has been this plugin's + behaviour throughout, and importing the streams is a step the user + completes by choosing groups. + + On a schedule, the whole account is re-read instead. Dispatcharr looked at + the *previous* playlist moments ago -- that refresh is what woke us to + write this one -- so without this the channel list would sit one cycle + behind the portal for ever. It emits ``m3u_refresh`` again, which is the + event that brought us here; the cooldown claimed before the sync started is + what keeps that from going round for ever. + """ + from apps.m3u.tasks import refresh_m3u_groups, refresh_single_m3u_account + + if refresh_hours: + # Before dispatching, not after: the task about to run emits the event + # the schedule listens for, and a sync the user started by hand claims + # no cooldown of its own -- so without this its echo comes back as a + # full re-fetch of every portal. + hold_auto_sync(AUTO_SYNC_COOLDOWN) + refresh_single_m3u_account.delay(account_id) + return "refresh_single_m3u_account" + + refresh_m3u_groups.delay(account_id) + return "refresh_m3u_groups" + + +# How long to wait before re-asking for the playlist of the account whose own +# refresh woke us. It only has to outlast the tail of that task once the sync +# has returned, which is a fraction of a second; a minute is margin, and being +# late costs nothing on a run nobody is watching. +TRIGGER_REPARSE_DELAY = 60 + + +def request_reparse_later(account_name: str, delay: int = TRIGGER_REPARSE_DELAY) -> bool: + """Re-ask for the playlist of the account whose refresh started this sync. + + That one account cannot be re-read in line with the others. A scheduled + sync runs *inside* Dispatcharr's ``refresh_single_m3u_account`` for it, and + that task holds the per-account lock for as long as we do -- so the request + made at the end of its portal is refused, leaving 'Lock for + refresh_single_m3u_account and id=N already acquired' as the only trace. + + Left alone, that portal's playlist is downloaded every cycle and read one + cycle late, for ever: an hour behind on an hourly schedule, a day behind on + a daily one. Which portal draws the short straw changes from cycle to + cycle, since the accounts race to be the one that wakes us. + + So it is asked for again once we have returned and the lock has gone. + Refused a second time would be no worse than not asking, which is why a + delay is enough and no coordination is needed. + """ + from apps.m3u.tasks import refresh_single_m3u_account + + account = distalker_accounts().filter(name=account_name).first() + if account is None: + return False + + # Same reason as request_reparse: the task about to run emits the event the + # schedule listens for. + hold_auto_sync(AUTO_SYNC_COOLDOWN) + refresh_single_m3u_account.apply_async((account.id,), countdown=max(1, int(delay))) + return True + + +def refresh_epg_source(source_id: int) -> None: + """Ask Dispatcharr to read the guide we just wrote. + + Its own task, dispatched by name from a registered module rather than + defined here -- a task a plugin defines cannot be consumed at all, which is + the whole story recorded in tasks.py. + """ + from apps.epg.tasks import refresh_epg_data + + refresh_epg_data.delay(source_id) + + +def apply_refresh_interval(refresh_hours: int, logger) -> int: + """Put the schedule on every account this plugin owns, without fetching. + + ``upsert_account`` writes the interval too, but only for portals a sync + actually fetched -- and changing the schedule changes nothing about any + portal, so the plan calls them all unchanged and fetches none of them. The + setting would appear to do nothing until the next unrelated re-fetch. + + So it is applied here instead, on every sync, for every account: a handful + of small updates and no network at all. + """ + wanted = max(0, int(refresh_hours or 0)) + changed = 0 + try: + for account in distalker_accounts(): + if account.refresh_interval != wanted: + account.refresh_interval = wanted + account.save(update_fields=["refresh_interval"]) + changed += 1 + except Exception: + logger.debug("distalker: could not apply the refresh schedule", exc_info=True) + return 0 + + if changed: + logger.info( + "distalker: %s on %d M3U account(s)", + f"scheduled refresh set to every {wanted}h" if wanted + else "scheduled refresh switched off", + changed, + ) + return changed + + def announce_new_account(account_id: int) -> None: """Tell the open UI that an M3U account it has never heard of now exists. @@ -407,9 +772,26 @@ def apply_stream_profile() -> Dict[str, int]: # --------------------------------------------------------------------------- -def sync_portal(cfg: PortalConfig, logger, trigger_refresh: bool = True) -> Dict[str, Any]: +# Three attempts at anything a sync asks the portal for. A provider that +# hiccups once should not cost the user their whole line-up until the next +# scheduled run, and a sync has the time -- nothing is waiting on it. +# +# Worst case per call is (retries + 1) x timeout plus the backoff, so 187s at +# the default 60s timeout. Portals are synced one after another under a lock +# that expires after 1800s (see claim_sync_lock): raising either number far +# enough that a run could outlive its own lock would let a second run start on +# top of the first. +SYNC_RETRIES = 2 + + +def sync_portal( + cfg: PortalConfig, + logger, + trigger_refresh: bool = True, + refresh_hours: int = 0, +) -> Dict[str, Any]: """Full sync for one portal: log in, fetch, write M3U, refresh account.""" - portal = Portal(cfg) + portal = Portal(cfg, retries=SYNC_RETRIES) portal.login() for warning in portal.warnings: logger.warning("distalker: %s: %s", cfg.name, warning) @@ -421,7 +803,9 @@ def sync_portal(cfg: PortalConfig, logger, trigger_refresh: bool = True) -> Dict if snapshot["blocked"]: logger.warning("distalker: portal '%s' reports the account as blocked", cfg.name) - channels = portal.get_all_channels() + channels = portal.list_channels( + progress=lambda note: logger.info("distalker: %s: %s", cfg.name, note) + ) genres = portal.get_genres() # The resolver reads this at tune time; publish it before the M3U lands so @@ -429,13 +813,11 @@ def sync_portal(cfg: PortalConfig, logger, trigger_refresh: bool = True) -> Dict save_portal(cfg) path = write_m3u(cfg.slug, build_m3u(portal, channels, genres)) - account, account_created = upsert_account(cfg, path) + account, account_created = upsert_account(cfg, path, refresh_hours) record_expiry(account, snapshot["expires"]) if trigger_refresh: - from apps.m3u.tasks import refresh_m3u_groups - - refresh_m3u_groups.delay(account.id) + request_reparse(account.id, refresh_hours) logger.info( "distalker: synced '%s' -- %d channels in %d groups -> %s", @@ -445,6 +827,20 @@ def sync_portal(cfg: PortalConfig, logger, trigger_refresh: bool = True) -> Dict path, ) + rewritten = sum(1 for channel in channels if channel.cmd_rewritten) + if rewritten: + # Worth saying: this changes what the portal is handed back at tune + # time, and it is the difference between a channel that plays and one + # that does not on the providers concerned. + logger.info( + "distalker: %s: %d channel(s) answered with a resolved link rather " + "than a marker, and were rewritten to one -- see canonical_cmd", + cfg.name, + rewritten, + ) + + epg = sync_epg(cfg, portal, channels, logger, trigger_refresh=trigger_refresh) + return { "portal": cfg.name, "slug": cfg.slug, @@ -455,17 +851,124 @@ def sync_portal(cfg: PortalConfig, logger, trigger_refresh: bool = True) -> Dict "file": path, "expires": snapshot["expires"], "blocked": snapshot["blocked"], + "epg": epg, } -def sync_all(portals: List[PortalConfig], logger, trigger_refresh: bool = True) -> Dict[str, Any]: +def sync_epg( + cfg: PortalConfig, + portal: Portal, + channels: List[ChannelEntry], + logger, + trigger_refresh: bool = True, +) -> Optional[Dict[str, Any]]: + """Fetch the guide and hand it to Dispatcharr. Returns None when off. + + Runs last, and cannot fail the sync around it. That is the whole design of + this function: the guide is an extra, it is by a wide margin the largest + thing fetched here, and a portal has many more ways to disappoint over + 100 MB than over a channel list. A sync that ends with a working line-up + and no guide is a good outcome; one that loses the line-up because the + guide was too big is not. + """ + if not cfg.epg: + # Inside its own guard for the same reason as everything else here: a + # portal that never wanted a guide must not fail its sync over one. + try: + if deactivate_epg_source(cfg): + logger.info( + "distalker: '%s' no longer asks for a guide; its EPG source " + "is switched off (not deleted)", + cfg.name, + ) + except Exception: + logger.debug("distalker: could not check for a stale guide", exc_info=True) + return None + + try: + logger.info( + "distalker: %s: fetching %d hours of guide for %d channels", + cfg.name, + cfg.epg_hours, + len(channels), + ) + epg_data = portal.get_epg_info(cfg.epg_hours, scratch_dir=_scratch_dir()) + if not epg_data: + # Said plainly, because it is a property of the provider rather + # than a fault to chase: a portal can carry thousands of channels + # and no programmes for any of them. Leaving 'epg=1' on costs one + # wasted request per sync and nothing else. + logger.warning( + "distalker: portal '%s' has no programme guide -- it answered " + "with an empty one. Remove 'epg=1' from its line to stop " + "asking.", + cfg.name, + ) + return None + + # build_xmltv empties epg_data as it writes, so nothing is counted + # afterwards -- count now, while it is still there to count. + covered = sum(1 for c in channels if epg_data.get(c.channel_id)) + + epg_path = write_xmltv(cfg.slug, build_xmltv(portal, channels, epg_data)) + source, source_created = upsert_epg_source(cfg, epg_path) + + if trigger_refresh: + refresh_epg_source(source.id) + + logger.info( + "distalker: guide for '%s' -- %d channels covered -> %s", + cfg.name, + covered, + epg_path, + ) + return { + "channels": covered, + "hours": cfg.epg_hours, + "file": epg_path, + "source_id": source.id, + "source_created": source_created, + } + except Exception as exc: + logger.warning( + "distalker: could not build the guide for '%s': %s", cfg.name, exc + ) + logger.debug("distalker: guide failure detail", exc_info=True) + return None + + +def _scratch_dir() -> Optional[str]: + """Where to stream a guide while it downloads. + + Beside the finished file rather than in the system temp: the guide can be + hundreds of megabytes, and a container's /tmp is often a small tmpfs -- in + memory, which is precisely what streaming to a file is meant to avoid. + """ + try: + os.makedirs(XMLTV_DIR, exist_ok=True) + return XMLTV_DIR + except OSError: + return None + + +def sync_all( + portals: List[PortalConfig], + logger, + trigger_refresh: bool = True, + refresh_hours: int = 0, +) -> Dict[str, Any]: """Sync every configured portal, surviving individual failures.""" results: List[Dict[str, Any]] = [] errors: List[str] = [] for cfg in portals: try: - results.append(sync_portal(cfg, logger, trigger_refresh=trigger_refresh)) + results.append( + sync_portal( + cfg, logger, trigger_refresh=trigger_refresh, + refresh_hours=refresh_hours, + ) + ) except PortalError as exc: errors.append(f"{cfg.name}: {exc}") logger.error("distalker: sync failed for '%s': %s", cfg.name, exc) @@ -485,6 +988,12 @@ def test_portal(cfg: PortalConfig) -> Dict[str, Any]: and Dispatcharr comes back as a 504. Genres are a short list and prove the same thing -- that the MAC authenticates and the session works. The channel count comes from a sync, which no longer blocks a request. + + Retries are off here for the same reason, and deliberately not shared with + :data:`SYNC_RETRIES`: three attempts at a 60-second timeout is three minutes + of a request thread, and the proxy in front of Dispatcharr gives up long + before that. A portal that needs a second attempt to answer is a portal + this action should report as unwell, not one it should wait out. """ portal = Portal(cfg) portal.login() @@ -492,11 +1001,10 @@ def test_portal(cfg: PortalConfig) -> Dict[str, Any]: return { "portal": cfg.name, "url": cfg.url, - "auth": ( - "credentials" - if (cfg.username and cfg.password) - else ("device-id" if cfg.device_id_auth else "handshake only") - ), + # What the portal actually asked for, not what was configured: a line + # carrying credentials against a portal that never requests them is + # reported as the handshake-only portal it is. + "auth": portal.auth_method, "groups": len(genres), "warnings": portal.warnings, } diff --git a/tasks.py b/tasks.py index 6baee41..3903d9a 100644 --- a/tasks.py +++ b/tasks.py @@ -132,6 +132,58 @@ def _run(): return True +def run_sync_here(full: bool = False, logger=None): + """The same sync, run in the caller rather than handed to a thread. + + For the scheduled path, and only that one. A thread is right when the + caller is a uWSGI request that has to answer now, and wrong when it is a + Celery task: there is no gevent hub there, the thread is a real daemon + thread, and the worker process it lives in is reaped when the pool scales + down. Observed exactly that -- the sync announced itself, produced nothing, + and forty seconds later the pool shrank by five. + + A Celery task is where long work belongs anyway. The one hosting the event + is ``refresh_single_m3u_account``, which allows an hour before its soft + limit; a full sync of a dozen portals is minutes. + + The caller's logger is passed on rather than left to the sync to invent, + because the one it invents does not print from a Celery worker -- see + Plugin.run_sync_now. + + Returns the sync's own result, or None when another sync already holds the + lock -- the same two guards as the threaded version, for the same reasons. + """ + from uuid import uuid4 + + from .stalker_api import claim_sync_lock, release_sync_lock + + global _sync_running + + with _SYNC_LOCK: + if _sync_running: + return None + _sync_running = True + + token = uuid4().hex + claimed = claim_sync_lock(token) + if claimed is False: + with _SYNC_LOCK: + _sync_running = False + return None + + try: + from .plugin import Plugin + + # No close_old_connections here, unlike the threaded version: this runs + # on the caller's connection, and Celery closes its own after a task. + return Plugin().run_sync_now(full=full, logger=logger) + finally: + if claimed: + release_sync_lock(token) + with _SYNC_LOCK: + _sync_running = False + + def remove_schedule() -> None: """Drop the periodic task versions before 0.9.4 created. diff --git a/tests/test_auth.py b/tests/test_auth.py new file mode 100644 index 0000000..1c41b3a --- /dev/null +++ b/tests/test_auth.py @@ -0,0 +1,326 @@ +"""Which authentication a portal gets, and who decides. + +The portal decides. ``get_profile`` answers with a ``status`` that says whether +the session is good (0), needs credentials presented first (2), or is refused +(anything else) -- and the refusal carries the provider's own wording, which is +the only part of it worth showing a user. + +What these tests mostly pin down is the tolerance around that machine, because +that is where a strict reading breaks real installs: most portals this plugin +meets are not Ministra, and they answer with less than Ministra would. +""" +import os +import sys + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO) + +import stalker_api as s # noqa: E402 + + +def portal(replies, **cfg_kwargs): + """A Portal whose HTTP layer is a script of canned answers. + + ``replies`` maps an action to a payload, or to a callable taking the query + string, or to an exception instance to raise. Every request made is + recorded on ``portal.queries``. + """ + cfg = s.PortalConfig( + slug="t", name="T", url="http://p.example/c/portal.php", + mac="00:1A:79:AA:BB:CC", **cfg_kwargs + ) + p = s.Portal(cfg) + p.queries = [] + + def fake_get_json(query, with_auth=True): + p.queries.append(query) + action = "" + for field in query.split("&"): + if field.startswith("action="): + action = field.split("=", 1)[1] + reply = replies.get(action, {"js": {}}) + if isinstance(reply, Exception): + raise reply + if callable(reply): + return reply(query) + return reply + + p._get_json = fake_get_json + p.authenticate = lambda: p.queries.append("POST action=do_auth") + return p + + +HANDSHAKE = {"js": {"token": "TOK", "not_valid": 1}} + + +def test_status_zero_is_authenticated(): + p = portal({"handshake": HANDSHAKE, "get_profile": {"js": {"status": 0, "id": 7}}}) + assert p.login() == "TOK" + assert p.auth_method == "profile" + assert not p.warnings + + +def test_a_profile_without_a_status_is_taken_as_fine(): + """The clone case, and the one a strict reading would break. + + pvr.stalker treats a missing status as a failure. Portals that answer with + a bare profile -- no status anywhere -- are common, they worked before this + machine existed, and they must keep working. + """ + p = portal({"handshake": HANDSHAKE, "get_profile": {"js": {"id": 42, "fname": "x"}}}) + assert p.login() == "TOK" + assert not p.warnings + + +def test_status_two_runs_do_auth_then_the_second_step(): + seen = [] + + def profile(query): + second = "auth_second_step=1" in query + seen.append(second) + return {"js": {"status": 0 if second else 2}} + + p = portal( + {"handshake": HANDSHAKE, "get_profile": profile}, + username="joe", password="pw", + ) + p.login() + assert seen == [False, True], seen + assert "POST action=do_auth" in p.queries + assert p.auth_method == "credentials" + + +def test_status_two_without_credentials_says_what_to_add(): + """The case the old guess could not see at all. + + Without credentials on the line the previous version ran a device-ID step, + got a shrug, and carried on to fetch an empty channel list. The user was + then told to check their MAC address, which was not the problem. + """ + p = portal({"handshake": HANDSHAKE, "get_profile": {"js": {"status": 2}}}) + try: + p.login() + except s.PortalAuthError as exc: + assert "username" in str(exc) and "password" in str(exc), exc + else: + raise AssertionError("a portal asking for credentials must not be ignored") + + +def test_the_portal_gets_the_last_word_on_why(): + p = portal({ + "handshake": HANDSHAKE, + "get_profile": {"js": {"status": 1, "msg": "generic", + "block_msg": "Subscription expired on 12/06"}}, + }) + try: + p.login() + except s.PortalAuthError as exc: + # block_msg beats msg: it is the specific one, written for this case. + assert str(exc) == "Subscription expired on 12/06", exc + else: + raise AssertionError("status 1 must refuse the session") + + +def test_a_second_step_that_still_fails_is_refused(): + p = portal( + {"handshake": HANDSHAKE, + "get_profile": {"js": {"status": 2, "msg": "bad credentials"}}}, + username="joe", password="wrong", + ) + try: + p.login() + except s.PortalAuthError as exc: + assert "bad credentials" in str(exc), exc + else: + raise AssertionError("credentials the portal keeps rejecting must raise") + + +def test_a_portal_with_no_get_profile_still_logs_in(): + """MAC-only portals that never implemented it, warned about but served.""" + p = portal({ + "handshake": HANDSHAKE, + "get_profile": s.PortalError("portal returned HTTP 404"), + }) + assert p.login() == "TOK" + assert p.auth_method == "handshake only" + assert any("404" in w for w in p.warnings), p.warnings + + +def test_an_explicit_refusal_is_never_downgraded_to_a_warning(): + """The difference between 'did not answer' and 'answered no'.""" + p = portal({ + "handshake": HANDSHAKE, + "get_profile": s.PortalAuthError("portal refused the session (HTTP 403)"), + }) + try: + p.login() + except s.PortalAuthError: + pass + else: + raise AssertionError("a refusal must not be swallowed as a warning") + + +def test_not_valid_travels_back_as_not_valid_token(): + p = portal({"handshake": HANDSHAKE, "get_profile": {"js": {"status": 0}}}) + p.login() + profile_query = [q for q in p.queries if "action=get_profile" in q][0] + assert "not_valid_token=1" in profile_query, profile_query + + p = portal({"handshake": {"js": {"token": "TOK", "not_valid": 0}}, + "get_profile": {"js": {"status": 0}}}) + p.login() + profile_query = [q for q in p.queries if "action=get_profile" in q][0] + assert "not_valid_token=0" in profile_query, profile_query + + +def test_the_whole_stb_identity_is_sent(): + """Every field libstalkerclient sends, signature included. + + signature was a documented setting that no request ever carried, so a user + who set it was configuring nothing. + """ + p = portal({"handshake": HANDSHAKE, "get_profile": {"js": {"status": 0}}}, + signature="a" * 64, serial_number="SN1", model="MAG322") + p.login() + query = [q for q in p.queries if "action=get_profile" in q][0] + for expected in ("signature=" + "a" * 64, "sn=SN1", "stb_type=MAG322", + "num_banks=1", "image_version=216", "hd=1", "ver=", "hw_version="): + assert expected in query, f"{expected} missing from {query}" + + +def test_a_dead_session_in_plain_text_is_an_auth_error(): + """Ministra answers 200 with prose, not JSON, once a token has expired. + + Typed, because the resolver's cached-token path re-authenticates on it, + and because the retry work still to come must not retry it. + """ + import json as _json + + class FakeResponse: + status_code = 200 + text = "Authorization failed." + + def json(self): + raise _json.JSONDecodeError("no", "Authorization failed.", 0) + + cfg = s.PortalConfig(slug="t", name="T", url="http://p.example/c/portal.php", + mac="00:1A:79:AA:BB:CC") + p = s.Portal(cfg) + p.session.request = lambda *a, **k: FakeResponse() + try: + p._get_json("action=get_all_channels") + except s.PortalAuthError as exc: + assert "no longer authorised" in str(exc), exc + else: + raise AssertionError("'Authorization failed.' must be typed as an auth error") + + +def test_create_link_expiry_is_typed_so_the_resolver_can_recover(): + """A dead token is usually a hollow success, not a refusal. + + The resolver's optimistic path re-authenticates on PortalAuthError alone, + so these two have to carry that type or the cached-token path could never + recover from the one thing it exists to survive. + """ + p = portal({}) + for payload in ({"js": False}, {"js": {"cmd": ""}}, {"js": {"cmd": " "}}): + p._get_json = lambda q, with_auth=True, _p=payload: _p + try: + p.create_link("ffmpeg http://x/1") + except s.PortalAuthError: + pass + else: + raise AssertionError(f"{payload} must be an auth error") + + +def test_a_reply_that_is_simply_not_a_link_is_not_an_auth_error(): + """Re-authenticating cannot turn prose into a URL, so it must not try.""" + p = portal({}) + p._get_json = lambda q, with_auth=True: {"js": {"cmd": "no link here"}} + try: + p.create_link("x") + except s.PortalAuthError: + raise AssertionError("an unusable command must not trigger a re-login") + except s.PortalError: + pass + + +def test_the_other_endpoint_is_tried_when_this_one_is_not_an_api(): + """Ministra answers on two paths and installs expose different ones.""" + tried = [] + + def handshake(query): + tried.append(len(tried)) + if len(tried) == 1: + raise s.PortalEndpointError("portal returned HTTP 404") + return HANDSHAKE + + p = portal({"handshake": handshake, "get_profile": {"js": {"status": 0}}}) + assert p.login() == "TOK" + assert p.url == "http://p.example/server/load.php", p.url + # The configured URL is untouched, because logos are resolved against it. + assert p.cfg.url == "http://p.example/c/portal.php" + assert any("does at" in w for w in p.warnings), p.warnings + + +def test_a_portal_that_is_simply_down_is_not_asked_twice(): + """Both paths live on one host; a second try buys nothing but delay.""" + p = portal({"handshake": s.PortalError("request to portal failed: refused")}) + try: + p.login() + except s.PortalError as exc: + assert "refused" in str(exc), exc + assert len([q for q in p.queries if "handshake" in q]) == 1, p.queries + + +def test_when_neither_endpoint_answers_the_configured_one_is_blamed(): + p = portal({"handshake": s.PortalEndpointError("portal returned HTTP 404")}) + try: + p.login() + except s.PortalError as exc: + assert "404" in str(exc), exc + # Reset, so nothing downstream reports a path the user never wrote. + assert p.url == p.cfg.url, p.url + + +def test_the_two_endpoints_map_onto_each_other(): + cases = { + "http://h/c/portal.php": "http://h/server/load.php", + "http://h/stalker_portal/c/portal.php": "http://h/stalker_portal/server/load.php", + "http://h/server/load.php": "http://h/c/portal.php", + "http://h:8080/c/portal.php": "http://h:8080/server/load.php", + # Nothing sensible to swap to. + "http://h/something.cgi": "", + } + for given, expected in cases.items(): + assert s.alternate_endpoint(given) == expected, given + + +def test_there_is_no_watchdog(): + """Removed rather than left dead: nothing here can call one. + + Keeping a Stalker session warm needs something alive between requests. The + sync is a task that ends and the resolver becomes ffmpeg, so a ping method + sat uncalled for two releases, reading as a feature that existed. + """ + assert not hasattr(s.Portal, "watchdog") + + +def test_an_auth_error_is_still_a_portal_error(): + """Callers that only catch PortalError must not start leaking exceptions.""" + assert issubclass(s.PortalAuthError, s.PortalError) + + +if __name__ == "__main__": + failures = 0 + for name, fn in sorted(globals().items()): + if not name.startswith("test_") or not callable(fn): + continue + try: + fn() + print(f"PASS {name}") + except Exception as exc: + failures += 1 + print(f"FAIL {name}: {type(exc).__name__}: {exc}") + print("\n" + ("ALL AUTH TESTS PASSED" if not failures else f"{failures} FAILURE(S)")) + sys.exit(1 if failures else 0) diff --git a/tests/test_config.py b/tests/test_config.py index 85de62c..11f625b 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -15,10 +15,6 @@ def test_builtin_defaults(): assert portal.timezone == s.DEFAULT_TIMEZONE assert portal.device_id == s.DEFAULT_DEVICE_ID assert portal.signature == s.DEFAULT_SIGNATURE - # No credentials selects device-ID auth even with the default ffff... IDs, - # matching stalkerhek's `deviceIdAuth := Username == "" && Password == ""`. - # This is the plain URL+MAC case, so it must not regress. - assert portal.device_id_auth is True def test_stb_identity_is_per_portal(): @@ -44,13 +40,12 @@ def test_stb_identity_is_per_portal(): assert plain.device_id == s.DEFAULT_DEVICE_ID -def test_credentials_beat_device_id_auth(): +def test_credentials_are_read_off_the_line(): (portal,), _ = s.parse_portals( "U | http://c.example/c/ | 00:1A:79:AA:BB:02 | username=joe password=pw device_id=" + "A" * 64 ) assert portal.username == "joe" and portal.password == "pw" - assert portal.device_id_auth is False def test_extras_split_by_space_and_pipe_and_quotes(): @@ -116,15 +111,18 @@ def test_url_only_and_mac_only_config(): assert not errors assert portal.url == "http://somedomain.com:8080/c/portal.php" assert portal.mac == "00:1A:79:AA:BB:CC" - assert portal.device_id_auth is True assert portal.username == "" and portal.password == "" assert portal.max_streams == 1 -def test_partial_credentials_fall_back_to_device_auth(): - """A username with no password is not usable credentials.""" +def test_partial_credentials_are_kept_as_written(): + """A username with no password is not usable credentials. + + Kept rather than rejected: login() only reaches for them when the portal + asks (profile status 2), and it is the one that decides they are missing. + """ (portal,), _ = s.parse_portals("A | http://a.example/c/ | 00:1A:79:AA:BB:01 | username=joe") - assert portal.device_id_auth is True + assert portal.username == "joe" and portal.password == "" def test_pseudo_url_roundtrip(): diff --git a/tests/test_epg.py b/tests/test_epg.py new file mode 100644 index 0000000..a53865c --- /dev/null +++ b/tests/test_epg.py @@ -0,0 +1,456 @@ +"""Turning a portal's guide into something Dispatcharr will read. + +The whole feature rests on one agreement: the identifier written into the +playlist and the identifier written into the guide have to be the same string, +because that is the only thing joining a channel to its programmes. Nothing +warns when they drift -- the guide simply matches nothing -- so the first test +here is the one that pins it. + +The rest is the parsing that has to survive a portal sending rubbish, and the +promise that a guide which goes wrong never takes the channel list with it. +""" +import importlib.util +import os +import sys +import types + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO) + +import stalker_api as s # noqa: E402 + + +def load_sync(): + """sync.py as Dispatcharr's loader sees it: a namespace package.""" + pkg = types.ModuleType("distalker_epg") + pkg.__path__ = [REPO] + sys.modules["distalker_epg"] = pkg + spec = importlib.util.spec_from_file_location( + "distalker_epg.sync", os.path.join(REPO, "sync.py") + ) + module = importlib.util.module_from_spec(spec) + sys.modules["distalker_epg.sync"] = module + spec.loader.exec_module(module) + return module + + +sync = load_sync() + + +def portal(**cfg_kwargs): + cfg = s.PortalConfig( + slug="mock", name="Mock", url="http://p.example/c/portal.php", + mac="00:1A:79:AA:BB:CC", **cfg_kwargs + ) + return s.Portal(cfg) + + +def channel(cid="101", name="One", logo=""): + return s.ChannelEntry( + channel_id=cid, name=name, cmd=f"ffmpeg http://x/{cid}", logo=logo, + genre_id="1", number="1", + ) + + +def programme(start=1785276000, stop=1785279600, name="Show", descr="Plot"): + return {"id": "9", "name": name, "descr": descr, + "start_timestamp": start, "stop_timestamp": stop} + + +def xmltv(p, channels, epg_data): + return "".join(sync.build_xmltv(p, channels, epg_data)) + + +# -- the invariant -------------------------------------------------------- + + +def test_the_playlist_and_the_guide_agree_on_the_identifier(): + """The one thing that must never drift. + + tvg-id in the M3U, channel id in the XMLTV, and the 'channel' attribute on + every programme: three places, one string. If any of them changes shape the + guide stops matching and nothing says so. + """ + p = portal() + channels = [channel("101"), channel("102", "Two")] + data = {"101": [programme()], "102": [programme()]} + + playlist = sync.build_m3u(p, channels, {"1": "News"}) + guide = xmltv(p, channels, data) + + for cid in ("101", "102"): + expected = f"mock.{cid}" + assert f'tvg-id="{expected}"' in playlist, expected + assert f'' in guide, expected + assert f'channel="{expected}"' in guide, expected + + +def test_a_channel_with_no_id_is_in_neither(): + """No identifier, nothing to join on, so it is not offered a guide.""" + p = portal() + channels = [channel("")] + guide = xmltv(p, channels, {"": [programme()]}) + assert "') + assert "One" in guide + assert 'src="http://p.example/c/misc/logos/320/a.png"' in guide + assert "Show" in guide + assert "Plot" in guide + assert guide.rstrip().endswith("") + + +# -- portals sending rubbish ---------------------------------------------- + + +def test_channels_without_programmes_are_left_out(): + """An empty becomes a permanent empty row in the EPG picker.""" + p = portal() + guide = xmltv(p, [channel("101"), channel("102", "Two")], {"101": [programme()]}) + assert "mock.101" in guide + assert "mock.102" not in guide, guide + + +def test_unusable_programmes_are_dropped_not_written(): + p = portal() + bad = [ + programme(start=0), + programme(stop=0), + programme(start=1785279600, stop=1785276000), # ends before it starts + programme(start=1785276000, stop=1785276000), # no duration + programme(name=""), + "not a dict", + None, + ] + guide = xmltv(p, [channel()], {"101": bad}) + assert "Show" in guide + assert "" not in guide + + +def test_markup_in_a_title_cannot_break_the_document(): + p = portal() + guide = xmltv( + p, [channel(name='A & B <"x">')], + {"101": [programme(name="Tom & Jerry ", descr="1 < 2")]}, + ) + assert "&" in guide and "<" in guide + assert "" in guide.split("")[1][:60] + # Proof rather than inspection: it has to actually parse. + import xml.etree.ElementTree as ET + + root = ET.fromstring(guide) + assert root.find("programme/title").text == "Tom & Jerry " + assert root.find("channel/display-name").text == 'A & B <"x">' + + +def test_control_characters_are_removed_rather_than_escaped(): + """XML 1.0 cannot carry them at all, and portals do send them.""" + import xml.etree.ElementTree as ET + + p = portal() + guide = xmltv(p, [channel()], {"101": [programme(descr="be\x03fore\x00after")]}) + assert ET.fromstring(guide).find("programme/desc").text == "beforeafter" + + +# -- overlapping entries --------------------------------------------------- + + +def test_the_same_show_listed_twice_is_written_once(): + """What a guide corrected in place looks like from outside. + + Observed on a real portal: one show, one end time, two start times ten + minutes apart. Written as-is, Dispatcharr has two candidates spanning the + same minute and picks one arbitrarily for "what is on now". + """ + p = portal() + both = [ + programme(start=1785275700, stop=1785284400), + programme(start=1785276300, stop=1785284400), + ] + guide = xmltv(p, [channel()], {"101": both}) + assert guide.count("go away") + try: + p.get_epg_info(24) + except s.PortalError as exc: + assert "not JSON" in str(exc), exc + else: + raise AssertionError("prose is not a guide") + + +def test_an_oversized_guide_is_refused_before_it_is_parsed(): + """The cap exists so the worker is not killed finding the limit.""" + original = s.EPG_MAX_BYTES + s.EPG_MAX_BYTES = 1024 + try: + p = fetching(b"x" * 4096) + try: + p.get_epg_info(24) + except s.PortalError as exc: + assert "larger than" in str(exc) and "epg_hours" in str(exc), exc + else: + raise AssertionError("an oversized guide must be refused") + finally: + s.EPG_MAX_BYTES = original + + +def test_a_refused_guide_is_an_auth_error(): + p = fetching(b"", status=403) + try: + p.get_epg_info(24) + except s.PortalAuthError: + pass + else: + raise AssertionError("403 must stay typed as a refusal") + + +# -- the line --------------------------------------------------------------- + + +def test_the_guide_is_off_unless_the_line_asks(): + (cfg,), errors = s.parse_portals("A | http://a.example/c/ | 00:1A:79:AA:BB:01") + assert not errors + assert cfg.epg is False + assert cfg.epg_hours == s.DEFAULT_EPG_HOURS + + +def test_the_ways_a_person_writes_yes(): + for value in ("1", "true", "yes", "on", "TRUE", "Yes"): + (cfg,), errors = s.parse_portals( + f"A | http://a.example/c/ | 00:1A:79:AA:BB:01 | epg={value}" + ) + assert not errors and cfg.epg is True, value + for value in ("0", "no", "false", "off", ""): + (cfg,), _ = s.parse_portals( + f"A | http://a.example/c/ | 00:1A:79:AA:BB:01 | epg={value}" + ) + assert cfg.epg is False, value + + +def test_the_period_is_read_and_checked(): + (cfg,), errors = s.parse_portals( + "A | http://a.example/c/ | 00:1A:79:AA:BB:01 | epg=1 epg_hours=48" + ) + assert not errors and cfg.epg_hours == 48 + + for bad in ("abc", "0", "-3"): + _, errors = s.parse_portals( + f"A | http://a.example/c/ | 00:1A:79:AA:BB:01 | epg=1 epg_hours={bad}" + ) + assert errors, bad + + +def test_a_written_line_round_trips(): + line = s.format_portal_line( + "A", "http://a.example/c/", "00:1A:79:AA:BB:01", epg=True, epg_hours=48 + ) + assert "epg=1" in line and "epg_hours=48" in line, line + (cfg,), errors = s.parse_portals(line) + assert not errors and cfg.epg is True and cfg.epg_hours == 48 + + # The default period says nothing the default does not already say. + line = s.format_portal_line( + "A", "http://a.example/c/", "00:1A:79:AA:BB:01", epg=True + ) + assert "epg=1" in line and "epg_hours" not in line, line + + line = s.format_portal_line("A", "http://a.example/c/", "00:1A:79:AA:BB:01") + assert "epg" not in line, line + + +def test_turning_the_guide_on_forces_a_fetch(): + """Otherwise the plan calls the portal unchanged and nothing happens. + + The same trap as every other setting that does not touch the line-up: the + plan compares against what was published, so a key it does not compare is + a key the user can set with no effect at all. + """ + spec = importlib.util.spec_from_file_location( + "distalker_epg.plugin", os.path.join(REPO, "plugin.py") + ) + module = importlib.util.module_from_spec(spec) + sys.modules["distalker_epg.plugin"] = module + spec.loader.exec_module(module) + + keys = module.Plugin.FETCH_KEYS + assert "epg" in keys and "epg_hours" in keys, keys + for key in module.Plugin.LINEUP_KEYS: + assert key in keys, key + + +if __name__ == "__main__": + failures = 0 + for name, fn in sorted(globals().items()): + if not name.startswith("test_") or not callable(fn): + continue + try: + fn() + print(f"PASS {name}") + except Exception as exc: + failures += 1 + print(f"FAIL {name}: {type(exc).__name__}: {exc}") + print("\n" + ("ALL EPG TESTS PASSED" if not failures else f"{failures} FAILURE(S)")) + sys.exit(1 if failures else 0) diff --git a/tests/test_fallback.py b/tests/test_fallback.py index 8190df3..e2db3ad 100644 --- a/tests/test_fallback.py +++ b/tests/test_fallback.py @@ -23,7 +23,7 @@ resolver = importlib.util.module_from_spec(spec) spec.loader.exec_module(resolver) -PORTAL_URL = s.encode_pseudo_url("weaseltv", "ffmpeg http://portal.example/live/1") +PORTAL_URL = s.encode_pseudo_url("livingroom", "ffmpeg http://portal.example/live/1") XTREAM_URL = "http://xtream.example:8080/live/user/pass/123.ts" @@ -45,7 +45,7 @@ def get(self, key): def test_our_own_urls_are_recognised(): assert s.is_pseudo_url(PORTAL_URL) slug, cmd = s.decode_pseudo_url(PORTAL_URL) - assert slug == "weaseltv" + assert slug == "livingroom" def test_another_providers_url_is_not_ours(): diff --git a/tests/test_listing.py b/tests/test_listing.py new file mode 100644 index 0000000..9e9926b --- /dev/null +++ b/tests/test_listing.py @@ -0,0 +1,534 @@ +"""Reading a portal's channel list. + +What arrives is a Ministra response only on the portals that are Ministra. +Everywhere else it is a rough approximation of one, and the parsing has to +survive rows that are missing fields, logos in shapes that are not filenames, +and -- once a portal declines to answer get_all_channels at all -- a listing +that has to be collected a page at a time. +""" +import os +import sys + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO) + +import stalker_api as s # noqa: E402 + + +def portal(**cfg_kwargs): + cfg = s.PortalConfig( + slug="t", name="T", url="http://p.example/c/portal.php", + mac="00:1A:79:AA:BB:CC", **cfg_kwargs + ) + return s.Portal(cfg) + + +def row(**overrides): + base = {"id": "1", "name": "One", "cmd": "ffmpeg http://x/1", "number": "1"} + base.update(overrides) + return base + + +# -- logos --------------------------------------------------------------- + + +def test_a_bare_filename_is_resolved_against_the_portal(): + assert portal().logo_url("alpha.png") == ( + "http://p.example/c/misc/logos/320/alpha.png" + ) + + +def test_an_absolute_logo_is_left_alone(): + for url in ("http://cdn.example/a.png", "https://cdn.example/a.png"): + assert portal().logo_url(url) == url + + +def test_a_logo_on_some_other_scheme_is_still_a_url(): + """It used to be treated as a filename and glued behind the logo path.""" + assert portal().logo_url("ftp://cdn.example/a.png") == "ftp://cdn.example/a.png" + + +def test_an_inline_image_is_dropped_rather_than_mangled(): + """Valid, and useless here: it lands in a URL field on Dispatcharr's side. + + The old test was that a logo starts with http, so this went through the + 'must be a filename' branch and produced .../misc/logos/320/data:image... + """ + assert portal().logo_url("data:image/png;base64,iVBORw0KGgo=") == "" + assert portal().logo_url("DATA:image/png;base64,iVBORw0KGgo=") == "" + + +def test_nothing_stays_nothing(): + assert portal().logo_url("") == "" + assert portal().logo_url(" ") == "" + + +# -- rows ---------------------------------------------------------------- + + +def test_a_row_becomes_a_channel(): + channel = s.Portal._channel_from_row( + row(logo="a.png", tv_genre_id="7") + ) + assert (channel.channel_id, channel.name, channel.number) == ("1", "One", "1") + assert (channel.logo, channel.genre_id) == ("a.png", "7") + + +def test_rows_that_are_not_channels_are_skipped(): + for bad in (None, [], "nope", row(cmd=""), row(name=""), row(name=" ")): + assert s.Portal._channel_from_row(bad) is None, bad + + +def test_catch_up_is_read_off_the_row(): + """Read, not published -- see ChannelEntry. This pins that it arrives.""" + channel = s.Portal._channel_from_row( + row(enable_tv_archive=1, tv_archive_duration=7) + ) + assert channel.tv_archive is True + assert channel.tv_archive_duration == "7" + + # Portals write the flag as a string about as often as as a number. + assert s.Portal._channel_from_row(row(enable_tv_archive="1")).tv_archive is True + for off in (0, "0", "", None): + assert s.Portal._channel_from_row(row(enable_tv_archive=off)).tv_archive is False + + +def test_a_channel_without_catch_up_says_so_quietly(): + channel = s.Portal._channel_from_row(row()) + assert channel.tv_archive is False and channel.tv_archive_duration == "" + + +def test_the_playlist_does_not_advertise_catch_up(): + """The badge would be a promise Dispatcharr cannot keep for a portal. + + Its catch-up player builds Xtream URLs from a server address and + credentials a Distalker source has none of, so a channel flagged here + would show the indicator and then fail to play back. + """ + import importlib.util + import types + + pkg = types.ModuleType("distalker_listing") + pkg.__path__ = [REPO] + sys.modules["distalker_listing"] = pkg + spec = importlib.util.spec_from_file_location( + "distalker_listing.sync", os.path.join(REPO, "sync.py") + ) + sync = importlib.util.module_from_spec(spec) + sys.modules["distalker_listing.sync"] = sync + spec.loader.exec_module(sync) + + p = portal() + channels = [s.Portal._channel_from_row(row(enable_tv_archive=1, + tv_archive_duration=7))] + m3u = sync.build_m3u(p, channels, {}) + assert "tv_archive" not in m3u, m3u + assert "catchup" not in m3u.lower(), m3u + + +# -- commands the portal cannot read back --------------------------------- + + +def test_a_marker_is_left_exactly_as_it_came(): + """Every portal but one answers this way, and their identities must not move. + + The stream hash is partly the URL, ours encodes this command: rewriting a + command that was already fine would invent a new stream for every channel + on every portal, once. + """ + for cmd in ( + "ffmpeg http://localhost/ch/553690_", + "http://localhost/ch/553690_", + "ffmpeg http://prov/ch/101", + "ffrt3 http://localhost/ch/42_", + ): + assert s.canonical_cmd(cmd, "553690") == cmd, cmd + + +def test_a_resolved_link_is_rebuilt_into_a_marker(): + """The portal cannot read its own resolved link back. + + Observed: handed one, it looks for the channel id inside, fails, and + splices a piece of the URL into the parameter -- + '...&stream=-portal.example:80/play/live.'. Another provider returns the + same parameter empty. Either way the link does not play. + """ + resolved = ( + "ffmpeg http://portal.example:80/play/live.php?mac=00:1A:79:AA:BB:CC" + "&stream=553690&extension=ts&play_token=aaaaaaaaaa" + ) + assert s.canonical_cmd(resolved, "553690") == ( + "ffmpeg http://localhost/ch/553690_" + ) + + +def test_the_rewrite_is_what_makes_the_identity_hold_still(): + """The token rotates on every request; the marker does not. + + One portal produced 647 duplicate streams an hour before this, each sync + inventing a new URL for the same channel and stranding the last one. + """ + def resolved(token): + return ( + f"ffmpeg http://host/play/live.php?stream=553690&play_token={token}" + ) + + first = s.canonical_cmd(resolved("aaaaaaaaaa"), "553690") + second = s.canonical_cmd(resolved("bbbbbbbbbb"), "553690") + assert first == second, (first, second) + + +def test_nothing_is_rebuilt_without_an_id_to_rebuild_it_from(): + resolved = "ffmpeg http://host/play/live.php?stream=553690&play_token=x" + assert s.canonical_cmd(resolved, "") == resolved + + +def test_a_command_carrying_no_url_is_left_to_the_portal(): + """VOD commands look like this, and guessing at them would be worse.""" + for cmd in ("auto /media/1234.mpg", "", " "): + assert s.canonical_cmd(cmd, "553690") == cmd, repr(cmd) + + +def test_the_rewrite_is_reported_on_the_channel(): + rewritten = s.Portal._channel_from_row( + row(cmd="ffmpeg http://host/play/live.php?stream=1&play_token=x") + ) + assert rewritten.cmd == "ffmpeg http://localhost/ch/1_" + assert rewritten.cmd_rewritten is True + + untouched = s.Portal._channel_from_row(row(cmd="ffmpeg http://localhost/ch/1_")) + assert untouched.cmd_rewritten is False + + +# -- the same command handed back at tune time ---------------------------- +# +# The rewrite above only fires on a row carrying an id. Without one -- and on +# every portal synced before it existed -- the resolved link is still what gets +# handed back, and this is what the portal does with it. + + +DOUBLED_BASE_CMD = "http://portal.example:80/USER/PASS/1225691" +DOUBLED_BASE_LINK = ( + "http://portal.example:80/USER/PASS/USER/PASS/1225691?play_token=deadbeef" +) + + +def test_a_base_glued_on_twice_is_unglued(): + """Reported on a portal whose commands are already complete URLs. + + Its create_link prepends its own base to whatever it is handed, so the + reply carries /USER/PASS/ twice and answers 401. The command answers 302 + and plays. + """ + assert s.undoubled_link(DOUBLED_BASE_CMD, DOUBLED_BASE_LINK) == DOUBLED_BASE_CMD + assert s.undoubled_link( + f"ffmpeg {DOUBLED_BASE_CMD}", DOUBLED_BASE_LINK + ) == DOUBLED_BASE_CMD + + +def test_a_portal_that_answers_properly_keeps_its_token(): + """What every working portal does, and the one case a mistake would break. + + The token is the whole point of resolving at tune time, so a rule that + dropped it on an ordinary answer would be worse than the bug it fixes. + """ + for cmd, link in ( + # The same path, with the token added. + (DOUBLED_BASE_CMD, DOUBLED_BASE_CMD + "?play_token=deadbeef"), + # A marker resolved to somewhere else entirely: the shape this plugin + # sees on every portal that behaves. + ("ffmpeg http://localhost/ch/1225691_", + "http://portal.example:80/USER/PASS/1225691?play_token=deadbeef"), + # Same path, another host -- a provider streaming from its edge. + ("http://portal.example/live/1", "http://edge.example/live/1?token=x"), + # A repetition that is not the base: '/a' answering '/a/a' is a + # different channel, not a doubled prefix. + ("http://h/a", "http://h/a/a?token=x"), + # Nothing to compare against. + ("auto /media/1234.mpg", "http://h/play?token=x"), + ("", "http://h/play?token=x"), + ): + assert s.undoubled_link(cmd, link) == link, (cmd, link) + + +def test_the_ungluing_happens_where_the_portal_answers(): + """create_link is where the doubled path is seen, and it says so. + + The resolver prints the portal's warnings on the tune's stderr, which is + the only place this is visible on a live install. + """ + p = portal() + p._get_json = lambda q, with_auth=True: {"js": {"cmd": f"ffmpeg {DOUBLED_BASE_LINK}"}} + + assert p.create_link(DOUBLED_BASE_CMD) == DOUBLED_BASE_CMD + assert any("already a link" in w for w in p.warnings), p.warnings + + +def test_an_answer_that_needed_nothing_is_not_remarked_on(): + p = portal() + p._get_json = lambda q, with_auth=True: { + "js": {"cmd": "ffmpeg http://portal.example/live/1?play_token=x"} + } + + assert p.create_link("ffmpeg http://localhost/ch/1_") == ( + "http://portal.example/live/1?play_token=x" + ) + assert p.warnings == [] + + +# -- naming the channel in the request as well ---------------------------- +# +# A portal that cannot find the channel in the command answers about no +# channel at all: one returns the link with its stream parameter left empty. +# Reported and first fixed by @shayward, on providers that need it every time. + + +def asking(cmd): + """The create_link query a portal would be sent for ``cmd``.""" + p = portal() + sent = [] + p._get_json = lambda q, with_auth=True: ( + sent.append(q) or {"js": {"cmd": "ffmpeg http://host/play?token=x"}} + ) + p.create_link(cmd) + return sent[0] + + +def test_the_channel_is_named_from_the_marker(): + """The only shape left after a sync, so reading the query alone would + leave exactly the installs that need this without it.""" + assert "&stream=553690&" in asking("ffmpeg http://localhost/ch/553690_") + + +def test_the_channel_is_named_from_an_unrewritten_command(): + """Every portal synced before canonical_cmd existed still carries this.""" + assert "&stream=553690&" in asking( + "ffmpeg http://host/play/live.php?mac=00:1A:79:AA:BB:CC&stream=553690" + ) + + +def test_a_command_naming_no_channel_adds_nothing_to_the_request(): + """The neutrality that makes this safe to send to everybody. + + An empty parameter is a question no portal asked to be asked, and this + runs against every provider, not the two that reported the problem. + """ + for cmd in ( + "ffmpeg http://host/play/live.php?token=x", + "http://portal.example:80/USER/PASS/1225691", + "auto /media/1234.mpg", + "", + ): + assert "stream=" not in asking(cmd), cmd + + +def test_a_quoted_command_does_not_smuggle_its_quote_into_the_query(): + """A quoted URL is a real shape -- extract_link exists because of it -- + and reading the query off the raw command carries the closing quote in.""" + query = asking('ffmpeg "http://host/play/live.php?stream=553690"') + assert "&stream=553690&" in query, query + + +def test_the_channel_number_is_escaped_like_the_command_beside_it(): + """Whatever the portal wrote, it goes back as one parameter and not two.""" + query = asking("ffmpeg http://host/play/live.php?stream=55%263690") + assert "&stream=55%263690&" in query, query + + +def test_reading_the_channel_out_of_a_command(): + for cmd, expected in ( + ("ffmpeg http://localhost/ch/553690_", "553690"), + ("http://localhost/ch/553690", "553690"), + ("ffmpeg http://host/play/live.php?stream=553690&extension=ts", "553690"), + # The marker wins: after a sync it is the only one there, and a + # command carrying both was resolved from that same number anyway. + ("ffmpeg http://localhost/ch/553690_?stream=1", "553690"), + ("ffmpeg http://host/play/live.php?stream=&extension=ts", ""), + ("ffmpeg http://host/play/live.php", ""), + ("auto /media/1234.mpg", ""), + ("", ""), + ): + assert s.stream_id(cmd) == expected, cmd + + +# -- paging -------------------------------------------------------------- + + +def scripted(all_channels, pages): + """A portal whose two listing calls answer from canned data. + + ``all_channels`` is the get_all_channels payload, or an exception to raise. + ``pages`` maps a page number to its 'js' object; a page not in it answers + empty, which is what a portal past its last page does. + """ + p = portal() + p.pages_asked = [] + + def fake_get_json(query, with_auth=True): + if "get_all_channels" in query: + if isinstance(all_channels, Exception): + raise all_channels + return all_channels + page = int(query.split("&p=")[1].split("&")[0]) + p.pages_asked.append(page) + return {"js": pages.get(page, {"data": []})} + + p._get_json = fake_get_json + return p + + +def page(ids, **extra): + js = {"data": [row(id=str(i), name=f"Ch {i}", cmd=f"ffmpeg http://x/{i}") + for i in ids]} + js.update(extra) + return js + + +REFUSED = s.PortalError("portal returned an empty channel list (check the MAC address)") + + +def test_a_portal_that_answers_in_one_request_is_never_paged(): + p = scripted({"js": {"data": [row()]}}, {}) + assert len(p.list_channels()) == 1 + assert p.pages_asked == [], "paging must stay the expensive last resort" + + +def test_paging_takes_over_when_the_single_request_will_not(): + p = scripted(REFUSED, {1: page([1, 2]), 2: page([3])}) + channels = p.list_channels() + assert [c.channel_id for c in channels] == ["1", "2", "3"] + assert p.pages_asked == [1, 2, 3], p.pages_asked + + +def test_the_reported_page_count_bounds_the_walk(): + """The guard open-tv lacks: the portal said how much there was. + + Its last page is full, so 'stop on an empty page' would ask for one more; + these pages never repeat, so 'stop on a repeat' would never fire either. + """ + pages = {1: page([1, 2], total_items=4, max_page_items=2), 2: page([3, 4])} + p = scripted(REFUSED, pages) + assert len(p.list_channels()) == 4 + assert p.pages_asked == [1, 2], p.pages_asked + + +def test_a_page_count_sent_as_strings_still_counts(): + pages = {1: page([1, 2], total_items="3", max_page_items="2"), 2: page([3])} + p = scripted(REFUSED, pages) + assert len(p.list_channels()) == 3 + assert p.pages_asked == [1, 2], p.pages_asked + + +def test_an_odd_remainder_gets_its_last_page(): + pages = {1: page([1, 2], total_items=5, max_page_items=2), + 2: page([3, 4]), 3: page([5])} + p = scripted(REFUSED, pages) + assert len(p.list_channels()) == 5 + assert p.pages_asked == [1, 2, 3], p.pages_asked + + +def test_a_portal_replaying_its_last_page_does_not_loop_forever(): + """Clamping 'p' instead of running out is common, and open-tv hangs on it.""" + p = portal() + p.pages_asked = [] + + def fake_get_json(query, with_auth=True): + if "get_all_channels" in query: + raise REFUSED + p.pages_asked.append(int(query.split("&p=")[1].split("&")[0])) + return {"js": page([1, 2])} # the same two channels, always + + p._get_json = fake_get_json + channels = p.list_channels() + assert [c.channel_id for c in channels] == ["1", "2"] + assert p.pages_asked == [1, 2], p.pages_asked + + +def test_an_empty_page_ends_it(): + p = scripted(REFUSED, {1: page([1, 2])}) + assert len(p.list_channels()) == 2 + assert p.pages_asked == [1, 2], p.pages_asked + + +def test_the_hard_cap_catches_a_portal_inventing_channels(): + """No total, never empty, never repeating: only the cap is left.""" + p = portal() + counter = [0] + + def fake_get_json(query, with_auth=True): + if "get_all_channels" in query: + raise REFUSED + counter[0] += 1 + return {"js": page([counter[0]])} + + p._get_json = fake_get_json + original = s.ORDERED_LIST_PAGE_CAP + s.ORDERED_LIST_PAGE_CAP = 5 + try: + assert len(p.list_channels()) == 5 + finally: + s.ORDERED_LIST_PAGE_CAP = original + + +def test_duplicates_across_pages_are_collapsed(): + p = scripted(REFUSED, {1: page([1, 2]), 2: page([2, 3])}) + assert [c.channel_id for c in p.list_channels()] == ["1", "2", "3"] + + +def test_when_neither_works_the_useful_message_survives(): + """Paging must not replace 'check the MAC' with something vaguer. + + An empty listing is far more often a wrong MAC than a portal that needs + paging, so the first failure stays the one the user is shown. + """ + p = scripted(REFUSED, {}) + try: + p.list_channels() + except s.PortalError as exc: + assert "MAC address" in str(exc), exc + + +def test_a_refused_session_is_not_paged_at_all(): + p = scripted(s.PortalAuthError("blocked"), {1: page([1])}) + try: + p.list_channels() + except s.PortalAuthError: + pass + else: + raise AssertionError("a refusal must not be retried by another route") + assert p.pages_asked == [], "asking again is how a MAC gets noticed" + + +def test_the_sync_is_told_what_is_happening(): + notes = [] + p = scripted(REFUSED, {1: page([1, 2], total_items=4, max_page_items=2), + 2: page([3, 4])}) + p.list_channels(progress=notes.append) + joined = " | ".join(notes) + assert "page at a time" in joined, notes + assert "2 pages" in joined, notes + assert "4 channels" in joined, notes + + +def test_the_page_request_asks_for_everything(): + p = portal() + asked = [] + p._get_json = lambda q, with_auth=True: asked.append(q) or {"js": {"data": []}} + p.get_ordered_list(3) + assert "genre=*" in asked[0] and "sortby=number" in asked[0], asked + assert "fav=0" in asked[0] and "&p=3" in asked[0], asked + + +if __name__ == "__main__": + failures = 0 + for name, fn in sorted(globals().items()): + if not name.startswith("test_") or not callable(fn): + continue + try: + fn() + print(f"PASS {name}") + except Exception as exc: + failures += 1 + print(f"FAIL {name}: {type(exc).__name__}: {exc}") + print("\n" + ("ALL LISTING TESTS PASSED" if not failures else f"{failures} FAILURE(S)")) + sys.exit(1 if failures else 0) diff --git a/tests/test_manifest.py b/tests/test_manifest.py index c137486..45480f7 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -51,10 +51,17 @@ def load_plugin_module(): class NullLogger: - def info(self, *a, **k): pass - def warning(self, *a, **k): pass - def error(self, *a, **k): pass - def exception(self, *a, **k): pass + """Swallows anything a logger is asked to do. + + A catch-all rather than a list of methods: naming them one by one means a + handler that reaches for a level nobody thought of raises AttributeError + *inside* run()'s own except clause, and the failure surfaces as the action + reporting something unrelated -- which is a long way to travel to discover + that a stub was missing a method. + """ + + def __getattr__(self, _name): + return lambda *a, **k: None class FakeTasks: @@ -109,7 +116,7 @@ def run(p, action, settings): return p.run(action, {}, {"settings": dict(settings), "logger": NullLogger()}) -PORTALS = "http://weaseltv.live/c/ | 00:1A:79:29:53:38\n" +PORTALS = "http://portal.example/c/ | 00:1A:79:AA:BB:CC\n" @contextlib.contextmanager @@ -249,7 +256,7 @@ def test_settings_never_travel_back_in_the_result(): p, store, _ = make_plugin({}) try: result = run(p, "sync_now", { - "portals": "http://weaseltv.live/c/ | 00:1A:79:29:53:38 | password=hunter2" + "portals": "http://portal.example/c/ | 00:1A:79:AA:BB:CC | password=hunter2" }) assert "settings" not in result assert "hunter2" not in json.dumps(result) @@ -494,7 +501,7 @@ def test_assigning_the_profile_also_republishes_the_portals(): {"event": "channel_error", "payload": {}}, {"settings": dict(store["settings"]), "logger": NullLogger()}, ) - assert published == ["weaseltv"] + assert published == ["portal"] finally: (plugin_mod.apply_stream_profile, plugin_mod.save_portal, plugin_mod.publish_fallback) = originals diff --git a/tests/test_mock_portal.py b/tests/test_mock_portal.py index a71a8ba..20b8d69 100644 --- a/tests/test_mock_portal.py +++ b/tests/test_mock_portal.py @@ -32,14 +32,19 @@ def _reply(self, payload): def _handle(self, params, method): action = params.get("action", [""])[0] - seen_requests.append((method, action, dict(self.headers))) + seen_requests.append((method, action, dict(self.headers), params)) if action == "handshake": - return self._reply({"js": {"token": "TESTTOKEN123", "not_valid": 0}}) + return self._reply({"js": {"token": "TESTTOKEN123", "not_valid": 1}}) if action == "do_auth": return self._reply({"js": True, "text": "authenticated"}) if action == "get_profile": - return self._reply({"js": {"id": 42, "fname": "Test User"}}) + # A portal that wants credentials: status 2 until do_auth has run + # and get_profile comes back with auth_second_step=1. This is the + # full state machine login() implements. + if params.get("auth_second_step", ["0"])[0] == "1": + return self._reply({"js": {"id": 42, "fname": "Test User", "status": 0}}) + return self._reply({"js": {"status": 2, "msg": "authorization required"}}) if action == "get_genres": return self._reply({"js": [ {"id": "1", "title": "FR| SPORT"}, @@ -47,9 +52,9 @@ def _handle(self, params, method): ]}) if action == "get_all_channels": return self._reply({"js": {"data": [ - {"id": "101", "name": 'Canal+ "HD"', "cmd": "ffmpeg http://prov/ch/101", - "logo": "canal.png", "tv_genre_id": "1", "number": "1"}, - {"id": "102", "name": "BBC One", "cmd": "ffmpeg http://prov/ch/102", + {"id": "101", "name": 'Alpha "HD"', "cmd": "ffmpeg http://prov/ch/101", + "logo": "alpha.png", "tv_genre_id": "1", "number": "1"}, + {"id": "102", "name": "Beta One", "cmd": "ffmpeg http://prov/ch/102", "logo": "", "tv_genre_id": "2", "number": "2"}, {"id": "103", "name": "No Genre", "cmd": "ffmpeg http://prov/ch/103", "logo": "ng.png", "tv_genre_id": "99", "number": ""}, @@ -58,6 +63,16 @@ def _handle(self, params, method): cmd = params.get("cmd", [""])[0] assert cmd.startswith("ffmpeg "), f"cmd not decoded properly: {cmd!r}" return self._reply({"js": {"cmd": "ffmpeg http://prov/live/101.m3u8?token=FRESH"}}) + if action == "get_epg_info": + assert params.get("period") == ["24"], params + return self._reply({"js": {"data": { + "101": [{"id": "1", "name": "Evening Report", "descr": "News & more", + "start_timestamp": 1785276000, "stop_timestamp": 1785279600}], + # A channel the playlist does not carry: the guide must not + # invent an entry for it. + "999": [{"id": "2", "name": "Ghost", "descr": "", + "start_timestamp": 1785276000, "stop_timestamp": 1785279600}], + }}}) if action == "get_events": return self._reply({"js": [], "text": ""}) return self._reply({"js": []}) @@ -121,7 +136,7 @@ def main(): print("\n--- generated M3U ---") print(m3u) - # The quote in 'Canal+ "HD"' must not break the attribute quoting. + # The quote in 'Alpha "HD"' must not break the attribute quoting. assert '"' not in m3u.split("\n")[1].split(",")[0].replace('tvg-id="', "").replace('"', "") or True for line in m3u.splitlines(): if line.startswith("#EXTINF"): @@ -129,7 +144,7 @@ def main(): assert 'group-title="FR| SPORT"' in m3u assert 'group-title="Other"' in m3u, "unknown genre must fall back to Other" assert 'tvg-id="mock.101"' in m3u - assert "misc/logos/320/canal.png" in m3u + assert "misc/logos/320/alpha.png" in m3u # Round-trip a generated URL exactly as resolver.py would. pseudo = [l for l in m3u.splitlines() if l.startswith("http://distalker.invalid")][0] @@ -138,13 +153,60 @@ def main(): print("pseudo-URL round-trip through the playlist: OK") # Auth header must be present on content calls but absent on handshake. - by_action = {a: h for _, a, h in seen_requests} + by_action = {a: h for _, a, h, _ in seen_requests} assert "Authorization" not in by_action["handshake"], "handshake must not send a token" assert by_action["get_all_channels"]["Authorization"] == "Bearer TESTTOKEN123" assert "MAG200 stbapp" in by_action["get_all_channels"]["User-Agent"] assert "mac=00%3A1A%3A79%3AAA%3ABB%3ACC" in by_action["get_all_channels"]["Cookie"] print("headers (UA / Bearer / MAC cookie): OK") + # The identity reaches a real socket in both forms, not just in the + # headers a unit test can inspect. + content = [p for _, a, _, p in seen_requests if a == "get_all_channels"][0] + assert content["mac"] == ["00:1A:79:AA:BB:CC"], content + assert content["token"] == ["TESTTOKEN123"], content + + # The portal asked for credentials and got them, in the right order. + actions = [a for _, a, _, _ in seen_requests] + assert actions[:4] == ["handshake", "get_profile", "do_auth", "get_profile"], actions + assert portal.auth_method == "credentials", portal.auth_method + + profiles = [p for _, a, _, p in seen_requests if a == "get_profile"] + # not_valid=1 from the handshake must come back as not_valid_token=1. + assert profiles[0]["not_valid_token"] == ["1"], profiles[0] + assert profiles[0]["auth_second_step"] == ["0"], profiles[0] + assert profiles[1]["auth_second_step"] == ["1"], profiles[1] + # The whole STB identity travels with it, signature included -- it used to + # be a setting nothing ever sent. + assert profiles[0]["signature"] == [s.DEFAULT_SIGNATURE], profiles[0] + assert profiles[0]["stb_type"] == [s.DEFAULT_MODEL], profiles[0] + assert profiles[0]["sn"] == [s.DEFAULT_SERIAL], profiles[0] + assert profiles[0]["hw_version"] == [s.STB_HW_VERSION], profiles[0] + assert "PORTAL version: 4.9.9" in profiles[0]["ver"][0], profiles[0] + print("get_profile state machine (status 2 -> do_auth -> second step): OK") + + # The guide, over a real socket: the streamed download and the scratch + # file are the parts a faked transport would never exercise. + epg_data = portal.get_epg_info(24) + assert set(epg_data) == {"101", "999"}, epg_data + guide = "".join(sync.build_xmltv(portal, channels, epg_data)) + print("\n--- generated XMLTV ---") + print(guide) + + import xml.etree.ElementTree as ET + + root = ET.fromstring(guide) + ids = [c.get("id") for c in root.findall("channel")] + assert ids == ["mock.101"], ids + assert root.find("programme").get("channel") == "mock.101" + assert root.find("programme/title").text == "Evening Report" + assert root.find("programme/desc").text == "News & more" + assert root.find("programme").get("start") == "20260728220000 +0000" + # Every tvg-id in the guide must exist in the playlist it accompanies. + for cid in ids: + assert f'tvg-id="{cid}"' in m3u, cid + print("XMLTV round-trip and tvg-id agreement: OK") + # ffmpeg argv construction, as resolver.py builds it. import resolver argv = resolver.build_ffmpeg_command(cfg, link) diff --git a/tests/test_registry.py b/tests/test_registry.py index 16dff7f..b6c6b98 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -44,7 +44,7 @@ def load_plugin_module(): import registry # noqa: E402 import stalker_api as s # noqa: E402 -PORTAL = "weaseltv | http://weaseltv.live/c/ | 00:1A:79:29:53:38 | max_streams=1\n" +PORTAL = "livingroom | http://portal.example/c/ | 00:1A:79:AA:BB:CC | max_streams=1\n" class NullLogger: @@ -87,15 +87,15 @@ def test_absent_key_means_clobbered_and_is_restored(): registry.save_registry(PORTAL) # The panel POSTed a state captured before the portal existed: no key at all. - p, store = make_plugin({"new_name": "weaseltv"}) - merged = {"new_name": "weaseltv", "portals": ""} # defaults merged by the loader + p, store = make_plugin({"new_name": "livingroom"}) + merged = {"new_name": "livingroom", "portals": ""} # defaults merged by the loader result = p._reconcile_registry(merged, NullLogger()) assert result["portals"] == PORTAL, "the clobbered list must come back" assert store["settings"]["portals"] == PORTAL, "and be written back to the DB" portals, errors = s.parse_portals(result["portals"]) - assert not errors and [x.name for x in portals] == ["weaseltv"] + assert not errors and [x.name for x in portals] == ["livingroom"] def test_hand_edited_textarea_wins(): @@ -162,7 +162,7 @@ def test_add_then_stale_click_does_not_lose_the_portal(): assert registry.load_registry() == PORTAL # 2. The panel, still holding its pre-add state, overwrites settings. - store["settings"] = {"new_name": "weaseltv", "new_url": "http://weaseltv.live/c/"} + store["settings"] = {"new_name": "livingroom", "new_url": "http://portal.example/c/"} # 3. The user clicks another action. result = p._reconcile_registry(dict(store["settings"], portals=""), NullLogger()) @@ -183,8 +183,8 @@ def test_stale_empty_textarea_does_not_erase_the_list(): # The panel PUTs the state it captured at page load, textarea and all. store["settings"] = { - "new_name": "weaseltv", - "new_url": "http://weaseltv.live/c/", + "new_name": "livingroom", + "new_url": "http://portal.example/c/", "portals": "", } diff --git a/tests/test_schedule.py b/tests/test_schedule.py new file mode 100644 index 0000000..9ea3114 --- /dev/null +++ b/tests/test_schedule.py @@ -0,0 +1,507 @@ +"""Waking the plugin up without a task it is allowed to register. + +A plugin cannot own a Celery task on a stock install -- the whole story is in +tasks.py -- so the schedule is borrowed: Dispatcharr refreshes each M3U account +on its own timer, and the m3u_refresh it ends with is dispatched to plugin +actions. Distalker answers that event by re-fetching every portal. + +Three guards decide whether an event is allowed to do that, and each covers a +different way the arrangement would otherwise misbehave. They are what this +file pins, because two of them protect other people's installs rather than +this feature: without the account check, refreshing any unrelated playlist +starts a round of portal logins; without the cooldown, the sync's own closing +refresh re-enters the event that started it. +""" +import importlib.util +import os +import sys +import types + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO) + +import stalker_api as s # noqa: E402 + + +def load_plugin(): + pkg = types.ModuleType("distalker_sched") + pkg.__path__ = [REPO] + pkg.__package__ = "distalker_sched" + sys.modules["distalker_sched"] = pkg + spec = importlib.util.spec_from_file_location( + "distalker_sched.plugin", os.path.join(REPO, "plugin.py") + ) + module = importlib.util.module_from_spec(spec) + sys.modules["distalker_sched.plugin"] = module + spec.loader.exec_module(module) + return module + + +plugin_mod = load_plugin() + +# plugin.py reaches stalker_api through a relative import, so the package has +# its own module object -- a different one from the `stalker_api` imported at +# the top of this file. Patching one does not touch the other, and the copy +# that matters is the one the plugin actually calls into. +plugin_api = sys.modules["distalker_sched.stalker_api"] + + +class FakeRedis: + """Enough of redis-py for a SET NX EX, and nothing else.""" + + def __init__(self): + self.store = {} + + def set(self, key, value, nx=False, ex=None): + if nx and key in self.store: + return None + self.store[key] = value + return True + + +class Recorder: + """A plugin whose sync is replaced by a note that it was asked for.""" + + def __init__(self, started=True): + self.plugin = plugin_mod.Plugin.__new__(plugin_mod.Plugin) + self.calls = [] + self.loggers = [] + self.re_asked = [] + self.started = started + + def resync(self, params, settings): + """Both routes stubbed: a click threads it, an event runs it here.""" + originals = (plugin_mod.tasks.run_sync_in_background, + plugin_mod.tasks.run_sync_here, + plugin_mod.request_reparse_later, + plugin_mod.Plugin._portals) + logger = _Logger() + + def fake_thread(full=False): + self.calls.append(("threaded", full)) + return self.started + + def fake_here(full=False, logger=None): + self.calls.append(("here", full)) + self.loggers.append(logger) + return {"message": "done"} if self.started else None + + def fake_later(account_name, delay=None): + self.re_asked.append(account_name) + return True + + plugin_mod.tasks.run_sync_in_background = fake_thread + plugin_mod.tasks.run_sync_here = fake_here + plugin_mod.request_reparse_later = fake_later + plugin_mod.Plugin._portals = lambda self, settings: [] + try: + return self.plugin._action_resync_all(params, settings, logger) + finally: + (plugin_mod.tasks.run_sync_in_background, + plugin_mod.tasks.run_sync_here, + plugin_mod.request_reparse_later, + plugin_mod.Plugin._portals) = originals + + +class _Logger: + def __getattr__(self, _name): + return lambda *a, **k: None + + +def event(account="Distalker: Mock"): + return {"event": "m3u_refresh", "payload": {"account_name": account}} + + +def fresh_redis(): + """Point the cooldown at a Redis nobody else has touched. + + Both module objects, since the plugin's copy is the one that counts and + the test's own asserts read through the other. + """ + client = FakeRedis() + modules = [s, plugin_api] + originals = [m._client_or_none for m in modules] + for module in modules: + module._client_or_none = lambda c=None: client + + def restore(): + for module, original in zip(modules, originals): + module._client_or_none = original + + return client, restore + + +# -- the guards ------------------------------------------------------------ + + +def test_a_button_press_always_syncs(): + """No event, no guards: a click is the user asking, and gets an answer.""" + client, restore = fresh_redis() + try: + rec = Recorder() + result = rec.resync({}, {"refresh_hours": 0}) + assert rec.calls == [("threaded", True)], rec.calls + assert result["changed"] is True + # Untouched: the cooldown is for the schedule, not for the button. + assert client.store == {}, client.store + finally: + restore() + + +def test_an_event_does_nothing_until_the_user_asks_for_a_schedule(): + """The default. An upgrade must not start contacting portals by itself.""" + client, restore = fresh_redis() + try: + rec = Recorder() + result = rec.resync(event(), {"refresh_hours": 0}) + assert rec.calls == [], rec.calls + assert result["changed"] is False + assert "off" in result["message"], result + finally: + restore() + + +def test_somebody_elses_playlist_is_not_our_business(): + """m3u_refresh fires for every account on the install, not just ours.""" + client, restore = fresh_redis() + try: + for name in ("Movies", "", "distalker: lowercase", "Not Distalker: x"): + rec = Recorder() + result = rec.resync(event(name), {"refresh_hours": 12}) + assert rec.calls == [], f"{name}: {rec.calls}" + assert "not one of ours" in result["message"], result + finally: + restore() + + +def test_the_second_event_of_a_cycle_is_the_one_we_caused(): + """The sync ends by asking for a re-read, which re-emits this event. + + The first event syncs; the second lands inside the cooldown and stops + there. Without this the two would take turns for ever, each round costing + a login and a full channel download. + """ + client, restore = fresh_redis() + try: + first = Recorder() + assert first.resync(event(), {"refresh_hours": 12})["changed"] is True + assert first.calls == [("here", True)] + + second = Recorder() + result = second.resync(event(), {"refresh_hours": 12}) + assert second.calls == [], second.calls + assert "too recently" in result["message"], result + finally: + restore() + + +def test_a_scheduled_run_refetches_everything(): + """Planned would find nothing: on a schedule the portal list never changed.""" + client, restore = fresh_redis() + try: + rec = Recorder() + rec.resync(event(), {"refresh_hours": 6}) + assert rec.calls == [("here", True)], "the schedule must force a full re-fetch" + finally: + restore() + + +def test_a_sync_already_running_is_not_started_twice(): + client, restore = fresh_redis() + try: + rec = Recorder(started=False) + result = rec.resync(event(), {"refresh_hours": 6}) + assert result["changed"] is False + assert "already running" in result["message"], result + finally: + restore() + + +def test_a_scheduled_sync_is_not_handed_to_a_thread(): + """The one that had to be learned from a log rather than from reading. + + A click arrives on a uWSGI request that must answer now, so its sync goes + to a thread. An event arrives inside a Celery task, where that same thread + is a real daemon thread in a worker the pool reaps when it scales down -- + the sync announced itself, produced nothing, and forty seconds later five + processes went away. Long work belongs in the Celery task, so that is + where it runs. + """ + client, restore = fresh_redis() + try: + rec = Recorder() + rec.resync(event(), {"refresh_hours": 6}) + assert rec.calls == [("here", True)], rec.calls + assert not any(kind == "threaded" for kind, _ in rec.calls), rec.calls + finally: + restore() + + +def test_a_scheduled_sync_reports_what_it_actually_did(): + """Running it here means the answer is a result, not a promise.""" + client, restore = fresh_redis() + try: + rec = Recorder() + result = rec.resync(event(), {"refresh_hours": 6}) + assert result["message"] == "done", result + assert "background" not in result["message"], result + finally: + restore() + + +def test_the_scheduled_sync_is_given_a_logger_that_prints(): + """The whole run went to nowhere, and looked exactly like no run at all. + + ``logging.getLogger(__name__)`` inside the plugin reaches the container + log from a uWSGI worker and not from a Celery one -- so every 'synced' + line ever recorded came from the button, and the scheduled path wrote its + channel counts, its guide sizes and its failures to a logger with nowhere + to put them. Twice that absence was read as a schedule that never fired. + Dispatcharr's own logger, handed to every action, prints from both. + """ + client, restore = fresh_redis() + try: + rec = Recorder() + rec.resync(event(), {"refresh_hours": 6}) + assert rec.loggers and rec.loggers[0] is not None, rec.loggers + assert isinstance(rec.loggers[0], _Logger), rec.loggers + finally: + restore() + + +def test_the_account_that_woke_us_is_asked_for_again(): + """It is the one account whose playlist our own sync cannot get re-read. + + We run inside Dispatcharr's refresh task for it, which holds that + account's lock for as long as we do, so the request made mid-sync is + refused. Unasked, that portal is downloaded every cycle and read one cycle + late for ever -- a day late on a daily schedule. + """ + client, restore = fresh_redis() + try: + rec = Recorder() + rec.resync(event(account="Distalker: Mock"), {"refresh_hours": 6}) + assert rec.re_asked == ["Distalker: Mock"], rec.re_asked + finally: + restore() + + +def test_a_button_press_asks_nobody_for_a_second_reading(): + """No task of ours holds a lock when the sync came from a click.""" + client, restore = fresh_redis() + try: + rec = Recorder() + rec.resync({}, {"refresh_hours": 6}) + assert rec.re_asked == [], rec.re_asked + finally: + restore() + + +def test_a_scheduled_sync_that_stood_down_asks_for_nothing(): + """Nothing was written, so there is nothing to have re-read.""" + client, restore = fresh_redis() + try: + rec = Recorder(started=False) + rec.resync(event(), {"refresh_hours": 6}) + assert rec.re_asked == [], rec.re_asked + finally: + restore() + + +def test_a_manual_sync_mutes_its_own_echo(): + """A sync the user started must not come back as a scheduled one. + + Every synced portal asks Dispatcharr to re-read its playlist, and that + re-read emits the event the schedule listens for. The scheduled path holds + a cooldown while it works, so its own echo lands inside it -- but a sync + started by hand holds nothing, and its echo was answered with a full + re-fetch of every portal. Seen once, on a single portal being added, and + prevented only by the sync lock happening to still be held. + """ + import importlib.util as _u + import types as _t + + held = [] + + class Task: + def delay(self, account_id): + pass + + fake_tasks = _t.ModuleType("apps.m3u.tasks") + fake_tasks.refresh_m3u_groups = Task() + fake_tasks.refresh_single_m3u_account = Task() + apps = _t.ModuleType("apps") + m3u = _t.ModuleType("apps.m3u") + saved = {k: sys.modules.get(k) for k in ("apps", "apps.m3u", "apps.m3u.tasks")} + sys.modules.update({"apps": apps, "apps.m3u": m3u, "apps.m3u.tasks": fake_tasks}) + + spec = _u.spec_from_file_location( + "distalker_sched.sync2", os.path.join(REPO, "sync.py") + ) + sync = _u.module_from_spec(spec) + sys.modules["distalker_sched.sync2"] = sync + try: + spec.loader.exec_module(sync) + sync.hold_auto_sync = lambda ttl=0: held.append(ttl) + + sync.request_reparse(7, 0) + assert held == [], "no schedule, nothing to mute" + + sync.request_reparse(7, 6) + assert held == [sync.AUTO_SYNC_COOLDOWN], held + finally: + for key, module in saved.items(): + if module is None: + sys.modules.pop(key, None) + else: + sys.modules[key] = module + + +def test_the_schedule_is_applied_on_the_path_that_fetches_nothing(): + """The one that matters, and the one that was missed. + + Changing the schedule changes no portal, so Sync's plan finds nothing to + fetch and returns early -- through _republish, not through the sync. Put + the interval only on the fetching path and the setting appears to do + nothing at all, which is exactly how it behaved before this test existed. + """ + calls = [] + originals = ( + plugin_mod.apply_refresh_interval, + plugin_mod.publish_fallback, + plugin_mod.Plugin._portals, + ) + plugin_mod.apply_refresh_interval = lambda hours, logger: calls.append(hours) + plugin_mod.publish_fallback = lambda name: "" + plugin_mod.Plugin._portals = lambda self, settings: [] + try: + plugin = plugin_mod.Plugin.__new__(plugin_mod.Plugin) + plugin._republish({"refresh_hours": 6}, _Logger()) + assert calls == [6], calls + finally: + (plugin_mod.apply_refresh_interval, + plugin_mod.publish_fallback, + plugin_mod.Plugin._portals) = originals + + +def test_only_one_reparse_task_is_ever_dispatched(): + """Both at once put them in a race for the same per-account lock. + + refresh_single_m3u_account refreshes the groups itself. Asking for + refresh_m3u_groups as well made the loser report "Failed to refresh M3U + groups" at the user and left the account stuck in Pending Setup -- which + looked like a portal problem and was not. + """ + import importlib.util as _u + import types as _t + + calls = [] + + class Task: + def __init__(self, name): + self.name = name + + def delay(self, account_id): + calls.append((self.name, account_id)) + + fake_tasks = _t.ModuleType("apps.m3u.tasks") + fake_tasks.refresh_m3u_groups = Task("refresh_m3u_groups") + fake_tasks.refresh_single_m3u_account = Task("refresh_single_m3u_account") + apps = _t.ModuleType("apps") + m3u = _t.ModuleType("apps.m3u") + saved = {k: sys.modules.get(k) for k in ("apps", "apps.m3u", "apps.m3u.tasks")} + sys.modules.update({"apps": apps, "apps.m3u": m3u, "apps.m3u.tasks": fake_tasks}) + + spec = _u.spec_from_file_location( + "distalker_sched.sync", os.path.join(REPO, "sync.py") + ) + sync = _u.module_from_spec(spec) + sys.modules["distalker_sched.sync"] = sync + try: + spec.loader.exec_module(sync) + + assert sync.request_reparse(7, 0) == "refresh_m3u_groups" + assert calls == [("refresh_m3u_groups", 7)], calls + + calls.clear() + assert sync.request_reparse(7, 6) == "refresh_single_m3u_account" + assert calls == [("refresh_single_m3u_account", 7)], calls + finally: + for key, module in saved.items(): + if module is None: + sys.modules.pop(key, None) + else: + sys.modules[key] = module + + +# -- the setting ----------------------------------------------------------- + + +def test_the_schedule_is_off_by_default_and_never_below_an_hour(): + """Below an hour the cooldown outlasts the interval, swallowing every + other run -- a schedule that quietly does half of what it says.""" + read = plugin_mod.Plugin._refresh_hours + assert read({}) == 0 + assert read({"refresh_hours": 0}) == 0 + assert read({"refresh_hours": ""}) == 0 + assert read({"refresh_hours": "nonsense"}) == 0 + assert read({"refresh_hours": -5}) == 0 + assert read({"refresh_hours": 1}) == 1 + assert read({"refresh_hours": "12"}) == 12 + + +def test_the_cooldown_outlasts_a_sync(): + assert plugin_mod.AUTO_SYNC_COOLDOWN >= 900, plugin_mod.AUTO_SYNC_COOLDOWN + + +def test_the_action_is_subscribed_to_the_event(): + """Without this in the manifest nothing is ever dispatched here.""" + actions = {a["id"]: a for a in plugin_mod.MANIFEST["actions"]} + assert "m3u_refresh" in (actions["resync_all"].get("events") or []), actions + + +def test_the_setting_exists_and_is_off(): + fields = {f["id"]: f for f in plugin_mod.MANIFEST["fields"]} + assert fields["refresh_hours"]["default"] == 0, fields["refresh_hours"] + + +# -- the cooldown itself --------------------------------------------------- + + +def test_the_cooldown_refuses_rather_than_assumes_when_redis_is_gone(): + """The opposite of the sync lock, and on purpose. + + That one guards a button the user just pressed and carries on when Redis + cannot answer. This one guards against a loop nobody asked for, so silence + means no. + """ + original = plugin_api._client_or_none + plugin_api._client_or_none = lambda c=None: None + try: + assert plugin_api.claim_auto_sync() is False + finally: + plugin_api._client_or_none = original + + +def test_the_cooldown_is_claimed_once(): + client, restore = fresh_redis() + try: + assert plugin_api.claim_auto_sync(ttl=60) is True + assert plugin_api.claim_auto_sync(ttl=60) is False + finally: + restore() + + +if __name__ == "__main__": + failures = 0 + for name, fn in sorted(globals().items()): + if not name.startswith("test_") or not callable(fn): + continue + try: + fn() + print(f"PASS {name}") + except Exception as exc: + failures += 1 + print(f"FAIL {name}: {type(exc).__name__}: {exc}") + print("\n" + ("ALL SCHEDULE TESTS PASSED" if not failures else f"{failures} FAILURE(S)")) + sys.exit(1 if failures else 0) diff --git a/tests/test_state.py b/tests/test_state.py index 8c27785..4aea2aa 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -11,6 +11,7 @@ Redis is repopulated from it, and that the one thing with no mirror -- the session token -- degrades to "no cache" rather than raising. """ +import importlib.util import os import shutil import sys @@ -25,11 +26,18 @@ import stalker_api as s # noqa: E402 +# resolver.py runs as a script, so it imports as a plain top-level module. +_spec = importlib.util.spec_from_file_location( + "resolver", os.path.join(REPO, "resolver.py") +) +resolver = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(resolver) + CFG = s.PortalConfig( - slug="weasel-tv", - name="Weasel TV", - url="http://weaseltv.live/c/", - mac="00:1A:79:29:53:38", + slug="livingroom", + name="Living Room", + url="http://portal.example/c/", + mac="00:1A:79:AA:BB:CC", username="jo", password="hunter2", timeout=90, @@ -239,6 +247,39 @@ def test_the_token_is_not_written_to_disk(): ] +def test_a_channel_the_portal_refuses_still_leaves_a_usable_token(): + """The handshake worked; only the link did not. + + Caching after the link would mean a portal refusing one channel -- a + connection limit, a subscription gap -- sent every later tune through a + fresh handshake to learn the same thing. + """ + reset() + client = FakeRedis() + s.save_portal(CFG, client) + + class Refusing(s.Portal): + def login(self): + self.token = "fresh" + return self.token + + def create_link(self, cmd): + raise s.PortalError("connection limit reached") + + original_portal, original_redis = s.Portal, s.get_redis + s.Portal, s.get_redis = Refusing, lambda: client + try: + resolver.resolve(CFG.slug, "ffmpeg http://localhost/ch/1_") + except s.PortalError: + pass + else: + raise AssertionError("the refusal must still reach the caller") + finally: + s.Portal, s.get_redis = original_portal, original_redis + + assert s.get_cached_token(CFG.slug, client) == "fresh" + + # -- the interpreter the stream profile is built with ------------------------- def test_a_python_is_chosen_even_when_the_sync_runs_under_uwsgi(): diff --git a/tests/test_transport.py b/tests/test_transport.py new file mode 100644 index 0000000..59f5fe5 --- /dev/null +++ b/tests/test_transport.py @@ -0,0 +1,270 @@ +"""What a portal request carries, and when it is worth making twice. + +Two changes share this file because they are the same decision seen from two +sides: what to do about a portal that does not behave like Ministra. One sends +the identity in every form a portal might read it in; the other accepts that a +portal can simply be having a bad minute. + +The retry half matters most for what it must *not* do. Retrying at tune time +would keep a dead source alive long enough to stop Dispatcharr failing over to +a working one, and retrying a refused login is how a MAC gets banned. +""" +import os +import sys + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO) + +import stalker_api as s # noqa: E402 + + +class FakeResponse: + def __init__(self, status=200, payload=None, text=""): + self.status_code = status + self._payload = payload + self.text = text + + def json(self): + if self._payload is None: + raise ValueError("not json") + return self._payload + + +class FakeSession: + """Replays a script of answers, recording every call and every sleep.""" + + def __init__(self, script): + self.script = list(script) + self.calls = [] + + def request(self, method, url, **kwargs): + self.calls.append((method, url, kwargs)) + answer = self.script.pop(0) if self.script else FakeResponse() + if isinstance(answer, Exception): + raise answer + return answer + + +def portal(script, retries=0, token="", **cfg_kwargs): + cfg = s.PortalConfig( + slug="t", name="T", url="http://p.example/c/portal.php", + mac="00:1A:79:AA:BB:CC", **cfg_kwargs + ) + p = s.Portal(cfg, token=token, retries=retries) + p.session = FakeSession(script) + return p + + +def no_sleeping(): + """Swap time.sleep out, collecting what would have been waited. + + Returns the list and the undo, so a backoff can be asserted on without the + test suite actually spending it. + """ + slept = [] + original = s.time.sleep + s.time.sleep = slept.append + return slept, (lambda: setattr(s.time, "sleep", original)) + + +# -- identity in the query string --------------------------------------- + + +def test_the_mac_travels_in_the_query_as_well_as_the_cookie(): + p = portal([FakeResponse(payload={"js": {}})]) + p._get_json("action=get_genres") + _, url, kwargs = p.session.calls[0] + assert "mac=00%3A1A%3A79%3AAA%3ABB%3ACC" in url, url + assert "mac=00%3A1A%3A79%3AAA%3ABB%3ACC" in kwargs["headers"]["Cookie"] + + +def test_the_token_travels_in_the_query_as_well_as_the_header(): + p = portal([FakeResponse(payload={"js": {}})], token="TOK") + p._get_json("action=get_genres") + _, url, kwargs = p.session.calls[0] + assert "token=TOK" in url, url + assert kwargs["headers"]["Authorization"] == "Bearer TOK" + + +def test_the_handshake_still_proves_nothing_it_has_not_earned(): + """No token in either place before there is one to send.""" + p = portal([FakeResponse(payload={"js": {"token": "NEW"}})]) + p.handshake() + _, url, kwargs = p.session.calls[0] + assert "Authorization" not in kwargs["headers"] + # handshake sends its own empty 'token=' by protocol; what must not appear + # is a second, authenticating one appended by the common parameters. + assert url.count("token=") == 1, url + assert "mac=" in url, "the MAC is still needed to be recognised" + + +def test_credentials_are_posted_with_the_identity_on_the_url(): + p = portal([FakeResponse(payload={"js": True})], token="TOK", + username="joe", password="pw") + p.authenticate() + method, url, kwargs = p.session.calls[0] + assert method == "POST" + assert "mac=" in url and "token=TOK" in url, url + # The password stays in the body, where a proxy log will not keep it. + assert "pw" not in url + assert kwargs["data"]["password"] == "pw" + + +# -- retrying ----------------------------------------------------------- + + +def test_nothing_is_retried_by_default(): + """The resolver's setting, and the one that protects failover.""" + import requests + + p = portal([requests.ConnectionError("down"), FakeResponse(payload={"js": {}})]) + try: + p._get_json("action=get_genres") + except s.PortalError: + pass + else: + raise AssertionError("a portal that is down must fail on the first try") + assert len(p.session.calls) == 1, p.session.calls + + +def test_a_transient_failure_is_ridden_out_when_retries_are_asked_for(): + import requests + + slept, restore = no_sleeping() + try: + p = portal( + [requests.ConnectionError("down"), + FakeResponse(502, text="bad gateway"), + FakeResponse(payload={"js": {"ok": 1}})], + retries=2, + ) + assert p._get_json("action=get_genres") == {"js": {"ok": 1}} + assert len(p.session.calls) == 3 + assert slept == [1.0, 2.0], slept + finally: + restore() + + +def test_the_attempts_do_run_out(): + slept, restore = no_sleeping() + try: + p = portal([FakeResponse(503, text="busy")] * 3, retries=2) + try: + p._get_json("action=get_genres") + except s.PortalError as exc: + assert "503" in str(exc), exc + else: + raise AssertionError("a portal that never answers must raise") + assert len(p.session.calls) == 3 + finally: + restore() + + +def test_a_refusal_is_never_retried(): + """Repeating a rejected login is how a MAC gets itself banned.""" + slept, restore = no_sleeping() + try: + p = portal([FakeResponse(403, text="no")] * 3, retries=2) + try: + p._get_json("action=get_genres") + except s.PortalAuthError: + pass + else: + raise AssertionError("403 must be an auth error") + assert len(p.session.calls) == 1, p.session.calls + assert slept == [] + finally: + restore() + + +def test_a_verdict_is_never_retried(): + """404 is the portal having made up its mind; asking again changes nothing.""" + slept, restore = no_sleeping() + try: + p = portal([FakeResponse(404, text="gone")] * 3, retries=2) + try: + p._get_json("action=get_genres") + except s.PortalError as exc: + assert "404" in str(exc), exc + assert len(p.session.calls) == 1, p.session.calls + finally: + restore() + + +def test_prose_instead_of_json_is_not_a_reason_to_ask_again(): + """It arrives with a 200 attached, so only the body says anything is wrong. + + Retrying would also delay the resolver's re-authentication, which is what + this particular body is supposed to trigger. + """ + slept, restore = no_sleeping() + try: + p = portal([FakeResponse(200, text="Authorization failed.")] * 3, retries=2) + try: + p._get_json("action=get_genres") + except s.PortalAuthError: + pass + else: + raise AssertionError("expected the session to be reported as dead") + assert len(p.session.calls) == 1, p.session.calls + finally: + restore() + + +def test_the_sync_asks_for_retries_and_the_test_action_does_not(): + """The one asymmetry that matters, pinned so a refactor keeps it. + + 'Test portals' runs on the request thread, where three attempts at a + 60-second timeout outlast any proxy in front of Dispatcharr. + """ + import importlib.util + import types + + pkg = types.ModuleType("distalker_probe") + pkg.__path__ = [REPO] + sys.modules["distalker_probe"] = pkg + spec = importlib.util.spec_from_file_location( + "distalker_probe.sync", os.path.join(REPO, "sync.py") + ) + sync = importlib.util.module_from_spec(spec) + sys.modules["distalker_probe.sync"] = sync + spec.loader.exec_module(sync) + + built = [] + original = sync.Portal + sync.Portal = lambda cfg, **kw: built.append(kw) or original(cfg, **kw) + _, restore = no_sleeping() # sync_portal would otherwise back off for real + try: + cfg = s.PortalConfig(slug="t", name="T", url="http://p.example/c/portal.php", + mac="00:1A:79:AA:BB:CC") + for call in (lambda: sync.test_portal(cfg), lambda: sync.sync_portal(cfg, _Logger())): + try: + call() + except Exception: + pass # no portal is listening; only the construction matters + finally: + sync.Portal = original + restore() + + assert built[0].get("retries", 0) == 0, f"test_portal must not retry: {built[0]}" + assert built[1].get("retries") == sync.SYNC_RETRIES, built[1] + + +class _Logger: + def __getattr__(self, _name): + return lambda *a, **k: None + + +if __name__ == "__main__": + failures = 0 + for name, fn in sorted(globals().items()): + if not name.startswith("test_") or not callable(fn): + continue + try: + fn() + print(f"PASS {name}") + except Exception as exc: + failures += 1 + print(f"FAIL {name}: {type(exc).__name__}: {exc}") + print("\n" + ("ALL TRANSPORT TESTS PASSED" if not failures else f"{failures} FAILURE(S)")) + sys.exit(1 if failures else 0)