-
Notifications
You must be signed in to change notification settings - Fork 0
fix: resolve episode audio from the podcast feed #8
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -20,7 +20,6 @@ | |
| }, | ||
| "dependencies": { | ||
| "@types/node": "^24.5.0", | ||
| "cheerio": "^1.0.0", | ||
| "music-metadata": "^11.10.3" | ||
| } | ||
| } | ||
This file was deleted.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,129 @@ | ||
| /** | ||
| * Media lookup via the podcast feed. | ||
| * | ||
| * The audio URL used to be scraped out of the rendered post page with a | ||
| * `audio source` selector. That coupled the worker to theme output: when an | ||
| * episode was published as a .wav, PowerPress fell back to a link-only player, | ||
| * the selector matched nothing, and every cron run threw for a day. | ||
| * | ||
| * PowerPress publishes the same URL in the feed's <enclosure> tag, which is a | ||
| * stable contract and additionally carries the file size — needed to decide how | ||
| * to hand the file to Telegram. | ||
| */ | ||
|
|
||
| const FEEDS: Partial<Record<string, string>> = { | ||
| FR: 'https://mhmic.org/feed/podcast/', | ||
| // JK: verify the per-category feed slug before enabling. | ||
| }; | ||
|
|
||
| export interface Enclosure { | ||
| url: string; | ||
| /** Bytes as declared by the feed; 0 when the feed omits the attribute. */ | ||
| length: number; | ||
| /** MIME type as declared by the feed. Unreliable — a .wav has been seen announced as audio/mpeg. */ | ||
| declaredType: string; | ||
| } | ||
|
|
||
| const decodeEntities = (value: string): string => | ||
| value | ||
| .replace(/</g, '<') | ||
| .replace(/>/g, '>') | ||
| .replace(/"/g, '"') | ||
| .replace(/�?39;/g, "'") | ||
| .replace(/'/g, "'") | ||
| .replace(/&/g, '&'); | ||
|
|
||
| const ITEM_RE = /<item\b[^>]*>([\s\S]*?)<\/item>/gi; | ||
|
|
||
| // Feed generators vary in how they quote attributes, so double, single and | ||
| // unquoted values are all accepted. The leading \b keeps `type` from matching | ||
| // the tail of an unrelated attribute name. | ||
| const attr = (tag: string, name: string): string | null => { | ||
| const match = tag.match(new RegExp(`\\b${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s"'>]+))`, 'i')); | ||
| if (!match) { | ||
| return null; | ||
| } | ||
| return decodeEntities(match[1] ?? match[2] ?? match[3]); | ||
| }; | ||
|
|
||
| const text = (item: string, tag: string): string | null => { | ||
| const match = item.match(new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)</${tag}>`, 'i')); | ||
| if (!match) { | ||
| return null; | ||
| } | ||
| return decodeEntities(match[1].replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1')).trim(); | ||
| }; | ||
|
|
||
| const toEnclosure = (item: string): Enclosure | null => { | ||
| const tag = item.match(/<enclosure\b[^>]*>/i)?.[0]; | ||
| if (!tag) { | ||
| return null; | ||
| } | ||
|
|
||
| const url = attr(tag, 'url'); | ||
| if (!url || !url.startsWith('http')) { | ||
| return null; | ||
| } | ||
|
|
||
| const length = Number.parseInt(attr(tag, 'length') || '0', 10); | ||
|
|
||
| return { | ||
| url, | ||
| length: Number.isFinite(length) ? length : 0, | ||
| declaredType: attr(tag, 'type') || '', | ||
| }; | ||
| }; | ||
|
|
||
| /** | ||
| * Resolves the media file for a post by matching the feed item whose link or | ||
| * guid contains the slug. | ||
| * | ||
| * Throws when no item matches. Feeds are cached more aggressively than the REST | ||
| * API and can lag a freshly published post by a few minutes, but falling back to | ||
| * the newest item would send the *previous* episode's audio — and the caller | ||
| * records the post as delivered on success, so the real audio would never go | ||
| * out. Failing instead leaves the position unchanged and the next cron tick | ||
| * retries once the feed catches up. | ||
| */ | ||
| export const getEnclosure = async (category: string, slug: string): Promise<Enclosure> => { | ||
| const feedUrl = FEEDS[category]; | ||
|
|
||
| if (!feedUrl) { | ||
| throw new Error(`No podcast feed configured for category: ${category}`); | ||
| } | ||
|
|
||
| console.log(`Fetching podcast feed for ${category}: ${feedUrl}`); | ||
|
|
||
| const response = await fetch(feedUrl, { headers: { accept: 'application/rss+xml, application/xml;q=0.9' } }); | ||
|
|
||
| if (!response.ok) { | ||
| throw new Error(`Podcast feed error: ${response.status} ${response.statusText} for URL: ${feedUrl}`); | ||
| } | ||
|
|
||
| const xml = await response.text(); | ||
| const items = [...xml.matchAll(ITEM_RE)].map((match) => match[1]); | ||
|
|
||
| if (items.length === 0) { | ||
| throw new Error(`Podcast feed contained no items: ${feedUrl}`); | ||
| } | ||
|
|
||
| const matched = items.find((item) => { | ||
| const link = text(item, 'link') || ''; | ||
| const guid = text(item, 'guid') || ''; | ||
| return link.includes(slug) || guid.includes(slug); | ||
| }); | ||
|
|
||
| if (!matched) { | ||
| throw new Error(`No feed item matched slug "${slug}" — the feed may not have caught up with the post yet`); | ||
| } | ||
|
|
||
| const enclosure = toEnclosure(matched); | ||
|
|
||
| if (!enclosure) { | ||
| throw new Error(`No audio enclosure in the podcast feed for "${slug}" — check the episode was published with a media file attached`); | ||
| } | ||
|
|
||
| console.log(`Resolved audio for ${slug}: ${enclosure.url} (${enclosure.length} bytes, ${enclosure.declaredType})`); | ||
|
|
||
| return enclosure; | ||
| }; | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,11 +1,41 @@ | ||
| import { env } from 'cloudflare:workers'; | ||
| import { extractAudioFilename } from './utils'; | ||
| import type { Enclosure } from './feed'; | ||
| import { parseBuffer } from 'music-metadata'; | ||
|
|
||
| // Telegram fetches the file itself at this size or below, so nothing passes | ||
| // through the worker. | ||
| const MAX_URL_SEND_BYTES = 20 * 1024 * 1024; | ||
|
|
||
| // Telegram rejects bot uploads larger than 50 MB, and the whole file is held in | ||
| // memory here, so oversized audio is rejected before it is downloaded. | ||
| // memory on the upload path, so oversized audio is rejected before download. | ||
| const MAX_AUDIO_BYTES = 50 * 1024 * 1024; | ||
|
|
||
| const MIME_BY_EXTENSION: Record<string, string> = { | ||
| mp3: 'audio/mpeg', | ||
| m4a: 'audio/mp4', | ||
| ogg: 'audio/ogg', | ||
| opus: 'audio/ogg', | ||
| wav: 'audio/wav', | ||
| flac: 'audio/flac', | ||
| }; | ||
|
|
||
| const extensionOf = (url: string): string => { | ||
| try { | ||
| return new URL(url).pathname.split('.').pop()?.toLowerCase() ?? ''; | ||
| } catch { | ||
| return ''; | ||
| } | ||
| }; | ||
|
|
||
| // The feed's declared type is unreliable — a .wav has been seen announced as | ||
| // audio/mpeg — so the extension wins. | ||
| const mimeFor = (url: string, declaredType: string): string => MIME_BY_EXTENSION[extensionOf(url)] || declaredType || 'application/octet-stream'; | ||
|
|
||
| // extractAudioFilename only strips a .mp3 suffix, which leaves the extension | ||
| // visible in the Telegram title for any other format. | ||
| const stripExtension = (title: string): string => title.replace(/\.(mp3|m4a|ogg|opus|wav|flac)$/i, ''); | ||
|
|
||
| interface TelegramErrorResponse { | ||
| description?: string; | ||
| } | ||
|
|
@@ -36,16 +66,50 @@ const sendMessage = async (chatId: string, text: string) => { | |
| await assertTelegramOk(response, 'Telegram sendMessage'); | ||
| }; | ||
|
|
||
| export const sendTelegramAudio = async (audioUrl: string) => { | ||
| if (!audioUrl || !audioUrl.startsWith('http')) { | ||
| throw new Error(`Invalid audio URL: ${audioUrl}`); | ||
| export const sendTelegramAudio = async (enclosure: Enclosure) => { | ||
| const { url, length, declaredType } = enclosure; | ||
|
|
||
| if (!url || !url.startsWith('http')) { | ||
| throw new Error(`Invalid audio URL: ${url}`); | ||
| } | ||
|
|
||
| const { filename, cleanTitle } = extractAudioFilename(url); | ||
| const title = stripExtension(cleanTitle); | ||
| const mimeType = mimeFor(url, declaredType); | ||
|
|
||
| if (length > MAX_AUDIO_BYTES) { | ||
| throw new Error( | ||
| `Audio is too large to send: ${length} bytes exceeds the ${MAX_AUDIO_BYTES} byte limit. ` + | ||
| 'Re-publish the episode as an mp3 — a full-length WAV will always exceed this.' | ||
| ); | ||
| } | ||
|
|
||
| // Preferred path: hand Telegram the URL. Downloading into the worker risks | ||
| // exhausting the 128 MB isolate long before the 50 MB guard below fires — a | ||
| // 104 MB WAV episode is what surfaced this. | ||
| if (length > 0 && length <= MAX_URL_SEND_BYTES) { | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. NIT — the URL-vs-buffer size branching (20 MB URL path, 50 MB guard, missing-length case) is the trickiest logic in this PR but has no unit coverage.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Added
One gap worth naming: this does not cover the metadata enrichment. |
||
| const response = await fetch(telegramUrl('sendAudio'), { | ||
| method: 'POST', | ||
| headers: { | ||
| 'Content-Type': 'application/json', | ||
| }, | ||
| body: JSON.stringify({ | ||
| chat_id: env.MAIN_CHAT_ID, | ||
| audio: url, | ||
| title, | ||
| }), | ||
| }); | ||
|
|
||
| await assertTelegramOk(response, 'Telegram sendAudio (by URL)'); | ||
| console.log(`Sent audio by URL: ${url} (${length} bytes)`); | ||
| return; | ||
| } | ||
|
|
||
| // Extract filename from URL | ||
| const { filename, cleanTitle } = extractAudioFilename(audioUrl); | ||
| // Fallback: buffer and upload. Used between 20 and 50 MB, or when the feed | ||
| // omitted the length and the URL limit cannot be ruled out. | ||
| console.log(`Buffering audio for upload: ${url} (${length || 'unknown'} bytes)`); | ||
|
|
||
| // Download the file and upload as multipart form data | ||
| const audioResponse = await fetch(audioUrl); | ||
| const audioResponse = await fetch(url); | ||
|
|
||
| if (!audioResponse.ok) { | ||
| throw new Error(`Failed to download audio: ${audioResponse.status} ${audioResponse.statusText}`); | ||
|
|
@@ -65,26 +129,26 @@ export const sendTelegramAudio = async (audioUrl: string) => { | |
|
|
||
| // Parse metadata | ||
| let duration: number | undefined; | ||
| let title = cleanTitle; | ||
| let uploadTitle = title; | ||
| let performer: string | undefined; | ||
|
|
||
| try { | ||
| const uint8Array = new Uint8Array(audioBuffer); | ||
| const metadata = await parseBuffer(uint8Array); | ||
| const metadata = await parseBuffer(uint8Array, mimeType); | ||
|
|
||
| if (metadata.format.duration) { | ||
| duration = Math.round(metadata.format.duration); | ||
| } | ||
|
|
||
| if (metadata.common.title) { | ||
| title = metadata.common.title; | ||
| uploadTitle = metadata.common.title; | ||
| } | ||
|
|
||
| if (metadata.common.artist) { | ||
| performer = metadata.common.artist; | ||
| } | ||
|
|
||
| console.log(`Extracted metadata - Duration: ${duration}s, Title: ${title}, Performer: ${performer}`); | ||
| console.log(`Extracted metadata - Duration: ${duration}s, Title: ${uploadTitle}, Performer: ${performer}`); | ||
| } catch (error) { | ||
| console.error('Failed to parse audio metadata:', error); | ||
| // Fallback to defaults if parsing fails | ||
|
|
@@ -93,8 +157,8 @@ export const sendTelegramAudio = async (audioUrl: string) => { | |
| // Create multipart form data | ||
| const formData = new FormData(); | ||
| formData.append('chat_id', env.MAIN_CHAT_ID); | ||
| formData.append('audio', new Blob([audioBuffer], { type: 'audio/mpeg' }), filename); | ||
| formData.append('title', title); | ||
| formData.append('audio', new Blob([audioBuffer], { type: mimeType }), filename); | ||
| formData.append('title', uploadTitle); | ||
|
|
||
| if (duration) { | ||
| formData.append('duration', duration.toString()); | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
NIT (optional) —
attr()only matches double-quoted attributes; a single-quoted or unquotedurlvalue would be ignored and the item treated as having no enclosure. Fine for most feed generators today.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Done in d809de4 —
attr()now accepts double-quoted, single-quoted, and unquoted values via an alternation, taking the first defined group (??rather than||, so an emptyurl=""stays""instead of falling through).Also added a
\bbefore the name while in there:attr(tag, 'type')would previously have matched the tail of an unrelated attribute likextype="...". Covered by a test with single-quoted and unquoted attributes.