diff --git a/package.json b/package.json index 55ed731..82137d3 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,6 @@ }, "dependencies": { "@types/node": "^24.5.0", - "cheerio": "^1.0.0", "music-metadata": "^11.10.3" } } diff --git a/src/audio.ts b/src/audio.ts deleted file mode 100644 index 330c3e5..0000000 --- a/src/audio.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { load } from 'cheerio'; - -const WEBSITE = 'https://mhmic.org'; -const FR_LINK = `${WEBSITE}/fajrreminders`; - -export const getHTML = async (slug: string) => { - try { - console.log(`Fetching HTML for slug: ${slug}`); - const url = `${FR_LINK}/${slug}`; - const data = await fetch(url); - - if (!data.ok) { - throw new Error(`HTTP ${data.status}: ${data.statusText} for URL: ${url}`); - } - - const html = await data.text(); - - if (!html || html.trim().length === 0) { - throw new Error(`Empty HTML response for slug: ${slug}`); - } - - console.log(`Successfully fetched HTML for slug: ${slug}`); - return html; - } catch (error) { - console.error(`Error fetching HTML for slug ${slug}:`, error); - throw error; - } -}; - -export const getAudioUrl = (html: string) => { - try { - console.log('Extracting audio URL from HTML'); - const $ = load(html); - const audioSrc = $('audio source').attr('src'); - - if (!audioSrc) { - throw new Error('No audio source found in HTML'); - } - - // Convert relative URLs to absolute URLs - const fullAudioUrl = audioSrc.startsWith('http') ? audioSrc : `${WEBSITE}${audioSrc}`; - - console.log(`Successfully extracted audio URL: ${fullAudioUrl}`); - return fullAudioUrl; - } catch (error) { - console.error('Error extracting audio URL:', error); - throw error; - } -}; diff --git a/src/feed.ts b/src/feed.ts new file mode 100644 index 0000000..3aece57 --- /dev/null +++ b/src/feed.ts @@ -0,0 +1,129 @@ +/** + * Media lookup via the podcast feed. + * + * The audio URL used to be scraped out of the rendered post page with a + * `audio source` selector. That coupled the worker to theme output: when an + * episode was published as a .wav, PowerPress fell back to a link-only player, + * the selector matched nothing, and every cron run threw for a day. + * + * PowerPress publishes the same URL in the feed's tag, which is a + * stable contract and additionally carries the file size — needed to decide how + * to hand the file to Telegram. + */ + +const FEEDS: Partial> = { + FR: 'https://mhmic.org/feed/podcast/', + // JK: verify the per-category feed slug before enabling. +}; + +export interface Enclosure { + url: string; + /** Bytes as declared by the feed; 0 when the feed omits the attribute. */ + length: number; + /** MIME type as declared by the feed. Unreliable — a .wav has been seen announced as audio/mpeg. */ + declaredType: string; +} + +const decodeEntities = (value: string): string => + value + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/�?39;/g, "'") + .replace(/'/g, "'") + .replace(/&/g, '&'); + +const ITEM_RE = /]*>([\s\S]*?)<\/item>/gi; + +// Feed generators vary in how they quote attributes, so double, single and +// unquoted values are all accepted. The leading \b keeps `type` from matching +// the tail of an unrelated attribute name. +const attr = (tag: string, name: string): string | null => { + const match = tag.match(new RegExp(`\\b${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s"'>]+))`, 'i')); + if (!match) { + return null; + } + return decodeEntities(match[1] ?? match[2] ?? match[3]); +}; + +const text = (item: string, tag: string): string | null => { + const match = item.match(new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)`, 'i')); + if (!match) { + return null; + } + return decodeEntities(match[1].replace(//g, '$1')).trim(); +}; + +const toEnclosure = (item: string): Enclosure | null => { + const tag = item.match(/]*>/i)?.[0]; + if (!tag) { + return null; + } + + const url = attr(tag, 'url'); + if (!url || !url.startsWith('http')) { + return null; + } + + const length = Number.parseInt(attr(tag, 'length') || '0', 10); + + return { + url, + length: Number.isFinite(length) ? length : 0, + declaredType: attr(tag, 'type') || '', + }; +}; + +/** + * Resolves the media file for a post by matching the feed item whose link or + * guid contains the slug. + * + * Throws when no item matches. Feeds are cached more aggressively than the REST + * API and can lag a freshly published post by a few minutes, but falling back to + * the newest item would send the *previous* episode's audio — and the caller + * records the post as delivered on success, so the real audio would never go + * out. Failing instead leaves the position unchanged and the next cron tick + * retries once the feed catches up. + */ +export const getEnclosure = async (category: string, slug: string): Promise => { + const feedUrl = FEEDS[category]; + + if (!feedUrl) { + throw new Error(`No podcast feed configured for category: ${category}`); + } + + console.log(`Fetching podcast feed for ${category}: ${feedUrl}`); + + const response = await fetch(feedUrl, { headers: { accept: 'application/rss+xml, application/xml;q=0.9' } }); + + if (!response.ok) { + throw new Error(`Podcast feed error: ${response.status} ${response.statusText} for URL: ${feedUrl}`); + } + + const xml = await response.text(); + const items = [...xml.matchAll(ITEM_RE)].map((match) => match[1]); + + if (items.length === 0) { + throw new Error(`Podcast feed contained no items: ${feedUrl}`); + } + + const matched = items.find((item) => { + const link = text(item, 'link') || ''; + const guid = text(item, 'guid') || ''; + return link.includes(slug) || guid.includes(slug); + }); + + if (!matched) { + throw new Error(`No feed item matched slug "${slug}" — the feed may not have caught up with the post yet`); + } + + const enclosure = toEnclosure(matched); + + if (!enclosure) { + throw new Error(`No audio enclosure in the podcast feed for "${slug}" — check the episode was published with a media file attached`); + } + + console.log(`Resolved audio for ${slug}: ${enclosure.url} (${enclosure.length} bytes, ${enclosure.declaredType})`); + + return enclosure; +}; diff --git a/src/handlers.ts b/src/handlers.ts index e927ea1..ad8306e 100644 --- a/src/handlers.ts +++ b/src/handlers.ts @@ -1,7 +1,7 @@ import { env } from 'cloudflare:workers'; import { getLatestPost, getPublishedAt, WordPressPost } from './wordpress'; import { sendTelegramAudio, sendErrorNotification } from './telegram'; -import { getHTML, getAudioUrl } from './audio'; +import { getEnclosure } from './feed'; import { getLastSentAt, setLastSentAt } from './storage'; import { secretsMatch } from './utils'; @@ -20,10 +20,9 @@ const describeError = (error: unknown) => (error instanceof Error ? error.messag export const send = async (post: WordPressPost) => { console.log(`Starting send operation for post: ${post.slug}`); - const html = await getHTML(post.slug); - const audioSrc = getAudioUrl(html); + const enclosure = await getEnclosure(CATEGORY, post.slug); - await sendTelegramAudio(audioSrc); + await sendTelegramAudio(enclosure); console.log('Send operation completed successfully'); }; diff --git a/src/telegram.ts b/src/telegram.ts index 3c4c31e..058e230 100644 --- a/src/telegram.ts +++ b/src/telegram.ts @@ -1,11 +1,41 @@ import { env } from 'cloudflare:workers'; import { extractAudioFilename } from './utils'; +import type { Enclosure } from './feed'; import { parseBuffer } from 'music-metadata'; +// Telegram fetches the file itself at this size or below, so nothing passes +// through the worker. +const MAX_URL_SEND_BYTES = 20 * 1024 * 1024; + // Telegram rejects bot uploads larger than 50 MB, and the whole file is held in -// memory here, so oversized audio is rejected before it is downloaded. +// memory on the upload path, so oversized audio is rejected before download. const MAX_AUDIO_BYTES = 50 * 1024 * 1024; +const MIME_BY_EXTENSION: Record = { + mp3: 'audio/mpeg', + m4a: 'audio/mp4', + ogg: 'audio/ogg', + opus: 'audio/ogg', + wav: 'audio/wav', + flac: 'audio/flac', +}; + +const extensionOf = (url: string): string => { + try { + return new URL(url).pathname.split('.').pop()?.toLowerCase() ?? ''; + } catch { + return ''; + } +}; + +// The feed's declared type is unreliable — a .wav has been seen announced as +// audio/mpeg — so the extension wins. +const mimeFor = (url: string, declaredType: string): string => MIME_BY_EXTENSION[extensionOf(url)] || declaredType || 'application/octet-stream'; + +// extractAudioFilename only strips a .mp3 suffix, which leaves the extension +// visible in the Telegram title for any other format. +const stripExtension = (title: string): string => title.replace(/\.(mp3|m4a|ogg|opus|wav|flac)$/i, ''); + interface TelegramErrorResponse { description?: string; } @@ -36,16 +66,50 @@ const sendMessage = async (chatId: string, text: string) => { await assertTelegramOk(response, 'Telegram sendMessage'); }; -export const sendTelegramAudio = async (audioUrl: string) => { - if (!audioUrl || !audioUrl.startsWith('http')) { - throw new Error(`Invalid audio URL: ${audioUrl}`); +export const sendTelegramAudio = async (enclosure: Enclosure) => { + const { url, length, declaredType } = enclosure; + + if (!url || !url.startsWith('http')) { + throw new Error(`Invalid audio URL: ${url}`); + } + + const { filename, cleanTitle } = extractAudioFilename(url); + const title = stripExtension(cleanTitle); + const mimeType = mimeFor(url, declaredType); + + if (length > MAX_AUDIO_BYTES) { + throw new Error( + `Audio is too large to send: ${length} bytes exceeds the ${MAX_AUDIO_BYTES} byte limit. ` + + 'Re-publish the episode as an mp3 — a full-length WAV will always exceed this.' + ); + } + + // Preferred path: hand Telegram the URL. Downloading into the worker risks + // exhausting the 128 MB isolate long before the 50 MB guard below fires — a + // 104 MB WAV episode is what surfaced this. + if (length > 0 && length <= MAX_URL_SEND_BYTES) { + const response = await fetch(telegramUrl('sendAudio'), { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + chat_id: env.MAIN_CHAT_ID, + audio: url, + title, + }), + }); + + await assertTelegramOk(response, 'Telegram sendAudio (by URL)'); + console.log(`Sent audio by URL: ${url} (${length} bytes)`); + return; } - // Extract filename from URL - const { filename, cleanTitle } = extractAudioFilename(audioUrl); + // Fallback: buffer and upload. Used between 20 and 50 MB, or when the feed + // omitted the length and the URL limit cannot be ruled out. + console.log(`Buffering audio for upload: ${url} (${length || 'unknown'} bytes)`); - // Download the file and upload as multipart form data - const audioResponse = await fetch(audioUrl); + const audioResponse = await fetch(url); if (!audioResponse.ok) { throw new Error(`Failed to download audio: ${audioResponse.status} ${audioResponse.statusText}`); @@ -65,26 +129,26 @@ export const sendTelegramAudio = async (audioUrl: string) => { // Parse metadata let duration: number | undefined; - let title = cleanTitle; + let uploadTitle = title; let performer: string | undefined; try { const uint8Array = new Uint8Array(audioBuffer); - const metadata = await parseBuffer(uint8Array); + const metadata = await parseBuffer(uint8Array, mimeType); if (metadata.format.duration) { duration = Math.round(metadata.format.duration); } if (metadata.common.title) { - title = metadata.common.title; + uploadTitle = metadata.common.title; } if (metadata.common.artist) { performer = metadata.common.artist; } - console.log(`Extracted metadata - Duration: ${duration}s, Title: ${title}, Performer: ${performer}`); + console.log(`Extracted metadata - Duration: ${duration}s, Title: ${uploadTitle}, Performer: ${performer}`); } catch (error) { console.error('Failed to parse audio metadata:', error); // Fallback to defaults if parsing fails @@ -93,8 +157,8 @@ export const sendTelegramAudio = async (audioUrl: string) => { // Create multipart form data const formData = new FormData(); formData.append('chat_id', env.MAIN_CHAT_ID); - formData.append('audio', new Blob([audioBuffer], { type: 'audio/mpeg' }), filename); - formData.append('title', title); + formData.append('audio', new Blob([audioBuffer], { type: mimeType }), filename); + formData.append('title', uploadTitle); if (duration) { formData.append('duration', duration.toString()); diff --git a/tests/unit/feed.test.ts b/tests/unit/feed.test.ts new file mode 100644 index 0000000..70062b5 --- /dev/null +++ b/tests/unit/feed.test.ts @@ -0,0 +1,109 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { getEnclosure } from '../../src/feed'; + +const feed = (items: string) => `${items}`; + +const item = (slug: string, url: string, length = '8600000', type = 'audio/mpeg') => ` + + ${slug} + https://mhmic.org/fajrreminders/${slug}/ + https://mhmic.org/?p=1&slug=${slug} + + `; + +const respondWith = (body: string, init: ResponseInit = {}) => + vi.stubGlobal( + 'fetch', + vi.fn(async () => new Response(body, { status: 200, ...init })) + ); + +beforeEach(() => { + vi.spyOn(console, 'log').mockImplementation(() => {}); +}); + +afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); + +describe('getEnclosure', () => { + it('should return the enclosure for the item matching the slug', async () => { + respondWith( + feed(item('newest-one', 'https://media.blubrry.com/x/newest.mp3') + item('emaan-and-yaqeen', 'https://media.blubrry.com/x/emaan.mp3')) + ); + + const enclosure = await getEnclosure('FR', 'emaan-and-yaqeen'); + + expect(enclosure.url).toBe('https://media.blubrry.com/x/emaan.mp3'); + expect(enclosure.length).toBe(8_600_000); + expect(enclosure.declaredType).toBe('audio/mpeg'); + }); + + // Sending the newest item instead would deliver the *previous* episode's + // audio and mark the new post as sent, so the real audio would never go out. + it('should throw rather than fall back when the feed has not caught up with the slug', async () => { + respondWith(feed(item('newest-one', 'https://media.blubrry.com/x/newest.mp3'))); + + await expect(getEnclosure('FR', 'not-in-the-feed-yet')).rejects.toThrow(/No feed item matched slug "not-in-the-feed-yet"/); + }); + + // The regression that started this: a wav episode still has an enclosure even + // though the page renders no