diff --git a/api/TimeAddressableMediaStore.yaml b/api/TimeAddressableMediaStore.yaml index 0fdc484f..b9e03fb9 100644 --- a/api/TimeAddressableMediaStore.yaml +++ b/api/TimeAddressableMediaStore.yaml @@ -1044,18 +1044,30 @@ paths: video: summary: Video Flow - H.264 Codec externalValue: examples/flow-get-200-video-h264.json - audio: - summary: Audio Flow - AAC Codec - externalValue: examples/flow-get-200-audio-aac.json rawvideo: summary: Video Flow - Uncompressed (Quicktime) externalValue: examples/flow-get-200-video-raw.json - jpegimage: - summary: Image Flow - JPEG Codec - externalValue: examples/flow-get-200-image-jpeg.json + mxf_jpeg2k: + summary: Video Flow - JPEG-2000 (MXF container) + externalValue: examples/flow-get-200-video-jpeg-mxf.json + mxf_h264: + summary: Video Flow - H.264 (MXF container) + externalValue: examples/flow-get-200-video-h264-mxf.json + jp2_jpeg2k_singleframe: + summary: Video Flow - JPEG-2000 (JP2 container, single frame segments) + externalValue: examples/flow-get-200-video-jpeg-jp2.json video_vfr: summary: Video Flow - H.264 Codec, Variable Frame Rate externalValue: examples/flow-get-200-video-h264-vfr.json + audio: + summary: Audio Flow - AAC Codec + externalValue: examples/flow-get-200-audio-aac.json + audio_wav: + summary: Audio Flow - PCM + externalValue: examples/flow-get-200-audio-wav.json + jpegimage: + summary: Image Flow - JPEG Codec + externalValue: examples/flow-get-200-image-jpeg.json ttml: summary: Data Flow - TTML description: TAMS can also be used for storing non-AV content such as subtitles or event data @@ -2127,8 +2139,15 @@ paths: application/json: schema: $ref: schemas/flow-storage.json - example: - $ref: examples/flow-storage-post-201.json + examples: + mpegts: + summary: MPEG-TS container Flow + description: Flow using MPEG-TS as the container format, writing to a cloud object store + externalValue: examples/flow-storage-post-201.json + wav_proxy: + summary: WAV Flow + description: Flow using WAV as the container format, writing to an HTTP interface running on the same origin as the TAMS API + externalValue: examples/flow-storage-post-201-wav.json "400": description: Bad request. Invalid Flow storage request JSON or the Flow 'container' is not set. If object_ids supplied, some or all already exist. "403": diff --git a/api/examples/flow-get-200-audio-wav.json b/api/examples/flow-get-200-audio-wav.json new file mode 100644 index 00000000..2d5c2ac7 --- /dev/null +++ b/api/examples/flow-get-200-audio-wav.json @@ -0,0 +1,27 @@ +{ + "id": "fd25a9fc-3b58-4dc1-93d4-81c52b206562", + "source_id": "8af9d4a3-aff7-44e8-b384-d2bfc0b93533", + "generation": 0, + "created": "2025-02-24T11:33:46Z", + "metadata_updated": "2025-02-24T11:33:46Z", + "segments_updated": "2025-02-24T11:48:22Z", + "description": "Radio off-air recording", + "label": "Radio off-air", + "format": "urn:x-nmos:format:audio", + "created_by": "tams-dev", + "updated_by": "tams-dev", + "tags": { + "input_quality": "contribution" + }, + "codec": "audio/x-raw-int", + "container": "audio/wav", + "avg_bit_rate": 192, + "essence_parameters": { + "sample_rate": 48000, + "channels": 2, + "bit_depth": 16, + "unc_parameters": { + "unc_type": "interleaved" + } + } +} \ No newline at end of file diff --git a/api/examples/flow-get-200-video-h264-mxf.json b/api/examples/flow-get-200-video-h264-mxf.json new file mode 100644 index 00000000..885c0ca1 --- /dev/null +++ b/api/examples/flow-get-200-video-h264-mxf.json @@ -0,0 +1,47 @@ +{ + "id": "1491ecfb-813d-4453-9554-e417d03161ba", + "source_id": "3e6201e2-4b38-402a-a08f-e2529ec98229", + "generation": 1, + "created": "2026-02-24T09:35:25Z", + "metadata_updated": "2026-02-24T09:36:02Z", + "segments_updated": "2026-02-24T15:38:21Z", + "description": "Video Flow in a MXF container, H264 codec", + "label": "VFX Render - Proxy Quality", + "format": "urn:x-nmos:format:video", + "created_by": "tams-dev", + "updated_by": "tams-dev", + "tags": { + "input_quality": "web" + }, + "codec": "video/h264", + "container": "application/mxf", + "avg_bit_rate": 6000, + "essence_parameters": { + "frame_rate": { + "numerator": 50, + "denominator": 1 + }, + "frame_width": 1920, + "frame_height": 1080, + "bit_depth": 8, + "interlace_mode": "progressive", + "colorspace": "BT709", + "transfer_characteristic": "SDR", + "aspect_ratio": { + "numerator": 16, + "denominator": 9 + }, + "pixel_aspect_ratio": { + "numerator": 1, + "denominator": 1 + }, + "component_type": "YCbCr", + "vert_chroma_subs": 2, + "horiz_chroma_subs": 2, + "avc_parameters": { + "profile": 100, + "level": 31, + "flags": 0 + } + } +} \ No newline at end of file diff --git a/api/examples/flow-get-200-video-h264.json b/api/examples/flow-get-200-video-h264.json index 5fa12015..4f1fc28a 100644 --- a/api/examples/flow-get-200-video-h264.json +++ b/api/examples/flow-get-200-video-h264.json @@ -17,6 +17,9 @@ "codec": "video/h264", "container": "video/mp2t", "avg_bit_rate": 2479, + "segment_duration": { + "numerator": 10 + }, "essence_parameters": { "frame_rate": diff --git a/api/examples/flow-get-200-video-jpeg-jp2.json b/api/examples/flow-get-200-video-jpeg-jp2.json new file mode 100644 index 00000000..e8a5976d --- /dev/null +++ b/api/examples/flow-get-200-video-jpeg-jp2.json @@ -0,0 +1,46 @@ +{ + "id": "49628282-d3c6-4faa-a31f-d35cff98b26c", + "source_id": "3e6201e2-4b38-402a-a08f-e2529ec98229", + "generation": 0, + "created": "2026-02-24T09:25:32Z", + "metadata_updated": "2026-02-24T09:36:54Z", + "segments_updated": "2026-02-24T14:29:14Z", + "description": "Video Flow in a JP2 container, JPEG-2000 codec, single frame segments", + "label": "VFX Render - Full Quality, single frame", + "format": "urn:x-nmos:format:video", + "created_by": "tams-dev", + "updated_by": "tams-dev", + "tags": { + "input_quality": "intermediate" + }, + "codec": "video/jp2", + "container": "image/jp2", + "avg_bit_rate": 250000, + "segment_duration": { + "numerator": 1, + "denominator": 50 + }, + "essence_parameters": { + "frame_rate": { + "numerator": 50, + "denominator": 1 + }, + "frame_width": 1920, + "frame_height": 1080, + "bit_depth": 10, + "interlace_mode": "progressive", + "colorspace": "BT709", + "transfer_characteristic": "SDR", + "aspect_ratio": { + "numerator": 16, + "denominator": 9 + }, + "pixel_aspect_ratio": { + "numerator": 1, + "denominator": 1 + }, + "component_type": "YCbCr", + "vert_chroma_subs": 1, + "horiz_chroma_subs": 2 + } +} \ No newline at end of file diff --git a/api/examples/flow-get-200-video-jpeg-mxf.json b/api/examples/flow-get-200-video-jpeg-mxf.json new file mode 100644 index 00000000..88bb1651 --- /dev/null +++ b/api/examples/flow-get-200-video-jpeg-mxf.json @@ -0,0 +1,45 @@ +{ + "id": "1a670176-5b40-433b-9d66-8f90efc026b6", + "source_id": "3e6201e2-4b38-402a-a08f-e2529ec98229", + "generation": 0, + "created": "2026-02-24T09:35:25Z", + "metadata_updated": "2026-02-24T09:35:25Z", + "segments_updated": "2026-02-24T14:28:11Z", + "description": "Video Flow in a MXF container, JPEG-2000 codec", + "label": "VFX Render - Full Quality", + "format": "urn:x-nmos:format:video", + "created_by": "tams-dev", + "updated_by": "tams-dev", + "tags": { + "input_quality": "intermediate" + }, + "codec": "video/jp2", + "container": "application/mxf", + "avg_bit_rate": 250000, + "segment_duration": { + "numerator": 60 + }, + "essence_parameters": { + "frame_rate": { + "numerator": 50, + "denominator": 1 + }, + "frame_width": 1920, + "frame_height": 1080, + "bit_depth": 10, + "interlace_mode": "progressive", + "colorspace": "BT709", + "transfer_characteristic": "SDR", + "aspect_ratio": { + "numerator": 16, + "denominator": 9 + }, + "pixel_aspect_ratio": { + "numerator": 1, + "denominator": 1 + }, + "component_type": "YCbCr", + "vert_chroma_subs": 1, + "horiz_chroma_subs": 2 + } +} \ No newline at end of file diff --git a/api/examples/flow-storage-post-201-wav.json b/api/examples/flow-storage-post-201-wav.json new file mode 100644 index 00000000..8580d868 --- /dev/null +++ b/api/examples/flow-storage-post-201-wav.json @@ -0,0 +1,25 @@ +{ + "media_objects": [ + { + "object_id": "e0a3df95-d7f5-4991-be40-500aa22a1ce3", + "put_url": { + "url": "https://example.com/tams/object-proxy/500aa22a1ce3", + "content-type": "audio/wav" + } + }, + { + "object_id": "89f565ce-edd1-49fd-b6b7-900b6e4b545f", + "put_url": { + "url": "https://example.com/tams/object-proxy/900b6e4b545f", + "content-type": "audio/wav" + } + }, + { + "object_id": "45342cb7-5fc0-4223-b2a8-de05c16453d1", + "put_url": { + "url": "https://example.com/tams/object-proxy/de05c16453d1", + "content-type": "audio/wav" + } + } + ] +} diff --git a/api/examples/flows-get-200.json b/api/examples/flows-get-200.json index ce7dde3c..3678ecf5 100644 --- a/api/examples/flows-get-200.json +++ b/api/examples/flows-get-200.json @@ -133,5 +133,132 @@ "vert_chroma_subs": 1, "horiz_chroma_subs": 2 } + }, + { + "id": "1a670176-5b40-433b-9d66-8f90efc026b6", + "source_id": "3e6201e2-4b38-402a-a08f-e2529ec98229", + "generation": 0, + "created": "2026-02-24T09:35:25Z", + "metadata_updated": "2026-02-24T09:35:25Z", + "segments_updated": "2026-02-24T14:28:11Z", + "description": "Video Flow in a MXF container, JPEG-2000 codec", + "label": "VFX Render - Full Quality", + "format": "urn:x-nmos:format:video", + "created_by": "tams-dev", + "updated_by": "tams-dev", + "tags": + { + "input_quality": "intermediate" + }, + "codec": "video/jp2", + "container": "application/mxf", + "avg_bit_rate": 250000, + "essence_parameters": + { + "frame_rate": + { + "numerator": 50, + "denominator": 1 + }, + "frame_width": 1920, + "frame_height": 1080, + "bit_depth": 10, + "interlace_mode": "progressive", + "colorspace": "BT709", + "transfer_characteristic": "SDR", + "aspect_ratio": + { + "numerator": 16, + "denominator": 9 + }, + "pixel_aspect_ratio": + { + "numerator": 1, + "denominator": 1 + }, + "component_type": "YCbCr", + "vert_chroma_subs": 1, + "horiz_chroma_subs": 1 + } + }, + { + "id": "1491ecfb-813d-4453-9554-e417d03161ba", + "source_id": "3e6201e2-4b38-402a-a08f-e2529ec98229", + "generation": 1, + "created": "2026-02-24T09:35:25Z", + "metadata_updated": "2026-02-24T09:36:02Z", + "segments_updated": "2026-02-24T15:38:21Z", + "description": "Video Flow in a MXF container, H264 codec", + "label": "VFX Render - Proxy Quality", + "format": "urn:x-nmos:format:video", + "created_by": "tams-dev", + "updated_by": "tams-dev", + "tags": + { + "input_quality": "web" + }, + "codec": "video/h264", + "container": "application/mxf", + "avg_bit_rate": 6000, + "essence_parameters": + { + "frame_rate": + { + "numerator": 50, + "denominator": 1 + }, + "frame_width": 1920, + "frame_height": 1080, + "bit_depth": 8, + "interlace_mode": "progressive", + "colorspace": "BT709", + "transfer_characteristic": "SDR", + "aspect_ratio": + { + "numerator": 16, + "denominator": 9 + }, + "pixel_aspect_ratio": + { + "numerator": 1, + "denominator": 1 + }, + "component_type": "YCbCr", + "vert_chroma_subs": 2, + "horiz_chroma_subs": 2, + "avc_parameters": + { + "profile": 100, + "level": 31, + "flags": 0 + } + } + }, + { + "id": "fd25a9fc-3b58-4dc1-93d4-81c52b206562", + "source_id": "8af9d4a3-aff7-44e8-b384-d2bfc0b93533", + "generation": 0, + "created": "2025-02-24T11:33:46Z", + "metadata_updated": "2025-02-24T11:33:46Z", + "segments_updated": "2025-02-24T11:48:22Z", + "description": "Radio off-air recording", + "label": "Radio off-air", + "format": "urn:x-nmos:format:audio", + "created_by": "tams-dev", + "updated_by": "tams-dev", + "tags": { + "input_quality": "contribution" + }, + "codec": "audio/x-raw-int", + "container": "audio/wav", + "avg_bit_rate": 192, + "essence_parameters": { + "sample_rate": 48000, + "channels": 2, + "bit_depth": 16, + "unc_parameters": { + "unc_type": "interleaved" + } + } } ] \ No newline at end of file diff --git a/examples/.gitignore b/examples/.gitignore index 0d623630..85298e49 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -1,6 +1,7 @@ -bbb_sunflower_1080p_30fps_normal.mp4 -bbb_sunflower_1080p_30fps_normal.mp4.zip +bbb_sunflower_1080p_30fps_normal* +tams-edit-test-film* sample_content/ +sample_content_segments/ venv/ __pycache__/ output.ts diff --git a/examples/Makefile b/examples/Makefile index 3381d34c..bd75b0b2 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -5,9 +5,6 @@ venv: python3 -m venv venv . venv/bin/activate && pip install -r requirements.txt && deactivate -sample_content: - ./hls_sample_content.sh - env_exports: @echo "export OAUTH2_URL=" @echo "export CLIENT_ID=" @@ -17,15 +14,41 @@ env_exports: clean: @rm -rf venv - @rm -f bbb_sunflower_1080p_30fps_normal.mp4 - @rm -f bbb_sunflower_1080p_30fps_normal.mp4.zip + @rm -f bbb_sunflower_1080p_30fps_normal* + @rm -f tams-edit-test-film.ts + @rm -rf sample_content_segments @rm -rf sample_content +sample_content_segments: + mkdir -p $@ + +tams-edit-test-film.ts: + wget -O $@ https://object.lon1.bbcis.uk/a4c52e22f45b4a0fa3ea3b9a42b35808:test-content/tams-edit-test-film-v4.ts + +bbb_sunflower_1080p_30fps_normal.mp4: + curl -o bbb_sunflower_1080p_30fps_normal.mp4.zip -s https://download.blender.org/demo/movies/BBB/bbb_sunflower_1080p_30fps_normal.mp4.zip + unzip -d . bbb_sunflower_1080p_30fps_normal.mp4.zip + rm bbb_sunflower_1080p_30fps_normal.mp4.zip + +sample_content_segments/hls_output.m3u8: bbb_sunflower_1080p_30fps_normal.mp4 sample_content_segments + ffmpeg -i bbb_sunflower_1080p_30fps_normal.mp4 -c:v copy -muxdelay 0 -an -f hls -hls_time 5 -hls_playlist_type vod $@ + +sample_content_segments/wav_pcm_flow.list: tams-edit-test-film.ts sample_content_segments + ffmpeg -i $< -vn -c:a pcm_s16le -f segment -segment_list $@ -segment_time 10 sample_content_segments/wav_pcm_flow_%03d.wav + +sample_content_segments/mov_h264_flow.list: tams-edit-test-film.ts sample_content_segments + ffmpeg -i $< -an -c:v copy -f segment -segment_list $@ -segment_time 60 sample_content_segments/mov_h264_flow_%03d.mov + +fetch_sample_media: tams-edit-test-film.ts bbb_sunflower_1080p_30fps_normal.mp4 + +sample_content: sample_content_segments/hls_output.m3u8 sample_content_segments/wav_pcm_flow.list sample_content_segments/mov_h264_flow.list + help: @echo "TAMS examples" @echo "make venv - Prepare a Python virtual environment in venv/ for running the examples" - @echo "make sample_content - Download and prepare content into sample_content/" + @echo "make fetch_sample_media - Download sample media files" + @echo "make sample_content - Download and prepare content into sample_content_segments/" @echo "make env_exports - Print environment variable exports that may used in the examples" @echo "make clean - Delete files that were created" -.PHONY: all help clean env_exports +.PHONY: all help clean env_exports fetch_sample_media sample_content diff --git a/examples/README.md b/examples/README.md index 511cb33f..4ca2c126 100644 --- a/examples/README.md +++ b/examples/README.md @@ -20,9 +20,13 @@ The following is required to run the example scripts: ### Sample Content -The [Big Buck Bunny](https://peach.blender.org/) short film is used as sample content for ingesting into TAMS. -Run `make sample_content` to download a file containing 30 fps HD and convert it to a video-only HLS playlist using ffmpeg. -The result can be found in the `sample_content/` folder. +A number of short films are provided as sample content for ingesting into TAMS. +Run `make sample_content` to download them and generate a number of playlists using `ffmpeg`. +The result can be found in the `sample_content_segments/` folder. + +- `sample_content_segments/hls_output.m3u8`: Big Buck Bunny as a video-only HLS playlist with 5-second segments +- `sample_content_segments/mov_h264_flow.list`: Short test film as a single H.264 video segment in a .mov container +- `sample_content_segments/wav_pcm_flow.list`: Audio from short test film as a series of .wav files containing PCM ### Virtual Environment @@ -81,7 +85,7 @@ The script makes some assumptions about the media content (e.g. the resulting Fl Run the script as follows (replace ``), ```bash -./ingest_hls.py --tams-url --hls-filename sample_content/hls_output.m3u8 +./ingest_hls.py --tams-url --hls-filename sample_content_segments/hls_output.m3u8 ``` The output Flow ID is logged as well as each segment timerange that is ingested from the HLS playlist. @@ -91,7 +95,7 @@ By default at most 30 segments will be ingested. The script follows these steps: * a new Flow is created with (hardcoded) properties that match the sample content -* the segment filenames are extracted from the playlist `sample_content/hls_output.m3u8` +* the segment filenames are extracted from the playlist `sample_content_segments/hls_output.m3u8` * each segment media file is read to extract the timerange * each segment media file is uploaded using the pre-signed URLs provided by the TAMS * each segment is registered in TAMS @@ -100,6 +104,12 @@ The script also has args to * change the start segment (`--hls-start-segment`) and number of segments (`--hls-segment-count`) limit, i.e. override the default 30 segment limit * set the Flow ID (`--flow-id`) and Source ID (`--source-id`) +* read a list of filenames rather than an HLS manifest (`--filename` and `--use-simple-list`) to ingest with formats other than HLS and MPEG-TS +* force the start time of the ensuing Flow (`--force-start-time`): will also force the timing of subsequent segments to make a contiguous Flow, regardless of internal timing + +The sample content also contains additional sets of segmented material, to demonstrate other codecs and container formats: +* H.264 video, MOV container, single segment: `./ingest_hls.py --tams-url --filename sample_content_segments/mov_h264_flow.list --use-simple-list --flow-params '{"label":"Demo Flow - MOV container","description":"Flow created to demonstrate manual upload of a single-segment Flow in a MOV container","format":"urn:x-nmos:format:video","codec":"video/h264","container":"video/quicktime","essence_parameters":{"frame_rate":{"numerator":50,"denominator":1},"frame_width":1920,"frame_height":1080,"bit_depth":8,"interlace_mode":"progressive","component_type":"YCbCr","horiz_chroma_subs":2,"vert_chroma_subs":2}}'` +* PCM audio, WAV container: `./ingest_hls.py --tams-url --filename sample_content_segments/wav_pcm_flow.list --use-simple-list --force-start-time "0:0" --flow-params '{"label":"Demo Flow - WAV container","description":"Flow created to demonstrate manual upload of PCM audio in a WAV container","format":"urn:x-nmos:format:audio","codec":"audio/x-raw-int","container":"audio/wav","essence_parameters":{"sample_rate":48000,"channels":2,"bit_depth":16,"unc_parameters":{"unc_type":"interleaved"}}}'` > The timerange extraction process as implemented is not optimal in terms of speed (e.g. it doesn't need to read all the frames) but at least it is more accurate than using the segment durations from the HLS playlist. > @@ -135,6 +145,8 @@ The script follows these steps: * the media timing is adjusted using the segment `ts_offset`, `sample_offset` and `sample_count` properties as required as well as timestamp rollover within the segment time period * the media is re-wrapped to the local MPEG-TS file +This script also supports the other container examples described above, but may not correctly support all codecs, containers and Flows. + > The script has not been optimised to download segments concurrently. ### Simple Edit ([simple_edit.py](./simple_edit.py)) diff --git a/examples/hls_sample_content.sh b/examples/hls_sample_content.sh deleted file mode 100755 index 6e6a726d..00000000 --- a/examples/hls_sample_content.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) - -if [ ! -f ${SCRIPT_DIR}/bbb_sunflower_1080p_30fps_normal.mp4 ] -then - curl -o ${SCRIPT_DIR}/bbb_sunflower_1080p_30fps_normal.mp4.zip -s https://download.blender.org/demo/movies/BBB/bbb_sunflower_1080p_30fps_normal.mp4.zip - unzip -d ${SCRIPT_DIR} ${SCRIPT_DIR}/bbb_sunflower_1080p_30fps_normal.mp4.zip -fi - -mkdir -p ${SCRIPT_DIR}/sample_content - -# The '-muxdelay 0' option below is used to avoid the 1.4 second delay added by the ffmpeg mpegts muxer. -# Note that the presentation start time of bbb_sunflower_1080p_30fps_normal.mp4 is 0.066667, not 0. - -ffmpeg -i ${SCRIPT_DIR}/bbb_sunflower_1080p_30fps_normal.mp4 -c:v copy -muxdelay 0 -an -f hls -hls_time 5 -hls_playlist_type vod ${SCRIPT_DIR}/sample_content/hls_output.m3u8 diff --git a/examples/ingest_hls.py b/examples/ingest_hls.py index 12306ce7..f3d23e24 100755 --- a/examples/ingest_hls.py +++ b/examples/ingest_hls.py @@ -30,7 +30,7 @@ "container": "video/mp2t", "essence_parameters": { "frame_rate": { - "numerator": 50, + "numerator": 30, "denominator": 1 }, "frame_width": 1920, @@ -66,8 +66,8 @@ async def put_flow( credentials, f"{tams_url}/flows/{flow_id}", json=flow_metadata - ) as resp: - resp.raise_for_status() + ): + pass # Context manager will raise on failure async def get_media_storage_urls( @@ -87,8 +87,7 @@ async def get_media_storage_urls( "limit": segment_count } ) as resp: - resp.raise_for_status() - + # Context manager will raise on failure media_storage = await resp.json() media_object_urls = media_storage["media_objects"] @@ -104,6 +103,13 @@ def get_hls_segment_filenames(hls_filename: str) -> Generator[str, None, None]: yield segment.uri +def get_segment_list_filenames(list_filename: str) -> Generator[str, None, None]: + """Return list of filenames from a plain list (rather than an HLS manifest above)""" + with open(list_filename, "r") as fp: + for line in fp: + yield line.rstrip() + + def extract_segment_timerange(filename: str) -> TimeRange: """Extract the presentation timerange from the media object @@ -153,15 +159,26 @@ async def ingest_segment( tams_url: str, flow_id: UUID, object_url: dict[str, Any], - filename: str -) -> None: - """Upload the segment's media object and register the segment""" - seg_tr = await asyncio.get_running_loop().run_in_executor( + filename: str, + start_timestamp_in_flow: Optional[Timestamp] = None +) -> TimeRange: + """Upload the segment's media object and register the segment + + Returns the ingested segment timerange + """ + media_tr = await asyncio.get_running_loop().run_in_executor( None, extract_segment_timerange, filename ) + if start_timestamp_in_flow: + seg_tr = TimeRange.from_start_length(start_timestamp_in_flow, media_tr.length, TimeRange.INCLUDE_START) + ts_offset = start_timestamp_in_flow - media_tr.start + else: + seg_tr = media_tr + ts_offset = Timestamp(0, 0) + first_object_url = object_url["put_url"]["url"] content_type = object_url["put_url"]["content-type"] with open(filename, "rb") as f: @@ -174,7 +191,7 @@ async def ingest_segment( ) as resp: resp.raise_for_status() - logger.info(f"Uploaded object to {object_url['object_id']}") + logger.info(f"Uploaded object to {object_url['object_id']} for timerange {seg_tr}") async with post_request( session, @@ -182,60 +199,75 @@ async def ingest_segment( f"{tams_url}/flows/{flow_id}/segments", json=mediajson.encode_value({ "object_id": object_url['object_id'], - "timerange": seg_tr + "timerange": seg_tr, + "ts_offset": ts_offset }) - ) as resp: - resp.raise_for_status() + ): + pass # Context manager will raise on failure logger.info(f"Created flow segment for {object_url['object_id']} at {seg_tr.to_sec_nsec_range()}") + return seg_tr -async def hls_ingest( +async def segment_ingest( tams_url: str, credentials: Credentials, - hls_filename: str, - hls_start_segment: int, - hls_segment_count: int, + manifest_filename: str, + start_segment: int, + segment_count: int, flow_id: UUID, source_id: UUID, - flow_params: Optional[dict] + flow_params: Optional[dict], + hls_mode: bool = True, + sequence_force_start_time: Optional[Timestamp] = None ) -> None: """Upload segments from the HLS playlist""" async with aiohttp.ClientSession(trust_env=True) as session: await put_flow(session, credentials, tams_url, flow_id, source_id, flow_params) - object_urls = get_media_storage_urls(session, credentials, tams_url, flow_id, hls_segment_count) + object_urls = get_media_storage_urls(session, credentials, tams_url, flow_id, segment_count) + + if hls_mode: + segment_filenames = get_hls_segment_filenames(manifest_filename) + else: + segment_filenames = get_segment_list_filenames(manifest_filename) - hls_segment_filenames = get_hls_segment_filenames(hls_filename) + position_in_flow = None + if sequence_force_start_time: + position_in_flow = sequence_force_start_time # This sequential upload process could be optimised by using asyncio tasks to # ingest segments concurrently count = 0 - for segment_filename in hls_segment_filenames: + for segment_filename in segment_filenames: count += 1 - if count <= hls_start_segment: + if count <= start_segment: continue - elif count > hls_start_segment + hls_segment_count: + elif count > start_segment + segment_count: break object_url = await anext(object_urls) - full_segment_filename = os.path.join(os.path.dirname(hls_filename), segment_filename) + full_segment_filename = os.path.join(os.path.dirname(manifest_filename), segment_filename) - await ingest_segment( + seg_tr = await ingest_segment( session, credentials, tams_url, flow_id, object_url, - full_segment_filename + full_segment_filename, + start_timestamp_in_flow=position_in_flow # Will be None if not used ) + if position_in_flow: + position_in_flow += seg_tr.length + if __name__ == "__main__": parser = ArgumentParser( prog="ingest_hls", - description="TAMS Flow ingest from HLS basic example" + description="TAMS Flow ingest from segment manifest example" ) parser.add_argument( @@ -262,18 +294,39 @@ async def hls_ingest( "--password", type=str, default=os.environ.get("PASSWORD"), help="Basic auth password. Defaults to the 'PASSWORD' environment variable" ) + + # Manifest input arguments parser.add_argument( - "--hls-filename", type=str, default="sample_content/hls_output.m3u8", - help="HLS playlist providing segment files" + "--filename", type=str, default="sample_content_segments/hls_output.m3u8", + help="File or HLS playlist providing segment files" ) parser.add_argument( - "--hls-start-segment", type=int, default=0, + "--use-simple-list", "-S", action="store_true", + help="Interpret the input file as a simple list of filenames, rather than an HLS manifest" + ) + parser.add_argument( + "--start-segment", type=int, default=0, help="Segment number to start ingesting from" ) parser.add_argument( - "--hls-segment-count", type=int, default=30, + "--segment-count", type=int, default=30, help="Maximum number of segments to ingest" ) + + # HLS-specific arguments (aliases of above) + parser.add_argument( + "--hls-filename", type=str, + help="HLS playlist providing segment files. Alias for `--filename`" + ) + parser.add_argument( + "--hls-start-segment", type=int, default=0, + help="Segment number to start ingesting from. Alias for `--start-segment`" + ) + parser.add_argument( + "--hls-segment-count", type=int, default=30, + help="Maximum number of segments to ingest. Alias for `--segment-count`" + ) + parser.add_argument( "--flow-id", type=UUID, help="Flow ID for the sample content. Default is to generate an ID" @@ -286,6 +339,10 @@ async def hls_ingest( "--flow-params", type=json.loads, help="JSON representation of Flow to write. Default is a basic video Flow" ) + parser.add_argument( + "--force-start-time", type=Timestamp.from_str, + help="Ignore timestamps in the input and ingest from this point, sequentially" + ) args = parser.parse_args() @@ -300,13 +357,20 @@ async def hls_ingest( "or basic credentials (--username, --password)" ) - output_timerange = asyncio.run(hls_ingest( + list_filename = args.hls_filename or args.filename + start_segment = args.hls_start_segment or args.start_segment + segment_count = args.hls_segment_count or args.segment_count + hls_mode = (args.hls_filename is not None) or not args.use_simple_list + + output_timerange = asyncio.run(segment_ingest( args.tams_url.rstrip("/"), credentials, - args.hls_filename, - args.hls_start_segment, - args.hls_segment_count, + list_filename, + start_segment, + segment_count, args.flow_id or uuid4(), args.source_id or uuid4(), - args.flow_params + args.flow_params, + hls_mode=hls_mode, + sequence_force_start_time=args.force_start_time )) diff --git a/examples/outgest_file.py b/examples/outgest_file.py index 43371676..f1c16114 100755 --- a/examples/outgest_file.py +++ b/examples/outgest_file.py @@ -148,7 +148,7 @@ def normalise_and_transfer_media( discarding_samples = skip_start_duration > 0 or skip_end_duration > 0 output_timerange = TimeRange.never() first_packet = True - with av.open(media_essence, mode="r", format="mpegts") as av_input: + with av.open(media_essence, mode="r") as av_input: for pkt in av_input.demux(**demux_kwargs): if pkt.dts is None and pkt.pts is None: continue @@ -271,12 +271,12 @@ async def outgest_file( if "container" not in flow: raise NotImplementedError("Flow without a container is not supported") if flow["container"] != "video/mp2t": - raise NotImplementedError(f"Flow container '{flow['container']}' is not supported") + logger.warning(f"Non-MPEGTS containers may work but are not well tested (Container is '{flow['container']}')") if flow["format"] not in ["urn:x-nmos:format:video", "urn:x-nmos:format:audio"]: raise NotImplementedError(f"Flow format '{flow['format']}' is not supported") output_timerange = TimeRange.never() - with av.open(output_filename, mode="w", format="mpegts") as av_output: + with av.open(output_filename, mode="w") as av_output: async with aiohttp.ClientSession(trust_env=True) as media_object_session: async for segment in get_flow_segments(tams_url, credentials, flow, timerange): try: diff --git a/examples/simple_edit.py b/examples/simple_edit.py index 2a25854a..46b9268e 100755 --- a/examples/simple_edit.py +++ b/examples/simple_edit.py @@ -62,8 +62,8 @@ async def put_flow( credentials, f"{tams_url}/flows/{flow_id}", json=flow_metadata - ) as resp: - resp.raise_for_status() + ): + pass # Context manager will raise on failure async def get_segments( @@ -79,7 +79,6 @@ async def get_segments( credentials, f"{tams_url}/flows/{flow_id}/segments?timerange={timerange!s}" ) as resp: - resp.raise_for_status() return await resp.json() @@ -94,7 +93,6 @@ async def get_flow( credentials, f"{tams_url}/flows/{flow_id}" ) as resp: - resp.raise_for_status() return await resp.json() @@ -125,7 +123,6 @@ async def simple_edit( "timerange": segment["timerange"] }) ) as resp: - resp.raise_for_status() print(f"Added segment from Flow {input_1_flow_id} from and to timerange {segment['timerange']}") # Add segments from input 2 to output after the input 1 segments @@ -174,7 +171,6 @@ async def simple_edit( "ts_offset": new_ts_offset }) ) as resp: - resp.raise_for_status() print(f"Added segment from Flow {input_2_flow_id} and timerange " "{segment['timerange']} to {new_seg_tr!s}") @@ -274,7 +270,6 @@ async def interval_edit( f"{tams_url}/flows/{output_flow_id}/segments", json=mediajson.encode_value(new_segment) ) as resp: - resp.raise_for_status() print(f"Added segment from Flow {current_seg['id']} and timerange {next_seg_tr} to {new_seg_tr!s}") # Advance our time pointer diff --git a/examples/utils/client.py b/examples/utils/client.py index 31b6b60a..a2064798 100644 --- a/examples/utils/client.py +++ b/examples/utils/client.py @@ -1,20 +1,35 @@ # This file provides functions to make HTTP requests to the TAMS API. # The functions include a retry on authentication failures when using renewable credentials. -from typing import AsyncGenerator +import dataclasses +from typing import AsyncGenerator, Optional from contextlib import asynccontextmanager import aiohttp +import aiohttp.client_exceptions from .credentials import Credentials, RenewableCredentials +@dataclasses.dataclass +class TAMSClientException(Exception): + error: aiohttp.client_exceptions.ClientResponseError + body: Optional[str] + + def __str__(self) -> str: + return f"{str(self.error)}, body={self.body}" + + def __repr__(self) -> str: + return f"{type(self).__name__}({self.error.__repr__}, body={self.body})" + + @asynccontextmanager async def request( session: aiohttp.ClientSession, credentials: Credentials, method: str, url: str, + raise_on_error: bool = True, **kwargs ) -> AsyncGenerator[aiohttp.ClientResponse, None]: """Execute a request and retry once if there is a credentials failure""" @@ -32,6 +47,11 @@ async def request( while True: async with session.request(method, url, headers=in_headers | credentials.header(), **kwargs) as resp: if resp.status != 401 or not isinstance(credentials, RenewableCredentials) or have_retried: + if raise_on_error: + try: + resp.raise_for_status() + except aiohttp.client_exceptions.ClientResponseError as e: + raise TAMSClientException(e, await resp.text()) yield resp break @@ -45,10 +65,11 @@ async def get_request( session: aiohttp.ClientSession, credentials: Credentials, url: str, + raise_on_error: bool = True, **kwargs ) -> AsyncGenerator[aiohttp.ClientResponse, None]: """Execute a GET request and retry once if there is a credentials failure""" - async with request(session, credentials, "GET", url, **kwargs) as resp: + async with request(session, credentials, "GET", url, raise_on_error=raise_on_error, **kwargs) as resp: yield resp @@ -57,10 +78,11 @@ async def put_request( session: aiohttp.ClientSession, credentials: Credentials, url: str, + raise_on_error: bool = True, **kwargs ) -> AsyncGenerator[aiohttp.ClientResponse, None]: """Execute a PUT request and retry once if there is a credentials failure""" - async with request(session, credentials, "PUT", url, **kwargs) as resp: + async with request(session, credentials, "PUT", url, raise_on_error=raise_on_error, **kwargs) as resp: yield resp @@ -70,10 +92,11 @@ async def post_request( credentials: Credentials, url: str, json: dict = {}, + raise_on_error: bool = True, **kwargs ) -> AsyncGenerator[aiohttp.ClientResponse, None]: """Execute a POST request and retry once if there is a credentials failure""" - async with request(session, credentials, "POST", url, json=json, **kwargs) as resp: + async with request(session, credentials, "POST", url, json=json, raise_on_error=raise_on_error, **kwargs) as resp: yield resp @@ -82,8 +105,9 @@ async def delete_request( session: aiohttp.ClientSession, credentials: Credentials, url: str, + raise_on_error: bool = True, **kwargs ) -> AsyncGenerator[aiohttp.ClientResponse, None]: """Execute a DELETE request and retry once if there is a credentials failure""" - async with request(session, credentials, "DELETE", url, **kwargs) as resp: + async with request(session, credentials, "DELETE", url, raise_on_error=raise_on_error, **kwargs) as resp: yield resp