From 0be6382723a1d9f0e28c7cc093bf1eb5bed6f9b1 Mon Sep 17 00:00:00 2001 From: Pavlo Strunkin Date: Mon, 2 Nov 2026 22:54:07 +0200 Subject: [PATCH 1/3] feat: add sourceBranch, customTags, comment, merge, multipart upload and 503 retry - Config.sourceBranch (VRT_SOURCEBRANCH, vrt.json "sourceBranch"): branch whose baselines the run is compared with; sent with every test run when set. - TestRun.customTags, comment and merge, also in the Playwright track options. - TestRun.imagePath (file) or imageBuffer (bytes) upload the image to /test-runs/multipart; imageBase64 still posts JSON. More than one image field raises ValueError. - The Playwright integration uploads its screenshots as multipart. - A 503 with Retry-After of at most 60 s is retried up to 3 times; the final ServerError says so. - 413 hints name the multipart limit or suggest multipart for JSON uploads. Co-Authored-By: Claude Opus 5.5 --- README.md | 29 ++- tests/test_config.py | 8 + tests/test_playwright.py | 29 +-- tests/test_playwright_async.py | 29 +-- tests/test_visualRegressionTracker.py | 191 +++++++++++++++++- visual_regression_tracker/config.py | 2 + visual_regression_tracker/playwright.py | 106 +++++----- visual_regression_tracker/types.py | 6 + .../visualRegressionTracker.py | 93 +++++++-- 9 files changed, 378 insertions(+), 115 deletions(-) diff --git a/README.md b/README.md index 384d9b9..d37c005 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ python -m playwright install | `ciBuildId` | `VRT_CIBUILDID` | none | CI build ID, such as the commit SHA; runs with the same ID share a build | | `enableSoftAssert` | `VRT_ENABLESOFTASSERT` | `false` | Log a failed check instead of raising `TestRunError` | | `httpTimeoutInSeconds` | `VRT_HTTPTIMEOUTINSECONDS` | `360` | How long to wait for the server's answer, `0` for no limit. Connecting is limited to 10 s. | +| `sourceBranch` | `VRT_SOURCEBRANCH` | the project's main branch | Branch whose baselines `branchName` is compared with, such as the target branch of a pull request | The server compares images while it answers `track()`. With VLM checks or a busy comparison queue that can take minutes, so keep `httpTimeoutInSeconds` above 300. @@ -55,7 +56,8 @@ override it. `Config.default(path)` reads another file. "branchName": "develop", "ciBuildId": "commit_sha", "enableSoftAssert": false, - "httpTimeoutInSeconds": 360 + "httpTimeoutInSeconds": 360, + "sourceBranch": "main" } ``` @@ -67,6 +69,7 @@ VRT_BRANCHNAME="develop" \ VRT_CIBUILDID="commit_sha" \ VRT_ENABLESOFTASSERT=true \ VRT_HTTPTIMEOUTINSECONDS=360 \ +VRT_SOURCEBRANCH="main" \ python -m pytest ``` @@ -91,10 +94,17 @@ Without the context manager, call `build = vrt.start()` (returns `Build(id, proj | Field | Meaning | |---|---| | `name` | Required. Name of the image in VRT | -| `imageBase64` | Required. The PNG image, base64 encoded | -| `os`, `browser`, `viewport`, `device` | Optional. Together with `name` they pick the baseline | +| `imageBase64` | The PNG image, base64 encoded, sent as JSON | +| `imagePath` | Or: path of a PNG file, sent as multipart | +| `imageBuffer` | Or: the PNG image as `bytes`, sent as multipart | +| `os`, `browser`, `viewport`, `device`, `customTags` | Optional. Together with `name` they pick the baseline | | `diffTollerancePercent` | Optional. Allowed difference in %, default 0. The spelling is the API's. | | `ignoreAreas` | Optional. List of `IgnoreArea(x, y, width, height)` in pixels | +| `comment` | Optional. Shown with the test run in the VRT app | +| `merge` | Optional. `True`: approving this run on a feature branch updates the baseline of the branch it was compared with | + +Set exactly one of the image fields. Multipart avoids the server's JSON size limit (`BODY_PARSER_JSON_LIMIT`) +and the base64 overhead, so prefer it for large screenshots. ### Playwright @@ -112,15 +122,15 @@ with sync_playwright() as playwright: vrt.trackElementHandle(page.query_selector('#touchnav-wrapper'), 'Navigation') ``` -`browser` is the Playwright browser name and `viewport` the page's viewport size. The async API has +The screenshot is sent as multipart. `browser` is the Playwright browser name and `viewport` the page's viewport size. The async API has `trackPageAsync` and `trackElementHandleAsync`. The last argument is optional: `PageTrackOptions` for `trackPage`, `ElementHandleTrackOptions` for `trackElementHandle`. Both take: -- `diffTollerancePercent` and `ignoreAreas`, as in `TestRun`; -- `agent: Agent(os, device, viewport)`: baseline fields Playwright doesn't know. `viewport` is used only for - element handles; +- `diffTollerancePercent`, `ignoreAreas`, `customTags`, `comment` and `merge`, as in `TestRun`; +- `agent: Agent(os, device, viewport)`: baseline fields Playwright doesn't know. `viewport` is used for + element handles and for pages without a viewport size; - `screenshotOptions`: arguments for Playwright's `screenshot()`. `PageScreenshotOptions` takes `full_page`, `clip`, `omit_background`, `timeout`, `type`, `quality` and `path`. `ElementHandleScreenshotOptions` takes the same, without `full_page` and `clip`. @@ -158,6 +168,11 @@ All errors derive from `VisualRegressionTrackerError`. | `MissingConfigurationError` | A required option is not set. Has the `field`. | | `InvalidConfigurationError` | An option has an invalid value. Has the `field`. | | `VisualRegressionTrackerError` | `track` or `stop` before `start`. | +| `ValueError` | A `TestRun` with more than one image field. | + +When the server answers 503 with a `Retry-After` of at most 60 seconds (the comparison queue is full, or the +server is restarting), the SDK waits that long and sends the request again, up to 3 times, logging a warning +each time. After that, or without a usable `Retry-After`, it raises `ServerError`. ## Development diff --git a/tests/test_config.py b/tests/test_config.py index b07f338..a3f1ba1 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -19,6 +19,7 @@ def config_file(tmpdir): 'apiKey': 'file api key', 'enableSoftAssert': True, 'httpTimeoutInSeconds': 5, + 'sourceBranch': 'file source branch', })) yield p @@ -33,6 +34,7 @@ def config_env(): 'VRT_APIKEY': 'env api key', 'VRT_ENABLESOFTASSERT': 'False', 'VRT_HTTPTIMEOUTINSECONDS': '7.5', + 'VRT_SOURCEBRANCH': 'env source branch', } yield env @@ -46,6 +48,7 @@ def test_config_from_file(config_file): assert cfg.apiKey == 'file api key' assert cfg.enableSoftAssert == True assert cfg.httpTimeoutInSeconds == 5 + assert cfg.sourceBranch == 'file source branch' def test_config_from_environment(config_env): @@ -57,6 +60,7 @@ def test_config_from_environment(config_env): assert cfg.apiKey == 'env api key' assert cfg.enableSoftAssert == False assert cfg.httpTimeoutInSeconds == 7.5 + assert cfg.sourceBranch == 'env source branch' def config_from_file(tmp_path, data): @@ -69,6 +73,10 @@ def test_config_timeout_defaults_to_360_seconds(): assert Config().httpTimeoutInSeconds == 360 +def test_config_has_no_source_branch_by_default(): + assert Config().sourceBranch is None + + def test_config_treats_empty_timeout_variable_as_unset(): assert Config.from_environment({'VRT_HTTPTIMEOUTINSECONDS': ''}).httpTimeoutInSeconds == 360 diff --git a/tests/test_playwright.py b/tests/test_playwright.py index 0801f3b..f983877 100644 --- a/tests/test_playwright.py +++ b/tests/test_playwright.py @@ -1,6 +1,4 @@ import pytest -import base64 -import binascii import subprocess import sys import visual_regression_tracker @@ -94,13 +92,14 @@ def test_pvrt_trackPage__default_options(pvrt, sync_page, sync_browserType): testRun = pvrt.track.call_args[0][0] assert testRun.name == 'image name' - assert is_base64(testRun.imageBase64) and len(testRun.imageBase64) > 1024 + assert testRun.imageBase64 is None and is_image(testRun.imageBuffer) assert testRun.os is None assert testRun.browser == sync_browserType.name assert testRun.viewport == '1024x768' assert testRun.device is None assert testRun.diffTollerancePercent is None assert testRun.ignoreAreas is None + assert (testRun.customTags, testRun.comment, testRun.merge) == (None, None, None) def test_pvrt_trackPage(pvrt, sync_page, sync_browserType): @@ -120,6 +119,9 @@ def test_pvrt_trackPage(pvrt, sync_page, sync_browserType): omit_background=False, full_page=True, ), + customTags='tag', + comment='why', + merge=True, agent=visual_regression_tracker.playwright.Agent( os='os', device='device', @@ -132,12 +134,13 @@ def test_pvrt_trackPage(pvrt, sync_page, sync_browserType): testRun = pvrt.track.call_args[0][0] assert testRun.name == 'image name' - assert is_base64(testRun.imageBase64) and len(testRun.imageBase64) > 1024 + assert testRun.imageBase64 is None and is_image(testRun.imageBuffer) assert testRun.os == 'os' assert testRun.browser == sync_browserType.name assert testRun.viewport == '1024x768' assert testRun.device == 'device' assert testRun.diffTollerancePercent == 10 + assert (testRun.customTags, testRun.comment, testRun.merge) == ('tag', 'why', True) assert testRun.ignoreAreas == [ visual_regression_tracker.types.IgnoreArea(1, 2, 3, 4), ] @@ -156,13 +159,14 @@ def test_pvrt_trackElementHandle__default_options(pvrt, sync_elementHandle, sync testRun = pvrt.track.call_args[0][0] assert testRun.name == 'image name' - assert is_base64(testRun.imageBase64) and len(testRun.imageBase64) > 1024 + assert testRun.imageBase64 is None and is_image(testRun.imageBuffer) assert testRun.os is None assert testRun.browser == sync_browserType.name assert testRun.viewport is None assert testRun.device is None assert testRun.diffTollerancePercent is None assert testRun.ignoreAreas is None + assert (testRun.customTags, testRun.comment, testRun.merge) == (None, None, None) def test_pvrt_trackElementHandle(pvrt, sync_elementHandle, sync_browserType): @@ -181,6 +185,9 @@ def test_pvrt_trackElementHandle(pvrt, sync_elementHandle, sync_browserType): quality=100, omit_background=False, ), + customTags='tag', + comment='why', + merge=True, agent=visual_regression_tracker.playwright.Agent( os='os', device='device', @@ -193,12 +200,13 @@ def test_pvrt_trackElementHandle(pvrt, sync_elementHandle, sync_browserType): testRun = pvrt.track.call_args[0][0] assert testRun.name == 'image name' - assert is_base64(testRun.imageBase64) and len(testRun.imageBase64) > 1024 + assert testRun.imageBase64 is None and is_image(testRun.imageBuffer) assert testRun.os == 'os' assert testRun.browser == sync_browserType.name assert testRun.viewport == '123x345' assert testRun.device == 'device' assert testRun.diffTollerancePercent == 10 + assert (testRun.customTags, testRun.comment, testRun.merge) == ('tag', 'why', True) assert testRun.ignoreAreas == [ visual_regression_tracker.types.IgnoreArea(1, 2, 3, 4), ] @@ -209,10 +217,5 @@ def sync_elementHandle(sync_page): return sync_page.query_selector('form') -def is_base64(val): - try: - assert isinstance(val, str) - base64.b64decode(val) - return True - except binascii.Error: - return False +def is_image(val): + return isinstance(val, bytes) and len(val) > 1024 diff --git a/tests/test_playwright_async.py b/tests/test_playwright_async.py index 8984a03..7e87a5f 100644 --- a/tests/test_playwright_async.py +++ b/tests/test_playwright_async.py @@ -1,5 +1,3 @@ -import base64 -import binascii import pytest import pytest_asyncio import visual_regression_tracker @@ -68,13 +66,14 @@ async def test_pvrt_trackPageAsync__default_options(pvrt, async_page, async_brow testRun = pvrt.track.call_args[0][0] assert testRun.name == 'image name' - assert is_base64(testRun.imageBase64) and len(testRun.imageBase64) > 1024 + assert testRun.imageBase64 is None and is_image(testRun.imageBuffer) assert testRun.os is None assert testRun.browser == async_browserType.name assert testRun.viewport == '1024x768' assert testRun.device is None assert testRun.diffTollerancePercent is None assert testRun.ignoreAreas is None + assert (testRun.customTags, testRun.comment, testRun.merge) == (None, None, None) @pytest.mark.asyncio(loop_scope='session') @@ -95,6 +94,9 @@ async def test_pvrt_trackPageAsync(pvrt, async_page, async_browserType): omit_background=False, full_page=True, ), + customTags='tag', + comment='why', + merge=True, agent=visual_regression_tracker.playwright.Agent( os='os', device='device', @@ -107,12 +109,13 @@ async def test_pvrt_trackPageAsync(pvrt, async_page, async_browserType): testRun = pvrt.track.call_args[0][0] assert testRun.name == 'image name' - assert is_base64(testRun.imageBase64) and len(testRun.imageBase64) > 1024 + assert testRun.imageBase64 is None and is_image(testRun.imageBuffer) assert testRun.os == 'os' assert testRun.browser == async_browserType.name assert testRun.viewport == '1024x768' assert testRun.device == 'device' assert testRun.diffTollerancePercent == 10 + assert (testRun.customTags, testRun.comment, testRun.merge) == ('tag', 'why', True) assert testRun.ignoreAreas == [ visual_regression_tracker.types.IgnoreArea(1, 2, 3, 4), ] @@ -133,13 +136,14 @@ async def test_pvrt_trackElementHandleAsync__default_options(pvrt, async_element testRun = pvrt.track.call_args[0][0] assert testRun.name == 'image name' - assert is_base64(testRun.imageBase64) and len(testRun.imageBase64) > 1024 + assert testRun.imageBase64 is None and is_image(testRun.imageBuffer) assert testRun.os is None assert testRun.browser == async_browserType.name assert testRun.viewport is None assert testRun.device is None assert testRun.diffTollerancePercent is None assert testRun.ignoreAreas is None + assert (testRun.customTags, testRun.comment, testRun.merge) == (None, None, None) @pytest.mark.asyncio(loop_scope='session') @@ -159,6 +163,9 @@ async def test_pvrt_trackElementHandleAsync(pvrt, async_elementHandle, async_bro quality=100, omit_background=False, ), + customTags='tag', + comment='why', + merge=True, agent=visual_regression_tracker.playwright.Agent( os='os', device='device', @@ -171,21 +178,17 @@ async def test_pvrt_trackElementHandleAsync(pvrt, async_elementHandle, async_bro testRun = pvrt.track.call_args[0][0] assert testRun.name == 'image name' - assert is_base64(testRun.imageBase64) and len(testRun.imageBase64) > 1024 + assert testRun.imageBase64 is None and is_image(testRun.imageBuffer) assert testRun.os == 'os' assert testRun.browser == async_browserType.name assert testRun.viewport == '123x345' assert testRun.device == 'device' assert testRun.diffTollerancePercent == 10 + assert (testRun.customTags, testRun.comment, testRun.merge) == ('tag', 'why', True) assert testRun.ignoreAreas == [ visual_regression_tracker.types.IgnoreArea(1, 2, 3, 4), ] -def is_base64(val): - try: - assert isinstance(val, str) - base64.b64decode(val) - return True - except binascii.Error: - return False +def is_image(val): + return isinstance(val, bytes) and len(val) > 1024 diff --git a/tests/test_visualRegressionTracker.py b/tests/test_visualRegressionTracker.py index 9b5ee77..f275214 100644 --- a/tests/test_visualRegressionTracker.py +++ b/tests/test_visualRegressionTracker.py @@ -324,11 +324,12 @@ def test__submitTestResults__should_throw_not_started(vrt, mocker): vrt._submitTestResult(testRun) -def mock_response(mocker, status_code, body=None, text='', reason=''): +def mock_response(mocker, status_code, body=None, text='', reason='', headers=None): response = mocker.Mock() response.url = 'http://vrt/builds' response.status_code = status_code response.reason = reason + response.headers = requests.structures.CaseInsensitiveDict(headers or {}) if body is None: response.text = text response.json.side_effect = ValueError('Expecting value') @@ -399,14 +400,192 @@ def test__http_request__wraps_request_exceptions_without_headers(mocker, excepti assert error.value.__cause__ is None and error.value.__suppress_context__ -@pytest.mark.parametrize('status_code,body,hint', [ - (413, None, 'BODY_PARSER_JSON_LIMIT'), - (422, {'message': 'Image is 20000x20000 (400000000 pixels), above MAX_IMAGE_PIXELS=100000000'}, 'MAX_IMAGE_PIXELS'), +@pytest.mark.parametrize('url,status_code,body,hint', [ + ('http://vrt/test-runs', 413, None, 'imagePath or imageBuffer'), + ('http://vrt/test-runs', 413, None, 'BODY_PARSER_JSON_LIMIT'), + ('http://vrt/test-runs/multipart', 413, None, 'MULTIPART_FILE_SIZE_LIMIT'), + ('http://vrt/test-runs', 422, + {'message': 'Image is 20000x20000 (400000000 pixels), above MAX_IMAGE_PIXELS=100000000'}, 'MAX_IMAGE_PIXELS'), ]) -def test__http_request__adds_hint_for_image_limits(mocker, status_code, body, hint): +def test__http_request__adds_hint_for_image_limits(mocker, url, status_code, body, hint): mocker.patch('requests.post', return_value=mock_response(mocker, status_code, body, 'request entity too large')) with pytest.raises(ServerError, match=re.escape(hint)) as error: - _http_request('url', 'post', {}, {}, 30) + _http_request(url, 'post', {}, {}, 30) assert error.value.status_code == status_code + + +def started_vrt(config=CONFIG): + vrt = VisualRegressionTracker(config) + vrt.buildId = '1312' + vrt.projectId = 'asd' + return vrt + + +def test__submitTestResults__sends_sourceBranch_customTags_comment_and_merge(mock_request): + vrt = started_vrt(dataclasses.replace(CONFIG, sourceBranch='feature')) + mock_request.return_value = {'status': 'ok'} + + vrt._submitTestResult(TestRun( + name='name', imageBase64='image', customTags='tag1,tag2', comment='why', merge=True)) + + mock_request.assert_called_once_with( + f'{CONFIG.apiUrl}/test-runs', + 'post', + { + 'name': 'name', + 'imageBase64': 'image', + 'customTags': 'tag1,tag2', + 'comment': 'why', + 'merge': True, + 'buildId': '1312', + 'projectId': 'asd', + 'branchName': CONFIG.branchName, + 'sourceBranch': 'feature', + }, + HEADERS, + TIMEOUT, + ) + + +def test__submitTestResults__leaves_out_unset_sourceBranch(mock_request): + mock_request.return_value = {'status': 'ok'} + + started_vrt()._submitTestResult(TestRun(name='name', imageBase64='image')) + + assert 'sourceBranch' not in mock_request.call_args[0][2] + + +def test__submitTestResults__sends_imageBuffer_as_multipart(mock_request): + vrt = started_vrt(dataclasses.replace(CONFIG, sourceBranch='feature')) + mock_request.return_value = {'status': 'ok', 'url': 'url'} + + result = vrt._submitTestResult(TestRun( + name='name', + imageBuffer=b'png', + os='os', + diffTollerancePercent=1.5, + ignoreAreas=[IgnoreArea(1, 2, 3, 4)], + customTags='tag', + comment='why', + merge=False, + )) + + assert result == TestRunResponse(status=TestRunStatus.OK, url='url') + mock_request.assert_called_once_with( + f'{CONFIG.apiUrl}/test-runs/multipart', + 'post', + { + 'name': 'name', + 'os': 'os', + 'diffTollerancePercent': '1.5', + 'ignoreAreas': '[{"x": 1, "y": 2, "width": 3, "height": 4}]', + 'customTags': 'tag', + 'comment': 'why', + 'merge': 'false', + 'buildId': '1312', + 'projectId': 'asd', + 'branchName': CONFIG.branchName, + 'sourceBranch': 'feature', + }, + HEADERS, + TIMEOUT, + files={'image': ('image.png', b'png', 'image/png')}, + ) + + +def test__submitTestResults__sends_imagePath_as_multipart(mock_request, tmp_path): + image = tmp_path / 'home page.png' + image.write_bytes(b'png from file') + mock_request.return_value = {'status': 'ok'} + + started_vrt()._submitTestResult(TestRun(name='name', imagePath=image)) + + assert mock_request.call_args[0][0] == f'{CONFIG.apiUrl}/test-runs/multipart' + assert mock_request.call_args[1]['files'] == {'image': ('home page.png', b'png from file', 'image/png')} + assert 'imagePath' not in mock_request.call_args[0][2] + + +@pytest.mark.parametrize('images', [ + {'imageBase64': 'image', 'imageBuffer': b'png'}, + {'imageBase64': 'image', 'imagePath': 'image.png'}, + {'imagePath': 'image.png', 'imageBuffer': b'png'}, +]) +def test__submitTestResults__rejects_more_than_one_image(mock_request, images): + with pytest.raises(ValueError, match='only one of imageBase64, imagePath and imageBuffer'): + started_vrt()._submitTestResult(TestRun(name='name', **images)) + + mock_request.assert_not_called() + + +def test__http_request__sends_files_as_multipart(mocker): + post = mocker.patch('requests.post', return_value=mock_response(mocker, 200, {'a': 'b'})) + files = {'image': ('image.png', b'png', 'image/png')} + + _http_request('url', 'post', {'name': 'n'}, {'apiKey': 'k'}, 7, files=files) + + post.assert_called_once_with('url', data={'name': 'n'}, files=files, headers={'apiKey': 'k'}, timeout=7) + + +@pytest.fixture +def sleep(mocker): + return mocker.patch('visual_regression_tracker.visualRegressionTracker.time.sleep') + + +def test__http_request__retries_503_after_retry_after_seconds(mocker, sleep): + post = mocker.patch('requests.post', side_effect=[ + mock_response(mocker, 503, {'message': 'Image diff queue is full'}, headers={'Retry-After': '5'}), + mock_response(mocker, 200, {'a': 'b'}), + ]) + + assert _http_request('url', 'post', {}, {}, 30) == {'a': 'b'} + + assert post.call_count == 2 + sleep.assert_called_once_with(5) + + +def test__http_request__reads_retry_after_as_http_date(mocker, sleep): + mocker.patch('visual_regression_tracker.visualRegressionTracker.time.time', + return_value=1793664000) # Tue, 03 Nov 2026 00:00:00 GMT + mocker.patch('requests.post', side_effect=[ + mock_response(mocker, 503, None, headers={'Retry-After': 'Tue, 03 Nov 2026 00:00:07 GMT'}), + mock_response(mocker, 200, {}), + ]) + + _http_request('url', 'post', {}, {}, 30) + + sleep.assert_called_once_with(7) + + +def test__http_request__gives_up_after_3_retries(mocker, sleep): + busy = mock_response(mocker, 503, {'message': 'Image diff queue is full'}, headers={'Retry-After': '5'}) + post = mocker.patch('requests.post', return_value=busy) + + with pytest.raises(ServerError, match='retried 3 times') as error: + _http_request('url', 'post', {}, {}, 30) + + assert post.call_count == 4 + assert sleep.call_count == 3 + assert error.value.status_code == 503 + + +@pytest.mark.parametrize('headers', [{}, {'Retry-After': '61'}, {'Retry-After': '-1'}, {'Retry-After': 'soon'}]) +def test__http_request__does_not_retry_503_without_a_usable_retry_after(mocker, sleep, headers): + post = mocker.patch('requests.post', return_value=mock_response(mocker, 503, None, reason='Service Unavailable', + headers=headers)) + + with pytest.raises(ServerError): + _http_request('url', 'post', {}, {}, 30) + + assert post.call_count == 1 + sleep.assert_not_called() + + +def test__http_request__does_not_retry_other_errors_with_retry_after(mocker, sleep): + post = mocker.patch('requests.post', return_value=mock_response(mocker, 429, None, headers={'Retry-After': '1'})) + + with pytest.raises(ServerError): + _http_request('url', 'post', {}, {}, 30) + + assert post.call_count == 1 diff --git a/visual_regression_tracker/config.py b/visual_regression_tracker/config.py index b6a2978..35e3e35 100644 --- a/visual_regression_tracker/config.py +++ b/visual_regression_tracker/config.py @@ -20,6 +20,7 @@ 'apiKey': 'VRT_APIKEY', 'enableSoftAssert': 'VRT_ENABLESOFTASSERT', 'httpTimeoutInSeconds': 'VRT_HTTPTIMEOUTINSECONDS', + 'sourceBranch': 'VRT_SOURCEBRANCH', } @@ -32,6 +33,7 @@ class Config: apiKey: str = None enableSoftAssert: bool = False httpTimeoutInSeconds: float = 360 + sourceBranch: str = None def __setattr__(self, name, value): if name == 'apiUrl' and isinstance(value, str): diff --git a/visual_regression_tracker/playwright.py b/visual_regression_tracker/playwright.py index 476fb68..9cf68fa 100644 --- a/visual_regression_tracker/playwright.py +++ b/visual_regression_tracker/playwright.py @@ -1,4 +1,3 @@ -import base64 import dataclasses import pathlib @@ -39,6 +38,9 @@ class PageTrackOptions: ignoreAreas: List[IgnoreArea] = None screenshotOptions: PageScreenshotOptions = None agent: Agent = None + customTags: str = None + comment: str = None + merge: bool = None @dataclasses.dataclass @@ -47,6 +49,9 @@ class ElementHandleTrackOptions: ignoreAreas: List[IgnoreArea] = None screenshotOptions: ElementHandleScreenshotOptions = None agent: Agent = None + customTags: str = None + comment: str = None + merge: bool = None class PlaywrightMixin: @@ -60,21 +65,8 @@ def trackPage( name: str, options: PageTrackOptions = None ): - viewportSize = page.viewport_size - screenshotOptions = _to_dict(options.screenshotOptions) if options else {} - screenshot = page.screenshot(**screenshotOptions) - imageBase64 = base64.b64encode(screenshot).decode('ascii') - - return self.track(TestRun( - name, - imageBase64, - options.agent.os if options and options.agent else None, - self.browser.name, - f'{viewportSize["width"]}x{viewportSize["height"]}' if viewportSize else None, - options.agent.device if options and options.agent else None, - options.diffTollerancePercent if options else None, - options.ignoreAreas if options else None, - )) + screenshot = page.screenshot(**_screenshotOptions(options)) + return self.track(self._testRun(name, screenshot, options, _viewport(page))) async def trackPageAsync( self, @@ -82,22 +74,8 @@ async def trackPageAsync( name: str, options: PageTrackOptions = None ): - viewportSize = page.viewport_size - - screenshotOptions = _to_dict(options.screenshotOptions) if options else {} - screenshot = await page.screenshot(**screenshotOptions) - imageBase64 = base64.b64encode(screenshot).decode('ascii') - - return self.track(TestRun( - name, - imageBase64, - options.agent.os if options and options.agent else None, - self.browser.name, - f'{viewportSize["width"]}x{viewportSize["height"]}' if viewportSize else None, - options.agent.device if options and options.agent else None, - options.diffTollerancePercent if options else None, - options.ignoreAreas if options else None, - )) + screenshot = await page.screenshot(**_screenshotOptions(options)) + return self.track(self._testRun(name, screenshot, options, _viewport(page))) def trackElementHandle( self, @@ -105,20 +83,8 @@ def trackElementHandle( name: str, options: ElementHandleTrackOptions = None ): - screenshotOptions = _to_dict(options.screenshotOptions) if options else {} - screenshot = elementHandle.screenshot(**screenshotOptions) - imageBase64 = base64.b64encode(screenshot).decode('ascii') - - return self.track(TestRun( - name, - imageBase64, - options.agent.os if options and options.agent else None, - self.browser.name, - options.agent.viewport if options and options.agent else None, - options.agent.device if options and options.agent else None, - options.diffTollerancePercent if options else None, - options.ignoreAreas if options else None, - )) + screenshot = elementHandle.screenshot(**_screenshotOptions(options)) + return self.track(self._testRun(name, screenshot, options)) async def trackElementHandleAsync( self, @@ -126,20 +92,40 @@ async def trackElementHandleAsync( name: str, options: ElementHandleTrackOptions = None ): - screenshotOptions = _to_dict(options.screenshotOptions) if options else {} - screenshot = await elementHandle.screenshot(**screenshotOptions) - imageBase64 = base64.b64encode(screenshot).decode('ascii') - - return self.track(TestRun( - name, - imageBase64, - options.agent.os if options and options.agent else None, - self.browser.name, - options.agent.viewport if options and options.agent else None, - options.agent.device if options and options.agent else None, - options.diffTollerancePercent if options else None, - options.ignoreAreas if options else None, - )) + screenshot = await elementHandle.screenshot(**_screenshotOptions(options)) + return self.track(self._testRun(name, screenshot, options)) + + def _testRun( + self, + name: str, + screenshot: bytes, + options: Union[PageTrackOptions, ElementHandleTrackOptions, None], + viewport: str = None, + ) -> TestRun: + options = options or ElementHandleTrackOptions() + agent = options.agent or Agent() + return TestRun( + name=name, + imageBuffer=screenshot, + os=agent.os, + browser=self.browser.name, + viewport=viewport or agent.viewport, + device=agent.device, + diffTollerancePercent=options.diffTollerancePercent, + ignoreAreas=options.ignoreAreas, + customTags=options.customTags, + comment=options.comment, + merge=options.merge, + ) + + +def _screenshotOptions(options) -> dict: + return _to_dict(options.screenshotOptions) if options and options.screenshotOptions else {} + + +def _viewport(page) -> str: + size = page.viewport_size + return f'{size["width"]}x{size["height"]}' if size else None class PlaywrightVisualRegressionTracker(PlaywrightMixin, VisualRegressionTracker): diff --git a/visual_regression_tracker/types.py b/visual_regression_tracker/types.py index 0441eab..c56cd17 100644 --- a/visual_regression_tracker/types.py +++ b/visual_regression_tracker/types.py @@ -1,6 +1,7 @@ import dataclasses import enum import logging +from os import PathLike import typing import dacite @@ -23,6 +24,11 @@ class TestRun: device: str = None diffTollerancePercent: float = None ignoreAreas: typing.List[IgnoreArea] = None + customTags: str = None + comment: str = None + merge: bool = None + imagePath: typing.Union[str, PathLike] = None + imageBuffer: bytes = None @dataclasses.dataclass diff --git a/visual_regression_tracker/visualRegressionTracker.py b/visual_regression_tracker/visualRegressionTracker.py index 7cfa07a..501d06d 100644 --- a/visual_regression_tracker/visualRegressionTracker.py +++ b/visual_regression_tracker/visualRegressionTracker.py @@ -1,4 +1,8 @@ +import email.utils +import json import logging +import pathlib +import time import requests @@ -10,6 +14,9 @@ from .config import Config CONNECT_TIMEOUT_SECONDS = 10 +MAX_RETRIES_ON_503 = 3 +MAX_RETRY_AFTER_SECONDS = 60 +IMAGE_FIELDS = ('imageBase64', 'imagePath', 'imageBuffer') class VisualRegressionTracker: @@ -34,13 +41,14 @@ def __init__(self, config: Config = None): def _isStarted(self): return self.buildId is not None and self.projectId is not None - def _request(self, method: str, path: str, data: dict) -> dict: + def _request(self, method: str, path: str, data: dict, **kwargs) -> dict: return _http_request( f'{self.config.apiUrl}/{path}', method, data, self.headers, (CONNECT_TIMEOUT_SECONDS, self.config.httpTimeoutInSeconds or None), + **kwargs, ) def start(self) -> Build: @@ -77,17 +85,25 @@ def _submitTestResult(self, test: TestRun) -> TestRunResponse: raise VisualRegressionTrackerError( "Visual Regression Tracker has not been started") + images = [field for field in IMAGE_FIELDS if getattr(test, field) is not None] + if len(images) > 1: + raise ValueError(f'TestRun takes only one of imageBase64, imagePath and imageBuffer, got {images}') + data = _to_dict(test) data.update( buildId=self.buildId, projectId=self.projectId, branchName=self.config.branchName, ) + if self.config.sourceBranch: + data['sourceBranch'] = self.config.sourceBranch - result = self._request('post', 'test-runs', data) - testRunResult = _from_dict(result, TestRunResponse) - - return testRunResult + if test.imagePath is None and test.imageBuffer is None: + result = self._request('post', 'test-runs', data) + else: + fields, image = _multipart(data) + result = self._request('post', 'test-runs/multipart', fields, files={'image': image}) + return _from_dict(result, TestRunResponse) def track(self, test: TestRun) -> TestRunResult: result = self._submitTestResult(test) @@ -107,23 +123,56 @@ def track(self, test: TestRun) -> TestRunResult: return TestRunResult(result, self.config.apiUrl) -_HINTS = { - 413: 'the request is larger than BODY_PARSER_JSON_LIMIT on the server; raise it or send a smaller screenshot', - 422: 'the image has more pixels than MAX_IMAGE_PIXELS on the server; raise it or send a smaller screenshot', -} +def _multipart(data: dict) -> tuple: + path = data.pop('imagePath', None) + image = pathlib.Path(path).read_bytes() if path is not None else data.pop('imageBuffer') + filename = pathlib.Path(path).name if path is not None else 'image.png' + fields = {key: _form_value(value) for key, value in data.items()} + return fields, (filename, image, 'image/png') + + +def _form_value(value) -> str: + if isinstance(value, bool): + return 'true' if value else 'false' + if isinstance(value, (list, dict)): + return json.dumps(value) + return str(value) + +def _hint(status: int, url: str, retries: int): + if status == 413 and url.endswith('/test-runs/multipart'): + return 'the image is larger than MULTIPART_FILE_SIZE_LIMIT on the server; raise it or send a smaller screenshot' + if status == 413: + return ('the request is larger than BODY_PARSER_JSON_LIMIT on the server; ' + 'track the image with imagePath or imageBuffer (multipart), raise the limit, or send a smaller screenshot') + if status == 422: + return 'the image has more pixels than MAX_IMAGE_PIXELS on the server; raise it or send a smaller screenshot' + if status == 503 and retries: + return f'the server is busy; retried {retries} times after its Retry-After' + return None -def _http_request(url: str, method: str, data: dict, headers: dict, timeout: tuple) -> dict: + +def _http_request(url: str, method: str, data: dict, headers: dict, timeout: tuple, files: dict = None) -> dict: request = getattr(requests, method.lower()) - try: - response = request(url, json=data, headers=headers, timeout=timeout) - except requests.RequestException as e: - raise RequestError(url, f'{method.upper()} {url} failed: {type(e).__name__}: {e}') from None - status = response.status_code + body = {'data': data, 'files': files} if files else {'json': data} + retries = 0 + while True: + try: + response = request(url, **body, headers=headers, timeout=timeout) + except requests.RequestException as e: + raise RequestError(url, f'{method.upper()} {url} failed: {type(e).__name__}: {e}') from None + status = response.status_code + delay = _retry_after(response) if status == 503 else None + if delay is None or retries == MAX_RETRIES_ON_503: + break + retries += 1 + logging.getLogger(__name__).warning( + '%s %s: HTTP 503, retry %d of %d in %g s', method.upper(), url, retries, MAX_RETRIES_ON_503, delay) + time.sleep(delay) if status >= 400: message = _error_message(response) - hint = _HINTS.get(status) + hint = _hint(status, url, retries) raise ServerError(status, f'{message} (hint: {hint})' if hint else message) try: @@ -140,3 +189,15 @@ def _error_message(response) -> str: if isinstance(message, list): message = ', '.join(map(str, message)) return message or response.text.strip() or response.reason or 'no error message' + + +def _retry_after(response): + value = response.headers.get('Retry-After', '').strip() + if value.isdigit(): + seconds = int(value) + else: + try: + seconds = email.utils.parsedate_to_datetime(value).timestamp() - time.time() + except (TypeError, ValueError): + seconds = None + return seconds if seconds is not None and 0 <= seconds <= MAX_RETRY_AFTER_SECONDS else None From d5ec1d2d40a126e19fe3ce367387e8a0e02a2990 Mon Sep 17 00:00:00 2001 From: Pavlo Strunkin Date: Mon, 2 Nov 2026 22:54:56 +0200 Subject: [PATCH 2/3] refactor: read the multipart image in one branch Co-Authored-By: Claude Opus 5.5 --- visual_regression_tracker/visualRegressionTracker.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/visual_regression_tracker/visualRegressionTracker.py b/visual_regression_tracker/visualRegressionTracker.py index 501d06d..3390441 100644 --- a/visual_regression_tracker/visualRegressionTracker.py +++ b/visual_regression_tracker/visualRegressionTracker.py @@ -125,8 +125,10 @@ def track(self, test: TestRun) -> TestRunResult: def _multipart(data: dict) -> tuple: path = data.pop('imagePath', None) - image = pathlib.Path(path).read_bytes() if path is not None else data.pop('imageBuffer') - filename = pathlib.Path(path).name if path is not None else 'image.png' + if path is None: + filename, image = 'image.png', data.pop('imageBuffer') + else: + filename, image = pathlib.Path(path).name, pathlib.Path(path).read_bytes() fields = {key: _form_value(value) for key, value in data.items()} return fields, (filename, image, 'image/png') From 203b6c3bac479b6b2aeb1ff72cbaceda41598857 Mon Sep 17 00:00:00 2001 From: Pavlo Strunkin Date: Mon, 2 Nov 2026 22:55:50 +0200 Subject: [PATCH 3/3] fix: read Retry-After without a zone as UTC and ignore non-ASCII digits Co-Authored-By: Claude Opus 5.5 --- tests/test_visualRegressionTracker.py | 19 ++++++++++++++++--- .../visualRegressionTracker.py | 8 +++++--- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/tests/test_visualRegressionTracker.py b/tests/test_visualRegressionTracker.py index f275214..2fef8a3 100644 --- a/tests/test_visualRegressionTracker.py +++ b/tests/test_visualRegressionTracker.py @@ -2,6 +2,7 @@ import json import logging import re +import time import pytest import requests @@ -545,11 +546,21 @@ def test__http_request__retries_503_after_retry_after_seconds(mocker, sleep): sleep.assert_called_once_with(5) -def test__http_request__reads_retry_after_as_http_date(mocker, sleep): +@pytest.fixture +def utc_plus_3(monkeypatch): + monkeypatch.setenv('TZ', 'Europe/Moscow') + time.tzset() + yield + monkeypatch.undo() + time.tzset() + + +@pytest.mark.parametrize('date', ['Tue, 03 Nov 2026 00:00:07 GMT', 'Tue, 03 Nov 2026 00:00:07 -0000']) +def test__http_request__reads_retry_after_as_http_date(mocker, sleep, utc_plus_3, date): mocker.patch('visual_regression_tracker.visualRegressionTracker.time.time', return_value=1793664000) # Tue, 03 Nov 2026 00:00:00 GMT mocker.patch('requests.post', side_effect=[ - mock_response(mocker, 503, None, headers={'Retry-After': 'Tue, 03 Nov 2026 00:00:07 GMT'}), + mock_response(mocker, 503, None, headers={'Retry-After': date}), mock_response(mocker, 200, {}), ]) @@ -570,7 +581,9 @@ def test__http_request__gives_up_after_3_retries(mocker, sleep): assert error.value.status_code == 503 -@pytest.mark.parametrize('headers', [{}, {'Retry-After': '61'}, {'Retry-After': '-1'}, {'Retry-After': 'soon'}]) +@pytest.mark.parametrize('headers', [ + {}, {'Retry-After': '61'}, {'Retry-After': '-1'}, {'Retry-After': 'soon'}, {'Retry-After': '\u00b2'}, +]) def test__http_request__does_not_retry_503_without_a_usable_retry_after(mocker, sleep, headers): post = mocker.patch('requests.post', return_value=mock_response(mocker, 503, None, reason='Service Unavailable', headers=headers)) diff --git a/visual_regression_tracker/visualRegressionTracker.py b/visual_regression_tracker/visualRegressionTracker.py index 3390441..dbdbc24 100644 --- a/visual_regression_tracker/visualRegressionTracker.py +++ b/visual_regression_tracker/visualRegressionTracker.py @@ -1,3 +1,4 @@ +import datetime import email.utils import json import logging @@ -195,11 +196,12 @@ def _error_message(response) -> str: def _retry_after(response): value = response.headers.get('Retry-After', '').strip() - if value.isdigit(): + if value.isascii() and value.isdigit(): seconds = int(value) else: try: - seconds = email.utils.parsedate_to_datetime(value).timestamp() - time.time() + date = email.utils.parsedate_to_datetime(value) except (TypeError, ValueError): - seconds = None + date = None + seconds = date and date.replace(tzinfo=date.tzinfo or datetime.timezone.utc).timestamp() - time.time() return seconds if seconds is not None and 0 <= seconds <= MAX_RETRY_AFTER_SECONDS else None