Fix: Mitigate WAF detection in HttpFetcher

TiktokExplode.All/TiktokExplode.All.csproj

@@ -10,7 +10,7 @@
 
   <PropertyGroup>
     <PackageId>TiktokExplode.All</PackageId>
-    <Version>1.2.0</Version>
+    <Version>1.2.1</Version>
     <Authors>Ts-Pytham</Authors>
     <Owners>Ts-Pytham</Owners>
     <Description>Meta-package that installs TiktokExplode (domain layer) and TiktokExplode.Infrastructure (HTTP/browser fetchers, parser, download client) in a single command.</Description>

TiktokExplode.Infrastructure/Clients/TiktokClient.cs

@@ -52,7 +52,7 @@ public async Task<Video> GetVideoAsync(string url, CancellationToken cancellatio
                 _downloadClient.InjectCookies(result.Cookies);
                 return await _parser.ParseAsync(result.HtmlContent);
             }
-            catch (TiktokWafException) when (attempt < options.MaxWafRetries)
+            catch (TiktokException) when (attempt < options.MaxWafRetries)
             {
                 await Task.Delay(options.RetryBaseDelay * (attempt + 1), cancellationToken);
             }

TiktokExplode.Infrastructure/Fetchers/HttpFetcher.cs

@@ -58,7 +58,7 @@ public HttpFetcher(HttpFetcherOptions options)
 
             MaxConnectionsPerServer = 10,
 
-            EnableMultipleHttp2Connections = true
+            EnableMultipleHttp2Connections = true,
         };
 
         _httpClient = new HttpClient(_handler);
@@ -79,18 +79,10 @@ private void ConfigureDefaultHeaders()
 
         headers.Clear();
 
-        headers.TryAddWithoutValidation("User-Agent",                _options.UserAgent);
-        headers.TryAddWithoutValidation("Accept",                    "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8");
-        headers.TryAddWithoutValidation("Accept-Language",           "en-US,en;q=0.9");
-        headers.TryAddWithoutValidation("Accept-Encoding",           "gzip, deflate, br");
-        headers.TryAddWithoutValidation("Upgrade-Insecure-Requests", "1");
-        headers.TryAddWithoutValidation("Sec-Fetch-Dest",            "document");
-        headers.TryAddWithoutValidation("Sec-Fetch-Mode",            "navigate");
-        headers.TryAddWithoutValidation("Sec-Fetch-Site",            "none");
-        headers.TryAddWithoutValidation("Sec-Fetch-User",            "?1");
-        headers.TryAddWithoutValidation("sec-ch-ua",                 "\"Chromium\";v=\"136\", \"Google Chrome\";v=\"136\", \"Not.A/Brand\";v=\"99\"");
-        headers.TryAddWithoutValidation("sec-ch-ua-mobile",          "?0");
-        headers.TryAddWithoutValidation("sec-ch-ua-platform",        "\"Windows\"");
+        headers.TryAddWithoutValidation("User-Agent", _options.UserAgent);
+        headers.TryAddWithoutValidation("Accept", "*/*");
+        headers.TryAddWithoutValidation("Cache-Control", "no-cache");
+        headers.TryAddWithoutValidation("Connection", "keep-alive");
     }
 
     /// <summary>
@@ -107,12 +99,10 @@ public async Task<PageFetchResult> FetchPageAsync(string url, CancellationToken
 
         using var request = new HttpRequestMessage(HttpMethod.Get, url)
         {
-            Version = HttpVersion.Version20,
-            VersionPolicy = HttpVersionPolicy.RequestVersionOrLower
+            Version = HttpVersion.Version11,
+            VersionPolicy = HttpVersionPolicy.RequestVersionExact
         };
 
-        request.Headers.Referrer = new Uri("https://www.tiktok.com/");
-
         using var response = await _httpClient.SendAsync(
             request,
             HttpCompletionOption.ResponseHeadersRead,
@@ -123,6 +113,9 @@ public async Task<PageFetchResult> FetchPageAsync(string url, CancellationToken
         if (content.Contains("_wafchallengeid", StringComparison.OrdinalIgnoreCase))
             throw new TiktokWafException("TikTok WAF challenge detected.");
 
+        if (!content.Contains("__UNIVERSAL_DATA_FOR_REHYDRATION__", StringComparison.OrdinalIgnoreCase))
+            throw new TiktokException("TikTok logo detected.");
+
         var cookies = _cookies.GetAllCookies()
             .Cast<Cookie>()
             .Select(c => new CookieData
@@ -157,8 +150,8 @@ private async Task EnsureWarmedUpAsync(CancellationToken cancellationToken)
 
             using var request = new HttpRequestMessage(HttpMethod.Get, "https://www.tiktok.com/")
             {
-                Version = HttpVersion.Version20,
-                VersionPolicy = HttpVersionPolicy.RequestVersionOrLower
+                Version = HttpVersion.Version11,
+                VersionPolicy = HttpVersionPolicy.RequestVersionExact
             };
 
             using var response = await _httpClient.SendAsync(

TiktokExplode.Infrastructure/Options/HttpFetcherOptions.cs

@@ -15,7 +15,6 @@ public sealed class HttpFetcherOptions
 
     /// <summary>
     /// The <c>User-Agent</c> header value sent with all HTTP requests.
-    /// Defaults to a recent Chrome on Windows UA string.
     /// </summary>
-    public string UserAgent { get; set; } = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36";
+    public string UserAgent { get; set; } = "TikTokExplode/1.2.0";
 }

TiktokExplode.Infrastructure/TiktokExplode.Infrastructure.csproj

@@ -12,7 +12,7 @@
 
   <PropertyGroup>
     <PackageId>TiktokExplode.Infrastructure</PackageId>
-    <Version>1.2.0</Version>
+    <Version>1.2.1</Version>
     <Authors>Ts-Pytham</Authors>
     <Owners>Ts-Pytham</Owners>
     <Description>Infrastructure layer for TiktokExplode: HTTP and Playwright-based page fetchers, TikTok video parser, CDN download client, and WAF-retry logic.</Description>

TiktokExplode.IntegrationTests/Usings.cs

@@ -3,4 +3,7 @@
 global using System.Text.Json.Nodes;
 global using TiktokExplode.Infrastructure.Fetchers.Search;
 global using TiktokExplode.IntegrationTests.Fixtures;
-global using TiktokExplode.Infrastructure.Clients;
+global using TiktokExplode.Infrastructure.Clients;
+global using TiktokExplode.Infrastructure.Fetchers;
+global using TiktokExplode.Domain.Exceptions;
+global using TiktokExplode.Infrastructure.Options;

TiktokExplode.IntegrationTests/Videos/HttpFetcherTests.cs

@@ -0,0 +1,33 @@
+namespace TiktokExplode.IntegrationTests.Videos;
+
+public sealed class HttpFetcherTests
+{
+    [Fact]
+    public async Task FetchPageAsync_Should_Bypass_Waf_And_Return_Page_Content()
+    {
+        // Arrange
+        using var fetcher = new HttpFetcher();
+
+        // Act
+        Func<Task> act = async () =>
+        {
+            var result = await fetcher.FetchPageAsync(
+                "https://www.tiktok.com/@js_nightwave/video/7579504710961548565");
+
+            // Assert
+            result.Should().NotBeNull();
+
+            result.HtmlContent.Should().NotBeNullOrWhiteSpace();
+
+            result.HtmlContent.Should()
+                .NotContain("_wafchallengeid");
+
+            result.HtmlContent.Length.Should().BeGreaterThan(10_000);
+
+            result.Cookies.Should().NotBeEmpty();
+        };
+
+        // Assert
+        await act.Should().NotThrowAsync<TiktokWafException>();
+    }
+}

TiktokExplode.IntegrationTests/Videos/TiktokClientTests.cs

@@ -0,0 +1,27 @@
+namespace TiktokExplode.IntegrationTests.Videos;
+
+public sealed class TiktokClientTests
+{
+    [Fact]
+    public async Task GetVideoAsync_Should_Return_Video()
+    {
+        var options = new TiktokOptions
+        {
+            MaxWafRetries = 10,
+            RetryBaseDelay = TimeSpan.FromMilliseconds(200),
+        };
+
+        await using var client = TiktokClient.CreateWithHttp(options: options);
+
+        var video = await client.GetVideoAsync(
+            "https://www.tiktok.com/@js_nightwave/video/7579504710961548565");
+
+        video.Should().NotBeNull();
+
+        video.Id.Should().NotBeNullOrWhiteSpace();
+
+        video.Info.DownloadLinks.Should().NotBeNull();
+
+        video.Author.Should().NotBeNull();
+    }
+}

TiktokExplode.Tests/Infrastructure/Clients/TiktokClientTests.cs

@@ -47,6 +47,25 @@ public async Task Should_RetryAndSucceed_When_FetcherThrowsWafOnFirstAttemptOnly
         await fetcher.Received(2).FetchPageAsync(ValidUrl, Arg.Any<CancellationToken>());
     }
 
+    [Fact]
+    public async Task Should_RetryAndSucceed_When_First_Attempt_Fails()
+    {
+        var fetcher = Substitute.For<IPageFetcher>();
+        var options = new TiktokOptions { MaxWafRetries = 2, RetryBaseDelay = TimeSpan.Zero };
+
+        fetcher.FetchPageAsync(Arg.Any<string>(), Arg.Any<CancellationToken>())
+               .Returns(
+                   Task.FromException<PageFetchResult>(new TiktokException()),
+                   Task.FromResult(TikTokFixtures.ValidVideoResult)
+               );
+
+        await using var client = new TiktokClient(fetcher, options);
+
+        var video = await client.GetVideoAsync(ValidUrl);
+
+        video.Should().NotBeNull();
+    }
+
     [Fact]
     public async Task Should_ThrowTiktokWafException_When_AllRetriesAreExhausted()
     {

TiktokExplode/TiktokExplode.csproj

@@ -11,7 +11,7 @@
 
   <PropertyGroup>
     <PackageId>TiktokExplode</PackageId>
-    <Version>1.2.0</Version>
+    <Version>1.2.1</Version>
     <Authors>Ts-Pytham</Authors>
     <Owners>Ts-Pytham</Owners>
     <Description>Domain layer for TiktokExplode: strongly-typed models, interfaces, and utilities for working with TikTok video metadata. Zero external dependencies.</Description>