StreamxMeshIntegrationTest.java

package com.streamx.blueprints.rewriter;

import static com.streamx.blueprints.rewriter.StreamxMeshIntegrationTest.UrlAssertions.assertAllUrlsAreAccessible;
import static com.streamx.blueprints.rewriter.StreamxMeshIntegrationTest.UrlAssertions.assertUrlContent;
import static com.streamx.blueprints.rewriter.StreamxMeshIntegrationTest.UrlAssertions.assertUrlContentContains;
import static com.streamx.blueprints.rewriter.StreamxMeshIntegrationTest.UrlAssertions.assertUrlContentLines;
import static com.streamx.blueprints.rewriter.StreamxMeshIntegrationTest.UrlAssertions.assertUrlIsAccessible;
import static com.streamx.blueprints.rewriter.StreamxMeshIntegrationTest.UrlAssertions.assertUrlIsNotAccessible;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatCode;
import static org.assertj.core.api.Assumptions.assumeThat;
import static org.awaitility.Awaitility.await;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.streamx.blueprints.cloudevents.utils.CloudEventUtils;
import com.streamx.blueprints.data.Asset;
import com.streamx.blueprints.data.Composition;
import com.streamx.blueprints.data.Data;
import com.streamx.blueprints.data.Fragment;
import com.streamx.blueprints.data.Layout;
import com.streamx.blueprints.data.Page;
import com.streamx.blueprints.data.Renderer;
import com.streamx.blueprints.data.RenderingContext;
import com.streamx.blueprints.data.RenderingContext.OutputFormat;
import com.streamx.blueprints.data.WebResource;
import com.streamx.clients.ingestion.StreamxClient;
import com.streamx.clients.ingestion.exceptions.StreamxClientException;
import com.streamx.clients.ingestion.publisher.Publisher;
import io.cloudevents.CloudEvent;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Field;
import java.time.Duration;
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.ThreadUtils;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;

/**
 * This test is disabled for mvn builds, but can be launched manually - e.g., via IDE. Prerequisite
 * to run this test: - running instance of StreamX, started with the project's /mesh.yaml file
 */
@Disabled
public class StreamxMeshIntegrationTest {

  private static final String INGESTION_URL = "http://localhost:8080";
  private static final String OPENSEARCH_URL = "http://localhost:9201";
  private static final ObjectMapper objectMapper = new ObjectMapper();

  private record ProjectSpecificSettings(
      String webServerUrl,
      String nginxUrl,
      String opensearchSinkUrl,
      String sitemapBaseUrl,
      boolean shouldRewriteKeys,
      boolean processExternalResourcesInPages,
      boolean processExternalResourcesInSitemaps,
      boolean hasSearch,
      boolean hasSitemaps,
      boolean hasImageOptimization
  ) { }

  private final ProjectSpecificSettings blueprintsSettings = new ProjectSpecificSettings(
      "http://localhost:8081",
      "http://localhost:8082",
      "http://localhost:8083",
      "http://localhost:8081",
      true,
      true,
      true,
      true,
      true,
      true
  );

  private final ProjectSpecificSettings acceleratorSettings = new ProjectSpecificSettings(
      "http://localhost:8083",
      "http://localhost:8084",
      "http://localhost:8082",
      "http://web.127.0.0.1.nip.io",
      false,
      false,
      false,
      true,
      true,
      true
  );

  private final ProjectSpecificSettings lightAcceleratorSettings = new ProjectSpecificSettings(
      acceleratorSettings.webServerUrl,
      acceleratorSettings.nginxUrl,
      acceleratorSettings.opensearchSinkUrl,
      acceleratorSettings.sitemapBaseUrl,
      acceleratorSettings.shouldRewriteKeys,
      acceleratorSettings.processExternalResourcesInPages,
      acceleratorSettings.processExternalResourcesInSitemaps,
      false,
      false,
      false
  );

  private final ProjectSpecificSettings settings = blueprintsSettings;

  @BeforeAll
  static void setApplicationName() {
    System.setProperty("quarkus.application.name",
        StreamxMeshIntegrationTest.class.getSimpleName());
  }

  @Test
  void shouldPublishSitemapFileAlongWithPagesAndTheirImages() throws Exception {
    assumeThat(settings.hasSitemaps).isTrue();

    // given
    final String key = "streamx-tech-sitemap.xml";
    final String payloadType = "web-resource";
    final String content = """
        <?xml version="1.0" encoding="UTF-8"?>
        <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
          <url>
            <loc>https://www.streamx.tech/blog.html</loc>
          </url>
          <url>
            <loc>https://www.streamx.tech/categories/accent-furniture.html</loc>
          </url>
        </urlset>""";

    WebResource sitemapResource = new WebResource(content, payloadType);

    // when
    publish(key, sitemapResource);

    // then
    assertUrlIsAccessible(
        // the sitemap file
        settings.webServerUrl + "/streamx-tech-sitemap.xml"
    );

    // and
    if (settings.processExternalResourcesInSitemaps) {
      assertAllUrlsAreAccessible(
          // two pages from the sitemap file
          settings.webServerUrl + "/blog.html",
          settings.webServerUrl + "/categories/accent-furniture.html",
          // referenced image
          settings.webServerUrl + "/assets/logo-footer.webp"
      );
    }
  }

  @Test
  void shouldComposePage() throws Exception {
    // given
    String layoutKey = "layout-for-foobar-pages";
    String layout = """
        <html>
          Hello {{#insert name="foo.html"}}.
          This is {{#insert name="bar.html"}}.
          FAQs:
            {{#insert name="faq.html"}}
              No data
            {{}}
        </html>
        """;

    String compositionKey = "/composition-for-john-streamx-foobar-page.html";
    String composition = """
        {{#define name="foo.html"}}
        <b>John</b>
        the
        developer

        {{#define name="bar.html"}}
        <b>StreamX</b>
        and how to use it
        """;

    // when
    publish(layoutKey, new Layout(layout, "layout-type"));
    publish(compositionKey, new Composition(composition, null, layoutKey));

    // then
    assertUrlContent(settings.webServerUrl + compositionKey, """
        <html>
          Hello <b>John</b>
        the
        developer.
          This is <b>StreamX</b>
        and how to use it.
          FAQs:
            No data
        </html>
        """);
  }

  @Test
  void shouldOptimizeImage() throws Exception {
    assumeThat(settings.hasImageOptimization).isTrue();

    // given
    String imageKey = "/images/mesh.jpg";
    String imagePath = "../optimized-images-generator/src/test/resources/images/mesh.jpg";
    Asset image = new Asset(FileUtils.readFileToByteArray(new File(imagePath)), "any");

    // when
    publish(imageKey, image);

    // then
    assertAllUrlsAreAccessible(
        settings.webServerUrl + imageKey,
        settings.webServerUrl + "/images/mesh-optimized.webp"
    );
    // and: allow the optimized asset to reach local-rerefences-rewriter before page to be adjusted
    sleepOneSecond();

    // when
    String pageKey = "/page-with-mesh-image.html";
    Page page = new Page("Hello. <img src='/images/mesh.jpg' />", "any");
    publish(pageKey, page);

    // then
    assertUrlContent(settings.webServerUrl + pageKey, """
        <html>
         <head></head>
         <body>
          Hello. <img src="/images/mesh-optimized.webp">
         </body>
        </html>""");
  }

  @Test
  void shouldGenerateSitemap() throws Exception {
    assumeThat(settings.hasSearch).isTrue();

    // given
    publish("/page-1.html", new Page("Page 1", "any"));
    publish("/page-2.html", new Page("Page 2", "any"));
    publish("/page-3.html", new Page("Page 3", "any"));

    // then
    assertAllUrlsAreAccessible(
        settings.webServerUrl + "/page-1.html",
        settings.webServerUrl + "/page-2.html",
        settings.webServerUrl + "/page-3.html",
        settings.webServerUrl + "/sitemap.xml"
    );

    // and
    assertUrlContentLines(settings.webServerUrl + "/sitemap.xml", List.of(
        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>",
        "<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">",
        "<url>",
        "<loc>%s/page-1.html</loc>".formatted(settings.sitemapBaseUrl),
        "</url>",
        "<url>",
        "<loc>%s/page-2.html</loc>".formatted(settings.sitemapBaseUrl),
        "</url>",
        "<url>",
        "<loc>%s/page-3.html</loc>".formatted(settings.sitemapBaseUrl),
        "</url>",
        "</urlset>"
    ));
  }

  @Test
  void shouldProduceIndexableResourceFromPageAndMakeItAvailableInSearch() throws Exception {
    assumeThat(settings.hasSearch).isTrue();

    // given
    String key = "/hello-page.html";
    String pageHtml = """
        <head>
            <title>Title</title>
        </head>
        <body>
            <h1>H1</h1>
            <h2>H2</h2>
            <p>Paragraph</p>
        </body>
        """;
    String pageType = "pages/test-page";

    // when
    publish(key, new Page(pageHtml, pageType));

    // then
    assertUrlContentContains(OPENSEARCH_URL + "/default/_search?q=_id:%22" + key + "%22",
        JsonUtils.makeSingleLine("""
            "hits": [
                {
                    "_index": "default",
                    "_id": "/hello-page.html",
                    "_score": 1.0,
                    "_source": {"""),
        JsonUtils.makeSingleLine("""
                        "fragments": []"""),
        JsonUtils.makeSingleLine("""
                        "namespace": null"""),
        JsonUtils.makeSingleLine("""
                        "type": "pages/test-page\""""),
        JsonUtils.makeSingleLine("""
                        "payload": {
                            "title": "Title",
                            "content": "Title\\n\\n\\n    H1\\n\\n    H2\\n\\n    Paragraph\"""")
    );

    // and: test using specialized endpoint
    assertUrlContentContains(settings.opensearchSinkUrl + "/search/query?query=paragr",
        JsonUtils.makeSingleLine("""
            "hits": [
                {
                    "_id": "/hello-page.html",
                    "_score": 2.0,
                    "_source": {
                        "payload": {
                            "title": "Title"
                        },
                        "namespace": null,
                        "type": "pages/test-page"
                    },
                    "highlight": {
                        "payload.content": [
                            "Title\\n\\n\\n    H1\\n\\n    H2\\n\\n    <em>Paragraph</em>"
                        ]
                    }
                }
            ]
            """
        ));
  }

  @Test
  void shouldUpdateFragmentsInSearch() throws Exception {
    assumeThat(settings.hasSearch).isTrue();

    // given
    final String pageType = "page";
    final String fragmentType = "fragment";

    final String pageKey = "page-%s.html".formatted(UUID.randomUUID());
    final String fragment1Key = "fragment-%s.html".formatted(UUID.randomUUID());
    final String fragment2Key = "fragment-%s.html".formatted(UUID.randomUUID());

    final String fragment1OriginalContent = "{\"content\": \"Fragment A original\"}";
    final String fragment2OriginalContent = "{\"content\": \"Fragment B original\"}";

    final String fragment1ModifiedContent = "{\"content\": \"Fragment A modified\"}";
    final String fragment2ModifiedContent = "{\"content\": \"Fragment B modified\"}";

    final String expectedSearchUrl = OPENSEARCH_URL + "/default/_search?q=_id:%22" + pageKey
                                     + "%22";

    // when
    publish(
        pageKey,
        new Page("""
            <html>Hello world!
              {{#include src="%s"}}
              {{#include src="%s"}}
            </html>""".formatted(fragment1Key, fragment2Key), pageType)
    );
    assertUrlContentContains(expectedSearchUrl, "Hello world!");

    publish(fragment1Key, new Fragment(fragment1OriginalContent, fragmentType));
    assertUrlContentContains(expectedSearchUrl, "Fragment A original");

    publish(fragment2Key, new Fragment(fragment2OriginalContent, fragmentType));
    assertUrlContentContains(expectedSearchUrl, "Fragment B original");

    // then
    await().atMost(Duration.ofSeconds(5)).alias(expectedSearchUrl).untilAsserted(() -> {
      String indexedJson = assertUrlIsAccessible(expectedSearchUrl);
      assertThat(indexedJson).contains("_source");
      JsonNode source = objectMapper.readTree(indexedJson).get("hits").get("hits").get(0)
          .get("_source");
      JsonUtils.assertNode(source, "payload", "title", pageKey);
      JsonUtils.assertNode(source, "payload", "content", "Hello world!\n  \n  ");

      JsonNode fragments = source.get("fragments");
      JsonUtils.assertNode(fragments.get(0), "key", fragment1Key);
      JsonUtils.assertNode(fragments.get(0), "payload", "content", fragment1OriginalContent);
      JsonUtils.assertNode(fragments.get(1), "key", fragment2Key);
      JsonUtils.assertNode(fragments.get(1), "payload", "content", fragment2OriginalContent);
    });

    // when 2: publish edited fragments
    publish(fragment1Key, new Fragment(fragment1ModifiedContent, fragmentType));
    assertUrlContentContains(expectedSearchUrl, "Fragment A modified");

    publish(fragment2Key, new Fragment(fragment2ModifiedContent, fragmentType));
    assertUrlContentContains(expectedSearchUrl, "Fragment B modified");

    // then 2
    await().atMost(Duration.ofSeconds(5)).alias(expectedSearchUrl).untilAsserted(() -> {
      String indexedJson = assertUrlIsAccessible(expectedSearchUrl);
      assertThat(indexedJson).contains("_source");
      JsonNode source = objectMapper.readTree(indexedJson).get("hits").get("hits").get(0)
          .get("_source");
      JsonUtils.assertNode(source, "payload", "title", pageKey);
      JsonUtils.assertNode(source, "payload", "content", "Hello world!\n  \n  ");

      JsonNode fragments = source.get("fragments");
      JsonUtils.assertNode(fragments.get(0), "key", fragment1Key);
      JsonUtils.assertNode(fragments.get(0), "payload", "content", fragment1ModifiedContent);
      JsonUtils.assertNode(fragments.get(1), "key", fragment2Key);
      JsonUtils.assertNode(fragments.get(1), "payload", "content", fragment2ModifiedContent);
    });
  }

  @Test
  void shouldConvertDataToIndexableResourceAvailableInSearch() throws Exception {
    assumeThat(settings.hasSearch).isTrue();

    // given
    String key = "cat:1";
    String dataContent = """
        {
          "id": "1",
          "name": "Bags"
        }""";
    Data data = new Data(dataContent, "category");

    // when
    publish(key, data);

    // then
    assertUrlContentContains(OPENSEARCH_URL + "/default/_search?q=_id:%22category:1%22",
        JsonUtils.makeSingleLine("""
            "hits": [
                {
                    "_index": "default",
                    "_id": "category:1",
                    "_score": 1.0,
                    "_source": {"""),
        JsonUtils.makeSingleLine("""
                        "fragments": []"""),
        JsonUtils.makeSingleLine("""
                        "namespace": "category\""""),
        JsonUtils.makeSingleLine("""
                        "type": "category\""""),
        JsonUtils.makeSingleLine("""
                        "payload": {"""),
        JsonUtils.makeSingleLine("""
                            "id": "1\""""),
        JsonUtils.makeSingleLine("""
                            "name": "Bags\"""")
    );
  }

  @Test
  void shouldRenderPageFromAggregatedData() throws Exception {
    // given
    final int productId = 1;
    final int productPrice = 200;
    final String productType = "product/simple";
    final String priceType = "prices/price";

    // and: publish product data and price to be aggregated
    publish(
        "pim:" + productId,
        new Data("{\"id\":\"%s\"}".formatted(productId), productType)
    );
    publish(
        "price:" + productId,
        new Data("{\"price\":\"%d\"}".formatted(productPrice), priceType)
    );
    sleepOneSecond();

    // and: publish RenderingContext
    String rendererKey = "test-renderer";
    RenderingContext renderingContext = new RenderingContext(
        rendererKey,
        "product:.*",
        productType,
        "product-pages/{{id}}.html",
        null,
        OutputFormat.PAGE
    );

    // and: publish Renderer
    String renderingContextKey = "test-rendering-context";
    publish(renderingContextKey, renderingContext);
    sleepOneSecond();

    Renderer renderer = new Renderer("Product with ID {{id}} has price {{price}}");
    publish(rendererKey, renderer);
    sleepOneSecond();

    // then: expect rendered page
    assertUrlContent(
        settings.webServerUrl + "/product-pages/" + productId + ".html",
        "Product with ID " + productId + " has price " + productPrice
    );
  }

  @Test
  void shouldCollectData() throws Exception {
    // given
    final String productType = "product/simple";
    final String product1Id = "Table";
    final String product2Id = "Recliner";
    final String product3Id = "Chair";

    final String product1 = """
        {
          "id": "Table",
          "name": "Ravenna Home Open Storage Table",
          "primaryImage": {
            "url": "https://storage.googleapis.com/puresight-pim-images/original/81/81pUqqzgTSL.jpg"
          },
          "categories": [
            { "name": "Furniture" }
          ],
          "price": { "value": 700 }
        }
        """;
    final String product2 = """
        {
          "id": "Recliner",
          "name": "LeatherSoft Recliner with Armrest Storage",
          "primaryImage": {
            "url": "https://storage.googleapis.com/puresight-pim-images/original/81/81VDBfCPBcL.jpg"
          },
          "categories": [
            { "name": "Furniture" }
          ],
          "price": { "value": 680 }
        }
        """;
    final String product3 = """
        {
          "id": "Chair",
          "name": "Rivet Bristol Natural Chair",
          "primaryImage": {
            "url": "https://storage.googleapis.com/puresight-pim-images/original/91/91t1Tzi3TNL.jpg"
          },
          "categories": [
            { "name": "Furniture" }
          ],
          "price": { "value": 670 }
        }
        """;

    // when
    publish("pim:" + product1Id, new Data(product1, productType));
    publish("pim:" + product2Id, new Data(product2, productType));
    publish("pim:" + product3Id, new Data(product3, productType));
    sleepOneSecond();

    // then
    String resultJson = assertUrlIsAccessible(
        settings.webServerUrl + "/collected_products_cheapest-by-category_Furniture.json"
    );

    // expecting products to be collected by price ascending
    String expectedCollectedDataJson = (
        "{"
        + "\"key\":\"collected_products_cheapest-by-category_Furniture\","
        + "\"values\":[" + product3 + "," + product2 + "," + product1 + "]"
        + "}"
    );
    if (settings.processExternalResourcesInPages) {
      expectedCollectedDataJson = expectedCollectedDataJson
          .replace("https://storage.googleapis.com", "/https_storage.googleapis.com");
    }

    assertThat(JsonUtils.formatJson(resultJson))
        .isEqualTo(JsonUtils.formatJson(expectedCollectedDataJson));
  }

  @Test
  void shouldReplaceUrlIncludes() throws Exception {
    // given
    String productsListHtml = """
        <ul>
          <li>Book</li>
          <li>Bag</li>
          <li>Car</li>
        </ul>
        """;

    String productsPageHtml = """
        <html>
          <body>
            Welcome at the products page.<br />
            {{#include src="products-list.html"}}
          </body>
        </html>
        """;

    // when
    publish("products-list.html", new Page(productsListHtml, "fragment"));
    publish("products-page.html", new Page(productsPageHtml, "page"));

    // then
    assertUrlContent(settings.nginxUrl + "/products-page.html", """
        <html>
          <body>
            Welcome at the products page.<br />
            <ul>
          <li>Book</li>
          <li>Bag</li>
          <li>Car</li>
        </ul>

          </body>
        </html>
        """);
  }

  @Test
  void shouldRewriteKeys() throws Exception {
    assumeThat(settings.shouldRewriteKeys).isTrue();

    // given
    String key = "/pages/to-be-rewritten/page1.html";
    String html = "<html />";

    // when
    publish(key, new Page(html, "pages/basic"));
    // then
    assertUrlContent(settings.webServerUrl + "/pages/rewritten/page1.html", html);

    // and
    unpublish(key, Page.class);
    // then
    assertUrlIsNotAccessible(key);
  }

  private void publish(String key, Object resource) throws Exception {
    publish(key, resource, Collections.emptyMap());
  }

  private void publish(String key, Object resource, Map<String, String> extensions)
      throws Exception {
    String eventType = getEventType(resource.getClass(), "TYPE_PUBLISHED");
    var builder = CloudEventUtils.baseBuilder(key, eventType, CloudEventUtils.getNow());
    extensions.forEach(builder::withExtension);
    CloudEvent event = CloudEventUtils.withData(builder, resource).build();
    send(event);
  }

  private void unpublish(String key, Class<?> resourceClass) throws Exception {
    String eventType = getEventType(resourceClass, "TYPE_UNPUBLISHED");
    CloudEvent event = CloudEventUtils.eventWithoutData(key, eventType);
    send(event);
  }

  private static String getEventType(Class<?> cls, String fieldName) throws  Exception {
    Field eventTypeField = cls.getDeclaredField(fieldName);
    eventTypeField.setAccessible(true);
    return (String) eventTypeField.get(null);
  }

  private static void send(CloudEvent event) throws StreamxClientException {
    try (StreamxClient client = StreamxClient.create(INGESTION_URL)) {
      Publisher publisher = client.newPublisher();
      publisher.send(event);
    }
  }

  private static void sleepOneSecond() {
    ThreadUtils.sleepQuietly(Duration.ofSeconds(1));
  }

  static class UrlAssertions {

    private static Consumer<String> emptyConsumer() {
      return string -> {
      };
    }

    static String assertUrlIsAccessible(String url) throws IOException {
      return assertUrlIsAccessible(url, emptyConsumer());
    }

    private static String assertUrlIsAccessible(String url, Consumer<String> bodyAssertion)
        throws IOException {
      return assertAllUrlsAreAccessible(Map.of(url, bodyAssertion)).get(url);
    }

    static void assertAllUrlsAreAccessible(String... urls) throws IOException {
      assertAllUrlsAreAccessible(
          Arrays.stream(urls).collect(Collectors.toMap(
              Function.identity(),
              url -> emptyConsumer()
          ))
      );
    }

    private static Map<String, String> assertAllUrlsAreAccessible(
        Map<String, Consumer<String>> urlsAndBodyPredicate) throws IOException {
      Map<String, String> urlContentMap = new LinkedHashMap<>();
      try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
        for (var entry : urlsAndBodyPredicate.entrySet()) {
          String url = entry.getKey();
          Consumer<String> bodyPredicate = entry.getValue();
          await().atMost(Duration.ofSeconds(10)).untilAsserted(() -> {
            AtomicInteger status = new AtomicInteger();
            AtomicReference<String> body = new AtomicReference<>();
            assertThatCode(() -> {
              HttpGet request = new HttpGet(url);
              CloseableHttpResponse response = httpClient.execute(request);
              status.set(response.getStatusLine().getStatusCode());
              body.set(new String(response.getEntity().getContent().readAllBytes()));
            }).doesNotThrowAnyException();

            assertThat(status).describedAs(url).hasValue(200);

            bodyPredicate.accept(body.get());

            urlContentMap.put(url, body.get());
          });
        }
      }
      return urlContentMap;
    }

    static void assertUrlIsNotAccessible(String url) throws IOException {
      try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
        await().atMost(Duration.ofSeconds(3)).untilAsserted(() -> {
          try {
            HttpGet request = new HttpGet(url);
            CloseableHttpResponse response = httpClient.execute(request);
            int status = response.getStatusLine().getStatusCode();

            assertThat(status).describedAs(url).isEqualTo(400);
          } catch (IOException ex) {
            assertThat(true).isTrue();
          }
        });
      }
    }

    static void assertUrlContent(String url, String expectedContent) throws IOException {
      assertUrlIsAccessible(
          url,
          actual -> assertThat(actual).isEqualTo(expectedContent)
      );
    }

    static void assertUrlContentContains(String url, String... expectedItems) {
      await().atMost(Duration.ofSeconds(5)).alias(url).untilAsserted(() -> {
        String indexedJson = assertUrlIsAccessible(url);
        assertThat(indexedJson).contains(expectedItems);
      });
    }

    static void assertUrlContentLines(String url, List<String> expectedContentLines)
        throws IOException {
      assertUrlIsAccessible(
          url,
          actual -> assertThat(actual.lines()).containsSubsequence(expectedContentLines)
      );
    }
  }

  static class JsonUtils {

    static String makeSingleLine(String json) {
      return json.replaceAll(" *\n *", "").replace(": ", ":").trim();
    }

    static String formatJson(String json) throws JsonProcessingException {
      JsonNode tree = objectMapper.readTree(json);
      return objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(tree);
    }

    static void assertNode(JsonNode node, String field, String expectedValue) {
      JsonNode jsonNode = node.get(field);
      assertThat(jsonNode).isNotNull();
      assertThat(jsonNode.textValue()).isEqualTo(expectedValue);
    }

    static void assertNode(JsonNode node, String field1, String field2,
        String expectedValue) {
      JsonNode node1 = node.get(field1);
      assertThat(node1).isNotNull();
      JsonNode node2 = node1.get(field2);
      assertThat(node2).isNotNull();
      assertThat(node2.textValue()).isEqualTo(expectedValue);
    }
  }
}