SXPR-145 Add external-resources-processing-service to ingest images f…

config/application-cloud.properties

@@ -8,3 +8,4 @@ streamx.kubernetes.controlled-resource-definitions=${apisix.resources}
 streamx.default.ingestion.replicas=2
 streamx.default.processing.replicas=1
 streamx.default.delivery.replicas=2
+streamx.rocksdb.path=/var/lib/rocksdb

config/application.properties

@@ -3,3 +3,4 @@ streamx.accelerator.ingestion.host=ingestion.${streamx.accelerator.ip}.nip.io
 streamx.accelerator.web.host=web.${streamx.accelerator.ip}.nip.io
 streamx.ingestion.url=http://${streamx.accelerator.ingestion.host}
 streamx.accelerator.web.url=http://${streamx.accelerator.web.host}
+streamx.rocksdb.path=/tmp

data/catalog/products.stream

@@ -48213,7 +48213,7 @@
   "action": "publish",
   "eventTime": null,
   "properties": {
-    "sx:type": "data/product"
+    "sx:type": "data/product_with_images"
   },
   "payload": {
     "dev.streamx.blueprints.data.Data": {

mesh/deployment.yaml

@@ -40,7 +40,7 @@ delivery:
       repository-volume:
         size: 10Gi
       rocksdb:
-        mountPath: "/var/lib/rocksdb"
+        mountPath: "${streamx.rocksdb.path}"
         size: 5Gi
     ingress:
       enabled: true
@@ -111,7 +111,7 @@ delivery:
     stateful: true
     volumes:
       rocksdb:
-        mountPath: "/var/lib/rocksdb"
+        mountPath: "${streamx.rocksdb.path}"
         size: 5Gi
     ingress:
       enabled: true

mesh/mesh.yaml

@@ -1,5 +1,5 @@
 defaultRegistry: ghcr.io/streamx-dev/streamx-blueprints
-defaultImageTag: 2.0.7-jvm
+defaultImageTag: 2.0.13-jvm
 
 sources:
   github:
@@ -117,18 +117,42 @@ processing:
       configs:
         - "image-optimization.properties"
 
+
+  blueprint-data-deps-rewriter:
+    image: external-resources-processing-service
+    incoming:
+      incoming-data:
+        topic: inboxes/data
+      incoming-pages:
+        topic: relays/fake-pages-channel
+      incoming-web-resources:
+        topic: relays/fake-web-resources
+    outgoing:
+      outgoing-data:
+        topic: relays/data
+      outgoing-assets:
+        topic: outboxes/assets
+      outgoing-pages:
+        topic: relays/fake-pages-channel
+      outgoing-web-resources:
+        topic: relays/fake-web-resources-channel
+    environment:
+      STREAMX_BLUEPRINTS_EXTERNAL_RESOURCES_PROCESSING_SERVICE_BASE_URL_FOR_RELATIVE_PATHS: "${streamx.accelerator.web.url}"
+      STREAMX_BLUEPRINTS_EXTERNAL_RESOURCES_PROCESSING_SERVICE_PROCESSABLE_SX_TYPES: "data/product_with_images"
+      STREAMX_BLUEPRINTS_EXTERNAL_RESOURCES_PROCESSING_SERVICE_JSON_EXTERNAL_RESOURCE_JSONPATH_SELECTORS: "$..primaryImage.url"
+
   blueprint-json-aggregator:
     image: json-aggregator-processing-service
     incoming:
       data:
-        topic: inboxes/data
+        topic: relays/data
       multivalued-data:
-        topic: inboxes/data
+        topic: relays/data
     outgoing:
       aggregated-data:
         topic: relays/aggregated-data
       aggregated-multivalued-data:
-        topic: inboxes/data
+        topic: relays/data
     environmentFrom:
       configs:
         - "aggregator.properties"
@@ -237,7 +261,7 @@ delivery:
       STREAMX_BLUEPRINTS_OPENSEARCH-DELIVERY-SERVICE_MIGRATION-SCRIPT-LOCATIONS: "classpath:opensearch/service-init,file:opensearch/service-init"
       # Keep stores up to date when container inside a pod restarts
       QUASAR_MESSAGING_STORE_BACKEND: "rocksdb"
-      QUASAR_MESSAGING_STORE_ROCKSDB_PATH: "/var/lib/rocksdb"
+      QUASAR_MESSAGING_STORE_ROCKSDB_PATH: "${streamx.rocksdb.path}"
     environmentFrom:
       secrets:
         - "opensearch-secrets.properties"
@@ -270,7 +294,7 @@ delivery:
       STREAMX_URL-INCLUDE-REPLACEMENT-PROVIDER_SSI-INCLUDE-FALLBACK-CONTENT: "<!-- Include Not Found -->"
       # Keep stores up to date when container inside a pod restarts
       QUASAR_MESSAGING_STORE_BACKEND: "rocksdb"
-      QUASAR_MESSAGING_STORE_ROCKSDB_PATH: "/var/lib/rocksdb"
+      QUASAR_MESSAGING_STORE_ROCKSDB_PATH: "${streamx.rocksdb.path}"
     repositoryVolume: "/srv/www"
     port: 8083
     components:

mesh/services/streamx-service-metadata/streamx-blueprints/2.0.6/composition-engine-processing-service/2.0.6-jvm.json → mesh/services/streamx-service-metadata/streamx-blueprints/2.0.13/composition-engine-processing-service/2.0.13-jvm.json

@@ -1,52 +1,37 @@
 {
-  "image": "ghcr.io/streamx-dev/streamx-blueprints/composition-engine-processing-service:2.0.6-jvm",
+  "image": "ghcr.io/streamx-dev/streamx-blueprints/composition-engine-processing-service:2.0.13-jvm",
   "type": "processing",
   "incoming": [
     {
       "name": "compositions",
-      "defaultSchemaId": "Composition_AVROSchema_4a3dd59dd13c43099ce84b485bf8965c5f18adf3",
+      "defaultSchemaId": "Composition_AVROSchema_777bc4dfb7d9f8890cea1e4ef60adf80f00cf1a9",
       "connector": "smallrye-pulsar"
     },
     {
       "name": "layouts",
-      "defaultSchemaId": "Layout_AVROSchema_e30b45af5d69094e955c34a0477d2da3da0320bf",
+      "defaultSchemaId": "Layout_AVROSchema_8db48ade1b3935c38328733b8a41e531cc09a8cd",
       "connector": "smallrye-pulsar"
     },
     {
       "name": "incoming-page-compose-requests",
-      "defaultSchemaId": "PageComposeRequest_AVROSchema_f1b67786d384de577ac5bcbca375d75c11bdc11c",
+      "defaultSchemaId": "PageComposeRequest_AVROSchema_4ddbe9e45593f752658beafaaf870efbbaf4e33c",
       "connector": "smallrye-pulsar"
     }
   ],
   "outgoing": [
     {
       "name": "outgoing-page-compose-requests",
-      "defaultSchemaId": "PageComposeRequest_AVROSchema_f1b67786d384de577ac5bcbca375d75c11bdc11c",
+      "defaultSchemaId": "PageComposeRequest_AVROSchema_4ddbe9e45593f752658beafaaf870efbbaf4e33c",
       "connector": "smallrye-pulsar"
     },
     {
       "name": "pages",
-      "defaultSchemaId": "Page_AVROSchema_09d27de42e84c61159a1c1f36510c9dc29f13500",
+      "defaultSchemaId": "Page_AVROSchema_18e8709c7e670b687b38be81b3dde84935ae3745",
       "connector": "smallrye-pulsar"
     }
   ],
   "schemas": {
-    "Composition_AVROSchema_4a3dd59dd13c43099ce84b485bf8965c5f18adf3": {
-      "type": "record",
-      "name": "Composition",
-      "namespace": "dev.streamx.blueprints.data",
-      "fields": [
-        {
-          "name": "layoutKey",
-          "type": "string"
-        },
-        {
-          "name": "content",
-          "type": "bytes"
-        }
-      ]
-    },
-    "Page_AVROSchema_09d27de42e84c61159a1c1f36510c9dc29f13500": {
+    "Page_AVROSchema_18e8709c7e670b687b38be81b3dde84935ae3745": {
       "type": "record",
       "name": "Page",
       "namespace": "dev.streamx.blueprints.data",
@@ -57,7 +42,7 @@
         }
       ]
     },
-    "Layout_AVROSchema_e30b45af5d69094e955c34a0477d2da3da0320bf": {
+    "Layout_AVROSchema_8db48ade1b3935c38328733b8a41e531cc09a8cd": {
       "type": "record",
       "name": "Layout",
       "namespace": "dev.streamx.blueprints.data",
@@ -68,7 +53,7 @@
         }
       ]
     },
-    "PageComposeRequest_AVROSchema_f1b67786d384de577ac5bcbca375d75c11bdc11c": {
+    "PageComposeRequest_AVROSchema_4ddbe9e45593f752658beafaaf870efbbaf4e33c": {
       "type": "record",
       "name": "PageComposeRequest",
       "namespace": "dev.streamx.blueprints.composition",
@@ -82,6 +67,21 @@
           "type": "string"
         }
       ]
+    },
+    "Composition_AVROSchema_777bc4dfb7d9f8890cea1e4ef60adf80f00cf1a9": {
+      "type": "record",
+      "name": "Composition",
+      "namespace": "dev.streamx.blueprints.data",
+      "fields": [
+        {
+          "name": "layoutKey",
+          "type": "string"
+        },
+        {
+          "name": "content",
+          "type": "bytes"
+        }
+      ]
     }
   }
 }

mesh/services/streamx-service-metadata/streamx-blueprints/2.0.6/data-collector-processing-service/2.0.6-jvm.json → mesh/services/streamx-service-metadata/streamx-blueprints/2.0.13/data-collector-processing-service/2.0.13-jvm.json

@@ -1,27 +1,27 @@
 {
-  "image": "ghcr.io/streamx-dev/streamx-blueprints/data-collector-processing-service:2.0.6-jvm",
+  "image": "ghcr.io/streamx-dev/streamx-blueprints/data-collector-processing-service:2.0.13-jvm",
   "type": "processing",
   "incoming": [
     {
       "name": "data",
-      "defaultSchemaId": "Data_AVROSchema_e8d9b0b0f0340f665d12aaabf62ea8c48c2c8c74",
+      "defaultSchemaId": "Data_AVROSchema_f7334b3902d61d92c614ea27ba284371d711a85c",
       "connector": "smallrye-pulsar"
     }
   ],
   "outgoing": [
     {
       "name": "web-resources",
-      "defaultSchemaId": "WebResource_AVROSchema_991c586ba1901a07df5004c0a80414839e50e59a",
+      "defaultSchemaId": "WebResource_AVROSchema_5b356d70b4744b8fb7b60caf075559b44690e295",
       "connector": "smallrye-pulsar"
     },
     {
       "name": "collected-data",
-      "defaultSchemaId": "Data_AVROSchema_e8d9b0b0f0340f665d12aaabf62ea8c48c2c8c74",
+      "defaultSchemaId": "Data_AVROSchema_f7334b3902d61d92c614ea27ba284371d711a85c",
       "connector": "smallrye-pulsar"
     }
   ],
   "schemas": {
-    "WebResource_AVROSchema_991c586ba1901a07df5004c0a80414839e50e59a": {
+    "WebResource_AVROSchema_5b356d70b4744b8fb7b60caf075559b44690e295": {
       "type": "record",
       "name": "WebResource",
       "namespace": "dev.streamx.blueprints.data",
@@ -32,7 +32,7 @@
         }
       ]
     },
-    "Data_AVROSchema_e8d9b0b0f0340f665d12aaabf62ea8c48c2c8c74": {
+    "Data_AVROSchema_f7334b3902d61d92c614ea27ba284371d711a85c": {
       "type": "record",
       "name": "Data",
       "namespace": "dev.streamx.blueprints.data",

mesh/services/streamx-service-metadata/streamx-blueprints/2.0.13/external-resources-processing-service/2.0.13-jvm.json

@@ -0,0 +1,89 @@
+{
+  "image": "ghcr.io/streamx-dev/streamx-blueprints/external-resources-processing-service:2.0.13-jvm",
+  "type": "processing",
+  "incoming": [
+    {
+      "name": "incoming-web-resources",
+      "defaultSchemaId": "WebResource_AVROSchema_ce805f0dc93f73300e87cee0f8323a9aa9469251",
+      "connector": "smallrye-pulsar"
+    },
+    {
+      "name": "incoming-pages",
+      "defaultSchemaId": "Page_AVROSchema_078dc3a84d247f1089e710b89f8d849e89480539",
+      "connector": "smallrye-pulsar"
+    },
+    {
+      "name": "incoming-data",
+      "defaultSchemaId": "Data_AVROSchema_22f7a0b8faabdf5ffacd6067836fb77ef06511c4",
+      "connector": "smallrye-pulsar"
+    }
+  ],
+  "outgoing": [
+    {
+      "name": "outgoing-web-resources",
+      "defaultSchemaId": "WebResource_AVROSchema_ce805f0dc93f73300e87cee0f8323a9aa9469251",
+      "connector": "smallrye-pulsar"
+    },
+    {
+      "name": "outgoing-pages",
+      "defaultSchemaId": "Page_AVROSchema_078dc3a84d247f1089e710b89f8d849e89480539",
+      "connector": "smallrye-pulsar"
+    },
+    {
+      "name": "outgoing-data",
+      "defaultSchemaId": "Data_AVROSchema_22f7a0b8faabdf5ffacd6067836fb77ef06511c4",
+      "connector": "smallrye-pulsar"
+    },
+    {
+      "name": "outgoing-assets",
+      "defaultSchemaId": "Asset_AVROSchema_13cc883c8fa6d556928578e480478921789dc327",
+      "connector": "smallrye-pulsar"
+    }
+  ],
+  "schemas": {
+    "Data_AVROSchema_22f7a0b8faabdf5ffacd6067836fb77ef06511c4": {
+      "type": "record",
+      "name": "Data",
+      "namespace": "dev.streamx.blueprints.data",
+      "fields": [
+        {
+          "name": "content",
+          "type": "bytes"
+        }
+      ]
+    },
+    "Asset_AVROSchema_13cc883c8fa6d556928578e480478921789dc327": {
+      "type": "record",
+      "name": "Asset",
+      "namespace": "dev.streamx.blueprints.data",
+      "fields": [
+        {
+          "name": "content",
+          "type": "bytes"
+        }
+      ]
+    },
+    "WebResource_AVROSchema_ce805f0dc93f73300e87cee0f8323a9aa9469251": {
+      "type": "record",
+      "name": "WebResource",
+      "namespace": "dev.streamx.blueprints.data",
+      "fields": [
+        {
+          "name": "content",
+          "type": "bytes"
+        }
+      ]
+    },
+    "Page_AVROSchema_078dc3a84d247f1089e710b89f8d849e89480539": {
+      "type": "record",
+      "name": "Page",
+      "namespace": "dev.streamx.blueprints.data",
+      "fields": [
+        {
+          "name": "content",
+          "type": "bytes"
+        }
+      ]
+    }
+  }
+}

mesh/services/streamx-service-metadata/streamx-blueprints/2.0.6/image-optimization-processing-service/2.0.6-jvm.json → mesh/services/streamx-service-metadata/streamx-blueprints/2.0.13/image-optimization-processing-service/2.0.13-jvm.json

@@ -1,32 +1,32 @@
 {
-  "image": "ghcr.io/streamx-dev/streamx-blueprints/image-optimization-processing-service:2.0.6-jvm",
+  "image": "ghcr.io/streamx-dev/streamx-blueprints/image-optimization-processing-service:2.0.13-jvm",
   "type": "processing",
   "incoming": [
     {
       "name": "incoming-assets",
-      "defaultSchemaId": "Asset_AVROSchema_9c1b46923b9301541888783b43958adc4300d7a9",
+      "defaultSchemaId": "Asset_AVROSchema_c5ec0ea9532a206c8425e4f408e616e6770ab00e",
       "connector": "smallrye-pulsar"
     },
     {
       "name": "incoming-pages",
-      "defaultSchemaId": "Page_AVROSchema_7df242c27c9c8d7857d4a788c97be58e7744d6c9",
+      "defaultSchemaId": "Page_AVROSchema_bf4bda261b3d2aa4c4cba247699ca235a59e86a0",
       "connector": "smallrye-pulsar"
     }
   ],
   "outgoing": [
     {
       "name": "optimized-assets",
-      "defaultSchemaId": "Asset_AVROSchema_9c1b46923b9301541888783b43958adc4300d7a9",
+      "defaultSchemaId": "Asset_AVROSchema_c5ec0ea9532a206c8425e4f408e616e6770ab00e",
       "connector": "smallrye-pulsar"
     },
     {
       "name": "outgoing-pages",
-      "defaultSchemaId": "Page_AVROSchema_7df242c27c9c8d7857d4a788c97be58e7744d6c9",
+      "defaultSchemaId": "Page_AVROSchema_bf4bda261b3d2aa4c4cba247699ca235a59e86a0",
       "connector": "smallrye-pulsar"
     }
   ],
   "schemas": {
-    "Page_AVROSchema_7df242c27c9c8d7857d4a788c97be58e7744d6c9": {
+    "Page_AVROSchema_bf4bda261b3d2aa4c4cba247699ca235a59e86a0": {
       "type": "record",
       "name": "Page",
       "namespace": "dev.streamx.blueprints.data",
@@ -37,7 +37,7 @@
         }
       ]
     },
-    "Asset_AVROSchema_9c1b46923b9301541888783b43958adc4300d7a9": {
+    "Asset_AVROSchema_c5ec0ea9532a206c8425e4f408e616e6770ab00e": {
       "type": "record",
       "name": "Asset",
       "namespace": "dev.streamx.blueprints.data",