From 243064b218bd3e6ac112cbfc09343418b2c0f866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Mon, 29 Dec 2025 14:34:09 +0100 Subject: [PATCH 01/36] DXP-2486 Add streamx run, stream and batch commands for StreamX 2.0 (and remove streamx publish/unpublish/cloud/dev/init commands for now) --- core/pom.xml | 13 +- .../java/dev/streamx/cli/BannerPrinter.java | 3 +- .../java/dev/streamx/cli/SchemaProvider.java | 96 - .../java/dev/streamx/cli/StreamxCommand.java | 21 +- .../command/cloud/KubernetesArguments.java | 30 - .../cli/command/cloud/KubernetesConfig.java | 24 - .../cli/command/cloud/KubernetesService.java | 147 - .../cli/command/cloud/MetadataUtils.java | 57 - .../command/cloud/ProjectPathsResolver.java | 38 - .../command/cloud/ServiceMeshResolver.java | 182 - .../collector/ClusterResourcesCollector.java | 84 - .../DirectoryResourcesCollector.java | 71 - .../KubernetesResourcesCollector.java | 10 - .../TypedClusterResourceCollector.java | 39 - .../cli/command/cloud/deploy/Config.java | 14 - .../command/cloud/deploy/ConfigService.java | 77 - .../cli/command/cloud/deploy/DataService.java | 99 - .../command/cloud/deploy/DeployCommand.java | 123 - .../deploy/ProjectResourcesExtractor.java | 80 - .../command/cloud/deploy/ResourceCleaner.java | 47 - .../cloud/undeploy/UndeployCommand.java | 38 - .../cli/command/dev/BrowserOpener.java | 36 - .../cli/command/dev/DashboardContainer.java | 31 - .../cli/command/dev/DashboardRunner.java | 56 - .../streamx/cli/command/dev/DevCommand.java | 135 - .../streamx/cli/command/dev/DevConfig.java | 20 - .../cli/command/dev/DockerValidator.java | 28 - .../command/dev/event/DashboardStarted.java | 5 - .../cli/command/dev/event/DevReady.java | 5 - .../ingestion/BaseIngestionCommand.java | 92 +- .../command/ingestion/CloudEventBuilder.java | 52 + .../ingestion/IngestionArgumentConfig.java | 45 - .../IngestionMessageJsonFactory.java | 52 - .../ingestion/IngestionTargetArguments.java | 21 - .../ingestion/StreamxClientProvider.java | 6 +- .../command/ingestion/batch/BatchCommand.java | 97 +- .../batch/BatchIngestionArguments.java | 14 +- .../batch/EventSourceDescriptor.java | 29 +- .../substitutor/StringSubstitutor.java | 5 +- .../resolver/substitutor/Substitutor.java | 3 +- .../walker/EventSourceFileTreeWalker.java | 4 +- .../ingestion/publish/PayloadArgument.java | 90 - .../ingestion/publish/PayloadArguments.java | 40 - .../ingestion/publish/PublishArguments.java | 24 - .../ingestion/publish/PublishCommand.java | 87 - .../publish/payload/JsonPathExtractor.java | 29 - .../publish/payload/PayloadResolver.java | 186 - ...tyCreatingJacksonJsonNodeJsonProvider.java | 33 - .../payload/source/ExactSourceResolver.java | 13 - .../payload/source/FileSourceResolver.java | 16 - .../publish/payload/source/RawPayload.java | 5 - .../payload/source/SourceResolver.java | 22 - .../publish/payload/typed/SourceType.java | 17 - .../publish/payload/typed/TypedPayload.java | 7 - .../typed/TypedPayloadFragmentResolver.java | 48 - .../ingestion/stream/StreamCommand.java | 51 +- .../stream/StreamIngestionArguments.java | 11 +- .../parser/StreamIngestionJsonParser.java | 4 +- .../parser/StreamIngestionProcessor.java | 4 +- .../ingestion/unpublish/UnpublishCommand.java | 39 - .../streamx/cli/command/init/GitClient.java | 67 - .../streamx/cli/command/init/InitCommand.java | 100 - .../cli/command/init/InitProjectConfig.java | 16 - .../ConfigurableProjectTemplateSource.java | 15 - .../template/ProdProjectTemplateSource.java | 12 - .../ProjectTemplateBeanConfiguration.java | 31 - .../template/ProjectTemplateSource.java | 6 - .../meshprocessing/ContainerWatcher.java | 12 +- .../MeshDefinitionResolver.java | 2 +- .../command/meshprocessing/MeshManager.java | 16 +- .../streamx/cli/command/run/RunCommand.java | 7 +- .../run/RunningMeshPropertiesGenerator.java | 2 +- .../cli/exception/DockerException.java | 4 +- .../streamx/cli/exception/GitException.java | 30 - .../JsonPathReplacementException.java | 70 - .../cli/exception/KubernetesException.java | 64 - ...bleToConnectIngestionServiceException.java | 28 - .../exception/UnknownChannelException.java | 20 - .../java/dev/streamx/cli/model/Resource.java | 8 + .../dev/streamx/cli/util/CollectionUtils.java | 13 - .../test/java/dev/streamx/cli/OsUtils.java | 5 +- .../dev/streamx/cli/command/MeshStopper.java | 2 +- .../cloud/ConfigurationInterpolationTest.java | 95 - .../cloud/KubernetesClientProfile.java | 12 - .../cloud/ProjectPathsResolverTest.java | 41 - .../cli/command/cloud/ProjectUtils.java | 33 - .../command/cloud/ServiceMeshCrdProvider.java | 25 - .../cloud/ServiceMeshResolverTest.java | 132 - .../cloud/deploy/ConfigServiceTest.java | 132 - .../command/cloud/deploy/DataServiceTest.java | 90 - .../command/cloud/deploy/DeployCommandIT.java | 27 - .../cloud/deploy/KubernetesServiceTest.java | 69 - .../deploy/ProjectResourcesExtractorTest.java | 70 - .../cloud/undeploy/UndeployCommandIT.java | 24 - .../cli/command/dev/DevCommandTest.java | 264 - .../cli/command/dev/DockerValidatorTest.java | 122 - .../ingestion/BaseIngestionCommandTest.java | 89 +- .../IngestionArgumentsValidationTest.java | 94 - .../batch/BatchPublishCommandTest.java | 201 +- .../batch/BatchUnpublishCommandTest.java | 107 +- .../batch/CloudEventJsonMatcher.java | 23 + .../ingestion/publish/PublishCommandTest.java | 268 - .../publish/PublishPayloadCommandTest.java | 305 - .../payload/JsonPathExtractorTest.java | 55 - .../publish/payload/PayloadResolverTest.java | 147 - .../ingestion/stream/StreamCommandTest.java | 98 +- .../unpublish/UnpublishCommandTest.java | 136 - .../cli/command/init/GitClientTest.java | 141 - .../cli/command/init/GitClientUtils.java | 22 - .../cli/command/init/InitCommandMainTest.java | 130 - .../cli/command/init/InitCommandTest.java | 131 - .../meshprocessing/MeshResolverTest.java | 16 +- .../cli/command/run/RunCommandTest.java | 14 +- .../cli/command/util/CommandTransformer.java | 33 - .../cli/command/util/MeshTestsUtils.java | 10 +- .../cli/license/LicenseAcceptingTest.java | 17 +- .../deploy/servicemeshes.streamx.dev-v1.yml | 5155 ----------------- .../project/configs/delivery/wds.properties | 1 - .../project/configs/delivery/wds/dir/file.txt | 1 - .../configs/delivery/wds/dir/file1.txt | 1 - .../project/configs/delivery/wds/file.txt | 1 - .../configs/delivery/wds/nginx.properties | 1 - .../configs/delivery/wds/nginx/file.txt | 1 - .../cloud/project/configs/global.properties | 2 - .../project/configs/ingestion/rest.properties | 1 - .../project/configs/ingestion/rest/file.txt | 1 - .../project/configs/invalid/invalid[]name.txt | 1 - .../invalid/invalid_prop_key.properties | 1 - .../configs/processing/relay.properties | 1 - .../project/configs/processing/relay/file.txt | 1 - .../cloud/project/configs/relay.properties | 1 - .../cloud/project/configs/shared.properties | 1 - .../shared/do_not_delete/DO_NOT_DELETE.txt | 1 - .../cloud/project/configs/shared/file.txt | 1 - .../cloud/project/configs/shared/file1.txt | 1 - .../project/configuration-interpolation.yaml | 31 - .../command/cloud/project/custom-name.yaml | 31 - ...eployment.configuration-interpolation.yaml | 19 - .../cloud/project/deployment.custom-name.yaml | 11 - .../project/deployment.empty-deployment.yaml | 0 .../project/deployment.with-configs.yaml | 19 - .../cli/command/cloud/project/deployment.yaml | 11 - .../cloud/project/empty-deployment.yaml | 31 - .../cli/command/cloud/project/empty-mesh.yaml | 0 .../cli/command/cloud/project/mesh.yaml | 33 - .../command/cloud/project/nodeployment.yaml | 31 - .../project/secrets/delivery/wds.properties | 1 - .../project/secrets/delivery/wds/dir/file.txt | 1 - .../secrets/delivery/wds/dir/file1.txt | 1 - .../project/secrets/delivery/wds/file.txt | 1 - .../secrets/delivery/wds/nginx.properties | 1 - .../secrets/delivery/wds/nginx/file.txt | 1 - .../cloud/project/secrets/global.properties | 1 - .../project/secrets/ingestion/rest.properties | 1 - .../project/secrets/ingestion/rest/file.txt | 1 - .../secrets/processing/relay.properties | 1 - .../project/secrets/processing/relay/file.txt | 1 - .../cloud/project/secrets/relay.properties | 1 - .../cloud/project/secrets/shared.properties | 1 - .../cloud/project/secrets/shared/file.txt | 1 - .../cloud/project/secrets/shared/file1.txt | 1 - .../command/cloud/project/with-configs.yaml | 91 - .../streamx/cli/command/dev/failing-mesh.yaml | 32 - .../dev/incremental-reloaded-mesh.yaml | 32 - .../streamx/cli/command/dev/initial-mesh.yaml | 32 - .../batch/invalid-channel/.eventsource.yaml | 7 - .../batch/invalid-channel/index.html | 1 - .../batch/invalid-json/.eventsource.yaml | 4 +- .../batch/unknown-channel/.eventsource.yaml | 7 - .../batch/unknown-channel/index.html | 1 - .../batch/valid-json/.eventsource.yaml | 3 +- .../command/ingestion/batch/valid/index.html | 1 - .../valid/{ => publish}/.eventsource.yaml | 7 +- .../batch/valid/{ => publish}/ignore.me | 0 .../ingestion/batch/valid/publish/index.html | 1 + .../batch/valid/unpublish/.eventsource.yaml | 10 + .../ingestion/batch/valid/unpublish/ignore.me | 1 + .../batch/valid/unpublish/index.html | 1 + .../publish/payload/example-image.png | Bin 2279 -> 0 bytes .../publish/payload/helloworld-payload.json | 1 - .../publish/payload/invalid-payload.json | 2 - .../ingestion/publish/payload/payload.json | 3 - .../ingestion/publish/payload/raw-text.txt | 1 - .../stream/invalid-separated-json.stream | 38 +- .../ingestion/stream/valid-json.stream | 43 +- .../resources/filtered/mesh-interpolated.yaml | 74 +- core/src/test/resources/filtered/mesh.yaml | 74 +- e2e-tests/pom.xml | 3 +- .../batch/publish/index.html | 1 + .../repository-volume/publish/index.html | 1 + .../test/tools/validators/HttpValidator.java | 6 +- .../validators/ProcessOutputValidator.java | 12 +- .../test/java/dev/streamx/cli/OsUtils.java | 2 +- .../dev/streamx/cli/StreamxCliLicenseIT.java | 13 +- .../streamx/cli/StreamxCliPublicationIT.java | 116 +- .../resources/batch/publish/.eventsource.yaml | 11 + .../test/resources/batch/publish/index.html | 1 + .../batch/unpublish/.eventsource.yaml | 8 + .../test/resources/batch/unpublish/index.html | 1 + .../resources/configs/overridden-nginx-conf | 10 - .../src/test/resources/file_param_page.json | 5 - .../src/test/resources/filtered/mesh.yaml | 93 +- .../resources/json_path_file_param_page.txt | 1 - e2e-tests/src/test/resources/payload.json | 5 - .../resources/stream/page-publish-event.json | 15 + .../stream/page-unpublish-event.json | 8 + pom.xml | 37 +- 207 files changed, 722 insertions(+), 11958 deletions(-) delete mode 100644 core/src/main/java/dev/streamx/cli/SchemaProvider.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/cloud/KubernetesArguments.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/cloud/KubernetesConfig.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/cloud/KubernetesService.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/cloud/MetadataUtils.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/cloud/ProjectPathsResolver.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/cloud/ServiceMeshResolver.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/cloud/collector/ClusterResourcesCollector.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/cloud/collector/DirectoryResourcesCollector.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/cloud/collector/KubernetesResourcesCollector.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/cloud/collector/TypedClusterResourceCollector.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/cloud/deploy/Config.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/cloud/deploy/ConfigService.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/cloud/deploy/DataService.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/cloud/deploy/DeployCommand.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/cloud/deploy/ProjectResourcesExtractor.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/cloud/deploy/ResourceCleaner.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/cloud/undeploy/UndeployCommand.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/dev/BrowserOpener.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/dev/DashboardContainer.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/dev/DashboardRunner.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/dev/DevCommand.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/dev/DevConfig.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/dev/DockerValidator.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/dev/event/DashboardStarted.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/dev/event/DevReady.java create mode 100644 core/src/main/java/dev/streamx/cli/command/ingestion/CloudEventBuilder.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/ingestion/IngestionArgumentConfig.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/ingestion/IngestionMessageJsonFactory.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/ingestion/IngestionTargetArguments.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/ingestion/publish/PayloadArgument.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/ingestion/publish/PayloadArguments.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/ingestion/publish/PublishArguments.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/ingestion/publish/PublishCommand.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/ingestion/publish/payload/JsonPathExtractor.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/ingestion/publish/payload/PayloadResolver.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/ingestion/publish/payload/PropertyCreatingJacksonJsonNodeJsonProvider.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/ingestion/publish/payload/source/ExactSourceResolver.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/ingestion/publish/payload/source/FileSourceResolver.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/ingestion/publish/payload/source/RawPayload.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/ingestion/publish/payload/source/SourceResolver.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/ingestion/publish/payload/typed/SourceType.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/ingestion/publish/payload/typed/TypedPayload.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/ingestion/publish/payload/typed/TypedPayloadFragmentResolver.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/ingestion/unpublish/UnpublishCommand.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/init/GitClient.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/init/InitCommand.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/init/InitProjectConfig.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/init/project/template/ConfigurableProjectTemplateSource.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/init/project/template/ProdProjectTemplateSource.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/init/project/template/ProjectTemplateBeanConfiguration.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/init/project/template/ProjectTemplateSource.java delete mode 100644 core/src/main/java/dev/streamx/cli/exception/GitException.java delete mode 100644 core/src/main/java/dev/streamx/cli/exception/JsonPathReplacementException.java delete mode 100644 core/src/main/java/dev/streamx/cli/exception/KubernetesException.java delete mode 100644 core/src/main/java/dev/streamx/cli/exception/UnableToConnectIngestionServiceException.java delete mode 100644 core/src/main/java/dev/streamx/cli/exception/UnknownChannelException.java create mode 100644 core/src/main/java/dev/streamx/cli/model/Resource.java delete mode 100644 core/src/main/java/dev/streamx/cli/util/CollectionUtils.java delete mode 100644 core/src/test/java/dev/streamx/cli/command/cloud/ConfigurationInterpolationTest.java delete mode 100644 core/src/test/java/dev/streamx/cli/command/cloud/KubernetesClientProfile.java delete mode 100644 core/src/test/java/dev/streamx/cli/command/cloud/ProjectPathsResolverTest.java delete mode 100644 core/src/test/java/dev/streamx/cli/command/cloud/ProjectUtils.java delete mode 100644 core/src/test/java/dev/streamx/cli/command/cloud/ServiceMeshCrdProvider.java delete mode 100644 core/src/test/java/dev/streamx/cli/command/cloud/ServiceMeshResolverTest.java delete mode 100644 core/src/test/java/dev/streamx/cli/command/cloud/deploy/ConfigServiceTest.java delete mode 100644 core/src/test/java/dev/streamx/cli/command/cloud/deploy/DataServiceTest.java delete mode 100644 core/src/test/java/dev/streamx/cli/command/cloud/deploy/DeployCommandIT.java delete mode 100644 core/src/test/java/dev/streamx/cli/command/cloud/deploy/KubernetesServiceTest.java delete mode 100644 core/src/test/java/dev/streamx/cli/command/cloud/deploy/ProjectResourcesExtractorTest.java delete mode 100644 core/src/test/java/dev/streamx/cli/command/cloud/undeploy/UndeployCommandIT.java delete mode 100644 core/src/test/java/dev/streamx/cli/command/dev/DevCommandTest.java delete mode 100644 core/src/test/java/dev/streamx/cli/command/dev/DockerValidatorTest.java delete mode 100644 core/src/test/java/dev/streamx/cli/command/ingestion/IngestionArgumentsValidationTest.java create mode 100644 core/src/test/java/dev/streamx/cli/command/ingestion/batch/CloudEventJsonMatcher.java delete mode 100644 core/src/test/java/dev/streamx/cli/command/ingestion/publish/PublishCommandTest.java delete mode 100644 core/src/test/java/dev/streamx/cli/command/ingestion/publish/PublishPayloadCommandTest.java delete mode 100644 core/src/test/java/dev/streamx/cli/command/ingestion/publish/payload/JsonPathExtractorTest.java delete mode 100644 core/src/test/java/dev/streamx/cli/command/ingestion/publish/payload/PayloadResolverTest.java delete mode 100644 core/src/test/java/dev/streamx/cli/command/ingestion/unpublish/UnpublishCommandTest.java delete mode 100644 core/src/test/java/dev/streamx/cli/command/init/GitClientTest.java delete mode 100644 core/src/test/java/dev/streamx/cli/command/init/GitClientUtils.java delete mode 100644 core/src/test/java/dev/streamx/cli/command/init/InitCommandMainTest.java delete mode 100644 core/src/test/java/dev/streamx/cli/command/init/InitCommandTest.java delete mode 100644 core/src/test/java/dev/streamx/cli/command/util/CommandTransformer.java delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/deploy/servicemeshes.streamx.dev-v1.yml delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/configs/delivery/wds.properties delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/configs/delivery/wds/dir/file.txt delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/configs/delivery/wds/dir/file1.txt delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/configs/delivery/wds/file.txt delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/configs/delivery/wds/nginx.properties delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/configs/delivery/wds/nginx/file.txt delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/configs/global.properties delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/configs/ingestion/rest.properties delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/configs/ingestion/rest/file.txt delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/configs/invalid/invalid[]name.txt delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/configs/invalid/invalid_prop_key.properties delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/configs/processing/relay.properties delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/configs/processing/relay/file.txt delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/configs/relay.properties delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/configs/shared.properties delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/configs/shared/do_not_delete/DO_NOT_DELETE.txt delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/configs/shared/file.txt delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/configs/shared/file1.txt delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/configuration-interpolation.yaml delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/custom-name.yaml delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/deployment.configuration-interpolation.yaml delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/deployment.custom-name.yaml delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/deployment.empty-deployment.yaml delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/deployment.with-configs.yaml delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/deployment.yaml delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/empty-deployment.yaml delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/empty-mesh.yaml delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/mesh.yaml delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/nodeployment.yaml delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/secrets/delivery/wds.properties delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/secrets/delivery/wds/dir/file.txt delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/secrets/delivery/wds/dir/file1.txt delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/secrets/delivery/wds/file.txt delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/secrets/delivery/wds/nginx.properties delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/secrets/delivery/wds/nginx/file.txt delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/secrets/global.properties delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/secrets/ingestion/rest.properties delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/secrets/ingestion/rest/file.txt delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/secrets/processing/relay.properties delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/secrets/processing/relay/file.txt delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/secrets/relay.properties delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/secrets/shared.properties delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/secrets/shared/file.txt delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/secrets/shared/file1.txt delete mode 100644 core/src/test/resources/dev/streamx/cli/command/cloud/project/with-configs.yaml delete mode 100644 core/src/test/resources/dev/streamx/cli/command/dev/failing-mesh.yaml delete mode 100644 core/src/test/resources/dev/streamx/cli/command/dev/incremental-reloaded-mesh.yaml delete mode 100644 core/src/test/resources/dev/streamx/cli/command/dev/initial-mesh.yaml delete mode 100644 core/src/test/resources/dev/streamx/cli/command/ingestion/batch/invalid-channel/.eventsource.yaml delete mode 100644 core/src/test/resources/dev/streamx/cli/command/ingestion/batch/invalid-channel/index.html delete mode 100644 core/src/test/resources/dev/streamx/cli/command/ingestion/batch/unknown-channel/.eventsource.yaml delete mode 100644 core/src/test/resources/dev/streamx/cli/command/ingestion/batch/unknown-channel/index.html delete mode 100644 core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/index.html rename core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/{ => publish}/.eventsource.yaml (66%) rename core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/{ => publish}/ignore.me (100%) create mode 100644 core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/publish/index.html create mode 100644 core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/unpublish/.eventsource.yaml create mode 100644 core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/unpublish/ignore.me create mode 100644 core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/unpublish/index.html delete mode 100644 core/src/test/resources/dev/streamx/cli/command/ingestion/publish/payload/example-image.png delete mode 100644 core/src/test/resources/dev/streamx/cli/command/ingestion/publish/payload/helloworld-payload.json delete mode 100644 core/src/test/resources/dev/streamx/cli/command/ingestion/publish/payload/invalid-payload.json delete mode 100644 core/src/test/resources/dev/streamx/cli/command/ingestion/publish/payload/payload.json delete mode 100644 core/src/test/resources/dev/streamx/cli/command/ingestion/publish/payload/raw-text.txt create mode 100644 e2e-tests/repository/web-server-sink/repository-volume/batch/publish/index.html create mode 100644 e2e-tests/repository/web-server-sink/repository-volume/publish/index.html create mode 100644 e2e-tests/src/test/resources/batch/publish/.eventsource.yaml create mode 100644 e2e-tests/src/test/resources/batch/publish/index.html create mode 100644 e2e-tests/src/test/resources/batch/unpublish/.eventsource.yaml create mode 100644 e2e-tests/src/test/resources/batch/unpublish/index.html delete mode 100644 e2e-tests/src/test/resources/configs/overridden-nginx-conf delete mode 100644 e2e-tests/src/test/resources/file_param_page.json delete mode 100644 e2e-tests/src/test/resources/json_path_file_param_page.txt delete mode 100644 e2e-tests/src/test/resources/payload.json create mode 100644 e2e-tests/src/test/resources/stream/page-publish-event.json create mode 100644 e2e-tests/src/test/resources/stream/page-unpublish-event.json diff --git a/core/pom.xml b/core/pom.xml index a39e505e..e8a45dc3 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -44,13 +44,9 @@ bcpkix-jdk18on - dev.streamx + com.streamx streamx-runner - - dev.streamx - streamx-operator-mesh-api - com.fasterxml.jackson.datatype jackson-datatype-jdk8 @@ -60,7 +56,7 @@ jackson-datatype-jsr310 - dev.streamx + com.streamx ingestion-client @@ -89,11 +85,6 @@ quarkus-junit5 test - - io.quarkus - quarkus-junit5-mockito - test - io.quarkus quarkus-junit5-component diff --git a/core/src/main/java/dev/streamx/cli/BannerPrinter.java b/core/src/main/java/dev/streamx/cli/BannerPrinter.java index ee98e3c7..b474e0d9 100644 --- a/core/src/main/java/dev/streamx/cli/BannerPrinter.java +++ b/core/src/main/java/dev/streamx/cli/BannerPrinter.java @@ -2,7 +2,6 @@ import static dev.streamx.cli.util.Output.print; -import dev.streamx.cli.command.dev.DevCommand; import dev.streamx.cli.command.run.RunCommand; import jakarta.enterprise.context.ApplicationScoped; import java.util.Set; @@ -13,7 +12,7 @@ public class BannerPrinter { private static final Set COMMANDS_REQUIRING_PRINTING_BANNER = - Set.of(DevCommand.COMMAND_NAME, RunCommand.COMMAND_NAME); + Set.of(RunCommand.COMMAND_NAME); private static final String BANNER = """ ____ _ __ __ diff --git a/core/src/main/java/dev/streamx/cli/SchemaProvider.java b/core/src/main/java/dev/streamx/cli/SchemaProvider.java deleted file mode 100644 index 42bd7630..00000000 --- a/core/src/main/java/dev/streamx/cli/SchemaProvider.java +++ /dev/null @@ -1,96 +0,0 @@ -package dev.streamx.cli; - -import static dev.streamx.cli.util.ExceptionUtils.sneakyThrow; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import dev.streamx.cli.command.ingestion.IngestionClientConfig; -import dev.streamx.cli.exception.IngestionClientException; -import dev.streamx.cli.exception.UnableToConnectIngestionServiceException; -import dev.streamx.cli.exception.UnknownChannelException; -import dev.streamx.clients.ingestion.exceptions.StreamxClientException; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.Map; -import javax.net.ssl.SSLHandshakeException; -import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpEntity; -import org.apache.http.HttpHeaders; -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.util.EntityUtils; - -@ApplicationScoped -public class SchemaProvider { - - private static final ObjectMapper objectMapper = new ObjectMapper(); - - @Inject - CloseableHttpClient httpClient; - - @Inject - IngestionClientConfig ingestionClientConfig; - - public JsonNode getSchema(String channel) throws UnknownChannelException { - Map schemas = fetchSchemas(ingestionClientConfig.url()); - if (schemas.containsKey(channel)) { - return schemas.get(channel); - } - throw new UnknownChannelException(channel, schemas.keySet().toString()); - } - - private Map fetchSchemas(String ingestionUrl) { - try { - URI channelsEndpointUri = buildUri(ingestionUrl); - HttpGet httpRequest = new HttpGet(channelsEndpointUri); - - ingestionClientConfig.authToken() - .ifPresent(authToken -> addAuthorizationHeader(httpRequest, authToken)); - - HttpResponse execute = httpClient.execute(httpRequest); - verifyStatusCode(execute); - - HttpEntity entity = execute.getEntity(); - - String body = EntityUtils.toString(entity, "UTF-8"); - - return objectMapper.readValue(body, new TypeReference<>() { - }); - } catch (SSLHandshakeException e) { - throw IngestionClientException.sslException(ingestionClientConfig.url()); - } catch (IOException e) { - throw new UnableToConnectIngestionServiceException(ingestionClientConfig.url(), e); - } catch (StreamxClientException e) { - throw sneakyThrow(e); - } - } - - private static void verifyStatusCode(HttpResponse execute) throws StreamxClientException { - if (execute.getStatusLine().getStatusCode() == HttpStatus.SC_UNAUTHORIZED) { - throw new StreamxClientException( - // this message is copy-pasted from StreamxIngestionClient - "Authentication failed. Make sure that the given token is valid."); - } - } - - private URI buildUri(String publicationsEndpointUri) { - String uriString = String.format("%s/ingestion/v1/channels", publicationsEndpointUri); - try { - return new URI(uriString); - } catch (URISyntaxException e) { - throw sneakyThrow(e); - } - } - - private void addAuthorizationHeader(HttpGet httpRequest, String authToken) { - if (StringUtils.isNotBlank(authToken)) { - httpRequest.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + authToken); - } - } -} diff --git a/core/src/main/java/dev/streamx/cli/StreamxCommand.java b/core/src/main/java/dev/streamx/cli/StreamxCommand.java index c16b95aa..75564240 100644 --- a/core/src/main/java/dev/streamx/cli/StreamxCommand.java +++ b/core/src/main/java/dev/streamx/cli/StreamxCommand.java @@ -1,13 +1,7 @@ package dev.streamx.cli; -import dev.streamx.cli.command.cloud.deploy.DeployCommand; -import dev.streamx.cli.command.cloud.undeploy.UndeployCommand; -import dev.streamx.cli.command.dev.DevCommand; import dev.streamx.cli.command.ingestion.batch.BatchCommand; -import dev.streamx.cli.command.ingestion.publish.PublishCommand; import dev.streamx.cli.command.ingestion.stream.StreamCommand; -import dev.streamx.cli.command.ingestion.unpublish.UnpublishCommand; -import dev.streamx.cli.command.init.InitCommand; import dev.streamx.cli.command.run.RunCommand; import dev.streamx.cli.config.ArgumentConfigSource; import dev.streamx.cli.config.validation.ConfigSourcesValidator; @@ -30,11 +24,8 @@ @Command(mixinStandardHelpOptions = true, name = "streamx", subcommands = { - InitCommand.class, - RunCommand.class, DevCommand.class, - PublishCommand.class, UnpublishCommand.class, + RunCommand.class, BatchCommand.class, StreamCommand.class, - DeployCommand.class, UndeployCommand.class, HelpCommand.class }, versionProvider = VersionProvider.class) @@ -55,14 +46,10 @@ public class StreamxCommand implements QuarkusApplication { @Inject ConfigSourcesValidator configSourcesValidator; - @Inject - BannerPrinter bannerPrinter; - @ArgGroup(exclusive = false) LicenseArguments licenseArguments; private CommandLine commandLine; - private String[] args; public static void main(String... args) { initializeArgumentConfigSource(args); @@ -82,9 +69,7 @@ private static void initializeArgumentConfigSource(String[] args) { } @Override - public int run(String... args) throws Exception { - this.args = args; - + public int run(String... args) { commandLine = new CommandLine(this, factory) .setParameterExceptionHandler(parameterExceptionHandler) .setExecutionExceptionHandler(executionExceptionHandler) @@ -122,8 +107,6 @@ private int executionStrategy(ParseResult parseResult) { } private void init() { - bannerPrinter.initialize(commandLine, args); - licenseProcessorEntrypoint.process(); } } diff --git a/core/src/main/java/dev/streamx/cli/command/cloud/KubernetesArguments.java b/core/src/main/java/dev/streamx/cli/command/cloud/KubernetesArguments.java deleted file mode 100644 index f5e98465..00000000 --- a/core/src/main/java/dev/streamx/cli/command/cloud/KubernetesArguments.java +++ /dev/null @@ -1,30 +0,0 @@ -package dev.streamx.cli.command.cloud; - -import dev.streamx.cli.config.ArgumentConfigSource; -import picocli.CommandLine.Option; - -public class KubernetesArguments { - - @Option(names = {"-n", "--namespace"}, paramLabel = "", - description = "Forces provided namespace.") - void namespace(String namespace) { - ArgumentConfigSource.registerValue(KubernetesConfig.STREAMX_KUBERNETES_NAMESPACE, namespace); - } - - @Option(names = {"-d", "--resources-directories"}, paramLabel = "", - description = "Specifies one or more comma-separated relative directory paths (from the " - + "mesh.yaml directory) where managed Kubernetes resource definitions are located.") - void resourceDirectories(String resourceDirectories) { - ArgumentConfigSource.registerValue(KubernetesConfig.STREAMX_KUBERNETES_RESOURCE_DIRECTORIES, - resourceDirectories); - } - - @Option(names = {"-r", - "--controlled-resource-definitions"}, paramLabel = "", - description = "Specifies one or more comma-separated definitions in the form of" - + " [cluster:]group/version/kind to be managed by CLI during undeploy.") - void controlledResourceDefinitions(String resourceDefinitions) { - ArgumentConfigSource.registerValue( - KubernetesConfig.STREAMX_KUBERNETES_CONTROLLED_RESOURCE_DEFINITIONS, resourceDefinitions); - } -} diff --git a/core/src/main/java/dev/streamx/cli/command/cloud/KubernetesConfig.java b/core/src/main/java/dev/streamx/cli/command/cloud/KubernetesConfig.java deleted file mode 100644 index 5ca85dc8..00000000 --- a/core/src/main/java/dev/streamx/cli/command/cloud/KubernetesConfig.java +++ /dev/null @@ -1,24 +0,0 @@ -package dev.streamx.cli.command.cloud; - -import io.smallrye.config.ConfigMapping; -import io.smallrye.config.WithName; -import java.util.Optional; - -@ConfigMapping -public interface KubernetesConfig { - - String STREAMX_KUBERNETES_NAMESPACE = "streamx.kubernetes.namespace"; - String STREAMX_KUBERNETES_RESOURCE_DIRECTORIES = "streamx.kubernetes.resource-directories"; - String STREAMX_KUBERNETES_CONTROLLED_RESOURCE_DEFINITIONS - = "streamx.kubernetes.controlled-resource-definitions"; - - @WithName(STREAMX_KUBERNETES_NAMESPACE) - Optional namespace(); - - @WithName(STREAMX_KUBERNETES_RESOURCE_DIRECTORIES) - Optional resourceDirectories(); - - @WithName(STREAMX_KUBERNETES_CONTROLLED_RESOURCE_DEFINITIONS) - Optional controlledResourceDefinitions(); - -} diff --git a/core/src/main/java/dev/streamx/cli/command/cloud/KubernetesService.java b/core/src/main/java/dev/streamx/cli/command/cloud/KubernetesService.java deleted file mode 100644 index 3d5f7289..00000000 --- a/core/src/main/java/dev/streamx/cli/command/cloud/KubernetesService.java +++ /dev/null @@ -1,147 +0,0 @@ -package dev.streamx.cli.command.cloud; - -import static dev.streamx.cli.command.cloud.MetadataUtils.CONFIG_TYPE_LABEL; -import static dev.streamx.cli.command.cloud.MetadataUtils.DEFAULT_K8S_NAMESPACE; -import static dev.streamx.cli.command.cloud.MetadataUtils.SERVICEMESH_CRD_NAME; -import static dev.streamx.cli.command.cloud.MetadataUtils.setLabel; -import static dev.streamx.cli.command.cloud.MetadataUtils.setMetadata; - -import dev.streamx.cli.command.cloud.collector.ClusterResourcesCollector; -import dev.streamx.cli.command.cloud.collector.TypedClusterResourceCollector; -import dev.streamx.cli.command.cloud.deploy.Config; -import dev.streamx.cli.exception.KubernetesException; -import dev.streamx.operator.Component; -import dev.streamx.operator.crd.ServiceMesh; -import io.fabric8.kubernetes.api.model.ConfigMap; -import io.fabric8.kubernetes.api.model.ConfigMapBuilder; -import io.fabric8.kubernetes.api.model.HasMetadata; -import io.fabric8.kubernetes.api.model.Secret; -import io.fabric8.kubernetes.api.model.SecretBuilder; -import io.fabric8.kubernetes.api.model.apiextensions.v1.CustomResourceDefinition; -import io.fabric8.kubernetes.client.KubernetesClient; -import io.fabric8.kubernetes.client.KubernetesClientException; -import io.fabric8.kubernetes.client.dsl.NonDeletingOperation; -import io.fabric8.kubernetes.client.utils.KubernetesResourceUtil; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; -import org.jetbrains.annotations.NotNull; - -@ApplicationScoped -public class KubernetesService { - - @Inject - KubernetesClient kubernetesClient; - @Inject - KubernetesConfig kubernetesConfig; - - public void deploy(List resources) { - resources.forEach(this::deploy); - } - - private void deploy(T resource) { - try { - kubernetesClient.resource(resource).inNamespace(getNamespace()) - .createOr(NonDeletingOperation::update); - } catch (KubernetesClientException e) { - throw KubernetesException.kubernetesClientException(e); - } - } - - public void undeploy(String meshName) { - undeploy(collectManagedResources(meshName)); - } - - public void undeploy(List resources) { - try { - resources.forEach(r -> kubernetesClient.resource(r).delete()); - } catch (KubernetesClientException e) { - throw KubernetesException.kubernetesClientException(e); - } - } - - public List collectManagedResources(String meshName) { - List result = new ArrayList<>(); - // Collect mesh - ServiceMesh mesh = kubernetesClient.resources(ServiceMesh.class).inNamespace(getNamespace()) - .withName(meshName).get(); - if (mesh != null) { - result.add(mesh); - } - - // Collect configs and secrets - result.addAll( - new TypedClusterResourceCollector(kubernetesClient, List.of(ConfigMap.class, Secret.class), - getNamespace()).collect(meshName)); - - // Collect other resources controlled by the CLI - result.addAll(new ClusterResourcesCollector(kubernetesClient, - getControlledResourceDefinitions(), getNamespace()).collect(meshName)); - - return result; - } - - public void validateCrdInstallation() { - try { - CustomResourceDefinition crd = kubernetesClient.apiextensions().v1() - .customResourceDefinitions() - .withName(SERVICEMESH_CRD_NAME) - .get(); - if (crd == null) { - throw KubernetesException.serviceMeshCrdNotFound(); - } - } catch (KubernetesClientException e) { - throw KubernetesException.kubernetesClientException(e); - } - } - - @NotNull - public ConfigMap buildConfigMap(String meshName, Config config) { - ConfigMap configMap = new ConfigMapBuilder() - .withNewMetadata() - .endMetadata() - .addToData(config.data()) - .build(); - String sanitizedName = KubernetesResourceUtil.sanitizeName(config.name()); - setMetadata(meshName, Component.EXTERNAL_CONFIG, sanitizedName, configMap); - setLabel(configMap, CONFIG_TYPE_LABEL, config.configType().getLabelValue()); - return configMap; - } - - @NotNull - public Secret buildSecret(String meshName, Config config) { - Secret secret = new SecretBuilder() - .withNewMetadata() - .endMetadata() - .withStringData(config.data()) - .build(); - String sanitizedName = KubernetesResourceUtil.sanitizeName(config.name()); - setMetadata(meshName, Component.EXTERNAL_SECRET, sanitizedName, secret); - setLabel(secret, CONFIG_TYPE_LABEL, config.configType().getLabelValue()); - return secret; - } - - public String getNamespace() { - return kubernetesConfig.namespace() - .orElse(Optional.ofNullable(kubernetesClient.getNamespace()).orElse(DEFAULT_K8S_NAMESPACE)); - } - - public List getResourcePaths() { - return kubernetesConfig.resourceDirectories().map(paths -> Arrays.stream(paths.split(",")) - .map(String::trim) - .filter(s -> !s.isEmpty()) - .collect(Collectors.toList())).orElse(List.of()); - } - - public List getControlledResourceDefinitions() { - return kubernetesConfig.controlledResourceDefinitions() - .map(paths -> Arrays.stream(paths.split(",")) - .map(String::trim) - .filter(s -> !s.isEmpty()) - .collect(Collectors.toList())).orElse(List.of()); - } -} diff --git a/core/src/main/java/dev/streamx/cli/command/cloud/MetadataUtils.java b/core/src/main/java/dev/streamx/cli/command/cloud/MetadataUtils.java deleted file mode 100644 index d2e3205d..00000000 --- a/core/src/main/java/dev/streamx/cli/command/cloud/MetadataUtils.java +++ /dev/null @@ -1,57 +0,0 @@ -package dev.streamx.cli.command.cloud; - -import dev.streamx.operator.Component; -import io.fabric8.kubernetes.api.model.HasMetadata; -import io.fabric8.kubernetes.client.utils.KubernetesResourceUtil; -import java.util.HashMap; -import java.util.Map; - -public class MetadataUtils { - - public static final String NAME_LABEL = "app.kubernetes.io/name"; - public static final String INSTANCE_LABEL = "app.kubernetes.io/instance"; - public static final String COMPONENT_LABEL = "app.kubernetes.io/component"; - public static final String MANAGED_BY_LABEL = "app.kubernetes.io/managed-by"; - public static final String MANAGED_BY_LABEL_VALUE = "streamx-cli"; - public static final String CONFIG_TYPE_LABEL = "mesh.streamx.dev/config-type"; - public static final String PART_OF_LABEL = "app.kubernetes.io/part-of"; - public static final String SERVICEMESH_CRD_NAME = "servicemeshes.streamx.dev"; - public static final String DEFAULT_K8S_NAMESPACE = "default"; - - private MetadataUtils() { - // No instances - } - - public static Map createPartOfAndManagedByLabels(String meshName) { - return Map.of( - PART_OF_LABEL, meshName, - MANAGED_BY_LABEL, MANAGED_BY_LABEL_VALUE - ); - } - - public static void setMetadata(String meshName, Component component, String name, - HasMetadata resource) { - String instanceName = getResourceName(meshName, component.getShortName(), name); - resource.getMetadata().setName(instanceName); - setLabel(resource, INSTANCE_LABEL, instanceName); - setLabel(resource, COMPONENT_LABEL, component.getName()); - setLabel(resource, NAME_LABEL, name); - setManagedByAndPartOfLabels(resource, meshName); - } - - public static void setManagedByAndPartOfLabels(HasMetadata resource, String meshName) { - setLabel(resource, PART_OF_LABEL, meshName); - setLabel(resource, MANAGED_BY_LABEL, MANAGED_BY_LABEL_VALUE); - } - - public static String getResourceName(String meshName, String componentName, String name) { - return KubernetesResourceUtil.sanitizeName(meshName + "-" + componentName + "-" + name); - } - - public static void setLabel(HasMetadata resource, String key, String value) { - if (resource.getMetadata().getLabels() == null) { - resource.getMetadata().setLabels(new HashMap<>()); - } - resource.getMetadata().getLabels().put(key, value); - } -} diff --git a/core/src/main/java/dev/streamx/cli/command/cloud/ProjectPathsResolver.java b/core/src/main/java/dev/streamx/cli/command/cloud/ProjectPathsResolver.java deleted file mode 100644 index 14d9b397..00000000 --- a/core/src/main/java/dev/streamx/cli/command/cloud/ProjectPathsResolver.java +++ /dev/null @@ -1,38 +0,0 @@ -package dev.streamx.cli.command.cloud; - -import dev.streamx.cli.command.meshprocessing.MeshResolver; -import jakarta.enterprise.context.ApplicationScoped; -import java.nio.file.Path; -import org.jetbrains.annotations.NotNull; - -@ApplicationScoped -public class ProjectPathsResolver { - - public static final String CONFIGS_DIRECTORY = "configs"; - public static final String SECRETS_DIRECTORY = "secrets"; - protected static final String YAML_EXT = ".yaml"; - static final String DEPLOYMENT = "deployment"; - static final String DEPLOYMENT_FILE_NAME = DEPLOYMENT + YAML_EXT; - - @NotNull - public Path resolveDeploymentPath(Path meshPath) { - String meshFileName = meshPath.getFileName().toString(); - String deploymentFileName = DEPLOYMENT_FILE_NAME; - if (!MeshResolver.MESH_YAML.equals(meshFileName)) { - deploymentFileName = DEPLOYMENT + "." + meshFileName; - } - return meshPath.getParent().resolve(deploymentFileName); - } - - public Path resolveSecretPath(Path projectPath, String sourcePath) { - return resolveSourcePath(projectPath, SECRETS_DIRECTORY, sourcePath); - } - - public Path resolveConfigPath(Path projectPath, String sourcePath) { - return resolveSourcePath(projectPath, CONFIGS_DIRECTORY, sourcePath); - } - - private Path resolveSourcePath(Path projectPath, String sourceDirectory, String sourcePath) { - return projectPath.resolve(sourceDirectory).resolve(sourcePath); - } -} diff --git a/core/src/main/java/dev/streamx/cli/command/cloud/ServiceMeshResolver.java b/core/src/main/java/dev/streamx/cli/command/cloud/ServiceMeshResolver.java deleted file mode 100644 index 0b963aa0..00000000 --- a/core/src/main/java/dev/streamx/cli/command/cloud/ServiceMeshResolver.java +++ /dev/null @@ -1,182 +0,0 @@ -package dev.streamx.cli.command.cloud; - -import com.fasterxml.jackson.databind.ObjectMapper; -import dev.streamx.cli.interpolation.Interpolating; -import dev.streamx.cli.util.ExceptionUtils; -import dev.streamx.mesh.model.AbstractContainer; -import dev.streamx.mesh.model.AbstractFromSource; -import dev.streamx.mesh.model.DeliveryService; -import dev.streamx.mesh.model.EnvironmentFrom; -import dev.streamx.mesh.model.VolumesFrom; -import dev.streamx.operator.crd.ServiceMesh; -import dev.streamx.operator.crd.ServiceMeshSpec; -import dev.streamx.operator.crd.deployment.ServiceMeshDeploymentConfig; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -@ApplicationScoped -public class ServiceMeshResolver { - - public static final String SERVICE_MESH_NAME = "sx"; - @Inject - @Interpolating - ObjectMapper objectMapper; - - @Inject - ProjectPathsResolver projectPathsResolver; - - @NotNull - public ServiceMesh resolveMesh(Path meshPath) { - File meshPathFile = meshPath.toFile(); - if (!meshPathFile.exists()) { - throw new RuntimeException("Mesh file with provided path '" + meshPath + "' does not exist."); - } - if (meshPathFile.length() < 1) { - throw new RuntimeException("Mesh file with provided path '" + meshPath + "' is empty."); - } - ServiceMesh serviceMesh = new ServiceMesh(); - try { - ServiceMeshSpec spec = objectMapper.readValue(meshPathFile, - ServiceMeshSpec.class); - ServiceMeshDeploymentConfig serviceMeshDeploymentConfig = readDeploymentConfig(meshPath); - spec.setDeploymentConfig(serviceMeshDeploymentConfig); - serviceMesh.setSpec(spec); - serviceMesh.getMetadata().setName(SERVICE_MESH_NAME); - } catch (IOException e) { - throw new RuntimeException( - ExceptionUtils.appendLogSuggestion( - "Unable to read mesh definition from '" + meshPath + "'.\n" - + "\n" - + "Details:\n" - + e.getMessage()), e); - } - return serviceMesh; - } - - @NotNull - public ConfigSourcesPaths extractConfigSourcesPaths(ServiceMesh serviceMesh) { - Set configEnvPaths = new HashSet<>(); - Set secretEnvPaths = new HashSet<>(); - Set configVolumePaths = new HashSet<>(); - Set secretVolumePaths = new HashSet<>(); - processGlobalEnvSources(serviceMesh, configEnvPaths, secretEnvPaths); - List containers = extractContainers(serviceMesh); - containers.forEach(container -> { - EnvironmentFrom environmentFrom = container.getEnvironmentFrom(); - configEnvPaths.addAll( - extractConfigSourcesPaths(environmentFrom, AbstractFromSource::getConfigs, null)); - secretEnvPaths.addAll( - extractConfigSourcesPaths(environmentFrom, AbstractFromSource::getSecrets, null)); - VolumesFrom volumesFrom = container.getVolumesFrom(); - configVolumePaths.addAll( - extractConfigSourcesPaths(volumesFrom, AbstractFromSource::getConfigs, - this::mapToHostPath)); - secretVolumePaths.addAll( - extractConfigSourcesPaths(volumesFrom, AbstractFromSource::getSecrets, - this::mapToHostPath)); - }); - - return new ConfigSourcesPaths(configEnvPaths, secretEnvPaths, configVolumePaths, - secretVolumePaths); - } - - @NotNull - private List extractConfigSourcesPaths(AbstractFromSource fromSource, - Function> pathsExtractor, Function mapper) { - List configsPaths = Collections.emptyList(); - if (fromSource != null) { - List extractedPaths = pathsExtractor.apply(fromSource); - if (extractedPaths != null) { - configsPaths = extractedPaths.stream().filter(Objects::nonNull).toList(); - if (mapper != null) { - configsPaths = configsPaths.stream().map(mapper).collect(Collectors.toList()); - } - } - } - return configsPaths; - } - - @Nullable - private ServiceMeshDeploymentConfig readDeploymentConfig(Path meshPath) { - Path deploymentPath = projectPathsResolver.resolveDeploymentPath(meshPath); - ServiceMeshDeploymentConfig serviceMeshDeploymentConfig = null; - File deploymentFile = deploymentPath.toFile(); - if (deploymentFile.exists() && deploymentFile.length() > 0) { - try { - serviceMeshDeploymentConfig = objectMapper.readValue(deploymentFile, - ServiceMeshDeploymentConfig.class); - } catch (IOException e) { - throw new RuntimeException( - ExceptionUtils.appendLogSuggestion( - "Unable to read deployment from '" + deploymentPath + "'.\n" - + "\n" - + "Details:\n" - + e.getMessage()), e); - } - } - return serviceMeshDeploymentConfig; - } - - @NotNull - List extractContainers(ServiceMesh serviceMesh) { - ServiceMeshSpec serviceMeshSpec = serviceMesh.getSpec(); - List containers = Stream.of( - serviceMeshSpec.getIngestion(), - serviceMeshSpec.getProcessing(), - serviceMeshSpec.getDelivery() - ).filter(Objects::nonNull).map(Map::values) - .flatMap(Collection::stream).collect(Collectors.toList()); - containers.addAll( - Optional.ofNullable(serviceMeshSpec.getDelivery()) - .orElse(Collections.emptyMap()) - .values() - .stream() - .map(DeliveryService::getComponents) - .filter(Objects::nonNull) - .map(Map::values) - .flatMap(Collection::stream) - .toList() - ); - return containers; - } - - private void processGlobalEnvSources(ServiceMesh serviceMesh, Set envConfigsPaths, - Set envSecretsPaths) { - EnvironmentFrom globalEnvironmentFrom = serviceMesh.getSpec().getEnvironmentFrom(); - if (globalEnvironmentFrom != null) { - List globalEnvironmentFromConfigs = globalEnvironmentFrom.getConfigs(); - if (globalEnvironmentFromConfigs != null) { - envConfigsPaths.addAll(globalEnvironmentFromConfigs); - } - List globalEnvironmentFromSecrets = globalEnvironmentFrom.getSecrets(); - if (globalEnvironmentFromSecrets != null) { - envSecretsPaths.addAll(globalEnvironmentFromSecrets); - } - } - } - - private String mapToHostPath(String volumeConf) { - return volumeConf.split(":")[0]; - } - - public record ConfigSourcesPaths(Set configEnvPaths, Set secretEnvPaths, - Set configVolumePaths, Set secretVolumePaths) { - - } -} diff --git a/core/src/main/java/dev/streamx/cli/command/cloud/collector/ClusterResourcesCollector.java b/core/src/main/java/dev/streamx/cli/command/cloud/collector/ClusterResourcesCollector.java deleted file mode 100644 index 6d0f38a4..00000000 --- a/core/src/main/java/dev/streamx/cli/command/cloud/collector/ClusterResourcesCollector.java +++ /dev/null @@ -1,84 +0,0 @@ -package dev.streamx.cli.command.cloud.collector; - -import static dev.streamx.cli.command.cloud.MetadataUtils.createPartOfAndManagedByLabels; -import static org.apache.commons.lang3.StringUtils.removeStart; - -import io.fabric8.kubernetes.api.model.HasMetadata; -import io.fabric8.kubernetes.client.KubernetesClient; -import io.fabric8.kubernetes.client.dsl.base.ResourceDefinitionContext; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * Collects controlled resources deployed to the Kubernetes cluster. Identifies the resources by the - * assigned labels. The scope of the class is the mesh namespace and the list of resources - * definitions, i.e: - * 
- *   - v1/Pod
- *   - apps/v1/Deployment
- *   - v1/Service
- *   - batch/v1/Job
- *   - cluster:rbac.authorization.k8s.io/v1/ClusterRole
- *
- */ -public class ClusterResourcesCollector implements KubernetesResourcesCollector { - - private static final String CLUSTER_RESOURCE_TYPE_MARKER = "cluster:"; - - private final KubernetesClient kubernetesClient; - private final List resourceDefinitions; - private final String namespace; - - public ClusterResourcesCollector(KubernetesClient kubernetesClient, - List resourceDefinitions, - String namespace) { - this.kubernetesClient = kubernetesClient; - this.resourceDefinitions = resourceDefinitions; - this.namespace = namespace; - } - - public List collect(String meshName) { - - List result = new ArrayList<>(); - // Split the property string by comma - resourceDefinitions.forEach(entry -> { - boolean clusterScoped = entry.startsWith(CLUSTER_RESOURCE_TYPE_MARKER); - String[] parts = (clusterScoped ? removeStart(entry, CLUSTER_RESOURCE_TYPE_MARKER) - : entry).split("/"); - - if (parts.length != 3 && parts.length != 2) { - throw new IllegalArgumentException("Invalid resource specification: " + entry - + ". Expected format: group/version/kind or version/kind"); - } - // For resources like Pod or ConfigMap, the group is skipped - boolean isCoreResource = parts.length == 2; - String group = isCoreResource ? "" : parts[0].trim(); - String version = isCoreResource ? parts[0].trim() : parts[1].trim(); - String kind = isCoreResource ? parts[1].trim() : parts[2].trim(); - - result.addAll(collectResourcesByType(meshName, group, version, kind, clusterScoped)); - }); - - return result; - } - - /** - * Collects resources of the given API group, version, and kind, using the global label selector. - */ - private List collectResourcesByType(String meshName, String group, String version, - String kind, boolean clusterScoped) { - Map selector = createPartOfAndManagedByLabels(meshName); - - ResourceDefinitionContext context = new ResourceDefinitionContext.Builder() - .withGroup(group) - .withVersion(version) - .withKind(kind) - .withNamespaced(!clusterScoped) - .build(); - - return new ArrayList<>(kubernetesClient.genericKubernetesResources(context) - .inNamespace(namespace) - .withLabels(selector).list().getItems()); - } -} diff --git a/core/src/main/java/dev/streamx/cli/command/cloud/collector/DirectoryResourcesCollector.java b/core/src/main/java/dev/streamx/cli/command/cloud/collector/DirectoryResourcesCollector.java deleted file mode 100644 index 6b8a511e..00000000 --- a/core/src/main/java/dev/streamx/cli/command/cloud/collector/DirectoryResourcesCollector.java +++ /dev/null @@ -1,71 +0,0 @@ -package dev.streamx.cli.command.cloud.collector; - -import static dev.streamx.cli.command.cloud.MetadataUtils.setManagedByAndPartOfLabels; - -import com.fasterxml.jackson.databind.MappingIterator; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.fabric8.kubernetes.api.model.HasMetadata; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Stream; - -/** - * Collects controlled resources defined in the resources directory. - */ -public class DirectoryResourcesCollector implements KubernetesResourcesCollector { - - private final ObjectMapper objectMapper; - private final Path projectPath; - private final List resourcesDirectories; - - - public DirectoryResourcesCollector(ObjectMapper objectMapper, Path projectPath, - List resourcesDirectories) { - this.objectMapper = objectMapper; - this.projectPath = projectPath; - this.resourcesDirectories = resourcesDirectories; - } - - @Override - public List collect(String meshName) { - List resources = new ArrayList<>(); - - return resourcesDirectories.stream().flatMap(resourcesDirectory -> { - Path dirPath = projectPath.resolve(resourcesDirectory); - - if (!Files.exists(dirPath) || !Files.isDirectory(dirPath)) { - throw new IllegalArgumentException( - "Kubernetes resources directory: %s does not exist".formatted(dirPath)); - } - - try (Stream files = Files.list(dirPath)) { - files.forEach(file -> processResourceFile(file, resources)); - } catch (IOException e) { - throw new RuntimeException("Error reading directory: " + dirPath, e); - } - - resources.forEach(r -> setManagedByAndPartOfLabels(r, meshName)); - return resources.stream(); - }).toList(); - } - - private void processResourceFile(Path file, List resources) { - String fileName = file.getFileName().toString(); - if (!fileName.endsWith(".yaml") && !fileName.endsWith(".yml")) { - return; - } - try (MappingIterator iterator = - this.objectMapper.readerFor(HasMetadata.class).readValues(file.toFile())) { - - while (iterator.hasNext()) { - resources.add(iterator.next()); - } - - } catch (IOException e) { - throw new RuntimeException("Failed to load resource from file: " + file, e); - } - } -} diff --git a/core/src/main/java/dev/streamx/cli/command/cloud/collector/KubernetesResourcesCollector.java b/core/src/main/java/dev/streamx/cli/command/cloud/collector/KubernetesResourcesCollector.java deleted file mode 100644 index d14f9864..00000000 --- a/core/src/main/java/dev/streamx/cli/command/cloud/collector/KubernetesResourcesCollector.java +++ /dev/null @@ -1,10 +0,0 @@ -package dev.streamx.cli.command.cloud.collector; - -import io.fabric8.kubernetes.api.model.HasMetadata; -import java.util.List; - -public interface KubernetesResourcesCollector { - - List collect(String meshName); - -} diff --git a/core/src/main/java/dev/streamx/cli/command/cloud/collector/TypedClusterResourceCollector.java b/core/src/main/java/dev/streamx/cli/command/cloud/collector/TypedClusterResourceCollector.java deleted file mode 100644 index 5738cc9b..00000000 --- a/core/src/main/java/dev/streamx/cli/command/cloud/collector/TypedClusterResourceCollector.java +++ /dev/null @@ -1,39 +0,0 @@ -package dev.streamx.cli.command.cloud.collector; - -import static dev.streamx.cli.command.cloud.MetadataUtils.createPartOfAndManagedByLabels; - -import io.fabric8.kubernetes.api.model.HasMetadata; -import io.fabric8.kubernetes.client.KubernetesClient; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * Collects controlled resources deployed to the Kubernetes cluster. Identifies the resources by the - * assigned labels. The scope of the class is the mesh namespace and the resource classes. - */ -public class TypedClusterResourceCollector implements KubernetesResourcesCollector { - - private final KubernetesClient kubernetesClient; - private final List> resourceDefinitions; - private final String namespace; - - public TypedClusterResourceCollector(KubernetesClient kubernetesClient, - List> resourceDefinitions, - String namespace) { - this.kubernetesClient = kubernetesClient; - this.resourceDefinitions = resourceDefinitions; - this.namespace = namespace; - } - - @Override - public List collect(String meshName) { - Map selector = createPartOfAndManagedByLabels(meshName); - List result = new ArrayList<>(); - resourceDefinitions.forEach(r -> - result.addAll(kubernetesClient.resources(r).inNamespace(namespace) - .withLabels(selector).list().getItems()) - ); - return result; - } -} diff --git a/core/src/main/java/dev/streamx/cli/command/cloud/deploy/Config.java b/core/src/main/java/dev/streamx/cli/command/cloud/deploy/Config.java deleted file mode 100644 index 3b51421e..00000000 --- a/core/src/main/java/dev/streamx/cli/command/cloud/deploy/Config.java +++ /dev/null @@ -1,14 +0,0 @@ -package dev.streamx.cli.command.cloud.deploy; - -import java.util.Map; - -public record Config(String name, Map data, ConfigType configType) { - - public enum ConfigType { - DIR, FILE; - - public String getLabelValue() { - return this.toString().toLowerCase(); - } - } -} diff --git a/core/src/main/java/dev/streamx/cli/command/cloud/deploy/ConfigService.java b/core/src/main/java/dev/streamx/cli/command/cloud/deploy/ConfigService.java deleted file mode 100644 index 44d31829..00000000 --- a/core/src/main/java/dev/streamx/cli/command/cloud/deploy/ConfigService.java +++ /dev/null @@ -1,77 +0,0 @@ -package dev.streamx.cli.command.cloud.deploy; - -import dev.streamx.cli.command.cloud.ProjectPathsResolver; -import dev.streamx.cli.command.cloud.deploy.Config.ConfigType; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; -import java.io.File; -import java.nio.file.Path; -import java.util.Map; -import java.util.function.Function; -import org.jetbrains.annotations.NotNull; - -@ApplicationScoped -public class ConfigService { - - @Inject - DataService dataService; - - @Inject - ProjectPathsResolver projectPathsResolver; - - @NotNull - public Config getSecretVolume(Path projectPath, String configPath) { - return getConfig(configPath, getSecretConfigPathMapper(projectPath), - dataService::loadDataFromFiles, this::getConfigType); - } - - @NotNull - public Config getConfigVolume(Path projectPath, String configPath) { - return getConfig(configPath, getConfigPathMapper(projectPath), - dataService::loadDataFromFiles, this::getConfigType); - } - - @NotNull - public Config getSecretEnv(Path projectPath, String configPath) { - return getConfig(configPath, getSecretConfigPathMapper(projectPath), - dataService::loadDataFromProperties, path -> ConfigType.FILE); - } - - @NotNull - public Config getConfigEnv(Path projectPath, String configPath) { - return getConfig(configPath, getConfigPathMapper(projectPath), - dataService::loadDataFromProperties, path -> ConfigType.FILE); - } - - @NotNull - private Function getSecretConfigPathMapper(Path projectPath) { - return (path) -> projectPathsResolver.resolveSecretPath(projectPath, path); - } - - @NotNull - private Function getConfigPathMapper(Path projectPath) { - return (path) -> projectPathsResolver.resolveConfigPath(projectPath, path); - } - - @NotNull - private Config getConfig(String configPath, Function pathMapper, - Function> dataMapper, Function configTypeMapper) { - Path mappedPath = pathMapper.apply(configPath); - Map data = dataMapper.apply(mappedPath); - ConfigType configType = configTypeMapper.apply(mappedPath); - return new Config(configPath, data, configType); - } - - @NotNull - ConfigType getConfigType(Path dataSourcePath) { - File dataSource = dataSourcePath.toFile(); - if (dataSource.isFile()) { - return ConfigType.FILE; - } - if (dataSource.isDirectory()) { - return ConfigType.DIR; - } - throw new IllegalStateException( - "Config source " + dataSource + " provided in Mesh should be file or directory."); - } -} diff --git a/core/src/main/java/dev/streamx/cli/command/cloud/deploy/DataService.java b/core/src/main/java/dev/streamx/cli/command/cloud/deploy/DataService.java deleted file mode 100644 index 8dce4263..00000000 --- a/core/src/main/java/dev/streamx/cli/command/cloud/deploy/DataService.java +++ /dev/null @@ -1,99 +0,0 @@ -package dev.streamx.cli.command.cloud.deploy; - -import jakarta.enterprise.context.ApplicationScoped; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Stream; - -@ApplicationScoped -public class DataService { - - private static final Pattern validKeyPattern = Pattern.compile("[-._a-zA-Z0-9]+"); - - public Map loadDataFromProperties(Path propertiesFilePath) { - File propertiesFile = propertiesFilePath.toFile(); - if (!propertiesFile.exists() || !propertiesFile.isFile()) { - throw new IllegalStateException("Path " + propertiesFilePath.normalize() - + " provided in Mesh must be a valid properties file."); - } - Properties properties = new Properties(); - try (FileInputStream fis = new FileInputStream(propertiesFile)) { - properties.load(fis); - Map data = new HashMap<>(); - for (Map.Entry entry : properties.entrySet()) { - String propertyKey = entry.getKey().toString(); - validatePropertyKey(propertiesFilePath.toString(), propertyKey); - data.put(propertyKey, entry.getValue().toString()); - } - return data; - } catch (IOException e) { - throw new IllegalStateException("Error reading properties file: " + propertiesFile, e); - } - } - - public Map loadDataFromFiles(Path path) { - Map data = new HashMap<>(); - try { - if (Files.isRegularFile(path)) { - loadDataFromFile(path.toString(), path, data); - } else if (Files.isDirectory(path)) { - try (Stream walk = Files.walk(path, 1)) { - walk - .filter(Files::isRegularFile) - .forEach(file -> { - try { - loadDataFromFile(path.toString(), file, data); - } catch (IOException e) { - throw new RuntimeException("Failed to read data file: " + file.toAbsolutePath(), - e); - } - }); - } - } else { - throw new IllegalArgumentException( - "Path " + path.normalize() + " provided in Mesh must be a file or a directory."); - } - } catch (IOException e) { - throw new IllegalStateException("Failed to convert " + path.normalize() + " to data", e); - } - - return data; - } - - private void loadDataFromFile(String configPath, Path filePath, Map data) - throws IOException { - String content = Files.readString(filePath); - String fileName = filePath.getFileName().toString(); - validateFileName(configPath, fileName); - data.put(fileName, content); - } - - private void validateFileName(String configPath, String fileName) { - if (isConfigDataKeyInvalid(fileName)) { - throw new IllegalArgumentException( - "Invalid file name: " + fileName + " in volumesFrom: " + configPath - + ". Valid file name must consist of alphanumeric characters, '-', '_' or '.'."); - } - } - - private void validatePropertyKey(String configPath, String key) { - if (isConfigDataKeyInvalid(key)) { - throw new IllegalArgumentException( - "Invalid properties key: " + key + " in environmentFrom: " + configPath - + ". Valid property key must consist of alphanumeric characters, '-', '_' or '.'."); - } - } - - private boolean isConfigDataKeyInvalid(String key) { - Matcher matcher = validKeyPattern.matcher(key); - return !matcher.matches(); - } -} diff --git a/core/src/main/java/dev/streamx/cli/command/cloud/deploy/DeployCommand.java b/core/src/main/java/dev/streamx/cli/command/cloud/deploy/DeployCommand.java deleted file mode 100644 index 4ec4112f..00000000 --- a/core/src/main/java/dev/streamx/cli/command/cloud/deploy/DeployCommand.java +++ /dev/null @@ -1,123 +0,0 @@ -package dev.streamx.cli.command.cloud.deploy; - -import static dev.streamx.cli.util.Output.printf; - -import com.fasterxml.jackson.databind.ObjectMapper; -import dev.streamx.cli.VersionProvider; -import dev.streamx.cli.command.cloud.KubernetesArguments; -import dev.streamx.cli.command.cloud.KubernetesService; -import dev.streamx.cli.command.cloud.ServiceMeshResolver; -import dev.streamx.cli.command.cloud.ServiceMeshResolver.ConfigSourcesPaths; -import dev.streamx.cli.command.cloud.collector.DirectoryResourcesCollector; -import dev.streamx.cli.command.cloud.collector.KubernetesResourcesCollector; -import dev.streamx.cli.command.meshprocessing.MeshConfig; -import dev.streamx.cli.command.meshprocessing.MeshResolver; -import dev.streamx.cli.command.meshprocessing.MeshSource; -import dev.streamx.cli.interpolation.Interpolating; -import dev.streamx.operator.crd.ServiceMesh; -import io.fabric8.kubernetes.api.model.HasMetadata; -import jakarta.inject.Inject; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; -import picocli.CommandLine.ArgGroup; -import picocli.CommandLine.Command; - -@Command( - name = DeployCommand.COMMAND_NAME, - mixinStandardHelpOptions = true, - versionProvider = VersionProvider.class, - description = "Deploy the StreamX project to the cloud.", - footer = DeployCommand.CLOUD_COMMAND_FOOTER -) -public class DeployCommand implements Runnable { - - public static final String COMMAND_NAME = "deploy"; - - public static final String CLOUD_COMMAND_FOOTER = """ - - The command automatically uses the cluster connection and namespace settings from the \ - current context in your @|italic kubeconfig|@ file. Ensure that your @|italic kubeconfig|@ \ - is configured correctly and pointing to the desired cluster and namespace. You can verify \ - your current context and namespace by running: - - @|yellow kubectl config current-context|@ - @|yellow kubectl config view --minify | grep namespace|@ - - If necessary, switch to the correct context using: - - @|yellow kubectl config use-context |@ - - This command assumes the StreamX Operator is installed and the required CRDs are available \ - on the target cluster. If not, please install the operator and ensure the cluster meets \ - the prerequisites before running this command."""; - - @ArgGroup - MeshSource meshSource; - - @ArgGroup(exclusive = false) - KubernetesArguments kubernetesArguments; - - @Inject - MeshConfig meshConfig; - - @Inject - MeshResolver meshResolver; - - @Inject - ServiceMeshResolver serviceMeshResolver; - - @Inject - KubernetesService kubernetesService; - - @Inject - ProjectResourcesExtractor projectResourcesExtractor; - - @Inject - @Interpolating - ObjectMapper objectMapper; - - @Override - public void run() { - Path meshPath = meshResolver.resolveMeshPath(meshConfig); - meshPath = meshPath.toAbsolutePath(); - ServiceMesh serviceMesh = serviceMeshResolver.resolveMesh(meshPath); - Path projectPath = meshPath.getParent(); - deploy(serviceMesh, projectPath); - } - - private void deploy(ServiceMesh serviceMesh, Path projectPath) { - kubernetesService.validateCrdInstallation(); - ConfigSourcesPaths configPaths = serviceMeshResolver.extractConfigSourcesPaths(serviceMesh); - String serviceMeshName = serviceMesh.getMetadata().getName(); - - List resourcesToDeploy = new ArrayList<>(); - List managedResources = kubernetesService.collectManagedResources(serviceMeshName); - - // Collect all resources to deploy - resourcesToDeploy.addAll(collectKubernetesResources(projectPath, serviceMeshName)); - resourcesToDeploy.addAll( - projectResourcesExtractor.getSecrets(projectPath, configPaths, serviceMeshName)); - resourcesToDeploy.addAll( - projectResourcesExtractor.getConfigMaps(projectPath, configPaths, serviceMeshName)); - resourcesToDeploy.add(serviceMesh); - - // Collect all resources to delete - ResourceCleaner cleaner = new ResourceCleaner(resourcesToDeploy, managedResources); - kubernetesService.deploy(resourcesToDeploy); - printf("Project %s successfully deployed to '%s' namespace.%n", - projectPath.toAbsolutePath().normalize(), kubernetesService.getNamespace()); - List orphanedResources = cleaner.getOrphanedResources(); - printf("Deleting %d orphaned resources.\n", orphanedResources.size()); - kubernetesService.undeploy(orphanedResources); - - } - - private List collectKubernetesResources(Path projectPath, String serviceMeshName) { - List resourcesDirectories = kubernetesService.getResourcePaths(); - - KubernetesResourcesCollector collector = new DirectoryResourcesCollector(objectMapper, - projectPath, resourcesDirectories); - return collector.collect(serviceMeshName); - } -} diff --git a/core/src/main/java/dev/streamx/cli/command/cloud/deploy/ProjectResourcesExtractor.java b/core/src/main/java/dev/streamx/cli/command/cloud/deploy/ProjectResourcesExtractor.java deleted file mode 100644 index 178ed241..00000000 --- a/core/src/main/java/dev/streamx/cli/command/cloud/deploy/ProjectResourcesExtractor.java +++ /dev/null @@ -1,80 +0,0 @@ -package dev.streamx.cli.command.cloud.deploy; - -import dev.streamx.cli.command.cloud.KubernetesService; -import dev.streamx.cli.command.cloud.ServiceMeshResolver.ConfigSourcesPaths; -import io.fabric8.kubernetes.api.model.ConfigMap; -import io.fabric8.kubernetes.api.model.Secret; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; -import java.nio.file.Path; -import java.util.List; -import java.util.Set; -import java.util.stream.Stream; -import org.jetbrains.annotations.NotNull; - -@ApplicationScoped -public class ProjectResourcesExtractor { - - @Inject - ConfigService configService; - - @Inject - KubernetesService kubernetesService; - - @NotNull - public List getSecrets(Path projectPath, ConfigSourcesPaths configSourcesPaths, - String serviceMeshName) { - List<@NotNull Secret> envSecrets = getEnvSecrets(projectPath, - configSourcesPaths.secretEnvPaths(), - serviceMeshName); - List<@NotNull Secret> volumeSecrets = getVolumeSecrets(projectPath, - configSourcesPaths.secretVolumePaths(), serviceMeshName); - return Stream.concat(envSecrets.stream(), volumeSecrets.stream()).toList(); - } - - @NotNull - public List getConfigMaps(Path projectPath, ConfigSourcesPaths configSourcesPaths, - String serviceMeshName) { - List<@NotNull ConfigMap> envConfigMaps = getEnvConfigMaps(projectPath, - configSourcesPaths.configEnvPaths(), serviceMeshName); - List<@NotNull ConfigMap> volumeConfigMaps = getVolumeConfigMaps(projectPath, - configSourcesPaths.configVolumePaths(), serviceMeshName); - return Stream.concat(envConfigMaps.stream(), volumeConfigMaps.stream()).toList(); - } - - @NotNull - private List<@NotNull Secret> getVolumeSecrets(Path projectPath, Set volumePaths, - String serviceMeshName) { - return volumePaths.stream() - .map(path -> configService.getSecretVolume(projectPath, path)) - .map(config -> kubernetesService.buildSecret(serviceMeshName, config)) - .toList(); - } - - @NotNull - private List<@NotNull Secret> getEnvSecrets(Path projectPath, Set envPaths, - String serviceMeshName) { - return envPaths.stream() - .map(path -> configService.getSecretEnv(projectPath, path)) - .map(config -> kubernetesService.buildSecret(serviceMeshName, config)) - .toList(); - } - - @NotNull - private List<@NotNull ConfigMap> getEnvConfigMaps(Path projectPath, Set envPaths, - String serviceMeshName) { - return envPaths.stream() - .map(path -> configService.getConfigEnv(projectPath, path)) - .map(config -> kubernetesService.buildConfigMap(serviceMeshName, config)) - .toList(); - } - - @NotNull - private List<@NotNull ConfigMap> getVolumeConfigMaps(Path projectPath, Set volumePaths, - String serviceMeshName) { - return volumePaths.stream() - .map(path -> configService.getConfigVolume(projectPath, path)) - .map(config -> kubernetesService.buildConfigMap(serviceMeshName, config)) - .toList(); - } -} diff --git a/core/src/main/java/dev/streamx/cli/command/cloud/deploy/ResourceCleaner.java b/core/src/main/java/dev/streamx/cli/command/cloud/deploy/ResourceCleaner.java deleted file mode 100644 index 5047c859..00000000 --- a/core/src/main/java/dev/streamx/cli/command/cloud/deploy/ResourceCleaner.java +++ /dev/null @@ -1,47 +0,0 @@ -package dev.streamx.cli.command.cloud.deploy; - -import io.fabric8.kubernetes.api.model.HasMetadata; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -public class ResourceCleaner { - - private final List resourcesToDeploy; - private final List managedResources; - - /** - * Constructor takes two lists: - * - * @param resourcesToDeploy list of resources planned for deployment - * @param managedResources list of currently managed resources - */ - public ResourceCleaner(List resourcesToDeploy, List managedResources) { - this.resourcesToDeploy = resourcesToDeploy; - this.managedResources = managedResources; - } - - /** - * Returns the list of orphaned managed resources, i.e. those managed resources that are not - * present in the deployment list, based on namespace and name. - * - * @return List of resources to remove. - */ - public List getOrphanedResources() { - // Build a set of keys (namespace/name) for resources to deploy. - Set deployKeys = resourcesToDeploy.stream() - .map(this::keyFor) - .collect(Collectors.toSet()); - - // Filter managedResources to keep only those not present in deployKeys. - return managedResources.stream() - .filter(resource -> !deployKeys.contains(keyFor(resource))) - .collect(Collectors.toList()); - } - - private String keyFor(HasMetadata resource) { - String api = resource.getApiVersion() + "/" + resource.getKind(); - String name = resource.getMetadata().getName(); - return api + "/" + name; - } -} \ No newline at end of file diff --git a/core/src/main/java/dev/streamx/cli/command/cloud/undeploy/UndeployCommand.java b/core/src/main/java/dev/streamx/cli/command/cloud/undeploy/UndeployCommand.java deleted file mode 100644 index eb439bac..00000000 --- a/core/src/main/java/dev/streamx/cli/command/cloud/undeploy/UndeployCommand.java +++ /dev/null @@ -1,38 +0,0 @@ -package dev.streamx.cli.command.cloud.undeploy; - -import static dev.streamx.cli.command.cloud.ServiceMeshResolver.SERVICE_MESH_NAME; -import static dev.streamx.cli.util.Output.printf; - -import dev.streamx.cli.VersionProvider; -import dev.streamx.cli.command.cloud.KubernetesArguments; -import dev.streamx.cli.command.cloud.KubernetesService; -import dev.streamx.cli.command.cloud.deploy.DeployCommand; -import jakarta.inject.Inject; -import picocli.CommandLine.ArgGroup; -import picocli.CommandLine.Command; - -@Command( - name = UndeployCommand.COMMAND_NAME, - mixinStandardHelpOptions = true, - versionProvider = VersionProvider.class, - description = "Undeploy the StreamX Project from the cloud.", - footer = DeployCommand.CLOUD_COMMAND_FOOTER -) -public class UndeployCommand implements Runnable { - - public static final String COMMAND_NAME = "undeploy"; - - @ArgGroup(exclusive = false) - KubernetesArguments kubernetesArguments; - @Inject - KubernetesService kubernetesService; - - @Override - public void run() { - kubernetesService.validateCrdInstallation(); - kubernetesService.undeploy(SERVICE_MESH_NAME); - - printf("StreamX project successfully undeployed from '%s' namespace.%n", - kubernetesService.getNamespace()); - } -} diff --git a/core/src/main/java/dev/streamx/cli/command/dev/BrowserOpener.java b/core/src/main/java/dev/streamx/cli/command/dev/BrowserOpener.java deleted file mode 100644 index 6448571a..00000000 --- a/core/src/main/java/dev/streamx/cli/command/dev/BrowserOpener.java +++ /dev/null @@ -1,36 +0,0 @@ -package dev.streamx.cli.command.dev; - -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; -import java.awt.Desktop; -import java.net.URI; -import org.jboss.logging.Logger; - -@ApplicationScoped -public class BrowserOpener { - - @Inject - Logger logger; - - @Inject - DevConfig devConfig; - - public void tryOpenBrowser() { - if (!devConfig.openOnStart()) { - return; - } - - try { - if (Desktop.isDesktopSupported()) { - Desktop desktop = Desktop.getDesktop(); - String entryUrl = "http://localhost:%d/overview".formatted(devConfig.dashboardPort()); - URI meshManagerUri = new URI(entryUrl); - desktop.browse(meshManagerUri); - } else { - logger.warn("Opening browser is not supported"); - } - } catch (Exception e) { - logger.error("Opening browser failed", e); - } - } -} diff --git a/core/src/main/java/dev/streamx/cli/command/dev/DashboardContainer.java b/core/src/main/java/dev/streamx/cli/command/dev/DashboardContainer.java deleted file mode 100644 index 7c032368..00000000 --- a/core/src/main/java/dev/streamx/cli/command/dev/DashboardContainer.java +++ /dev/null @@ -1,31 +0,0 @@ -package dev.streamx.cli.command.dev; - -import java.util.List; -import org.testcontainers.containers.BindMode; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.utility.DockerImageName; - -public class DashboardContainer extends GenericContainer { - - public static final String CONTAINER_NAME = "streamx-dashboard"; - - private static final int DASHBOARDS_CONTAINER_PORT = 8080; - - public DashboardContainer(String fullImageName, int exposedPort, String meshPath, - String meshDirectory, String projectDirectory) { - super(DockerImageName.parse(fullImageName)); - setExposedPorts(List.of(DASHBOARDS_CONTAINER_PORT)); - addFixedExposedPort(exposedPort, DASHBOARDS_CONTAINER_PORT); - - withFileSystemBind(meshPath, "/data/mesh.yaml", BindMode.READ_WRITE); - withFileSystemBind(meshDirectory, "/data/mesh", BindMode.READ_WRITE); - - if (projectDirectory != null) { - withFileSystemBind(projectDirectory, "/data/project", BindMode.READ_WRITE); - } - - withCreateContainerCmdModifier(cmd -> cmd.withName(CONTAINER_NAME)); - withEnv("streamx.platform.mesh.services-metadata-registry-roots", - "/data/project/services,/data/mesh/services"); - } -} diff --git a/core/src/main/java/dev/streamx/cli/command/dev/DashboardRunner.java b/core/src/main/java/dev/streamx/cli/command/dev/DashboardRunner.java deleted file mode 100644 index f2c2a171..00000000 --- a/core/src/main/java/dev/streamx/cli/command/dev/DashboardRunner.java +++ /dev/null @@ -1,56 +0,0 @@ -package dev.streamx.cli.command.dev; - -import static dev.streamx.cli.util.Output.print; - -import dev.streamx.cli.command.dev.event.DashboardStarted; -import dev.streamx.cli.util.StreamxMavenPropertiesUtils; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.enterprise.event.Event; -import jakarta.inject.Inject; -import java.time.Duration; -import org.testcontainers.containers.Network; -import org.testcontainers.containers.wait.strategy.Wait; - -@ApplicationScoped -public class DashboardRunner { - private static final long CONTAINER_TIMEOUT_IN_SECS = 60_000L; - - @Inject - DevConfig devConfig; - - @Inject - BrowserOpener browserOpener; - - @Inject - Event dashboardStartedEvent; - - private DashboardContainer dashboardContainer; - - public void startStreamxDashboard(String meshPath, String meshDirectory, - String projectDirectory, Network network) { - dashboardContainer = new DashboardContainer( - StreamxMavenPropertiesUtils.getDashboardImage(), - devConfig.dashboardPort(), - meshPath, - meshDirectory, - projectDirectory - ) - .withNetwork(network) - .waitingFor(Wait.forHttp("/q/health") - .forPort(8080) - ) - .withStartupTimeout(Duration.ofSeconds(CONTAINER_TIMEOUT_IN_SECS)); - - dashboardContainer.start(); - - print("StreamX Dashboard started on http://localhost:" + devConfig.dashboardPort()); - - browserOpener.tryOpenBrowser(); - dashboardStartedEvent.fire(new DashboardStarted()); - } - - public void stopStreamxDashboard() { - dashboardContainer.stop(); - print("StreamX Dashboard stopped"); - } -} diff --git a/core/src/main/java/dev/streamx/cli/command/dev/DevCommand.java b/core/src/main/java/dev/streamx/cli/command/dev/DevCommand.java deleted file mode 100644 index 0a0e8b69..00000000 --- a/core/src/main/java/dev/streamx/cli/command/dev/DevCommand.java +++ /dev/null @@ -1,135 +0,0 @@ -package dev.streamx.cli.command.dev; - -import static dev.streamx.cli.util.Output.print; - -import dev.streamx.cli.BannerPrinter; -import dev.streamx.cli.VersionProvider; -import dev.streamx.cli.command.dev.event.DevReady; -import dev.streamx.cli.command.meshprocessing.MeshConfig; -import dev.streamx.cli.command.meshprocessing.MeshManager; -import dev.streamx.cli.command.meshprocessing.MeshResolver; -import dev.streamx.cli.command.meshprocessing.MeshSource; -import dev.streamx.cli.command.meshprocessing.MeshWatcher; -import dev.streamx.cli.exception.DockerException; -import dev.streamx.runner.StreamxRunner; -import dev.streamx.runner.container.PulsarContainer; -import dev.streamx.runner.exception.ContainerStartupTimeoutException; -import io.quarkus.runtime.Quarkus; -import jakarta.enterprise.event.Event; -import jakarta.inject.Inject; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Set; -import org.jboss.logging.Logger; -import picocli.CommandLine; -import picocli.CommandLine.ArgGroup; -import picocli.CommandLine.Command; -import picocli.CommandLine.Spec; - -@Command(name = DevCommand.COMMAND_NAME, - mixinStandardHelpOptions = true, - versionProvider = VersionProvider.class, - description = "Develop a StreamX Mesh locally.") -public class DevCommand implements Runnable { - - public static final String COMMAND_NAME = "dev"; - - @Inject - Logger logger; - - @ArgGroup - MeshSource meshSource; - - @Spec - CommandLine.Model.CommandSpec spec; - - @Inject - MeshConfig meshConfig; - - @Inject - MeshResolver meshResolver; - - @Inject - DockerValidator dockerValidator; - - @Inject - StreamxRunner runner; - - @Inject - MeshWatcher meshWatcher; - - @Inject - BannerPrinter bannerPrinter; - - @Inject - MeshManager meshManager; - - @Inject - DashboardRunner dashboardRunner; - - @Inject - Event devReadyEvent; - - @Override - public void run() { - try { - Path meshPath = meshResolver.resolveMeshPath(meshConfig, false); - - dockerValidator.validateDockerEnvironment(Set.of( - DashboardContainer.CONTAINER_NAME, - PulsarContainer.NAME - )); - - bannerPrinter.printBanner(); - boolean meshFileExists = meshPath.toFile().exists(); - if (!meshFileExists) { - Files.createFile(meshPath); - } - - this.runner.initializeBase(); - startDashboard(meshPath); - - meshManager.initializeDevMode(meshPath, spec.commandLine()); - meshWatcher.watchMeshChanges(meshPath); - - if (meshFileExists) { - meshManager.start(); - } - - devReadyEvent.fire(new DevReady()); - - Quarkus.waitForExit(); - } catch (ContainerStartupTimeoutException e) { - throw DockerException.containerStartupFailed( - e.getContainerName(), - runner.getContext().getStreamxBaseConfig().getContainerStartupTimeout()); - } catch (IOException e) { - // handle creat file - throw new RuntimeException(e); - } - } - - private void startDashboard(Path meshPath) { - print("Setting up StreamX Dashboard..."); - var meshPathAsString = meshPath.toAbsolutePath().normalize().toString(); - Path meshDirectory = meshPath.resolve(".."); - Path projectDirectory = null; - if (Files.exists(meshDirectory.resolve("..").normalize())) { - projectDirectory = meshDirectory.resolve("..").normalize(); - } - - var meshDirectoryAsString = meshDirectory.toAbsolutePath().normalize().toString(); - var projectDirectoryAsString = projectDirectory != null - ? projectDirectory.toAbsolutePath().normalize().toString() - : null; - - logger.infov("Resolved mesh {0}, mesh directory {1} and project directory {2}", - meshPathAsString, meshDirectoryAsString, projectDirectoryAsString); - dashboardRunner.startStreamxDashboard( - meshPathAsString, - meshDirectoryAsString, - projectDirectoryAsString, - runner.getContext().getNetwork()); - } -} diff --git a/core/src/main/java/dev/streamx/cli/command/dev/DevConfig.java b/core/src/main/java/dev/streamx/cli/command/dev/DevConfig.java deleted file mode 100644 index 616bc97d..00000000 --- a/core/src/main/java/dev/streamx/cli/command/dev/DevConfig.java +++ /dev/null @@ -1,20 +0,0 @@ -package dev.streamx.cli.command.dev; - -import io.smallrye.config.ConfigMapping; -import io.smallrye.config.WithDefault; -import io.smallrye.config.WithName; - -@ConfigMapping -public interface DevConfig { - - String STREAMX_DEV_DASHBOARD_PORT = "streamx.dev.dashboard.port"; - String STREAMX_DEV_DASHBOARD_OPEN_ON_STARTUP = "streamx.dev.dashboard.open-on-startup"; - - @WithName(STREAMX_DEV_DASHBOARD_PORT) - @WithDefault("9088") - int dashboardPort(); - - @WithName(STREAMX_DEV_DASHBOARD_OPEN_ON_STARTUP) - @WithDefault("true") - boolean openOnStart(); -} diff --git a/core/src/main/java/dev/streamx/cli/command/dev/DockerValidator.java b/core/src/main/java/dev/streamx/cli/command/dev/DockerValidator.java deleted file mode 100644 index 7dfb0c47..00000000 --- a/core/src/main/java/dev/streamx/cli/command/dev/DockerValidator.java +++ /dev/null @@ -1,28 +0,0 @@ -package dev.streamx.cli.command.dev; - -import com.github.dockerjava.api.DockerClient; -import dev.streamx.runner.validation.DockerContainerValidator; -import dev.streamx.runner.validation.DockerEnvironmentValidator; -import jakarta.enterprise.context.Dependent; -import jakarta.inject.Inject; -import java.util.Set; -import org.jboss.logging.Logger; - -@Dependent -public class DockerValidator { - - private static final Logger LOG = Logger.getLogger(DockerValidator.class); - - @Inject - DockerEnvironmentValidator dockerEnvironmentValidator; - - @Inject - DockerContainerValidator dockerContainerValidator; - - public void validateDockerEnvironment(Set validatedContainerNames) { - LOG.info("Validating environment..."); - - DockerClient client = dockerEnvironmentValidator.validateDockerClient(); - dockerContainerValidator.verifyExistingContainers(client, validatedContainerNames); - } -} diff --git a/core/src/main/java/dev/streamx/cli/command/dev/event/DashboardStarted.java b/core/src/main/java/dev/streamx/cli/command/dev/event/DashboardStarted.java deleted file mode 100644 index d23e7b49..00000000 --- a/core/src/main/java/dev/streamx/cli/command/dev/event/DashboardStarted.java +++ /dev/null @@ -1,5 +0,0 @@ -package dev.streamx.cli.command.dev.event; - -public class DashboardStarted { - -} diff --git a/core/src/main/java/dev/streamx/cli/command/dev/event/DevReady.java b/core/src/main/java/dev/streamx/cli/command/dev/event/DevReady.java deleted file mode 100644 index 0a754e0e..00000000 --- a/core/src/main/java/dev/streamx/cli/command/dev/event/DevReady.java +++ /dev/null @@ -1,5 +0,0 @@ -package dev.streamx.cli.command.dev.event; - -public class DevReady { - -} diff --git a/core/src/main/java/dev/streamx/cli/command/ingestion/BaseIngestionCommand.java b/core/src/main/java/dev/streamx/cli/command/ingestion/BaseIngestionCommand.java index d0e88f12..a4bcb91a 100644 --- a/core/src/main/java/dev/streamx/cli/command/ingestion/BaseIngestionCommand.java +++ b/core/src/main/java/dev/streamx/cli/command/ingestion/BaseIngestionCommand.java @@ -1,25 +1,21 @@ package dev.streamx.cli.command.ingestion; +import static java.nio.charset.StandardCharsets.UTF_8; + import com.fasterxml.jackson.databind.JsonNode; -import dev.streamx.cli.SchemaProvider; +import com.streamx.clients.ingestion.StreamxClient; +import com.streamx.clients.ingestion.exceptions.StreamxClientException; +import com.streamx.clients.ingestion.publisher.Publisher; import dev.streamx.cli.exception.IngestionClientException; -import dev.streamx.cli.exception.UnableToConnectIngestionServiceException; -import dev.streamx.cli.exception.UnknownChannelException; +import dev.streamx.cli.model.Resource; import dev.streamx.cli.util.ExceptionUtils; -import dev.streamx.clients.ingestion.StreamxClient; -import dev.streamx.clients.ingestion.exceptions.StreamxClientConnectionException; -import dev.streamx.clients.ingestion.exceptions.StreamxClientException; -import dev.streamx.clients.ingestion.exceptions.UnsupportedChannelException; -import dev.streamx.clients.ingestion.publisher.Publisher; +import io.cloudevents.CloudEvent; +import io.cloudevents.jackson.JsonCloudEventData; import jakarta.inject.Inject; -import java.util.List; +import java.nio.ByteBuffer; import javax.net.ssl.SSLHandshakeException; -import org.apache.avro.Schema; -import org.apache.avro.Schema.Field; -import org.apache.avro.Schema.Type; import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Model.CommandSpec; -import picocli.CommandLine.ParameterException; import picocli.CommandLine.Spec; public abstract class BaseIngestionCommand implements Runnable { @@ -33,24 +29,16 @@ public abstract class BaseIngestionCommand implements Runnable { @Inject StreamxClientProvider streamxClientProvider; - @Inject - SchemaProvider schemaProvider; - @Inject IngestionClientConfig ingestionClientConfig; - protected abstract String getChannel(); - - protected abstract void perform(Publisher publisher) throws StreamxClientException; + protected abstract void perform(Publisher publisher) throws StreamxClientException; @Override public final void run() { try (StreamxClient client = streamxClientProvider.createStreamxClient(ingestionClientConfig)) { - doRun(client); - } catch (UnsupportedChannelException e) { - throw new ParameterException(spec.commandLine(), e.getMessage()); - } catch (StreamxClientConnectionException e) { - throw new UnableToConnectIngestionServiceException(ingestionClientConfig.url(), e); + Publisher publisher = client.newPublisher(); + perform(publisher); } catch (StreamxClientException e) { if (e.getCause() instanceof SSLHandshakeException) { throw IngestionClientException.sslException(ingestionClientConfig.url()); @@ -59,39 +47,37 @@ public final void run() { } } - protected void doRun(StreamxClient client) throws StreamxClientException { - Publisher publisher = client.newPublisher(getChannel(), JsonNode.class); - perform(publisher); - } - - protected String getPayloadPropertyName() { - JsonNode schemaJson = getSchemaForChannel(); - return getPayloadPropertyName(schemaJson); + protected static CloudEvent withAdjustedData(CloudEvent event) { + JsonCloudEventData data = (JsonCloudEventData) event.getData(); + Resource resource = extractResource(data); + if (resource != null) { + // convert data to format expected by StreamX + return CloudEventBuilder.copyWithNewData(event, resource); + } + return event; } - private String getPayloadPropertyName(JsonNode schemaJson) { - Schema.Parser parser = new Schema.Parser(); - Schema channelSchema = parser.parse(schemaJson.toString()); - Field payload = channelSchema.getField("payload"); - Schema payloadSchema = payload.schema(); - if (payloadSchema.getType() == Type.UNION) { - List unionSchemas = payloadSchema.getTypes(); - for (Schema schema : unionSchemas) { - if (schema.getType() == Type.RECORD) { - return schema.getFullName(); + private static Resource extractResource(JsonCloudEventData data) { + if (data != null) { + JsonNode dataNode = data.getNode(); + if (dataNode != null) { + JsonNode contentNode = dataNode.get("content"); + JsonNode typeNode = dataNode.get("type"); + String type = typeNode != null ? typeNode.asText() : null; + if (contentNode != null) { + JsonNode bytesNode = contentNode.get("bytes"); + if (bytesNode != null) { + String content = bytesNode.asText(); + if (content != null) { + // TODO: detect if the content.bytes field is already base 64 encoded + // and if it is encoded - decode it first. + // For now assume it's in clear text + return new Resource(ByteBuffer.wrap(content.getBytes(UTF_8)), type); + } + } } } } - return payloadSchema.getFullName(); - } - - private JsonNode getSchemaForChannel() { - try { - return schemaProvider.getSchema(getChannel()); - } catch (UnknownChannelException e) { - throw new ParameterException(spec.commandLine(), - "Channel '" + e.getChannel() + "' not found. " - + "Available channels: " + e.getAvailableChannels()); - } + return null; } } diff --git a/core/src/main/java/dev/streamx/cli/command/ingestion/CloudEventBuilder.java b/core/src/main/java/dev/streamx/cli/command/ingestion/CloudEventBuilder.java new file mode 100644 index 00000000..b610f9e0 --- /dev/null +++ b/core/src/main/java/dev/streamx/cli/command/ingestion/CloudEventBuilder.java @@ -0,0 +1,52 @@ +package dev.streamx.cli.command.ingestion; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.NullNode; +import dev.streamx.cli.model.Resource; +import io.cloudevents.CloudEvent; +import io.cloudevents.jackson.JsonCloudEventData; +import java.net.URI; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.util.UUID; + +public class CloudEventBuilder { + + private static final ObjectMapper objectMapper = new ObjectMapper(); + + private CloudEventBuilder() { + // no instance + } + + public static CloudEvent build(String subject, String type, String source, Object data) { + JsonNode dataNode = objectMapper.valueToTree(data); + return build(subject, type, source, dataNode); + } + + public static CloudEvent build(String subject, String type, String source, JsonNode data) { + var builder = io.cloudevents.core.builder.CloudEventBuilder.v1() + .withId(UUID.randomUUID().toString()) + .withSource(URI.create(source)) + .withSubject(subject) + .withType(type) + .withTime(OffsetDateTime.now(ZoneOffset.UTC)); + + if (data == null || data instanceof NullNode) { + return builder.withoutData().build(); + } + + return builder + .withDataContentType("application/json") + .withData(JsonCloudEventData.wrap(data)) + .build(); + } + + public static CloudEvent copyWithNewData(CloudEvent cloudEvent, Resource data) { + return io.cloudevents.core.builder.CloudEventBuilder.v1(cloudEvent) + .withDataContentType("application/json") + .withData(JsonCloudEventData.wrap(objectMapper.valueToTree(data))) + .build(); + } + +} diff --git a/core/src/main/java/dev/streamx/cli/command/ingestion/IngestionArgumentConfig.java b/core/src/main/java/dev/streamx/cli/command/ingestion/IngestionArgumentConfig.java deleted file mode 100644 index bfe4b687..00000000 --- a/core/src/main/java/dev/streamx/cli/command/ingestion/IngestionArgumentConfig.java +++ /dev/null @@ -1,45 +0,0 @@ -package dev.streamx.cli.command.ingestion; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.jayway.jsonpath.Configuration; -import com.jayway.jsonpath.Option; -import com.jayway.jsonpath.spi.json.JsonProvider; -import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; -import com.jayway.jsonpath.spi.mapper.MappingProvider; -import dev.streamx.cli.command.ingestion.publish.payload.PropertyCreatingJacksonJsonNodeJsonProvider; -import io.quarkus.runtime.Startup; -import jakarta.enterprise.context.ApplicationScoped; -import java.util.Set; - -@ApplicationScoped -@Startup -public class IngestionArgumentConfig { - - private final JsonProvider jsonProvider; - private final MappingProvider mappingProvider; - - public IngestionArgumentConfig(@PayloadProcessing ObjectMapper objectMapper) { - jsonProvider = new PropertyCreatingJacksonJsonNodeJsonProvider(objectMapper); - mappingProvider = new JacksonMappingProvider(objectMapper); - configureDefaults(); - } - - private void configureDefaults() { - Configuration.setDefaults(new Configuration.Defaults() { - @Override - public JsonProvider jsonProvider() { - return jsonProvider; - } - - @Override - public Set - dev.streamx + com.streamx streamx-runner test @@ -64,7 +64,6 @@ src/test/resources/filtered mesh.yaml - overridden-nginx-conf diff --git a/e2e-tests/repository/web-server-sink/repository-volume/batch/publish/index.html b/e2e-tests/repository/web-server-sink/repository-volume/batch/publish/index.html new file mode 100644 index 00000000..9a1eb8d2 --- /dev/null +++ b/e2e-tests/repository/web-server-sink/repository-volume/batch/publish/index.html @@ -0,0 +1 @@ +

Hello World

\ No newline at end of file diff --git a/e2e-tests/repository/web-server-sink/repository-volume/publish/index.html b/e2e-tests/repository/web-server-sink/repository-volume/publish/index.html new file mode 100644 index 00000000..9a1eb8d2 --- /dev/null +++ b/e2e-tests/repository/web-server-sink/repository-volume/publish/index.html @@ -0,0 +1 @@ +

Hello World

\ No newline at end of file diff --git a/e2e-tests/src/main/java/dev/streamx/cli/test/tools/validators/HttpValidator.java b/e2e-tests/src/main/java/dev/streamx/cli/test/tools/validators/HttpValidator.java index 8829f8c4..5bca3d82 100644 --- a/e2e-tests/src/main/java/dev/streamx/cli/test/tools/validators/HttpValidator.java +++ b/e2e-tests/src/main/java/dev/streamx/cli/test/tools/validators/HttpValidator.java @@ -1,13 +1,13 @@ package dev.streamx.cli.test.tools.validators; import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.SECONDS; import static org.awaitility.Awaitility.await; import jakarta.annotation.PreDestroy; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import java.io.IOException; +import java.time.Duration; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; @@ -22,9 +22,9 @@ public class HttpValidator { @Inject CloseableHttpClient httpClient; - public void validate(String url, int expectedStatusCode, String expectedBody, int timeout) { + public void validate(String url, int expectedStatusCode, String expectedBody, Duration timeout) { await() - .atMost(timeout, SECONDS) + .atMost(timeout) .alias("Assertion of response from url: " + url + " with expecting status:" + expectedStatusCode + " body:" + expectedBody diff --git a/e2e-tests/src/main/java/dev/streamx/cli/test/tools/validators/ProcessOutputValidator.java b/e2e-tests/src/main/java/dev/streamx/cli/test/tools/validators/ProcessOutputValidator.java index dd39218b..2cda2c77 100644 --- a/e2e-tests/src/main/java/dev/streamx/cli/test/tools/validators/ProcessOutputValidator.java +++ b/e2e-tests/src/main/java/dev/streamx/cli/test/tools/validators/ProcessOutputValidator.java @@ -1,10 +1,10 @@ package dev.streamx.cli.test.tools.validators; import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.SECONDS; import static org.awaitility.Awaitility.await; import jakarta.enterprise.context.ApplicationScoped; +import java.time.Duration; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -21,12 +21,12 @@ public class ProcessOutputValidator { + "[\\p{Alnum}.,%_=?&#\\-+()\\[\\]\\*$~@!:/{};']*)", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE | Pattern.DOTALL); - public void validate(List output, String expectedContent, long timeout) { + public void validate(List output, String expectedContent, Duration timeout) { try { await() - .atMost(timeout, SECONDS) + .atMost(timeout) .pollInterval(100, MILLISECONDS) - .alias("Finding expectedContent:" + expectedContent) + .alias("Finding expectedContent: " + expectedContent) .until(() -> output .stream() @@ -38,9 +38,9 @@ public void validate(List output, String expectedContent, long timeout) } } - public String validateContainsUrl(List output, long timeout) { + public String validateContainsUrl(List output, Duration timeout) { await() - .atMost(timeout, SECONDS) + .atMost(timeout) .pollInterval(100, MILLISECONDS) .alias("Finding any url") .until(() -> diff --git a/e2e-tests/src/test/java/dev/streamx/cli/OsUtils.java b/e2e-tests/src/test/java/dev/streamx/cli/OsUtils.java index f6b7a575..0a531834 100644 --- a/e2e-tests/src/test/java/dev/streamx/cli/OsUtils.java +++ b/e2e-tests/src/test/java/dev/streamx/cli/OsUtils.java @@ -1,6 +1,6 @@ package dev.streamx.cli; -import dev.streamx.runner.validation.DockerEnvironmentValidator; +import com.streamx.runner.validation.DockerEnvironmentValidator; public class OsUtils { public static boolean isDockerAvailable() { diff --git a/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliLicenseIT.java b/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliLicenseIT.java index c2141d0f..28244190 100644 --- a/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliLicenseIT.java +++ b/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliLicenseIT.java @@ -12,6 +12,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.Duration; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @@ -20,7 +21,7 @@ @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class StreamxCliLicenseIT { - private static final int CLI_SHORT_TIMEOUT_IN_SEC = 5; + private static final Duration CLI_TIMEOUT = Duration.ofSeconds(10); @Inject @Named("StreamxCommandRunner") @@ -85,9 +86,9 @@ public void shouldAcceptingLicenseAllowUsingWithoutMoreQuestions() { private void assertIfLicenseIsAccessible(ShellProcess process) { String url = processOutputValidator.validateContainsUrl( process.getCurrentOutputLines(), - CLI_SHORT_TIMEOUT_IN_SEC); + CLI_TIMEOUT); httpValidator.validate(url, 200, "License", - CLI_SHORT_TIMEOUT_IN_SEC); + CLI_TIMEOUT); } private static void declineLicense(ShellProcess p) { @@ -102,12 +103,12 @@ private void assertIfAcceptanceLicenseErrorPresent(ShellProcess p) { processOutputValidator.validate( p.getCurrentErrorLines(), "License acceptance is required for using StreamX", - CLI_SHORT_TIMEOUT_IN_SEC); + CLI_TIMEOUT); } private void assertIfStreamxCommandWork(ShellProcess p) { processOutputValidator.validate(p.getCurrentOutputLines(), "streamx-cli version", - CLI_SHORT_TIMEOUT_IN_SEC); + CLI_TIMEOUT); } private ShellProcess runStreamxCommand() { @@ -118,7 +119,7 @@ private void assertIfAcceptLicenseQuestionPresent(ShellProcess p) { processOutputValidator.validate( p.getCurrentOutputLines(), "Do you accept the license agreement?", - CLI_SHORT_TIMEOUT_IN_SEC); + CLI_TIMEOUT); } private static void resetLicenseAcceptance() { diff --git a/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliPublicationIT.java b/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliPublicationIT.java index 9aa8b477..e5866fdc 100644 --- a/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliPublicationIT.java +++ b/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliPublicationIT.java @@ -1,8 +1,6 @@ package dev.streamx.cli; - import static dev.streamx.cli.test.tools.ResourcePathResolver.absolutePath; -import static org.junit.jupiter.params.provider.Arguments.arguments; import dev.streamx.cli.test.tools.terminal.TerminalCommandRunner; import dev.streamx.cli.test.tools.terminal.process.ShellProcess; @@ -11,26 +9,21 @@ import io.quarkus.test.junit.QuarkusTest; import jakarta.inject.Inject; import jakarta.inject.Named; -import java.util.List; -import java.util.stream.Stream; +import java.time.Duration; import org.eclipse.microprofile.config.inject.ConfigProperty; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.condition.EnabledIf; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; @QuarkusTest @EnabledIf("dev.streamx.cli.OsUtils#isDockerAvailable") @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class StreamxCliPublicationIT { - private static final int CLI_SHORT_TIMEOUT_IN_SEC = 5; + private static final Duration CLI_TIMEOUT = Duration.ofSeconds(10); @ConfigProperty(name = "streamx.cli.e2e.web.delivery.url", defaultValue = "http://localhost:8087/") String webDeliveryPortUrl; - @ConfigProperty(name = "streamx.cli.e2e.nginx.url", defaultValue = "http://localhost:8089/overridden/") - String nginxPortUrl; @ConfigProperty(name = "streamx.cli.e2e.setup.timeoutInSec", defaultValue = "60") int setupTimeoutInSec; @@ -48,85 +41,62 @@ public class StreamxCliPublicationIT { @BeforeAll public void setup() { runStreamxCommand( - "--accept-license run -f " + absolutePath("mesh.yaml"), + "--accept-license run_v2 -f " + absolutePath("mesh.yaml"), "STREAMX IS READY!", - setupTimeoutInSec); + Duration.ofSeconds(setupTimeoutInSec)); } - private void runStreamxCommand(String command, String expectedOutput, long timeoutInS) { + private void runStreamxCommand(String command, String expectedOutput, Duration timeout) { ShellProcess p = terminalCommandRunner.run(command); - processOutputValidator.validate(p.getCurrentOutputLines(), expectedOutput, timeoutInS); + processOutputValidator.validate(p.getCurrentOutputLines(), expectedOutput, timeout); } + private void runStreamxIngestionCommand(String commandName, String path, String expectedOutput) { + String command = "--accept-license %s %s".formatted(commandName, path); + runStreamxCommand(command, expectedOutput, CLI_TIMEOUT); + } - @ParameterizedTest - @MethodSource("testCases") - public void shouldTestPublishAndUnpublishPageOnStreamx( - String pageName, - String commandContentPart, - String expectedPageContent - ) { - - runStreamxCommand( - "--accept-license publish pages " + pageName + " " + commandContentPart, - "Sent data publication message to", - CLI_SHORT_TIMEOUT_IN_SEC); + @Test + public void shouldPublishAndUnpublishPageUsingStreamOperation() { + runStreamxIngestionCommand( + "stream_v2", + "src/test/resources/stream/page-publish-event.json", + "Sent com.streamx.blueprints.page.published.v1 event using stream with key 'hello.html'" + ); - validateStreamxPage(pageName, 200, expectedPageContent); + validateStreamxPage("hello.html", 200, "Hello World!"); - runStreamxCommand( - "--accept-license unpublish pages " + pageName, - "Sent data unpublication message to", - CLI_SHORT_TIMEOUT_IN_SEC); + runStreamxIngestionCommand( + "stream_v2", + "src/test/resources/stream/page-unpublish-event.json", + "Sent com.streamx.blueprints.page.unpublished.v1 event using stream with key 'hello.html'" + ); - validateStreamxPage(pageName, 404, ""); + validateStreamxPage("hello.html", 404, ""); } - static Stream testCases() { - return Stream.of( - arguments( - "third_param_page.html", - absolutePath("payload.json"), - "third_param_page" - ), - arguments( - "exact_param_page.html", - "-j '{\"content\":{\"bytes\":\"exact_param_page\"}}'", - "exact_param_page" - ), - arguments( - "json_path_exact_param_page.html", - "-s content.bytes='Json exact page'", - "Json exact page" - ), - arguments( - "json_path_exact_param_page.html", - "-b content.bytes='Json exact page'", - "Json exact page" - ), - arguments( - "file_param_page.html", - "-j file://" + absolutePath("file_param_page.json"), - "file_param_page" - ), - arguments( - "json_path_file_param_page.html", - "-s content.bytes=file://" + absolutePath("json_path_file_param_page.txt"), - "json_path_file_param_page" - ), - arguments( - "json_path_file_param_page.html", - "-b content.bytes=file://" + absolutePath("json_path_file_param_page.txt"), - "json_path_file_param_page" - ) + @Test + public void shouldPublishAndUnpublishPageUsingBatchOperation() { + runStreamxIngestionCommand( + "batch_v2", + "src/test/resources/batch/publish", + "Sent com.streamx.blueprints.page.published.v1 event using batch with key 'index.html'" + ); + + validateStreamxPage("index.html", 200, "

Hello World!

"); + + runStreamxIngestionCommand( + "batch_v2", + "src/test/resources/batch/unpublish", + "Sent com.streamx.blueprints.page.unpublished.v1 event using batch with key 'index.html'" ); + + validateStreamxPage("index.html", 404, ""); } private void validateStreamxPage(String resourcePath, int expectedStatusCode, String expectedBody) { - List.of(webDeliveryPortUrl, nginxPortUrl).forEach( - url -> httpValidator.validate(url + resourcePath, expectedStatusCode, expectedBody, - CLI_SHORT_TIMEOUT_IN_SEC) - ); + String url = webDeliveryPortUrl + resourcePath; + httpValidator.validate(url, expectedStatusCode, expectedBody, CLI_TIMEOUT); } } diff --git a/e2e-tests/src/test/resources/batch/publish/.eventsource.yaml b/e2e-tests/src/test/resources/batch/publish/.eventsource.yaml new file mode 100644 index 00000000..cecd6f94 --- /dev/null +++ b/e2e-tests/src/test/resources/batch/publish/.eventsource.yaml @@ -0,0 +1,11 @@ +eventType: com.streamx.blueprints.page.published.v1 +eventSource: source + +relativePathLevel: 0 +key: ${relativePath} + +payload: + content: + bytes: file://${payloadPath} + type: page/index-page + diff --git a/e2e-tests/src/test/resources/batch/publish/index.html b/e2e-tests/src/test/resources/batch/publish/index.html new file mode 100644 index 00000000..ba7c290e --- /dev/null +++ b/e2e-tests/src/test/resources/batch/publish/index.html @@ -0,0 +1 @@ +

Hello World!

\ No newline at end of file diff --git a/e2e-tests/src/test/resources/batch/unpublish/.eventsource.yaml b/e2e-tests/src/test/resources/batch/unpublish/.eventsource.yaml new file mode 100644 index 00000000..b6db9ffc --- /dev/null +++ b/e2e-tests/src/test/resources/batch/unpublish/.eventsource.yaml @@ -0,0 +1,8 @@ +eventType: com.streamx.blueprints.page.unpublished.v1 +eventSource: source + +relativePathLevel: 0 +key: ${relativePath} + +payload: + type: page/index-page diff --git a/e2e-tests/src/test/resources/batch/unpublish/index.html b/e2e-tests/src/test/resources/batch/unpublish/index.html new file mode 100644 index 00000000..ba7c290e --- /dev/null +++ b/e2e-tests/src/test/resources/batch/unpublish/index.html @@ -0,0 +1 @@ +

Hello World!

\ No newline at end of file diff --git a/e2e-tests/src/test/resources/configs/overridden-nginx-conf b/e2e-tests/src/test/resources/configs/overridden-nginx-conf deleted file mode 100644 index 329f89b7..00000000 --- a/e2e-tests/src/test/resources/configs/overridden-nginx-conf +++ /dev/null @@ -1,10 +0,0 @@ -server { - listen 80; - listen [::]:80; - server_name localhost; - - location /overridden { - alias /usr/share/nginx/html; - } -} - diff --git a/e2e-tests/src/test/resources/file_param_page.json b/e2e-tests/src/test/resources/file_param_page.json deleted file mode 100644 index 6cb37e37..00000000 --- a/e2e-tests/src/test/resources/file_param_page.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "content": { - "bytes": "file_param_page" - } -} \ No newline at end of file diff --git a/e2e-tests/src/test/resources/filtered/mesh.yaml b/e2e-tests/src/test/resources/filtered/mesh.yaml index 71e1ac93..8d741dd7 100644 --- a/e2e-tests/src/test/resources/filtered/mesh.yaml +++ b/e2e-tests/src/test/resources/filtered/mesh.yaml @@ -1,44 +1,61 @@ -defaultRegistry: ghcr.io/streamx-dev/streamx -defaultImageTag: ${streamx.version}-jvm +descriptors: + relay: + type: "processing" + containers: + service: + incoming: + events: {} + outgoing: + relayed-events: {} + + web-server-sink: + type: "edge" + volumes: + repository-volume: {} + containers: + sink: + image: "ghcr.io/streamx-com/streamx-blueprints/web-server-sink:3.0.7-native" + incoming: + resources: {} + servicePorts: + - "8087:8080" + volumes: + - "repository-volume:/srv/www" sources: - cli: + e2e: outgoing: - - "pages" + - ref: "inbox.pages" + produces: + - "com.streamx.blueprints.page.published.v1" + - "com.streamx.blueprints.page.unpublished.v1" -processing: - relay: - image: sample-relay-processing-service - incoming: - incoming-pages: - topic: inboxes/pages - outgoing: - outgoing-pages: - topic: outboxes/pages +ingestion: + rest-ingestion: + containers: + proxy: + environment: + QUARKUS_HTTP_AUTH_PERMISSION_BEARER_POLICY: permit -delivery: - web-delivery: - image: sample-web-delivery-service - incoming: - pages: - topic: outboxes/pages - port: 8087 - composite-web-delivery: - image: sample-web-delivery-service - incoming: - pages: - topic: outboxes/pages - port: 8088 - environment: - REPOSITORY_RESOURCE_ROOT_DIRECTORY: "/home/runner/work/streamx-cli/streamx-cli/srv/www" - repositoryVolume: "/home/runner/work/streamx-cli/streamx-cli/srv/www" - components: - webserver: - image: "docker.io/library/nginx:1.26.0" - ports: - - ${streamx.cli.e2e.port}:80 - repositoryVolume: "/usr/share/nginx/html" - volumesFrom: - configs: - - "overridden-nginx-conf:/etc/nginx/conf.d/default.conf" +processing: + pages-relay: + descriptor: relay + containers: + service: + incoming: + events: + ref: inbox.pages + outgoing: + relayed-events: + ref: outbox.pages +edge: + web-server-sink: + descriptor: web-server-sink + containers: + sink: + incoming: + resources: + ref: outbox.pages + environment: + STREAMX_BLUEPRINTS_WEB_SERVER_SINK_STORAGE_ROOT_DIRECTORY: "/srv/www" \ No newline at end of file diff --git a/e2e-tests/src/test/resources/json_path_file_param_page.txt b/e2e-tests/src/test/resources/json_path_file_param_page.txt deleted file mode 100644 index bb24ea82..00000000 --- a/e2e-tests/src/test/resources/json_path_file_param_page.txt +++ /dev/null @@ -1 +0,0 @@ -json_path_file_param_page \ No newline at end of file diff --git a/e2e-tests/src/test/resources/payload.json b/e2e-tests/src/test/resources/payload.json deleted file mode 100644 index d685f897..00000000 --- a/e2e-tests/src/test/resources/payload.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "content": { - "bytes": "third_param_page" - } -} \ No newline at end of file diff --git a/e2e-tests/src/test/resources/stream/page-publish-event.json b/e2e-tests/src/test/resources/stream/page-publish-event.json new file mode 100644 index 00000000..044b9a96 --- /dev/null +++ b/e2e-tests/src/test/resources/stream/page-publish-event.json @@ -0,0 +1,15 @@ +{ + "specversion" : "1.0", + "id" : "11111111-327e-4bee-96a1-3e4224a1e71d", + "source" : "source", + "type" : "com.streamx.blueprints.page.published.v1", + "datacontenttype" : "application/json", + "subject" : "hello.html", + "time" : "2025-12-23T10:28:23.435253Z", + "data" : { + "content" : { + "bytes" : "Hello World!" + }, + "type" : "page/test" + } +} \ No newline at end of file diff --git a/e2e-tests/src/test/resources/stream/page-unpublish-event.json b/e2e-tests/src/test/resources/stream/page-unpublish-event.json new file mode 100644 index 00000000..ec767e45 --- /dev/null +++ b/e2e-tests/src/test/resources/stream/page-unpublish-event.json @@ -0,0 +1,8 @@ +{ + "specversion" : "1.0", + "id" : "11111111-327e-4bee-96a1-3e4224a1e71d", + "source" : "source", + "type" : "com.streamx.blueprints.page.unpublished.v1", + "subject" : "hello.html", + "time" : "2025-12-23T10:28:23.435253Z" +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index e0ec4b0a..437e1dce 100644 --- a/pom.xml +++ b/pom.xml @@ -41,9 +41,7 @@ io.quarkus.platform 3.18.3 - 1.0.16 - 1.1.1 - 0.0.16 + 2.0.6 2.9.0 3.27.3 3.12.1 @@ -76,20 +74,14 @@ - dev.streamx + com.streamx streamx-runner ${streamx.version} - dev.streamx + com.streamx ingestion-client - ${ingestion-client.version} - - - - dev.streamx - streamx-operator-mesh-api - ${streamx-operator.version} + ${streamx.version} com.jayway.jsonpath @@ -183,6 +175,27 @@ + + org.apache.maven.plugins + maven-surefire-plugin + 3.2.5 + + + -javaagent:${settings.localRepository}/org/mockito/mockito-core/5.15.2/mockito-core-5.15.2.jar + + + + maven-failsafe-plugin + 3.5.4 + + + + integration-test + verify + + + + org.apache.maven.plugins maven-checkstyle-plugin From eb63811e5e9ccb167b57b66cc8a7a544585d4fe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Mon, 29 Dec 2025 14:43:27 +0100 Subject: [PATCH 02/36] mockito --- core/pom.xml | 5 +++++ pom.xml | 9 --------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index e8a45dc3..ee996e94 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -85,6 +85,11 @@ quarkus-junit5 test + + io.quarkus + quarkus-junit5-mockito + test + io.quarkus quarkus-junit5-component diff --git a/pom.xml b/pom.xml index 437e1dce..9d5c3185 100644 --- a/pom.xml +++ b/pom.xml @@ -175,15 +175,6 @@ - - org.apache.maven.plugins - maven-surefire-plugin - 3.2.5 - - - -javaagent:${settings.localRepository}/org/mockito/mockito-core/5.15.2/mockito-core-5.15.2.jar - - maven-failsafe-plugin 3.5.4 From 288ccf845926132cec77016cfcfc03d45c3bc7e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Mon, 29 Dec 2025 15:04:38 +0100 Subject: [PATCH 03/36] java 21 --- .github/workflows/ci-test-build.yaml | 8 ++++---- .../mvn-build-and-check-streamx-cli-commands-linux.yaml | 2 +- .github/workflows/release-preview.yaml | 4 ++-- .github/workflows/release.yaml | 4 ++-- .github/workflows/streamx-homebrew-instalation-check.yaml | 4 ++-- .../workflows/streamx-scoop-instalation-check.yaml.yaml | 4 ++-- core/pom.xml | 2 +- distribution/pom.xml | 4 ++-- distribution/src/main/resources/jreleaser-preview.yml | 2 +- distribution/src/main/resources/jreleaser.yml | 2 +- e2e-tests/pom.xml | 4 ++-- .../src/main/java/dev/streamx/cli/EntrypointMain.java | 4 ++-- .../src/test/java/dev/streamx/cli/EntrypointMainTest.java | 2 +- 13 files changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci-test-build.yaml b/.github/workflows/ci-test-build.yaml index 70f3f7d8..d86506f5 100644 --- a/.github/workflows/ci-test-build.yaml +++ b/.github/workflows/ci-test-build.yaml @@ -26,10 +26,10 @@ jobs: workload_identity_provider: ${{ secrets.GCP_STREAMX_RELEASES_WORKLOAD_IDENTITY_PROVIDER }} service_account: ${{ secrets.GCP_STREAMX_RELEASES_READ_SA }} - - name: Set up JDK 17 + - name: Set up JDK 21 uses: actions/setup-java@v3 with: - java-version: '17' + java-version: '21' distribution: 'adopt' cache: 'maven' @@ -51,10 +51,10 @@ jobs: workload_identity_provider: ${{ secrets.GCP_STREAMX_RELEASES_WORKLOAD_IDENTITY_PROVIDER }} service_account: ${{ secrets.GCP_STREAMX_RELEASES_READ_SA }} - - name: Set up JDK 17 + - name: Set up JDK 21 uses: actions/setup-java@v3 with: - java-version: '17' + java-version: '21' distribution: 'adopt' cache: 'maven' diff --git a/.github/workflows/mvn-build-and-check-streamx-cli-commands-linux.yaml b/.github/workflows/mvn-build-and-check-streamx-cli-commands-linux.yaml index 8bbd4b51..2601f61f 100644 --- a/.github/workflows/mvn-build-and-check-streamx-cli-commands-linux.yaml +++ b/.github/workflows/mvn-build-and-check-streamx-cli-commands-linux.yaml @@ -35,7 +35,7 @@ jobs: - name: Set up JDK 17s uses: actions/setup-java@v4 with: - java-version: '17' + java-version: '21' distribution: 'adopt' - name: Set up homebrew diff --git a/.github/workflows/release-preview.yaml b/.github/workflows/release-preview.yaml index a70d7d89..83f4b719 100644 --- a/.github/workflows/release-preview.yaml +++ b/.github/workflows/release-preview.yaml @@ -35,10 +35,10 @@ jobs: printf '%s\n' "${{ secrets.SSH_SECRET_KEY }}" > ~/.ssh/id_rsa chmod 0600 ~/.ssh/id_rsa - - name: Set up JDK 17 + - name: Set up JDK 21 uses: actions/setup-java@v3 with: - java-version: '17' + java-version: '21' distribution: 'adopt' - name: Preview release prepare diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a584a3b8..29a3ac3d 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -31,10 +31,10 @@ jobs: printf '%s\n' "${{ secrets.SSH_SECRET_KEY }}" > ~/.ssh/id_rsa chmod 0600 ~/.ssh/id_rsa - - name: Set up JDK 17 + - name: Set up JDK 21 uses: actions/setup-java@v3 with: - java-version: '17' + java-version: '21' distribution: 'adopt' - name: Release prepare diff --git a/.github/workflows/streamx-homebrew-instalation-check.yaml b/.github/workflows/streamx-homebrew-instalation-check.yaml index 66f1bfe2..834629b0 100644 --- a/.github/workflows/streamx-homebrew-instalation-check.yaml +++ b/.github/workflows/streamx-homebrew-instalation-check.yaml @@ -14,10 +14,10 @@ jobs: runs-on: ${{ matrix.os }} steps: - - name: Set up JDK 17 + - name: Set up JDK 21 uses: actions/setup-java@v4 with: - java-version: '17' + java-version: '21' distribution: 'adopt' - name: Set up homebrew diff --git a/.github/workflows/streamx-scoop-instalation-check.yaml.yaml b/.github/workflows/streamx-scoop-instalation-check.yaml.yaml index ffeedcf0..83ac6c0b 100644 --- a/.github/workflows/streamx-scoop-instalation-check.yaml.yaml +++ b/.github/workflows/streamx-scoop-instalation-check.yaml.yaml @@ -9,10 +9,10 @@ jobs: runs-on: windows-latest steps: - - name: Set up JDK 17 + - name: Set up JDK 21 uses: actions/setup-java@v4 with: - java-version: '17' + java-version: '21' distribution: 'adopt' - name: Set up Scoop diff --git a/core/pom.xml b/core/pom.xml index ee996e94..d3c4a941 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -12,7 +12,7 @@ StreamX Cli : Core - 17 + 21 true uber-jar diff --git a/distribution/pom.xml b/distribution/pom.xml index f88258bb..b86d5770 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -12,8 +12,8 @@ StreamX Cli : Distribution - 17 - 17 + 21 + 21 UTF-8 1.17.0 diff --git a/distribution/src/main/resources/jreleaser-preview.yml b/distribution/src/main/resources/jreleaser-preview.yml index e2becadb..b815a80c 100644 --- a/distribution/src/main/resources/jreleaser-preview.yml +++ b/distribution/src/main/resources/jreleaser-preview.yml @@ -15,7 +15,7 @@ project: java: groupId: dev.streamx.cli artifactId: streamx-cli - version: 17 + version: 21 inceptionYear: 2024 tags: diff --git a/distribution/src/main/resources/jreleaser.yml b/distribution/src/main/resources/jreleaser.yml index 6fd51cba..7e6192bd 100644 --- a/distribution/src/main/resources/jreleaser.yml +++ b/distribution/src/main/resources/jreleaser.yml @@ -13,7 +13,7 @@ project: java: groupId: dev.streamx.cli artifactId: streamx-cli - version: 17 + version: 21 inceptionYear: 2024 tags: diff --git a/e2e-tests/pom.xml b/e2e-tests/pom.xml index d651b8cf..86e2c4cd 100644 --- a/e2e-tests/pom.xml +++ b/e2e-tests/pom.xml @@ -13,8 +13,8 @@ StreamX Cli : E2e Tests - 17 - 17 + 21 + 21 UTF-8 diff --git a/entrypoint/src/main/java/dev/streamx/cli/EntrypointMain.java b/entrypoint/src/main/java/dev/streamx/cli/EntrypointMain.java index 81edfde6..cc3f86f1 100644 --- a/entrypoint/src/main/java/dev/streamx/cli/EntrypointMain.java +++ b/entrypoint/src/main/java/dev/streamx/cli/EntrypointMain.java @@ -14,8 +14,8 @@ public class EntrypointMain { public static void main(String[] args) { int javaVersion = getJavaVersion(); - if (javaVersion < 17) { - System.out.println("Java 17 or higher is required!"); + if (javaVersion < 21) { + System.out.println("Java 21 or higher is required!"); return; } diff --git a/entrypoint/src/test/java/dev/streamx/cli/EntrypointMainTest.java b/entrypoint/src/test/java/dev/streamx/cli/EntrypointMainTest.java index a790c2e5..e6daad54 100644 --- a/entrypoint/src/test/java/dev/streamx/cli/EntrypointMainTest.java +++ b/entrypoint/src/test/java/dev/streamx/cli/EntrypointMainTest.java @@ -53,7 +53,7 @@ void shouldFailTooLowJavaVersions(String javaVersion) { // then Assertions.assertFalse(StreamxCommand.isLaunched()); Assertions.assertTrue( - byteArrayOutputStream.toString().contains("Java 17 or higher is required!") + byteArrayOutputStream.toString().contains("Java 21 or higher is required!") ); } From e8ac81a56728cbb699318df943530ec9d39fc62f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Mon, 29 Dec 2025 15:09:56 +0100 Subject: [PATCH 04/36] add missing validation --- .../ingestion/batch/walker/EventSourceFileTreeWalker.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/main/java/dev/streamx/cli/command/ingestion/batch/walker/EventSourceFileTreeWalker.java b/core/src/main/java/dev/streamx/cli/command/ingestion/batch/walker/EventSourceFileTreeWalker.java index 26ab2afe..b4b8c33a 100644 --- a/core/src/main/java/dev/streamx/cli/command/ingestion/batch/walker/EventSourceFileTreeWalker.java +++ b/core/src/main/java/dev/streamx/cli/command/ingestion/batch/walker/EventSourceFileTreeWalker.java @@ -54,6 +54,8 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th "Missing required 'key' property in config file '%s'.'".formatted(configFile)); Objects.requireNonNull(descriptor.getEventType(), "Missing required 'eventType' property in config file '%s'.'".formatted(configFile)); + Objects.requireNonNull(descriptor.getEventSource(), + "Missing required 'eventSource' property in config file '%s'.'".formatted(configFile)); Objects.requireNonNull(descriptor.getPayload(), "Missing required 'payload' property in config file '%s'.'".formatted(configFile)); descriptor.setSource(configFile); From 9761683590fd8460f22b53b4023ed0d722eb282e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Mon, 29 Dec 2025 15:49:43 +0100 Subject: [PATCH 05/36] Remove Resource class --- .../ingestion/BaseIngestionCommand.java | 26 ++++++++----------- .../command/ingestion/CloudEventBuilder.java | 5 ++-- .../ingestion/stream/StreamCommand.java | 2 -- .../license/LicenseProcessorEntrypoint.java | 6 ++++- .../java/dev/streamx/cli/model/Resource.java | 8 ------ .../cli/license/LicenseAcceptingTest.java | 14 +++++++--- .../cli/license/input/LicenseInputConfig.java | 17 ------------ 7 files changed, 28 insertions(+), 50 deletions(-) delete mode 100644 core/src/main/java/dev/streamx/cli/model/Resource.java delete mode 100644 core/src/test/java/dev/streamx/cli/license/input/LicenseInputConfig.java diff --git a/core/src/main/java/dev/streamx/cli/command/ingestion/BaseIngestionCommand.java b/core/src/main/java/dev/streamx/cli/command/ingestion/BaseIngestionCommand.java index a4bcb91a..03b06378 100644 --- a/core/src/main/java/dev/streamx/cli/command/ingestion/BaseIngestionCommand.java +++ b/core/src/main/java/dev/streamx/cli/command/ingestion/BaseIngestionCommand.java @@ -3,16 +3,17 @@ import static java.nio.charset.StandardCharsets.UTF_8; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.streamx.clients.ingestion.StreamxClient; import com.streamx.clients.ingestion.exceptions.StreamxClientException; import com.streamx.clients.ingestion.publisher.Publisher; import dev.streamx.cli.exception.IngestionClientException; -import dev.streamx.cli.model.Resource; import dev.streamx.cli.util.ExceptionUtils; import io.cloudevents.CloudEvent; import io.cloudevents.jackson.JsonCloudEventData; import jakarta.inject.Inject; -import java.nio.ByteBuffer; +import java.util.Base64; import javax.net.ssl.SSLHandshakeException; import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Model.CommandSpec; @@ -20,6 +21,8 @@ public abstract class BaseIngestionCommand implements Runnable { + protected static final ObjectMapper objectMapper = new ObjectMapper(); + @ArgGroup(exclusive = false) IngestionArguments ingestionArguments; @@ -49,21 +52,10 @@ public final void run() { protected static CloudEvent withAdjustedData(CloudEvent event) { JsonCloudEventData data = (JsonCloudEventData) event.getData(); - Resource resource = extractResource(data); - if (resource != null) { - // convert data to format expected by StreamX - return CloudEventBuilder.copyWithNewData(event, resource); - } - return event; - } - - private static Resource extractResource(JsonCloudEventData data) { if (data != null) { JsonNode dataNode = data.getNode(); if (dataNode != null) { JsonNode contentNode = dataNode.get("content"); - JsonNode typeNode = dataNode.get("type"); - String type = typeNode != null ? typeNode.asText() : null; if (contentNode != null) { JsonNode bytesNode = contentNode.get("bytes"); if (bytesNode != null) { @@ -72,12 +64,16 @@ private static Resource extractResource(JsonCloudEventData data) { // TODO: detect if the content.bytes field is already base 64 encoded // and if it is encoded - decode it first. // For now assume it's in clear text - return new Resource(ByteBuffer.wrap(content.getBytes(UTF_8)), type); + String encoded = Base64.getEncoder().encodeToString(content.getBytes(UTF_8)); + + ObjectNode dataNodeObj = (ObjectNode) dataNode; + dataNodeObj.put("content", encoded); + return CloudEventBuilder.copyWithNewData(event, dataNodeObj); } } } } } - return null; + return event; } } diff --git a/core/src/main/java/dev/streamx/cli/command/ingestion/CloudEventBuilder.java b/core/src/main/java/dev/streamx/cli/command/ingestion/CloudEventBuilder.java index b610f9e0..a08e009f 100644 --- a/core/src/main/java/dev/streamx/cli/command/ingestion/CloudEventBuilder.java +++ b/core/src/main/java/dev/streamx/cli/command/ingestion/CloudEventBuilder.java @@ -3,7 +3,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.NullNode; -import dev.streamx.cli.model.Resource; import io.cloudevents.CloudEvent; import io.cloudevents.jackson.JsonCloudEventData; import java.net.URI; @@ -42,10 +41,10 @@ public static CloudEvent build(String subject, String type, String source, JsonN .build(); } - public static CloudEvent copyWithNewData(CloudEvent cloudEvent, Resource data) { + public static CloudEvent copyWithNewData(CloudEvent cloudEvent, JsonNode data) { return io.cloudevents.core.builder.CloudEventBuilder.v1(cloudEvent) .withDataContentType("application/json") - .withData(JsonCloudEventData.wrap(objectMapper.valueToTree(data))) + .withData(JsonCloudEventData.wrap(data)) .build(); } diff --git a/core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamCommand.java b/core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamCommand.java index 8cd396a4..fda195eb 100644 --- a/core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamCommand.java +++ b/core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamCommand.java @@ -4,7 +4,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import com.streamx.ce.serialization.DeserializerException; import com.streamx.ce.serialization.json.CloudEventJsonDeserializer; import com.streamx.clients.ingestion.exceptions.StreamxClientException; @@ -33,7 +32,6 @@ public class StreamCommand extends BaseIngestionCommand { // TODO "_v2" is a temporary postfix for now public static final String COMMAND_NAME = "stream_v2"; - private static final ObjectMapper objectMapper = new ObjectMapper(); private static final CloudEventJsonDeserializer deserializer = new CloudEventJsonDeserializer(); @ArgGroup(exclusive = false, multiplicity = "1") diff --git a/core/src/main/java/dev/streamx/cli/license/LicenseProcessorEntrypoint.java b/core/src/main/java/dev/streamx/cli/license/LicenseProcessorEntrypoint.java index 7c0e25d5..6a0a6ab6 100644 --- a/core/src/main/java/dev/streamx/cli/license/LicenseProcessorEntrypoint.java +++ b/core/src/main/java/dev/streamx/cli/license/LicenseProcessorEntrypoint.java @@ -84,7 +84,7 @@ private void proceedLicenseAcceptance(LicenseSettings licenseSettings, LocalDate return; } - if (acceptingStrategy.isLicenseAccepted()) { + if (getAcceptingStrategy().isLicenseAccepted()) { licenseSettingsStore.acceptLicense(licenseSettings, now); } else { throw LicenseException.licenseAcceptanceRejectedException(); @@ -108,4 +108,8 @@ private static String getLicenseText(LastLicenseFetch licenseSettings) { licenseSettings.licenseUrl() ); } + + AcceptingStrategy getAcceptingStrategy() { + return acceptingStrategy; + } } diff --git a/core/src/main/java/dev/streamx/cli/model/Resource.java b/core/src/main/java/dev/streamx/cli/model/Resource.java deleted file mode 100644 index 0b3cea23..00000000 --- a/core/src/main/java/dev/streamx/cli/model/Resource.java +++ /dev/null @@ -1,8 +0,0 @@ - -package dev.streamx.cli.model; - -import java.nio.ByteBuffer; - -public record Resource(ByteBuffer content, String type) { - -} diff --git a/core/src/test/java/dev/streamx/cli/license/LicenseAcceptingTest.java b/core/src/test/java/dev/streamx/cli/license/LicenseAcceptingTest.java index 54abb157..1a7e703e 100644 --- a/core/src/test/java/dev/streamx/cli/license/LicenseAcceptingTest.java +++ b/core/src/test/java/dev/streamx/cli/license/LicenseAcceptingTest.java @@ -5,6 +5,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; import dev.streamx.cli.exception.LicenseException; import dev.streamx.cli.license.LicenseTestProfiles.AcceptProceedingTestProfile; @@ -16,6 +17,7 @@ import dev.streamx.cli.settings.SettingsStore; import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.TestProfile; +import io.quarkus.test.junit.mockito.InjectSpy; import jakarta.inject.Inject; import java.io.File; import java.time.LocalDateTime; @@ -25,6 +27,7 @@ import org.assertj.core.api.Assertions; import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -35,16 +38,19 @@ class LicenseAcceptingTest { public static final String OLD_URL = "http://old.streamx.dev/license.html"; public static final String OLD_NAME = "oldLicense"; - LicenseArguments licenseArguments = new LicenseArguments(); + private final LicenseArguments licenseArguments = new LicenseArguments(); + private final AcceptingStrategy acceptingStrategy = mock(AcceptingStrategy.class); @Inject SettingsStore settingsStore; - @Inject + @InjectSpy LicenseProcessorEntrypoint entrypoint; - @Inject - AcceptingStrategy acceptingStrategy; + @BeforeEach + void setup() { + doReturn(acceptingStrategy).when(entrypoint).getAcceptingStrategy(); + } @AfterEach void shutdown() { diff --git a/core/src/test/java/dev/streamx/cli/license/input/LicenseInputConfig.java b/core/src/test/java/dev/streamx/cli/license/input/LicenseInputConfig.java deleted file mode 100644 index c162a58e..00000000 --- a/core/src/test/java/dev/streamx/cli/license/input/LicenseInputConfig.java +++ /dev/null @@ -1,17 +0,0 @@ -package dev.streamx.cli.license.input; - -import io.quarkus.arc.profile.IfBuildProfile; -import jakarta.enterprise.context.Dependent; -import jakarta.inject.Singleton; -import org.mockito.Mockito; - -@Dependent -public class LicenseInputConfig { - - @Singleton - @IfBuildProfile("test") - AcceptingStrategy mockedAcceptingStrategy() { - return Mockito.mock(AcceptingStrategy.class); - } - -} From 46a6874fd4831b269714ded20ea39b99232eee8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Mon, 29 Dec 2025 15:56:57 +0100 Subject: [PATCH 06/36] Restore banner --- core/src/main/java/dev/streamx/cli/StreamxCommand.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/src/main/java/dev/streamx/cli/StreamxCommand.java b/core/src/main/java/dev/streamx/cli/StreamxCommand.java index 75564240..d196fb73 100644 --- a/core/src/main/java/dev/streamx/cli/StreamxCommand.java +++ b/core/src/main/java/dev/streamx/cli/StreamxCommand.java @@ -46,10 +46,14 @@ public class StreamxCommand implements QuarkusApplication { @Inject ConfigSourcesValidator configSourcesValidator; + @Inject + BannerPrinter bannerPrinter; + @ArgGroup(exclusive = false) LicenseArguments licenseArguments; private CommandLine commandLine; + private String[] args; public static void main(String... args) { initializeArgumentConfigSource(args); @@ -70,6 +74,8 @@ private static void initializeArgumentConfigSource(String[] args) { @Override public int run(String... args) { + this.args = args; + commandLine = new CommandLine(this, factory) .setParameterExceptionHandler(parameterExceptionHandler) .setExecutionExceptionHandler(executionExceptionHandler) @@ -107,6 +113,8 @@ private int executionStrategy(ParseResult parseResult) { } private void init() { + bannerPrinter.initialize(commandLine, args); + licenseProcessorEntrypoint.process(); } } From 74f6c6d2d22db11f215a441afc40ae6e69e67e45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Mon, 29 Dec 2025 16:25:34 +0100 Subject: [PATCH 07/36] Remove unnecessary plugin --- pom.xml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/pom.xml b/pom.xml index 9d5c3185..c6dbc29e 100644 --- a/pom.xml +++ b/pom.xml @@ -175,18 +175,6 @@ - - maven-failsafe-plugin - 3.5.4 - - - - integration-test - verify - - - - org.apache.maven.plugins maven-checkstyle-plugin From cad43e560c40e15d7e7757d8e8cd6a9c7ba283f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Mon, 29 Dec 2025 16:57:53 +0100 Subject: [PATCH 08/36] Use cli source --- core/src/test/resources/filtered/mesh.yaml | 2 +- e2e-tests/src/test/resources/filtered/mesh.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/test/resources/filtered/mesh.yaml b/core/src/test/resources/filtered/mesh.yaml index 8d741dd7..ce6c786c 100644 --- a/core/src/test/resources/filtered/mesh.yaml +++ b/core/src/test/resources/filtered/mesh.yaml @@ -23,7 +23,7 @@ descriptors: - "repository-volume:/srv/www" sources: - e2e: + cli: outgoing: - ref: "inbox.pages" produces: diff --git a/e2e-tests/src/test/resources/filtered/mesh.yaml b/e2e-tests/src/test/resources/filtered/mesh.yaml index 8d741dd7..ce6c786c 100644 --- a/e2e-tests/src/test/resources/filtered/mesh.yaml +++ b/e2e-tests/src/test/resources/filtered/mesh.yaml @@ -23,7 +23,7 @@ descriptors: - "repository-volume:/srv/www" sources: - e2e: + cli: outgoing: - ref: "inbox.pages" produces: From c8ca778ebd091cf519e0614db42c0b7dcfc62d39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Mon, 29 Dec 2025 16:58:16 +0100 Subject: [PATCH 09/36] Increase test coverage --- .../batch/walker/EventSourceFileTreeWalker.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/dev/streamx/cli/command/ingestion/batch/walker/EventSourceFileTreeWalker.java b/core/src/main/java/dev/streamx/cli/command/ingestion/batch/walker/EventSourceFileTreeWalker.java index b4b8c33a..08f4d2d7 100644 --- a/core/src/main/java/dev/streamx/cli/command/ingestion/batch/walker/EventSourceFileTreeWalker.java +++ b/core/src/main/java/dev/streamx/cli/command/ingestion/batch/walker/EventSourceFileTreeWalker.java @@ -5,7 +5,6 @@ import dev.streamx.cli.command.ingestion.batch.EventSourceDescriptor; import dev.streamx.cli.command.ingestion.batch.exception.EventSourceDescriptorException; import dev.streamx.cli.util.FileUtils; -import io.quarkus.logging.Log; import java.io.IOException; import java.nio.file.DirectoryStream; import java.nio.file.FileVisitResult; @@ -16,10 +15,12 @@ import java.util.List; import java.util.Objects; import java.util.Stack; +import org.jboss.logging.Logger; import org.jetbrains.annotations.NotNull; public class EventSourceFileTreeWalker extends SimpleFileVisitor { + private final Logger logger = Logger.getLogger(EventSourceFileTreeWalker.class); private final Stack eventSourceStack = new Stack<>(); private final ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory()); @@ -36,7 +37,7 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th EventSourceDescriptor descriptor = parseDescriptor(configFile); eventSourceStack.push(descriptor); - Log.debugf("Found event source in %s: %s", dir, descriptor); + logger.debugf("Found event source in %s: %s", dir, descriptor); } // Process all files in this directory with the current active event source (if any) @@ -70,7 +71,7 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) { Path configFile = dir.resolve(EventSourceDescriptor.FILENAME); if (Files.exists(configFile) && Files.isRegularFile(configFile)) { EventSourceDescriptor popped = eventSourceStack.pop(); - Log.debugf("Leaving %s. Reverting event source: %s", dir, popped); + logger.debugf("Leaving %s. Reverting event source: %s", dir, popped); } return FileVisitResult.CONTINUE; } @@ -83,9 +84,9 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) { */ private void processDirectory(Path dir, EventSourceDescriptor currentDescriptor) throws IOException { - Log.debugf("Processing directory: %s", dir); + logger.debugf("Processing directory: %s", dir); if (currentDescriptor != null) { - Log.debugf("Active event source: %s", currentDescriptor); + logger.debugf("Active event source: %s", currentDescriptor); // Iterate through all entries in the directory. try (DirectoryStream stream = Files.newDirectoryStream(dir)) { @@ -100,7 +101,7 @@ && noIgnorePatternMatches(entry, currentDescriptor)) { } } else { - Log.debugf("No active event source."); + logger.debugf("No active event source."); } } From 9776c77ff270450581970314b7315cd371261d8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Mon, 29 Dec 2025 17:04:37 +0100 Subject: [PATCH 10/36] Use jvm --- core/src/test/resources/filtered/mesh.yaml | 2 +- e2e-tests/src/test/resources/filtered/mesh.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/test/resources/filtered/mesh.yaml b/core/src/test/resources/filtered/mesh.yaml index ce6c786c..e06315cd 100644 --- a/core/src/test/resources/filtered/mesh.yaml +++ b/core/src/test/resources/filtered/mesh.yaml @@ -14,7 +14,7 @@ descriptors: repository-volume: {} containers: sink: - image: "ghcr.io/streamx-com/streamx-blueprints/web-server-sink:3.0.7-native" + image: "ghcr.io/streamx-com/streamx-blueprints/web-server-sink:3.0.7-jvm" incoming: resources: {} servicePorts: diff --git a/e2e-tests/src/test/resources/filtered/mesh.yaml b/e2e-tests/src/test/resources/filtered/mesh.yaml index ce6c786c..e06315cd 100644 --- a/e2e-tests/src/test/resources/filtered/mesh.yaml +++ b/e2e-tests/src/test/resources/filtered/mesh.yaml @@ -14,7 +14,7 @@ descriptors: repository-volume: {} containers: sink: - image: "ghcr.io/streamx-com/streamx-blueprints/web-server-sink:3.0.7-native" + image: "ghcr.io/streamx-com/streamx-blueprints/web-server-sink:3.0.7-jvm" incoming: resources: {} servicePorts: From dca7a186fc978a2fac7c239bbb95369b895858ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Tue, 30 Dec 2025 09:06:31 +0100 Subject: [PATCH 11/36] Use temurin --- .github/workflows/ci-test-build.yaml | 4 ++-- .../mvn-build-and-check-streamx-cli-commands-linux.yaml | 2 +- .github/workflows/release-preview.yaml | 2 +- .github/workflows/release.yaml | 2 +- .github/workflows/streamx-homebrew-instalation-check.yaml | 2 +- .github/workflows/streamx-scoop-instalation-check.yaml.yaml | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-test-build.yaml b/.github/workflows/ci-test-build.yaml index d86506f5..f36e4654 100644 --- a/.github/workflows/ci-test-build.yaml +++ b/.github/workflows/ci-test-build.yaml @@ -30,7 +30,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: '21' - distribution: 'adopt' + distribution: 'temurin' cache: 'maven' - name: Build project @@ -55,7 +55,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: '21' - distribution: 'adopt' + distribution: 'temurin' cache: 'maven' - name: Build project diff --git a/.github/workflows/mvn-build-and-check-streamx-cli-commands-linux.yaml b/.github/workflows/mvn-build-and-check-streamx-cli-commands-linux.yaml index 2601f61f..ab61944f 100644 --- a/.github/workflows/mvn-build-and-check-streamx-cli-commands-linux.yaml +++ b/.github/workflows/mvn-build-and-check-streamx-cli-commands-linux.yaml @@ -36,7 +36,7 @@ jobs: uses: actions/setup-java@v4 with: java-version: '21' - distribution: 'adopt' + distribution: 'temurin' - name: Set up homebrew if: inputs.streamx_source == 'installed' diff --git a/.github/workflows/release-preview.yaml b/.github/workflows/release-preview.yaml index 83f4b719..a383f8c6 100644 --- a/.github/workflows/release-preview.yaml +++ b/.github/workflows/release-preview.yaml @@ -39,7 +39,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: '21' - distribution: 'adopt' + distribution: 'temurin' - name: Preview release prepare run: ./mvnw -B clean install -DskipTests -P preview diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 29a3ac3d..b1c23342 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -35,7 +35,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: '21' - distribution: 'adopt' + distribution: 'temurin' - name: Release prepare id: prepare-release diff --git a/.github/workflows/streamx-homebrew-instalation-check.yaml b/.github/workflows/streamx-homebrew-instalation-check.yaml index 834629b0..376e6076 100644 --- a/.github/workflows/streamx-homebrew-instalation-check.yaml +++ b/.github/workflows/streamx-homebrew-instalation-check.yaml @@ -18,7 +18,7 @@ jobs: uses: actions/setup-java@v4 with: java-version: '21' - distribution: 'adopt' + distribution: 'temurin' - name: Set up homebrew uses: 'Homebrew/actions/setup-homebrew@master' diff --git a/.github/workflows/streamx-scoop-instalation-check.yaml.yaml b/.github/workflows/streamx-scoop-instalation-check.yaml.yaml index 83ac6c0b..e0f0a013 100644 --- a/.github/workflows/streamx-scoop-instalation-check.yaml.yaml +++ b/.github/workflows/streamx-scoop-instalation-check.yaml.yaml @@ -13,7 +13,7 @@ jobs: uses: actions/setup-java@v4 with: java-version: '21' - distribution: 'adopt' + distribution: 'temurin' - name: Set up Scoop uses: MinoruSekine/setup-scoop@v4 From 18fdda47c62b71bd33dc111e594e3e567f396fb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Tue, 30 Dec 2025 10:13:28 +0100 Subject: [PATCH 12/36] Move test file, add test for default values --- ...shDefinitionResolverInterpolationTest.java | 60 +++++++++++++++---- .../{filtered => }/mesh-interpolated.yaml | 6 +- 2 files changed, 52 insertions(+), 14 deletions(-) rename core/src/test/resources/{filtered => }/mesh-interpolated.yaml (93%) diff --git a/core/src/test/java/dev/streamx/cli/command/meshprocessing/MeshDefinitionResolverInterpolationTest.java b/core/src/test/java/dev/streamx/cli/command/meshprocessing/MeshDefinitionResolverInterpolationTest.java index 7c73106e..a085bbfd 100644 --- a/core/src/test/java/dev/streamx/cli/command/meshprocessing/MeshDefinitionResolverInterpolationTest.java +++ b/core/src/test/java/dev/streamx/cli/command/meshprocessing/MeshDefinitionResolverInterpolationTest.java @@ -1,40 +1,78 @@ package dev.streamx.cli.command.meshprocessing; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertThrowsExactly; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.fasterxml.jackson.databind.JsonMappingException; +import com.streamx.mesh.model.ServiceMesh; import io.quarkus.test.junit.QuarkusTest; import jakarta.inject.Inject; import java.io.IOException; import java.nio.file.Path; import java.util.NoSuchElementException; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @QuarkusTest class MeshDefinitionResolverInterpolationTest { - private static final String TEST_MESH_LOCATION = "target/test-classes/mesh-interpolated.yaml"; + private static final String TEST_MESH_LOCATION = "src/test/resources/mesh-interpolated.yaml"; private static final Path TEST_MESH_PATH = Path.of(TEST_MESH_LOCATION); @Inject MeshDefinitionResolver uut; - @Test - void shouldFailWithPropertyUndefined() { + @BeforeEach + void clearSystemProperties() { System.clearProperty("config.image.interpolated"); + System.clearProperty("config.source.interpolated"); + } + + @Test + void shouldFailWithMandatoryPropertyUndefined() { + assertThatThrownBy(() -> uut.resolve(TEST_MESH_PATH)) + .isInstanceOf(JsonMappingException.class) + .hasRootCauseInstanceOf(NoSuchElementException.class) + .hasRootCauseMessage("Could not expand value config.image.interpolated" + + " in expression ${config.image.interpolated}"); + } - JsonMappingException ex = assertThrowsExactly(JsonMappingException.class, - () -> uut.resolve(TEST_MESH_PATH)); - assertThat(ex).hasRootCauseExactlyInstanceOf(NoSuchElementException.class); + @Test + void shouldResolveWithMandatoryPropertyDefinedAndOptionalPropertyUndefined() throws IOException { + System.setProperty("config.image.interpolated", "image-1"); + + ServiceMesh result = uut.resolve(TEST_MESH_PATH); + assertSinkImage(result, "image-1"); + assertSourceRef(result, "inbox.pages"); } @Test - void shouldResolveWithPropertyDefined() throws IOException { - System.setProperty("config.image.interpolated", "value"); + void shouldResolveWithMandatoryAndOptionalPropertiesDefined() throws IOException { + System.setProperty("config.image.interpolated", "image-1"); + System.setProperty("config.source.interpolated", "source-1"); - var result = uut.resolve(TEST_MESH_PATH); + ServiceMesh result = uut.resolve(TEST_MESH_PATH); + assertSinkImage(result, "image-1"); + assertSourceRef(result, "source-1"); + } + + private static void assertSinkImage(ServiceMesh result, String expected) { + String actual = result + .getDescriptors() + .get("web-server-sink") + .getContainers() + .get("sink") + .getImage(); + assertThat(actual).isEqualTo(expected); + } - assertThat(result).isNotNull(); + private static void assertSourceRef(ServiceMesh result, String expected) { + String actual = result + .getSources() + .get("cli") + .getOutgoing() + .getFirst() + .getRef(); + assertThat(actual).isEqualTo(expected); } } diff --git a/core/src/test/resources/filtered/mesh-interpolated.yaml b/core/src/test/resources/mesh-interpolated.yaml similarity index 93% rename from core/src/test/resources/filtered/mesh-interpolated.yaml rename to core/src/test/resources/mesh-interpolated.yaml index c56edd5b..d8481695 100644 --- a/core/src/test/resources/filtered/mesh-interpolated.yaml +++ b/core/src/test/resources/mesh-interpolated.yaml @@ -16,16 +16,16 @@ descriptors: sink: image: "${config.image.interpolated}" incoming: - resources: {} + resources: servicePorts: - "8087:8080" volumes: - "repository-volume:/srv/www" sources: - e2e: + cli: outgoing: - - ref: "inbox.pages" + - ref: "${config.source.interpolated:inbox.pages}" produces: - "com.streamx.blueprints.page.published.v1" - "com.streamx.blueprints.page.unpublished.v1" From f08a25d4c58899e93199610e51518730a762e8e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Tue, 30 Dec 2025 10:14:15 +0100 Subject: [PATCH 13/36] Better newline diff comparison --- .../cli/command/ingestion/BaseIngestionCommandTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/test/java/dev/streamx/cli/command/ingestion/BaseIngestionCommandTest.java b/core/src/test/java/dev/streamx/cli/command/ingestion/BaseIngestionCommandTest.java index e704c15a..ed54075d 100644 --- a/core/src/test/java/dev/streamx/cli/command/ingestion/BaseIngestionCommandTest.java +++ b/core/src/test/java/dev/streamx/cli/command/ingestion/BaseIngestionCommandTest.java @@ -48,7 +48,8 @@ protected static void expectSuccess(LaunchResult result) { protected static void expectError(LaunchResult result, String expectedErrorOutput) { assertThat(result.exitCode()).isNotZero(); - assertThat(result.getErrorOutput().replace("\r\n", "\n")).isEqualTo(expectedErrorOutput); + assertThat(result.getErrorOutput().trim()) + .isEqualToNormalizingNewlines(expectedErrorOutput.trim()); } protected static void setupMockPublicationResponse(CloudEvent response) { From 16501c8ef831d1e77c392e135b52d958a6f6db6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Tue, 30 Dec 2025 10:15:15 +0100 Subject: [PATCH 14/36] Use native --- .github/workflows/ci-test-build.yaml | 3 ++- core/src/test/resources/filtered/mesh.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-test-build.yaml b/.github/workflows/ci-test-build.yaml index f36e4654..d61692ef 100644 --- a/.github/workflows/ci-test-build.yaml +++ b/.github/workflows/ci-test-build.yaml @@ -35,7 +35,8 @@ jobs: - name: Build project run: | - ./mvnw clean verify -P all-tests + ./mvnw clean verify -P all-tests -Dnative + windows-test-build: runs-on: windows-latest steps: diff --git a/core/src/test/resources/filtered/mesh.yaml b/core/src/test/resources/filtered/mesh.yaml index e06315cd..ce6c786c 100644 --- a/core/src/test/resources/filtered/mesh.yaml +++ b/core/src/test/resources/filtered/mesh.yaml @@ -14,7 +14,7 @@ descriptors: repository-volume: {} containers: sink: - image: "ghcr.io/streamx-com/streamx-blueprints/web-server-sink:3.0.7-jvm" + image: "ghcr.io/streamx-com/streamx-blueprints/web-server-sink:3.0.7-native" incoming: resources: {} servicePorts: From dd8a24bc6e23d0940252b8d801374906808f5568 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Tue, 30 Dec 2025 11:52:06 +0100 Subject: [PATCH 15/36] restore {} --- core/src/test/resources/mesh-interpolated.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/resources/mesh-interpolated.yaml b/core/src/test/resources/mesh-interpolated.yaml index d8481695..9dd776a4 100644 --- a/core/src/test/resources/mesh-interpolated.yaml +++ b/core/src/test/resources/mesh-interpolated.yaml @@ -16,7 +16,7 @@ descriptors: sink: image: "${config.image.interpolated}" incoming: - resources: + resources: {} servicePorts: - "8087:8080" volumes: From 0b9fcd3d733359cca6286a4334210751c8087f1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Wed, 31 Dec 2025 10:34:15 +0100 Subject: [PATCH 16/36] Improve handling of ByteBuffer fields --- .../ingestion/BaseIngestionCommand.java | 37 ----------- .../command/ingestion/CloudEventBuilder.java | 7 --- .../command/ingestion/batch/BatchCommand.java | 5 +- .../resolver/step/BinaryResolverStep.java | 39 ++++++------ .../ingestion/stream/StreamCommand.java | 10 ++- .../ingestion/stream/StreamProperties.java | 44 +++++++++++++ .../stream/parser/JsonBase64Encoder.java | 43 +++++++++++++ .../cli/exception/PayloadException.java | 9 --- .../dev/streamx/cli/util/FileSourceUtils.java | 38 ------------ .../batch/BatchPublishCommandTest.java | 51 ++++++++++++--- .../batch/BatchUnpublishCommandTest.java | 8 +-- .../input/StdInLineReadStrategyTest.java | 3 +- .../valid/publish/image/.eventsource.yaml | 12 ++++ .../batch/valid/publish/image/ds.png | Bin 0 -> 794 bytes .../batch/valid/publish/{ => image}/ignore.me | 0 .../publish/{ => page}/.eventsource.yaml | 3 +- .../{unpublish => publish/page}/ignore.me | 0 .../batch/valid/publish/{ => page}/index.html | 0 .../unpublish/{ => page}/.eventsource.yaml | 0 .../batch/valid/unpublish/page/ignore.me | 1 + .../valid/unpublish/{ => page}/index.html | 0 .../ingestion/stream/.stream.properties | 1 + .../ingestion/stream/valid-json.stream | 4 +- e2e-tests/pom.xml | 5 ++ .../test/tools/validators/HttpValidator.java | 52 +++++++++------- .../validators/ProcessOutputValidator.java | 20 +++--- .../dev/streamx/cli/StreamxCliLicenseIT.java | 6 +- .../streamx/cli/StreamxCliPublicationIT.java | 58 ++++++++++++++---- .../batch/publish/asset/.eventsource.yaml | 9 +++ .../test/resources/batch/publish/asset/ds.png | Bin 0 -> 794 bytes .../publish/{ => page}/.eventsource.yaml | 4 +- .../batch/publish/{ => page}/index.html | 0 .../batch/unpublish/asset/.eventsource.yaml | 9 +++ .../resources/batch/unpublish/asset/ds.png | Bin 0 -> 794 bytes .../unpublish/{ => page}/.eventsource.yaml | 0 .../batch/unpublish/{ => page}/index.html | 0 .../src/test/resources/filtered/mesh.yaml | 17 ++++- .../test/resources/stream/.stream.properties | 1 + .../resources/stream/page-publish-event.json | 4 +- 39 files changed, 315 insertions(+), 185 deletions(-) create mode 100644 core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamProperties.java create mode 100644 core/src/main/java/dev/streamx/cli/command/ingestion/stream/parser/JsonBase64Encoder.java delete mode 100644 core/src/main/java/dev/streamx/cli/util/FileSourceUtils.java create mode 100644 core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/publish/image/.eventsource.yaml create mode 100644 core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/publish/image/ds.png rename core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/publish/{ => image}/ignore.me (100%) rename core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/publish/{ => page}/.eventsource.yaml (78%) rename core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/{unpublish => publish/page}/ignore.me (100%) rename core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/publish/{ => page}/index.html (100%) rename core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/unpublish/{ => page}/.eventsource.yaml (100%) create mode 100644 core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/unpublish/page/ignore.me rename core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/unpublish/{ => page}/index.html (100%) create mode 100644 core/src/test/resources/dev/streamx/cli/command/ingestion/stream/.stream.properties create mode 100644 e2e-tests/src/test/resources/batch/publish/asset/.eventsource.yaml create mode 100644 e2e-tests/src/test/resources/batch/publish/asset/ds.png rename e2e-tests/src/test/resources/batch/publish/{ => page}/.eventsource.yaml (76%) rename e2e-tests/src/test/resources/batch/publish/{ => page}/index.html (100%) create mode 100644 e2e-tests/src/test/resources/batch/unpublish/asset/.eventsource.yaml create mode 100644 e2e-tests/src/test/resources/batch/unpublish/asset/ds.png rename e2e-tests/src/test/resources/batch/unpublish/{ => page}/.eventsource.yaml (100%) rename e2e-tests/src/test/resources/batch/unpublish/{ => page}/index.html (100%) create mode 100644 e2e-tests/src/test/resources/stream/.stream.properties diff --git a/core/src/main/java/dev/streamx/cli/command/ingestion/BaseIngestionCommand.java b/core/src/main/java/dev/streamx/cli/command/ingestion/BaseIngestionCommand.java index 03b06378..24bdbe48 100644 --- a/core/src/main/java/dev/streamx/cli/command/ingestion/BaseIngestionCommand.java +++ b/core/src/main/java/dev/streamx/cli/command/ingestion/BaseIngestionCommand.java @@ -1,19 +1,11 @@ package dev.streamx.cli.command.ingestion; -import static java.nio.charset.StandardCharsets.UTF_8; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import com.streamx.clients.ingestion.StreamxClient; import com.streamx.clients.ingestion.exceptions.StreamxClientException; import com.streamx.clients.ingestion.publisher.Publisher; import dev.streamx.cli.exception.IngestionClientException; import dev.streamx.cli.util.ExceptionUtils; -import io.cloudevents.CloudEvent; -import io.cloudevents.jackson.JsonCloudEventData; import jakarta.inject.Inject; -import java.util.Base64; import javax.net.ssl.SSLHandshakeException; import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Model.CommandSpec; @@ -21,8 +13,6 @@ public abstract class BaseIngestionCommand implements Runnable { - protected static final ObjectMapper objectMapper = new ObjectMapper(); - @ArgGroup(exclusive = false) IngestionArguments ingestionArguments; @@ -49,31 +39,4 @@ public final void run() { throw ExceptionUtils.sneakyThrow(e); } } - - protected static CloudEvent withAdjustedData(CloudEvent event) { - JsonCloudEventData data = (JsonCloudEventData) event.getData(); - if (data != null) { - JsonNode dataNode = data.getNode(); - if (dataNode != null) { - JsonNode contentNode = dataNode.get("content"); - if (contentNode != null) { - JsonNode bytesNode = contentNode.get("bytes"); - if (bytesNode != null) { - String content = bytesNode.asText(); - if (content != null) { - // TODO: detect if the content.bytes field is already base 64 encoded - // and if it is encoded - decode it first. - // For now assume it's in clear text - String encoded = Base64.getEncoder().encodeToString(content.getBytes(UTF_8)); - - ObjectNode dataNodeObj = (ObjectNode) dataNode; - dataNodeObj.put("content", encoded); - return CloudEventBuilder.copyWithNewData(event, dataNodeObj); - } - } - } - } - } - return event; - } } diff --git a/core/src/main/java/dev/streamx/cli/command/ingestion/CloudEventBuilder.java b/core/src/main/java/dev/streamx/cli/command/ingestion/CloudEventBuilder.java index a08e009f..8b1ca539 100644 --- a/core/src/main/java/dev/streamx/cli/command/ingestion/CloudEventBuilder.java +++ b/core/src/main/java/dev/streamx/cli/command/ingestion/CloudEventBuilder.java @@ -41,11 +41,4 @@ public static CloudEvent build(String subject, String type, String source, JsonN .build(); } - public static CloudEvent copyWithNewData(CloudEvent cloudEvent, JsonNode data) { - return io.cloudevents.core.builder.CloudEventBuilder.v1(cloudEvent) - .withDataContentType("application/json") - .withData(JsonCloudEventData.wrap(data)) - .build(); - } - } diff --git a/core/src/main/java/dev/streamx/cli/command/ingestion/batch/BatchCommand.java b/core/src/main/java/dev/streamx/cli/command/ingestion/batch/BatchCommand.java index e66ec207..b4f94aac 100644 --- a/core/src/main/java/dev/streamx/cli/command/ingestion/batch/BatchCommand.java +++ b/core/src/main/java/dev/streamx/cli/command/ingestion/batch/BatchCommand.java @@ -100,10 +100,9 @@ private void send(State state, Publisher publisher) throws StreamxClientExceptio state.eventSubject, state.eventType, state.eventSource, - state.payload + state.payload // TODO rename payload to data everywhere ); - CloudEvent adjustedEvent = withAdjustedData(inputEvent); - CloudEvent responseEvent = publisher.send(adjustedEvent); + CloudEvent responseEvent = publisher.send(inputEvent); printf("Sent %s event using batch with key '%s' at %s%n", state.eventType, state.eventSubject(), diff --git a/core/src/main/java/dev/streamx/cli/command/ingestion/batch/resolver/step/BinaryResolverStep.java b/core/src/main/java/dev/streamx/cli/command/ingestion/batch/resolver/step/BinaryResolverStep.java index 812c03a2..43e3e3db 100644 --- a/core/src/main/java/dev/streamx/cli/command/ingestion/batch/resolver/step/BinaryResolverStep.java +++ b/core/src/main/java/dev/streamx/cli/command/ingestion/batch/resolver/step/BinaryResolverStep.java @@ -1,15 +1,15 @@ package dev.streamx.cli.command.ingestion.batch.resolver.step; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.BinaryNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.util.TokenBuffer; import dev.streamx.cli.exception.PayloadException; -import dev.streamx.cli.util.FileSourceUtils; import jakarta.enterprise.context.ApplicationScoped; import java.io.IOException; -import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; @@ -17,7 +17,7 @@ @ApplicationScoped public class BinaryResolverStep implements ResolverStep { - private static final ObjectMapper BINARY_SERIALIZATION_OBJECT_MAPPER = new ObjectMapper(); + private static final String FILE_STRATEGY_PREFIX = "file://"; @Override public JsonNode resolve(JsonNode payload, Map variables) { @@ -31,8 +31,9 @@ public JsonNode resolve(JsonNode payload, Map variables) { private JsonNode evaluateBinary(JsonNode evaluatedPayload) { if (evaluatedPayload.isTextual()) { String text = evaluatedPayload.textValue(); - if (FileSourceUtils.applies(text)) { - return toJsonNode(FileSourceUtils.resolve(text)); + if (isFileReference(text)) { + byte[] content = readFileContent(text); + return new BinaryNode(content); } else { return evaluatedPayload; } @@ -55,18 +56,20 @@ private JsonNode evaluateBinary(JsonNode evaluatedPayload) { } } - /** - * Converts binary data into a JSON node by writing it as an ISO_8859_1 string. - */ - private static JsonNode toJsonNode(byte[] datum) { - if (datum == null) { - return null; - } - try (var generator = new TokenBuffer(BINARY_SERIALIZATION_OBJECT_MAPPER, false)) { - generator.writeString(new String(datum, StandardCharsets.ISO_8859_1)); - return BINARY_SERIALIZATION_OBJECT_MAPPER.readTree(generator.asParser()); + public static boolean isFileReference(String rawSource) { + return rawSource != null && rawSource.startsWith(FILE_STRATEGY_PREFIX); + } + + public static byte[] readFileContent(String rawSource) { + String sourceFile = rawSource.substring(FILE_STRATEGY_PREFIX.length()); + Path path = Path.of(sourceFile); + try { + return Files.readAllBytes(path); + } catch (NoSuchFileException e) { + throw PayloadException.noSuchFileException(e, path); } catch (IOException e) { - throw PayloadException.ioException(e); + throw PayloadException.fileReadingException(e, path); } } + } diff --git a/core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamCommand.java b/core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamCommand.java index fda195eb..9a973633 100644 --- a/core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamCommand.java +++ b/core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamCommand.java @@ -4,12 +4,14 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.streamx.ce.serialization.DeserializerException; import com.streamx.ce.serialization.json.CloudEventJsonDeserializer; import com.streamx.clients.ingestion.exceptions.StreamxClientException; import com.streamx.clients.ingestion.publisher.Publisher; import dev.streamx.cli.VersionProvider; import dev.streamx.cli.command.ingestion.BaseIngestionCommand; +import dev.streamx.cli.command.ingestion.stream.parser.JsonBase64Encoder; import dev.streamx.cli.command.ingestion.stream.parser.StreamIngestionJsonParser; import dev.streamx.cli.util.ExceptionUtils; import dev.streamx.cli.util.FileUtils; @@ -19,6 +21,7 @@ import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.List; import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Command; @@ -32,6 +35,7 @@ public class StreamCommand extends BaseIngestionCommand { // TODO "_v2" is a temporary postfix for now public static final String COMMAND_NAME = "stream_v2"; + private static final ObjectMapper objectMapper = new ObjectMapper(); private static final CloudEventJsonDeserializer deserializer = new CloudEventJsonDeserializer(); @ArgGroup(exclusive = false, multiplicity = "1") @@ -43,10 +47,13 @@ public class StreamCommand extends BaseIngestionCommand { @Override protected void perform(Publisher publisher) throws StreamxClientException { Path streamFile = Paths.get(streamIngestionArguments.getSourceFile()); + List jsonPathsToEncodeToBase64 = StreamProperties + .getJsonPathsToEncodeToBase64(streamFile); try (FileInputStream fis = new FileInputStream(streamFile.toFile())) { ingestionJsonParser.parse(fis, cloudEventNode -> { + JsonBase64Encoder.encodeFields(cloudEventNode, jsonPathsToEncodeToBase64); CloudEvent inputEvent = toCloudEvent(cloudEventNode); CloudEvent responseEvent = publisher.send(inputEvent); printf("Sent %s event using stream with key '%s' at %s%n", @@ -66,7 +73,6 @@ protected void perform(Publisher publisher) throws StreamxClientException { private static CloudEvent toCloudEvent(JsonNode cloudEventNode) throws JsonProcessingException { byte[] cloudEventJsonBytes = objectMapper.writeValueAsBytes(cloudEventNode); - CloudEvent inputEvent = deserializer.deserialize(cloudEventJsonBytes); - return withAdjustedData(inputEvent); + return deserializer.deserialize(cloudEventJsonBytes); } } diff --git a/core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamProperties.java b/core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamProperties.java new file mode 100644 index 00000000..e0a7509b --- /dev/null +++ b/core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamProperties.java @@ -0,0 +1,44 @@ +package dev.streamx.cli.command.ingestion.stream; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import org.apache.commons.lang3.StringUtils; +import org.jboss.logging.Logger; + +class StreamProperties { + + private static final Logger logger = Logger.getLogger(StreamProperties.class); + private static final String PROPERTIES_FILE_NAME = ".stream.properties"; + private static final String BASE64_PATHS_PROPERTY_NAME = "json.paths.to.encode.to.base64"; + + private StreamProperties() { + // no instances + } + + public static List getJsonPathsToEncodeToBase64(Path streamFile) { + Path propertiesFile = streamFile.resolveSibling(PROPERTIES_FILE_NAME); + if (Files.exists(propertiesFile)) { + try (InputStream fileStream = Files.newInputStream(propertiesFile)) { + Properties properties = new Properties(); + properties.load(fileStream); + String paths = properties.getProperty(BASE64_PATHS_PROPERTY_NAME); + if (StringUtils.isNotBlank(paths)) { + return Arrays.stream(paths.split(",")) + .map(String::trim) + .filter(StringUtils::isNotEmpty) + .toList(); + } + } catch (IOException ex) { + logger.warn("Error reading properties from " + propertiesFile, ex); + } + } + return Collections.emptyList(); + } + +} \ No newline at end of file diff --git a/core/src/main/java/dev/streamx/cli/command/ingestion/stream/parser/JsonBase64Encoder.java b/core/src/main/java/dev/streamx/cli/command/ingestion/stream/parser/JsonBase64Encoder.java new file mode 100644 index 00000000..01d84b2d --- /dev/null +++ b/core/src/main/java/dev/streamx/cli/command/ingestion/stream/parser/JsonBase64Encoder.java @@ -0,0 +1,43 @@ +package dev.streamx.cli.command.ingestion.stream.parser; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import java.util.Base64; +import java.util.List; + +public class JsonBase64Encoder { + + private JsonBase64Encoder() { + // no instances + } + + public static void encodeFields(JsonNode root, List jsonPaths) { + for (String jsonPath : jsonPaths) { + JsonNode node = root.at(jsonPath); + + if (node.isMissingNode() || !node.isTextual()) { + continue; + } + + TextNode encodedTextNode = createTextNodeWithEncodedContent(node); + replaceNode(root, jsonPath, encodedTextNode); + } + } + + private static TextNode createTextNodeWithEncodedContent(JsonNode node) { + String encodedText = Base64.getEncoder().encodeToString(node.asText().getBytes()); + return TextNode.valueOf(encodedText); + } + + private static void replaceNode(JsonNode root, String jsonPath, JsonNode newValue) { + int lastSlash = jsonPath.lastIndexOf('/'); + String parentNodePath = jsonPath.substring(0, lastSlash); + String fieldName = jsonPath.substring(lastSlash + 1); + + JsonNode parentNode = root.at(parentNodePath); + if (parentNode instanceof ObjectNode obj) { + obj.set(fieldName, newValue); + } + } +} \ No newline at end of file diff --git a/core/src/main/java/dev/streamx/cli/exception/PayloadException.java b/core/src/main/java/dev/streamx/cli/exception/PayloadException.java index 1f6550d3..fd0ed959 100644 --- a/core/src/main/java/dev/streamx/cli/exception/PayloadException.java +++ b/core/src/main/java/dev/streamx/cli/exception/PayloadException.java @@ -17,15 +17,6 @@ private PayloadException(String message) { super(message); } - public static PayloadException payloadNotFound() { - return new PayloadException(""" - Payload definition not found. - - Make sure that: - * there is an optional third parameter that points to the file containing the payload, - * or there is at least one Payload Defining Option (-s/-b/-j)"""); - } - public static PayloadException jsonParseException(JsonParseException exception, String payload) { return new PayloadException(""" Payload could not be parsed. diff --git a/core/src/main/java/dev/streamx/cli/util/FileSourceUtils.java b/core/src/main/java/dev/streamx/cli/util/FileSourceUtils.java deleted file mode 100644 index 8e4ff5b9..00000000 --- a/core/src/main/java/dev/streamx/cli/util/FileSourceUtils.java +++ /dev/null @@ -1,38 +0,0 @@ -package dev.streamx.cli.util; - -import dev.streamx.cli.exception.PayloadException; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; - -public class FileSourceUtils { - - public static final String FILE_STRATEGY_PREFIX = "file://"; - - private FileSourceUtils() { - // No instances - } - - - public static boolean applies(String rawSource) { - return rawSource != null && rawSource.startsWith(FILE_STRATEGY_PREFIX); - } - - public static byte[] resolve(String rawSource) { - String source = rawSource.substring(FILE_STRATEGY_PREFIX.length()); - - return readFile(source); - } - - private static byte[] readFile(String data) { - Path path = Path.of(data); - try { - return Files.readAllBytes(path); - } catch (NoSuchFileException e) { - throw PayloadException.noSuchFileException(e, path); - } catch (IOException e) { - throw PayloadException.fileReadingException(e, path); - } - } -} diff --git a/core/src/test/java/dev/streamx/cli/command/ingestion/batch/BatchPublishCommandTest.java b/core/src/test/java/dev/streamx/cli/command/ingestion/batch/BatchPublishCommandTest.java index daf9bfde..a6ae6f7a 100644 --- a/core/src/test/java/dev/streamx/cli/command/ingestion/batch/BatchPublishCommandTest.java +++ b/core/src/test/java/dev/streamx/cli/command/ingestion/batch/BatchPublishCommandTest.java @@ -2,7 +2,6 @@ import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; -import static java.nio.charset.StandardCharsets.UTF_8; import com.github.tomakehurst.wiremock.matching.ContainsPattern; import dev.streamx.cli.command.ingestion.AuthorizedProfile; @@ -12,6 +11,9 @@ import io.quarkus.test.junit.main.LaunchResult; import io.quarkus.test.junit.main.QuarkusMainLauncher; import io.quarkus.test.junit.main.QuarkusMainTest; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import org.jose4j.base64url.Base64; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -47,11 +49,11 @@ public void shouldRejectInvalidDataJson(QuarkusMainLauncher launcher) { } @Test - public void shouldBatchPublishValidDirectory(QuarkusMainLauncher launcher) { + public void shouldBatchPublishValidDirectoryWithPage(QuarkusMainLauncher launcher) { // when LaunchResult result = launcher.launch( BatchCommand.COMMAND_NAME, "--ingestion-url=" + getIngestionUrl(), - "target/test-classes/dev/streamx/cli/command/ingestion/batch/valid/publish" + "target/test-classes/dev/streamx/cli/command/ingestion/batch/valid/publish/page" ); // then @@ -64,14 +66,47 @@ public void shouldBatchPublishValidDirectory(QuarkusMainLauncher launcher) { "source" : "source", "type" : "page_publish", "datacontenttype" : "application/json", - "subject" : "publish/index.html", + "subject" : "page/index.html", "time" : "2025-12-23T10:28:23.435253Z", "data" : { "content" : "%s", "type" : "page/sub-page" } } - """.formatted(Base64.encode("

Hello World!

".getBytes(UTF_8))))) + """.formatted(Base64.encode("

Hello World!

".getBytes())))) + .withoutHeader("Authorization")); + wm.verify(1, postRequestedFor(urlEqualTo(PUBLICATION_PATH))); + } + + @Test + public void shouldBatchPublishValidDirectoryWithImage(QuarkusMainLauncher launcher) + throws IOException { + // when + LaunchResult result = launcher.launch( + BatchCommand.COMMAND_NAME, "--ingestion-url=" + getIngestionUrl(), + "target/test-classes/dev/streamx/cli/command/ingestion/batch/valid/publish/image" + ); + + // then + expectSuccess(result); + wm.verify(postRequestedFor(urlEqualTo(PUBLICATION_PATH)) + .withRequestBody(new CloudEventJsonMatcher(""" + { + "specversion" : "1.0", + "id" : "75a90fb7-327e-4bee-96a1-3e4224a1e71d", + "source" : "source", + "type" : "image_publish", + "datacontenttype" : "application/json", + "subject" : "image/ds.png", + "time" : "2025-12-23T10:28:23.435253Z", + "data" : { + "content" : "%s", + "type" : "assets/image" + } + } + """.formatted(Base64.encode(Files.readAllBytes(Path.of( + "target/test-classes/dev/streamx/cli/command/ingestion/batch/valid/" + + "publish/image/ds.png")))))) .withoutHeader("Authorization")); wm.verify(1, postRequestedFor(urlEqualTo(PUBLICATION_PATH))); } @@ -119,12 +154,12 @@ public void shouldRejectWrongIngestionUrl(QuarkusMainLauncher launcher) { LaunchResult result = launcher.launch( BatchCommand.COMMAND_NAME, "--ingestion-url=" + ingestionServiceUrl, - "target/test-classes/dev/streamx/cli/command/ingestion/batch/valid/publish"); + "target/test-classes/dev/streamx/cli/command/ingestion/batch/valid/publish/page"); // then expectError(result, "Error performing batch publication while processing 'target/test-classes/" - + "dev/streamx/cli/command/ingestion/batch/valid/publish/index.html' file.\n" + + "dev/streamx/cli/command/ingestion/batch/valid/publish/page/index.html' file.\n" + "\n" + "Details:\n" + "Ingestion REST error: unknown host\n" @@ -144,7 +179,7 @@ public void shouldPublishAuthorizedUsing(QuarkusMainLauncher launcher) { // when LaunchResult result = launcher.launch(BatchCommand.COMMAND_NAME, "--ingestion-url=" + getIngestionUrl(), - "target/test-classes/dev/streamx/cli/command/ingestion/batch/valid/publish"); + "target/test-classes/dev/streamx/cli/command/ingestion/batch/valid/publish/page"); // then expectSuccess(result); diff --git a/core/src/test/java/dev/streamx/cli/command/ingestion/batch/BatchUnpublishCommandTest.java b/core/src/test/java/dev/streamx/cli/command/ingestion/batch/BatchUnpublishCommandTest.java index 733c4c32..a43f65ed 100644 --- a/core/src/test/java/dev/streamx/cli/command/ingestion/batch/BatchUnpublishCommandTest.java +++ b/core/src/test/java/dev/streamx/cli/command/ingestion/batch/BatchUnpublishCommandTest.java @@ -27,7 +27,7 @@ public void shouldUnpublishUsingIngestionClient(QuarkusMainLauncher launcher) { LaunchResult result = launcher.launch( BatchCommand.COMMAND_NAME, "--ingestion-url=" + getIngestionUrl(), - "target/test-classes/dev/streamx/cli/command/ingestion/batch/valid/unpublish"); + "target/test-classes/dev/streamx/cli/command/ingestion/batch/valid/unpublish/page"); // then expectSuccess(result); @@ -39,7 +39,7 @@ public void shouldUnpublishUsingUnauthorizedIngestionClient(QuarkusMainLauncher LaunchResult result = launcher.launch( BatchCommand.COMMAND_NAME, "--ingestion-url=" + getIngestionUrl(), - "target/test-classes/dev/streamx/cli/command/ingestion/batch/valid/unpublish"); + "target/test-classes/dev/streamx/cli/command/ingestion/batch/valid/unpublish/page"); // then expectSuccess(result); @@ -60,7 +60,7 @@ public void shouldUnpublishUsingAuthorizedIngestionClient(QuarkusMainLauncher la LaunchResult result = launcher.launch( BatchCommand.COMMAND_NAME, "--ingestion-url=" + getIngestionUrl(), - "target/test-classes/dev/streamx/cli/command/ingestion/batch/valid/unpublish"); + "target/test-classes/dev/streamx/cli/command/ingestion/batch/valid/unpublish/page"); // then expectSuccess(result); @@ -74,7 +74,7 @@ public void shouldUnpublishUsingAuthorizedIngestionClient(QuarkusMainLauncher la "source" : "source", "type" : "page_unpublish", "datacontenttype":"application/json", - "subject" : "unpublish/index.html", + "subject" : "page/index.html", "time" : "2025-12-23T11:59:29.946127Z", "data" : { "type" : "page/sub-page" diff --git a/core/src/test/java/dev/streamx/cli/license/input/StdInLineReadStrategyTest.java b/core/src/test/java/dev/streamx/cli/license/input/StdInLineReadStrategyTest.java index 95442b1e..2b3cbf54 100644 --- a/core/src/test/java/dev/streamx/cli/license/input/StdInLineReadStrategyTest.java +++ b/core/src/test/java/dev/streamx/cli/license/input/StdInLineReadStrategyTest.java @@ -1,7 +1,6 @@ package dev.streamx.cli.license.input; import java.io.ByteArrayInputStream; -import java.nio.charset.StandardCharsets; import org.assertj.core.api.Assertions; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -50,6 +49,6 @@ void shouldReject(String source) { } private static void givenStdIn(String source) { - System.setIn(new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8))); + System.setIn(new ByteArrayInputStream(source.getBytes())); } } \ No newline at end of file diff --git a/core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/publish/image/.eventsource.yaml b/core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/publish/image/.eventsource.yaml new file mode 100644 index 00000000..d1047825 --- /dev/null +++ b/core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/publish/image/.eventsource.yaml @@ -0,0 +1,12 @@ +eventType: image_publish +eventSource: source + +relativePathLevel: 1 +key: ${relativePath} +ignorePatterns: + - ".*/ignore\\.me" + +payload: + content: file://${payloadPath} + type: assets/image + diff --git a/core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/publish/image/ds.png b/core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/publish/image/ds.png new file mode 100644 index 0000000000000000000000000000000000000000..4b85fd5595a31d4159a76523de8ead1227e8dccf GIT binary patch literal 794 zcmV+#1LgdQP)0AF5+scm83Xka=X>Vqda*6P2?PzVB$#VJ{eSkr_gB5QvxO(;mQiq z%XyqZaTBx*Y8SI^T(}6Wg6i2XchGb`bjQx~58bd=b4^u1klmaPEa!Pu1%Y;?!kysz z(ZIA5y5*p5sFCXxj9>m6Ph@7sy)DpTPnjix52M8F>aOTvU?XKvpSjRL#(0}u?#$z& z!Aid4$R^67UfmgEce&7m>1uF@M;&<&HBgvE>FZ={fa-i`SA*i(3BlS*a6K)3os8)z zD3nubpwGcjc2byDsF}tZp|j8%tJ00zYVejvJ=jNKra~R*u~6XC0Ik&z->sH@bdl*4 z=%cuvOk9B37rUnySpk)2LSJ<-FnT@M&l?V?UvZI7VxBN#({sF$xP$3l@@nA;3HPM* zsWgqO*@m;*q8UCV&3>5@NLLBOaU zItu-=dj#=A%nN`z?h_46o1s0>Bdetw51|fd68q)?8qE>&pZbQFw+)P*sINi$CF0Hg YA0Yu2>W4%%jsO4v07*qoM6N<$g4geS5&!@I literal 0 HcmV?d00001 diff --git a/core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/publish/ignore.me b/core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/publish/image/ignore.me similarity index 100% rename from core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/publish/ignore.me rename to core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/publish/image/ignore.me diff --git a/core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/publish/.eventsource.yaml b/core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/publish/page/.eventsource.yaml similarity index 78% rename from core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/publish/.eventsource.yaml rename to core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/publish/page/.eventsource.yaml index 18ab39df..ee93d29e 100644 --- a/core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/publish/.eventsource.yaml +++ b/core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/publish/page/.eventsource.yaml @@ -7,7 +7,6 @@ ignorePatterns: - ".*/ignore\\.me" payload: - content: - bytes: file://${payloadPath} + content: file://${payloadPath} type: page/sub-page diff --git a/core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/unpublish/ignore.me b/core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/publish/page/ignore.me similarity index 100% rename from core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/unpublish/ignore.me rename to core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/publish/page/ignore.me diff --git a/core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/publish/index.html b/core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/publish/page/index.html similarity index 100% rename from core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/publish/index.html rename to core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/publish/page/index.html diff --git a/core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/unpublish/.eventsource.yaml b/core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/unpublish/page/.eventsource.yaml similarity index 100% rename from core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/unpublish/.eventsource.yaml rename to core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/unpublish/page/.eventsource.yaml diff --git a/core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/unpublish/page/ignore.me b/core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/unpublish/page/ignore.me new file mode 100644 index 00000000..8195dbc0 --- /dev/null +++ b/core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/unpublish/page/ignore.me @@ -0,0 +1 @@ +nevermind - ignore me! diff --git a/core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/unpublish/index.html b/core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/unpublish/page/index.html similarity index 100% rename from core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/unpublish/index.html rename to core/src/test/resources/dev/streamx/cli/command/ingestion/batch/valid/unpublish/page/index.html diff --git a/core/src/test/resources/dev/streamx/cli/command/ingestion/stream/.stream.properties b/core/src/test/resources/dev/streamx/cli/command/ingestion/stream/.stream.properties new file mode 100644 index 00000000..c872efe1 --- /dev/null +++ b/core/src/test/resources/dev/streamx/cli/command/ingestion/stream/.stream.properties @@ -0,0 +1 @@ +json.paths.to.encode.to.base64=/data/content \ No newline at end of file diff --git a/core/src/test/resources/dev/streamx/cli/command/ingestion/stream/valid-json.stream b/core/src/test/resources/dev/streamx/cli/command/ingestion/stream/valid-json.stream index 31d20c86..d5785ffe 100644 --- a/core/src/test/resources/dev/streamx/cli/command/ingestion/stream/valid-json.stream +++ b/core/src/test/resources/dev/streamx/cli/command/ingestion/stream/valid-json.stream @@ -7,9 +7,7 @@ "subject" : "homepage.html", "time" : "2025-12-23T10:28:23.435253Z", "data" : { - "content" : { - "bytes" : "

This is Home Page!

" - }, + "content" : "

This is Home Page!

", "type" : "page/content" } } diff --git a/e2e-tests/pom.xml b/e2e-tests/pom.xml index 86e2c4cd..fe6b6163 100644 --- a/e2e-tests/pom.xml +++ b/e2e-tests/pom.xml @@ -30,6 +30,11 @@ org.awaitility awaitility + + org.assertj + assertj-core + compile + org.jboss.logging commons-logging-jboss-logging diff --git a/e2e-tests/src/main/java/dev/streamx/cli/test/tools/validators/HttpValidator.java b/e2e-tests/src/main/java/dev/streamx/cli/test/tools/validators/HttpValidator.java index 5bca3d82..385e8e14 100644 --- a/e2e-tests/src/main/java/dev/streamx/cli/test/tools/validators/HttpValidator.java +++ b/e2e-tests/src/main/java/dev/streamx/cli/test/tools/validators/HttpValidator.java @@ -1,6 +1,8 @@ package dev.streamx.cli.test.tools.validators; import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; import static org.awaitility.Awaitility.await; import jakarta.annotation.PreDestroy; @@ -8,10 +10,12 @@ import jakarta.inject.Inject; import java.io.IOException; import java.time.Duration; +import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.util.EntityUtils; +import org.assertj.core.api.ThrowingConsumer; import org.jboss.logging.Logger; @ApplicationScoped @@ -23,31 +27,35 @@ public class HttpValidator { CloseableHttpClient httpClient; public void validate(String url, int expectedStatusCode, String expectedBody, Duration timeout) { + validate(url, expectedStatusCode, timeout, httpEntity -> { + String responseBody = EntityUtils.toString(httpEntity); + assertThat(responseBody).describedAs(url).contains(expectedBody); + }); + } + + public void validate(String url, int expectedStatusCode, byte[] expectedBody, Duration timeout) { + validate(url, expectedStatusCode, timeout, httpEntity -> { + byte[] responseBody = EntityUtils.toByteArray(httpEntity); + assertThat(responseBody).describedAs(url).containsExactly(expectedBody); + }); + } + + public void validate(String url, int expectedStatusCode, Duration timeout, + ThrowingConsumer responseEntityAssertion) { await() .atMost(timeout) - .alias("Assertion of response from url: " + url - + " with expecting status:" + expectedStatusCode - + " body:" + expectedBody - + " timeout:" + timeout) .pollInterval(100, MILLISECONDS) - .until(() -> - validate(url, expectedStatusCode, expectedBody) - ); - } - - private boolean validate(String url, int expectedStatusCode, String expectedBody) { - HttpGet request = new HttpGet(url); - try (CloseableHttpResponse response = httpClient.execute(request)) { - int actualStatusCode = response.getStatusLine().getStatusCode(); - String responseBody = EntityUtils.toString(response.getEntity()); - logger.info("Request to " + url - + " return statusCode " + actualStatusCode - + " and body " + responseBody); - return actualStatusCode == expectedStatusCode && responseBody.contains(expectedBody); - } catch (IOException e) { - logger.error("Request to " + url + "failed: " + e.getMessage(), e); - throw new RuntimeException("Can not make request:" + url, e); - } + .untilAsserted(() -> { + HttpGet request = new HttpGet(url); + try (CloseableHttpResponse response = httpClient.execute(request)) { + int statusCode = response.getStatusLine().getStatusCode(); + assertThat(statusCode).describedAs(url).isEqualTo(expectedStatusCode); + HttpEntity responseEntity = response.getEntity(); + responseEntityAssertion.accept(responseEntity); + } catch (IOException e) { + fail("Request to " + url + "failed", e); + } + }); } @PreDestroy diff --git a/e2e-tests/src/main/java/dev/streamx/cli/test/tools/validators/ProcessOutputValidator.java b/e2e-tests/src/main/java/dev/streamx/cli/test/tools/validators/ProcessOutputValidator.java index 2cda2c77..d1c58a69 100644 --- a/e2e-tests/src/main/java/dev/streamx/cli/test/tools/validators/ProcessOutputValidator.java +++ b/e2e-tests/src/main/java/dev/streamx/cli/test/tools/validators/ProcessOutputValidator.java @@ -1,8 +1,10 @@ package dev.streamx.cli.test.tools.validators; import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; +import dev.streamx.cli.test.tools.terminal.process.ShellProcess; import jakarta.enterprise.context.ApplicationScoped; import java.time.Duration; import java.util.List; @@ -21,15 +23,19 @@ public class ProcessOutputValidator { + "[\\p{Alnum}.,%_=?&#\\-+()\\[\\]\\*$~@!:/{};']*)", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE | Pattern.DOTALL); - public void validate(List output, String expectedContent, Duration timeout) { + public void validate(ShellProcess process, List output, String expectedContent, + Duration timeout) { try { await() .atMost(timeout) .pollInterval(100, MILLISECONDS) .alias("Finding expectedContent: " + expectedContent) - .until(() -> - output - .stream() + .untilAsserted(() -> + assertThat(output) + .describedAs(() -> "Full output is:\n" + + String.join("\n", process.getCurrentOutputLines()) + + "\n" + + String.join("\n", process.getCurrentErrorLines())) .anyMatch(line -> line.contains(expectedContent)) ); } catch (ConditionTimeoutException e) { @@ -43,10 +49,8 @@ public String validateContainsUrl(List output, Duration timeout) { .atMost(timeout) .pollInterval(100, MILLISECONDS) .alias("Finding any url") - .until(() -> - output - .stream() - .anyMatch(line -> urlPattern.matcher(line).find()) + .untilAsserted(() -> + assertThat(output).anyMatch(line -> urlPattern.matcher(line).find()) ); return output.stream() diff --git a/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliLicenseIT.java b/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliLicenseIT.java index 28244190..e5055942 100644 --- a/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliLicenseIT.java +++ b/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliLicenseIT.java @@ -100,14 +100,14 @@ private static void acceptLicense(ShellProcess p) { } private void assertIfAcceptanceLicenseErrorPresent(ShellProcess p) { - processOutputValidator.validate( + processOutputValidator.validate(p, p.getCurrentErrorLines(), "License acceptance is required for using StreamX", CLI_TIMEOUT); } private void assertIfStreamxCommandWork(ShellProcess p) { - processOutputValidator.validate(p.getCurrentOutputLines(), "streamx-cli version", + processOutputValidator.validate(p, p.getCurrentOutputLines(), "streamx-cli version", CLI_TIMEOUT); } @@ -116,7 +116,7 @@ private ShellProcess runStreamxCommand() { } private void assertIfAcceptLicenseQuestionPresent(ShellProcess p) { - processOutputValidator.validate( + processOutputValidator.validate(p, p.getCurrentOutputLines(), "Do you accept the license agreement?", CLI_TIMEOUT); diff --git a/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliPublicationIT.java b/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliPublicationIT.java index e5866fdc..cca684ef 100644 --- a/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliPublicationIT.java +++ b/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliPublicationIT.java @@ -9,8 +9,12 @@ import io.quarkus.test.junit.QuarkusTest; import jakarta.inject.Inject; import jakarta.inject.Named; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.time.Duration; import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @@ -23,7 +27,7 @@ public class StreamxCliPublicationIT { private static final Duration CLI_TIMEOUT = Duration.ofSeconds(10); @ConfigProperty(name = "streamx.cli.e2e.web.delivery.url", defaultValue = "http://localhost:8087/") - String webDeliveryPortUrl; + String webDeliveryUrl; @ConfigProperty(name = "streamx.cli.e2e.setup.timeoutInSec", defaultValue = "60") int setupTimeoutInSec; @@ -48,7 +52,7 @@ public void setup() { private void runStreamxCommand(String command, String expectedOutput, Duration timeout) { ShellProcess p = terminalCommandRunner.run(command); - processOutputValidator.validate(p.getCurrentOutputLines(), expectedOutput, timeout); + processOutputValidator.validate(p, p.getCurrentOutputLines(), expectedOutput, timeout); } private void runStreamxIngestionCommand(String commandName, String path, String expectedOutput) { @@ -64,7 +68,7 @@ public void shouldPublishAndUnpublishPageUsingStreamOperation() { "Sent com.streamx.blueprints.page.published.v1 event using stream with key 'hello.html'" ); - validateStreamxPage("hello.html", 200, "Hello World!"); + validateStreamxPage("hello.html", "Hello World!"); runStreamxIngestionCommand( "stream_v2", @@ -72,31 +76,61 @@ public void shouldPublishAndUnpublishPageUsingStreamOperation() { "Sent com.streamx.blueprints.page.unpublished.v1 event using stream with key 'hello.html'" ); - validateStreamxPage("hello.html", 404, ""); + validateStreamxPageNotAvailable("hello.html"); } @Test public void shouldPublishAndUnpublishPageUsingBatchOperation() { runStreamxIngestionCommand( "batch_v2", - "src/test/resources/batch/publish", + "src/test/resources/batch/publish/page", "Sent com.streamx.blueprints.page.published.v1 event using batch with key 'index.html'" ); - validateStreamxPage("index.html", 200, "

Hello World!

"); + validateStreamxPage("index.html", "

Hello World!

"); runStreamxIngestionCommand( "batch_v2", - "src/test/resources/batch/unpublish", + "src/test/resources/batch/unpublish/page", "Sent com.streamx.blueprints.page.unpublished.v1 event using batch with key 'index.html'" ); - validateStreamxPage("index.html", 404, ""); + validateStreamxPageNotAvailable("index.html"); } - private void validateStreamxPage(String resourcePath, int expectedStatusCode, - String expectedBody) { - String url = webDeliveryPortUrl + resourcePath; - httpValidator.validate(url, expectedStatusCode, expectedBody, CLI_TIMEOUT); + @Test + public void shouldPublishAndUnpublishAssetUsingBatchOperation() throws IOException { + runStreamxIngestionCommand( + "batch_v2", + "src/test/resources/batch/publish/asset", + "Sent com.streamx.blueprints.asset.published.v1 event using batch with key 'ds.png'" + ); + + validateStreamxPage("ds.png", + Files.readAllBytes(Path.of("src/test/resources/batch/publish/asset/ds.png"))); + + runStreamxIngestionCommand( + "batch_v2", + "src/test/resources/batch/unpublish/asset", + "Sent com.streamx.blueprints.asset.unpublished.v1 event using batch with key 'ds.png'" + ); + + validateStreamxPageNotAvailable("ds.png"); + } + + private void validateStreamxPage(String resourcePath, String expectedBody) { + httpValidator.validate(url(resourcePath), 200, expectedBody, CLI_TIMEOUT); + } + + private void validateStreamxPage(String resourcePath, byte[] expectedBody) { + httpValidator.validate(url(resourcePath), 200, expectedBody, CLI_TIMEOUT); + } + + private void validateStreamxPageNotAvailable(String resourcePath) { + httpValidator.validate(url(resourcePath), 404, "", CLI_TIMEOUT); + } + + private String url(String resourcePath) { + return webDeliveryUrl + resourcePath; } } diff --git a/e2e-tests/src/test/resources/batch/publish/asset/.eventsource.yaml b/e2e-tests/src/test/resources/batch/publish/asset/.eventsource.yaml new file mode 100644 index 00000000..0d4ee277 --- /dev/null +++ b/e2e-tests/src/test/resources/batch/publish/asset/.eventsource.yaml @@ -0,0 +1,9 @@ +eventType: com.streamx.blueprints.asset.published.v1 +eventSource: source + +relativePathLevel: 0 +key: ${relativePath} + +payload: + content: file://${payloadPath} + type: assets/image diff --git a/e2e-tests/src/test/resources/batch/publish/asset/ds.png b/e2e-tests/src/test/resources/batch/publish/asset/ds.png new file mode 100644 index 0000000000000000000000000000000000000000..4b85fd5595a31d4159a76523de8ead1227e8dccf GIT binary patch literal 794 zcmV+#1LgdQP)0AF5+scm83Xka=X>Vqda*6P2?PzVB$#VJ{eSkr_gB5QvxO(;mQiq z%XyqZaTBx*Y8SI^T(}6Wg6i2XchGb`bjQx~58bd=b4^u1klmaPEa!Pu1%Y;?!kysz z(ZIA5y5*p5sFCXxj9>m6Ph@7sy)DpTPnjix52M8F>aOTvU?XKvpSjRL#(0}u?#$z& z!Aid4$R^67UfmgEce&7m>1uF@M;&<&HBgvE>FZ={fa-i`SA*i(3BlS*a6K)3os8)z zD3nubpwGcjc2byDsF}tZp|j8%tJ00zYVejvJ=jNKra~R*u~6XC0Ik&z->sH@bdl*4 z=%cuvOk9B37rUnySpk)2LSJ<-FnT@M&l?V?UvZI7VxBN#({sF$xP$3l@@nA;3HPM* zsWgqO*@m;*q8UCV&3>5@NLLBOaU zItu-=dj#=A%nN`z?h_46o1s0>Bdetw51|fd68q)?8qE>&pZbQFw+)P*sINi$CF0Hg YA0Yu2>W4%%jsO4v07*qoM6N<$g4geS5&!@I literal 0 HcmV?d00001 diff --git a/e2e-tests/src/test/resources/batch/publish/.eventsource.yaml b/e2e-tests/src/test/resources/batch/publish/page/.eventsource.yaml similarity index 76% rename from e2e-tests/src/test/resources/batch/publish/.eventsource.yaml rename to e2e-tests/src/test/resources/batch/publish/page/.eventsource.yaml index cecd6f94..eca8625a 100644 --- a/e2e-tests/src/test/resources/batch/publish/.eventsource.yaml +++ b/e2e-tests/src/test/resources/batch/publish/page/.eventsource.yaml @@ -5,7 +5,5 @@ relativePathLevel: 0 key: ${relativePath} payload: - content: - bytes: file://${payloadPath} + content: file://${payloadPath} type: page/index-page - diff --git a/e2e-tests/src/test/resources/batch/publish/index.html b/e2e-tests/src/test/resources/batch/publish/page/index.html similarity index 100% rename from e2e-tests/src/test/resources/batch/publish/index.html rename to e2e-tests/src/test/resources/batch/publish/page/index.html diff --git a/e2e-tests/src/test/resources/batch/unpublish/asset/.eventsource.yaml b/e2e-tests/src/test/resources/batch/unpublish/asset/.eventsource.yaml new file mode 100644 index 00000000..e7f18c2f --- /dev/null +++ b/e2e-tests/src/test/resources/batch/unpublish/asset/.eventsource.yaml @@ -0,0 +1,9 @@ +eventType: com.streamx.blueprints.asset.unpublished.v1 +eventSource: source + +relativePathLevel: 0 +key: ${relativePath} + +payload: + content: file://${payloadPath} + type: assets/image diff --git a/e2e-tests/src/test/resources/batch/unpublish/asset/ds.png b/e2e-tests/src/test/resources/batch/unpublish/asset/ds.png new file mode 100644 index 0000000000000000000000000000000000000000..4b85fd5595a31d4159a76523de8ead1227e8dccf GIT binary patch literal 794 zcmV+#1LgdQP)0AF5+scm83Xka=X>Vqda*6P2?PzVB$#VJ{eSkr_gB5QvxO(;mQiq z%XyqZaTBx*Y8SI^T(}6Wg6i2XchGb`bjQx~58bd=b4^u1klmaPEa!Pu1%Y;?!kysz z(ZIA5y5*p5sFCXxj9>m6Ph@7sy)DpTPnjix52M8F>aOTvU?XKvpSjRL#(0}u?#$z& z!Aid4$R^67UfmgEce&7m>1uF@M;&<&HBgvE>FZ={fa-i`SA*i(3BlS*a6K)3os8)z zD3nubpwGcjc2byDsF}tZp|j8%tJ00zYVejvJ=jNKra~R*u~6XC0Ik&z->sH@bdl*4 z=%cuvOk9B37rUnySpk)2LSJ<-FnT@M&l?V?UvZI7VxBN#({sF$xP$3l@@nA;3HPM* zsWgqO*@m;*q8UCV&3>5@NLLBOaU zItu-=dj#=A%nN`z?h_46o1s0>Bdetw51|fd68q)?8qE>&pZbQFw+)P*sINi$CF0Hg YA0Yu2>W4%%jsO4v07*qoM6N<$g4geS5&!@I literal 0 HcmV?d00001 diff --git a/e2e-tests/src/test/resources/batch/unpublish/.eventsource.yaml b/e2e-tests/src/test/resources/batch/unpublish/page/.eventsource.yaml similarity index 100% rename from e2e-tests/src/test/resources/batch/unpublish/.eventsource.yaml rename to e2e-tests/src/test/resources/batch/unpublish/page/.eventsource.yaml diff --git a/e2e-tests/src/test/resources/batch/unpublish/index.html b/e2e-tests/src/test/resources/batch/unpublish/page/index.html similarity index 100% rename from e2e-tests/src/test/resources/batch/unpublish/index.html rename to e2e-tests/src/test/resources/batch/unpublish/page/index.html diff --git a/e2e-tests/src/test/resources/filtered/mesh.yaml b/e2e-tests/src/test/resources/filtered/mesh.yaml index e06315cd..b11638e4 100644 --- a/e2e-tests/src/test/resources/filtered/mesh.yaml +++ b/e2e-tests/src/test/resources/filtered/mesh.yaml @@ -29,6 +29,10 @@ sources: produces: - "com.streamx.blueprints.page.published.v1" - "com.streamx.blueprints.page.unpublished.v1" + - ref: "inbox.assets" + produces: + - "com.streamx.blueprints.asset.published.v1" + - "com.streamx.blueprints.asset.unpublished.v1" ingestion: rest-ingestion: @@ -49,6 +53,17 @@ processing: relayed-events: ref: outbox.pages + assets-relay: + descriptor: relay + containers: + service: + incoming: + events: + ref: inbox.assets + outgoing: + relayed-events: + ref: outbox.assets + edge: web-server-sink: descriptor: web-server-sink @@ -56,6 +71,6 @@ edge: sink: incoming: resources: - ref: outbox.pages + ref: outbox.pages,outbox.assets environment: STREAMX_BLUEPRINTS_WEB_SERVER_SINK_STORAGE_ROOT_DIRECTORY: "/srv/www" \ No newline at end of file diff --git a/e2e-tests/src/test/resources/stream/.stream.properties b/e2e-tests/src/test/resources/stream/.stream.properties new file mode 100644 index 00000000..c872efe1 --- /dev/null +++ b/e2e-tests/src/test/resources/stream/.stream.properties @@ -0,0 +1 @@ +json.paths.to.encode.to.base64=/data/content \ No newline at end of file diff --git a/e2e-tests/src/test/resources/stream/page-publish-event.json b/e2e-tests/src/test/resources/stream/page-publish-event.json index 044b9a96..c8b1f8f3 100644 --- a/e2e-tests/src/test/resources/stream/page-publish-event.json +++ b/e2e-tests/src/test/resources/stream/page-publish-event.json @@ -7,9 +7,7 @@ "subject" : "hello.html", "time" : "2025-12-23T10:28:23.435253Z", "data" : { - "content" : { - "bytes" : "Hello World!" - }, + "content" : "Hello World!", "type" : "page/test" } } \ No newline at end of file From 2145fc8721a9f99b2aab2eda48e3982efc48ba3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Wed, 31 Dec 2025 13:09:00 +0100 Subject: [PATCH 17/36] CloudEventBuilder fixes --- .../cli/command/ingestion/batch/BatchCommand.java | 1 - .../ingestion/{ => batch}/CloudEventBuilder.java | 12 ++---------- .../command/ingestion/BaseIngestionCommandTest.java | 4 +++- 3 files changed, 5 insertions(+), 12 deletions(-) rename core/src/main/java/dev/streamx/cli/command/ingestion/{ => batch}/CloudEventBuilder.java (72%) diff --git a/core/src/main/java/dev/streamx/cli/command/ingestion/batch/BatchCommand.java b/core/src/main/java/dev/streamx/cli/command/ingestion/batch/BatchCommand.java index b4f94aac..b1b88ee8 100644 --- a/core/src/main/java/dev/streamx/cli/command/ingestion/batch/BatchCommand.java +++ b/core/src/main/java/dev/streamx/cli/command/ingestion/batch/BatchCommand.java @@ -7,7 +7,6 @@ import com.streamx.clients.ingestion.publisher.Publisher; import dev.streamx.cli.VersionProvider; import dev.streamx.cli.command.ingestion.BaseIngestionCommand; -import dev.streamx.cli.command.ingestion.CloudEventBuilder; import dev.streamx.cli.command.ingestion.batch.exception.EventSourceDescriptorException; import dev.streamx.cli.command.ingestion.batch.exception.FileIngestionException; import dev.streamx.cli.command.ingestion.batch.resolver.BatchPayloadResolver; diff --git a/core/src/main/java/dev/streamx/cli/command/ingestion/CloudEventBuilder.java b/core/src/main/java/dev/streamx/cli/command/ingestion/batch/CloudEventBuilder.java similarity index 72% rename from core/src/main/java/dev/streamx/cli/command/ingestion/CloudEventBuilder.java rename to core/src/main/java/dev/streamx/cli/command/ingestion/batch/CloudEventBuilder.java index 8b1ca539..71c8257e 100644 --- a/core/src/main/java/dev/streamx/cli/command/ingestion/CloudEventBuilder.java +++ b/core/src/main/java/dev/streamx/cli/command/ingestion/batch/CloudEventBuilder.java @@ -1,7 +1,6 @@ -package dev.streamx.cli.command.ingestion; +package dev.streamx.cli.command.ingestion.batch; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.NullNode; import io.cloudevents.CloudEvent; import io.cloudevents.jackson.JsonCloudEventData; @@ -12,17 +11,10 @@ public class CloudEventBuilder { - private static final ObjectMapper objectMapper = new ObjectMapper(); - private CloudEventBuilder() { // no instance } - public static CloudEvent build(String subject, String type, String source, Object data) { - JsonNode dataNode = objectMapper.valueToTree(data); - return build(subject, type, source, dataNode); - } - public static CloudEvent build(String subject, String type, String source, JsonNode data) { var builder = io.cloudevents.core.builder.CloudEventBuilder.v1() .withId(UUID.randomUUID().toString()) @@ -41,4 +33,4 @@ public static CloudEvent build(String subject, String type, String source, JsonN .build(); } -} +} \ No newline at end of file diff --git a/core/src/test/java/dev/streamx/cli/command/ingestion/BaseIngestionCommandTest.java b/core/src/test/java/dev/streamx/cli/command/ingestion/BaseIngestionCommandTest.java index ed54075d..c7ed654b 100644 --- a/core/src/test/java/dev/streamx/cli/command/ingestion/BaseIngestionCommandTest.java +++ b/core/src/test/java/dev/streamx/cli/command/ingestion/BaseIngestionCommandTest.java @@ -6,10 +6,12 @@ import static com.streamx.clients.ingestion.StreamxClient.INGESTION_ENDPOINT_PATH_V2; import static org.assertj.core.api.Assertions.assertThat; +import com.fasterxml.jackson.databind.node.TextNode; import com.github.tomakehurst.wiremock.client.ResponseDefinitionBuilder; import com.github.tomakehurst.wiremock.client.WireMock; import com.github.tomakehurst.wiremock.junit5.WireMockExtension; import com.streamx.ce.serialization.json.CloudEventJsonSerializer; +import dev.streamx.cli.command.ingestion.batch.CloudEventBuilder; import io.cloudevents.CloudEvent; import io.quarkus.test.junit.main.LaunchResult; import org.apache.hc.core5.http.HttpStatus; @@ -33,7 +35,7 @@ void setup() { private void initializeWiremock() { setupMockPublicationResponse( - CloudEventBuilder.build("index.html", "some-event-type", "source", "mock-response") + CloudEventBuilder.build("index.html", "some-event-type", "source", new TextNode("mock-response")) ); } From 403fd2336f5fbf6941fe384e529ac627ba3bb8fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Wed, 31 Dec 2025 13:37:15 +0100 Subject: [PATCH 18/36] fixes --- .../cli/command/ingestion/BaseIngestionCommandTest.java | 3 ++- .../MeshDefinitionResolverInterpolationTest.java | 2 +- core/src/test/resources/{ => filtered}/mesh-interpolated.yaml | 0 3 files changed, 3 insertions(+), 2 deletions(-) rename core/src/test/resources/{ => filtered}/mesh-interpolated.yaml (100%) diff --git a/core/src/test/java/dev/streamx/cli/command/ingestion/BaseIngestionCommandTest.java b/core/src/test/java/dev/streamx/cli/command/ingestion/BaseIngestionCommandTest.java index c7ed654b..e1e42218 100644 --- a/core/src/test/java/dev/streamx/cli/command/ingestion/BaseIngestionCommandTest.java +++ b/core/src/test/java/dev/streamx/cli/command/ingestion/BaseIngestionCommandTest.java @@ -35,7 +35,8 @@ void setup() { private void initializeWiremock() { setupMockPublicationResponse( - CloudEventBuilder.build("index.html", "some-event-type", "source", new TextNode("mock-response")) + CloudEventBuilder.build( + "index.html", "some-event-type", "source", new TextNode("mock-response")) ); } diff --git a/core/src/test/java/dev/streamx/cli/command/meshprocessing/MeshDefinitionResolverInterpolationTest.java b/core/src/test/java/dev/streamx/cli/command/meshprocessing/MeshDefinitionResolverInterpolationTest.java index a085bbfd..ef030ade 100644 --- a/core/src/test/java/dev/streamx/cli/command/meshprocessing/MeshDefinitionResolverInterpolationTest.java +++ b/core/src/test/java/dev/streamx/cli/command/meshprocessing/MeshDefinitionResolverInterpolationTest.java @@ -16,7 +16,7 @@ @QuarkusTest class MeshDefinitionResolverInterpolationTest { - private static final String TEST_MESH_LOCATION = "src/test/resources/mesh-interpolated.yaml"; + private static final String TEST_MESH_LOCATION = "src/test/resources/filtered/mesh-interpolated.yaml"; private static final Path TEST_MESH_PATH = Path.of(TEST_MESH_LOCATION); @Inject diff --git a/core/src/test/resources/mesh-interpolated.yaml b/core/src/test/resources/filtered/mesh-interpolated.yaml similarity index 100% rename from core/src/test/resources/mesh-interpolated.yaml rename to core/src/test/resources/filtered/mesh-interpolated.yaml From d3f7eae53b52df6e5a331303be54c044ae0a9be4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Wed, 31 Dec 2025 14:17:10 +0100 Subject: [PATCH 19/36] fix codestyle --- .../meshprocessing/MeshDefinitionResolverInterpolationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/java/dev/streamx/cli/command/meshprocessing/MeshDefinitionResolverInterpolationTest.java b/core/src/test/java/dev/streamx/cli/command/meshprocessing/MeshDefinitionResolverInterpolationTest.java index ef030ade..4ec28753 100644 --- a/core/src/test/java/dev/streamx/cli/command/meshprocessing/MeshDefinitionResolverInterpolationTest.java +++ b/core/src/test/java/dev/streamx/cli/command/meshprocessing/MeshDefinitionResolverInterpolationTest.java @@ -16,7 +16,7 @@ @QuarkusTest class MeshDefinitionResolverInterpolationTest { - private static final String TEST_MESH_LOCATION = "src/test/resources/filtered/mesh-interpolated.yaml"; + private static final String TEST_MESH_LOCATION = "target/test-classes/mesh-interpolated.yaml"; private static final Path TEST_MESH_PATH = Path.of(TEST_MESH_LOCATION); @Inject From 1f13cc6d954e620456c1887a25a0afa8a47cf2e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Wed, 31 Dec 2025 14:40:23 +0100 Subject: [PATCH 20/36] Small fix --- .../cli/interpolation/InterpolatingMapperTest.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/core/src/test/java/dev/streamx/cli/interpolation/InterpolatingMapperTest.java b/core/src/test/java/dev/streamx/cli/interpolation/InterpolatingMapperTest.java index ac3156f6..47f93279 100644 --- a/core/src/test/java/dev/streamx/cli/interpolation/InterpolatingMapperTest.java +++ b/core/src/test/java/dev/streamx/cli/interpolation/InterpolatingMapperTest.java @@ -10,7 +10,6 @@ import java.util.Set; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; -import org.testcontainers.shaded.org.bouncycastle.oer.its.etsi102941.Url; @QuarkusTest public class InterpolatingMapperTest { @@ -114,7 +113,7 @@ void testMixedTypesInterpolation() throws Exception { assertEquals("interpolatedValue", result.getString()); assertEquals(-1, result.getLongField()); assertEquals(-1, result.getInteger()); - assertEquals("http://interpolatedValue", result.getUrl().getUrl()); + assertEquals("http://interpolatedValue", result.getUrl()); } @@ -124,7 +123,7 @@ static class MixedTypesTestClass { private String string; private Integer integer; private Long longField; - private Url url; + private String url; public boolean isBool() { return bool; @@ -158,11 +157,11 @@ public void setLongField(Long longField) { this.longField = longField; } - public Url getUrl() { + public String getUrl() { return url; } - public void setUrl(Url url) { + public void setUrl(String url) { this.url = url; } } From 811cab5ac599254b03b5a77ed1ec16ab12947378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Mon, 5 Jan 2026 09:31:00 +0100 Subject: [PATCH 21/36] Use new streamx-service-mesh version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c6dbc29e..060e1317 100644 --- a/pom.xml +++ b/pom.xml @@ -41,7 +41,7 @@ io.quarkus.platform 3.18.3 - 2.0.6 + 2.0.8 2.9.0 3.27.3 3.12.1 From 6b6e62daadf17997897161c8c1dd2d371e2af275 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Mon, 5 Jan 2026 09:45:47 +0100 Subject: [PATCH 22/36] Don't use native --- .github/workflows/ci-test-build.yaml | 2 +- core/src/test/resources/filtered/mesh.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-test-build.yaml b/.github/workflows/ci-test-build.yaml index d61692ef..1a22a2f9 100644 --- a/.github/workflows/ci-test-build.yaml +++ b/.github/workflows/ci-test-build.yaml @@ -35,7 +35,7 @@ jobs: - name: Build project run: | - ./mvnw clean verify -P all-tests -Dnative + ./mvnw clean verify -P all-tests windows-test-build: runs-on: windows-latest diff --git a/core/src/test/resources/filtered/mesh.yaml b/core/src/test/resources/filtered/mesh.yaml index ce6c786c..e06315cd 100644 --- a/core/src/test/resources/filtered/mesh.yaml +++ b/core/src/test/resources/filtered/mesh.yaml @@ -14,7 +14,7 @@ descriptors: repository-volume: {} containers: sink: - image: "ghcr.io/streamx-com/streamx-blueprints/web-server-sink:3.0.7-native" + image: "ghcr.io/streamx-com/streamx-blueprints/web-server-sink:3.0.7-jvm" incoming: resources: {} servicePorts: From ec213a64df269ed07d41172c0314fe8c81ec2e76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Mon, 5 Jan 2026 10:21:55 +0100 Subject: [PATCH 23/36] Minor fixes --- .../dev/streamx/cli/StreamxTerminalCommandProducer.java | 6 +++++- .../cli/test/tools/terminal/OsTerminalCommandRunner.java | 2 +- .../test/java/dev/streamx/cli/StreamxCliPublicationIT.java | 1 - 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/e2e-tests/src/main/java/dev/streamx/cli/StreamxTerminalCommandProducer.java b/e2e-tests/src/main/java/dev/streamx/cli/StreamxTerminalCommandProducer.java index 082e099c..c24370ce 100644 --- a/e2e-tests/src/main/java/dev/streamx/cli/StreamxTerminalCommandProducer.java +++ b/e2e-tests/src/main/java/dev/streamx/cli/StreamxTerminalCommandProducer.java @@ -1,6 +1,7 @@ package dev.streamx.cli; import java.io.File; +import java.nio.file.Path; import java.util.regex.Pattern; import java.util.stream.Stream; @@ -39,7 +40,10 @@ private String findStreamxJar() { .filter(File::exists) .map(File::listFiles) .flatMap(Stream::of) - .map(File::getAbsolutePath) + .map(File::toPath) + .map(Path::toAbsolutePath) + .map(Path::normalize) + .map(Path::toString) .filter(p -> pattern.matcher(p.replace("\\", "/")).matches()) .findFirst() .orElseThrow(() -> new RuntimeException("Could not find streamx Jar")); diff --git a/e2e-tests/src/main/java/dev/streamx/cli/test/tools/terminal/OsTerminalCommandRunner.java b/e2e-tests/src/main/java/dev/streamx/cli/test/tools/terminal/OsTerminalCommandRunner.java index 30a2a335..390434e1 100644 --- a/e2e-tests/src/main/java/dev/streamx/cli/test/tools/terminal/OsTerminalCommandRunner.java +++ b/e2e-tests/src/main/java/dev/streamx/cli/test/tools/terminal/OsTerminalCommandRunner.java @@ -25,7 +25,6 @@ public OsTerminalCommandRunner() { } public ShellProcess run(String command) { - logger.info("Running terminal command: " + command); ProcessBuilder processBuilder = osCommand.create(command); try { ShellProcess shellProcess = ShellProcess.run(processBuilder); @@ -33,6 +32,7 @@ public ShellProcess run(String command) { processes.add(shellProcess); return shellProcess; } catch (IOException e) { + logger.info("Error running terminal command: " + command); throw new RuntimeException(e); } } diff --git a/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliPublicationIT.java b/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliPublicationIT.java index cca684ef..1d398963 100644 --- a/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliPublicationIT.java +++ b/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliPublicationIT.java @@ -14,7 +14,6 @@ import java.nio.file.Path; import java.time.Duration; import org.eclipse.microprofile.config.inject.ConfigProperty; -import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; From c6eba111bd6c3fcd26c846711a9b384be585373e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Mon, 5 Jan 2026 10:45:59 +0100 Subject: [PATCH 24/36] add debug logging --- .../streamx/cli/StreamxCliPublicationIT.java | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliPublicationIT.java b/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliPublicationIT.java index 1d398963..0bb505ab 100644 --- a/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliPublicationIT.java +++ b/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliPublicationIT.java @@ -1,6 +1,7 @@ package dev.streamx.cli; import static dev.streamx.cli.test.tools.ResourcePathResolver.absolutePath; +import static org.assertj.core.api.Assertions.fail; import dev.streamx.cli.test.tools.terminal.TerminalCommandRunner; import dev.streamx.cli.test.tools.terminal.process.ShellProcess; @@ -10,10 +11,14 @@ import jakarta.inject.Inject; import jakarta.inject.Named; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.jboss.logging.Logger; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @@ -67,7 +72,19 @@ public void shouldPublishAndUnpublishPageUsingStreamOperation() { "Sent com.streamx.blueprints.page.published.v1 event using stream with key 'hello.html'" ); - validateStreamxPage("hello.html", "Hello World!"); + // TODO remove the catch + try { + validateStreamxPage("hello.html", "Hello World!"); + } catch (Throwable t) { + String dockerPsOutput = readProcessOutput("docker ps"); + String containerLine = dockerPsOutput.lines() + .filter(line -> line.contains("web-server-sink")) + .findFirst().orElseThrow(); + String containerId = StringUtils.substringBefore(containerLine, " "); + String logs = readProcessOutput("docker logs " + containerId); + Logger log = Logger.getLogger(StreamxCliPublicationIT.class); + log.info("\n---- DOCKER CONTAINER LOGS ---\n" + logs); + } runStreamxIngestionCommand( "stream_v2", @@ -78,6 +95,18 @@ public void shouldPublishAndUnpublishPageUsingStreamOperation() { validateStreamxPageNotAvailable("hello.html"); } + private static String readProcessOutput(String command) { + String[] words = command.split(" "); + ProcessBuilder builder = new ProcessBuilder(words); + builder.redirectErrorStream(true); // merge STDOUT + STDERR + try { + Process process = builder.start(); + return IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8); + } catch (IOException ex) { + return fail("Error reading output of command", ex); + } + } + @Test public void shouldPublishAndUnpublishPageUsingBatchOperation() { runStreamxIngestionCommand( From cd8b68461efa54365b8330758ff68df28c839cd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Mon, 5 Jan 2026 11:08:37 +0100 Subject: [PATCH 25/36] Remove repository volume --- core/src/test/resources/filtered/mesh-interpolated.yaml | 4 ---- core/src/test/resources/filtered/mesh.yaml | 4 ---- e2e-tests/src/test/resources/filtered/mesh.yaml | 4 ---- 3 files changed, 12 deletions(-) diff --git a/core/src/test/resources/filtered/mesh-interpolated.yaml b/core/src/test/resources/filtered/mesh-interpolated.yaml index 9dd776a4..3e10157e 100644 --- a/core/src/test/resources/filtered/mesh-interpolated.yaml +++ b/core/src/test/resources/filtered/mesh-interpolated.yaml @@ -19,8 +19,6 @@ descriptors: resources: {} servicePorts: - "8087:8080" - volumes: - - "repository-volume:/srv/www" sources: cli: @@ -57,5 +55,3 @@ edge: incoming: resources: ref: outbox.pages - environment: - STREAMX_BLUEPRINTS_WEB_SERVER_SINK_STORAGE_ROOT_DIRECTORY: "/srv/www" \ No newline at end of file diff --git a/core/src/test/resources/filtered/mesh.yaml b/core/src/test/resources/filtered/mesh.yaml index e06315cd..8801f750 100644 --- a/core/src/test/resources/filtered/mesh.yaml +++ b/core/src/test/resources/filtered/mesh.yaml @@ -19,8 +19,6 @@ descriptors: resources: {} servicePorts: - "8087:8080" - volumes: - - "repository-volume:/srv/www" sources: cli: @@ -57,5 +55,3 @@ edge: incoming: resources: ref: outbox.pages - environment: - STREAMX_BLUEPRINTS_WEB_SERVER_SINK_STORAGE_ROOT_DIRECTORY: "/srv/www" \ No newline at end of file diff --git a/e2e-tests/src/test/resources/filtered/mesh.yaml b/e2e-tests/src/test/resources/filtered/mesh.yaml index b11638e4..fb1774f9 100644 --- a/e2e-tests/src/test/resources/filtered/mesh.yaml +++ b/e2e-tests/src/test/resources/filtered/mesh.yaml @@ -19,8 +19,6 @@ descriptors: resources: {} servicePorts: - "8087:8080" - volumes: - - "repository-volume:/srv/www" sources: cli: @@ -72,5 +70,3 @@ edge: incoming: resources: ref: outbox.pages,outbox.assets - environment: - STREAMX_BLUEPRINTS_WEB_SERVER_SINK_STORAGE_ROOT_DIRECTORY: "/srv/www" \ No newline at end of file From 976a3d64b3c7af9c6af78946c9b3929077b7c890 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Mon, 5 Jan 2026 11:21:32 +0100 Subject: [PATCH 26/36] Volumes cleanup --- .../resources/filtered/mesh-interpolated.yaml | 2 -- core/src/test/resources/filtered/mesh.yaml | 2 -- .../batch/publish/index.html | 1 - .../repository-volume/publish/index.html | 1 - .../streamx/cli/StreamxCliPublicationIT.java | 31 +------------------ .../src/test/resources/filtered/mesh.yaml | 2 -- 6 files changed, 1 insertion(+), 38 deletions(-) delete mode 100644 e2e-tests/repository/web-server-sink/repository-volume/batch/publish/index.html delete mode 100644 e2e-tests/repository/web-server-sink/repository-volume/publish/index.html diff --git a/core/src/test/resources/filtered/mesh-interpolated.yaml b/core/src/test/resources/filtered/mesh-interpolated.yaml index 3e10157e..7f8ea1f6 100644 --- a/core/src/test/resources/filtered/mesh-interpolated.yaml +++ b/core/src/test/resources/filtered/mesh-interpolated.yaml @@ -10,8 +10,6 @@ descriptors: web-server-sink: type: "edge" - volumes: - repository-volume: {} containers: sink: image: "${config.image.interpolated}" diff --git a/core/src/test/resources/filtered/mesh.yaml b/core/src/test/resources/filtered/mesh.yaml index 8801f750..0e07ab37 100644 --- a/core/src/test/resources/filtered/mesh.yaml +++ b/core/src/test/resources/filtered/mesh.yaml @@ -10,8 +10,6 @@ descriptors: web-server-sink: type: "edge" - volumes: - repository-volume: {} containers: sink: image: "ghcr.io/streamx-com/streamx-blueprints/web-server-sink:3.0.7-jvm" diff --git a/e2e-tests/repository/web-server-sink/repository-volume/batch/publish/index.html b/e2e-tests/repository/web-server-sink/repository-volume/batch/publish/index.html deleted file mode 100644 index 9a1eb8d2..00000000 --- a/e2e-tests/repository/web-server-sink/repository-volume/batch/publish/index.html +++ /dev/null @@ -1 +0,0 @@ -

Hello World

\ No newline at end of file diff --git a/e2e-tests/repository/web-server-sink/repository-volume/publish/index.html b/e2e-tests/repository/web-server-sink/repository-volume/publish/index.html deleted file mode 100644 index 9a1eb8d2..00000000 --- a/e2e-tests/repository/web-server-sink/repository-volume/publish/index.html +++ /dev/null @@ -1 +0,0 @@ -

Hello World

\ No newline at end of file diff --git a/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliPublicationIT.java b/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliPublicationIT.java index 0bb505ab..1d398963 100644 --- a/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliPublicationIT.java +++ b/e2e-tests/src/test/java/dev/streamx/cli/StreamxCliPublicationIT.java @@ -1,7 +1,6 @@ package dev.streamx.cli; import static dev.streamx.cli.test.tools.ResourcePathResolver.absolutePath; -import static org.assertj.core.api.Assertions.fail; import dev.streamx.cli.test.tools.terminal.TerminalCommandRunner; import dev.streamx.cli.test.tools.terminal.process.ShellProcess; @@ -11,14 +10,10 @@ import jakarta.inject.Inject; import jakarta.inject.Named; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; import org.eclipse.microprofile.config.inject.ConfigProperty; -import org.jboss.logging.Logger; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @@ -72,19 +67,7 @@ public void shouldPublishAndUnpublishPageUsingStreamOperation() { "Sent com.streamx.blueprints.page.published.v1 event using stream with key 'hello.html'" ); - // TODO remove the catch - try { - validateStreamxPage("hello.html", "Hello World!"); - } catch (Throwable t) { - String dockerPsOutput = readProcessOutput("docker ps"); - String containerLine = dockerPsOutput.lines() - .filter(line -> line.contains("web-server-sink")) - .findFirst().orElseThrow(); - String containerId = StringUtils.substringBefore(containerLine, " "); - String logs = readProcessOutput("docker logs " + containerId); - Logger log = Logger.getLogger(StreamxCliPublicationIT.class); - log.info("\n---- DOCKER CONTAINER LOGS ---\n" + logs); - } + validateStreamxPage("hello.html", "Hello World!"); runStreamxIngestionCommand( "stream_v2", @@ -95,18 +78,6 @@ public void shouldPublishAndUnpublishPageUsingStreamOperation() { validateStreamxPageNotAvailable("hello.html"); } - private static String readProcessOutput(String command) { - String[] words = command.split(" "); - ProcessBuilder builder = new ProcessBuilder(words); - builder.redirectErrorStream(true); // merge STDOUT + STDERR - try { - Process process = builder.start(); - return IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8); - } catch (IOException ex) { - return fail("Error reading output of command", ex); - } - } - @Test public void shouldPublishAndUnpublishPageUsingBatchOperation() { runStreamxIngestionCommand( diff --git a/e2e-tests/src/test/resources/filtered/mesh.yaml b/e2e-tests/src/test/resources/filtered/mesh.yaml index fb1774f9..b163ea17 100644 --- a/e2e-tests/src/test/resources/filtered/mesh.yaml +++ b/e2e-tests/src/test/resources/filtered/mesh.yaml @@ -10,8 +10,6 @@ descriptors: web-server-sink: type: "edge" - volumes: - repository-volume: {} containers: sink: image: "ghcr.io/streamx-com/streamx-blueprints/web-server-sink:3.0.7-jvm" From 0158222e435650920d437c2ec493b73c9e5c2406 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Mon, 5 Jan 2026 11:26:27 +0100 Subject: [PATCH 27/36] Temporarily allow releasing preview on demand --- .github/workflows/release-preview.yaml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release-preview.yaml b/.github/workflows/release-preview.yaml index a383f8c6..d18dceea 100644 --- a/.github/workflows/release-preview.yaml +++ b/.github/workflows/release-preview.yaml @@ -1,10 +1,12 @@ name: 'Release: new CLI preview version using JReleaser' on: - workflow_run: - workflows: ["Test: CI build"] - types: - - completed + # Temporarily allow running on demand. TODO: Restore workflow_run, remove workflow_dispatch + workflow_dispatch: + # workflow_run: + # workflows: ["Test: CI build"] + # types: + # - completed permissions: id-token: write @@ -13,7 +15,8 @@ permissions: jobs: release: runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.head_branch == 'main' }} + # Temporarily allow running on demand. TODO: restore this condition + # if: ${{ github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.head_branch == 'main' }} steps: - name: Checkout uses: actions/checkout@v4 From 58337de2d73ae905613911de36119096ef5a58a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Mon, 5 Jan 2026 12:03:13 +0100 Subject: [PATCH 28/36] Temporarily embed releasing preview on demand into PR CI --- .github/workflows/ci-test-build.yaml | 24 +++++++++++++++++++++++- .github/workflows/release-preview.yaml | 13 +++++-------- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci-test-build.yaml b/.github/workflows/ci-test-build.yaml index 1a22a2f9..c671d4e4 100644 --- a/.github/workflows/ci-test-build.yaml +++ b/.github/workflows/ci-test-build.yaml @@ -8,7 +8,8 @@ on: permissions: id-token: write - contents: read + # TODO restore to "contents: read" after removing the release steps + contents: write jobs: linux-test-build: @@ -37,6 +38,27 @@ jobs: run: | ./mvnw clean verify -P all-tests +# TODO remove the release preview section + - name: Configure Git + run: | + git config --global user.email "github.actions@streamx.dev" + git config --global user.name "${GITHUB_ACTOR}" + mkdir -p ~/.ssh + printf '%s\n' "${{ secrets.SSH_SECRET_KEY }}" > ~/.ssh/id_rsa + chmod 0600 ~/.ssh/id_rsa + + - name: Preview release prepare + run: ./mvnw -B clean install -DskipTests -P preview + + - name: Preview release perform + env: + JRELEASER_GITHUB_TOKEN: ${{ secrets.JRELEASER_PREVIEW_GITHUB_TOKEN }} + JRELEASER_HOMEBREW_GITHUB_TOKEN: ${{ secrets.JRELEASER_HOMEBREW_PREVIEW_GITHUB_TOKEN }} + JRELEASER_SCOOP_GITHUB_TOKEN: ${{ secrets.JRELEASER_SCOOP_PREVIEW_GITHUB_TOKEN }} + run: | + ./mvnw -pl distribution jreleaser:assemble jreleaser:full-release -P preview +# TODO end + windows-test-build: runs-on: windows-latest steps: diff --git a/.github/workflows/release-preview.yaml b/.github/workflows/release-preview.yaml index d18dceea..a383f8c6 100644 --- a/.github/workflows/release-preview.yaml +++ b/.github/workflows/release-preview.yaml @@ -1,12 +1,10 @@ name: 'Release: new CLI preview version using JReleaser' on: - # Temporarily allow running on demand. TODO: Restore workflow_run, remove workflow_dispatch - workflow_dispatch: - # workflow_run: - # workflows: ["Test: CI build"] - # types: - # - completed + workflow_run: + workflows: ["Test: CI build"] + types: + - completed permissions: id-token: write @@ -15,8 +13,7 @@ permissions: jobs: release: runs-on: ubuntu-latest - # Temporarily allow running on demand. TODO: restore this condition - # if: ${{ github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.head_branch == 'main' }} + if: ${{ github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.head_branch == 'main' }} steps: - name: Checkout uses: actions/checkout@v4 From 8e1100e463c14e53abbddd2479c459065ccb5457 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Mon, 5 Jan 2026 12:17:31 +0100 Subject: [PATCH 29/36] Make release preview to use specific branch --- .github/workflows/ci-test-build.yaml | 11 ++++------- distribution/src/main/resources/jreleaser-preview.yml | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-test-build.yaml b/.github/workflows/ci-test-build.yaml index c671d4e4..3aa053b3 100644 --- a/.github/workflows/ci-test-build.yaml +++ b/.github/workflows/ci-test-build.yaml @@ -8,7 +8,6 @@ on: permissions: id-token: write - # TODO restore to "contents: read" after removing the release steps contents: write jobs: @@ -34,11 +33,6 @@ jobs: distribution: 'temurin' cache: 'maven' - - name: Build project - run: | - ./mvnw clean verify -P all-tests - -# TODO remove the release preview section - name: Configure Git run: | git config --global user.email "github.actions@streamx.dev" @@ -47,6 +41,10 @@ jobs: printf '%s\n' "${{ secrets.SSH_SECRET_KEY }}" > ~/.ssh/id_rsa chmod 0600 ~/.ssh/id_rsa + - name: Build project + run: | + ./mvnw clean verify -P all-tests + - name: Preview release prepare run: ./mvnw -B clean install -DskipTests -P preview @@ -57,7 +55,6 @@ jobs: JRELEASER_SCOOP_GITHUB_TOKEN: ${{ secrets.JRELEASER_SCOOP_PREVIEW_GITHUB_TOKEN }} run: | ./mvnw -pl distribution jreleaser:assemble jreleaser:full-release -P preview -# TODO end windows-test-build: runs-on: windows-latest diff --git a/distribution/src/main/resources/jreleaser-preview.yml b/distribution/src/main/resources/jreleaser-preview.yml index b815a80c..c83bf878 100644 --- a/distribution/src/main/resources/jreleaser-preview.yml +++ b/distribution/src/main/resources/jreleaser-preview.yml @@ -28,7 +28,7 @@ release: owner: streamx-dev name: streamx-cli-preview tagName: "${git.commit.id.describe}" - branch: main + branch: DXP-2486-add-run-stream-and-batch-commands-for-streamx-2.0 overwrite: false update: enabled: true From 3ba20b4e2e2d647c6d8747e5cd108fd1c8228343 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Mon, 5 Jan 2026 12:37:10 +0100 Subject: [PATCH 30/36] Restore recent changes regarding preview releasing --- .github/workflows/ci-test-build.yaml | 21 +------------------ .../src/main/resources/jreleaser-preview.yml | 2 +- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci-test-build.yaml b/.github/workflows/ci-test-build.yaml index 3aa053b3..1a22a2f9 100644 --- a/.github/workflows/ci-test-build.yaml +++ b/.github/workflows/ci-test-build.yaml @@ -8,7 +8,7 @@ on: permissions: id-token: write - contents: write + contents: read jobs: linux-test-build: @@ -33,29 +33,10 @@ jobs: distribution: 'temurin' cache: 'maven' - - name: Configure Git - run: | - git config --global user.email "github.actions@streamx.dev" - git config --global user.name "${GITHUB_ACTOR}" - mkdir -p ~/.ssh - printf '%s\n' "${{ secrets.SSH_SECRET_KEY }}" > ~/.ssh/id_rsa - chmod 0600 ~/.ssh/id_rsa - - name: Build project run: | ./mvnw clean verify -P all-tests - - name: Preview release prepare - run: ./mvnw -B clean install -DskipTests -P preview - - - name: Preview release perform - env: - JRELEASER_GITHUB_TOKEN: ${{ secrets.JRELEASER_PREVIEW_GITHUB_TOKEN }} - JRELEASER_HOMEBREW_GITHUB_TOKEN: ${{ secrets.JRELEASER_HOMEBREW_PREVIEW_GITHUB_TOKEN }} - JRELEASER_SCOOP_GITHUB_TOKEN: ${{ secrets.JRELEASER_SCOOP_PREVIEW_GITHUB_TOKEN }} - run: | - ./mvnw -pl distribution jreleaser:assemble jreleaser:full-release -P preview - windows-test-build: runs-on: windows-latest steps: diff --git a/distribution/src/main/resources/jreleaser-preview.yml b/distribution/src/main/resources/jreleaser-preview.yml index c83bf878..b815a80c 100644 --- a/distribution/src/main/resources/jreleaser-preview.yml +++ b/distribution/src/main/resources/jreleaser-preview.yml @@ -28,7 +28,7 @@ release: owner: streamx-dev name: streamx-cli-preview tagName: "${git.commit.id.describe}" - branch: DXP-2486-add-run-stream-and-batch-commands-for-streamx-2.0 + branch: main overwrite: false update: enabled: true From ee83d919fba075e436ee78bd2107498b22cf3af2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Mon, 12 Jan 2026 17:21:04 +0100 Subject: [PATCH 31/36] Rename json.paths.to.encode.to.base64 to json.fields.as-base64 --- .../ingestion/stream/StreamCommand.java | 6 +- .../ingestion/stream/StreamProperties.java | 10 +-- .../stream/parser/JsonBase64Encoder.java | 19 ++--- .../stream/parser/JsonBase64EncoderTest.java | 78 +++++++++++++++++++ .../ingestion/stream/.stream.properties | 2 +- .../test/resources/stream/.stream.properties | 2 +- 6 files changed, 98 insertions(+), 19 deletions(-) create mode 100644 core/src/test/java/dev/streamx/cli/command/ingestion/stream/parser/JsonBase64EncoderTest.java diff --git a/core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamCommand.java b/core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamCommand.java index 9a973633..74ec9864 100644 --- a/core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamCommand.java +++ b/core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamCommand.java @@ -47,13 +47,13 @@ public class StreamCommand extends BaseIngestionCommand { @Override protected void perform(Publisher publisher) throws StreamxClientException { Path streamFile = Paths.get(streamIngestionArguments.getSourceFile()); - List jsonPathsToEncodeToBase64 = StreamProperties - .getJsonPathsToEncodeToBase64(streamFile); + List jsonFieldsToEncodeToBase64 = StreamProperties + .getJsonFieldsToEncodeToBase64(streamFile); try (FileInputStream fis = new FileInputStream(streamFile.toFile())) { ingestionJsonParser.parse(fis, cloudEventNode -> { - JsonBase64Encoder.encodeFields(cloudEventNode, jsonPathsToEncodeToBase64); + JsonBase64Encoder.encodeFields(cloudEventNode, jsonFieldsToEncodeToBase64); CloudEvent inputEvent = toCloudEvent(cloudEventNode); CloudEvent responseEvent = publisher.send(inputEvent); printf("Sent %s event using stream with key '%s' at %s%n", diff --git a/core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamProperties.java b/core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamProperties.java index e0a7509b..a98d5c96 100644 --- a/core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamProperties.java +++ b/core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamProperties.java @@ -15,21 +15,21 @@ class StreamProperties { private static final Logger logger = Logger.getLogger(StreamProperties.class); private static final String PROPERTIES_FILE_NAME = ".stream.properties"; - private static final String BASE64_PATHS_PROPERTY_NAME = "json.paths.to.encode.to.base64"; + private static final String JSON_FIELDS_AS_BASE64 = "json.fields.as-base64"; private StreamProperties() { // no instances } - public static List getJsonPathsToEncodeToBase64(Path streamFile) { + public static List getJsonFieldsToEncodeToBase64(Path streamFile) { Path propertiesFile = streamFile.resolveSibling(PROPERTIES_FILE_NAME); if (Files.exists(propertiesFile)) { try (InputStream fileStream = Files.newInputStream(propertiesFile)) { Properties properties = new Properties(); properties.load(fileStream); - String paths = properties.getProperty(BASE64_PATHS_PROPERTY_NAME); - if (StringUtils.isNotBlank(paths)) { - return Arrays.stream(paths.split(",")) + String fields = properties.getProperty(JSON_FIELDS_AS_BASE64); + if (StringUtils.isNotBlank(fields)) { + return Arrays.stream(fields.split(",")) .map(String::trim) .filter(StringUtils::isNotEmpty) .toList(); diff --git a/core/src/main/java/dev/streamx/cli/command/ingestion/stream/parser/JsonBase64Encoder.java b/core/src/main/java/dev/streamx/cli/command/ingestion/stream/parser/JsonBase64Encoder.java index 01d84b2d..a6906d1d 100644 --- a/core/src/main/java/dev/streamx/cli/command/ingestion/stream/parser/JsonBase64Encoder.java +++ b/core/src/main/java/dev/streamx/cli/command/ingestion/stream/parser/JsonBase64Encoder.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.node.TextNode; import java.util.Base64; import java.util.List; +import org.apache.commons.lang3.StringUtils; public class JsonBase64Encoder { @@ -12,16 +13,17 @@ private JsonBase64Encoder() { // no instances } - public static void encodeFields(JsonNode root, List jsonPaths) { - for (String jsonPath : jsonPaths) { - JsonNode node = root.at(jsonPath); + public static void encodeFields(JsonNode root, List fieldPaths) { + for (String fieldPath : fieldPaths) { + fieldPath = StringUtils.prependIfMissing(fieldPath, "/"); + JsonNode node = root.at(fieldPath); if (node.isMissingNode() || !node.isTextual()) { continue; } TextNode encodedTextNode = createTextNodeWithEncodedContent(node); - replaceNode(root, jsonPath, encodedTextNode); + replaceNode(root, fieldPath, encodedTextNode); } } @@ -30,12 +32,11 @@ private static TextNode createTextNodeWithEncodedContent(JsonNode node) { return TextNode.valueOf(encodedText); } - private static void replaceNode(JsonNode root, String jsonPath, JsonNode newValue) { - int lastSlash = jsonPath.lastIndexOf('/'); - String parentNodePath = jsonPath.substring(0, lastSlash); - String fieldName = jsonPath.substring(lastSlash + 1); + private static void replaceNode(JsonNode root, String fieldPath, JsonNode newValue) { + String parentPath = StringUtils.substringBeforeLast(fieldPath, "/"); + String fieldName = StringUtils.substringAfterLast(fieldPath, "/"); - JsonNode parentNode = root.at(parentNodePath); + JsonNode parentNode = root.at(parentPath); if (parentNode instanceof ObjectNode obj) { obj.set(fieldName, newValue); } diff --git a/core/src/test/java/dev/streamx/cli/command/ingestion/stream/parser/JsonBase64EncoderTest.java b/core/src/test/java/dev/streamx/cli/command/ingestion/stream/parser/JsonBase64EncoderTest.java new file mode 100644 index 00000000..2fef08b4 --- /dev/null +++ b/core/src/test/java/dev/streamx/cli/command/ingestion/stream/parser/JsonBase64EncoderTest.java @@ -0,0 +1,78 @@ +package dev.streamx.cli.command.ingestion.stream.parser; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.params.provider.Arguments.arguments; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class JsonBase64EncoderTest { + + private static final ObjectMapper objectMapper = new ObjectMapper(); + public static final String INITIAL_JSON = """ + { + "foo" : { + "bar" : "foobar" + }, + "foo2" : "bar2" + }"""; + + static Stream testArguments() { + return Stream.of( + // happy path + arguments( + INITIAL_JSON, + "/foo/bar", + INITIAL_JSON.replace("foobar", "Zm9vYmFy") + ), + + // missing leading slash in the jsonFieldsAsBase64 should be automatically added + arguments( + INITIAL_JSON, + "foo/bar", + INITIAL_JSON.replace("foobar", "Zm9vYmFy") + ), + + // attempting to encode a parent node that is not a text node + arguments( + INITIAL_JSON, + "/foo", + INITIAL_JSON + ), + + // json doesn't contain the requested field + arguments( + INITIAL_JSON, + "bar", + INITIAL_JSON + ), + + // invalid path of field + arguments( + INITIAL_JSON, + "/a/)#Q(*%Y@W$Tbngow23p0t[';'\\325/b", + INITIAL_JSON + ) + ); + } + + @ParameterizedTest + @MethodSource("testArguments") + void shouldEncodeFields(String sourceJson, String jsonFieldsAsBase64, String expectedResultJson) + throws Exception { + // given + JsonNode jsonNode = objectMapper.readTree(sourceJson); + + // when + JsonBase64Encoder.encodeFields(jsonNode, List.of(jsonFieldsAsBase64)); + + // then + assertThat(jsonNode.toPrettyString()).isEqualTo(expectedResultJson); + } + +} \ No newline at end of file diff --git a/core/src/test/resources/dev/streamx/cli/command/ingestion/stream/.stream.properties b/core/src/test/resources/dev/streamx/cli/command/ingestion/stream/.stream.properties index c872efe1..15b64ec4 100644 --- a/core/src/test/resources/dev/streamx/cli/command/ingestion/stream/.stream.properties +++ b/core/src/test/resources/dev/streamx/cli/command/ingestion/stream/.stream.properties @@ -1 +1 @@ -json.paths.to.encode.to.base64=/data/content \ No newline at end of file +json.fields.as-base64=/data/content \ No newline at end of file diff --git a/e2e-tests/src/test/resources/stream/.stream.properties b/e2e-tests/src/test/resources/stream/.stream.properties index c872efe1..15b64ec4 100644 --- a/e2e-tests/src/test/resources/stream/.stream.properties +++ b/e2e-tests/src/test/resources/stream/.stream.properties @@ -1 +1 @@ -json.paths.to.encode.to.base64=/data/content \ No newline at end of file +json.fields.as-base64=/data/content \ No newline at end of file From a0b9cc6ec986a7784a4afacb502cb80c689c5fe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Wed, 14 Jan 2026 11:23:51 +0100 Subject: [PATCH 32/36] Formatting agnostic verification --- .../ingestion/stream/parser/JsonBase64EncoderTest.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/core/src/test/java/dev/streamx/cli/command/ingestion/stream/parser/JsonBase64EncoderTest.java b/core/src/test/java/dev/streamx/cli/command/ingestion/stream/parser/JsonBase64EncoderTest.java index 2fef08b4..0f3e0a07 100644 --- a/core/src/test/java/dev/streamx/cli/command/ingestion/stream/parser/JsonBase64EncoderTest.java +++ b/core/src/test/java/dev/streamx/cli/command/ingestion/stream/parser/JsonBase64EncoderTest.java @@ -16,10 +16,10 @@ class JsonBase64EncoderTest { private static final ObjectMapper objectMapper = new ObjectMapper(); public static final String INITIAL_JSON = """ { - "foo" : { - "bar" : "foobar" + "foo": { + "bar": "foobar" }, - "foo2" : "bar2" + "foo2": "bar2" }"""; static Stream testArguments() { @@ -72,7 +72,8 @@ void shouldEncodeFields(String sourceJson, String jsonFieldsAsBase64, String exp JsonBase64Encoder.encodeFields(jsonNode, List.of(jsonFieldsAsBase64)); // then - assertThat(jsonNode.toPrettyString()).isEqualTo(expectedResultJson); + JsonNode expectedNode = objectMapper.readTree(expectedResultJson); + assertThat(jsonNode.toPrettyString()).isEqualTo(expectedNode.toPrettyString()); } } \ No newline at end of file From 09e184b7cab228c6b65c8b178d05c4039fc101fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Wed, 11 Feb 2026 10:10:48 +0100 Subject: [PATCH 33/36] Bump streamx-core version to the one that has GatewayRunner --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 060e1317..8ddac26c 100644 --- a/pom.xml +++ b/pom.xml @@ -41,7 +41,7 @@ io.quarkus.platform 3.18.3 - 2.0.8 + 2.0.11 2.9.0 3.27.3 3.12.1 From 05e10d477555e2d8b85b94b02d59d2b53479d6bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Wed, 11 Feb 2026 10:32:05 +0100 Subject: [PATCH 34/36] Remove interpolation --- .../MeshDefinitionResolver.java | 2 - .../cli/interpolation/Interpolating.java | 16 -- .../InterpolatingYamlFactory.java | 47 ---- .../InterpolatingYamlParser.java | 43 ---- .../interpolation/InterpolationSupport.java | 60 ----- .../interpolation/ObjectMapperProducer.java | 8 +- ...shDefinitionResolverInterpolationTest.java | 78 ------- .../InterpolatingMapperTest.java | 208 ------------------ .../InterpolationSupportTest.java | 144 ------------ .../src/test/resources/application.properties | 5 - .../resources/filtered/mesh-interpolated.yaml | 55 ----- e2e-tests/.env | 2 - 12 files changed, 2 insertions(+), 666 deletions(-) delete mode 100644 core/src/main/java/dev/streamx/cli/interpolation/Interpolating.java delete mode 100644 core/src/main/java/dev/streamx/cli/interpolation/InterpolatingYamlFactory.java delete mode 100644 core/src/main/java/dev/streamx/cli/interpolation/InterpolatingYamlParser.java delete mode 100644 core/src/main/java/dev/streamx/cli/interpolation/InterpolationSupport.java delete mode 100644 core/src/test/java/dev/streamx/cli/command/meshprocessing/MeshDefinitionResolverInterpolationTest.java delete mode 100644 core/src/test/java/dev/streamx/cli/interpolation/InterpolatingMapperTest.java delete mode 100644 core/src/test/java/dev/streamx/cli/interpolation/InterpolationSupportTest.java delete mode 100644 core/src/test/resources/filtered/mesh-interpolated.yaml delete mode 100644 e2e-tests/.env diff --git a/core/src/main/java/dev/streamx/cli/command/meshprocessing/MeshDefinitionResolver.java b/core/src/main/java/dev/streamx/cli/command/meshprocessing/MeshDefinitionResolver.java index 9090474d..fa66e4cb 100644 --- a/core/src/main/java/dev/streamx/cli/command/meshprocessing/MeshDefinitionResolver.java +++ b/core/src/main/java/dev/streamx/cli/command/meshprocessing/MeshDefinitionResolver.java @@ -2,7 +2,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.streamx.mesh.model.ServiceMesh; -import dev.streamx.cli.interpolation.Interpolating; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import java.io.IOException; @@ -12,7 +11,6 @@ public class MeshDefinitionResolver { @Inject - @Interpolating ObjectMapper objectMapper; public ServiceMesh resolve(Path meshPath) throws IOException { diff --git a/core/src/main/java/dev/streamx/cli/interpolation/Interpolating.java b/core/src/main/java/dev/streamx/cli/interpolation/Interpolating.java deleted file mode 100644 index 41c9b1f7..00000000 --- a/core/src/main/java/dev/streamx/cli/interpolation/Interpolating.java +++ /dev/null @@ -1,16 +0,0 @@ -package dev.streamx.cli.interpolation; - -import static java.lang.annotation.ElementType.FIELD; -import static java.lang.annotation.ElementType.METHOD; -import static java.lang.annotation.ElementType.PARAMETER; -import static java.lang.annotation.ElementType.TYPE; -import static java.lang.annotation.RetentionPolicy.RUNTIME; - -import jakarta.inject.Qualifier; -import java.lang.annotation.Retention; -import java.lang.annotation.Target; - -@Qualifier -@Retention(RUNTIME) -@Target({TYPE, METHOD, FIELD, PARAMETER}) -public @interface Interpolating {} diff --git a/core/src/main/java/dev/streamx/cli/interpolation/InterpolatingYamlFactory.java b/core/src/main/java/dev/streamx/cli/interpolation/InterpolatingYamlFactory.java deleted file mode 100644 index be5fb3d1..00000000 --- a/core/src/main/java/dev/streamx/cli/interpolation/InterpolatingYamlFactory.java +++ /dev/null @@ -1,47 +0,0 @@ -package dev.streamx.cli.interpolation; - -import com.fasterxml.jackson.core.io.IOContext; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import com.fasterxml.jackson.dataformat.yaml.YAMLParser; -import java.io.CharArrayReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.Reader; - -public class InterpolatingYamlFactory extends YAMLFactory { - - private final InterpolationSupport interpolationSupport; - - public InterpolatingYamlFactory(InterpolationSupport interpolationSupport) { - this.interpolationSupport = interpolationSupport; - } - - @Override - protected YAMLParser _createParser(InputStream in, IOContext context) throws IOException { - return new InterpolatingYamlParser(context, this._parserFeatures, this._yamlParserFeatures, - this._loaderOptions, this._objectCodec, this._createReader(in, null, context), - interpolationSupport); - } - - @Override - protected YAMLParser _createParser(Reader r, IOContext context) { - return new InterpolatingYamlParser(context, this._parserFeatures, this._yamlParserFeatures, - this._loaderOptions, this._objectCodec, r, interpolationSupport); - } - - @Override - protected YAMLParser _createParser(char[] data, int offset, int len, IOContext context, - boolean recyclable) { - return new InterpolatingYamlParser(context, this._parserFeatures, this._yamlParserFeatures, - this._loaderOptions, this._objectCodec, new CharArrayReader(data, offset, len), - interpolationSupport); - } - - @Override - protected YAMLParser _createParser(byte[] data, int offset, int len, IOContext context) - throws IOException { - return new InterpolatingYamlParser(context, this._parserFeatures, this._yamlParserFeatures, - this._loaderOptions, this._objectCodec, - this._createReader(data, offset, len, null, context), interpolationSupport); - } -} diff --git a/core/src/main/java/dev/streamx/cli/interpolation/InterpolatingYamlParser.java b/core/src/main/java/dev/streamx/cli/interpolation/InterpolatingYamlParser.java deleted file mode 100644 index def7af71..00000000 --- a/core/src/main/java/dev/streamx/cli/interpolation/InterpolatingYamlParser.java +++ /dev/null @@ -1,43 +0,0 @@ -package dev.streamx.cli.interpolation; - -import com.fasterxml.jackson.core.ObjectCodec; -import com.fasterxml.jackson.core.io.IOContext; -import com.fasterxml.jackson.dataformat.yaml.YAMLParser; -import java.io.IOException; -import java.io.Reader; -import org.yaml.snakeyaml.LoaderOptions; - -public class InterpolatingYamlParser extends YAMLParser { - - private final InterpolationSupport interpolationSupport; - - public InterpolatingYamlParser(IOContext context, int parserFeatures, int formatFeatures, - LoaderOptions loaderOptions, ObjectCodec codec, Reader reader, - InterpolationSupport interpolationSupport) { - super(context, parserFeatures, formatFeatures, loaderOptions, codec, reader); - this.interpolationSupport = interpolationSupport; - } - - @Override - public String getText() throws IOException { - final String value = super.getText(); - if (value != null) { - return interpolationSupport.expand(value); - } - return null; - } - - @Override - public String getValueAsString() throws IOException { - return getValueAsString(null); - } - - @Override - public String getValueAsString(final String defaultValue) throws IOException { - final String value = super.getValueAsString(defaultValue); - if (value != null) { - return interpolationSupport.expand(value); - } - return null; - } -} diff --git a/core/src/main/java/dev/streamx/cli/interpolation/InterpolationSupport.java b/core/src/main/java/dev/streamx/cli/interpolation/InterpolationSupport.java deleted file mode 100644 index 356b7a9e..00000000 --- a/core/src/main/java/dev/streamx/cli/interpolation/InterpolationSupport.java +++ /dev/null @@ -1,60 +0,0 @@ -package dev.streamx.cli.interpolation; - -import static io.smallrye.common.expression.Expression.Flag.LENIENT_SYNTAX; -import static io.smallrye.common.expression.Expression.Flag.NO_SMART_BRACES; -import static io.smallrye.common.expression.Expression.Flag.NO_TRIM; - -import io.smallrye.common.expression.Expression; -import jakarta.enterprise.context.Dependent; -import java.util.NoSuchElementException; -import java.util.Objects; -import java.util.Optional; -import org.eclipse.microprofile.config.Config; -import org.eclipse.microprofile.config.spi.ConfigProviderResolver; - -@Dependent -public class InterpolationSupport { - - /** - * Adapted from {@link io.smallrye.config.ExpressionConfigSourceInterceptor} - */ - String expand(final String rawValue) { - Objects.requireNonNull(rawValue, "Expression cannot be null"); - - // Avoid extra StringBuilder allocations from Expression - if (rawValue.indexOf('$') == -1) { - return rawValue; - } - - final Config config = ConfigProviderResolver.instance().getConfig(); - final Expression expression = Expression.compile(escapeDollarIfExists(rawValue), LENIENT_SYNTAX, - NO_TRIM, NO_SMART_BRACES); - return expression.evaluate((resolveContext, stringBuilder) -> { - Optional resolve = config.getOptionalValue(resolveContext.getKey(), String.class); - if (resolve.isPresent()) { - stringBuilder.append(resolve.get()); - } else if (resolveContext.hasDefault()) { - resolveContext.expandDefault(); - } else { - throw new NoSuchElementException(String.format("Could not expand value %s in expression %s", - resolveContext.getKey(), rawValue)); - } - }); - } - - private String escapeDollarIfExists(final String value) { - int index = value.indexOf("\\$"); - if (index != -1) { - int start = 0; - StringBuilder builder = new StringBuilder(); - while (index != -1) { - builder.append(value, start, index).append("$$"); - start = index + 2; - index = value.indexOf("\\$", start); - } - builder.append(value.substring(start)); - return builder.toString(); - } - return value; - } -} diff --git a/core/src/main/java/dev/streamx/cli/interpolation/ObjectMapperProducer.java b/core/src/main/java/dev/streamx/cli/interpolation/ObjectMapperProducer.java index dd6c7b0a..0f2a690d 100644 --- a/core/src/main/java/dev/streamx/cli/interpolation/ObjectMapperProducer.java +++ b/core/src/main/java/dev/streamx/cli/interpolation/ObjectMapperProducer.java @@ -1,22 +1,18 @@ package dev.streamx.cli.interpolation; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import io.quarkus.arc.All; import io.quarkus.jackson.ObjectMapperCustomizer; import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; import java.util.List; @ApplicationScoped public class ObjectMapperProducer { - @Inject - InterpolationSupport interpolationSupport; - - @Interpolating @ApplicationScoped ObjectMapper produce(@All List customizers) { - ObjectMapper mapper = new ObjectMapper(new InterpolatingYamlFactory(interpolationSupport)); + ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); // Apply all ObjectMapperCustomizer beans (incl. Quarkus) for (ObjectMapperCustomizer customizer : customizers) { diff --git a/core/src/test/java/dev/streamx/cli/command/meshprocessing/MeshDefinitionResolverInterpolationTest.java b/core/src/test/java/dev/streamx/cli/command/meshprocessing/MeshDefinitionResolverInterpolationTest.java deleted file mode 100644 index 4ec28753..00000000 --- a/core/src/test/java/dev/streamx/cli/command/meshprocessing/MeshDefinitionResolverInterpolationTest.java +++ /dev/null @@ -1,78 +0,0 @@ -package dev.streamx.cli.command.meshprocessing; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -import com.fasterxml.jackson.databind.JsonMappingException; -import com.streamx.mesh.model.ServiceMesh; -import io.quarkus.test.junit.QuarkusTest; -import jakarta.inject.Inject; -import java.io.IOException; -import java.nio.file.Path; -import java.util.NoSuchElementException; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -@QuarkusTest -class MeshDefinitionResolverInterpolationTest { - - private static final String TEST_MESH_LOCATION = "target/test-classes/mesh-interpolated.yaml"; - private static final Path TEST_MESH_PATH = Path.of(TEST_MESH_LOCATION); - - @Inject - MeshDefinitionResolver uut; - - @BeforeEach - void clearSystemProperties() { - System.clearProperty("config.image.interpolated"); - System.clearProperty("config.source.interpolated"); - } - - @Test - void shouldFailWithMandatoryPropertyUndefined() { - assertThatThrownBy(() -> uut.resolve(TEST_MESH_PATH)) - .isInstanceOf(JsonMappingException.class) - .hasRootCauseInstanceOf(NoSuchElementException.class) - .hasRootCauseMessage("Could not expand value config.image.interpolated" - + " in expression ${config.image.interpolated}"); - } - - @Test - void shouldResolveWithMandatoryPropertyDefinedAndOptionalPropertyUndefined() throws IOException { - System.setProperty("config.image.interpolated", "image-1"); - - ServiceMesh result = uut.resolve(TEST_MESH_PATH); - assertSinkImage(result, "image-1"); - assertSourceRef(result, "inbox.pages"); - } - - @Test - void shouldResolveWithMandatoryAndOptionalPropertiesDefined() throws IOException { - System.setProperty("config.image.interpolated", "image-1"); - System.setProperty("config.source.interpolated", "source-1"); - - ServiceMesh result = uut.resolve(TEST_MESH_PATH); - assertSinkImage(result, "image-1"); - assertSourceRef(result, "source-1"); - } - - private static void assertSinkImage(ServiceMesh result, String expected) { - String actual = result - .getDescriptors() - .get("web-server-sink") - .getContainers() - .get("sink") - .getImage(); - assertThat(actual).isEqualTo(expected); - } - - private static void assertSourceRef(ServiceMesh result, String expected) { - String actual = result - .getSources() - .get("cli") - .getOutgoing() - .getFirst() - .getRef(); - assertThat(actual).isEqualTo(expected); - } -} diff --git a/core/src/test/java/dev/streamx/cli/interpolation/InterpolatingMapperTest.java b/core/src/test/java/dev/streamx/cli/interpolation/InterpolatingMapperTest.java deleted file mode 100644 index 47f93279..00000000 --- a/core/src/test/java/dev/streamx/cli/interpolation/InterpolatingMapperTest.java +++ /dev/null @@ -1,208 +0,0 @@ -package dev.streamx.cli.interpolation; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.fasterxml.jackson.databind.ObjectMapper; -import io.quarkus.test.junit.QuarkusTest; -import jakarta.inject.Inject; -import java.util.Set; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; - -@QuarkusTest -public class InterpolatingMapperTest { - - @Inject - @Interpolating - ObjectMapper mapper; - - @AfterEach - void cleanup() { - var propertiesToClear = Set.of("test.interpolatedValue", "test.nestedValue", - "test.arrayValue1", "test.arrayValue2", "test.string", "test.int", "test.boolean"); - - propertiesToClear.forEach(System::clearProperty); - } - - @Test - void testInterpolateStringFields() throws Exception { - System.setProperty("test.interpolatedValue", "interpolatedValue"); - String yaml = """ - field: ${test.interpolatedValue} - """; - TestClass result = mapper.readValue(yaml, TestClass.class); - - assertEquals("interpolatedValue", result.getField()); - } - - @Test - void testInterpolateFieldWithDefault() throws Exception { - String yaml = """ - field: ${test.interpolatedValue:default} - """; - TestClass result = mapper.readValue(yaml, TestClass.class); - - assertEquals("default", result.getField()); - } - - @Test - void testNotInterpolatingNullField() throws Exception { - String yaml = """ - field: null - """; - TestClass result = mapper.readValue(yaml, TestClass.class); - - assertNull(result.getField()); - } - - @Test - void testNotInterpolatingFieldWithoutPlaceholder() throws Exception { - String yaml = """ - field: "field" - """; - TestClass result = mapper.readValue(yaml, TestClass.class); - - assertEquals("field", result.getField()); - } - - @Test - void testNestedInterpolation() throws Exception { - System.setProperty("test.nestedValue", "nestedValue"); - String yaml = """ - nested: - field: ${test.nestedValue} - """; - - NestedTestClass result = mapper.readValue(yaml, NestedTestClass.class); - - assertEquals("nestedValue", result.getNested().getField()); - } - - @Test - void testArrayInterpolation() throws Exception { - System.setProperty("test.arrayValue1", "interpolatedValue1"); - System.setProperty("test.arrayValue2", "interpolatedValue2"); - - String yaml = """ - - "${test.arrayValue1}" - - "${test.arrayValue2}" - """; - String[] result = mapper.readValue(yaml, String[].class); - - assertEquals("interpolatedValue1", result[0]); - assertEquals("interpolatedValue2", result[1]); - } - - @Test - void testMixedTypesInterpolation() throws Exception { - System.setProperty("test.string", "interpolatedValue"); - System.setProperty("test.int", "-1"); - System.setProperty("test.boolean", "true"); - - String yaml = """ - bool: true - string: ${test.string} - longField: ${test.int} - integer: ${test.int} - url: http://${test.string} - """; - MixedTypesTestClass result = mapper.readValue(yaml, MixedTypesTestClass.class); - assertTrue(result.isBool()); - assertEquals("interpolatedValue", result.getString()); - assertEquals(-1, result.getLongField()); - assertEquals(-1, result.getInteger()); - assertEquals("http://interpolatedValue", result.getUrl()); - } - - - static class MixedTypesTestClass { - - private boolean bool; - private String string; - private Integer integer; - private Long longField; - private String url; - - public boolean isBool() { - return bool; - } - - public void setBool(boolean bool) { - this.bool = bool; - } - - public String getString() { - return string; - } - - public void setString(String string) { - this.string = string; - } - - public Integer getInteger() { - return integer; - } - - public void setInteger(Integer integer) { - this.integer = integer; - } - - public Long getLongField() { - return longField; - } - - public void setLongField(Long longField) { - this.longField = longField; - } - - public String getUrl() { - return url; - } - - public void setUrl(String url) { - this.url = url; - } - } - - static class TestClass { - - private String field; - - public TestClass() { - } - - public TestClass(String field) { - this.field = field; - } - - public String getField() { - return field; - } - - public void setField(String field) { - this.field = field; - } - } - - static class NestedTestClass { - - private TestClass nested; - - public NestedTestClass() { - } - - public NestedTestClass(TestClass nested) { - this.nested = nested; - } - - public TestClass getNested() { - return nested; - } - - public void setNested(TestClass nested) { - this.nested = nested; - } - } -} diff --git a/core/src/test/java/dev/streamx/cli/interpolation/InterpolationSupportTest.java b/core/src/test/java/dev/streamx/cli/interpolation/InterpolationSupportTest.java deleted file mode 100644 index 3e8c0c65..00000000 --- a/core/src/test/java/dev/streamx/cli/interpolation/InterpolationSupportTest.java +++ /dev/null @@ -1,144 +0,0 @@ -package dev.streamx.cli.interpolation; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import io.quarkus.test.component.QuarkusComponentTest; -import jakarta.inject.Inject; -import java.util.NoSuchElementException; -import java.util.Set; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; - -@QuarkusComponentTest -public class InterpolationSupportTest { - - @Inject - InterpolationSupport interpolationSupport; - - @AfterEach - void cleanup() { - var propertiesToClear = Set.of( - "this.is.a.property", "prop1", "prop2", "nested", "inner", "whitespace.property"); - - propertiesToClear.forEach(System::clearProperty); - } - - @Test - void testExpandWithValidProperty() { - // Setup: Define the environment or system properties - System.setProperty("this.is.a.property", "validValue"); - - // Valid case - String rawValue = "${this.is.a.property}"; - String result = interpolationSupport.expand(rawValue); - - assertEquals("validValue", result); - } - - @Test - void testExpandWithMultipleConfigSources() { - // Setup: Define the environment or system properties - System.setProperty("this.is.a.property", "From application.properties ->"); - - // Valid case - String rawValue = "${this.is.a.property} ${streamx.cli.settings.root-dir}"; - String result = interpolationSupport.expand(rawValue); - - assertEquals("From application.properties -> ./target/test-classes/dev.streamx.cli.settings", - result); - } - - @Test - void testExpandWithMultipleProperties() { - // Setup multiple properties - System.setProperty("prop1", "value1"); - System.setProperty("prop2", "value2"); - - String rawValue = "Property1: ${prop1}, Property2: ${prop2}"; - String result = interpolationSupport.expand(rawValue); - - assertEquals("Property1: value1, Property2: value2", result); - } - - @Test - void testExpandWithNestedProperties() { - // Setup nested properties - System.setProperty("nested", "${inner}"); - System.setProperty("inner", "finalValue"); - - String rawValue = "${nested}"; - String result = interpolationSupport.expand(rawValue); - - assertEquals("finalValue", result); - } - - @Test - void testExpandWithMissingProperty() { - // Case where property is not defined - String rawValue = "${undefined.property}"; - - assertThrows(NoSuchElementException.class, - () -> interpolationSupport.expand(rawValue), - "Could not expand value undefined.property in expression ${undefined.property}"); - } - - @Test - void testExpandWithPartialProperty() { - System.setProperty("this.is.a.property", "validValueToo"); - - // Case with partially closed property - String rawValue = "${this.is.a.property"; // Missing closing brace - - String result = interpolationSupport.expand(rawValue); - - assertEquals("validValueToo", result); - } - - @Test - void testExpandWithNoInterpolation() { - // Case with no interpolation markers - String rawValue = "No interpolation here."; - String result = interpolationSupport.expand(rawValue); - - assertEquals("No interpolation here.", result); - } - - @Test - void testExpandWithEmptyString() { - // Case with empty string - String rawValue = ""; - String result = interpolationSupport.expand(rawValue); - - assertEquals("", result); - } - - @Test - void testExpandWithNullInput() { - // Case with null input - String rawValue = null; - - Exception exception = assertThrows(NullPointerException.class, - () -> interpolationSupport.expand(rawValue), "Expression cannot be null"); - } - - @Test - void testExpandWithEscapedPropertySyntax() { - // Case with escaped property syntax - String rawValue = "\\${this.is.a.property}"; // Escaped $ to avoid interpolation - String result = interpolationSupport.expand(rawValue); - - assertEquals("${this.is.a.property}", result); - } - - @Test - void testExpandWithWhitespaceAroundProperty() { - // Case with whitespace - System.setProperty("whitespace.property", "value"); - - String rawValue = " ${whitespace.property} "; - String result = interpolationSupport.expand(rawValue); - - assertEquals(" value ", result, "Whitespace outside property markers should be preserved"); - } -} diff --git a/core/src/test/resources/application.properties b/core/src/test/resources/application.properties index cbe8bcb1..1994f3f5 100644 --- a/core/src/test/resources/application.properties +++ b/core/src/test/resources/application.properties @@ -4,8 +4,3 @@ streamx.dev.dashboard.open-on-startup=false # stdout is written to console by default, so writing same to logs is redundant %test.quarkus.log.category."dev.streamx.cli.util.Output".level=OFF - -# Used by interpolation tests -string.array.property=stringArrayValueFromFile -string.property=stringValueFromFile -integer.property=1 diff --git a/core/src/test/resources/filtered/mesh-interpolated.yaml b/core/src/test/resources/filtered/mesh-interpolated.yaml deleted file mode 100644 index 7f8ea1f6..00000000 --- a/core/src/test/resources/filtered/mesh-interpolated.yaml +++ /dev/null @@ -1,55 +0,0 @@ -descriptors: - relay: - type: "processing" - containers: - service: - incoming: - events: {} - outgoing: - relayed-events: {} - - web-server-sink: - type: "edge" - containers: - sink: - image: "${config.image.interpolated}" - incoming: - resources: {} - servicePorts: - - "8087:8080" - -sources: - cli: - outgoing: - - ref: "${config.source.interpolated:inbox.pages}" - produces: - - "com.streamx.blueprints.page.published.v1" - - "com.streamx.blueprints.page.unpublished.v1" - -ingestion: - rest-ingestion: - containers: - proxy: - environment: - QUARKUS_HTTP_AUTH_PERMISSION_BEARER_POLICY: permit - -processing: - pages-relay: - descriptor: relay - containers: - service: - incoming: - events: - ref: inbox.pages - outgoing: - relayed-events: - ref: outbox.pages - -edge: - web-server-sink: - descriptor: web-server-sink - containers: - sink: - incoming: - resources: - ref: outbox.pages diff --git a/e2e-tests/.env b/e2e-tests/.env deleted file mode 100644 index 3b120f79..00000000 --- a/e2e-tests/.env +++ /dev/null @@ -1,2 +0,0 @@ -# Property used in configuration interpolation in e2e tests -streamx.cli.e2e.port=8089 \ No newline at end of file From 6988d4f833031e51ea8e769145b5ce61b461085f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Wed, 11 Feb 2026 10:42:40 +0100 Subject: [PATCH 35/36] Remove unused MeshWatcher class to increase coverage --- .../command/meshprocessing/MeshWatcher.java | 111 ------------------ 1 file changed, 111 deletions(-) delete mode 100644 core/src/main/java/dev/streamx/cli/command/meshprocessing/MeshWatcher.java diff --git a/core/src/main/java/dev/streamx/cli/command/meshprocessing/MeshWatcher.java b/core/src/main/java/dev/streamx/cli/command/meshprocessing/MeshWatcher.java deleted file mode 100644 index 256d6839..00000000 --- a/core/src/main/java/dev/streamx/cli/command/meshprocessing/MeshWatcher.java +++ /dev/null @@ -1,111 +0,0 @@ -package dev.streamx.cli.command.meshprocessing; - -import static dev.streamx.cli.util.Output.print; - -import io.quarkus.scheduler.Scheduled.ConcurrentExecution; -import io.quarkus.scheduler.Scheduler; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; -import java.io.IOException; -import java.nio.file.FileSystems; -import java.nio.file.Path; -import java.nio.file.StandardWatchEventKinds; -import java.nio.file.WatchEvent; -import java.nio.file.WatchKey; -import java.nio.file.WatchService; -import org.jboss.logging.Logger; - -@ApplicationScoped -public class MeshWatcher { - - public static final String MESH_WATCHER_JOB_NAME = "meshWatcher"; - - @Inject - Logger log; - - @Inject - MeshManager meshManager; - - @Inject - Scheduler scheduler; - - public void watchMeshChanges(Path meshPath) { - try { - Path meshDir = meshPath.toAbsolutePath().normalize().getParent(); - - WatchService watchService = FileSystems.getDefault().newWatchService(); - WatchKey watchKey = meshDir.register(watchService, - StandardWatchEventKinds.ENTRY_CREATE, - StandardWatchEventKinds.ENTRY_MODIFY, - StandardWatchEventKinds.ENTRY_DELETE - ); - watchKey.reset(); - - scheduler.newJob(MESH_WATCHER_JOB_NAME) - .setConcurrentExecution(ConcurrentExecution.SKIP) - .setTask(task -> { - try { - ActionToPerform action = checkModifications(meshPath, watchService); - if (action != null) { - switch (action) { - case RELOAD -> { - meshManager.reload(); - } - case STOP -> { - print(""); - print("Mesh file deleted. Stopping..."); - meshManager.stop(); - print("Mesh stopped."); - } - default -> { - print("Unknown action: " + action + ". Skipping..."); - } - } - } - } catch (Exception e) { - log.error(e); - } - }) - .setInterval("500ms") - .schedule(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private static ActionToPerform checkModifications(Path meshPath, WatchService watchService) { - ActionToPerform lastAction = null; - WatchKey polledKey = watchService.poll(); - if (polledKey != null) { - for (var event : polledKey.pollEvents()) { - if (pathsMatches(meshPath, event)) { - if (event.kind() == StandardWatchEventKinds.ENTRY_CREATE) { - lastAction = ActionToPerform.RELOAD; - } else if (event.kind() == StandardWatchEventKinds.ENTRY_MODIFY) { - lastAction = ActionToPerform.RELOAD; - } else if (event.kind() == StandardWatchEventKinds.ENTRY_DELETE) { - lastAction = ActionToPerform.STOP; - } - } - } - polledKey.reset(); - } - return lastAction; - } - - private enum ActionToPerform { - RELOAD, - STOP - } - - private static boolean pathsMatches(Path meshPath, WatchEvent event) { - Path modifiedFile = (Path) event.context(); - Path normalizedModifiedPath = meshPath.toAbsolutePath().normalize() - .getParent().resolve(modifiedFile) - .toAbsolutePath().normalize(); - - Path normalizedMeshPath = meshPath.toAbsolutePath().normalize(); - - return normalizedModifiedPath.equals(normalizedMeshPath); - } -} From edb3661f402bf4727bc035e75deb095c7527bd80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Ruchlicki?= Date: Thu, 12 Feb 2026 14:25:52 +0100 Subject: [PATCH 36/36] Remove json.fields.as-base64 feature --- .../ingestion/stream/StreamCommand.java | 6 -- .../ingestion/stream/StreamProperties.java | 44 ----------- .../stream/parser/JsonBase64Encoder.java | 44 ----------- .../stream/parser/JsonBase64EncoderTest.java | 79 ------------------- .../ingestion/stream/.stream.properties | 1 - .../ingestion/stream/valid-json.stream | 2 +- .../test/resources/stream/.stream.properties | 1 - .../resources/stream/page-publish-event.json | 2 +- 8 files changed, 2 insertions(+), 177 deletions(-) delete mode 100644 core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamProperties.java delete mode 100644 core/src/main/java/dev/streamx/cli/command/ingestion/stream/parser/JsonBase64Encoder.java delete mode 100644 core/src/test/java/dev/streamx/cli/command/ingestion/stream/parser/JsonBase64EncoderTest.java delete mode 100644 core/src/test/resources/dev/streamx/cli/command/ingestion/stream/.stream.properties delete mode 100644 e2e-tests/src/test/resources/stream/.stream.properties diff --git a/core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamCommand.java b/core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamCommand.java index 74ec9864..d4fb28c0 100644 --- a/core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamCommand.java +++ b/core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamCommand.java @@ -11,7 +11,6 @@ import com.streamx.clients.ingestion.publisher.Publisher; import dev.streamx.cli.VersionProvider; import dev.streamx.cli.command.ingestion.BaseIngestionCommand; -import dev.streamx.cli.command.ingestion.stream.parser.JsonBase64Encoder; import dev.streamx.cli.command.ingestion.stream.parser.StreamIngestionJsonParser; import dev.streamx.cli.util.ExceptionUtils; import dev.streamx.cli.util.FileUtils; @@ -21,7 +20,6 @@ import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.List; import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Command; @@ -47,13 +45,9 @@ public class StreamCommand extends BaseIngestionCommand { @Override protected void perform(Publisher publisher) throws StreamxClientException { Path streamFile = Paths.get(streamIngestionArguments.getSourceFile()); - List jsonFieldsToEncodeToBase64 = StreamProperties - .getJsonFieldsToEncodeToBase64(streamFile); - try (FileInputStream fis = new FileInputStream(streamFile.toFile())) { ingestionJsonParser.parse(fis, cloudEventNode -> { - JsonBase64Encoder.encodeFields(cloudEventNode, jsonFieldsToEncodeToBase64); CloudEvent inputEvent = toCloudEvent(cloudEventNode); CloudEvent responseEvent = publisher.send(inputEvent); printf("Sent %s event using stream with key '%s' at %s%n", diff --git a/core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamProperties.java b/core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamProperties.java deleted file mode 100644 index a98d5c96..00000000 --- a/core/src/main/java/dev/streamx/cli/command/ingestion/stream/StreamProperties.java +++ /dev/null @@ -1,44 +0,0 @@ -package dev.streamx.cli.command.ingestion.stream; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Properties; -import org.apache.commons.lang3.StringUtils; -import org.jboss.logging.Logger; - -class StreamProperties { - - private static final Logger logger = Logger.getLogger(StreamProperties.class); - private static final String PROPERTIES_FILE_NAME = ".stream.properties"; - private static final String JSON_FIELDS_AS_BASE64 = "json.fields.as-base64"; - - private StreamProperties() { - // no instances - } - - public static List getJsonFieldsToEncodeToBase64(Path streamFile) { - Path propertiesFile = streamFile.resolveSibling(PROPERTIES_FILE_NAME); - if (Files.exists(propertiesFile)) { - try (InputStream fileStream = Files.newInputStream(propertiesFile)) { - Properties properties = new Properties(); - properties.load(fileStream); - String fields = properties.getProperty(JSON_FIELDS_AS_BASE64); - if (StringUtils.isNotBlank(fields)) { - return Arrays.stream(fields.split(",")) - .map(String::trim) - .filter(StringUtils::isNotEmpty) - .toList(); - } - } catch (IOException ex) { - logger.warn("Error reading properties from " + propertiesFile, ex); - } - } - return Collections.emptyList(); - } - -} \ No newline at end of file diff --git a/core/src/main/java/dev/streamx/cli/command/ingestion/stream/parser/JsonBase64Encoder.java b/core/src/main/java/dev/streamx/cli/command/ingestion/stream/parser/JsonBase64Encoder.java deleted file mode 100644 index a6906d1d..00000000 --- a/core/src/main/java/dev/streamx/cli/command/ingestion/stream/parser/JsonBase64Encoder.java +++ /dev/null @@ -1,44 +0,0 @@ -package dev.streamx.cli.command.ingestion.stream.parser; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.node.TextNode; -import java.util.Base64; -import java.util.List; -import org.apache.commons.lang3.StringUtils; - -public class JsonBase64Encoder { - - private JsonBase64Encoder() { - // no instances - } - - public static void encodeFields(JsonNode root, List fieldPaths) { - for (String fieldPath : fieldPaths) { - fieldPath = StringUtils.prependIfMissing(fieldPath, "/"); - JsonNode node = root.at(fieldPath); - - if (node.isMissingNode() || !node.isTextual()) { - continue; - } - - TextNode encodedTextNode = createTextNodeWithEncodedContent(node); - replaceNode(root, fieldPath, encodedTextNode); - } - } - - private static TextNode createTextNodeWithEncodedContent(JsonNode node) { - String encodedText = Base64.getEncoder().encodeToString(node.asText().getBytes()); - return TextNode.valueOf(encodedText); - } - - private static void replaceNode(JsonNode root, String fieldPath, JsonNode newValue) { - String parentPath = StringUtils.substringBeforeLast(fieldPath, "/"); - String fieldName = StringUtils.substringAfterLast(fieldPath, "/"); - - JsonNode parentNode = root.at(parentPath); - if (parentNode instanceof ObjectNode obj) { - obj.set(fieldName, newValue); - } - } -} \ No newline at end of file diff --git a/core/src/test/java/dev/streamx/cli/command/ingestion/stream/parser/JsonBase64EncoderTest.java b/core/src/test/java/dev/streamx/cli/command/ingestion/stream/parser/JsonBase64EncoderTest.java deleted file mode 100644 index 0f3e0a07..00000000 --- a/core/src/test/java/dev/streamx/cli/command/ingestion/stream/parser/JsonBase64EncoderTest.java +++ /dev/null @@ -1,79 +0,0 @@ -package dev.streamx.cli.command.ingestion.stream.parser; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.params.provider.Arguments.arguments; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; -import java.util.stream.Stream; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; - -class JsonBase64EncoderTest { - - private static final ObjectMapper objectMapper = new ObjectMapper(); - public static final String INITIAL_JSON = """ - { - "foo": { - "bar": "foobar" - }, - "foo2": "bar2" - }"""; - - static Stream testArguments() { - return Stream.of( - // happy path - arguments( - INITIAL_JSON, - "/foo/bar", - INITIAL_JSON.replace("foobar", "Zm9vYmFy") - ), - - // missing leading slash in the jsonFieldsAsBase64 should be automatically added - arguments( - INITIAL_JSON, - "foo/bar", - INITIAL_JSON.replace("foobar", "Zm9vYmFy") - ), - - // attempting to encode a parent node that is not a text node - arguments( - INITIAL_JSON, - "/foo", - INITIAL_JSON - ), - - // json doesn't contain the requested field - arguments( - INITIAL_JSON, - "bar", - INITIAL_JSON - ), - - // invalid path of field - arguments( - INITIAL_JSON, - "/a/)#Q(*%Y@W$Tbngow23p0t[';'\\325/b", - INITIAL_JSON - ) - ); - } - - @ParameterizedTest - @MethodSource("testArguments") - void shouldEncodeFields(String sourceJson, String jsonFieldsAsBase64, String expectedResultJson) - throws Exception { - // given - JsonNode jsonNode = objectMapper.readTree(sourceJson); - - // when - JsonBase64Encoder.encodeFields(jsonNode, List.of(jsonFieldsAsBase64)); - - // then - JsonNode expectedNode = objectMapper.readTree(expectedResultJson); - assertThat(jsonNode.toPrettyString()).isEqualTo(expectedNode.toPrettyString()); - } - -} \ No newline at end of file diff --git a/core/src/test/resources/dev/streamx/cli/command/ingestion/stream/.stream.properties b/core/src/test/resources/dev/streamx/cli/command/ingestion/stream/.stream.properties deleted file mode 100644 index 15b64ec4..00000000 --- a/core/src/test/resources/dev/streamx/cli/command/ingestion/stream/.stream.properties +++ /dev/null @@ -1 +0,0 @@ -json.fields.as-base64=/data/content \ No newline at end of file diff --git a/core/src/test/resources/dev/streamx/cli/command/ingestion/stream/valid-json.stream b/core/src/test/resources/dev/streamx/cli/command/ingestion/stream/valid-json.stream index d5785ffe..cb02c209 100644 --- a/core/src/test/resources/dev/streamx/cli/command/ingestion/stream/valid-json.stream +++ b/core/src/test/resources/dev/streamx/cli/command/ingestion/stream/valid-json.stream @@ -7,7 +7,7 @@ "subject" : "homepage.html", "time" : "2025-12-23T10:28:23.435253Z", "data" : { - "content" : "

This is Home Page!

", + "content" : "PGgxPlRoaXMgaXMgSG9tZSBQYWdlITwvaDE+", "type" : "page/content" } } diff --git a/e2e-tests/src/test/resources/stream/.stream.properties b/e2e-tests/src/test/resources/stream/.stream.properties deleted file mode 100644 index 15b64ec4..00000000 --- a/e2e-tests/src/test/resources/stream/.stream.properties +++ /dev/null @@ -1 +0,0 @@ -json.fields.as-base64=/data/content \ No newline at end of file diff --git a/e2e-tests/src/test/resources/stream/page-publish-event.json b/e2e-tests/src/test/resources/stream/page-publish-event.json index c8b1f8f3..f915e048 100644 --- a/e2e-tests/src/test/resources/stream/page-publish-event.json +++ b/e2e-tests/src/test/resources/stream/page-publish-event.json @@ -7,7 +7,7 @@ "subject" : "hello.html", "time" : "2025-12-23T10:28:23.435253Z", "data" : { - "content" : "Hello World!", + "content" : "PGI+SGVsbG8gV29ybGQhPC9iPg==", "type" : "page/test" } } \ No newline at end of file