From 9f91d7bc269a2b41f241408e1008aa3cd86d9632 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Wed, 26 Oct 2022 17:10:32 +0800 Subject: [PATCH 01/22] [fix][test] Fix ClientTlsTest license (#18204) --- .../org/apache/pulsar/tests/integration/tls/ClientTlsTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/tls/ClientTlsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/tls/ClientTlsTest.java index fe5c3ad40b5e1..59ff978cafa06 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/tls/ClientTlsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/tls/ClientTlsTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information From 29461bd49c3ba7bf8d1221c05343d90527fef780 Mon Sep 17 00:00:00 2001 From: Jiaqi Shen <18863662628@163.com> Date: Wed, 26 Oct 2022 22:50:16 +0800 Subject: [PATCH 02/22] [doc] [client] [go] Add chunking to go-client doc (#17789) * add go-client chunking doc * Update site2/docs/client-libraries-go.md Co-authored-by: Zixuan Liu * Update site2/docs/client-libraries-go.md Co-authored-by: Anonymitaet <50226895+Anonymitaet@users.noreply.github.com> * Update site2/docs/client-libraries-go.md Co-authored-by: Anonymitaet <50226895+Anonymitaet@users.noreply.github.com> * Update site2/docs/client-libraries-go.md Co-authored-by: Anonymitaet <50226895+Anonymitaet@users.noreply.github.com> Co-authored-by: Zixuan Liu Co-authored-by: Anonymitaet <50226895+Anonymitaet@users.noreply.github.com> --- site2/docs/client-libraries-go.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/site2/docs/client-libraries-go.md b/site2/docs/client-libraries-go.md index 8ca9f269f5e26..14b331409ce5e 100644 --- a/site2/docs/client-libraries-go.md +++ b/site2/docs/client-libraries-go.md @@ -204,6 +204,34 @@ for i := 0; i < 10; i++ { } ``` +#### How to use chunking in producer + +```go +client, err := pulsar.NewClient(pulsar.ClientOptions{ + URL: serviceURL, +}) + +if err != nil { + log.Fatal(err) +} +defer client.Close() + +// The message chunking feature is OFF by default. +// By default, a producer chunks the large message based on the max message size (`maxMessageSize`) configured at the broker side (for example, 5MB). +// Client can also configure the max chunked size using the producer configuration `ChunkMaxMessageSize`. +// Note: to enable chunking, you need to disable batching (`DisableBatching=true`) concurrently. +producer, err := client.CreateProducer(pulsar.ProducerOptions{ + Topic: "my-topic", + DisableBatching: true, + EnableChunking: true, +}) + +if err != nil { + log.Fatal(err) +} +defer producer.Close() +``` + #### How to use schema interface in producer ```go From 6657fe42149be30365db3e99abc877a22173b52c Mon Sep 17 00:00:00 2001 From: tison Date: Thu, 27 Oct 2022 09:00:55 +0800 Subject: [PATCH 03/22] [fix][sql] Fix jline version to 3.21.0 (#18207) --- pom.xml | 6 ++++++ .../org/apache/pulsar/PulsarStandalone.java | 2 +- pulsar-client-tools/pom.xml | 1 - pulsar-sql/pom.xml | 17 +++++++++++++++++ pulsar-sql/presto-distribution/LICENSE | 6 +++--- 5 files changed, 27 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 23092d983b6c2..1121742ae3140 100644 --- a/pom.xml +++ b/pom.xml @@ -290,6 +290,12 @@ flexible messaging model and an intuitive client API. + + org.jline + jline + ${jline3.version} + + org.asynchttpclient async-http-client diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java index 70110766e4e77..d58cdc502a3cf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java @@ -439,7 +439,7 @@ private void startBookieWithMetadataStore() throws Exception { log.info("Starting BK with RocksDb metadata store"); metadataStoreUrl = "rocksdb://" + Paths.get(metadataDir).toAbsolutePath(); } else { - log.info("Starting BK with metadata store:", metadataStoreUrl); + log.info("Starting BK with metadata store: {}", metadataStoreUrl); } ServerConfiguration bkServerConf = new ServerConfiguration(); diff --git a/pulsar-client-tools/pom.xml b/pulsar-client-tools/pom.xml index 2553f1c2488ab..c81c0da4c0b25 100644 --- a/pulsar-client-tools/pom.xml +++ b/pulsar-client-tools/pom.xml @@ -126,7 +126,6 @@ org.jline jline - ${jline3.version} net.java.dev.jna diff --git a/pulsar-sql/pom.xml b/pulsar-sql/pom.xml index b546b7c6b532d..3f8f3b5e044fd 100644 --- a/pulsar-sql/pom.xml +++ b/pulsar-sql/pom.xml @@ -63,6 +63,23 @@ ${okio.version} + + + org.jline + jline-reader + ${jline3.version} + + + org.jline + jline-terminal + ${jline3.version} + + + org.jline + jline-terminal-jna + ${jline3.version} + + org.slf4j diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 18c6420f0c00b..ea7a1c02deede 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -492,9 +492,9 @@ BSD License - asm-tree-6.2.1.jar - asm-util-6.2.1.jar * JLine - - jline-reader-3.17.1.jar - - jline-terminal-3.17.1.jar - - jline-terminal-jna-3.17.1.jar + - jline-reader-3.21.0.jar + - jline-terminal-3.21.0.jar + - jline-terminal-jna-3.21.0.jar MIT License * PCollections From a48bc8b07fec31a03d3e96042d2e4d8a8e7f22cf Mon Sep 17 00:00:00 2001 From: momo-jun <60642177+momo-jun@users.noreply.github.com> Date: Thu, 27 Oct 2022 09:06:27 +0800 Subject: [PATCH 04/22] [improve][doc] Improve the authentication enablement workflow across multiple providers (#18035) * Update security-athenz.md * improve auth overview * address review comments * Improve OAuth2 authentication * improve Kerberos authentication * Update security-athenz.md --- site2/docs/client-libraries-java.md | 2 +- site2/docs/security-athenz.md | 163 ++++++-- site2/docs/security-extending.md | 8 +- site2/docs/security-jwt.md | 105 +++--- site2/docs/security-kerberos.md | 432 +++++++++------------- site2/docs/security-oauth2.md | 169 ++++----- site2/docs/security-overview.md | 39 +- site2/docs/security-tls-authentication.md | 15 +- 8 files changed, 475 insertions(+), 458 deletions(-) diff --git a/site2/docs/client-libraries-java.md b/site2/docs/client-libraries-java.md index 2135c165a45b3..e36a3957b1af9 100644 --- a/site2/docs/client-libraries-java.md +++ b/site2/docs/client-libraries-java.md @@ -1262,7 +1262,7 @@ Pulsar currently supports the following authentication mechansims: * [TLS](security-tls-authentication.md#configure-tls-authentication-in-pulsar-clients) * [JWT](security-jwt.md#configure-jwt-authentication-in-pulsar-clients) * [Athenz](security-athenz.md#configure-athenz-authentication-in-pulsar-clients) -* [Kerberos](security-kerberos.md#java-client-and-java-admin-client) +* [Kerberos](security-kerberos.md#configure-kerberos-authentication-in-pulsar-clients) * [OAuth2](security-oauth2.md#configure-oauth2-authentication-in-pulsar-clients) * [HTTP basic](security-basic-auth.md#configure-basic-authentication-in-pulsar-clients) diff --git a/site2/docs/security-athenz.md b/site2/docs/security-athenz.md index 60849db06ab83..3bccffd70b738 100644 --- a/site2/docs/security-athenz.md +++ b/site2/docs/security-athenz.md @@ -6,42 +6,43 @@ sidebar_label: "Authentication using Athenz" [Athenz](https://github.com/AthenZ/athenz) is a role-based authentication/authorization system. In Pulsar, you can use Athenz role tokens (also known as *z-tokens*) to establish the identity of the client. -## Enable Athenz authentication - A [decentralized Athenz system](https://github.com/AthenZ/athenz/blob/master/docs/decent_authz_flow.md) contains an [authori**Z**ation **M**anagement **S**ystem](https://github.com/AthenZ/athenz/blob/master/docs/setup_zms.md) (ZMS) server and an [authori**Z**ation **T**oken **S**ystem](https://github.com/AthenZ/athenz/blob/master/docs/setup_zts) (ZTS) server. -To begin, you need to set up Athenz service access control. You need to create domains for the *provider* (which provides some resources to other services with some authentication/authorization policies) and the *tenant* (which is provisioned to access some resources in a provider). In this case, the provider corresponds to the Pulsar service itself and the tenant corresponds to each application using Pulsar (typically, a [tenant](reference-terminology.md#tenant) in Pulsar). +## Prerequisites + +To begin, you need to set up Athenz service access control by creating domains for the *provider* (which provides some resources to other services with some authentication/authorization policies) and the *tenant* (which is provisioned to access some resources in a provider). In this case, the provider corresponds to the Pulsar service itself and the tenant corresponds to each application using Pulsar (typically, a [tenant](reference-terminology.md#tenant) in Pulsar). -### Create the tenant domain and service +### Create a tenant domain and service -On the [tenant](reference-terminology.md#tenant) side, you need to do the following things: +On the tenant side, do the followings: -1. Create a domain, such as `shopping` -2. Generate a private/public key pair -3. Create a service, such as `some_app`, on the domain with the public key +1. Create a domain, such as `shopping`. +2. Generate a private/public key pair. +3. Create a service, such as `some_app`, on the domain with the public key. -Note that you need to specify the private key generated in step 2 when the Pulsar client connects to the [broker](reference-terminology.md#broker) (see client configuration examples for [Java](client-libraries-java.md) and [C++](client-libraries-cpp.md)). +Note that you need to specify the private key generated in step 2 when the Pulsar client connects to the broker. For more specific steps involving the Athenz UI, refer to [Example Service Access Control Setup](https://github.com/AthenZ/athenz/blob/master/docs/example_service_athenz_setup.md#client-tenant-domain). -### Create the provider domain and add the tenant service to some role members +### Create a provider domain and add the tenant service to role members On the provider side, you need to do the following things: -1. Create a domain, such as `pulsar` -2. Create a role -3. Add the tenant service to members of the role +1. Create a domain, such as `pulsar`. +2. Create a role. +3. Add the tenant service to the members of the role. Note that you can specify any action and resource in step 2 since they are not used on Pulsar. In other words, Pulsar uses the Athenz role token only for authentication, *not* for authorization. -For more specific steps involving UI, refer to [Example Service Access Control Setup](https://github.com/AthenZ/athenz/blob/master/docs/example_service_athenz_setup.md#server-provider-domain). +For more specific steps involving the Athenz UI, refer to [Example Service Access Control Setup](https://github.com/AthenZ/athenz/blob/master/docs/example_service_athenz_setup.md#server-provider-domain). ## Enable Athenz authentication on brokers -> ### TLS encryption -> -> Note that when you are using Athenz as an authentication provider, you had better use TLS encryption -> as it can protect role tokens from being intercepted and reused. (for more details involving TLS encryption see [Architecture - Data Model](https://github.com/AthenZ/athenz/blob/master/docs/data_model)). +:::note + +When you are using Athenz as an authentication provider, it's highly recommended to use [TLS encryption](security-tls-transport.md) as it can protect role tokens from being intercepted and reused. For more details involving TLS encryption, see [Architecture - Data Model](https://github.com/AthenZ/athenz/blob/master/docs/data_model). + +::: In the `conf/broker.conf` configuration file in your Pulsar installation, you need to provide the class name of the Athenz authentication provider as well as a comma-separated list of provider domain names. @@ -52,11 +53,6 @@ authorizationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderAthenz athenzDomainNames=pulsar -# Enable TLS -tlsEnabled=true -tlsCertificateFilePath=/path/to/broker-cert.pem -tlsKeyFilePath=/path/to/broker-key.pem - # Authentication settings of the broker itself. Used when the broker connects to other brokers, either in same or other clusters brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationAthenz brokerClientAuthenticationParameters={"tenantDomain":"shopping","tenantService":"some_app","providerDomain":"pulsar","privateKey":"file:///path/to/private.pem","keyId":"v1"} @@ -65,18 +61,49 @@ brokerClientAuthenticationParameters={"tenantDomain":"shopping","tenantService": > A full listing of parameters is available in the `conf/broker.conf` file, you can also find the default > values for those parameters in [Broker Configuration](reference-configuration.md#broker). +## Enable Athenz authentication on proxies + +Configure the required parameters in the `conf/proxy.conf` file in your Pulsar installation. + +```properties +# Add the Athenz auth provider +authenticationEnabled=true +authorizationEnabled=true +authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderAthenz +athenzDomainNames=pulsar + +brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationAthenz +brokerClientAuthenticationParameters={"tenantDomain":"shopping","tenantService":"some_app","providerDomain":"pulsar","privateKey":"file:///path/to/private.pem","keyId":"v1"} +``` + ## Configure Athenz authentication in Pulsar clients -To use Athenz as an authentication provider, you need to [use TLS](#tls-encryption) and provide values for four parameters in a hash: +To use Athenz as an authentication provider, you need to provide values for four parameters in a hash: * `tenantDomain` * `tenantService` * `providerDomain` * `privateKey` +:::tip + +The `privateKey` parameter supports the following three pattern formats: +* `file:///path/to/file` +* `file:/path/to/file` +* `data:application/x-pem-file;base64,` + +::: + You can also set an optional `keyId`. The following is an example. +````mdx-code-block + + + ```java Map authParams = new HashMap(); +authParams.put("ztsUrl", "http://localhost:9998"); authParams.put("tenantDomain", "shopping"); // Tenant domain name authParams.put("tenantService", "some_app"); // Tenant service name authParams.put("providerDomain", "pulsar"); // Provider domain name @@ -87,17 +114,88 @@ Authentication athenzAuth = AuthenticationFactory .create(AuthenticationAthenz.class.getName(), authParams); PulsarClient client = PulsarClient.builder() - .serviceUrl("pulsar+ssl://my-broker.com:6651") - .tlsTrustCertsFilePath("/path/to/cacert.pem") + .serviceUrl("pulsar://my-broker.com:6650") .authentication(athenzAuth) .build(); ``` -#### Supported pattern formats -The `privateKey` parameter supports the following three pattern formats: -* `file:///path/to/file` -* `file:/path/to/file` -* `data:application/x-pem-file;base64,` + + + +```python +authPlugin = "athenz" +authParams = """ +{ +"tenantDomain": "shopping", +"tenantService": "some_app", +"providerDomain": "pulsar", +"privateKey": "file:///path/to/private.pem", +"ztsUrl": "http://localhost:9998" +} +""" + +client = Client( + "pulsar://my-broker.com:6650", + authentication=Authentication(authPlugin, authParams), +) +``` + + + + +```cpp +std::string params = R"({ + "tenantDomain": "shopping", + "tenantService": "some_app", + "providerDomain": "pulsar", + "privateKey": "file:///path/to/private.pem", + "ztsUrl": "http://localhost:9998" + })"; +pulsar::AuthenticationPtr auth = pulsar::AuthAthenz::create(params); +ClientConfiguration config = ClientConfiguration(); +config.setAuth(auth); +Client client("pulsar://my-broker.com:6650", config); +``` + + + + +```javascript +const auth = new Pulsar.AuthenticationAthenz({ + tenantDomain: "shopping", + tenantService: "some_app", + providerDomain: "pulsar", + privateKey: "file:///path/to/private.pem", + ztsUrl: "http://localhost:9998" +}); + +const client = new Pulsar.Client({ + serviceUrl: 'pulsar://my-broker.com:6650', + authentication: auth +}); +``` + + + + +```go +provider := pulsar.NewAuthenticationAthenz( + "pulsar", + "shopping", + "some_app", + "file:///path/to/private.pem", + "v1", + "", + "http://localhost:9998") +client, err := pulsarNewClient(ClientOptions{ + URL: "pulsar://my-broker.com:6650", + Authentication: basicAuth, + }) +``` + + + +```` ## Configure Athenz authentication in CLI tools @@ -117,5 +215,4 @@ authParams={"tenantDomain":"shopping","tenantService":"some_app","providerDomain useTls=true tlsAllowInsecureConnection=false tlsTrustCertsFilePath=/path/to/cacert.pem -``` - +``` \ No newline at end of file diff --git a/site2/docs/security-extending.md b/site2/docs/security-extending.md index 2955c44a92253..b3470307416d0 100644 --- a/site2/docs/security-extending.md +++ b/site2/docs/security-extending.md @@ -9,8 +9,8 @@ Pulsar provides a way to use custom authentication and authorization mechanisms. ## Authentication You can use a custom authentication mechanism by providing the implementation in the form of two plugins. -* Client authentication plugin -* Proxy/Broker authentication plugin +* Client authentication plugin `org.apache.pulsar.client.api.AuthenticationDataProvider` provides the authentication data for broker/proxy. +* Broker/Proxy authentication plugin `org.apache.pulsar.broker.authentication.AuthenticationProvider` authenticates the authentication data from clients. ### Client authentication plugin @@ -37,9 +37,9 @@ You can find the following examples for different client authentication plugins: * [OAuth 2.0](https://github.com/apache/pulsar/blob/master/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationOAuth2.java) * [Basic auth](https://github.com/apache/pulsar/blob/master/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationBasic.java) -### Proxy/Broker authentication plugin +### Broker/Proxy authentication plugin -On the proxy/broker side, you need to configure the corresponding plugin to validate the credentials that the client sends. The proxy and broker can support multiple authentication providers at the same time. +On the broker/proxy side, you need to configure the corresponding plugin to validate the credentials that the client sends. The proxy and broker can support multiple authentication providers at the same time. In `conf/broker.conf`, you can choose to specify a list of valid providers: diff --git a/site2/docs/security-jwt.md b/site2/docs/security-jwt.md index 1764887827e5b..88d0ac408d9ef 100644 --- a/site2/docs/security-jwt.md +++ b/site2/docs/security-jwt.md @@ -1,6 +1,6 @@ --- id: security-jwt -title: Client authentication using tokens based on JSON Web Tokens +title: Authentication using tokens based on JSON Web Tokens sidebar_label: "Authentication using JWT" --- @@ -9,43 +9,34 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; ```` +Pulsar supports authenticating clients using security tokens based on [JSON Web Tokens](https://jwt.io/introduction/) ([RFC-7519](https://tools.ietf.org/html/rfc7519)), including all the algorithms that the [Java JWT library](https://github.com/jwtk/jjwt#signature-algorithms-keys) supports. -## Token authentication overview +A token is a credential associated with a user. The association is done through a "principal" or "role". In the case of JWT tokens, it typically refers to a **subject**. You can use a token to identify a Pulsar client and associate it with a **subject** that is permitted to do specific actions, such as publish messages to a topic or consume messages from a topic. An alternative is to pass a "token supplier" (a function that returns the token when the client library needs one). -Pulsar supports authenticating clients using security tokens that are based on [JSON Web Tokens](https://jwt.io/introduction/) ([RFC-7519](https://tools.ietf.org/html/rfc7519)). - -You can use tokens to identify a Pulsar client and associate with some "principal" (or "role") that -is permitted to do some actions (eg: publish to a topic or consume from a topic). - -A user typically gets a token string from the administrator (or some automated service). - -The compact representation of a signed JWT is a string that looks like the following: +The application specifies the token when you create the client instance. The user typically gets the token string from the administrator. The compact representation of a signed JWT is a string that looks like the following: ``` eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJKb2UifQ.ipevRNuRP6HflG8cFKnmUPtypruRC4fb1DWtoLL62SY ``` -Application specifies the token when you create the client instance. An alternative is to pass a "token supplier" (a function that returns the token when the client library needs one). - :::note -Always use TLS transport encryption when you connect to the Pulsar service, because sending a token is equivalent to sending a password over the wire. See [Transport Encryption using TLS](security-tls-transport.md) for more details. +Always use [TLS encryption](security-tls-transport.md) when connecting to the Pulsar service, because sending a token is equivalent to sending a password over the wire. ::: -## Enable token authentication +## Create client certificates -JWT supports two different kinds of keys to generate and validate the tokens: +JWT authentication supports two different kinds of keys to generate and validate the tokens: - * Symmetric: - - You can use a single ***Secret*** key to generate and validate tokens. - * Asymmetric: A pair of keys consists of the Private key and the Public key. - - You can use ***Private*** key to generate tokens. - - You can use ***Public*** key to validate tokens. +- Symmetric: A single ***secret*** key. +- Asymmetric: A key pair, including: + - a ***private*** key to generate tokens. + - a ***public*** key to validate tokens. ### Create a secret key -When you use a secret key, the administrator creates the key and uses the key to generate the client tokens. You can also configure this key to brokers to validate the clients. +The administrators create the secret key and use it to generate the client tokens. You can also configure this key for brokers to validate the clients. The output file is generated in the root of your Pulsar installation directory. You can also provide an absolute path for the output file using the command below. @@ -53,7 +44,7 @@ The output file is generated in the root of your Pulsar installation directory. bin/pulsar tokens create-secret-key --output my-secret.key ``` -Enter this command to generate a base64 encoded private key. +To generate a base64-encoded private key, enter the following command. ```shell bin/pulsar tokens create-secret-key --output /opt/my-secret.key --base64 @@ -61,54 +52,48 @@ bin/pulsar tokens create-secret-key --output /opt/my-secret.key --base64 ### Create a key pair -With Public and Private keys, you need to create a pair of keys. Pulsar supports all algorithms that the Java JWT library (shown [here](https://github.com/jwtk/jjwt#signature-algorithms-keys)) supports. - -The output file is generated in the root of your Pulsar installation directory. You can also provide an absolute path for the output file using the command below. +To use asymmetric key encryption, you need to create a pair of keys. The output file is generated in the root of your Pulsar installation directory. You can also provide an absolute path for the output file using the command below. ```shell bin/pulsar tokens create-key-pair --output-private-key my-private.key --output-public-key my-public.key ``` - * Store `my-private.key` in a safe location and only administrators can use `my-private.key` to generate new tokens. - * `my-public.key` is distributed to all Pulsar brokers. You can publicly share this file without any security concerns. + * Store `my-private.key` in a safe location and only the administrators can use this private key to generate new tokens. + * The public key file `my-public.key` is distributed to all Pulsar brokers. You can publicly share it without any security concerns. ### Generate tokens -A token is a credential associated with a user. The association is done through the "principal" or "role". In the case of JWT tokens, this field is typically referred as **subject**, though they are the same concept. - -Then, you need to use this command to require the generated token to have a **subject** fieldset. - -```shell -bin/pulsar tokens create --secret-key file:///path/to/my-secret.key \ - --subject test-user -``` +1. Use this command to require the generated token to have a **subject** fieldset. This command prints the token string on `stdout`. -This command prints the token string on stdout. + ```shell + bin/pulsar tokens create --secret-key file:///path/to/my-secret.key \ + --subject test-user + ``` -Similarly, you can create a token by passing the "private" key using the command below: +2. Create a token by passing the "private" key using the command below: -```shell -bin/pulsar tokens create --private-key file:///path/to/my-private.key \ - --subject test-user -``` + ```shell + bin/pulsar tokens create --private-key file:///path/to/my-private.key \ + --subject test-user + ``` -Finally, you can enter the following command to create a token with a pre-defined TTL. And then the token is automatically invalidated. +3. Create a token with a pre-defined TTL. Then the token is automatically invalidated. -```shell -bin/pulsar tokens create --secret-key file:///path/to/my-secret.key \ - --subject test-user \ - --expiry-time 1y -``` + ```shell + bin/pulsar tokens create --secret-key file:///path/to/my-secret.key \ + --subject test-user \ + --expiry-time 1y + ``` :::tip -The token itself does not have any permission associated. The authorization engine determines whether the token can have permissions or not. You need to [enable authorization and assign superusers](security-authorization.md#enable-authorization-and-assign-superusers), and then use the `bin/pulsar-admin namespaces grant-permission` command to grant permissions for tokens. +The token itself does not have any permission associated. You need to [enable authorization and assign superusers](security-authorization.md#enable-authorization-and-assign-superusers), and use the `bin/pulsar-admin namespaces grant-permission` command to grant permissions to the token. ::: -### Enable token authentication on brokers +## Enable JWT authentication on brokers -To configure brokers to authenticate clients, add the following parameters to the `conf/broker.conf` or `conf/standalone.conf` file. +To configure brokers to authenticate clients using JWT, add the following parameters to the `conf/broker.conf` or `conf/standalone.conf` file. ```properties # Configuration to enable authentication @@ -143,9 +128,9 @@ Equivalent to `brokerClientAuthenticationParameters`, you need to configure `aut ::: -### Enable token authentication on proxies +## Enable JWT authentication on proxies -To configure proxies to authenticate clients, add the following parameters to the `conf/proxy.conf` file. +To configure proxies to authenticate clients using JWT, add the following parameters to the `conf/proxy.conf` file. ```properties # For clients connecting to the proxy @@ -162,16 +147,20 @@ brokerClientAuthenticationParameters={"token":"eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ0 # brokerClientAuthenticationParameters=file:///path/to/token ``` +:::note + The proxy uses its own token when connecting to brokers. You need to configure the role token for this key pair in the `proxyRoles` of the brokers. For more details, see [authorization](security-authorization.md). -### Configure JWT authentication in CLI Tools +::: + +## Configure JWT authentication in CLI Tools [Command-line tools](reference-cli-tools.md) like [`pulsar-admin`](/tools/pulsar-admin/), [`pulsar-perf`](reference-cli-tools.md), and [`pulsar-client`](reference-cli-tools.md) use the `conf/client.conf` config file in a Pulsar installation. -You need to add the following parameters to that file to use the token authentication with CLI tools of Pulsar: +You need to add the following parameters to the `conf/client.conf` config file to use the JWT authentication with CLI tools of Pulsar: -```conf -webServiceUrl=http://broker.example.com:8080/ +```properties +webServiceUrl=https://broker.example.com:8443/ brokerServiceUrl=pulsar://broker.example.com:6650/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationToken authParams=token:eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJKb2UifQ.ipevRNuRP6HflG8cFKnmUPtypruRC4fb1DWtoLL62SY @@ -179,11 +168,11 @@ authParams=token:eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJKb2UifQ.ipevRNuRP6HflG8cFKnmUPt The token string can also be read from a file, for example: -```conf +```properties authParams=file:///path/to/token/file ``` -### Configure JWT authentication in Pulsar clients +## Configure JWT authentication in Pulsar clients You can use tokens to authenticate the following Pulsar clients. diff --git a/site2/docs/security-kerberos.md b/site2/docs/security-kerberos.md index 506dc1573c69b..9d7b7ddf1aa67 100644 --- a/site2/docs/security-kerberos.md +++ b/site2/docs/security-kerberos.md @@ -4,28 +4,34 @@ title: Authentication using Kerberos sidebar_label: "Authentication using Kerberos" --- -[Kerberos](https://web.mit.edu/kerberos/) is a network authentication protocol. By using secret-key cryptography, [Kerberos](https://web.mit.edu/kerberos/) is designed to provide strong authentication for client applications and server applications. +[Kerberos](https://web.mit.edu/kerberos/) is a network authentication protocol designed to provide strong authentication for client applications and server applications by using secret-key cryptography. -In Pulsar, you can use Kerberos with [SASL](https://en.wikipedia.org/wiki/Simple_Authentication_and_Security_Layer) as a choice for authentication. And Pulsar uses the [Java Authentication and Authorization Service (JAAS)](https://en.wikipedia.org/wiki/Java_Authentication_and_Authorization_Service) for SASL configuration. You need to provide JAAS configurations for Kerberos authentication. +In Pulsar, you can use Kerberos with [SASL](https://en.wikipedia.org/wiki/Simple_Authentication_and_Security_Layer) as a choice for authentication. Since Pulsar uses the [Java Authentication and Authorization Service (JAAS)](https://en.wikipedia.org/wiki/Java_Authentication_and_Authorization_Service) for SASL configuration, you need to provide JAAS configurations for Kerberos authentication. -This document introduces how to configure `Kerberos` with `SASL` between Pulsar clients and brokers and how to configure Kerberos for Pulsar proxy in detail. +:::note -## Configure Kerberos between client and broker +Kerberos authentication uses the authenticated principal as the role token for [Pulsar authorization](security-authorization.md). If you've enabled `authorizationEnabled`, you need to set `superUserRoles` in `broker.conf` that corresponds to the name registered in KDC. For example: -### Prerequisites +```bash +superUserRoles=client/{clientIp}@EXAMPLE.COM +``` -To begin, you need to set up (or already have) a [Key Distribution Center(KDC)](https://en.wikipedia.org/wiki/Key_distribution_center). Also you need to configure and run the [Key Distribution Center(KDC)](https://en.wikipedia.org/wiki/Key_distribution_center)in advance. +::: -If your organization already uses a Kerberos server (for example, by using `Active Directory`), you do not have to install a new server for Pulsar. If your organization does not use a Kerberos server, you need to install one. Your Linux vendor might have packages for `Kerberos`. On how to install and configure Kerberos, refer to [Ubuntu](https://help.ubuntu.com/community/Kerberos), +## Prerequisites + +- Set up and run a [Key Distribution Center(KDC)](https://en.wikipedia.org/wiki/Key_distribution_center). +- Install a Kerberos server if your organization doesn't have one. Your Linux vendor might have packages for `Kerberos`. For how to install and configure Kerberos, see [Ubuntu](https://help.ubuntu.com/community/Kerberos) and [Redhat](https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_Linux/6/html/Managing_Smart_Cards/installing-kerberos.html). +- If you use Oracle Java, you need to download JCE policy files for your Java version and copy them to the `$JAVA_HOME/jre/lib/security` directory. -Note that if you use Oracle Java, you need to download JCE policy files for your Java version and copy them to the `$JAVA_HOME/jre/lib/security` directory. +## Enable Kerberos authentication on brokers -#### Kerberos principals +### Create Kerberos principals -If you use the existing Kerberos system, ask your Kerberos administrator for a principal for each brokers in your cluster and for every operating system user that accesses Pulsar with Kerberos authentication(via clients and tools). +If you use the existing Kerberos system, ask your Kerberos administrator to obtain a principal for each broker in your cluster and for every operating system user that accesses Pulsar with Kerberos authentication (via clients and CLI tools). -If you have installed your own Kerberos system, you can create these principals with the following commands: +If you have installed your own Kerberos system, you need to create these principals with the following commands: ```shell ### add Principals for broker @@ -36,35 +42,32 @@ sudo /usr/sbin/kadmin.local -q 'addprinc -randkey client/{hostname}@{REALM}' sudo /usr/sbin/kadmin.local -q "ktadd -k /etc/security/keytabs/{client-keytabname}.keytab client/{hostname}@{REALM}" ``` -Note that *Kerberos* requires that all your hosts can be resolved with their FQDNs. - -The first part of broker principal (for example, `broker` in `broker/{hostname}@{REALM}`) is the `serverType` of each host. The suggested values of `serverType` are `broker` (host machine runs service Pulsar brokers) and `proxy` (host machine runs service Pulsar Proxy). +The first part of broker principal (for example, `broker` in `broker/{hostname}@{REALM}`) is the `serverType` of each host. The suggested values of `serverType` are `broker` (host machine runs Pulsar broker service) and `proxy` (host machine runs Pulsar Proxy service). -#### Connect to KDC - -You need to enter the command below to specify the path to the `krb5.conf` file for the client side and the broker side. The content of `krb5.conf` file indicates the default Realm and KDC information. See [JDK’s Kerberos Requirements](https://docs.oracle.com/javase/8/docs/technotes/guides/security/jgss/tutorials/KerberosReq.html) for more details. - -```shell --Djava.security.krb5.conf=/etc/pulsar/krb5.conf -``` +Note that *Kerberos* requires that all your hosts can be resolved with their FQDNs. -Here is an example of the krb5.conf file. `EXAMPLE.COM` is the default realm; `kdc = localhost:62037` is the kdc server URL for realm `EXAMPLE.COM `. +### Configure brokers + +In the `broker.conf` file, set Kerberos-related configurations. Here is an example: ```conf -[libdefaults] - default_realm = EXAMPLE.COM +authenticationEnabled=true +authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderSasl +saslJaasClientAllowedIds=.*client.* ## regex for principals that are allowed to connect to brokers +saslJaasServerSectionName=PulsarBroker ## corresponds to the section in the JAAS configuration file for brokers -[realms] - EXAMPLE.COM = { - kdc = localhost:62037 - } +## Authentication settings of the broker itself. Used when the broker connects to other brokers +brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationSasl +brokerClientAuthenticationParameters={"saslJaasClientSectionName":"PulsarClient", "serverType":"broker"} ``` -Usually machines configured with kerberos already have a system-wide configuration and this configuration is optional. +To make Pulsar internal admin client work properly, you need to: +- Set `brokerClientAuthenticationPlugin` to client plugin `AuthenticationSasl`; +- Set `brokerClientAuthenticationParameters` to value in JSON string `{"saslJaasClientSectionName":"PulsarClient", "serverType":"broker"}`, in which `PulsarClient` is the section name in the `pulsar_jaas.conf` file, and `"serverType":"broker"` indicates that the internal admin client connects to a broker. -#### Configure JAAS configuration file +### Configure JAAS -You need the JAAS configuration file for the client side and the broker side. JAAS configuration file provides the section of information that is used to connect KDC. Here is an example named `pulsar_jaas.conf`: +JAAS configuration file provides the information to connect KDC. Here is an example named `pulsar_jaas.conf`: ```conf PulsarBroker { @@ -86,174 +89,90 @@ You need the JAAS configuration file for the client side and the broker side. JA }; ``` -You need to set the `JAAS` configuration file path as JVM parameter for client and broker. For example: +In the above example: +- `PulsarBroker` is a section name in the JAAS file that each broker uses. This section tells the broker to use which principal inside Kerberos and the location of the keytab where the principal is stored. +- `PulsarClient` is a section name in the JASS file that each client uses. This section tells the client to use which principal inside Kerberos and the location of the keytab where the principal is stored. + +You need to set the `pulsar_jaas.conf` file path as a JVM parameter. For example: ```shell -Djava.security.auth.login.config=/etc/pulsar/pulsar_jaas.conf ``` -In the `pulsar_jaas.conf` file above - -1. `PulsarBroker` is a section name in the JAAS file that each broker uses. This section tells the broker to use which principal inside Kerberos and the location of the keytab where the principal is stored. `PulsarBroker` allows the broker to use the keytab specified in this section. -2. `PulsarClient` is a section name in the JASS file that each broker uses. This section tells the client to use which principal inside Kerberos and the location of the keytab where the principal is stored. `PulsarClient` allows the client to use the keytab specified in this section. - The following example also reuses this `PulsarClient` section in both the Pulsar internal admin configuration and in CLI command of `bin/pulsar-client`, `bin/pulsar-perf` and `bin/pulsar-admin`. You can also add different sections for different use cases. - -You can have 2 separate JAAS configuration files: -* the file for a broker that has sections of both `PulsarBroker` and `PulsarClient`; -* the file for a client that only has a `PulsarClient` section. - - -### Configure Kerberos authentication for brokers +### Connect to KDC -#### Configure the `broker.conf` file - -In the `broker.conf` file, set Kerberos-related configurations. - -- Set `authenticationEnabled` to `true`; -- Set `authenticationProviders` to choose `AuthenticationProviderSasl`; -- Set `saslJaasClientAllowedIds` regex for principals that are allowed to connect to brokers; -- Set `saslJaasServerSectionName` that corresponds to the section in the JAAS configuration file for brokers; - -To make Pulsar internal admin client work properly, you need to set the configuration in the `broker.conf` file as below: -- Set `brokerClientAuthenticationPlugin` to client plugin `AuthenticationSasl`; -- Set `brokerClientAuthenticationParameters` to value in JSON string `{"saslJaasClientSectionName":"PulsarClient", "serverType":"broker"}`, in which `PulsarClient` is the section name in the `pulsar_jaas.conf` file, and `"serverType":"broker"` indicates that the internal admin client connects to a Pulsar Broker; - -Here is an example: +:::note -```conf -authenticationEnabled=true -authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderSasl -saslJaasClientAllowedIds=.*client.* -saslJaasServerSectionName=PulsarBroker +If your machines configured with Kerberos already have a system-wide configuration, you can skip this configuration. -## Authentication settings of the broker itself. Used when the broker connects to other brokers -brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationSasl -brokerClientAuthenticationParameters={"saslJaasClientSectionName":"PulsarClient", "serverType":"broker"} -``` +::: -#### Set broker JVM parameters +The content of `krb5.conf` file indicates the default Realm and KDC information. See [JDK’s Kerberos Requirements](https://docs.oracle.com/javase/8/docs/technotes/guides/security/jgss/tutorials/KerberosReq.html) for more details. -Set JVM parameters for the JAAS configuration file and krb5 configuration file with additional options. +To specify the path to the `krb5.conf` file for brokers, enter the command below. ```shell --Djava.security.auth.login.config=/etc/pulsar/pulsar_jaas.conf -Djava.security.krb5.conf=/etc/pulsar/krb5.conf +-Djava.security.krb5.conf=/etc/pulsar/krb5.conf ``` -You can add this at the end of `PULSAR_EXTRA_OPTS` in the file [`pulsar_env.sh`](https://github.com/apache/pulsar/blob/master/conf/pulsar_env.sh) - -You must ensure that the operating system user who starts broker can reach the keytabs configured in the `pulsar_jaas.conf` file and kdc server in the `krb5.conf` file. - -### Configure Kerberos authentication for clients +Here is an example of the `krb5.conf` file. -#### Java Client and Java Admin Client - -In client applications, include `pulsar-client-auth-sasl` in your project dependency. +```conf +[libdefaults] + default_realm = EXAMPLE.COM -```xml - - org.apache.pulsar - pulsar-client-auth-sasl - ${pulsar.version} - +[realms] + EXAMPLE.COM = { + kdc = localhost:62037 + } ``` -Configure the authentication type to use `AuthenticationSasl`, and also provide the authentication parameters to it. - -You need 2 parameters: -- `saslJaasClientSectionName`. This parameter corresponds to the section in the JAAS configuration file for clients; -- `serverType`. This parameter stands for whether this client connects to brokers or proxies. And client uses this parameter to know which server-side principal should be used. - -When you authenticate between client and broker with the setting in the above JAAS configuration file, we need to set `saslJaasClientSectionName` to `PulsarClient` and set `serverType` to `broker`. +In the above example: +- `EXAMPLE.COM` is the default Realm; +- `kdc = localhost:62037` is the KDC server URL for the `EXAMPLE.COM` Realm. -The following is an example of creating a Java client: +## Enable Kerberos authentication on proxies - ```java - System.setProperty("java.security.auth.login.config", "/etc/pulsar/pulsar_jaas.conf"); - System.setProperty("java.security.krb5.conf", "/etc/pulsar/krb5.conf"); +If you want to use proxies between brokers and clients, Pulsar proxies (as a SASL server in Kerberos) will authenticate clients (as a SASL client in Kerberos) before brokers authenticate proxies. - Map authParams = Maps.newHashMap(); - authParams.put("saslJaasClientSectionName", "PulsarClient"); - authParams.put("serverType", "broker"); +### Create Kerberos principals - Authentication saslAuth = AuthenticationFactory - .create(org.apache.pulsar.client.impl.auth.AuthenticationSasl.class.getName(), authParams); - - PulsarClient client = PulsarClient.builder() - .serviceUrl("pulsar://my-broker.com:6650") - .authentication(saslAuth) - .build(); - ``` - -> The first two lines in the example above are hard-coded. Alternatively, you can set additional JVM parameters for JAAS and krb5 configuration file when you run the application like below: +Add new principals for Pulsar proxies. ```shell -java -cp -Djava.security.auth.login.config=/etc/pulsar/pulsar_jaas.conf -Djava.security.krb5.conf=/etc/pulsar/krb5.conf $APP-jar-with-dependencies.jar $CLASSNAME +### add Principals for Pulsar Proxy +sudo /usr/sbin/kadmin.local -q 'addprinc -randkey proxy/{hostname}@{REALM}' +sudo /usr/sbin/kadmin.local -q "ktadd -k /etc/security/keytabs/{proxy-keytabname}.keytab proxy/{hostname}@{REALM}" ``` -You must ensure that the operating system user who starts Pulsar client can reach the keytabs configured in the `pulsar_jaas.conf` file and kdc server in the `krb5.conf` file. +For principals set for brokers and clients, see [here](#create-kerberos-principals). -#### Configure CLI tools +### Configure proxies -If you use a command-line tool (such as `bin/pulsar-client`, `bin/pulsar-perf` and `bin/pulsar-admin`), you need to perform the following steps: - -Step 1. Enter the command below to configure your `client.conf`. - -```shell -authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationSasl -authParams={"saslJaasClientSectionName":"PulsarClient", "serverType":"broker"} -``` - -Step 2. Enter the command below to set JVM parameters for the JAAS configuration file and krb5 configuration file with additional options. +In the `proxy.conf` file, set Kerberos-related configuration. ```shell --Djava.security.auth.login.config=/etc/pulsar/pulsar_jaas.conf -Djava.security.krb5.conf=/etc/pulsar/krb5.conf -``` - -You can add this at the end of `PULSAR_EXTRA_OPTS` in the file [`pulsar_tools_env.sh`](https://github.com/apache/pulsar/blob/master/conf/pulsar_tools_env.sh), -or add this line `OPTS="$OPTS -Djava.security.auth.login.config=/etc/pulsar/pulsar_jaas.conf -Djava.security.krb5.conf=/etc/pulsar/krb5.conf "` directly to the CLI tool script. - -The meaning of configurations is the same as the meaning of configurations in Java client section. - -## Configure Kerberos authentication for proxies - -With the above configuration, clients and brokers can do authentication using Kerberos. - -A client that connects to Pulsar Proxy is a little different. Pulsar Proxy (as a SASL Server in Kerberos) authenticates Client (as a SASL client in Kerberos) first, and then Pulsar broker authenticates Pulsar Proxy. - -Now in comparison with the above configuration between client and broker, we show you how to configure Pulsar Proxy as follows. - -### Create principals in Kerberos - -You need to add new principals for Pulsar proxy compared with the above configuration. If you already have principals for client and broker, you only need to add the proxy principal here. +## related to authenticate client. +authenticationEnabled=true +authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderSasl +saslJaasClientAllowedIds=.*client.* +saslJaasServerSectionName=PulsarProxy -```shell -### add Principals for Pulsar Proxy -sudo /usr/sbin/kadmin.local -q 'addprinc -randkey proxy/{hostname}@{REALM}' -sudo /usr/sbin/kadmin.local -q "ktadd -k /etc/security/keytabs/{proxy-keytabname}.keytab proxy/{hostname}@{REALM}" -### add Principals for broker -sudo /usr/sbin/kadmin.local -q 'addprinc -randkey broker/{hostname}@{REALM}' -sudo /usr/sbin/kadmin.local -q "ktadd -k /etc/security/keytabs/{broker-keytabname}.keytab broker/{hostname}@{REALM}" -### add Principals for client -sudo /usr/sbin/kadmin.local -q 'addprinc -randkey client/{hostname}@{REALM}' -sudo /usr/sbin/kadmin.local -q "ktadd -k /etc/security/keytabs/{client-keytabname}.keytab client/{hostname}@{REALM}" +## related to be authenticated by broker +brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationSasl +brokerClientAuthenticationParameters={"saslJaasClientSectionName":"PulsarProxy", "serverType":"broker"} +forwardAuthorizationCredentials=true ``` -### Add a section in JAAS configuration file +In the above example: +- The first part relates to the authentication between clients and proxies. In this phase, clients work as SASL clients, while proxies work as SASL servers. +- The second part relates to the authentication between proxies and brokers. In this phase, proxies work as SASL clients, while brokers work as SASL servers. -In comparison with the above configuration, add a new section for Pulsar Proxy in the JAAS configuration file. +### Configure JAAS -Here is an example named `pulsar_jaas.conf`: +Add a new section for proxies in the `pulsar_jaas.conf` file. Here is an example: ```conf - PulsarBroker { - com.sun.security.auth.module.Krb5LoginModule required - useKeyTab=true - storeKey=true - useTicketCache=false - keyTab="/etc/security/keytabs/pulsarbroker.keytab" - principal="broker/localhost@EXAMPLE.COM"; -}; - PulsarProxy { com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true @@ -262,133 +181,134 @@ Here is an example named `pulsar_jaas.conf`: keyTab="/etc/security/keytabs/pulsarproxy.keytab" principal="proxy/localhost@EXAMPLE.COM"; }; - - PulsarClient { - com.sun.security.auth.module.Krb5LoginModule required - useKeyTab=true - storeKey=true - useTicketCache=false - keyTab="/etc/security/keytabs/pulsarclient.keytab" - principal="client/localhost@EXAMPLE.COM"; -}; ``` -### Configure proxy clients +## Configure Kerberos authentication in Java clients -Pulsar client configuration is similar to client and broker configuration, except that you need to set `serverType` to `proxy` instead of `broker`, for the reason that you need to do the Kerberos authentication between client and proxy. +:::note - ```java - System.setProperty("java.security.auth.login.config", "/etc/pulsar/pulsar_jaas.conf"); - System.setProperty("java.security.krb5.conf", "/etc/pulsar/krb5.conf"); +Ensure that the operating system user who starts Pulsar clients can access the keytabs configured in the `pulsar_jaas.conf` file and the KDC server configured in the `krb5.conf` file. - Map authParams = Maps.newHashMap(); - authParams.put("saslJaasClientSectionName", "PulsarClient"); - authParams.put("serverType", "proxy"); // ** here is the different ** +::: - Authentication saslAuth = AuthenticationFactory - .create(org.apache.pulsar.client.impl.auth.AuthenticationSasl.class.getName(), authParams); - - PulsarClient client = PulsarClient.builder() - .serviceUrl("pulsar://my-broker.com:6650") - .authentication(saslAuth) - .build(); - ``` +1. In client applications, include `pulsar-client-auth-sasl` in your project dependency. -> The first two lines in the example above are hard coded, alternatively, you can set additional JVM parameters for JAAS and krb5 configuration file when you run the application like below: + ```xml + + org.apache.pulsar + pulsar-client-auth-sasl + ${pulsar.version} + + ``` -```shell -java -cp -Djava.security.auth.login.config=/etc/pulsar/pulsar_jaas.conf -Djava.security.krb5.conf=/etc/pulsar/krb5.conf $APP-jar-with-dependencies.jar $CLASSNAME -``` +2. Configure the authentication type to use `AuthenticationSasl` and provide the following parameters. + - set `saslJaasClientSectionName` to `PulsarClient`; + - set `serverType` to `broker`. `serverType` stands for whether this client connects to brokers or proxies. Clients use this parameter to know which server-side principal should be used. -### Configure proxy service + The following is an example of configuring a Java client: -In the `proxy.conf` file, set Kerberos-related configuration. Here is an example: + ```java + System.setProperty("java.security.auth.login.config", "/etc/pulsar/pulsar_jaas.conf"); + System.setProperty("java.security.krb5.conf", "/etc/pulsar/krb5.conf"); -```shell -## related to authenticate client. -authenticationEnabled=true -authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderSasl -saslJaasClientAllowedIds=.*client.* -saslJaasServerSectionName=PulsarProxy + Map authParams = Maps.newHashMap(); + authParams.put("saslJaasClientSectionName", "PulsarClient"); + authParams.put("serverType", "broker"); -## related to be authenticated by broker -brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationSasl -brokerClientAuthenticationParameters={"saslJaasClientSectionName":"PulsarProxy", "serverType":"broker"} -forwardAuthorizationCredentials=true -``` + Authentication saslAuth = AuthenticationFactory + .create(org.apache.pulsar.client.impl.auth.AuthenticationSasl.class.getName(), authParams); + + PulsarClient client = PulsarClient.builder() + .serviceUrl("pulsar://my-broker.com:6650") + .authentication(saslAuth) + .build(); + ``` -The first part relates to authenticating between clients and Pulsar proxies. In this phase, client works as SASL client, while Pulsar Proxy works as SASL server. + :::note + + - To configure clients for proxies, you need to set `serverType` to `proxy` instead of `broker`. + - The first two lines in the above example are hard-coded. Alternatively, you can set additional JVM parameters for `pulsar_jaas.conf` and `krb5.conf` files when you run the application like below: -The second part relates to authenticating between Pulsar proxies and Pulsar brokers. In this phase, Pulsar Proxy works as SASL client, while Pulsar broker works as SASL server. + ```shell + java -cp -Djava.security.auth.login.config=/etc/pulsar/pulsar_jaas.conf -Djava.security.krb5.conf=/etc/pulsar/krb5.conf $APP-jar-with-dependencies.jar $CLASSNAME + ``` -### Configure brokers + ::: -The broker-side configuration file is the same as the above `broker.conf`, you do not need special configurations for Pulsar Proxy. +## Configure Kerberos authentication in CLI tools -```conf -authenticationEnabled=true -authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderSasl -saslJaasClientAllowedIds=.*client.* -saslJaasServerSectionName=PulsarBroker -``` +[Command-line tools](reference-cli-tools.md) like [`pulsar-admin`](/tools/pulsar-admin/), [`pulsar-perf`](reference-cli-tools.md#pulsar-perf), and [`pulsar-client`](reference-cli-tools.md#pulsar-client) use the `conf/client.conf` file in a Pulsar installation. -## Authorization and role token +When using command-line tools, you need to perform the following steps: -For Kerberos authentication, we usually use the authenticated principal as the role token for Pulsar authorization. For more information on authorization in Pulsar, see [security authorization](security-authorization.md). +1. Configure the `conf/client.conf` file. -If you enable 'authorizationEnabled', you need to set `superUserRoles` in `broker.conf` that corresponds to the name registered in kdc. + ```shell + authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationSasl + authParams={"saslJaasClientSectionName":"PulsarClient", "serverType":"broker"} + ``` -For example: +2. Set JVM parameters for the `pulsar_jaas.conf` file and `krb5.conf` files with additional options. -```bash -superUserRoles=client/{clientIp}@EXAMPLE.COM -``` + ```shell + -Djava.security.auth.login.config=/etc/pulsar/pulsar_jaas.conf -Djava.security.krb5.conf=/etc/pulsar/krb5.conf + ``` + + You can add this at the end of `PULSAR_EXTRA_OPTS` in the file [`pulsar_tools_env.sh`](https://github.com/apache/pulsar/blob/master/conf/pulsar_tools_env.sh), or add this line `OPTS="$OPTS -Djava.security.auth.login.config=/etc/pulsar/pulsar_jaas.conf -Djava.security.krb5.conf=/etc/pulsar/krb5.conf"` directly to the CLI tool script. The meaning of configurations is the same as the meaning of configurations in Java client section. ## Configure Kerberos authentication between ZooKeeper and broker -Pulsar broker acts as a Kerberos client when you authenticate with Zookeeper. According to [ZooKeeper document](https://cwiki.apache.org/confluence/display/ZOOKEEPER/Client-Server+mutual+authentication), you need these settings in `conf/zookeeper.conf`: +Pulsar broker acts as a Kerberos client when authenticating with Zookeeper. -```conf -authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider -requireClientAuthScheme=sasl -``` +1. Add the settings in `conf/zookeeper.conf`. -Enter the following commands to add a section of `Client` configurations in the file `pulsar_jaas.conf`, which Pulsar broker uses: + ```conf + authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider + requireClientAuthScheme=sasl + ``` -``` - Client { - com.sun.security.auth.module.Krb5LoginModule required - useKeyTab=true - storeKey=true - useTicketCache=false - keyTab="/etc/security/keytabs/pulsarbroker.keytab" - principal="broker/localhost@EXAMPLE.COM"; -}; -``` +2. Enter the following commands to add a section of `Client` configurations in `pulsar_jaas.conf` that Pulsar broker uses: -In this setting, the principal of Pulsar broker and keyTab file indicates the role of brokers when you authenticate with ZooKeeper. + ``` + Client { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + useTicketCache=false + keyTab="/etc/security/keytabs/pulsarbroker.keytab" + principal="broker/localhost@EXAMPLE.COM"; + }; + ``` -## Configure Kerberos authentication between BookKeeper and broker + In this setting, the principal of Pulsar broker and keytab file indicates the role of brokers when you authenticate with ZooKeeper. -Pulsar broker acts as a Kerberos client when you authenticate with Bookie. According to [BookKeeper document](https://bookkeeper.apache.org/docs/next/security/sasl/), you need to add `bookkeeperClientAuthenticationPlugin` parameter in `broker.conf`: +For more information, see [ZooKeeper document](https://cwiki.apache.org/confluence/display/ZOOKEEPER/Client-Server+mutual+authentication) -```conf -bookkeeperClientAuthenticationPlugin=org.apache.bookkeeper.sasl.SASLClientProviderFactory -``` +## Configure Kerberos authentication for BookKeeper and broker -In this setting, `SASLClientProviderFactory` creates a BookKeeper SASL client in a broker, and the broker uses the created SASL client to authenticate with a Bookie node. +Pulsar broker acts as a Kerberos client when authenticating with Bookie. -Enter the following commands to add a section of `BookKeeper` configurations in the `pulsar_jaas.conf` that Pulsar broker uses: +1. Add the `bookkeeperClientAuthenticationPlugin` parameter in `broker.conf`. -```conf - BookKeeper { - com.sun.security.auth.module.Krb5LoginModule required - useKeyTab=true - storeKey=true - useTicketCache=false - keyTab="/etc/security/keytabs/pulsarbroker.keytab" - principal="broker/localhost@EXAMPLE.COM"; -}; -``` + ```conf + bookkeeperClientAuthenticationPlugin=org.apache.bookkeeper.sasl.SASLClientProviderFactory + ``` + + `SASLClientProviderFactory` creates a BookKeeper SASL client in a broker, and the broker uses the created SASL client to authenticate with a Bookie node. + +2. Add a section of `BookKeeper` configurations in the `pulsar_jaas.conf` file that broker/proxy uses. + + ```conf + BookKeeper { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + useTicketCache=false + keyTab="/etc/security/keytabs/pulsarbroker.keytab" + principal="broker/localhost@EXAMPLE.COM"; + }; + ``` + + In this setting, the principal of Pulsar broker and keytab file indicates the role of brokers when you authenticate with Bookie. -In this setting, the principal of Pulsar broker and keytab file indicates the role of brokers when you authenticate with Bookie. +For more information, see [BookKeeper document](https://bookkeeper.apache.org/docs/next/security/sasl/). \ No newline at end of file diff --git a/site2/docs/security-oauth2.md b/site2/docs/security-oauth2.md index ddb7bb4fc07b6..abedc3bba577c 100644 --- a/site2/docs/security-oauth2.md +++ b/site2/docs/security-oauth2.md @@ -1,74 +1,34 @@ --- id: security-oauth2 -title: Client authentication using OAuth 2.0 access tokens +title: Authentication using OAuth 2.0 access tokens sidebar_label: "Authentication using OAuth 2.0 access tokens" --- -Pulsar supports authenticating clients using OAuth 2.0 access tokens. You can use OAuth 2.0 access tokens to identify a Pulsar client and associate the Pulsar client with some "principal" (or "role"), which is permitted to do some actions, such as publishing messages to a topic or consuming messages from a topic. - -This module is used to support the [Pulsar client authentication plugin](security-extending.md#client-authentication-plugin) for OAuth 2.0. After communicating with the OAuth 2.0 server, the Pulsar client gets an `access token` from the OAuth 2.0 server, and passes this `access token` to the Pulsar broker to do the authentication. The broker can use the `org.apache.pulsar.broker.authentication.AuthenticationProviderToken`. Or, you can add your own `AuthenticationProvider` to make it with this module. - -## Authentication provider configuration - -This library allows you to authenticate the Pulsar client by using an access token that is obtained from an OAuth 2.0 authorization service, which acts as a _token issuer_. - -### Authentication types - -The authentication type determines how to obtain an access token through an OAuth 2.0 authorization flow. - -:::note - -Currently, the Pulsar Java client only supports the `client_credentials` authentication type. - -::: - -#### Client credentials - -The following table lists parameters supported for the `client credentials` authentication type. - -| Parameter | Description | Example | Required or not | -| --- | --- | --- | --- | -| `type` | OAuth 2.0 authentication type. | `client_credentials` (default) | Optional | -| `issuerUrl` | URL of the authentication provider which allows the Pulsar client to obtain an access token | `https://accounts.google.com` | Required | -| `privateKey` | URL to a JSON credentials file | Support the following pattern formats:
  • `file:///path/to/file`
  • `file:/path/to/file`
  • `data:application/json;base64,`
    • | Required | -| `audience` | An OAuth 2.0 "resource server" identifier for the Pulsar cluster | `https://broker.example.com` | Optional | -| `scope` | Scope of an access request.
    For more information, see [access token scope](https://datatracker.ietf.org/doc/html/rfc6749#section-3.3). | api://pulsar-cluster-1/.default | Optional | +````mdx-code-block +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +```` -The credentials file contains service account credentials used with the client authentication type. The following shows an example of a credentials file `credentials_file.json`. +Pulsar supports authenticating clients using OAuth 2.0 access tokens. Using an access token obtained from an OAuth 2.0 authorization service (acts as a token issuer), you can identify a Pulsar client and associate it with a "principal" (or "role") that is permitted to do some actions, such as publishing messages to a topic or consuming messages from a topic. -```json -{ - "type": "client_credentials", - "client_id": "d9ZyX97q1ef8Cr81WHVC4hFQ64vSlDK3", - "client_secret": "on1uJ...k6F6R", - "client_email": "1234567890-abcdefghijklmnopqrstuvwxyz@developer.gserviceaccount.com", - "issuer_url": "https://accounts.google.com" -} -``` +After communicating with the OAuth 2.0 server, the Pulsar client gets an access token from the server and passes this access token to brokers for authentication. By default, brokers can use the `org.apache.pulsar.broker.authentication.AuthenticationProviderToken`. Alternatively, you can customize the value of `AuthenticationProvider`. -In the above example, the authentication type is set to `client_credentials` by default. And the fields "client_id" and "client_secret" are required. +## Enable OAuth2 authentication on brokers/proxies -### Typical original OAuth2 request mapping +To configure brokers to authenticate clients using OAuth2， add the following parameters to the `conf/broker.conf` and `conf/proxy.conf` file. -The following shows a typical original OAuth2 request, which is used to obtain the access token from the OAuth2 server. - -```bash -curl --request POST \ - --url https://dev-kt-aa9ne.us.auth0.com/oauth/token \ - --header 'content-type: application/json' \ - --data '{ - "client_id":"Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x", - "client_secret":"rT7ps7WY8uhdVuBTKWZkttwLdQotmdEliaM5rLfmgNibvqziZ-g07ZH52N_poGAb", - "audience":"https://dev-kt-aa9ne.us.auth0.com/api/v2/", - "grant_type":"client_credentials"}' +```properties +# Configuration to enable authentication +authenticationEnabled=true +authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken +tokenPublicKey=/path/to/publicKey +# Authentication settings of the broker itself. Used when the broker connects to other brokers, +# either in same or other clusters +brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 +brokerClientAuthenticationParameters={"privateKey":"/path/to/privateKey",\ + "audience":"https://dev-kt-aa9ne.us.auth0.com/api/v2/","issuerUrl":"https://dev-kt-aa9ne.us.auth0.com"} ``` -In the above example, the mapping relationship is shown as below. - -- The `issuerUrl` parameter in this plugin is mapped to `--url https://dev-kt-aa9ne.us.auth0.com`. -- The `privateKey` file parameter in this plugin should at least contains the `client_id` and `client_secret` fields. -- The `audience` parameter in this plugin is mapped to `"audience":"https://dev-kt-aa9ne.us.auth0.com/api/v2/"`. This field is only used by some identity providers. - ## Configure OAuth2 authentication in Pulsar clients You can use the OAuth2 authentication provider with the following Pulsar clients. @@ -204,28 +164,15 @@ client, err := pulsar.NewClient(pulsar.ClientOptions{ ```` -## Broker configuration -To enable OAuth2 authentication in brokers, add the following parameters to the `broker.conf` or `standalone.conf` file. - -```properties -# Configuration to enable authentication -authenticationEnabled=true -authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken -tokenPublicKey=/path/to/publicKey -# Authentication settings of the broker itself. Used when the broker connects to other brokers, -# either in same or other clusters -brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 -brokerClientAuthenticationParameters={"privateKey":"/path/to/privateKey",\ - "audience":"https://dev-kt-aa9ne.us.auth0.com/api/v2/","issuerUrl":"https://dev-kt-aa9ne.us.auth0.com"} -``` - ## Configure OAuth2 authentication in CLI tools This section describes how to use Pulsar CLI tools to connect a cluster through OAuth2 authentication plugin. -### pulsar-admin - -This example shows how to use pulsar-admin to connect to a cluster through OAuth2 authentication plugin. +````mdx-code-block + + ```shell bin/pulsar-admin --admin-url https://streamnative.cloud:443 \ @@ -236,12 +183,8 @@ bin/pulsar-admin --admin-url https://streamnative.cloud:443 \ tenants list ``` -Set the `admin-url` parameter to the Web service URL. A Web service URL is a combination of the protocol, hostname and port ID, such as `pulsar://localhost:6650`. -Set the `privateKey`, `issuerUrl`, and `audience` parameters to the values based on the configuration in the key file. For details, see [authentication types](#authentication-types). - -### pulsar-client - -This example shows how to use pulsar-client to connect to a cluster through OAuth2 authentication plugin. + + ```shell bin/pulsar-client \ @@ -253,12 +196,8 @@ bin/pulsar-client \ produce test-topic -m "test-message" -n 10 ``` -Set the `admin-url` parameter to the Web service URL. A Web service URL is a combination of the protocol, hostname and port ID, such as `pulsar://localhost:6650`. -Set the `privateKey`, `issuerUrl`, and `audience` parameters to the values based on the configuration in the key file. For details, see [authentication types](#authentication-types). - -### pulsar-perf - -This example shows how to use pulsar-perf to connect to a cluster through OAuth2 authentication plugin. + + ```shell bin/pulsar-perf produce --service-url pulsar+ssl://streamnative.cloud:6651 \ @@ -269,5 +208,53 @@ bin/pulsar-perf produce --service-url pulsar+ssl://streamnative.cloud:6651 \ -r 1000 -s 1024 test-topic ``` -Set the `admin-url` parameter to the Web service URL. A Web service URL is a combination of the protocol, hostname and port ID, such as `pulsar://localhost:6650`. -Set the `privateKey`, `issuerUrl`, and `audience` parameters to the values based on the configuration in the key file. For details, see [authentication types](#authentication-types). + + +```` + +* Set the `admin-url` parameter to the Web service URL. A Web service URL is a combination of the protocol, hostname and port ID, such as `pulsar://localhost:6650`. +* Set the `privateKey`, `issuerUrl`, and `audience` parameters to the values based on the configuration in the key file. For details, see [authentication types](#authentication-types). + +#### Authentication types + +Currently, Pulsar clients only support the `client_credentials` authentication type. The authentication type determines how to obtain an access token through an OAuth 2.0 authorization service. + +The following table outlines the parameters of the `client_credentials` authentication type. + +| Parameter | Description | Example | Required or not | +| --- | --- | --- | --- | +| `type` | OAuth 2.0 authentication type. | `client_credentials` (default) | Optional | +| `issuerUrl` | The URL of the authentication provider which allows the Pulsar client to obtain an access token. | `https://accounts.google.com` | Required | +| `privateKey` | The URL to the JSON credentials file. | Support the following pattern formats:
  • `file:///path/to/file`
  • `file:/path/to/file`
  • `data:application/json;base64,`
    • | Required | +| `audience` | The OAuth 2.0 "resource server" identifier for a Pulsar cluster. | `https://broker.example.com` | Optional | +| `scope` | The scope of an access request.
    For more information, see [access token scope](https://datatracker.ietf.org/doc/html/rfc6749#section-3.3). | api://pulsar-cluster-1/.default | Optional | + +The credentials file `credentials_file.json` contains the service account credentials used with the client authentication type. The following is an example of the credentials file. The authentication type is set to `client_credentials` by default. And the fields "client_id" and "client_secret" are required. + +```json +{ + "type": "client_credentials", + "client_id": "d9ZyX97q1ef8Cr81WHVC4hFQ64vSlDK3", + "client_secret": "on1uJ...k6F6R", + "client_email": "1234567890-abcdefghijklmnopqrstuvwxyz@developer.gserviceaccount.com", + "issuer_url": "https://accounts.google.com" +} +``` + +The following is an example of a typical original OAuth2 request, which is used to obtain an access token from the OAuth2 server. + +```bash +curl --request POST \ + --url https://dev-kt-aa9ne.us.auth0.com/oauth/token \ + --header 'content-type: application/json' \ + --data '{ + "client_id":"Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x", + "client_secret":"rT7ps7WY8uhdVuBTKWZkttwLdQotmdEliaM5rLfmgNibvqziZ-g07ZH52N_poGAb", + "audience":"https://dev-kt-aa9ne.us.auth0.com/api/v2/", + "grant_type":"client_credentials"}' +``` + +In the above example, the mapping relationship is shown below. +- The `issuerUrl` parameter is mapped to `--url https://dev-kt-aa9ne.us.auth0.com`. +- The `privateKey` parameter should contain the `client_id` and `client_secret` fields at least. +- The `audience` parameter is mapped to `"audience":"https://dev-kt-aa9ne.us.auth0.com/api/v2/"`. This field is only used by some identity providers. diff --git a/site2/docs/security-overview.md b/site2/docs/security-overview.md index 37d96d65b01fe..917b7ac7af2d8 100644 --- a/site2/docs/security-overview.md +++ b/site2/docs/security-overview.md @@ -21,28 +21,41 @@ Encryption ensures that if an attacker gets access to your data, the attacker ca **What's next?** -* To configure end-to-end encryption, see [End-to-end encryption](security-encryption.md) for more details. -* To configure transport layer encryption, see [TLS encryption](security-tls-transport.md) for more details. +- To configure end-to-end encryption, see [End-to-end encryption](security-encryption.md) for more details. +- To configure transport layer encryption, see [TLS encryption](security-tls-transport.md) for more details. ## Authentication -Authentication is the process of verifying the identity of clients. In Pulsar, the authentication provider is responsible for properly identifying clients and associating the clients with role tokens. If you only enable authentication, an authenticated role token can access all resources in the cluster. +Authentication is the process of verifying the identity of clients. In Pulsar, the authentication provider is responsible for properly identifying clients and associating them with role tokens. Note that if you only enable authentication, an authenticated role token can access all resources in the cluster. -Pulsar supports a pluggable authentication mechanism, and Pulsar clients use this mechanism to authenticate with brokers and proxies. +**How it works in Pulsar** -Pulsar broker validates the authentication credentials when a connection is established. After the initial connection is authenticated, the "principal" token is stored for authorization though the connection is not re-authenticated. The broker periodically checks the expiration status of every `ServerCnx` object. By default, the `authenticationRefreshCheckSeconds` is set to 60s. When the authentication is expired, the broker re-authenticates the connection. If the re-authentication fails, the broker disconnects the client. +Pulsar provides a pluggable authentication framework, and Pulsar brokers/proxies use this mechanism to authenticate clients. -Pulsar broker supports learning whether a particular client supports authentication refreshing. If a client supports authentication refreshing and the credential is expired, the authentication provider calls the `refreshAuthentication` method to initiate the refreshing process. If a client does not support authentication refreshing and the credential is expired, the broker disconnects the client. +The way how each client passes its authentication data to brokers varies depending on the protocols it uses. Brokers validate the authentication credentials when a connection is established and check whether the authentication data is expired. +- When using HTTP/HTTPS protocol for cluster management, each client passes the authentication data based on the HTTP/HTTPS request header, and brokers check the data upon request. +- When using [Pulsar protocol](developing-binary-protocol.md) for productions/consumptions, each client passes the authentication data by sending the `CommandConnect` command when connecting to brokers. Brokers cache the data and periodically check whether the data has expired and learn whether the client supports authentication refreshing. By default, the `authenticationRefreshCheckSeconds` is set to 60s. + - If a client supports authentication refreshing and the credential is expired, brokers send the `CommandAuthChallenge` command to exchange the authentication data with the client. If the next check finds that the previous authentication exchange has not been returned, brokers disconnect the client. + - If a client does not support authentication refreshing and the credential is expired, brokers disconnect the client. + +:::note + +When you use proxies between clients and brokers, brokers only authenticate proxies (known as **self-authentication**) by default. To forward the authentication data from clients to brokers for client authentication (known as **original authentication**), you need to: +1. Set `forwardAuthorizationCredentials` to `true` in the `conf/proxy.conf` file. +2. Set `authenticateOriginalAuthData` to `true` in the `conf/broker.conf` file, which ensures that brokers recheck the client authentication. + +::: **What's next?** -Pulsar supports the following authentication providers, and you can configure multiple authentication providers. -- [TLS authentication](security-tls-authentication.md) -- [Athenz authentication](security-athenz.md) -- [Kerberos authentication](security-kerberos.md) -- [JSON Web Token (JWT) authentication](security-jwt.md) -- [OAuth 2.0 authentication](security-oauth2.md) -- [HTTP basic authentication](security-basic-auth.md) +- To configure built-in authentication plugins, read: + - [TLS authentication](security-tls-authentication.md) + - [Athenz authentication](security-athenz.md) + - [Kerberos authentication](security-kerberos.md) + - [JSON Web Token (JWT) authentication](security-jwt.md) + - [OAuth 2.0 authentication](security-oauth2.md) + - [HTTP basic authentication](security-basic-auth.md) +- To customize an authentication plugin, read [extended authentication](security-extending). :::note diff --git a/site2/docs/security-tls-authentication.md b/site2/docs/security-tls-authentication.md index 4f907933abe56..7b86ca8e90640 100644 --- a/site2/docs/security-tls-authentication.md +++ b/site2/docs/security-tls-authentication.md @@ -102,12 +102,12 @@ brokerClientAuthenticationParameters=tlsCertFile:/path/to/proxy.cert.pem,tlsKeyF ## Configure TLS authentication in Pulsar clients -When you use TLS authentication, client connects via TLS transport. You need to configure the client to use `https://` and 8443 port for the web service URL, `pulsar+ssl://` and 6651 port for the broker service URL. +When using TLS authentication, clients connect via TLS transport. You need to configure clients to use `https://` and the `8443` port for the web service URL, use `pulsar+ssl://` and the `6651` port for the broker service URL. ````mdx-code-block + values={[{"label":"Java","value":"Java"},{"label":"Python","value":"Python"},{"label":"C++","value":"C++"},{"label":"Node.js","value":"Node.js"},{"label":"Go","value":"Go"},{"label":"C#","value":"C#"}]}> ```java @@ -172,6 +172,17 @@ const Pulsar = require('pulsar-client'); })(); ``` + + + +```go +client, err := pulsar.NewClient(ClientOptions{ + URL: "pulsar+ssl://broker.example.com:6651/", + TLSTrustCertsFilePath: "/path/to/ca.cert.pem", + Authentication: pulsar.NewAuthenticationTLS("/path/to/my-role.cert.pem", "/path/to/my-role.key-pk8.pem"), + }) +``` + From 59e00bab889bec3749a1cf9266353b8f8bc3d5af Mon Sep 17 00:00:00 2001 From: tison Date: Thu, 27 Oct 2022 09:32:56 +0800 Subject: [PATCH 05/22] [improve][doc] cherry-pick cpp client docs installation section (#18188) * [improve][doc] cherry-pick cpp client docs installation section Signed-off-by: tison * Apply suggestions from code review Co-authored-by: momo-jun <60642177+momo-jun@users.noreply.github.com> * Update client-libraries-cpp.md Signed-off-by: tison Co-authored-by: momo-jun <60642177+momo-jun@users.noreply.github.com> --- site2/docs/client-libraries-cpp.md | 2 +- .../version-2.10.x/client-libraries-cpp.md | 370 ++++-------------- .../version-2.7.5/client-libraries-cpp.md | 164 +++----- .../version-2.8.x/client-libraries-cpp.md | 270 +++---------- .../version-2.9.x/client-libraries-cpp.md | 343 ++++------------ 5 files changed, 253 insertions(+), 896 deletions(-) diff --git a/site2/docs/client-libraries-cpp.md b/site2/docs/client-libraries-cpp.md index 58d1cef4b3c20..7f56d3cc72770 100644 --- a/site2/docs/client-libraries-cpp.md +++ b/site2/docs/client-libraries-cpp.md @@ -78,7 +78,7 @@ This package contains shared libraries: `libpulsar.so` and `libpulsarnossl.so`. wget @pulsar:dist_rpm:client-debuginfo@ ``` -This package contains debug symbols for `libpulsar.so` +This package contains debug symbols for `libpulsar.so`. diff --git a/site2/website/versioned_docs/version-2.10.x/client-libraries-cpp.md b/site2/website/versioned_docs/version-2.10.x/client-libraries-cpp.md index e8bcfa0abe118..e88df87b65453 100644 --- a/site2/website/versioned_docs/version-2.10.x/client-libraries-cpp.md +++ b/site2/website/versioned_docs/version-2.10.x/client-libraries-cpp.md @@ -5,341 +5,111 @@ sidebar_label: "C++" original_id: client-libraries-cpp --- -You can use Pulsar C++ client to create Pulsar producers and consumers in C++. +````mdx-code-block +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +```` -All the methods in producer, consumer, and reader of a C++ client are thread-safe. +You can use a Pulsar C++ client to create producers, consumers, and readers. -## Supported platforms +All the methods in producer, consumer, and reader of a C++ client are thread-safe. You can read the [API docs](/api/cpp) for the C++ client. -Pulsar C++ client is supported on **Linux** ,**macOS** and **Windows** platforms. +## Installation -[Doxygen](http://www.doxygen.nl/)-generated API docs for the C++ client are available [here](/api/cpp). +Use one of the following methods to install a Pulsar C++ client. +### Brew -## Linux - -:::note - -You can choose one of the following installation methods based on your needs: Compilation, Install RPM or Install Debian. - -::: - -### Compilation - -#### System requirements - -You need to install the following components before using the C++ client: - -* [CMake](https://cmake.org/) -* [Boost](http://www.boost.org/) -* [Protocol Buffers](https://developers.google.com/protocol-buffers/) >= 3 -* [libcurl](https://curl.se/libcurl/) -* [Google Test](https://github.com/google/googletest) - -1. Clone the Pulsar repository. - -```shell - -$ git clone https://github.com/apache/pulsar - -``` - -2. Install all necessary dependencies. - -```shell - -$ apt-get install cmake libssl-dev libcurl4-openssl-dev liblog4cxx-dev \ - libprotobuf-dev protobuf-compiler libboost-all-dev google-mock libgtest-dev libjsoncpp-dev - -``` - -3. Compile and install [Google Test](https://github.com/google/googletest). - -```shell - -# libgtest-dev version is 1.18.0 or above -$ cd /usr/src/googletest -$ sudo cmake . -$ sudo make -$ sudo cp ./googlemock/libgmock.a ./googlemock/gtest/libgtest.a /usr/lib/ - -# less than 1.18.0 -$ cd /usr/src/gtest -$ sudo cmake . -$ sudo make -$ sudo cp libgtest.a /usr/lib - -$ cd /usr/src/gmock -$ sudo cmake . -$ sudo make -$ sudo cp libgmock.a /usr/lib - -``` - -4. Compile the Pulsar client library for C++ inside the Pulsar repository. - -```shell - -$ cd pulsar-client-cpp -$ cmake . -$ make - -``` - -After you install the components successfully, the files `libpulsar.so` and `libpulsar.a` are in the `lib` folder of the repository. The tools `perfProducer` and `perfConsumer` are in the `perf` directory. - -### Install Dependencies - -> Since 2.1.0 release, Pulsar ships pre-built RPM and Debian packages. You can download and install those packages directly. - -After you download and install RPM or DEB, the `libpulsar.so`, `libpulsarnossl.so`, `libpulsar.a`, and `libpulsarwithdeps.a` libraries are in your `/usr/lib` directory. - -By default, they are built in code path `${PULSAR_HOME}/pulsar-client-cpp`. You can build with the command below. - - `cmake . -DBUILD_TESTS=OFF -DLINK_STATIC=ON && make pulsarShared pulsarSharedNossl pulsarStatic pulsarStaticWithDeps -j 3`. - -These libraries rely on some other libraries. If you want to get detailed version of dependencies, see [RPM](https://github.com/apache/pulsar/blob/master/pulsar-client-cpp/pkg/rpm/Dockerfile) or [DEB](https://github.com/apache/pulsar/blob/master/pulsar-client-cpp/pkg/deb/Dockerfile) files. - -1. `libpulsar.so` is a shared library, containing statically linked `boost` and `openssl`. It also dynamically links all other necessary libraries. You can use this Pulsar library with the command below. +Use [Homebrew](http://brew.sh/) to install the latest tagged version with the library and headers: ```bash - - g++ --std=c++11 PulsarTest.cpp -o test /usr/lib/libpulsar.so -I/usr/local/ssl/include - -``` - -2. `libpulsarnossl.so` is a shared library, similar to `libpulsar.so` except that the libraries `openssl` and `crypto` are dynamically linked. You can use this Pulsar library with the command below. - -```bash - - g++ --std=c++11 PulsarTest.cpp -o test /usr/lib/libpulsarnossl.so -lssl -lcrypto -I/usr/local/ssl/include -L/usr/local/ssl/lib - +brew install libpulsar ``` -3. `libpulsar.a` is a static library. You need to load dependencies before using this library. You can use this Pulsar library with the command below. +### Deb -```bash +1. Download any one of the Deb packages: - g++ --std=c++11 PulsarTest.cpp -o test /usr/lib/libpulsar.a -lssl -lcrypto -ldl -lpthread -I/usr/local/ssl/include -L/usr/local/ssl/lib -lboost_system -lboost_regex -lcurl -lprotobuf -lzstd -lz - -``` - -4. `libpulsarwithdeps.a` is a static library, based on `libpulsar.a`. It is archived in the dependencies of `libboost_regex`, `libboost_system`, `libcurl`, `libprotobuf`, `libzstd` and `libz`. You can use this Pulsar library with the command below. + + ```bash - - g++ --std=c++11 PulsarTest.cpp -o test /usr/lib/libpulsarwithdeps.a -lssl -lcrypto -ldl -lpthread -I/usr/local/ssl/include -L/usr/local/ssl/lib - +wget @pulsar:deb:client@ ``` -The `libpulsarwithdeps.a` does not include library openssl related libraries `libssl` and `libcrypto`, because these two libraries are related to security. It is more reasonable and easier to use the versions provided by the local system to handle security issues and upgrade libraries. - -### Install RPM +This package contains shared libraries `libpulsar.so` and `libpulsarnossl.so`. -1. Download an RPM package from the links in the table. - -| Link | Crypto files | -|------|--------------| -| [client](@pulsar:dist_rpm:client@) | [asc](@pulsar:dist_rpm:client@.asc), [sha512](@pulsar:dist_rpm:client@.sha512) | -| [client-debuginfo](@pulsar:dist_rpm:client-debuginfo@) | [asc](@pulsar:dist_rpm:client-debuginfo@.asc), [sha512](@pulsar:dist_rpm:client-debuginfo@.sha512) | -| [client-devel](@pulsar:dist_rpm:client-devel@) | [asc](@pulsar:dist_rpm:client-devel@.asc), [sha512](@pulsar:dist_rpm:client-devel@.sha512) | - -2. Install the package using the following command. + + ```bash - -$ rpm -ivh apache-pulsar-client*.rpm - -``` - -After you install RPM successfully, Pulsar libraries are in the `/usr/lib` directory, for example： - -```bash - -lrwxrwxrwx 1 root root 18 Dec 30 22:21 libpulsar.so -> libpulsar.so.2.9.1 -lrwxrwxrwx 1 root root 23 Dec 30 22:21 libpulsarnossl.so -> libpulsarnossl.so.2.9.1 - +wget @pulsar:deb:client-devel@ ``` -:::note +This package contains static libraries: `libpulsar.a`, `libpulsarwithdeps.a` and C/C++ headers. -If you get the error that `libpulsar.so: cannot open shared object file: No such file or directory` when starting Pulsar client, you may need to run `ldconfig` first. + + -::: - -2. Install the GCC and g++ using the following command, otherwise errors would occur in installing Node.js. +2. Install the package using the following command: ```bash - -$ sudo yum -y install gcc automake autoconf libtool make -$ sudo yum -y install gcc-c++ - +apt install ./apache-pulsar-client*.deb ``` -### Install Debian +Now, you can see Pulsar C++ client libraries installed under the `/usr/lib` directory. -1. Download a Debian package from the links in the table. +### RPM -| Link | Crypto files | -|------|--------------| -| [client](@pulsar:deb:client@) | [asc](@pulsar:dist_deb:client@.asc), [sha512](@pulsar:dist_deb:client@.sha512) | -| [client-devel](@pulsar:deb:client-devel@) | [asc](@pulsar:dist_deb:client-devel@.asc), [sha512](@pulsar:dist_deb:client-devel@.sha512) | +1. Download any one of the RPM packages: -2. Install the package using the following command. + + ```bash - -$ apt install ./apache-pulsar-client*.deb - +wget @pulsar:dist_rpm:client@ ``` -After you install DEB successfully, Pulsar libraries are in the `/usr/lib` directory. - -### Build +This package contains shared libraries: `libpulsar.so` and `libpulsarnossl.so`. -> If you want to build RPM and Debian packages from the latest master, follow the instructions below. You should run all the instructions at the root directory of your cloned Pulsar repository. - -There are recipes that build RPM and Debian packages containing a -statically linked `libpulsar.so` / `libpulsarnossl.so` / `libpulsar.a` / `libpulsarwithdeps.a` with all required dependencies. - -To build the C++ library packages, you need to build the Java packages first. - -```shell - -mvn install -DskipTests - -``` - -#### RPM - -To build the RPM inside a Docker container, use the command below. The RPMs are in the `pulsar-client-cpp/pkg/rpm/RPMS/x86_64/` path. - -```shell - -pulsar-client-cpp/pkg/rpm/docker-build-rpm.sh - -``` - -| Package name | Content | -|-----|-----| -| pulsar-client | Shared library `libpulsar.so` and `libpulsarnossl.so` | -| pulsar-client-devel | Static library `libpulsar.a`, `libpulsarwithdeps.a`and C++ and C headers | -| pulsar-client-debuginfo | Debug symbols for `libpulsar.so` | - -#### Debian - -To build Debian packages, enter the following command. - -```shell - -pulsar-client-cpp/pkg/deb/docker-build-deb.sh - -``` - -Debian packages are created in the `pulsar-client-cpp/pkg/deb/BUILD/DEB/` path. - -| Package name | Content | -|-----|-----| -| pulsar-client | Shared library `libpulsar.so` and `libpulsarnossl.so` | -| pulsar-client-dev | Static library `libpulsar.a`, `libpulsarwithdeps.a` and C++ and C headers | - -## MacOS - -### Compilation - -1. Clone the Pulsar repository. - -```shell - -$ git clone https://github.com/apache/pulsar - -``` - -2. Install all necessary dependencies. - -```shell - -# OpenSSL installation -$ brew install openssl -$ export OPENSSL_INCLUDE_DIR=/usr/local/opt/openssl/include/ -$ export OPENSSL_ROOT_DIR=/usr/local/opt/openssl/ - -# Protocol Buffers installation -$ brew install protobuf boost boost-python log4cxx -# If you are using python3, you need to install boost-python3 - -# Google Test installation -$ git clone https://github.com/google/googletest.git -$ cd googletest -$ git checkout release-1.12.1 -$ cmake . -$ make install - -``` - -3. Compile the Pulsar client library in the repository that you cloned. - -```shell - -$ cd pulsar-client-cpp -$ cmake . -$ make - -``` - -### Install `libpulsar` - -Pulsar releases are available in the [Homebrew](https://brew.sh/) core repository. You can install the C++ client library with the following command. The package is installed with the library and headers. - -```shell - -brew install libpulsar + + +```bash +wget @pulsar:dist_rpm:client-debuginfo@ ``` -## Windows (64-bit) - -### Compilation - -1. Clone the Pulsar repository. +This package contains debug symbols for `libpulsar.so`. -```shell - -$ git clone https://github.com/apache/pulsar + + +```bash +wget @pulsar:dist_rpm:client-devel@ ``` -2. Install all necessary dependencies. +This package contains static libraries: `libpulsar.a`, `libpulsarwithdeps.a` and C/C++ headers. -```shell + + -cd ${PULSAR_HOME}/pulsar-client-cpp -vcpkg install --feature-flags=manifests --triplet x64-windows +2. Install the package using the following command: +```bash +rpm -ivh apache-pulsar-client*.rpm ``` -3. Build C++ libraries. - -```shell - -cmake -B ./build -A x64 -DBUILD_PYTHON_WRAPPER=OFF -DBUILD_TESTS=OFF -DVCPKG_TRIPLET=x64-windows -DCMAKE_BUILD_TYPE=Release -S . -cmake --build ./build --config Release - -``` +Now, you can see Pulsar C++ client libraries installed under the `/usr/lib` directory. -> **NOTE** -> -> 1. For Windows 32-bit, you need to use `-A Win32` and `-DVCPKG_TRIPLET=x86-windows`. -> 2. For MSVC Debug mode, you need to replace `Release` with `Debug` for both `CMAKE_BUILD_TYPE` variable and `--config` option. +:::note -4. Client libraries are available in the following places. +If you get an error like "libpulsar.so: cannot open shared object file: No such file or directory" when starting a Pulsar client, you need to run `ldconfig` first. -``` +::: -${PULSAR_HOME}/pulsar-client-cpp/build/lib/Release/pulsar.lib -${PULSAR_HOME}/pulsar-client-cpp/build/lib/Release/pulsar.dll +### Source -``` +For how to build Pulsar C++ client on different platforms from source code, see [compliation](https://github.com/apache/pulsar-client-cpp#compilation). ## Connection URLs @@ -353,7 +123,7 @@ pulsar://localhost:6650 ``` -In a Pulsar cluster in production, the URL looks as follows. +In a Pulsar cluster in production, the URL looks as follows. ```http @@ -504,7 +274,7 @@ producerConf.setLazyStartPartitionedProducers(true); ### Enable chunking -Message [chunking](concepts-messaging.md#chunking) enables Pulsar to process large payload messages by splitting the message into chunks at the producer side and aggregating chunked messages at the consumer side. +Message [chunking](concepts-messaging.md#chunking) enables Pulsar to process large payload messages by splitting the message into chunks at the producer side and aggregating chunked messages at the consumer side. The message chunking feature is OFF by default. The following is an example about how to enable message chunking when creating a producer. @@ -624,7 +394,7 @@ int main() { ### Configure chunking -You can limit the maximum number of chunked messages a consumer maintains concurrently by configuring the `setMaxPendingChunkedMessage` and `setAutoAckOldestChunkedMessageOnQueueFull` parameters. When the threshold is reached, the consumer drops pending messages by silently acknowledging them or asking the broker to redeliver them later. +You can limit the maximum number of chunked messages a consumer maintains concurrently by configuring the `setMaxPendingChunkedMessage` and `setAutoAckOldestChunkedMessageOnQueueFull` parameters. When the threshold is reached, the consumer drops pending messages by silently acknowledging them or asking the broker to redeliver them later. The following is an example of how to configure message chunking. @@ -666,7 +436,7 @@ schema, see [Pulsar schema](schema-get-started.md). - The following example shows how to create a producer with an Avro schema. ```cpp - + static const std::string exampleSchema = "{\"type\":\"record\",\"name\":\"Example\",\"namespace\":\"test\"," "\"fields\":[{\"name\":\"a\",\"type\":\"int\"},{\"name\":\"b\",\"type\":\"int\"}]}"; @@ -674,13 +444,13 @@ schema, see [Pulsar schema](schema-get-started.md). ProducerConfiguration producerConf; producerConf.setSchema(SchemaInfo(AVRO, "Avro", exampleSchema)); client.createProducer("topic-avro", producerConf, producer); - + ``` - The following example shows how to create a consumer with an Avro schema. ```cpp - + static const std::string exampleSchema = "{\"type\":\"record\",\"name\":\"Example\",\"namespace\":\"test\"," "\"fields\":[{\"name\":\"a\",\"type\":\"int\"},{\"name\":\"b\",\"type\":\"int\"}]}"; @@ -688,14 +458,14 @@ schema, see [Pulsar schema](schema-get-started.md). Consumer consumer; consumerConf.setSchema(SchemaInfo(AVRO, "Avro", exampleSchema)); client.subscribe("topic-avro", "sub-2", consumerConf, consumer) - + ``` ### ProtobufNative schema The following example shows how to create a producer and a consumer with a ProtobufNative schema. ​ -1. Generate the `User` class using Protobuf3. +1. Generate the `User` class using Protobuf3. :::note @@ -706,14 +476,14 @@ The following example shows how to create a producer and a consumer with a Proto ​ ```protobuf - + syntax = "proto3"; - + message User { string name = 1; int32 age = 2; } - + ``` ​ @@ -721,9 +491,9 @@ The following example shows how to create a producer and a consumer with a Proto ​ ```cpp - + #include - + ``` ​ @@ -731,7 +501,7 @@ The following example shows how to create a producer and a consumer with a Proto ​ ```cpp - + ProducerConfiguration producerConf; producerConf.setSchema(createProtobufNativeSchema(User::GetDescriptor())); Producer producer; @@ -742,7 +512,7 @@ The following example shows how to create a producer and a consumer with a Proto std::string content; user.SerializeToString(&content); producer.send(MessageBuilder().setContent(content).build()); - + ``` ​ @@ -750,7 +520,7 @@ The following example shows how to create a producer and a consumer with a Proto ​ ```cpp - + ConsumerConfiguration consumerConf; consumerConf.setSchema(createProtobufNativeSchema(User::GetDescriptor())); consumerConf.setSubscriptionInitialPosition(InitialPositionEarliest); @@ -760,6 +530,6 @@ The following example shows how to create a producer and a consumer with a Proto consumer.receive(msg); User user2; user2.ParseFromArray(msg.getData(), msg.getLength()); - + ``` diff --git a/site2/website/versioned_docs/version-2.7.5/client-libraries-cpp.md b/site2/website/versioned_docs/version-2.7.5/client-libraries-cpp.md index 71dbaa19c153e..204b63f3442b8 100644 --- a/site2/website/versioned_docs/version-2.7.5/client-libraries-cpp.md +++ b/site2/website/versioned_docs/version-2.7.5/client-libraries-cpp.md @@ -5,161 +5,111 @@ sidebar_label: "C++" original_id: client-libraries-cpp --- -You can use Pulsar C++ client to create Pulsar producers and consumers in C++. +````mdx-code-block +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +```` -All the methods in producer, consumer, and reader of a C++ client are thread-safe. +You can use a Pulsar C++ client to create producers, consumers, and readers. -## Supported platforms +All the methods in producer, consumer, and reader of a C++ client are thread-safe. You can read the [API docs](/api/cpp) for the C++ client. -Pulsar C++ client is supported on **Linux** and **MacOS** platforms. +## Installation -[Doxygen](http://www.doxygen.nl/)-generated API docs for the C++ client are available [here](/api/cpp). +Use one of the following methods to install a Pulsar C++ client. -## Linux +### Brew -> Since 2.1.0 release, Pulsar ships pre-built RPM and Debian packages. You can download and install those packages directly. - -Four kind of libraries `libpulsar.so` / `libpulsarnossl.so` / `libpulsar.a` / `libpulsarwithdeps.a` are included in your `/usr/lib` after rpm/deb download and install. -By default, they are build under code path `${PULSAR_HOME}/pulsar-client-cpp`, using command - `cmake . -DBUILD_TESTS=OFF -DLINK_STATIC=ON && make pulsarShared pulsarSharedNossl pulsarStatic pulsarStaticWithDeps -j 3` -These libraries rely on some other libraries, if you want to get detailed version of dependencies libraries, please reference [these](https://github.com/apache/pulsar/blob/master/pulsar-client-cpp/pkg/rpm/Dockerfile) [files](https://github.com/apache/pulsar/blob/master/pulsar-client-cpp/pkg/deb/Dockerfile). - -1. `libpulsar.so` is the Shared library, it contains statically linked `boost` and `openssl`, and will also dynamically link all other needed libraries. -The command the when use this pulsar library is like this: - -```bash - - g++ --std=c++11 PulsarTest.cpp -o test /usr/lib/libpulsar.so -I/usr/local/ssl/include - -``` - -2. `libpulsarnossl.so` is the Shared library that similar to `libpulsar.so` except that the library `openssl` and `crypto` are dynamically linked. -The command the when use this pulsar library is like this: +Use [Homebrew](http://brew.sh/) to install the latest tagged version with the library and headers: ```bash - - g++ --std=c++11 PulsarTest.cpp -o test /usr/lib/libpulsarnossl.so -lssl -lcrypto -I/usr/local/ssl/include -L/usr/local/ssl/lib - +brew install libpulsar ``` -3. `libpulsar.a` is the Static library, it need to load some dependencies library when using it. -The command the when use this pulsar library is like this: - -```bash +### Deb - g++ --std=c++11 PulsarTest.cpp -o test /usr/lib/libpulsar.a -lssl -lcrypto -ldl -lpthread -I/usr/local/ssl/include -L/usr/local/ssl/lib -lboost_system -lboost_regex -lcurl -lprotobuf -lzstd -lz - -``` +1. Download any one of the Deb packages: -4. `libpulsarwithdeps.a` is the Static library, base on `libpulsar.a`, and archived in the dependencies libraries of `libboost_regex`, `libboost_system`, `libcurl`, `libprotobuf`, `libzstd` and `libz`, -The command the when use this pulsar library is like this: + + ```bash - - g++ --std=c++11 PulsarTest.cpp -o test /usr/lib/libpulsarwithdeps.a -lssl -lcrypto -ldl -lpthread -I/usr/local/ssl/include -L/usr/local/ssl/lib - +wget @pulsar:deb:client@ ``` -`libpulsarwithdeps.a` does not include library openssl related libraries: `libssl` and `libcrypto`, because these 2 library is related to security, -by using user local system provided version is more reasonable, and more easy for user to handling security issue and library upgrade. - -### Install RPM - -1. Download a RPM package from the links in the table. - -| Link | Crypto files | -|------|--------------| -| [client](@pulsar:dist_rpm:client@) | [asc](@pulsar:dist_rpm:client@.asc), [sha512](@pulsar:dist_rpm:client@.sha512) | -| [client-debuginfo](@pulsar:dist_rpm:client-debuginfo@) | [asc](@pulsar:dist_rpm:client-debuginfo@.asc), [sha512](@pulsar:dist_rpm:client-debuginfo@.sha512) | -| [client-devel](@pulsar:dist_rpm:client-devel@) | [asc](@pulsar:dist_rpm:client-devel@.asc), [sha512](@pulsar:dist_rpm:client-devel@.sha512) | +This package contains shared libraries `libpulsar.so` and `libpulsarnossl.so`. -2. Install the package using the following command. + + ```bash - -$ rpm -ivh apache-pulsar-client*.rpm - +wget @pulsar:deb:client-devel@ ``` -After install, Pulsar libraries will be placed under `/usr/lib`. - -### Install Debian +This package contains static libraries: `libpulsar.a`, `libpulsarwithdeps.a` and C/C++ headers. -1. Download a Debian package from the links in the table. - -| Link | Crypto files | -|------|--------------| -| [client](@pulsar:deb:client@) | [asc](@pulsar:dist_deb:client@.asc), [sha512](@pulsar:dist_deb:client@.sha512) | -| [client-devel](@pulsar:deb:client-devel@) | [asc](@pulsar:dist_deb:client-devel@.asc), [sha512](@pulsar:dist_deb:client-devel@.sha512) | + + 2. Install the package using the following command: ```bash - -$ apt install ./apache-pulsar-client*.deb - +apt install ./apache-pulsar-client*.deb ``` -After install, Pulsar libraries will be placed under `/usr/lib`. - -### Build - -> If you want to build RPM and Debian packages from the latest master, follow the instructions below. All the instructions are run at the root directory of your cloned Pulsar repository. +Now, you can see Pulsar C++ client libraries installed under the `/usr/lib` directory. -There are recipes that build RPM and Debian packages containing a -statically linked `libpulsar.so` / `libpulsarnossl.so` / `libpulsar.a` / `libpulsarwithdeps.a` with all the required -dependencies. +### RPM -To build the C++ library packages, build the Java packages first. +1. Download any one of the RPM packages: -```shell - -mvn install -DskipTests + + +```bash +wget @pulsar:dist_rpm:client@ ``` -#### RPM +This package contains shared libraries: `libpulsar.so` and `libpulsarnossl.so`. -```shell - -pulsar-client-cpp/pkg/rpm/docker-build-rpm.sh + + +```bash +wget @pulsar:dist_rpm:client-debuginfo@ ``` -This builds the RPM inside a Docker container and it leaves the RPMs in `pulsar-client-cpp/pkg/rpm/RPMS/x86_64/`. +This package contains debug symbols for `libpulsar.so`. -| Package name | Content | -|-----|-----| -| pulsar-client | Shared library `libpulsar.so` and `libpulsarnossl.so` | -| pulsar-client-devel | Static library `libpulsar.a`, `libpulsarwithdeps.a`and C++ and C headers | -| pulsar-client-debuginfo | Debug symbols for `libpulsar.so` | + + -#### Debian +```bash +wget @pulsar:dist_rpm:client-devel@ +``` -To build Debian packages, enter the following command. +This package contains static libraries: `libpulsar.a`, `libpulsarwithdeps.a` and C/C++ headers. -```shell + + -pulsar-client-cpp/pkg/deb/docker-build-deb.sh +2. Install the package using the following command: +```bash +rpm -ivh apache-pulsar-client*.rpm ``` -Debian packages are created at `pulsar-client-cpp/pkg/deb/BUILD/DEB/`. - -| Package name | Content | -|-----|-----| -| pulsar-client | Shared library `libpulsar.so` and `libpulsarnossl.so` | -| pulsar-client-dev | Static library `libpulsar.a`, `libpulsarwithdeps.a` and C++ and C headers | +Now, you can see Pulsar C++ client libraries installed under the `/usr/lib` directory. -## MacOS +:::note -Pulsar releases are available in the [Homebrew](https://brew.sh/) core repository. You can install the C++ client library with the following command. The package is installed with the library and headers. +If you get an error like "libpulsar.so: cannot open shared object file: No such file or directory" when starting a Pulsar client, you need to run `ldconfig` first. -```shell +::: -brew install libpulsar +### Source -``` +For how to build Pulsar C++ client on different platforms from source code, see [compliation](https://github.com/apache/pulsar-client-cpp#compilation). ## Connection URLs @@ -173,7 +123,7 @@ pulsar://localhost:6650 ``` -In a Pulsar cluster in production, the URL looks as follows: +In a Pulsar cluster in production, the URL looks as follows: ```http @@ -190,7 +140,7 @@ pulsar+ssl://pulsar.us-west.example.com:6651 ``` ## Create a consumer -To connect to Pulsar as a consumer, you need to create a consumer on the C++ client. The following is an example. +To connect to Pulsar as a consumer, you need to create a consumer on the C++ client. The following is an example. ```c++ @@ -218,7 +168,7 @@ client.close(); ``` ## Create a producer -To connect to Pulsar as a producer, you need to create a producer on the C++ client. The following is an example. +To connect to Pulsar as a producer, you need to create a producer on the C++ client. The following is an example. ```c++ diff --git a/site2/website/versioned_docs/version-2.8.x/client-libraries-cpp.md b/site2/website/versioned_docs/version-2.8.x/client-libraries-cpp.md index b4fe26e3d37cd..4f8903ae0b09d 100644 --- a/site2/website/versioned_docs/version-2.8.x/client-libraries-cpp.md +++ b/site2/website/versioned_docs/version-2.8.x/client-libraries-cpp.md @@ -5,267 +5,111 @@ sidebar_label: "C++" original_id: client-libraries-cpp --- -You can use Pulsar C++ client to create Pulsar producers and consumers in C++. +````mdx-code-block +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +```` -All the methods in producer, consumer, and reader of a C++ client are thread-safe. +You can use a Pulsar C++ client to create producers, consumers, and readers. -## Supported platforms +All the methods in producer, consumer, and reader of a C++ client are thread-safe. You can read the [API docs](/api/cpp) for the C++ client. -Pulsar C++ client is supported on **Linux** and **macOS** platforms. +## Installation -[Doxygen](http://www.doxygen.nl/)-generated API docs for the C++ client are available [here](/api/cpp). +Use one of the following methods to install a Pulsar C++ client. -## System requirements +### Brew -You need to install the following components before using the C++ client: - -* [CMake](https://cmake.org/) -* [Boost](http://www.boost.org/) -* [Protocol Buffers](https://developers.google.com/protocol-buffers/) 2.6 -* [libcurl](https://curl.haxx.se/libcurl/) -* [Google Test](https://github.com/google/googletest) - -## Linux - -### Compilation - -1. Clone the Pulsar repository. - -```shell - -$ git clone https://github.com/apache/pulsar - -``` - -2. Install all necessary dependencies. - -```shell - -$ apt-get install cmake libssl-dev libcurl4-openssl-dev liblog4cxx-dev \ - libprotobuf-dev protobuf-compiler libboost-all-dev google-mock libgtest-dev libjsoncpp-dev - -``` - -3. Compile and install [Google Test](https://github.com/google/googletest). - -```shell - -# libgtest-dev version is 1.18.0 or above -$ cd /usr/src/googletest -$ sudo cmake . -$ sudo make -$ sudo cp ./googlemock/libgmock.a ./googlemock/gtest/libgtest.a /usr/lib/ - -# less than 1.18.0 -$ cd /usr/src/gtest -$ sudo cmake . -$ sudo make -$ sudo cp libgtest.a /usr/lib - -$ cd /usr/src/gmock -$ sudo cmake . -$ sudo make -$ sudo cp libgmock.a /usr/lib - -``` - -4. Compile the Pulsar client library for C++ inside the Pulsar repository. - -```shell - -$ cd pulsar-client-cpp -$ cmake . -$ make - -``` - -After you install the components successfully, the files `libpulsar.so` and `libpulsar.a` are in the `lib` folder of the repository. The tools `perfProducer` and `perfConsumer` are in the `perf` directory. - -### Install Dependencies - -> Since 2.1.0 release, Pulsar ships pre-built RPM and Debian packages. You can download and install those packages directly. - -After you download and install RPM or DEB, the `libpulsar.so`, `libpulsarnossl.so`, `libpulsar.a`, and `libpulsarwithdeps.a` libraries are in your `/usr/lib` directory. - -By default, they are built in code path `${PULSAR_HOME}/pulsar-client-cpp`. You can build with the command below. - - `cmake . -DBUILD_TESTS=OFF -DLINK_STATIC=ON && make pulsarShared pulsarSharedNossl pulsarStatic pulsarStaticWithDeps -j 3`. - -These libraries rely on some other libraries. If you want to get detailed version of dependencies, see [RPM](https://github.com/apache/pulsar/blob/master/pulsar-client-cpp/pkg/rpm/Dockerfile) or [DEB](https://github.com/apache/pulsar/blob/master/pulsar-client-cpp/pkg/deb/Dockerfile) files. - -1. `libpulsar.so` is a shared library, containing statically linked `boost` and `openssl`. It also dynamically links all other necessary libraries. You can use this Pulsar library with the command below. +Use [Homebrew](http://brew.sh/) to install the latest tagged version with the library and headers: ```bash - - g++ --std=c++11 PulsarTest.cpp -o test /usr/lib/libpulsar.so -I/usr/local/ssl/include - -``` - -2. `libpulsarnossl.so` is a shared library, similar to `libpulsar.so` except that the libraries `openssl` and `crypto` are dynamically linked. You can use this Pulsar library with the command below. - -```bash - - g++ --std=c++11 PulsarTest.cpp -o test /usr/lib/libpulsarnossl.so -lssl -lcrypto -I/usr/local/ssl/include -L/usr/local/ssl/lib - +brew install libpulsar ``` -3. `libpulsar.a` is a static library. You need to load dependencies before using this library. You can use this Pulsar library with the command below. - -```bash - - g++ --std=c++11 PulsarTest.cpp -o test /usr/lib/libpulsar.a -lssl -lcrypto -ldl -lpthread -I/usr/local/ssl/include -L/usr/local/ssl/lib -lboost_system -lboost_regex -lcurl -lprotobuf -lzstd -lz +### Deb -``` +1. Download any one of the Deb packages: -4. `libpulsarwithdeps.a` is a static library, based on `libpulsar.a`. It is archived in the dependencies of `libboost_regex`, `libboost_system`, `libcurl`, `libprotobuf`, `libzstd` and `libz`. You can use this Pulsar library with the command below. + + ```bash - - g++ --std=c++11 PulsarTest.cpp -o test /usr/lib/libpulsarwithdeps.a -lssl -lcrypto -ldl -lpthread -I/usr/local/ssl/include -L/usr/local/ssl/lib - +wget @pulsar:deb:client@ ``` -The `libpulsarwithdeps.a` does not include library openssl related libraries `libssl` and `libcrypto`, because these two libraries are related to security. It is more reasonable and easier to use the versions provided by the local system to handle security issues and upgrade libraries. - -### Install RPM +This package contains shared libraries `libpulsar.so` and `libpulsarnossl.so`. -1. Download an RPM package from the links in the table. - -| Link | Crypto files | -|------|--------------| -| [client](@pulsar:dist_rpm:client@) | [asc](@pulsar:dist_rpm:client@.asc), [sha512](@pulsar:dist_rpm:client@.sha512) | -| [client-debuginfo](@pulsar:dist_rpm:client-debuginfo@) | [asc](@pulsar:dist_rpm:client-debuginfo@.asc), [sha512](@pulsar:dist_rpm:client-debuginfo@.sha512) | -| [client-devel](@pulsar:dist_rpm:client-devel@) | [asc](@pulsar:dist_rpm:client-devel@.asc), [sha512](@pulsar:dist_rpm:client-devel@.sha512) | - -2. Install the package using the following command. + + ```bash - -$ rpm -ivh apache-pulsar-client*.rpm - +wget @pulsar:deb:client-devel@ ``` -After you install RPM successfully, Pulsar libraries are in the `/usr/lib` directory. - -### Install Debian +This package contains static libraries: `libpulsar.a`, `libpulsarwithdeps.a` and C/C++ headers. -1. Download a Debian package from the links in the table. + + -| Link | Crypto files | -|------|--------------| -| [client](@pulsar:deb:client@) | [asc](@pulsar:dist_deb:client@.asc), [sha512](@pulsar:dist_deb:client@.sha512) | -| [client-devel](@pulsar:deb:client-devel@) | [asc](@pulsar:dist_deb:client-devel@.asc), [sha512](@pulsar:dist_deb:client-devel@.sha512) | - -2. Install the package using the following command. +2. Install the package using the following command: ```bash - -$ apt install ./apache-pulsar-client*.deb - +apt install ./apache-pulsar-client*.deb ``` -After you install DEB successfully, Pulsar libraries are in the `/usr/lib` directory. - -### Build - -> If you want to build RPM and Debian packages from the latest master, follow the instructions below. You should run all the instructions at the root directory of your cloned Pulsar repository. - -There are recipes that build RPM and Debian packages containing a -statically linked `libpulsar.so` / `libpulsarnossl.so` / `libpulsar.a` / `libpulsarwithdeps.a` with all required dependencies. - -To build the C++ library packages, you need to build the Java packages first. - -```shell - -mvn install -DskipTests - -``` +Now, you can see Pulsar C++ client libraries installed under the `/usr/lib` directory. -#### RPM +### RPM -To build the RPM inside a Docker container, use the command below. The RPMs are in the `pulsar-client-cpp/pkg/rpm/RPMS/x86_64/` path. +1. Download any one of the RPM packages: -```shell - -pulsar-client-cpp/pkg/rpm/docker-build-rpm.sh + + +```bash +wget @pulsar:dist_rpm:client@ ``` -| Package name | Content | -|-----|-----| -| pulsar-client | Shared library `libpulsar.so` and `libpulsarnossl.so` | -| pulsar-client-devel | Static library `libpulsar.a`, `libpulsarwithdeps.a`and C++ and C headers | -| pulsar-client-debuginfo | Debug symbols for `libpulsar.so` | - -#### Debian +This package contains shared libraries: `libpulsar.so` and `libpulsarnossl.so`. -To build Debian packages, enter the following command. - -```shell - -pulsar-client-cpp/pkg/deb/docker-build-deb.sh + + +```bash +wget @pulsar:dist_rpm:client-debuginfo@ ``` -Debian packages are created in the `pulsar-client-cpp/pkg/deb/BUILD/DEB/` path. - -| Package name | Content | -|-----|-----| -| pulsar-client | Shared library `libpulsar.so` and `libpulsarnossl.so` | -| pulsar-client-dev | Static library `libpulsar.a`, `libpulsarwithdeps.a` and C++ and C headers | - -## MacOS - -### Compilation - -1. Clone the Pulsar repository. - -```shell +This package contains debug symbols for `libpulsar.so`. -$ git clone https://github.com/apache/pulsar + + +```bash +wget @pulsar:dist_rpm:client-devel@ ``` -2. Install all necessary dependencies. - -```shell - -# OpenSSL installation -$ brew install openssl -$ export OPENSSL_INCLUDE_DIR=/usr/local/opt/openssl/include/ -$ export OPENSSL_ROOT_DIR=/usr/local/opt/openssl/ +This package contains static libraries: `libpulsar.a`, `libpulsarwithdeps.a` and C/C++ headers. -$ brew install protobuf boost boost-python log4cxx -# If you are using python3, you need to install boost-python3 + + -# Google Test installation -$ git clone https://github.com/google/googletest.git -$ cd googletest -$ git checkout release-1.12.1 -$ cmake . -$ make install +2. Install the package using the following command: +```bash +rpm -ivh apache-pulsar-client*.rpm ``` -3. Compile the Pulsar client library in the repository that you cloned. - -```shell +Now, you can see Pulsar C++ client libraries installed under the `/usr/lib` directory. -$ cd pulsar-client-cpp -$ cmake . -$ make +:::note -``` - -### Install `libpulsar` +If you get an error like "libpulsar.so: cannot open shared object file: No such file or directory" when starting a Pulsar client, you need to run `ldconfig` first. -Pulsar releases are available in the [Homebrew](https://brew.sh/) core repository. You can install the C++ client library with the following command. The package is installed with the library and headers. +::: -```shell +### Source -brew install libpulsar - -``` +For how to build Pulsar C++ client on different platforms from source code, see [compliation](https://github.com/apache/pulsar-client-cpp#compilation). ## Connection URLs @@ -279,7 +123,7 @@ pulsar://localhost:6650 ``` -In a Pulsar cluster in production, the URL looks as follows. +In a Pulsar cluster in production, the URL looks as follows. ```http @@ -297,7 +141,7 @@ pulsar+ssl://pulsar.us-west.example.com:6651 ## Create a consumer -To use Pulsar as a consumer, you need to create a consumer on the C++ client. The following is an example. +To use Pulsar as a consumer, you need to create a consumer on the C++ client. The following is an example. ```cpp @@ -326,7 +170,7 @@ client.close(); ## Create a producer -To use Pulsar as a producer, you need to create a producer on the C++ client. The following is an example. +To use Pulsar as a producer, you need to create a producer on the C++ client. The following is an example. ```cpp diff --git a/site2/website/versioned_docs/version-2.9.x/client-libraries-cpp.md b/site2/website/versioned_docs/version-2.9.x/client-libraries-cpp.md index 59a33b743496e..aabf7f0969b0e 100644 --- a/site2/website/versioned_docs/version-2.9.x/client-libraries-cpp.md +++ b/site2/website/versioned_docs/version-2.9.x/client-libraries-cpp.md @@ -5,318 +5,111 @@ sidebar_label: "C++" original_id: client-libraries-cpp --- -You can use Pulsar C++ client to create Pulsar producers and consumers in C++. +````mdx-code-block +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +```` -All the methods in producer, consumer, and reader of a C++ client are thread-safe. +You can use a Pulsar C++ client to create producers, consumers, and readers. -## Supported platforms +All the methods in producer, consumer, and reader of a C++ client are thread-safe. You can read the [API docs](/api/cpp) for the C++ client. -Pulsar C++ client is supported on **Linux** ,**macOS** and **Windows** platforms. +## Installation -[Doxygen](http://www.doxygen.nl/)-generated API docs for the C++ client are available [here](/api/cpp). +Use one of the following methods to install a Pulsar C++ client. -## System requirements +### Brew -You need to install the following components before using the C++ client: - -* [CMake](https://cmake.org/) -* [Boost](http://www.boost.org/) -* [Protocol Buffers](https://developers.google.com/protocol-buffers/) >= 3 -* [libcurl](https://curl.se/libcurl/) -* [Google Test](https://github.com/google/googletest) - -## Linux - -### Compilation - -1. Clone the Pulsar repository. - -```shell - -$ git clone https://github.com/apache/pulsar +Use [Homebrew](http://brew.sh/) to install the latest tagged version with the library and headers: +```bash +brew install libpulsar ``` -2. Install all necessary dependencies. +### Deb -```shell +1. Download any one of the Deb packages: -$ apt-get install cmake libssl-dev libcurl4-openssl-dev liblog4cxx-dev \ - libprotobuf-dev protobuf-compiler libboost-all-dev google-mock libgtest-dev libjsoncpp-dev + + +```bash +wget @pulsar:deb:client@ ``` -3. Compile and install [Google Test](https://github.com/google/googletest). - -```shell - -# libgtest-dev version is 1.18.0 or above -$ cd /usr/src/googletest -$ sudo cmake . -$ sudo make -$ sudo cp ./googlemock/libgmock.a ./googlemock/gtest/libgtest.a /usr/lib/ - -# less than 1.18.0 -$ cd /usr/src/gtest -$ sudo cmake . -$ sudo make -$ sudo cp libgtest.a /usr/lib +This package contains shared libraries `libpulsar.so` and `libpulsarnossl.so`. -$ cd /usr/src/gmock -$ sudo cmake . -$ sudo make -$ sudo cp libgmock.a /usr/lib + + +```bash +wget @pulsar:deb:client-devel@ ``` -4. Compile the Pulsar client library for C++ inside the Pulsar repository. +This package contains static libraries: `libpulsar.a`, `libpulsarwithdeps.a` and C/C++ headers. -```shell + + -$ cd pulsar-client-cpp -$ cmake . -$ make +2. Install the package using the following command: +```bash +apt install ./apache-pulsar-client*.deb ``` -After you install the components successfully, the files `libpulsar.so` and `libpulsar.a` are in the `lib` folder of the repository. The tools `perfProducer` and `perfConsumer` are in the `perf` directory. - -### Install Dependencies - -> Since 2.1.0 release, Pulsar ships pre-built RPM and Debian packages. You can download and install those packages directly. +Now, you can see Pulsar C++ client libraries installed under the `/usr/lib` directory. -After you download and install RPM or DEB, the `libpulsar.so`, `libpulsarnossl.so`, `libpulsar.a`, and `libpulsarwithdeps.a` libraries are in your `/usr/lib` directory. +### RPM -By default, they are built in code path `${PULSAR_HOME}/pulsar-client-cpp`. You can build with the command below. +1. Download any one of the RPM packages: - `cmake . -DBUILD_TESTS=OFF -DLINK_STATIC=ON && make pulsarShared pulsarSharedNossl pulsarStatic pulsarStaticWithDeps -j 3`. - -These libraries rely on some other libraries. If you want to get detailed version of dependencies, see [RPM](https://github.com/apache/pulsar/blob/master/pulsar-client-cpp/pkg/rpm/Dockerfile) or [DEB](https://github.com/apache/pulsar/blob/master/pulsar-client-cpp/pkg/deb/Dockerfile) files. - -1. `libpulsar.so` is a shared library, containing statically linked `boost` and `openssl`. It also dynamically links all other necessary libraries. You can use this Pulsar library with the command below. + + ```bash - - g++ --std=c++11 PulsarTest.cpp -o test /usr/lib/libpulsar.so -I/usr/local/ssl/include - +wget @pulsar:dist_rpm:client@ ``` -2. `libpulsarnossl.so` is a shared library, similar to `libpulsar.so` except that the libraries `openssl` and `crypto` are dynamically linked. You can use this Pulsar library with the command below. - -```bash - - g++ --std=c++11 PulsarTest.cpp -o test /usr/lib/libpulsarnossl.so -lssl -lcrypto -I/usr/local/ssl/include -L/usr/local/ssl/lib - -``` +This package contains shared libraries: `libpulsar.so` and `libpulsarnossl.so`. -3. `libpulsar.a` is a static library. You need to load dependencies before using this library. You can use this Pulsar library with the command below. + + ```bash - - g++ --std=c++11 PulsarTest.cpp -o test /usr/lib/libpulsar.a -lssl -lcrypto -ldl -lpthread -I/usr/local/ssl/include -L/usr/local/ssl/lib -lboost_system -lboost_regex -lcurl -lprotobuf -lzstd -lz - +wget @pulsar:dist_rpm:client-debuginfo@ ``` -4. `libpulsarwithdeps.a` is a static library, based on `libpulsar.a`. It is archived in the dependencies of `libboost_regex`, `libboost_system`, `libcurl`, `libprotobuf`, `libzstd` and `libz`. You can use this Pulsar library with the command below. - -```bash +This package contains debug symbols for `libpulsar.so`. - g++ --std=c++11 PulsarTest.cpp -o test /usr/lib/libpulsarwithdeps.a -lssl -lcrypto -ldl -lpthread -I/usr/local/ssl/include -L/usr/local/ssl/lib + + +```bash +wget @pulsar:dist_rpm:client-devel@ ``` -The `libpulsarwithdeps.a` does not include library openssl related libraries `libssl` and `libcrypto`, because these two libraries are related to security. It is more reasonable and easier to use the versions provided by the local system to handle security issues and upgrade libraries. - -### Install RPM - -1. Download a RPM package from the links in the table. +This package contains static libraries: `libpulsar.a`, `libpulsarwithdeps.a` and C/C++ headers. -| Link | Crypto files | -|------|--------------| -| [client](@pulsar:dist_rpm:client@) | [asc](@pulsar:dist_rpm:client@.asc), [sha512](@pulsar:dist_rpm:client@.sha512) | -| [client-debuginfo](@pulsar:dist_rpm:client-debuginfo@) | [asc](@pulsar:dist_rpm:client-debuginfo@.asc), [sha512](@pulsar:dist_rpm:client-debuginfo@.sha512) | -| [client-devel](@pulsar:dist_rpm:client-devel@) | [asc](@pulsar:dist_rpm:client-devel@.asc), [sha512](@pulsar:dist_rpm:client-devel@.sha512) | + + -2. Install the package using the following command. +2. Install the package using the following command: ```bash - -$ rpm -ivh apache-pulsar-client*.rpm - +rpm -ivh apache-pulsar-client*.rpm ``` -After you install RPM successfully, Pulsar libraries are in the `/usr/lib` directory. +Now, you can see Pulsar C++ client libraries installed under the `/usr/lib` directory. :::note -If you get the error that `libpulsar.so: cannot open shared object file: No such file or directory` when starting Pulsar client, you may need to run `ldconfig` first. +If you get an error like "libpulsar.so: cannot open shared object file: No such file or directory" when starting a Pulsar client, you need to run `ldconfig` first. ::: -### Install Debian - -1. Download a Debian package from the links in the table. - -| Link | Crypto files | -|------|--------------| -| [client](@pulsar:deb:client@) | [asc](@pulsar:dist_deb:client@.asc), [sha512](@pulsar:dist_deb:client@.sha512) | -| [client-devel](@pulsar:deb:client-devel@) | [asc](@pulsar:dist_deb:client-devel@.asc), [sha512](@pulsar:dist_deb:client-devel@.sha512) | - -2. Install the package using the following command. - -```bash - -$ apt install ./apache-pulsar-client*.deb - -``` - -After you install DEB successfully, Pulsar libraries are in the `/usr/lib` directory. - -### Build - -> If you want to build RPM and Debian packages from the latest master, follow the instructions below. You should run all the instructions at the root directory of your cloned Pulsar repository. - -There are recipes that build RPM and Debian packages containing a -statically linked `libpulsar.so` / `libpulsarnossl.so` / `libpulsar.a` / `libpulsarwithdeps.a` with all required dependencies. - -To build the C++ library packages, you need to build the Java packages first. - -```shell - -mvn install -DskipTests - -``` - -#### RPM - -To build the RPM inside a Docker container, use the command below. The RPMs are in the `pulsar-client-cpp/pkg/rpm/RPMS/x86_64/` path. - -```shell - -pulsar-client-cpp/pkg/rpm/docker-build-rpm.sh - -``` - -| Package name | Content | -|-----|-----| -| pulsar-client | Shared library `libpulsar.so` and `libpulsarnossl.so` | -| pulsar-client-devel | Static library `libpulsar.a`, `libpulsarwithdeps.a`and C++ and C headers | -| pulsar-client-debuginfo | Debug symbols for `libpulsar.so` | - -#### Debian - -To build Debian packages, enter the following command. - -```shell - -pulsar-client-cpp/pkg/deb/docker-build-deb.sh - -``` - -Debian packages are created in the `pulsar-client-cpp/pkg/deb/BUILD/DEB/` path. - -| Package name | Content | -|-----|-----| -| pulsar-client | Shared library `libpulsar.so` and `libpulsarnossl.so` | -| pulsar-client-dev | Static library `libpulsar.a`, `libpulsarwithdeps.a` and C++ and C headers | - -## MacOS - -### Compilation - -1. Clone the Pulsar repository. - -```shell - -$ git clone https://github.com/apache/pulsar - -``` - -2. Install all necessary dependencies. - -```shell - -# OpenSSL installation -$ brew install openssl -$ export OPENSSL_INCLUDE_DIR=/usr/local/opt/openssl/include/ -$ export OPENSSL_ROOT_DIR=/usr/local/opt/openssl/ - -# Protocol Buffers installation -$ brew install protobuf boost boost-python log4cxx -# If you are using python3, you need to install boost-python3 - -# Google Test installation -$ git clone https://github.com/google/googletest.git -$ cd googletest -$ git checkout release-1.12.1 -$ cmake . -$ make install - -``` - -3. Compile the Pulsar client library in the repository that you cloned. - -```shell - -$ cd pulsar-client-cpp -$ cmake . -$ make - -``` - -### Install `libpulsar` - -Pulsar releases are available in the [Homebrew](https://brew.sh/) core repository. You can install the C++ client library with the following command. The package is installed with the library and headers. - -```shell - -brew install libpulsar - -``` - -## Windows (64-bit) - -### Compilation - -1. Clone the Pulsar repository. - -```shell +### Source -$ git clone https://github.com/apache/pulsar - -``` - -2. Install all necessary dependencies. - -```shell - -cd ${PULSAR_HOME}/pulsar-client-cpp -vcpkg install --feature-flags=manifests --triplet x64-windows - -``` - -3. Build C++ libraries. - -```shell - -cmake -B ./build -A x64 -DBUILD_PYTHON_WRAPPER=OFF -DBUILD_TESTS=OFF -DVCPKG_TRIPLET=x64-windows -DCMAKE_BUILD_TYPE=Release -S . -cmake --build ./build --config Release - -``` - -> **NOTE** -> -> 1. For Windows 32-bit, you need to use `-A Win32` and `-DVCPKG_TRIPLET=x86-windows`. -> 2. For MSVC Debug mode, you need to replace `Release` with `Debug` for both `CMAKE_BUILD_TYPE` variable and `--config` option. - -4. Client libraries are available in the following places. - -``` - -${PULSAR_HOME}/pulsar-client-cpp/build/lib/Release/pulsar.lib -${PULSAR_HOME}/pulsar-client-cpp/build/lib/Release/pulsar.dll - -``` +For how to build Pulsar C++ client on different platforms from source code, see [compliation](https://github.com/apache/pulsar-client-cpp#compilation). ## Connection URLs @@ -330,7 +123,7 @@ pulsar://localhost:6650 ``` -In a Pulsar cluster in production, the URL looks as follows. +In a Pulsar cluster in production, the URL looks as follows. ```http @@ -609,7 +402,7 @@ schema, see [Pulsar schema](schema-get-started.md). - The following example shows how to create a producer with an Avro schema. ```cpp - + static const std::string exampleSchema = "{\"type\":\"record\",\"name\":\"Example\",\"namespace\":\"test\"," "\"fields\":[{\"name\":\"a\",\"type\":\"int\"},{\"name\":\"b\",\"type\":\"int\"}]}"; @@ -617,13 +410,13 @@ schema, see [Pulsar schema](schema-get-started.md). ProducerConfiguration producerConf; producerConf.setSchema(SchemaInfo(AVRO, "Avro", exampleSchema)); client.createProducer("topic-avro", producerConf, producer); - + ``` - The following example shows how to create a consumer with an Avro schema. ```cpp - + static const std::string exampleSchema = "{\"type\":\"record\",\"name\":\"Example\",\"namespace\":\"test\"," "\"fields\":[{\"name\":\"a\",\"type\":\"int\"},{\"name\":\"b\",\"type\":\"int\"}]}"; @@ -631,14 +424,14 @@ schema, see [Pulsar schema](schema-get-started.md). Consumer consumer; consumerConf.setSchema(SchemaInfo(AVRO, "Avro", exampleSchema)); client.subscribe("topic-avro", "sub-2", consumerConf, consumer) - + ``` ### ProtobufNative schema The following example shows how to create a producer and a consumer with a ProtobufNative schema. ​ -1. Generate the `User` class using Protobuf3. +1. Generate the `User` class using Protobuf3. :::note @@ -649,14 +442,14 @@ The following example shows how to create a producer and a consumer with a Proto ​ ```protobuf - + syntax = "proto3"; - + message User { string name = 1; int32 age = 2; } - + ``` ​ @@ -664,9 +457,9 @@ The following example shows how to create a producer and a consumer with a Proto ​ ```cpp - + #include - + ``` ​ @@ -674,7 +467,7 @@ The following example shows how to create a producer and a consumer with a Proto ​ ```cpp - + ProducerConfiguration producerConf; producerConf.setSchema(createProtobufNativeSchema(User::GetDescriptor())); Producer producer; @@ -685,7 +478,7 @@ The following example shows how to create a producer and a consumer with a Proto std::string content; user.SerializeToString(&content); producer.send(MessageBuilder().setContent(content).build()); - + ``` ​ @@ -693,7 +486,7 @@ The following example shows how to create a producer and a consumer with a Proto ​ ```cpp - + ConsumerConfiguration consumerConf; consumerConf.setSchema(createProtobufNativeSchema(User::GetDescriptor())); consumerConf.setSubscriptionInitialPosition(InitialPositionEarliest); @@ -703,6 +496,6 @@ The following example shows how to create a producer and a consumer with a Proto consumer.receive(msg); User user2; user2.ParseFromArray(msg.getData(), msg.getLength()); - + ``` From 5c1f8afcaa970886030322754cefe0c1ea0c2b4c Mon Sep 17 00:00:00 2001 From: Lishen Yao Date: Thu, 27 Oct 2022 09:37:44 +0800 Subject: [PATCH 06/22] [improve][ci] Add schedule trigger to get master branch code coverage daily (#18081) --- .github/workflows/pulsar-ci.yaml | 2 ++ build/run_unit_group.sh | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index f996725e3bec8..8e205298b1464 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -22,6 +22,8 @@ on: pull_request: branches: - master + schedule: + - cron: '0 12 * * *' workflow_dispatch: concurrency: diff --git a/build/run_unit_group.sh b/build/run_unit_group.sh index 6d8cafbee2c7f..1901c18d4b94e 100755 --- a/build/run_unit_group.sh +++ b/build/run_unit_group.sh @@ -24,7 +24,7 @@ set -e set -o pipefail set -o errexit -MVN_TEST_OPTIONS='mvn -Pcoverage -B -ntp -DskipSourceReleaseAssembly=true -DskipBuildDistribution=true -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true' +MVN_TEST_OPTIONS='mvn -B -ntp -DskipSourceReleaseAssembly=true -DskipBuildDistribution=true -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true' function mvn_test() { ( @@ -33,7 +33,11 @@ function mvn_test() { clean_arg="clean" shift fi - TARGET=verify + if echo "${FUNCNAME[@]}" | grep "flaky"; then + TARGET="verify" + else + TARGET="verify -Pcoverage" + fi if [[ "$1" == "--install" ]]; then TARGET="install" shift From 3d7f9e539535a5150d880a83c88ad89e1fb57eef Mon Sep 17 00:00:00 2001 From: Anonymitaet <50226895+Anonymitaet@users.noreply.github.com> Date: Thu, 27 Oct 2022 09:46:50 +0800 Subject: [PATCH 07/22] [fix][doc] Optimize URLs for CLI tools page (#18101) --- site2/docs/about.md | 2 +- site2/docs/reference-cli-tools.md | 25 ++++++++++++++----------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/site2/docs/about.md b/site2/docs/about.md index f198373497800..6a1dd53837275 100644 --- a/site2/docs/about.md +++ b/site2/docs/about.md @@ -55,7 +55,7 @@ You’ll notice an Edit button at the bottom and top of each page. Click it to o :::tip -For how to make contributions to documentation, see [Pulsar Documentation Contribution Guide](https://docs.google.com/document/d/11DTnNPpvcPrebLkMAFcDEIFlD8ARD-k6F-LXoIwdD9Y/edit#). +For how to make contributions to documentation, see [Pulsar Documentation Contribution Guide](https://github.com/apache/pulsar/blob/master/site2/README.md). ::: diff --git a/site2/docs/reference-cli-tools.md b/site2/docs/reference-cli-tools.md index cde841484a4f4..7d28866076e8b 100644 --- a/site2/docs/reference-cli-tools.md +++ b/site2/docs/reference-cli-tools.md @@ -6,14 +6,19 @@ sidebar_label: "Pulsar CLI tools" Pulsar offers several command-line tools that you can use for managing Pulsar installations, performance testing, using command-line producers and consumers, and more. -* [`pulsar-admin`](https://pulsar.apache.org/reference/#/latest/pulsar-admin/) -* [`pulsar`](https://pulsar.apache.org/reference/#/latest/pulsar/) -* [`pulsar-client`](https://pulsar.apache.org/reference/#/latest/pulsar-client/) -* [`pulsar-perf`](https://pulsar.apache.org/reference/#/latest/pulsar-perf/) -* [`pulsar-daemon`](reference-cli-pulsar-daemon.md) -* [`pulsar-shell`](reference-cli-pulsar-shell.md) -* [`bookkeeper`](reference-cli-bookkeeper.md) -* [`broker-tool`](reference-cli-broker-tool.md) +* `pulsar-admin` +* `pulsar` +* `pulsar-client` +* `pulsar-daemon` +* `pulsar-perf` +* `pulsar-shell` +* `bookkeeper` + +::: tip + +For the latest and complete information about command-line tools, including commands, flags, descriptions, and more information, see [Pulsar Reference](https://pulsar.apache.org/reference). + +::: All Pulsar command-line tools can be run from the `bin` directory of your [installed Pulsar package](getting-started-standalone.md). @@ -21,6 +26,4 @@ You can get help for any CLI tool, command, or subcommand using the `--help` fla ```shell bin/pulsar broker --help -``` - - +``` \ No newline at end of file From 1c78e0aabf576145c5f7831e931d89ae795bc2cc Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Thu, 27 Oct 2022 09:49:37 +0800 Subject: [PATCH 08/22] [improve][broker] Support setting forceDeleteTenantAllowed dynamically (#18192) --- .../apache/pulsar/broker/ServiceConfiguration.java | 1 + .../pulsar/broker/service/BrokerServiceTest.java | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index f0725353dd717..95c1a763d4e8e 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -643,6 +643,7 @@ The delayed message index bucket time step(in seconds) in per bucket snapshot se @FieldContext( category = CATEGORY_POLICIES, + dynamic = true, doc = "Allow forced deletion of tenants. Default is false." ) private boolean forceDeleteTenantAllowed = false; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java index eb9447b47fe0c..24e38438c5329 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java @@ -1508,4 +1508,16 @@ public void testDynamicConfigurationsForceDeleteNamespaceAllowed() throws Except assertTrue(conf.isForceDeleteNamespaceAllowed()); }); } + + @Test + public void testDynamicConfigurationsForceDeleteTenantAllowed() throws Exception { + cleanup(); + conf.setForceDeleteTenantAllowed(false); + setup(); + admin.brokers() + .updateDynamicConfiguration("forceDeleteTenantAllowed", "true"); + Awaitility.await().untilAsserted(()->{ + assertTrue(conf.isForceDeleteTenantAllowed()); + }); + } } From 5b7c5c62965151c35d9e5b9f0b50bb93b0beb2c3 Mon Sep 17 00:00:00 2001 From: feynmanlin <315157973@qq.com> Date: Thu, 27 Oct 2022 11:29:53 +0800 Subject: [PATCH 09/22] [improve][broker] Add UncaughtExceptionHandler for every thread pool (#18211) --- .../apache/pulsar/broker/PulsarService.java | 11 ++++--- .../TransactionMetadataStoreService.java | 4 +-- .../impl/ModularLoadManagerImpl.java | 5 +-- .../impl/SimpleLoadManagerImpl.java | 4 +-- .../pulsar/broker/service/BrokerService.java | 33 ++++++++++--------- .../metrics/PrometheusMetricsProvider.java | 5 +-- .../broker/tools/LoadReportCommand.java | 4 ++- .../client/impl/AutoClusterFailover.java | 4 +-- .../impl/ControlledClusterFailover.java | 4 +-- .../pulsar/client/impl/PulsarClientImpl.java | 5 ++- .../pulsar/client/util/ExecutorProvider.java | 8 +++-- .../functions/instance/InstanceCache.java | 4 +-- .../worker/ClusterServiceCoordinator.java | 4 +-- 13 files changed, 53 insertions(+), 42 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 17fb80d9ee442..e31256177b023 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -315,13 +315,13 @@ public PulsarService(ServiceConfiguration config, this.config = config; this.processTerminator = processTerminator; this.loadManagerExecutor = Executors - .newSingleThreadScheduledExecutor(new DefaultThreadFactory("pulsar-load-manager")); + .newSingleThreadScheduledExecutor(new ExecutorProvider.ExtendedThreadFactory("pulsar-load-manager")); this.workerConfig = workerConfig; this.functionWorkerService = functionWorkerService; this.executor = Executors.newScheduledThreadPool(config.getNumExecutorThreadPoolSize(), - new DefaultThreadFactory("pulsar")); + new ExecutorProvider.ExtendedThreadFactory("pulsar")); this.cacheExecutor = Executors.newScheduledThreadPool(config.getNumCacheExecutorThreadPoolSize(), - new DefaultThreadFactory("zk-cache-callback")); + new ExecutorProvider.ExtendedThreadFactory("zk-cache-callback")); if (config.isTransactionCoordinatorEnabled()) { this.transactionExecutorProvider = new ExecutorProvider(this.getConfiguration() @@ -615,7 +615,7 @@ private synchronized void resetMetricsServlet() { private CompletableFuture addTimeoutHandling(CompletableFuture future) { ScheduledExecutorService shutdownExecutor = Executors.newSingleThreadScheduledExecutor( - new DefaultThreadFactory(getClass().getSimpleName() + "-shutdown")); + new ExecutorProvider.ExtendedThreadFactory(getClass().getSimpleName() + "-shutdown")); FutureUtil.addTimeoutHandling(future, Duration.ofMillis(Math.max(1L, getConfiguration().getBrokerShutdownTimeoutMs())), shutdownExecutor, () -> FutureUtil.createTimeoutException("Timeout in close", getClass(), "close")); @@ -1425,7 +1425,8 @@ public BookKeeperClientFactory getBookKeeperClientFactory() { protected synchronized ScheduledExecutorService getCompactorExecutor() { if (this.compactorExecutor == null) { - compactorExecutor = Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory("compaction")); + compactorExecutor = Executors.newSingleThreadScheduledExecutor( + new ExecutorProvider.ExtendedThreadFactory("compaction")); } return this.compactorExecutor; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java index 9ee1657f137e7..3d9e6924d1168 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java @@ -24,7 +24,6 @@ import com.google.common.annotations.VisibleForTesting; import io.netty.util.HashedWheelTimer; import io.netty.util.Timer; -import io.netty.util.concurrent.DefaultThreadFactory; import java.util.Collections; import java.util.Deque; import java.util.List; @@ -52,6 +51,7 @@ import org.apache.pulsar.client.api.transaction.TransactionBufferClientException.ReachMaxPendingOpsException; import org.apache.pulsar.client.api.transaction.TransactionBufferClientException.RequestTimeoutException; import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.client.util.ExecutorProvider; import org.apache.pulsar.common.api.proto.TxnAction; import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.util.FutureUtil; @@ -93,7 +93,7 @@ public class TransactionMetadataStoreService { private static final long HANDLE_PENDING_CONNECT_TIME_OUT = 30000L; private final ThreadFactory threadFactory = - new DefaultThreadFactory("transaction-coordinator-thread-factory"); + new ExecutorProvider.ExtendedThreadFactory("transaction-coordinator-thread-factory"); public TransactionMetadataStoreService(TransactionMetadataStoreProvider transactionMetadataStoreProvider, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java index d04c64c163ad2..c14768eed5e9f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java @@ -20,7 +20,6 @@ import com.google.common.collect.Multimap; import com.google.common.collect.Sets; -import io.netty.util.concurrent.DefaultThreadFactory; import java.util.ArrayList; import java.util.Collection; import java.util.ConcurrentModificationException; @@ -60,6 +59,7 @@ import org.apache.pulsar.broker.resources.ClusterResources; import org.apache.pulsar.broker.stats.prometheus.metrics.Summary; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.util.ExecutorProvider; import org.apache.pulsar.common.naming.NamespaceBundleFactory; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.ServiceUnitId; @@ -212,7 +212,8 @@ public ModularLoadManagerImpl() { loadData = new LoadData(); loadSheddingPipeline = new ArrayList<>(); preallocatedBundleToBroker = new ConcurrentHashMap<>(); - scheduler = Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory("pulsar-modular-load-manager")); + scheduler = Executors.newSingleThreadScheduledExecutor( + new ExecutorProvider.ExtendedThreadFactory("pulsar-modular-load-manager")); this.brokerToFailureDomainMap = new HashMap<>(); this.brokerTopicLoadingPredicate = new BrokerTopicLoadingPredicate() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java index 5b2098b8e4d57..c2c0d1947c93e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java @@ -26,7 +26,6 @@ import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import com.google.common.collect.TreeMultimap; -import io.netty.util.concurrent.DefaultThreadFactory; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Collections; @@ -55,6 +54,7 @@ import org.apache.pulsar.broker.loadbalance.PlacementStrategy; import org.apache.pulsar.broker.loadbalance.ResourceUnit; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared.BrokerTopicLoadingPredicate; +import org.apache.pulsar.client.util.ExecutorProvider; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.ServiceUnitId; import org.apache.pulsar.common.policies.data.ResourceQuota; @@ -189,7 +189,7 @@ public class SimpleLoadManagerImpl implements LoadManager, Consumer()); this.currentLoadReports = new HashMap<>(); this.resourceUnitRankings = new HashMap<>(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 14fb9a9a4b4ee..b410fb48b22fa 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -127,6 +127,7 @@ import org.apache.pulsar.client.impl.ClientBuilderImpl; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.client.internal.PropertiesUtils; +import org.apache.pulsar.client.util.ExecutorProvider; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.configuration.BindAddress; import org.apache.pulsar.common.configuration.FieldContext; @@ -311,13 +312,14 @@ public BrokerService(PulsarService pulsar, EventLoopGroup eventLoopGroup) throws this.topicOrderedExecutor = OrderedExecutor.newBuilder() .numThreads(pulsar.getConfiguration().getNumWorkerThreadsForNonPersistentTopic()) .name("broker-topic-workers").build(); - final DefaultThreadFactory acceptorThreadFactory = new DefaultThreadFactory("pulsar-acceptor"); + final DefaultThreadFactory acceptorThreadFactory = + new ExecutorProvider.ExtendedThreadFactory("pulsar-acceptor"); this.acceptorGroup = EventLoopUtil.newEventLoopGroup( pulsar.getConfiguration().getNumAcceptorThreads(), false, acceptorThreadFactory); this.workerGroup = eventLoopGroup; - this.statsUpdater = Executors - .newSingleThreadScheduledExecutor(new DefaultThreadFactory("pulsar-stats-updater")); + this.statsUpdater = Executors.newSingleThreadScheduledExecutor( + new ExecutorProvider.ExtendedThreadFactory("pulsar-stats-updater")); this.authorizationService = new AuthorizationService( pulsar.getConfiguration(), pulsar().getPulsarResources()); if (!pulsar.getConfiguration().getEntryFilterNames().isEmpty()) { @@ -327,22 +329,22 @@ public BrokerService(PulsarService pulsar, EventLoopGroup eventLoopGroup) throws pulsar.getLocalMetadataStore().registerListener(this::handleMetadataChanges); pulsar.getConfigurationMetadataStore().registerListener(this::handleMetadataChanges); - this.inactivityMonitor = Executors - .newSingleThreadScheduledExecutor(new DefaultThreadFactory("pulsar-inactivity-monitor")); - this.messageExpiryMonitor = Executors - .newSingleThreadScheduledExecutor(new DefaultThreadFactory("pulsar-msg-expiry-monitor")); + this.inactivityMonitor = Executors.newSingleThreadScheduledExecutor( + new ExecutorProvider.ExtendedThreadFactory("pulsar-inactivity-monitor")); + this.messageExpiryMonitor = Executors.newSingleThreadScheduledExecutor( + new ExecutorProvider.ExtendedThreadFactory("pulsar-msg-expiry-monitor")); this.compactionMonitor = Executors.newSingleThreadScheduledExecutor( - new DefaultThreadFactory("pulsar-compaction-monitor")); - this.consumedLedgersMonitor = Executors - .newSingleThreadScheduledExecutor(new DefaultThreadFactory("consumed-Ledgers-monitor")); + new ExecutorProvider.ExtendedThreadFactory("pulsar-compaction-monitor")); + this.consumedLedgersMonitor = Executors.newSingleThreadScheduledExecutor( + new ExecutorProvider.ExtendedThreadFactory("consumed-Ledgers-monitor")); this.topicPublishRateLimiterMonitor = new PublishRateLimiterMonitor("pulsar-topic-publish-rate-limiter-monitor"); this.brokerPublishRateLimiterMonitor = new PublishRateLimiterMonitor("pulsar-broker-publish-rate-limiter-monitor"); this.backlogQuotaManager = new BacklogQuotaManager(pulsar); - this.backlogQuotaChecker = Executors - .newSingleThreadScheduledExecutor(new DefaultThreadFactory("pulsar-backlog-quota-checker")); + this.backlogQuotaChecker = Executors.newSingleThreadScheduledExecutor( + new ExecutorProvider.ExtendedThreadFactory("pulsar-backlog-quota-checker")); this.authenticationService = new AuthenticationService(pulsar.getConfiguration()); this.blockedDispatchers = ConcurrentOpenHashSet.newBuilder().build(); @@ -429,7 +431,8 @@ private void startProtocolHandler(String protocol, bootstrap.childOption(ChannelOption.RCVBUF_ALLOCATOR, new AdaptiveRecvByteBufAllocator(1024, 16 * 1024, 1 * 1024 * 1024)); EventLoopUtil.enableTriggeredMode(bootstrap); - DefaultThreadFactory defaultThreadFactory = new DefaultThreadFactory("pulsar-ph-" + protocol); + DefaultThreadFactory defaultThreadFactory = + new ExecutorProvider.ExtendedThreadFactory("pulsar-ph-" + protocol); EventLoopGroup dedicatedWorkerGroup = EventLoopUtil.newEventLoopGroup(configuration.getNumIOThreads(), false, defaultThreadFactory); bootstrap.channel(EventLoopUtil.getServerSocketChannelClass(dedicatedWorkerGroup)); @@ -551,7 +554,7 @@ protected void startDeduplicationSnapshotMonitor() { int interval = pulsar().getConfiguration().getBrokerDeduplicationSnapshotFrequencyInSeconds(); if (interval > 0 && pulsar().getConfiguration().isBrokerDeduplicationEnabled()) { this.deduplicationSnapshotMonitor = - Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory( + Executors.newSingleThreadScheduledExecutor(new ExecutorProvider.ExtendedThreadFactory( "deduplication-snapshot-monitor")); deduplicationSnapshotMonitor.scheduleAtFixedRate(safeRun(() -> forEachTopic( Topic::checkDeduplicationSnapshot)) @@ -685,7 +688,7 @@ synchronized void startOrUpdate(long tickTimeMs, Runnable checkTask, Runnable re stop(); } //start monitor. - scheduler = Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory(name)); + scheduler = Executors.newSingleThreadScheduledExecutor(new ExecutorProvider.ExtendedThreadFactory(name)); // schedule task that sums up publish-rate across all cnx on a topic , // and check the rate limit exceeded or not. scheduler.scheduleAtFixedRate(safeRun(checkTask), tickTimeMs, tickTimeMs, TimeUnit.MILLISECONDS); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusMetricsProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusMetricsProvider.java index 3097d2613130a..73c0609c556a5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusMetricsProvider.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusMetricsProvider.java @@ -20,7 +20,6 @@ import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.google.common.annotations.VisibleForTesting; -import io.netty.util.concurrent.DefaultThreadFactory; import io.prometheus.client.Collector; import java.io.IOException; import java.io.Writer; @@ -34,6 +33,7 @@ import org.apache.bookkeeper.stats.StatsProvider; import org.apache.commons.configuration.Configuration; import org.apache.commons.lang.StringUtils; +import org.apache.pulsar.client.util.ExecutorProvider; /** * A Prometheus based {@link StatsProvider} implementation. @@ -90,7 +90,8 @@ public String getStatsName(String... statsComponents) { @Override public void start(Configuration conf) { - executor = Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory("metrics")); + executor = Executors.newSingleThreadScheduledExecutor( + new ExecutorProvider.ExtendedThreadFactory("metrics")); int latencyRolloverSeconds = conf.getInt(PROMETHEUS_STATS_LATENCY_ROLLOVER_SECONDS, DEFAULT_PROMETHEUS_STATS_LATENCY_ROLLOVER_SECONDS); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/tools/LoadReportCommand.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/tools/LoadReportCommand.java index 9eaf8c1196da6..935e3a9f2fa1a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/tools/LoadReportCommand.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/tools/LoadReportCommand.java @@ -32,6 +32,7 @@ import org.apache.pulsar.broker.loadbalance.impl.GenericBrokerHostUsageImpl; import org.apache.pulsar.broker.loadbalance.impl.LinuxBrokerHostUsageImpl; import org.apache.pulsar.broker.tools.LoadReportCommand.Flags; +import org.apache.pulsar.client.util.ExecutorProvider; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; @@ -88,7 +89,8 @@ private boolean apply(Flags flags) { spec.console().println("--------------------------------------"); spec.console().println(); - ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); + ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor( + new ExecutorProvider.ExtendedThreadFactory("load-report")); BrokerHostUsage hostUsage; try { if (isLinux) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AutoClusterFailover.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AutoClusterFailover.java index f2b4449aa0e11..94e8026b7010e 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AutoClusterFailover.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AutoClusterFailover.java @@ -20,7 +20,6 @@ import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.google.common.base.Strings; -import io.netty.util.concurrent.DefaultThreadFactory; import java.io.IOException; import java.net.InetSocketAddress; import java.net.Socket; @@ -38,6 +37,7 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.ServiceUrlProvider; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.client.util.ExecutorProvider; @Slf4j @Data @@ -80,7 +80,7 @@ private AutoClusterFailover(AutoClusterFailoverBuilderImpl builder) { this.intervalMs = builder.checkIntervalMs; this.resolver = new PulsarServiceNameResolver(); this.executor = Executors.newSingleThreadScheduledExecutor( - new DefaultThreadFactory("pulsar-service-provider")); + new ExecutorProvider.ExtendedThreadFactory("pulsar-service-provider")); } @Override diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ControlledClusterFailover.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ControlledClusterFailover.java index 3fb503cd0f75f..4ab1977d0fb26 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ControlledClusterFailover.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ControlledClusterFailover.java @@ -25,7 +25,6 @@ import com.google.common.base.Strings; import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.codec.http.HttpResponse; -import io.netty.util.concurrent.DefaultThreadFactory; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; @@ -44,6 +43,7 @@ import org.apache.pulsar.client.api.ControlledClusterFailoverBuilder; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.ServiceUrlProvider; +import org.apache.pulsar.client.util.ExecutorProvider; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.asynchttpclient.AsyncHttpClient; import org.asynchttpclient.AsyncHttpClientConfig; @@ -74,7 +74,7 @@ private ControlledClusterFailover(ControlledClusterFailoverBuilderImpl builder) this.currentPulsarServiceUrl = builder.defaultServiceUrl; this.interval = builder.interval; this.executor = Executors.newSingleThreadScheduledExecutor( - new DefaultThreadFactory("pulsar-service-provider")); + new ExecutorProvider.ExtendedThreadFactory("pulsar-service-provider")); this.httpClient = buildHttpClient(); this.requestBuilder = httpClient.prepareGet(builder.urlProvider) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java index 08adad4613665..d964328d59cbd 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java @@ -26,7 +26,6 @@ import io.netty.channel.EventLoopGroup; import io.netty.util.HashedWheelTimer; import io.netty.util.Timer; -import io.netty.util.concurrent.DefaultThreadFactory; import java.io.IOException; import java.net.InetSocketAddress; import java.time.Clock; @@ -756,7 +755,7 @@ public CompletableFuture closeAsync() { // would happen CompletableFuture combinedFuture = FutureUtil.waitForAll(futures); ScheduledExecutorService shutdownExecutor = Executors.newSingleThreadScheduledExecutor( - new DefaultThreadFactory("pulsar-client-shutdown-timeout-scheduler")); + new ExecutorProvider.ExtendedThreadFactory("pulsar-client-shutdown-timeout-scheduler")); FutureUtil.addTimeoutHandling(combinedFuture, Duration.ofSeconds(CLOSE_TIMEOUT_SECONDS), shutdownExecutor, () -> FutureUtil.createTimeoutException("Closing producers and consumers timed out.", PulsarClientImpl.class, "closeAsync")); @@ -1087,7 +1086,7 @@ private static EventLoopGroup getEventLoopGroup(ClientConfigurationData conf) { } private static ThreadFactory getThreadFactory(String poolName) { - return new DefaultThreadFactory(poolName, Thread.currentThread().isDaemon()); + return new ExecutorProvider.ExtendedThreadFactory(poolName, Thread.currentThread().isDaemon()); } void cleanupProducer(ProducerBase producer) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/util/ExecutorProvider.java b/pulsar-client/src/main/java/org/apache/pulsar/client/util/ExecutorProvider.java index a7a9734700295..037aef411a0e9 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/util/ExecutorProvider.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/util/ExecutorProvider.java @@ -40,10 +40,12 @@ public class ExecutorProvider { private final String poolName; private volatile boolean isShutdown; - protected static class ExtendedThreadFactory extends DefaultThreadFactory { - + public static class ExtendedThreadFactory extends DefaultThreadFactory { @Getter private Thread thread; + public ExtendedThreadFactory(String poolName) { + super(poolName, false); + } public ExtendedThreadFactory(String poolName, boolean daemon) { super(poolName, daemon); } @@ -51,6 +53,8 @@ public ExtendedThreadFactory(String poolName, boolean daemon) { @Override public Thread newThread(Runnable r) { thread = super.newThread(r); + thread.setUncaughtExceptionHandler((t, e) -> + log.error("Thread {} got uncaught Exception", t.getName(), e)); return thread; } } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceCache.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceCache.java index 988f7ce2e2852..c9aea1148b17f 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceCache.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceCache.java @@ -18,11 +18,11 @@ */ package org.apache.pulsar.functions.instance; -import io.netty.util.concurrent.DefaultThreadFactory; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import lombok.Getter; +import org.apache.pulsar.client.util.ExecutorProvider; public class InstanceCache { @@ -33,7 +33,7 @@ public class InstanceCache { private InstanceCache() { ThreadFactory namedThreadFactory = - new DefaultThreadFactory("function-timer-thread"); + new ExecutorProvider.ExtendedThreadFactory("function-timer-thread"); scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(namedThreadFactory); } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/ClusterServiceCoordinator.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/ClusterServiceCoordinator.java index f52259c629623..e01b5bc6943be 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/ClusterServiceCoordinator.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/ClusterServiceCoordinator.java @@ -20,7 +20,6 @@ import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.google.common.annotations.VisibleForTesting; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.util.HashMap; import java.util.Map; import java.util.concurrent.Executors; @@ -30,6 +29,7 @@ import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.client.util.ExecutorProvider; @Slf4j public class ClusterServiceCoordinator implements AutoCloseable { @@ -54,7 +54,7 @@ public TimerTaskInfo(long interval, Runnable task) { public ClusterServiceCoordinator(String workerId, LeaderService leaderService, Supplier isLeader) { this(workerId, leaderService, isLeader, Executors.newSingleThreadScheduledExecutor( - new ThreadFactoryBuilder().setNameFormat("cluster-service-coordinator-timer").build())); + new ExecutorProvider.ExtendedThreadFactory("cluster-service-coordinator-timer"))); } @VisibleForTesting From 2c9e7296bf259191ee059e2a7cd7720c256b8c3c Mon Sep 17 00:00:00 2001 From: HuangZeGui Date: Thu, 27 Oct 2022 11:57:20 +0800 Subject: [PATCH 10/22] [improve][ml] Remove the redundant judgment logic of ManagedCursorImpl (#18205) --- .../mledger/impl/ManagedCursorImpl.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index ce0619a986584..4fa73a2027326 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -566,7 +566,7 @@ protected void recoverFromLedger(final ManagedCursorInfo info, final VoidCallbac if (positionInfo.getIndividualDeletedMessagesCount() > 0) { recoverIndividualDeletedMessages(positionInfo.getIndividualDeletedMessagesList()); } - if (config.isDeletionAtBatchIndexLevelEnabled() && batchDeletedIndexes != null + if (config.isDeletionAtBatchIndexLevelEnabled() && positionInfo.getBatchedEntryDeletionIndexInfoCount() > 0) { recoverBatchDeletedIndexes(positionInfo.getBatchedEntryDeletionIndexInfoList()); } @@ -1227,7 +1227,7 @@ public void operationComplete() { lastMarkDeleteEntry = new MarkDeleteEntry(newMarkDeletePosition, isCompactionCursor() ? getProperties() : Collections.emptyMap(), null, null); individualDeletedMessages.clear(); - if (config.isDeletionAtBatchIndexLevelEnabled() && batchDeletedIndexes != null) { + if (config.isDeletionAtBatchIndexLevelEnabled()) { batchDeletedIndexes.values().forEach(BitSetRecyclable::recycle); batchDeletedIndexes.clear(); long[] resetWords = newPosition.ackSet; @@ -1866,7 +1866,7 @@ public void asyncMarkDelete(final Position position, Map propertie PositionImpl newPosition = (PositionImpl) position; - if (config.isDeletionAtBatchIndexLevelEnabled() && batchDeletedIndexes != null) { + if (config.isDeletionAtBatchIndexLevelEnabled()) { if (newPosition.ackSet != null) { AtomicReference bitSetRecyclable = new AtomicReference<>(); BitSetRecyclable givenBitSet = BitSetRecyclable.create().resetWords(newPosition.ackSet); @@ -2049,7 +2049,7 @@ public void operationComplete() { try { individualDeletedMessages.removeAtMost(mdEntry.newPosition.getLedgerId(), mdEntry.newPosition.getEntryId()); - if (config.isDeletionAtBatchIndexLevelEnabled() && batchDeletedIndexes != null) { + if (config.isDeletionAtBatchIndexLevelEnabled()) { Map subMap = batchDeletedIndexes.subMap(PositionImpl.EARLIEST, false, PositionImpl.get(mdEntry.newPosition.getLedgerId(), mdEntry.newPosition.getEntryId()), true); @@ -2178,7 +2178,7 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb if (individualDeletedMessages.contains(position.getLedgerId(), position.getEntryId()) || position.compareTo(markDeletePosition) <= 0) { - if (config.isDeletionAtBatchIndexLevelEnabled() && batchDeletedIndexes != null) { + if (config.isDeletionAtBatchIndexLevelEnabled()) { BitSetRecyclable bitSetRecyclable = batchDeletedIndexes.remove(position); if (bitSetRecyclable != null) { bitSetRecyclable.recycle(); @@ -2190,7 +2190,7 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb continue; } if (position.ackSet == null) { - if (config.isDeletionAtBatchIndexLevelEnabled() && batchDeletedIndexes != null) { + if (config.isDeletionAtBatchIndexLevelEnabled()) { BitSetRecyclable bitSetRecyclable = batchDeletedIndexes.remove(position); if (bitSetRecyclable != null) { bitSetRecyclable.recycle(); @@ -2207,7 +2207,7 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb log.debug("[{}] [{}] Individually deleted messages: {}", ledger.getName(), name, individualDeletedMessages); } - } else if (config.isDeletionAtBatchIndexLevelEnabled() && batchDeletedIndexes != null) { + } else if (config.isDeletionAtBatchIndexLevelEnabled()) { BitSetRecyclable givenBitSet = BitSetRecyclable.create().resetWords(position.ackSet); BitSetRecyclable bitSet = batchDeletedIndexes.computeIfAbsent(position, (v) -> givenBitSet); if (givenBitSet != bitSet) { @@ -2862,8 +2862,7 @@ private List buildIndividualDeletedMessageRanges() { private List buildBatchEntryDeletionIndexInfoList() { lock.readLock().lock(); try { - if (!config.isDeletionAtBatchIndexLevelEnabled() || batchDeletedIndexes == null - || batchDeletedIndexes.isEmpty()) { + if (!config.isDeletionAtBatchIndexLevelEnabled() || batchDeletedIndexes.isEmpty()) { return Collections.emptyList(); } MLDataFormats.NestedPositionInfo.Builder nestedPositionBuilder = MLDataFormats.NestedPositionInfo @@ -3314,7 +3313,7 @@ private ManagedCursorImpl cursorImpl() { @Override public long[] getDeletedBatchIndexesAsLongArray(PositionImpl position) { - if (config.isDeletionAtBatchIndexLevelEnabled() && batchDeletedIndexes != null) { + if (config.isDeletionAtBatchIndexLevelEnabled()) { BitSetRecyclable bitSet = batchDeletedIndexes.get(position); return bitSet == null ? null : bitSet.toLongArray(); } else { From c7990b9eb0e2e550a612212431748431b15fa856 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Thu, 27 Oct 2022 14:41:47 +0800 Subject: [PATCH 11/22] [fix][test] Fix flaky test `testDoNotGetOffloadPoliciesMultipleTimesWhenTrimLedgers` (#18147) --- .../apache/bookkeeper/mledger/impl/ManagedLedgerTest.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java index 744f50add3d63..29e30a958bd02 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java @@ -3626,10 +3626,13 @@ public void testDoNotGetOffloadPoliciesMultipleTimesWhenTrimLedgers() throws Exc LedgerOffloader ledgerOffloader = mock(NullLedgerOffloader.class); OffloadPoliciesImpl offloadPolicies = mock(OffloadPoliciesImpl.class); when(ledgerOffloader.getOffloadPolicies()).thenReturn(offloadPolicies); + when(ledgerOffloader.getOffloadPolicies().getManagedLedgerOffloadThresholdInBytes()).thenReturn(-1L); + when(ledgerOffloader.getOffloadPolicies().getManagedLedgerOffloadThresholdInSeconds()).thenReturn(-1L); when(ledgerOffloader.getOffloadDriverName()).thenReturn("s3"); config.setLedgerOffloader(ledgerOffloader); - ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open( - "testDoNotGetOffloadPoliciesMultipleTimesWhenTrimLedgers", config); + ManagedLedgerImpl ledger = spy((ManagedLedgerImpl)factory.open( + "testDoNotGetOffloadPoliciesMultipleTimesWhenTrimLedgers", config)); + doNothing().when(ledger).trimConsumedLedgersInBackground(any(CompletableFuture.class)); // Retain the data. ledger.openCursor("test-cursor"); From b061c6ac5833c21e483368febebd0d30679a35e1 Mon Sep 17 00:00:00 2001 From: Lei Zhiyuan Date: Thu, 27 Oct 2022 14:43:32 +0800 Subject: [PATCH 12/22] [improve][broker] Remove locallyAcquiredLock when removeOwnership (#18197) --- .../apache/pulsar/broker/namespace/OwnershipCache.java | 9 ++++++++- .../pulsar/broker/namespace/OwnershipCacheTest.java | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java index 1ca9a0494555f..a9dd44d4589d9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java @@ -21,6 +21,7 @@ import com.github.benmanes.caffeine.cache.AsyncCacheLoader; import com.github.benmanes.caffeine.cache.AsyncLoadingCache; import com.github.benmanes.caffeine.cache.Caffeine; +import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.MoreExecutors; import java.util.ArrayList; import java.util.List; @@ -207,7 +208,7 @@ public CompletableFuture tryAcquiringOwnership(Namespace * */ public CompletableFuture removeOwnership(NamespaceBundle bundle) { - ResourceLock lock = locallyAcquiredLocks.get(bundle); + ResourceLock lock = locallyAcquiredLocks.remove(bundle); if (lock == null) { // We don't own the specified bundle anymore return CompletableFuture.completedFuture(null); @@ -328,6 +329,12 @@ public void invalidateLocalOwnerCache(NamespaceBundle namespaceBundle) { this.ownedBundlesCache.synchronous().invalidate(namespaceBundle); } + @VisibleForTesting + public Map> getLocallyAcquiredLocks() { + return locallyAcquiredLocks; + } + + public synchronized boolean refreshSelfOwnerInfo() { this.selfOwnerInfo = new NamespaceEphemeralData(pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls(), pulsar.getSafeWebServiceAddress(), diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnershipCacheTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnershipCacheTest.java index b5092784103c5..8b2cb96e267c5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnershipCacheTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnershipCacheTest.java @@ -367,6 +367,7 @@ public void testRemoveOwnership() throws Exception { Awaitility.await().untilAsserted(() -> { assertTrue(cache.getOwnedBundles().isEmpty()); assertFalse(store.exists(ServiceUnitUtils.path(bundle)).join()); + assertNull(cache.getLocallyAcquiredLocks().get(bundle)); }); } From 5e3f8ba5e5c2b2375880b679abebb2b03e738cd8 Mon Sep 17 00:00:00 2001 From: Rajan Dhabalia Date: Thu, 27 Oct 2022 05:54:12 -0700 Subject: [PATCH 13/22] [improve][client] support aggregate metrics for partition topic stats (#18214) --- .../client/impl/ProducerStatsRecorderImpl.java | 17 +++++++++++++++-- .../impl/ProducerStatsRecorderImplTest.java | 13 +++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImpl.java index 01ed84f55030e..a7e541e5de178 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImpl.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.text.DecimalFormat; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.DoubleAdder; import java.util.concurrent.atomic.LongAdder; import org.apache.pulsar.client.api.ProducerStats; import org.apache.pulsar.client.impl.conf.ProducerConfigurationData; @@ -50,6 +51,8 @@ public class ProducerStatsRecorderImpl implements ProducerStatsRecorder { private final LongAdder totalBytesSent; private final LongAdder totalSendFailed; private final LongAdder totalAcksReceived; + private final DoubleAdder sendMsgsRateAggregate; + private final DoubleAdder sendBytesRateAggregate; private static final DecimalFormat DEC = new DecimalFormat("0.000"); private static final DecimalFormat THROUGHPUT_FORMAT = new DecimalFormat("0.00"); private final transient DoublesSketch ds; @@ -58,6 +61,7 @@ public class ProducerStatsRecorderImpl implements ProducerStatsRecorder { private volatile double sendMsgsRate; private volatile double sendBytesRate; + private int partitions = 0; private volatile double[] latencyPctValues = new double[PERCENTILES.length]; private volatile double[] batchSizePctValues = new double[PERCENTILES.length]; private volatile double[] msgSizePctValues = new double[PERCENTILES.length]; @@ -73,6 +77,8 @@ public ProducerStatsRecorderImpl() { totalBytesSent = new LongAdder(); totalSendFailed = new LongAdder(); totalAcksReceived = new LongAdder(); + sendMsgsRateAggregate = new DoubleAdder(); + sendBytesRateAggregate = new DoubleAdder(); ds = DoublesSketch.builder().build(256); batchSizeDs = DoublesSketch.builder().build(256); msgSizeDs = DoublesSketch.builder().build(256); @@ -91,6 +97,8 @@ public ProducerStatsRecorderImpl(PulsarClientImpl pulsarClient, ProducerConfigur totalBytesSent = new LongAdder(); totalSendFailed = new LongAdder(); totalAcksReceived = new LongAdder(); + sendMsgsRateAggregate = new DoubleAdder(); + sendBytesRateAggregate = new DoubleAdder(); ds = DoublesSketch.builder().build(256); batchSizeDs = DoublesSketch.builder().build(256); msgSizeDs = DoublesSketch.builder().build(256); @@ -239,6 +247,7 @@ void reset() { totalBytesSent.reset(); totalSendFailed.reset(); totalAcksReceived.reset(); + partitions = 0; } void updateCumulativeStats(ProducerStats stats) { @@ -253,6 +262,10 @@ void updateCumulativeStats(ProducerStats stats) { totalBytesSent.add(stats.getTotalBytesSent()); totalSendFailed.add(stats.getTotalSendFailed()); totalAcksReceived.add(stats.getTotalAcksReceived()); + // update rates + sendMsgsRateAggregate.add(stats.getSendMsgsRate()); + sendBytesRateAggregate.add(stats.getSendBytesRate()); + partitions++; } @Override @@ -293,12 +306,12 @@ public long getTotalAcksReceived() { @Override public double getSendMsgsRate() { - return sendMsgsRate; + return partitions != 0 ? sendMsgsRateAggregate.doubleValue() / partitions : sendMsgsRate; } @Override public double getSendBytesRate() { - return sendBytesRate; + return partitions != 0 ? sendBytesRateAggregate.doubleValue() / partitions : sendBytesRate; } @Override diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImplTest.java index daf4aa4473bd9..28f47105f864b 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImplTest.java @@ -26,7 +26,9 @@ import java.util.concurrent.TimeUnit; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; @@ -74,4 +76,15 @@ public void testGetStatsAndCancelStatsTimeoutWithoutArriveUpdateInterval() { recorder.cancelStatsTimeout(); assertEquals(1000.0, recorder.getSendLatencyMillisMax(), 0.5); } + + @Test + public void testPartitionTopicAggegationStats() { + ProducerStatsRecorderImpl recorder1 = spy(new ProducerStatsRecorderImpl()); + ProducerStatsRecorderImpl recorder2 = new ProducerStatsRecorderImpl(); + when(recorder1.getSendMsgsRate()).thenReturn(1000.0); + when(recorder1.getSendBytesRate()).thenReturn(1000.0); + recorder2.updateCumulativeStats(recorder1); + assertTrue(recorder2.getSendBytesRate() > 0); + assertTrue(recorder2.getSendMsgsRate() > 0); + } } From fad3cccf87480a7a8c3a938cf5ca539b9a033106 Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Thu, 27 Oct 2022 23:01:37 +0800 Subject: [PATCH 14/22] =?UTF-8?q?[fix]=20[pulsar-client]=20Fix=20pendingLo?= =?UTF-8?q?okupRequestSemaphore=20leak=20when=20Ser=E2=80=A6=20(#18219)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Motivation https://github.com/apache/pulsar/blob/b061c6ac5833c21e483368febebd0d30679a35e1/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java#L748-L774 The `pendingLookupRequestSemaphore` will leak when handleError. There are `LookUpRequestSemaphore` not released when removing it from `pendingRequests` related PR: #17856 ### Modifications We can't easily release the semaphore in `handleError`, because there are not only `LookUpRequest`. So release the semaphore when LookupException ### Verifying this change Add unit test case to cover this change ### Documentation - [ ] `doc-required` (Your PR needs to update docs and you will update later) - [x] `doc-not-needed` bug fixs, no need doc - [ ] `doc` (Your PR contains doc changes) - [ ] `doc-complete` (Docs have been already added) --- .../apache/pulsar/client/impl/ClientCnx.java | 3 +- .../pulsar/client/impl/ClientCnxTest.java | 43 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index e8682e69f018b..76908a7c5caaf 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -807,7 +807,8 @@ public CompletableFuture newLookup(ByteBuf request, long reque if (pendingLookupRequestSemaphore.tryAcquire()) { future.whenComplete((lookupDataResult, throwable) -> { - if (throwable instanceof ConnectException) { + if (throwable instanceof ConnectException + || throwable instanceof PulsarClientException.LookupException) { pendingLookupRequestSemaphore.release(); } }); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java index 793e77d79bcb5..22220805814f5 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java @@ -120,6 +120,49 @@ public void testPendingLookupRequestSemaphore() throws Exception { eventLoop.shutdownGracefully(); } + @Test + public void testPendingLookupRequestSemaphoreServiceNotReady() throws Exception { + EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, new DefaultThreadFactory("testClientCnxTimeout")); + ClientConfigurationData conf = new ClientConfigurationData(); + conf.setOperationTimeoutMs(10_000); + conf.setKeepAliveIntervalSeconds(0); + ClientCnx cnx = new ClientCnx(conf, eventLoop); + + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + Channel channel = mock(Channel.class); + when(ctx.channel()).thenReturn(channel); + ChannelFuture listenerFuture = mock(ChannelFuture.class); + when(listenerFuture.addListener(any())).thenReturn(listenerFuture); + when(ctx.writeAndFlush(any())).thenReturn(listenerFuture); + cnx.channelActive(ctx); + cnx.state = ClientCnx.State.Ready; + CountDownLatch countDownLatch = new CountDownLatch(1); + CompletableFuture completableFuture = new CompletableFuture<>(); + new Thread(() -> { + try { + Thread.sleep(1_000); + CompletableFuture future = + cnx.newLookup(null, 123); + countDownLatch.countDown(); + future.get(); + } catch (Exception e) { + completableFuture.complete(e); + } + }).start(); + countDownLatch.await(); + CommandError commandError = new CommandError(); + commandError.setRequestId(123L); + commandError.setError(ServerError.ServiceNotReady); + commandError.setMessage("Service not ready"); + cnx.handleError(commandError); + assertTrue(completableFuture.get().getCause() instanceof PulsarClientException.LookupException); + // wait for subsequent calls over + Awaitility.await().untilAsserted(() -> { + assertEquals(cnx.getPendingLookupRequestSemaphore().availablePermits(), conf.getConcurrentLookupRequest()); + }); + eventLoop.shutdownGracefully(); + } + @Test public void testPendingWaitingLookupRequestSemaphore() throws Exception { EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, new DefaultThreadFactory("testClientCnxTimeout")); From b193051bb70f0432bd68cc64b9025a653abab3d8 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Thu, 27 Oct 2022 20:05:29 +0200 Subject: [PATCH 15/22] Add HTTP Sink (#17581) --- .../terraform-ansible/deploy-pulsar.yaml | 1 + distribution/io/src/assemble/io.xml | 1 + pom.xml | 1 + pulsar-io/docs/pom.xml | 5 + pulsar-io/http/pom.xml | 87 ++++ .../org/apache/pulsar/io/http/HttpSink.java | 130 +++++ .../apache/pulsar/io/http/HttpSinkConfig.java | 58 +++ .../apache/pulsar/io/http/JsonConverter.java | 241 ++++++++++ .../apache/pulsar/io/http/package-info.java | 19 + .../META-INF/services/pulsar-io.yaml | 23 + .../apache/pulsar/io/http/HttpSinkTest.java | 445 ++++++++++++++++++ pulsar-io/pom.xml | 2 + 12 files changed, 1013 insertions(+) create mode 100644 pulsar-io/http/pom.xml create mode 100644 pulsar-io/http/src/main/java/org/apache/pulsar/io/http/HttpSink.java create mode 100644 pulsar-io/http/src/main/java/org/apache/pulsar/io/http/HttpSinkConfig.java create mode 100644 pulsar-io/http/src/main/java/org/apache/pulsar/io/http/JsonConverter.java create mode 100644 pulsar-io/http/src/main/java/org/apache/pulsar/io/http/package-info.java create mode 100644 pulsar-io/http/src/main/resources/META-INF/services/pulsar-io.yaml create mode 100644 pulsar-io/http/src/test/java/org/apache/pulsar/io/http/HttpSinkTest.java diff --git a/deployment/terraform-ansible/deploy-pulsar.yaml b/deployment/terraform-ansible/deploy-pulsar.yaml index ae1243ad66e63..db2fd1257ca41 100644 --- a/deployment/terraform-ansible/deploy-pulsar.yaml +++ b/deployment/terraform-ansible/deploy-pulsar.yaml @@ -154,6 +154,7 @@ # - jdbc-mariadb # - jdbc-postgres # - jdbc-sqlite +# - http - kafka # - kafka-connect-adaptor # - kinesis diff --git a/distribution/io/src/assemble/io.xml b/distribution/io/src/assemble/io.xml index 7657b35ae0919..33ca4e79ba942 100644 --- a/distribution/io/src/assemble/io.xml +++ b/distribution/io/src/assemble/io.xml @@ -47,6 +47,7 @@ ${basedir}/../../pulsar-io/cassandra/target/pulsar-io-cassandra-${project.version}.nar ${basedir}/../../pulsar-io/twitter/target/pulsar-io-twitter-${project.version}.nar ${basedir}/../../pulsar-io/kafka/target/pulsar-io-kafka-${project.version}.nar + ${basedir}/../../pulsar-io/http/target/pulsar-io-http-${project.version}.nar ${basedir}/../../pulsar-io/kinesis/target/pulsar-io-kinesis-${project.version}.nar ${basedir}/../../pulsar-io/rabbitmq/target/pulsar-io-rabbitmq-${project.version}.nar ${basedir}/../../pulsar-io/nsq/target/pulsar-io-nsq-${project.version}.nar diff --git a/pom.xml b/pom.xml index 1121742ae3140..53b7b2b2ee72b 100644 --- a/pom.xml +++ b/pom.xml @@ -248,6 +248,7 @@ flexible messaging model and an intuitive client API. 4.2.0 1.2.22 1.5.1 + 2.33.2 0.6.1 diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml index 3ed776035fc02..dca0edbc759d3 100644 --- a/pulsar-io/docs/pom.xml +++ b/pulsar-io/docs/pom.xml @@ -157,6 +157,11 @@ pulsar-io-jdbc-openmldb ${project.version}     + + ${project.groupId} + pulsar-io-http + ${project.version} + ${project.groupId} pulsar-io-kafka diff --git a/pulsar-io/http/pom.xml b/pulsar-io/http/pom.xml new file mode 100644 index 0000000000000..cd5a38d0adadc --- /dev/null +++ b/pulsar-io/http/pom.xml @@ -0,0 +1,87 @@ + + + 4.0.0 + + org.apache.pulsar + pulsar-io + 2.11.0-SNAPSHOT + + + pulsar-io-http + Pulsar IO :: HTTP + + + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + + com.fasterxml.jackson.core + jackson-databind + + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + ${jackson.version} + + + + org.apache.avro + avro + ${avro.version} + + + + org.apache.pulsar + pulsar-client-original + ${project.version} + test + + + + com.github.tomakehurst + wiremock-jre8 + ${wiremock.version} + test + + + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + + diff --git a/pulsar-io/http/src/main/java/org/apache/pulsar/io/http/HttpSink.java b/pulsar-io/http/src/main/java/org/apache/pulsar/io/http/HttpSink.java new file mode 100644 index 0000000000000..31b5053ba7a18 --- /dev/null +++ b/pulsar-io/http/src/main/java/org/apache/pulsar/io/http/HttpSink.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.io.http; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.schema.GenericObject; +import org.apache.pulsar.client.api.schema.KeyValueSchema; +import org.apache.pulsar.functions.api.Record; +import org.apache.pulsar.io.core.Sink; +import org.apache.pulsar.io.core.SinkContext; + +/** + * A Sink that makes a POST request to a configured HTTP endpoint for each record (webhook). + * The body of the HTTP request is the JSON representation of the record value. + * Some headers are added to the HTTP request: + *
      + *
    • PulsarTopic: the topic of the record
      • + *
    • PulsarKey: the key of the record
      • + *
    • PulsarEventTime: the event time of the record
      • + *
    • PulsarPublishTime: the publish time of the record
      • + *
    • PulsarMessageId: the ID of the message contained in the record
      • + *
    • PulsarProperties-*: each record property is passed with the property name prefixed by PulsarProperties-
      • + *
    + */ +public class HttpSink implements Sink { + + HttpSinkConfig httpSinkConfig; + private HttpClient httpClient; + private ObjectMapper mapper; + private URI uri; + + @Override + public void open(Map config, SinkContext sinkContext) throws Exception { + httpSinkConfig = HttpSinkConfig.load(config); + uri = new URI(httpSinkConfig.getUrl()); + httpClient = HttpClient.newHttpClient(); + mapper = new ObjectMapper().registerModule(new JavaTimeModule()); + } + + @Override + public void write(Record record) throws Exception { + Object json = toJsonSerializable(record.getSchema(), record.getValue().getNativeObject()); + byte[] bytes = mapper.writeValueAsBytes(json); + HttpRequest.Builder builder = HttpRequest.newBuilder() + .uri(uri) + .POST(HttpRequest.BodyPublishers.ofByteArray(bytes)); + httpSinkConfig.getHeaders().forEach(builder::header); + record.getProperties().forEach((k, v) -> builder.header("PulsarProperties-" + k, v)); + record.getTopicName().ifPresent(topic -> builder.header("PulsarTopic", topic)); + record.getEventTime().ifPresent(eventTime -> builder.header("PulsarEventTime", eventTime.toString())); + record.getKey().ifPresent(key -> builder.header("PulsarKey", key)); + record.getMessage().ifPresent( + message -> { + if (message.getMessageId() != null) { + String messageId = Base64.getEncoder().encodeToString(message.getMessageId().toByteArray()); + builder.header("PulsarMessageId", messageId); + } + if (message.getPublishTime() != 0) { + builder.header("PulsarPublishTime", String.valueOf(message.getPublishTime())); + } + } + ); + builder.header("Content-Type", "application/json"); + + HttpResponse response = httpClient.send(builder.build(), HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() < 200 || response.statusCode() >= 300) { + throw new IOException( + String.format("HTTP call to %s failed with status code %s", uri, response.statusCode())); + } + + } + + private static Object toJsonSerializable(Schema schema, Object val) { + if (schema == null || schema.getSchemaInfo().getType().isPrimitive()) { + return val; + } + switch (schema.getSchemaInfo().getType()) { + case KEY_VALUE: + KeyValueSchema keyValueSchema = (KeyValueSchema) schema; + org.apache.pulsar.common.schema.KeyValue keyValue = + (org.apache.pulsar.common.schema.KeyValue) val; + Map jsonKeyValue = new HashMap<>(); + Object key = keyValue.getKey(); + Object value = keyValue.getValue(); + jsonKeyValue.put("key", toJsonSerializable(keyValueSchema.getKeySchema(), + key instanceof GenericObject ? ((GenericObject) key).getNativeObject() : key)); + jsonKeyValue.put("value", toJsonSerializable(keyValueSchema.getValueSchema(), + value instanceof GenericObject ? ((GenericObject) value).getNativeObject() : value)); + return jsonKeyValue; + case AVRO: + return JsonConverter.toJson((org.apache.avro.generic.GenericRecord) val); + case JSON: + return val; + default: + throw new UnsupportedOperationException("Unsupported schema type =" + + schema.getSchemaInfo().getType()); + } + } + + @Override + public void close() {} +} diff --git a/pulsar-io/http/src/main/java/org/apache/pulsar/io/http/HttpSinkConfig.java b/pulsar-io/http/src/main/java/org/apache/pulsar/io/http/HttpSinkConfig.java new file mode 100644 index 0000000000000..2113aec720686 --- /dev/null +++ b/pulsar-io/http/src/main/java/org/apache/pulsar/io/http/HttpSinkConfig.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.io.http; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import java.io.File; +import java.io.IOException; +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; +import lombok.Data; +import lombok.experimental.Accessors; +import org.apache.pulsar.io.core.annotations.FieldDoc; + +@Data +@Accessors(chain = true) +public class HttpSinkConfig implements Serializable { + + private static final long serialVersionUID = 1L; + + @FieldDoc( + defaultValue = "http://localhost", + help = "The URL of the HTTP server") + private String url = "http://localhost"; + + @FieldDoc( + defaultValue = "", + help = "The list of default headers added to each request") + private Map headers = new HashMap<>(); + + public static HttpSinkConfig load(String yamlFile) throws IOException { + ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + return mapper.readValue(new File(yamlFile), HttpSinkConfig.class); + } + + public static HttpSinkConfig load(Map map) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + return mapper.readValue(new ObjectMapper().writeValueAsString(map), HttpSinkConfig.class); + } +} diff --git a/pulsar-io/http/src/main/java/org/apache/pulsar/io/http/JsonConverter.java b/pulsar-io/http/src/main/java/org/apache/pulsar/io/http/JsonConverter.java new file mode 100644 index 0000000000000..65ae6e876047e --- /dev/null +++ b/pulsar-io/http/src/main/java/org/apache/pulsar/io/http/JsonConverter.java @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.io.http; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.apache.avro.Conversion; +import org.apache.avro.Conversions; +import org.apache.avro.Schema; +import org.apache.avro.data.TimeConversions; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericFixed; +import org.apache.avro.generic.GenericRecord; + +/** + * Convert an AVRO GenericRecord to a JsonNode. + */ +public class JsonConverter { + + private static final Map> logicalTypeConverters = new HashMap<>(); + private static final JsonNodeFactory jsonNodeFactory = JsonNodeFactory.withExactBigDecimals(true); + + public static JsonNode topLevelMerge(JsonNode n1, JsonNode n2) { + ObjectNode objectNode = jsonNodeFactory.objectNode(); + n1.fieldNames().forEachRemaining(f -> objectNode.put(f, n1.get(f))); + n2.fieldNames().forEachRemaining(f -> objectNode.put(f, n2.get(f))); + return objectNode; + } + + public static JsonNode toJson(GenericRecord genericRecord) { + if (genericRecord == null) { + return null; + } + ObjectNode objectNode = jsonNodeFactory.objectNode(); + for (Schema.Field field : genericRecord.getSchema().getFields()) { + objectNode.set(field.name(), toJson(field.schema(), genericRecord.get(field.name()))); + } + return objectNode; + } + + public static JsonNode toJson(Schema schema, Object value) { + if (schema.getLogicalType() != null && logicalTypeConverters.containsKey(schema.getLogicalType().getName())) { + return logicalTypeConverters.get(schema.getLogicalType().getName()).toJson(schema, value); + } + if (value == null) { + return jsonNodeFactory.nullNode(); + } + switch(schema.getType()) { + case NULL: // this should not happen + return jsonNodeFactory.nullNode(); + case INT: + return jsonNodeFactory.numberNode((Integer) value); + case LONG: + return jsonNodeFactory.numberNode((Long) value); + case DOUBLE: + return jsonNodeFactory.numberNode((Double) value); + case FLOAT: + return jsonNodeFactory.numberNode((Float) value); + case BOOLEAN: + return jsonNodeFactory.booleanNode((Boolean) value); + case BYTES: + return jsonNodeFactory.binaryNode((byte[]) value); + case FIXED: + return jsonNodeFactory.binaryNode(((GenericFixed) value).bytes()); + case ENUM: // GenericEnumSymbol + case STRING: + return jsonNodeFactory.textNode(value.toString()); // can be a String or org.apache.avro.util.Utf8 + case ARRAY: { + Schema elementSchema = schema.getElementType(); + ArrayNode arrayNode = jsonNodeFactory.arrayNode(); + Object[] iterable; + if (value instanceof GenericData.Array) { + iterable = ((GenericData.Array) value).toArray(); + } else { + iterable = (Object[]) value; + } + for (Object elem : iterable) { + JsonNode fieldValue = toJson(elementSchema, elem); + arrayNode.add(fieldValue); + } + return arrayNode; + } + case MAP: { + Map map = (Map) value; + ObjectNode objectNode = jsonNodeFactory.objectNode(); + for (Map.Entry entry : map.entrySet()) { + JsonNode jsonNode = toJson(schema.getValueType(), entry.getValue()); + // can be a String or org.apache.avro.util.Utf8 + final String entryKey = entry.getKey() == null ? null : entry.getKey().toString(); + objectNode.set(entryKey, jsonNode); + } + return objectNode; + } + case RECORD: + return toJson((GenericRecord) value); + case UNION: + for (Schema s : schema.getTypes()) { + if (s.getType() == Schema.Type.NULL) { + continue; + } + return toJson(s, value); + } + // this case should not happen + return jsonNodeFactory.textNode(value.toString()); + default: + throw new UnsupportedOperationException("Unknown AVRO schema type=" + schema.getType()); + } + } + + abstract static class LogicalTypeConverter { + final Conversion conversion; + + public LogicalTypeConverter(Conversion conversion) { + this.conversion = conversion; + } + + abstract JsonNode toJson(Schema schema, Object value); + } + + static { + logicalTypeConverters.put("decimal", new LogicalTypeConverter( + new Conversions.DecimalConversion()) { + @Override + JsonNode toJson(Schema schema, Object value) { + if (!(value instanceof BigDecimal)) { + throw new IllegalArgumentException("Invalid type for Decimal, expected BigDecimal but was " + + value.getClass()); + } + BigDecimal decimal = (BigDecimal) value; + return jsonNodeFactory.numberNode(decimal); + } + }); + logicalTypeConverters.put("date", new LogicalTypeConverter( + new TimeConversions.DateConversion()) { + @Override + JsonNode toJson(Schema schema, Object value) { + if (!(value instanceof Integer)) { + throw new IllegalArgumentException("Invalid type for date, expected Integer but was " + + value.getClass()); + } + Integer daysFromEpoch = (Integer) value; + return jsonNodeFactory.numberNode(daysFromEpoch); + } + }); + logicalTypeConverters.put("time-millis", new LogicalTypeConverter( + new TimeConversions.TimeMillisConversion()) { + @Override + JsonNode toJson(Schema schema, Object value) { + if (!(value instanceof Integer)) { + throw new IllegalArgumentException("Invalid type for time-millis, expected Integer but was " + + value.getClass()); + } + Integer timeMillis = (Integer) value; + return jsonNodeFactory.numberNode(timeMillis); + } + }); + logicalTypeConverters.put("time-micros", new LogicalTypeConverter( + new TimeConversions.TimeMicrosConversion()) { + @Override + JsonNode toJson(Schema schema, Object value) { + if (!(value instanceof Long)) { + throw new IllegalArgumentException("Invalid type for time-micros, expected Long but was " + + value.getClass()); + } + Long timeMicro = (Long) value; + return jsonNodeFactory.numberNode(timeMicro); + } + }); + logicalTypeConverters.put("timestamp-millis", new LogicalTypeConverter( + new TimeConversions.TimestampMillisConversion()) { + @Override + JsonNode toJson(Schema schema, Object value) { + if (!(value instanceof Long)) { + throw new IllegalArgumentException("Invalid type for timestamp-millis, expected Long but was " + + value.getClass()); + } + Long epochMillis = (Long) value; + return jsonNodeFactory.numberNode(epochMillis); + } + }); + logicalTypeConverters.put("timestamp-micros", new LogicalTypeConverter( + new TimeConversions.TimestampMicrosConversion()) { + @Override + JsonNode toJson(Schema schema, Object value) { + if (!(value instanceof Long)) { + throw new IllegalArgumentException("Invalid type for timestamp-micros, expected Long but was " + + value.getClass()); + } + Long epochMillis = (Long) value; + return jsonNodeFactory.numberNode(epochMillis); + } + }); + logicalTypeConverters.put("uuid", new LogicalTypeConverter( + new Conversions.UUIDConversion()) { + @Override + JsonNode toJson(Schema schema, Object value) { + return jsonNodeFactory.textNode(value == null ? null : value.toString()); + } + }); + } + + public static ArrayNode toJsonArray(JsonNode jsonNode, List fields) { + ArrayNode arrayNode = jsonNodeFactory.arrayNode(); + Iterator it = jsonNode.fieldNames(); + while (it.hasNext()) { + String fieldName = it.next(); + if (fields.contains(fieldName)) { + arrayNode.add(jsonNode.get(fieldName)); + } + } + return arrayNode; + } + +} diff --git a/pulsar-io/http/src/main/java/org/apache/pulsar/io/http/package-info.java b/pulsar-io/http/src/main/java/org/apache/pulsar/io/http/package-info.java new file mode 100644 index 0000000000000..87131fdf3be8c --- /dev/null +++ b/pulsar-io/http/src/main/java/org/apache/pulsar/io/http/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.io.http; diff --git a/pulsar-io/http/src/main/resources/META-INF/services/pulsar-io.yaml b/pulsar-io/http/src/main/resources/META-INF/services/pulsar-io.yaml new file mode 100644 index 0000000000000..bdd1712b9249a --- /dev/null +++ b/pulsar-io/http/src/main/resources/META-INF/services/pulsar-io.yaml @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: http +description: Writes data to an HTTP server (Webhook) +sinkClass: org.apache.pulsar.io.http.HttpSink +sinkConfigClass: org.apache.pulsar.io.http.HttpSinkConfig diff --git a/pulsar-io/http/src/test/java/org/apache/pulsar/io/http/HttpSinkTest.java b/pulsar-io/http/src/test/java/org/apache/pulsar/io/http/HttpSinkTest.java new file mode 100644 index 0000000000000..d5de27d628407 --- /dev/null +++ b/pulsar-io/http/src/test/java/org/apache/pulsar/io/http/HttpSinkTest.java @@ -0,0 +1,445 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.io.http; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.configureFor; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.verify; +import com.github.tomakehurst.wiremock.WireMockServer; +import java.io.IOException; +import java.sql.Time; +import java.sql.Timestamp; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.schema.GenericObject; +import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.client.api.schema.GenericSchema; +import org.apache.pulsar.client.api.schema.RecordSchemaBuilder; +import org.apache.pulsar.client.api.schema.SchemaBuilder; +import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.client.impl.schema.KeyValueSchemaImpl; +import org.apache.pulsar.common.api.EncryptionContext; +import org.apache.pulsar.common.schema.KeyValue; +import org.apache.pulsar.common.schema.KeyValueEncodingType; +import org.apache.pulsar.common.schema.SchemaType; +import org.apache.pulsar.functions.api.Record; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class HttpSinkTest { + + WireMockServer server; + + @BeforeClass + public void setUp() { + server = new WireMockServer(0); + server.start(); + configureFor(server.port()); + stubFor(post(urlPathEqualTo("/")) + .willReturn(aResponse().withStatus(200))); + } + + @AfterClass + public void tearDown() { + server.stop(); + } + + @DataProvider(name = "primitives") + public Object[][] primitives() { + return new Object[][]{ + new Object[] {Schema.STRING, "test-string", "\"test-string\""}, + new Object[] {Schema.INT8, (byte) 42, "42"}, + new Object[] {Schema.INT16, (short) 42, "42"}, + new Object[] {Schema.INT32, 42, "42"}, + new Object[] {Schema.INT64, 42L, "42"}, + new Object[] {Schema.BOOL, true, "true"}, + new Object[] {Schema.FLOAT, 0.1F, "0.1"}, + new Object[] {Schema.DOUBLE, 0.1D, "0.1"}, + new Object[] {Schema.DATE, new Date(1662418008047L), "1662418008047"}, + new Object[] {Schema.TIME, new Time(0, 46, 48), "\"00:46:48\""}, + new Object[] {Schema.TIMESTAMP, new Timestamp(1662418008047L), "1662418008047"}, + new Object[] {Schema.INSTANT, Instant.ofEpochMilli(1662418008047L), "1662418008.047000000"}, + new Object[] {Schema.LOCAL_DATE, LocalDate.of(2022, 1, 1), "[2022,1,1]"}, + new Object[] {Schema.LOCAL_TIME, LocalTime.of(11, 12), "[11,12]"}, + new Object[] {Schema.LOCAL_DATE_TIME, new Timestamp(1662418008047L), "1662418008047"}, + }; + } + + @Test(dataProvider = "primitives") + public void testPrimitives(Schema schema, Object value, String responseBody) throws Exception { + GenericObject genericObject = new GenericObject() { + @Override + public SchemaType getSchemaType() { + return null; + } + + @Override + public Object getNativeObject() { + return value; + } + }; + test(schema, genericObject, responseBody); + } + + @DataProvider(name = "schema") + public Object[][] schema() { + return new Object[][]{ + new Object[]{Schema.JSON(Object.class)}, + new Object[]{Schema.AVRO(Object.class)}, + }; + } + + @Test(dataProvider = "schema") + public void testGenericRecord(Schema schema) throws Exception { + SchemaType schemaType = schema.getSchemaInfo().getType(); + RecordSchemaBuilder valueSchemaBuilder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("value"); + valueSchemaBuilder.field("c").type(SchemaType.STRING).optional().defaultValue(null); + valueSchemaBuilder.field("d").type(SchemaType.INT32).optional().defaultValue(null); + RecordSchemaBuilder udtSchemaBuilder = SchemaBuilder.record("type1"); + udtSchemaBuilder.field("a").type(SchemaType.STRING).optional().defaultValue(null); + udtSchemaBuilder.field("b").type(SchemaType.BOOLEAN).optional().defaultValue(null); + udtSchemaBuilder.field("d").type(SchemaType.DOUBLE).optional().defaultValue(null); + udtSchemaBuilder.field("f").type(SchemaType.FLOAT).optional().defaultValue(null); + udtSchemaBuilder.field("i").type(SchemaType.INT32).optional().defaultValue(null); + udtSchemaBuilder.field("l").type(SchemaType.INT64).optional().defaultValue(null); + GenericSchema udtGenericSchema = Schema.generic(udtSchemaBuilder.build(schemaType)); + valueSchemaBuilder.field("e", udtGenericSchema).type(schemaType).optional().defaultValue(null); + GenericSchema valueSchema = Schema.generic(valueSchemaBuilder.build(schemaType)); + + GenericRecord valueGenericRecord = valueSchema.newRecordBuilder() + .set("c", "1") + .set("d", 1) + .set("e", udtGenericSchema.newRecordBuilder() + .set("a", "a") + .set("b", true) + .set("d", 1.0) + .set("f", 1.0f) + .set("i", 1) + .set("l", 10L) + .build()) + .build(); + + String responseBody = + "{\"c\":\"1\",\"d\":1,\"e\":{\"a\":\"a\",\"b\":true,\"d\":1.0,\"f\":1.0,\"i\":1,\"l\":10}}"; + test(schema, valueGenericRecord, responseBody); + } + + @Test + public void testKeyValuePrimitives() throws Exception { + Schema> keyValueSchema = KeyValueSchemaImpl.of(Schema.STRING, Schema.STRING); + GenericObject genericObject = new GenericObject() { + @Override + public SchemaType getSchemaType() { + return null; + } + + @Override + public Object getNativeObject() { + return new KeyValue<>("test-key", "test-value"); + } + }; + test(keyValueSchema, genericObject, "{\"value\":\"test-value\",\"key\":\"test-key\"}"); + } + + @Test(dataProvider = "schema") + public void testKeyValueGenericRecord(Schema schema) throws Exception { + SchemaType schemaType = schema.getSchemaInfo().getType(); + RecordSchemaBuilder keySchemaBuilder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("key"); + keySchemaBuilder.field("a").type(SchemaType.STRING).optional().defaultValue(null); + keySchemaBuilder.field("b").type(SchemaType.INT32).optional().defaultValue(null); + GenericSchema keySchema = Schema.generic(keySchemaBuilder.build(schemaType)); + GenericRecord keyGenericRecord = keySchema.newRecordBuilder() + .set("a", "1") + .set("b", 1) + .build(); + + RecordSchemaBuilder valueSchemaBuilder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("value"); + valueSchemaBuilder.field("c").type(SchemaType.STRING).optional().defaultValue(null); + valueSchemaBuilder.field("d").type(SchemaType.INT32).optional().defaultValue(null); + RecordSchemaBuilder udtSchemaBuilder = SchemaBuilder.record("type1"); + udtSchemaBuilder.field("a").type(SchemaType.STRING).optional().defaultValue(null); + udtSchemaBuilder.field("b").type(SchemaType.BOOLEAN).optional().defaultValue(null); + udtSchemaBuilder.field("d").type(SchemaType.DOUBLE).optional().defaultValue(null); + udtSchemaBuilder.field("f").type(SchemaType.FLOAT).optional().defaultValue(null); + udtSchemaBuilder.field("i").type(SchemaType.INT32).optional().defaultValue(null); + udtSchemaBuilder.field("l").type(SchemaType.INT64).optional().defaultValue(null); + GenericSchema udtGenericSchema = Schema.generic(udtSchemaBuilder.build(schemaType)); + valueSchemaBuilder.field("e", udtGenericSchema).type(schemaType).optional().defaultValue(null); + GenericSchema valueSchema = Schema.generic(valueSchemaBuilder.build(schemaType)); + + GenericRecord valueGenericRecord = valueSchema.newRecordBuilder() + .set("c", "1") + .set("d", 1) + .set("e", udtGenericSchema.newRecordBuilder() + .set("a", "a") + .set("b", true) + .set("d", 1.0) + .set("f", 1.0f) + .set("i", 1) + .set("l", 10L) + .build()) + .build(); + + Schema> keyValueSchema = Schema.KeyValue(keySchema, valueSchema, KeyValueEncodingType.INLINE); + KeyValue keyValue = new KeyValue<>(keyGenericRecord, valueGenericRecord); + GenericObject genericObject = new GenericObject() { + @Override + public SchemaType getSchemaType() { + return SchemaType.KEY_VALUE; + } + + @Override + public Object getNativeObject() { + return keyValue; + } + }; + String responseBody = "{\"value\":{\"c\":\"1\",\"d\":1,\"e\":{\"a\":\"a\",\"b\":true,\"d\":1.0,\"f\":1.0," + + "\"i\":1,\"l\":10}},\"key\":{\"a\":\"1\",\"b\":1}}"; + test(keyValueSchema, genericObject, responseBody); + } + + private void test(Schema schema, GenericObject genericObject, String responseBody) throws Exception { + HttpSink httpSink = new HttpSink(); + Map config = new HashMap<>(); + config.put("url", server.baseUrl()); + Map headers = new HashMap<>(); + headers.put("header-name", "header-value"); + config.put("headers", headers); + httpSink.open(config, null); + + long now = 1662418008000L; + Map messageProperties = new HashMap<>(); + messageProperties.put("prop-name", "prop-value"); + + Record record = new Record<>() { + @Override + public GenericObject getValue() { + return genericObject; + } + + @Override + public Schema getSchema() { + return schema; + } + + @Override + public Optional getEventTime() { + return Optional.of(now); + } + + @Override + public Map getProperties() { + return messageProperties; + } + + @Override + public Optional getTopicName() { + return Optional.of("test-topic"); + } + + @Override + public Optional getKey() { + return Optional.of("test-key"); + } + + @Override + public Optional> getMessage() { + return Optional.of(new Message<>() { + + @Override + public Map getProperties() { + return null; + } + + @Override + public boolean hasProperty(String name) { + return false; + } + + @Override + public String getProperty(String name) { + return null; + } + + @Override + public byte[] getData() { + return new byte[0]; + } + + @Override + public int size() { + return 0; + } + + @Override + public GenericObject getValue() { + return null; + } + + @Override + public MessageId getMessageId() { + return new MessageIdImpl(1, 2, 3); + } + + @Override + public long getPublishTime() { + return now + 1; + } + + @Override + public long getEventTime() { + return 0; + } + + @Override + public long getSequenceId() { + return 0; + } + + @Override + public String getProducerName() { + return null; + } + + @Override + public boolean hasKey() { + return false; + } + + @Override + public String getKey() { + return null; + } + + @Override + public boolean hasBase64EncodedKey() { + return false; + } + + @Override + public byte[] getKeyBytes() { + return new byte[0]; + } + + @Override + public boolean hasOrderingKey() { + return false; + } + + @Override + public byte[] getOrderingKey() { + return new byte[0]; + } + + @Override + public String getTopicName() { + return null; + } + + @Override + public Optional getEncryptionCtx() { + return Optional.empty(); + } + + @Override + public int getRedeliveryCount() { + return 0; + } + + @Override + public byte[] getSchemaVersion() { + return new byte[0]; + } + + @Override + public boolean isReplicated() { + return false; + } + + @Override + public String getReplicatedFrom() { + return null; + } + + @Override + public void release() { + + } + + @Override + public boolean hasBrokerPublishTime() { + return false; + } + + @Override + public Optional getBrokerPublishTime() { + return Optional.empty(); + } + + @Override + public boolean hasIndex() { + return false; + } + + @Override + public Optional getIndex() { + return Optional.empty(); + } + }); + } + }; + httpSink.write(record); + + verify(postRequestedFor(urlEqualTo("/")) + .withRequestBody(equalTo(responseBody)) + .withHeader("Content-Type", equalTo("application/json")) + .withHeader("header-name", equalTo("header-value")) + .withHeader("PulsarTopic", equalTo("test-topic")) + .withHeader("PulsarKey", equalTo("test-key")) + .withHeader("PulsarEventTime", equalTo("1662418008000")) + .withHeader("PulsarPublishTime", equalTo("1662418008001")) + .withHeader("PulsarMessageId", equalTo("CAEQAhgDMAA=")) + .withHeader("PulsarProperties-prop-name", equalTo("prop-value")) + ); + } + + @Test(expectedExceptions = IOException.class) + public void testRequestFailure() throws Exception { + stubFor(post(urlPathEqualTo("/")) + .willReturn(aResponse().withStatus(500))); + + testKeyValuePrimitives(); + } +} diff --git a/pulsar-io/pom.xml b/pulsar-io/pom.xml index b2c400117a50a..a5e096aff59a7 100644 --- a/pulsar-io/pom.xml +++ b/pulsar-io/pom.xml @@ -51,6 +51,7 @@ twitter cassandra aerospike + http kafka rabbitmq kinesis @@ -88,6 +89,7 @@ twitter cassandra aerospike + http kafka rabbitmq kinesis From 0bfbda8ce14fcee9463fecbb94a0fc32822bd761 Mon Sep 17 00:00:00 2001 From: Vineeth Date: Wed, 19 Oct 2022 09:07:01 -0700 Subject: [PATCH 16/22] =?UTF-8?q?[improve][pulsar-broker]=20Add=20option?= =?UTF-8?q?=20to=C2=A0=20unloadNamespaceBundle=20with=20bundle=20Affinity?= =?UTF-8?q?=20broker=20url?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../broker/admin/impl/NamespacesBase.java | 29 ++++++++- .../pulsar/broker/admin/v1/Namespaces.java | 4 +- .../pulsar/broker/admin/v2/Namespaces.java | 4 +- .../broker/loadbalance/LoadManager.java | 4 ++ .../loadbalance/ModularLoadManager.java | 6 ++ .../broker/loadbalance/NoopLoadManager.java | 13 ++++ .../impl/ModularLoadManagerImpl.java | 19 +++++- .../impl/ModularLoadManagerWrapper.java | 29 +++++++-- .../impl/SimpleLoadManagerImpl.java | 15 +++++ .../broker/namespace/NamespaceService.java | 1 + .../pulsar/broker/web/PulsarWebResource.java | 2 +- .../pulsar/broker/admin/NamespacesTest.java | 4 +- .../ModularLoadManagerImplTest.java | 63 ++++++++++++++++++- .../pulsar/client/admin/Namespaces.java | 27 ++++++++ .../client/admin/internal/NamespacesImpl.java | 12 ++++ .../pulsar/admin/cli/CmdNamespaces.java | 11 +++- .../pulsar/tests/integration/cli/CLITest.java | 2 +- 17 files changed, 227 insertions(+), 18 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 59e1226e16c71..47790eafb39c8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -56,6 +56,7 @@ import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.admin.AdminResource; import org.apache.pulsar.broker.authorization.AuthorizationService; +import org.apache.pulsar.broker.loadbalance.LeaderBroker; import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException; import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.Topic; @@ -852,6 +853,29 @@ protected BookieAffinityGroupData internalGetBookieAffinityGroup() { } } + public void setNamespaceBundleAffinity (String bundleRange, String brokerUrl) { + if (brokerUrl != null) { + if (!this.isLeaderBroker()) { + LeaderBroker leaderBroker = pulsar().getLeaderElectionService().getCurrentLeader().get(); + String leaderBrokerUrl = leaderBroker.getServiceUrl(); + try { + URL redirectUrl = new URL(leaderBrokerUrl); + URI redirect = UriBuilder.fromUri(uri.getRequestUri()).host(redirectUrl.getHost()) + .port(redirectUrl.getPort()).replaceQueryParam("authoritative", + false).build(); + + // Redirect + log.debug("Redirecting the rest call to {}, bundleRange - {}", redirect, bundleRange); + throw new WebApplicationException(Response.temporaryRedirect(redirect).build()); + } catch (MalformedURLException exception) { + log.error("The leader broker url is malformed - {}", leaderBrokerUrl); + throw new RestException(exception); + } + } + pulsar().getLoadManager().get().setNamespaceBundleAffinity(bundleRange, brokerUrl); + } + } + public CompletableFuture internalUnloadNamespaceBundleAsync(String bundleRange, boolean authoritative) { return validateSuperUserAccessAsync() .thenAccept(__ -> { @@ -898,8 +922,9 @@ public CompletableFuture internalUnloadNamespaceBundleAsync(String bundleR } return validateNamespaceBundleOwnershipAsync(namespaceName, policies.bundles, bundleRange, authoritative, true) - .thenCompose(nsBundle -> - pulsar().getNamespaceService().unloadNamespaceBundle(nsBundle)); + .thenCompose(nsBundle -> { + return pulsar().getNamespaceService().unloadNamespaceBundle(nsBundle); + }); })); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java index 8f81c24502904..463a9ab721fc8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java @@ -885,8 +885,10 @@ public void unloadNamespace(@Suspended final AsyncResponse asyncResponse, @PathP public void unloadNamespaceBundle(@Suspended final AsyncResponse asyncResponse, @PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @PathParam("bundle") String bundleRange, - @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, + @QueryParam("destinationBroker") String destinationBroker) { validateNamespaceName(property, cluster, namespace); + setNamespaceBundleAffinity(bundleRange, destinationBroker); internalUnloadNamespaceBundleAsync(bundleRange, authoritative) .thenAccept(__ -> { log.info("[{}] Successfully unloaded namespace bundle {}", clientAppId(), bundleRange); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java index 63120e47b8d84..9396aa8a0e66d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java @@ -812,8 +812,10 @@ public void unloadNamespace(@Suspended final AsyncResponse asyncResponse, public void unloadNamespaceBundle(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("bundle") String bundleRange, - @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, + @QueryParam("destinationBroker") String destinationBroker) { validateNamespaceName(tenant, namespace); + setNamespaceBundleAffinity(bundleRange, destinationBroker); internalUnloadNamespaceBundleAsync(bundleRange, authoritative) .thenAccept(__ -> { log.info("[{}] Successfully unloaded namespace bundle {}", clientAppId(), bundleRange); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadManager.java index e34215d199648..6d2f5c52454ef 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadManager.java @@ -118,6 +118,10 @@ default void writeLoadReportOnZookeeper(boolean force) throws Exception { Set getAvailableBrokers() throws Exception; CompletableFuture> getAvailableBrokersAsync(); + + void setNamespaceBundleAffinity(String bundle, String broker); + + String removeNamespaceBundleAffinity(String bundle); void stop() throws PulsarServerException; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManager.java index fa6895568e918..461ed2dd3ce16 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManager.java @@ -140,4 +140,10 @@ default void writeBrokerDataOnZooKeeper(boolean force) { * @return bundle data */ BundleData getBundleDataOrDefault(String bundle); + + String getNamespaceBundleAffinity(String bundle); + + void setNamespaceBundleAffinity(String bundle, String broker); + + String removeNamespaceBundleAffinity(String bundle); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java index 1ab56b50cdef4..0b852772fa34c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java @@ -24,6 +24,7 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import java.util.concurrent.ConcurrentHashMap; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; @@ -43,11 +44,13 @@ public class NoopLoadManager implements LoadManager { private String lookupServiceAddress; private ResourceUnit localResourceUnit; private LockManager lockManager; + private ConcurrentHashMap bundleBrokerAffinityMap; @Override public void initialize(PulsarService pulsar) { this.pulsar = pulsar; this.lockManager = pulsar.getCoordinationService().getLockManager(LocalBrokerData.class); + this.bundleBrokerAffinityMap = new ConcurrentHashMap<>(); } @Override @@ -142,4 +145,14 @@ public void stop() throws PulsarServerException { } } + @Override + public void setNamespaceBundleAffinity(String bundle, String broker) { + broker = broker.replaceFirst("http[s]?://", ""); + this.bundleBrokerAffinityMap.put(bundle, broker); + } + + @Override + public String removeNamespaceBundleAffinity(String bundle) { + return this.bundleBrokerAffinityMap.remove(bundle); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java index c14768eed5e9f..68b0f4113aadc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java @@ -197,6 +197,7 @@ public class ModularLoadManagerImpl implements ModularLoadManager { private final Lock lock = new ReentrantLock(); private Set knownBrokers = ConcurrentHashMap.newKeySet(); + private Map bundleBrokerAffinityMap; /** * Initializes fields which do not depend on PulsarService. initialize(PulsarService) should subsequently be called. @@ -215,7 +216,7 @@ public ModularLoadManagerImpl() { scheduler = Executors.newSingleThreadScheduledExecutor( new ExecutorProvider.ExtendedThreadFactory("pulsar-modular-load-manager")); this.brokerToFailureDomainMap = new HashMap<>(); - + this.bundleBrokerAffinityMap = new ConcurrentHashMap<>(); this.brokerTopicLoadingPredicate = new BrokerTopicLoadingPredicate() { @Override public boolean isEnablePersistentTopics(String brokerUrl) { @@ -1214,4 +1215,20 @@ public List getLoadBalancingMetrics() { return metricsCollection; } + + @Override + public String getNamespaceBundleAffinity(String bundle) { + return this.bundleBrokerAffinityMap.get(bundle); + } + + @Override + public void setNamespaceBundleAffinity(String bundle, String broker) { + broker = broker.replaceFirst("http[s]?://", ""); + this.bundleBrokerAffinityMap.put(bundle, broker); + } + + @Override + public String removeNamespaceBundleAffinity(String bundle) { + return this.bundleBrokerAffinityMap.remove(bundle); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerWrapper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerWrapper.java index 5f7cd5b8c38fa..80814ee8186fb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerWrapper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerWrapper.java @@ -65,13 +65,13 @@ public LoadManagerReport generateLoadReport() { @Override public Optional getLeastLoaded(final ServiceUnitId serviceUnit) { + String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(serviceUnit.toString()); + String affinityBroker = loadManager.removeNamespaceBundleAffinity(bundleRange); + if (affinityBroker != null) { + return Optional.of(buildBrokerResourceUnit(affinityBroker)); + } Optional leastLoadedBroker = loadManager.selectBrokerForAssignment(serviceUnit); - return leastLoadedBroker.map(s -> { - String webServiceUrl = getBrokerWebServiceUrl(s); - String brokerZnodeName = getBrokerZnodeName(s, webServiceUrl); - return new SimpleResourceUnit(webServiceUrl, - new PulsarResourceDescription(), Map.of(ResourceUnit.PROPERTY_KEY_BROKER_ZNODE_NAME, brokerZnodeName)); - }); + return leastLoadedBroker.map(this::buildBrokerResourceUnit); } private String getBrokerWebServiceUrl(String broker) { @@ -146,4 +146,21 @@ public Set getAvailableBrokers() throws Exception { public CompletableFuture> getAvailableBrokersAsync() { return loadManager.getAvailableBrokersAsync(); } + + private SimpleResourceUnit buildBrokerResourceUnit (String broker) { + String webServiceUrl = getBrokerWebServiceUrl(broker); + String brokerZnodeName = getBrokerZnodeName(broker, webServiceUrl); + return new SimpleResourceUnit(webServiceUrl, + new PulsarResourceDescription(), Map.of(ResourceUnit.PROPERTY_KEY_BROKER_ZNODE_NAME, brokerZnodeName)); + } + + @Override + public void setNamespaceBundleAffinity(String bundle, String broker) { + loadManager.setNamespaceBundleAffinity(bundle, broker); + } + + @Override + public String removeNamespaceBundleAffinity(String bundle) { + return loadManager.removeNamespaceBundleAffinity(bundle); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java index c2c0d1947c93e..350e2d50ea011 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java @@ -40,6 +40,7 @@ import java.util.TreeSet; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; @@ -186,6 +187,8 @@ public class SimpleLoadManagerImpl implements LoadManager, Consumer updateRankingHandle; + private ConcurrentHashMap bundleBrokerAffinityMap; + // Perform initializations which may be done without a PulsarService. public SimpleLoadManagerImpl() { scheduler = Executors.newSingleThreadScheduledExecutor( @@ -251,6 +254,7 @@ public Long load(String key) throws Exception { } }); this.pulsar = pulsar; + this.bundleBrokerAffinityMap = new ConcurrentHashMap<>(); } public SimpleLoadManagerImpl(PulsarService pulsar) { @@ -1442,6 +1446,17 @@ public void doNamespaceBundleSplit() throws Exception { this.setLoadReportForceUpdateFlag(); } } + + @Override + public void setNamespaceBundleAffinity(String bundle, String broker) { + broker = broker.replaceFirst("http[s]?://", ""); + this.bundleBrokerAffinityMap.put(bundle, broker); + } + + @Override + public String removeNamespaceBundleAffinity(String bundle) { + return this.bundleBrokerAffinityMap.remove(bundle); + } @Override public void stop() throws PulsarServerException { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 2e193823d9f39..1c64ea108126f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -712,6 +712,7 @@ private Optional> getLeastLoadedFromLoadManager(ServiceUnit String lookupAddress = leastLoadedBroker.get().getResourceId(); String advertisedAddr = (String) leastLoadedBroker.get() .getProperty(ResourceUnit.PROPERTY_KEY_BROKER_ZNODE_NAME); + if (LOG.isDebugEnabled()) { LOG.debug("{} : redirecting to the least loaded broker, lookup address={}", pulsar.getSafeWebServiceAddress(), diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java index 4f53b296e22a0..8657c28ce75f5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java @@ -710,7 +710,7 @@ public CompletableFuture validateBundleOwnershipAsync(NamespaceBundle bund // Replace the host and port of the current request and redirect URI redirect = UriBuilder.fromUri(uri.getRequestUri()).host(webUrl.get().getHost()) .port(webUrl.get().getPort()).replaceQueryParam("authoritative", - newAuthoritative).build(); + newAuthoritative).replaceQueryParam("brokerUrl", null).build(); log.debug("{} is not a service unit owned", bundle); // Redirect diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java index 51c68ea0e89df..e05ddc4f8a400 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java @@ -687,7 +687,7 @@ public void testNamespacesApiRedirects() throws Exception { doReturn(uri).when(uriInfo).getRequestUri(); namespaces.unloadNamespaceBundle(response, this.testTenant, this.testOtherCluster, - this.testLocalNamespaces.get(2).getLocalName(), "0x00000000_0xffffffff", false); + this.testLocalNamespaces.get(2).getLocalName(), "0x00000000_0xffffffff", false, null); captor = ArgumentCaptor.forClass(WebApplicationException.class); verify(response, timeout(5000).atLeast(1)).resume(captor.capture()); assertEquals(captor.getValue().getResponse().getStatus(), Status.TEMPORARY_REDIRECT.getStatusCode()); @@ -996,7 +996,7 @@ public void testUnloadNamespaceWithBundles() throws Exception { doReturn(CompletableFuture.completedFuture(null)).when(nsSvc).unloadNamespaceBundle(testBundle); AsyncResponse response = mock(AsyncResponse.class); namespaces.unloadNamespaceBundle(response, testTenant, testLocalCluster, bundledNsLocal, "0x00000000_0x80000000", - false); + false, null); verify(response, timeout(5000).times(1)).resume(any(RestException.class)); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerImplTest.java index 05e82484226ca..5a1f2b2167f5d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerImplTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.loadbalance; +import static java.lang.Thread.sleep; import static org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl.TIME_AVERAGE_BROKER_ZPATH; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.spy; @@ -40,6 +41,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Random; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; @@ -85,6 +87,7 @@ import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; import org.awaitility.Awaitility; import org.mockito.Mockito; +import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -188,7 +191,7 @@ void setup() throws Exception { primaryLoadManager = (ModularLoadManagerImpl) getField(pulsar1.getLoadManager().get(), "loadManager"); secondaryLoadManager = (ModularLoadManagerImpl) getField(pulsar2.getLoadManager().get(), "loadManager"); nsFactory = new NamespaceBundleFactory(pulsar1, Hashing.crc32()); - Thread.sleep(100); + sleep(100); } @AfterMethod(alwaysRun = true) @@ -284,6 +287,62 @@ public void testEvenBundleDistribution() throws Exception { } } + + + @Test + public void testBrokerAffinity() throws Exception { + // Start broker 3 + ServiceConfiguration config = new ServiceConfiguration(); + config.setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); + config.setLoadBalancerLoadSheddingStrategy("org.apache.pulsar.broker.loadbalance.impl.OverloadShedder"); + config.setClusterName("use"); + config.setWebServicePort(Optional.of(0)); + config.setMetadataStoreUrl("zk:127.0.0.1:" + bkEnsemble.getZookeeperPort()); + config.setAdvertisedAddress("localhost"); + config.setBrokerShutdownTimeoutMs(0L); + config.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); + config.setBrokerServicePort(Optional.of(0)); + config.setBrokerServicePortTls(Optional.of(0)); + config.setWebServicePortTls(Optional.of(0)); + PulsarService pulsar3 = new PulsarService(config); + pulsar3.start(); + + final String tenant = "test"; + final String cluster = "test"; + String namespace = tenant + "/" + cluster + "/" + "test"; + String topic = "persistent://" + namespace + "/my-topic1"; + admin1.clusters().createCluster(cluster, ClusterData.builder().serviceUrl("http://" + pulsar1.getAdvertisedAddress()).build()); + admin1.tenants().createTenant(tenant, + new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet(cluster))); + admin1.namespaces().createNamespace(namespace, 16); + + String topicLookup = admin1.lookups().lookupTopic(topic); + String bundleRange = admin1.lookups().getBundleRange(topic); + + String brokerServiceUrl = pulsar1.getBrokerServiceUrl(); + String brokerUrl = pulsar1.getSafeWebServiceAddress(); + Random rand=new Random(); + + if (topicLookup.equals(brokerServiceUrl)) { + int x = rand.nextInt(2); + if (x == 0) { + brokerUrl = pulsar2.getSafeWebServiceAddress(); + brokerServiceUrl = pulsar2.getBrokerServiceUrl(); + } + else { + brokerUrl = pulsar3.getSafeWebServiceAddress(); + brokerServiceUrl = pulsar3.getBrokerServiceUrl(); + } + } + + admin1.namespaces().unloadNamespaceBundle(namespace, bundleRange, brokerUrl); + + String topicLookupAfterUnload = admin1.lookups().lookupTopic(topic); + + Assert.assertEquals(brokerServiceUrl, topicLookupAfterUnload); + pulsar3.close(); + } + /** * It verifies that once broker owns max-number of topics: load-manager doesn't allocates new bundles to that broker * unless all the brokers are in same state. @@ -345,7 +404,7 @@ public void testLoadShedding() throws Exception { // Need to update all the bundle data for the shredder to see the spy. primaryLoadManager.handleDataNotification(new Notification(NotificationType.Created, LoadManager.LOADBALANCE_BROKERS_ROOT + "/broker:8080")); - Thread.sleep(100); + sleep(100); localBrokerData.setCpu(new ResourceUsage(80, 100)); primaryLoadManager.doLoadShedding(); diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java index 16d4e155635d8..8e49400d4384b 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java @@ -2090,6 +2090,20 @@ void setBookieAffinityGroup(String namespace, BookieAffinityGroupData bookieAffi */ void unloadNamespaceBundle(String namespace, String bundle) throws PulsarAdminException; + /** + * Unload namespace bundle and assign the bundle to specified broker. + * + * @param namespace + * @param bundle + * range of bundle to unload + * @param brokerUrl + * Target broker url to which the bundle should be assigned to + * @throws PulsarAdminException + * Unexpected error + */ + void unloadNamespaceBundle(String namespace, String bundle, String brokerUrl) throws PulsarAdminException; + + /** * Unload namespace bundle asynchronously. * @@ -2101,6 +2115,19 @@ void setBookieAffinityGroup(String namespace, BookieAffinityGroupData bookieAffi */ CompletableFuture unloadNamespaceBundleAsync(String namespace, String bundle); + /** + * Unload namespace bundle asynchronously. + * + * @param namespace + * @param bundle + * range of bundle to unload + * @param brokerUrl + * Target broker url to which the bundle should be assigned to + * + * @return a future that can be used to track when the bundle is unloaded + */ + CompletableFuture unloadNamespaceBundleAsync(String namespace, String bundle, String brokerUrl); + /** * Split namespace bundle. * diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java index 3d193c827732e..256170aee0b8b 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java @@ -814,6 +814,11 @@ public void unloadNamespaceBundle(String namespace, String bundle) throws Pulsar sync(() -> unloadNamespaceBundleAsync(namespace, bundle)); } + @Override + public void unloadNamespaceBundle(String namespace, String bundle, String brokerUrl) throws PulsarAdminException { + sync(() -> unloadNamespaceBundleAsync(namespace, bundle, brokerUrl)); + } + @Override public CompletableFuture unloadNamespaceBundleAsync(String namespace, String bundle) { NamespaceName ns = NamespaceName.get(namespace); @@ -821,6 +826,13 @@ public CompletableFuture unloadNamespaceBundleAsync(String namespace, Stri return asyncPutRequest(path, Entity.entity("", MediaType.APPLICATION_JSON)); } + @Override + public CompletableFuture unloadNamespaceBundleAsync(String namespace, String bundle, String brokerUrl) { + NamespaceName ns = NamespaceName.get(namespace); + WebTarget path = namespacePath(ns, bundle, "unload").queryParam("brokerUrl", brokerUrl); + return asyncPutRequest(path, Entity.entity("", MediaType.APPLICATION_JSON)); + } + @Override public void splitNamespaceBundle(String namespace, String bundle, boolean unloadSplitBundles, String splitAlgorithmName) throws PulsarAdminException { diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index aba0a6cda547c..95a0edfa69378 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -880,13 +880,22 @@ private class Unload extends CliCommand { @Parameter(names = { "--bundle", "-b" }, description = "{start-boundary}_{end-boundary}") private String bundle; + @Parameter(names = { "--brokerUrl", "-u" }, + description = "Target brokerWebServiceAddress to which the bundle has to be allocated to") + private String broker; + @Override void run() throws PulsarAdminException { String namespace = validateNamespace(params); if (bundle == null) { getAdmin().namespaces().unload(namespace); } else { - getAdmin().namespaces().unloadNamespaceBundle(namespace, bundle); + if (broker == null) { + getAdmin().namespaces().unloadNamespaceBundle(namespace, bundle); + } else { + getAdmin().namespaces().unloadNamespaceBundle(namespace, bundle, broker); + } + } } } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/CLITest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/CLITest.java index 1c65d897fe930..c1aa407412cba 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/CLITest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/CLITest.java @@ -250,7 +250,7 @@ public void testCreateUpdateSubscriptionWithPropertiesCommand() throws Exception @Test public void testTopicTerminationOnTopicsWithoutConnectedConsumers() throws Exception { String topicName = "persistent://public/default/test-topic-termination"; - BrokerContainer container = pulsarCluster.getAnyBroker(); + BrokerContarun_integration_groupiner container = pulsarCluster.getAnyBroker(); container.execCmd( PulsarCluster.ADMIN_SCRIPT, "topics", From 6b3d6061ca1381ad535e584abd66453d4d9fe521 Mon Sep 17 00:00:00 2001 From: Vineeth Date: Tue, 25 Oct 2022 15:23:36 -0700 Subject: [PATCH 17/22] =?UTF-8?q?[improve][pulsar-broker]=20Add=20option?= =?UTF-8?q?=20to=C2=A0=20unloadNamespaceBundle=20with=20bundle=20Affinity?= =?UTF-8?q?=20broker=20url?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pulsar/broker/loadbalance/ModularLoadManager.java | 4 +--- .../apache/pulsar/broker/loadbalance/NoopLoadManager.java | 3 ++- .../broker/loadbalance/impl/ModularLoadManagerImpl.java | 7 +------ .../broker/loadbalance/impl/SimpleLoadManagerImpl.java | 2 +- 4 files changed, 5 insertions(+), 11 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManager.java index 461ed2dd3ce16..ce89a3367daac 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManager.java @@ -140,9 +140,7 @@ default void writeBrokerDataOnZooKeeper(boolean force) { * @return bundle data */ BundleData getBundleDataOrDefault(String bundle); - - String getNamespaceBundleAffinity(String bundle); - + void setNamespaceBundleAffinity(String bundle, String broker); String removeNamespaceBundleAffinity(String bundle); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java index 0b852772fa34c..4d2d86b95b2a0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java @@ -20,6 +20,7 @@ import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -44,7 +45,7 @@ public class NoopLoadManager implements LoadManager { private String lookupServiceAddress; private ResourceUnit localResourceUnit; private LockManager lockManager; - private ConcurrentHashMap bundleBrokerAffinityMap; + private Map bundleBrokerAffinityMap; @Override public void initialize(PulsarService pulsar) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java index 68b0f4113aadc..97c2f0007eabd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java @@ -1215,12 +1215,7 @@ public List getLoadBalancingMetrics() { return metricsCollection; } - - @Override - public String getNamespaceBundleAffinity(String bundle) { - return this.bundleBrokerAffinityMap.get(bundle); - } - + @Override public void setNamespaceBundleAffinity(String bundle, String broker) { broker = broker.replaceFirst("http[s]?://", ""); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java index 350e2d50ea011..fadfcd918fd07 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java @@ -187,7 +187,7 @@ public class SimpleLoadManagerImpl implements LoadManager, Consumer updateRankingHandle; - private ConcurrentHashMap bundleBrokerAffinityMap; + private Map bundleBrokerAffinityMap; // Perform initializations which may be done without a PulsarService. public SimpleLoadManagerImpl() { From d51fc1f49a799fb1763e3f36dcc03f5924d24de9 Mon Sep 17 00:00:00 2001 From: Vineeth Date: Tue, 25 Oct 2022 15:28:52 -0700 Subject: [PATCH 18/22] =?UTF-8?q?[improve][pulsar-broker]=20Add=20option?= =?UTF-8?q?=20to=C2=A0=20unloadNamespaceBundle=20with=20bundle=20Affinity?= =?UTF-8?q?=20broker=20url?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/org/apache/pulsar/tests/integration/cli/CLITest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/CLITest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/CLITest.java index c1aa407412cba..1c65d897fe930 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/CLITest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/CLITest.java @@ -250,7 +250,7 @@ public void testCreateUpdateSubscriptionWithPropertiesCommand() throws Exception @Test public void testTopicTerminationOnTopicsWithoutConnectedConsumers() throws Exception { String topicName = "persistent://public/default/test-topic-termination"; - BrokerContarun_integration_groupiner container = pulsarCluster.getAnyBroker(); + BrokerContainer container = pulsarCluster.getAnyBroker(); container.execCmd( PulsarCluster.ADMIN_SCRIPT, "topics", From 31ce1e5e1e1f7505f13913de7a17f0eed974053f Mon Sep 17 00:00:00 2001 From: Vineeth Date: Tue, 25 Oct 2022 15:30:35 -0700 Subject: [PATCH 19/22] =?UTF-8?q?[improve][pulsar-broker]=20Add=20option?= =?UTF-8?q?=20to=C2=A0=20unloadNamespaceBundle=20with=20bundle=20Affinity?= =?UTF-8?q?=20broker=20url?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/org/apache/pulsar/client/admin/Namespaces.java | 8 ++++---- .../pulsar/client/admin/internal/NamespacesImpl.java | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java index 8e49400d4384b..dd14c200758a2 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java @@ -2096,12 +2096,12 @@ void setBookieAffinityGroup(String namespace, BookieAffinityGroupData bookieAffi * @param namespace * @param bundle * range of bundle to unload - * @param brokerUrl + * @param destinationBroker * Target broker url to which the bundle should be assigned to * @throws PulsarAdminException * Unexpected error */ - void unloadNamespaceBundle(String namespace, String bundle, String brokerUrl) throws PulsarAdminException; + void unloadNamespaceBundle(String namespace, String bundle, String destinationBroker) throws PulsarAdminException; /** @@ -2121,12 +2121,12 @@ void setBookieAffinityGroup(String namespace, BookieAffinityGroupData bookieAffi * @param namespace * @param bundle * range of bundle to unload - * @param brokerUrl + * @param destinationBroker * Target broker url to which the bundle should be assigned to * * @return a future that can be used to track when the bundle is unloaded */ - CompletableFuture unloadNamespaceBundleAsync(String namespace, String bundle, String brokerUrl); + CompletableFuture unloadNamespaceBundleAsync(String namespace, String bundle, String destinationBroker); /** * Split namespace bundle. diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java index 256170aee0b8b..0a59e6558b715 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java @@ -815,8 +815,8 @@ public void unloadNamespaceBundle(String namespace, String bundle) throws Pulsar } @Override - public void unloadNamespaceBundle(String namespace, String bundle, String brokerUrl) throws PulsarAdminException { - sync(() -> unloadNamespaceBundleAsync(namespace, bundle, brokerUrl)); + public void unloadNamespaceBundle(String namespace, String bundle, String destinationBroker) throws PulsarAdminException { + sync(() -> unloadNamespaceBundleAsync(namespace, bundle, destinationBroker)); } @Override @@ -827,9 +827,9 @@ public CompletableFuture unloadNamespaceBundleAsync(String namespace, Stri } @Override - public CompletableFuture unloadNamespaceBundleAsync(String namespace, String bundle, String brokerUrl) { + public CompletableFuture unloadNamespaceBundleAsync(String namespace, String bundle, String destinationBroker) { NamespaceName ns = NamespaceName.get(namespace); - WebTarget path = namespacePath(ns, bundle, "unload").queryParam("brokerUrl", brokerUrl); + WebTarget path = namespacePath(ns, bundle, "unload").queryParam("destinationBroker", destinationBroker); return asyncPutRequest(path, Entity.entity("", MediaType.APPLICATION_JSON)); } From eb3c6e1c7ed5e48c0cd5fabe92641647d274619c Mon Sep 17 00:00:00 2001 From: Vineeth Date: Tue, 25 Oct 2022 15:44:06 -0700 Subject: [PATCH 20/22] =?UTF-8?q?[improve][pulsar-broker]=20Add=20option?= =?UTF-8?q?=20to=C2=A0=20unloadNamespaceBundle=20with=20bundle=20Affinity?= =?UTF-8?q?=20broker=20url?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../org/apache/pulsar/broker/admin/impl/NamespacesBase.java | 6 +++--- .../org/apache/pulsar/broker/web/PulsarWebResource.java | 2 +- .../java/org/apache/pulsar/admin/cli/CmdNamespaces.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 47790eafb39c8..3f39d9169bde9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -853,8 +853,8 @@ protected BookieAffinityGroupData internalGetBookieAffinityGroup() { } } - public void setNamespaceBundleAffinity (String bundleRange, String brokerUrl) { - if (brokerUrl != null) { + public void setNamespaceBundleAffinity (String bundleRange, String destinationBroker) { + if (destinationBroker != null) { if (!this.isLeaderBroker()) { LeaderBroker leaderBroker = pulsar().getLeaderElectionService().getCurrentLeader().get(); String leaderBrokerUrl = leaderBroker.getServiceUrl(); @@ -872,7 +872,7 @@ public void setNamespaceBundleAffinity (String bundleRange, String brokerUrl) { throw new RestException(exception); } } - pulsar().getLoadManager().get().setNamespaceBundleAffinity(bundleRange, brokerUrl); + pulsar().getLoadManager().get().setNamespaceBundleAffinity(bundleRange, destinationBroker); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java index 8657c28ce75f5..068713b4a22e6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java @@ -710,7 +710,7 @@ public CompletableFuture validateBundleOwnershipAsync(NamespaceBundle bund // Replace the host and port of the current request and redirect URI redirect = UriBuilder.fromUri(uri.getRequestUri()).host(webUrl.get().getHost()) .port(webUrl.get().getPort()).replaceQueryParam("authoritative", - newAuthoritative).replaceQueryParam("brokerUrl", null).build(); + newAuthoritative).replaceQueryParam("destinationBroker", null).build(); log.debug("{} is not a service unit owned", bundle); // Redirect diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index 95a0edfa69378..0ae1668a106ce 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -880,7 +880,7 @@ private class Unload extends CliCommand { @Parameter(names = { "--bundle", "-b" }, description = "{start-boundary}_{end-boundary}") private String bundle; - @Parameter(names = { "--brokerUrl", "-u" }, + @Parameter(names = { "--destinationBroker", "-d" }, description = "Target brokerWebServiceAddress to which the bundle has to be allocated to") private String broker; From 99cbd4a9c385337edd5ceb0884f1b40910290755 Mon Sep 17 00:00:00 2001 From: Vineeth Date: Thu, 27 Oct 2022 10:40:24 -0700 Subject: [PATCH 21/22] =?UTF-8?q?[improve][pulsar-broker]=20Add=20option?= =?UTF-8?q?=20to=C2=A0=20unloadNamespaceBundle=20with=20bundle=20Affinity?= =?UTF-8?q?=20broker=20url?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../broker/admin/impl/NamespacesBase.java | 2 +- .../broker/loadbalance/LoadManager.java | 2 +- .../loadbalance/ModularLoadManager.java | 2 +- .../impl/ModularLoadManagerImpl.java | 2 +- .../impl/SimpleLoadManagerImpl.java | 2 +- .../pulsar/broker/web/PulsarWebResource.java | 4 +- .../ModularLoadManagerImplTest.java | 50 ++++++++++++------- 7 files changed, 40 insertions(+), 24 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 3f39d9169bde9..0992b8c86f080 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -865,7 +865,7 @@ public void setNamespaceBundleAffinity (String bundleRange, String destinationBr false).build(); // Redirect - log.debug("Redirecting the rest call to {}, bundleRange - {}", redirect, bundleRange); + log.debug("Redirecting the rest call to leader - {}, bundleRange - {}", redirect, bundleRange); throw new WebApplicationException(Response.temporaryRedirect(redirect).build()); } catch (MalformedURLException exception) { log.error("The leader broker url is malformed - {}", leaderBrokerUrl); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadManager.java index 6d2f5c52454ef..45a5cffc3f027 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadManager.java @@ -118,7 +118,7 @@ default void writeLoadReportOnZookeeper(boolean force) throws Exception { Set getAvailableBrokers() throws Exception; CompletableFuture> getAvailableBrokersAsync(); - + void setNamespaceBundleAffinity(String bundle, String broker); String removeNamespaceBundleAffinity(String bundle); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManager.java index ce89a3367daac..55754c0a0a672 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManager.java @@ -140,7 +140,7 @@ default void writeBrokerDataOnZooKeeper(boolean force) { * @return bundle data */ BundleData getBundleDataOrDefault(String bundle); - + void setNamespaceBundleAffinity(String bundle, String broker); String removeNamespaceBundleAffinity(String bundle); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java index 97c2f0007eabd..8d41e9c9dedf4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java @@ -1215,7 +1215,7 @@ public List getLoadBalancingMetrics() { return metricsCollection; } - + @Override public void setNamespaceBundleAffinity(String bundle, String broker) { broker = broker.replaceFirst("http[s]?://", ""); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java index fadfcd918fd07..b457c3653fb0e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java @@ -1446,7 +1446,7 @@ public void doNamespaceBundleSplit() throws Exception { this.setLoadReportForceUpdateFlag(); } } - + @Override public void setNamespaceBundleAffinity(String bundle, String broker) { broker = broker.replaceFirst("http[s]?://", ""); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java index 068713b4a22e6..53b6d049c83a6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java @@ -710,8 +710,8 @@ public CompletableFuture validateBundleOwnershipAsync(NamespaceBundle bund // Replace the host and port of the current request and redirect URI redirect = UriBuilder.fromUri(uri.getRequestUri()).host(webUrl.get().getHost()) .port(webUrl.get().getPort()).replaceQueryParam("authoritative", - newAuthoritative).replaceQueryParam("destinationBroker", null).build(); - + newAuthoritative).replaceQueryParam("destinationBroker", + null).build(); log.debug("{} is not a service unit owned", bundle); // Redirect log.debug("Redirecting the rest call to {}", redirect); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerImplTest.java index 5a1f2b2167f5d..e283d7cd87064 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerImplTest.java @@ -105,6 +105,8 @@ public class ModularLoadManagerImplTest { private PulsarService pulsar2; private PulsarAdmin admin2; + private PulsarService pulsar3; + private String primaryHost; private String secondaryHost; @@ -184,6 +186,20 @@ void setup() throws Exception { pulsar2 = new PulsarService(config2); pulsar2.start(); + ServiceConfiguration config = new ServiceConfiguration(); + config.setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); + config.setLoadBalancerLoadSheddingStrategy("org.apache.pulsar.broker.loadbalance.impl.OverloadShedder"); + config.setClusterName("use"); + config.setWebServicePort(Optional.of(0)); + config.setMetadataStoreUrl("zk:127.0.0.1:" + bkEnsemble.getZookeeperPort()); + config.setAdvertisedAddress("localhost"); + config.setBrokerShutdownTimeoutMs(0L); + config.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); + config.setBrokerServicePort(Optional.of(0)); + config.setBrokerServicePortTls(Optional.of(0)); + config.setWebServicePortTls(Optional.of(0)); + pulsar3 = new PulsarService(config); + secondaryHost = String.format("%s:%d", "localhost", pulsar2.getListenPortHTTP().get()); url2 = new URL(pulsar2.getWebServiceAddress()); admin2 = PulsarAdmin.builder().serviceHttpUrl(url2.toString()).build(); @@ -204,6 +220,10 @@ void shutdown() throws Exception { pulsar2.close(); pulsar1.close(); + + if (pulsar3.isRunning()) { + pulsar3.close(); + } bkEnsemble.stop(); } @@ -292,19 +312,6 @@ public void testEvenBundleDistribution() throws Exception { @Test public void testBrokerAffinity() throws Exception { // Start broker 3 - ServiceConfiguration config = new ServiceConfiguration(); - config.setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); - config.setLoadBalancerLoadSheddingStrategy("org.apache.pulsar.broker.loadbalance.impl.OverloadShedder"); - config.setClusterName("use"); - config.setWebServicePort(Optional.of(0)); - config.setMetadataStoreUrl("zk:127.0.0.1:" + bkEnsemble.getZookeeperPort()); - config.setAdvertisedAddress("localhost"); - config.setBrokerShutdownTimeoutMs(0L); - config.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); - config.setBrokerServicePort(Optional.of(0)); - config.setBrokerServicePortTls(Optional.of(0)); - config.setWebServicePortTls(Optional.of(0)); - PulsarService pulsar3 = new PulsarService(config); pulsar3.start(); final String tenant = "test"; @@ -321,6 +328,7 @@ public void testBrokerAffinity() throws Exception { String brokerServiceUrl = pulsar1.getBrokerServiceUrl(); String brokerUrl = pulsar1.getSafeWebServiceAddress(); + log.debug("initial broker service url - {}", topicLookup); Random rand=new Random(); if (topicLookup.equals(brokerServiceUrl)) { @@ -334,13 +342,21 @@ public void testBrokerAffinity() throws Exception { brokerServiceUrl = pulsar3.getBrokerServiceUrl(); } } + log.debug("destination broker service url - {}, broker url - {}", brokerServiceUrl, brokerUrl); + String leaderServiceUrl = admin1.brokers().getLeaderBroker().getServiceUrl(); + log.debug("leader serviceUrl - {}, broker1 service url - {}", leaderServiceUrl, pulsar1.getSafeWebServiceAddress()); + //Make a call to broker which is not a leader + if (!leaderServiceUrl.equals(pulsar1.getSafeWebServiceAddress())) { + admin1.namespaces().unloadNamespaceBundle(namespace, bundleRange, brokerUrl); + } + else { + admin2.namespaces().unloadNamespaceBundle(namespace, bundleRange, brokerUrl); + } - admin1.namespaces().unloadNamespaceBundle(namespace, bundleRange, brokerUrl); - + sleep(2000); String topicLookupAfterUnload = admin1.lookups().lookupTopic(topic); - + log.debug("final broker service url - {}", topicLookupAfterUnload); Assert.assertEquals(brokerServiceUrl, topicLookupAfterUnload); - pulsar3.close(); } /** From 396be61b278d2658714da34704b5341789159dbc Mon Sep 17 00:00:00 2001 From: Vineeth Date: Thu, 27 Oct 2022 12:55:31 -0700 Subject: [PATCH 22/22] =?UTF-8?q?[improve][pulsar-broker]=20Add=20option?= =?UTF-8?q?=20to=C2=A0=20unloadNamespaceBundle=20with=20bundle=20Affinity?= =?UTF-8?q?=20broker=20url?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../apache/pulsar/client/admin/internal/NamespacesImpl.java | 6 ++++-- .../java/org/apache/pulsar/admin/cli/CmdNamespaces.java | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java index 0a59e6558b715..a28850ca81d64 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java @@ -815,7 +815,8 @@ public void unloadNamespaceBundle(String namespace, String bundle) throws Pulsar } @Override - public void unloadNamespaceBundle(String namespace, String bundle, String destinationBroker) throws PulsarAdminException { + public void unloadNamespaceBundle(String namespace, + String bundle, String destinationBroker) throws PulsarAdminException { sync(() -> unloadNamespaceBundleAsync(namespace, bundle, destinationBroker)); } @@ -827,7 +828,8 @@ public CompletableFuture unloadNamespaceBundleAsync(String namespace, Stri } @Override - public CompletableFuture unloadNamespaceBundleAsync(String namespace, String bundle, String destinationBroker) { + public CompletableFuture unloadNamespaceBundleAsync(String namespace, + String bundle, String destinationBroker) { NamespaceName ns = NamespaceName.get(namespace); WebTarget path = namespacePath(ns, bundle, "unload").queryParam("destinationBroker", destinationBroker); return asyncPutRequest(path, Entity.entity("", MediaType.APPLICATION_JSON)); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index 0ae1668a106ce..2e56a29bf9691 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -882,7 +882,7 @@ private class Unload extends CliCommand { @Parameter(names = { "--destinationBroker", "-d" }, description = "Target brokerWebServiceAddress to which the bundle has to be allocated to") - private String broker; + private String destinationBroker; @Override void run() throws PulsarAdminException { @@ -890,10 +890,10 @@ void run() throws PulsarAdminException { if (bundle == null) { getAdmin().namespaces().unload(namespace); } else { - if (broker == null) { + if (destinationBroker == null) { getAdmin().namespaces().unloadNamespaceBundle(namespace, bundle); } else { - getAdmin().namespaces().unloadNamespaceBundle(namespace, bundle, broker); + getAdmin().namespaces().unloadNamespaceBundle(namespace, bundle, destinationBroker); } }