diff --git a/iceberg-catalog-migrator/cli/BUNDLE-LICENSE b/iceberg-catalog-migrator/cli/BUNDLE-LICENSE index dcaa665f..5a03d8e9 100644 --- a/iceberg-catalog-migrator/cli/BUNDLE-LICENSE +++ b/iceberg-catalog-migrator/cli/BUNDLE-LICENSE @@ -1239,6 +1239,13 @@ License: BSD 3-Clause -------------------------------------------------------------------------------- +This artifact bundles iq80 LevelDB (Java port of LevelDB). + +Project URL: https://github.com/dain/leveldb +License: Apache License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt + +-------------------------------------------------------------------------------- + This artifact bundles Project Nessie. Project URL: https://github.com/projectnessie/nessie diff --git a/iceberg-catalog-migrator/cli/BUNDLE-NOTICE b/iceberg-catalog-migrator/cli/BUNDLE-NOTICE index f45c7511..b8be7ed5 100644 --- a/iceberg-catalog-migrator/cli/BUNDLE-NOTICE +++ b/iceberg-catalog-migrator/cli/BUNDLE-NOTICE @@ -9,6 +9,32 @@ to the ASF by Dremio Corporation. (https://www.dremio.com/) copyright 2022. ------------------------------------------------------------------------- +This artifact bundles Apache Iceberg with the following in its NOTICE: +| Apache Iceberg +| Copyright 2017-2026 The Apache Software Foundation +| +| This product includes software developed at +| The Apache Software Foundation (http://www.apache.org/). +| +| This project includes code from Kite, developed at Cloudera, Inc. with +| the following copyright notice: +| +| Copyright 2013 Cloudera Inc. +| +| Licensed under the Apache License, Version 2.0 (the "License"); +| you may not use this file except in compliance with the License. +| You may obtain a copy of the License at +| +| http://www.apache.org/licenses/LICENSE-2.0 +| +| Unless required by applicable law or agreed to in writing, software +| distributed under the License is distributed on an "AS IS" BASIS, +| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +| See the License for the specific language governing permissions and +| limitations under the License. + +------------------------------------------------------------------------- + This artifact bundles reload4j with the following in its NOTICE: | Apache log4j | Copyright 2007 The Apache Software Foundation @@ -165,3 +191,59 @@ This artifact bundles Netty with the following in its NOTICE: | * HOMEPAGE: | * https://github.com/JCTools/JCTools +------------------------------------------------------------------------- + +This artifact bundles Snappy Java with the following in its NOTICE: +| This product includes software developed by Google Snappy. +| (http://code.google.com/p/snappy) +| +| This product includes software developed by Apache +| PureJavaCrc32C from apache-hadoop-common. +| (http://hadoop.apache.org) + +------------------------------------------------------------------------- + +This artifact bundles HawtJNI with the following in its NOTICE: +| This product includes software developed by FuseSource Corp. +| http://fusesource.com +| +| This product includes software developed at +| Progress Software Corporation and/or its subsidiaries or affiliates. +| +| This product includes software developed by IBM Corporation and others. + +------------------------------------------------------------------------- + +This artifact bundles BoneCP with the following in its NOTICE: +| BoneCP +| Copyright 2010 Wallace Wadge + +------------------------------------------------------------------------- + +This artifact bundles Eigenbase Properties with the following in its NOTICE: +| eigenbase-properties +| Copyright (C) 2012-2020, Julian Hyde +| This product includes software from the Eigenbase project, licensed from +| DynamoBI Corporation. +| Copyright (C) 2005 Dynamo BI Corporation + +------------------------------------------------------------------------- + +This artifact bundles Groovy with the following in its NOTICE: +| Apache Groovy +| Copyright 2003-2015 The Apache Software Foundation +| +| This product includes software developed at +| The Apache Software Foundation (http://www.apache.org/). +| +| It includes the following other software: +| +| Antlr 2 (http://www.antlr2.org/) +| ASM (http://asm.ow2.org/) +| GPars (http://www.gpars.org/) +| Hamcrest (https://github.com/hamcrest/JavaHamcrest) +| JCommander (http://jcommander.org/) +| Openbeans (https://code.google.com/p/openbeans/) +| QDox (http://qdox.codehaus.org/) +| TestNG (http://testng.org/) +| XStream (http://xstream.codehaus.org/) diff --git a/iceberg-catalog-migrator/docs/examples.md b/iceberg-catalog-migrator/docs/examples.md index b54b9d48..91230e34 100644 --- a/iceberg-catalog-migrator/docs/examples.md +++ b/iceberg-catalog-migrator/docs/examples.md @@ -32,7 +32,7 @@ Below are some examples of registering tables from one catalog to another. ### Registering All Tables from Hadoop Catalog to Polaris Catalog ```shell -java -jar iceberg-catalog-migrator-cli-0.0.1.jar register \ +java -jar iceberg-catalog-migrator-cli-1.6.0.jar register \ --source-catalog-type HADOOP \ --source-catalog-properties warehouse=/tmp/warehouse,type=hadoop \ --target-catalog-type REST \ @@ -46,7 +46,7 @@ java -jar iceberg-catalog-migrator-cli-0.0.1.jar register \ In this example, only tables t1 and t2 in namespace foo will be migrated. ```shell -java -jar iceberg-catalog-migrator-cli-0.0.1.jar migrate \ +java -jar iceberg-catalog-migrator-cli-1.6.0.jar migrate \ --source-catalog-type HADOOP \ --source-catalog-properties warehouse=/tmp/warehouse,type=hadoop \ --target-catalog-type REST \ @@ -56,7 +56,7 @@ java -jar iceberg-catalog-migrator-cli-0.0.1.jar migrate \ ### Migrate from GLUE Catalog to Polaris Catalog ```shell -java -jar iceberg-catalog-migrator-cli-0.0.1.jar migrate \ +java -jar iceberg-catalog-migrator-cli-1.6.0.jar migrate \ --source-catalog-type GLUE \ --source-catalog-properties warehouse=s3a://some-bucket/wh/,io-impl=org.apache.iceberg.aws.s3.S3FileIO \ --target-catalog-type REST \ @@ -65,7 +65,7 @@ java -jar iceberg-catalog-migrator-cli-0.0.1.jar migrate \ ### Migrate from HIVE Catalog to Polaris Catalog ```shell -java -jar iceberg-catalog-migrator-cli-0.0.1.jar migrate \ +java -jar iceberg-catalog-migrator-cli-1.6.0.jar migrate \ --source-catalog-type HIVE \ --source-catalog-properties warehouse=s3a://some-bucket/wh/,io-impl=org.apache.iceberg.aws.s3.S3FileIO,uri=thrift://localhost:9083 \ --target-catalog-type REST \ @@ -78,7 +78,7 @@ source catalog directory in `storage_configuration_info`. ### Migrate from DYNAMODB Catalog to Polaris Catalog ```shell -java -jar iceberg-catalog-migrator-cli-0.0.1.jar migrate \ +java -jar iceberg-catalog-migrator-cli-1.6.0.jar migrate \ --source-catalog-type DYNAMODB \ --source-catalog-properties warehouse=s3a://some-bucket/wh/,io-impl=org.apache.iceberg.aws.s3.S3FileIO \ --target-catalog-type REST \ @@ -87,7 +87,7 @@ java -jar iceberg-catalog-migrator-cli-0.0.1.jar migrate \ ### Migrate from JDBC Catalog to Polaris Catalog ```shell -java -jar iceberg-catalog-migrator-cli-0.0.1.jar migrate \ +java -jar iceberg-catalog-migrator-cli-1.6.0.jar migrate \ --source-catalog-type JDBC \ --source-catalog-properties warehouse=/tmp/warehouseJdbc,jdbc.user=root,jdbc.password=pass,uri=jdbc:mysql://localhost:3306/db1,name=catalogName \ --target-catalog-type REST \ @@ -96,7 +96,7 @@ java -jar iceberg-catalog-migrator-cli-0.0.1.jar migrate \ ### Migrate Only Tables Starting with "foo" ```shell -java -jar iceberg-catalog-migrator-cli-0.0.1.jar migrate \ +java -jar iceberg-catalog-migrator-cli-1.6.0.jar migrate \ --source-catalog-type HIVE \ --source-catalog-properties warehouse=s3a://some-bucket/wh/,io-impl=org.apache.iceberg.aws.s3.S3FileIO,uri=thrift://localhost:9083 \ --target-catalog-type NESSIE \ @@ -109,7 +109,7 @@ java -jar iceberg-catalog-migrator-cli-0.0.1.jar migrate \ The file idx.txt contains each table identifier to migrate delimited by a newline. ```shell -java -jar iceberg-catalog-migrator-cli-0.0.1.jar migrate \ +java -jar iceberg-catalog-migrator-cli-1.6.0.jar migrate \ --source-catalog-type HIVE \ --source-catalog-properties warehouse=/tmp/warehouse,type=hadoop \ --target-catalog-type NESSIE \ diff --git a/iceberg-catalog-migrator/docs/getting-started.md b/iceberg-catalog-migrator/docs/getting-started.md index ffbcd419..e5f65772 100644 --- a/iceberg-catalog-migrator/docs/getting-started.md +++ b/iceberg-catalog-migrator/docs/getting-started.md @@ -50,7 +50,7 @@ These commands: 3. Build the tool 4. Create a JAR file in `iceberg-catalog-migrator/cli/build/libs/` directory -The JAR file will be created with name `iceberg-catalog-migrator-cli-.jar` where `` is the version of the tool found in the `iceberg-catalog-migrator/version.txt` file. For the examples below, we will assume the version is `0.0.1-SNAPSHOT`, so the JAR file name will be `iceberg-catalog-migrator-cli-0.0.1-SNAPSHOT.jar`. +The JAR file will be created with name `iceberg-catalog-migrator-cli-.jar` where `` is the version of the tool found in the `iceberg-catalog-migrator/version.txt` file. For the examples below, we will assume the version is `1.6.0`, so the JAR file name will be `iceberg-catalog-migrator-cli-1.6.0.jar`. ### Step 2: Set the Object Storage Environment Variables The tool will need access to the underlying object storage via environmental variables. For this example, we will use AWS S3 with an access key and id: @@ -84,12 +84,12 @@ export TOKEN_TARGET=xxxxxxx ### Step 4: Validate the Migration Execute the following command to understand how to migrate the tables: ```shell -java -jar ./cli/build/libs/iceberg-catalog-migrator-cli-0.0.1-SNAPSHOT.jar register -h +java -jar ./cli/build/libs/iceberg-catalog-migrator-cli-1.6.0.jar register -h ``` In the example, execute the following command to perform a dry run migration. This will not migrate the tables but will provide information on the operation: ```shell -java -jar ./cli/build/libs/iceberg-catalog-migrator-cli-0.0.1-SNAPSHOT.jar register \ +java -jar ./cli/build/libs/iceberg-catalog-migrator-cli-1.6.0.jar register \ --source-catalog-type REST \ --source-catalog-properties uri=http://sourcecatalog:8181/api/catalog,warehouse=test,token=$TOKEN_SOURCE \ --target-catalog-type REST \ @@ -103,7 +103,7 @@ After validating all inputs, the console will display a list of table identifier In the example, execute the following command to perform a migration: ```shell -java -jar ./cli/build/libs/iceberg-catalog-migrator-cli-0.0.1-SNAPSHOT.jar migrate \ +java -jar ./cli/build/libs/iceberg-catalog-migrator-cli-1.6.0.jar migrate \ --source-catalog-type REST \ --source-catalog-properties uri=http://sourcecatalog:8181/api/catalog,warehouse=test,token=$TOKEN_SOURCE \ --target-catalog-type REST \ diff --git a/iceberg-catalog-migrator/gradle/libs.versions.toml b/iceberg-catalog-migrator/gradle/libs.versions.toml index 3e90124a..99181b20 100644 --- a/iceberg-catalog-migrator/gradle/libs.versions.toml +++ b/iceberg-catalog-migrator/gradle/libs.versions.toml @@ -26,7 +26,7 @@ googleJavaFormat = "1.25.2" guava = "33.4.0-jre" hadoop = "3.4.1" # this is in mapping with iceberg repo. hive = { strictly = "2.3.10"} # this is in mapping with iceberg repo. -iceberg = "1.10.0" +iceberg = "1.11.0" immutables = "2.10.1" jacoco = "0.8.12" jandex = "3.2.3" diff --git a/iceberg-catalog-migrator/version.txt b/iceberg-catalog-migrator/version.txt index 4adba7ed..dc1e644a 100644 --- a/iceberg-catalog-migrator/version.txt +++ b/iceberg-catalog-migrator/version.txt @@ -1 +1 @@ -0.0.1-SNAPSHOT \ No newline at end of file +1.6.0