From dfa6afb32f6a6e97a1d648f2a3ddff2564b474c0 Mon Sep 17 00:00:00 2001 From: Hemit Shah Date: Mon, 15 Sep 2025 13:37:20 -0700 Subject: [PATCH 01/10] SNOW-2339275: update version to 1.17.0 --- build.sbt | 2 +- scripts/deploy-fips.sh | 120 ++++++++++++++++++ scripts/deploy.sh | 9 +- scripts/utils.sh | 2 +- .../com/snowflake/snowpark/UtilsSuite.scala | 2 +- 5 files changed, 127 insertions(+), 8 deletions(-) create mode 100755 scripts/deploy-fips.sh diff --git a/build.sbt b/build.sbt index 173a074c..e0e5f7c3 100644 --- a/build.sbt +++ b/build.sbt @@ -11,7 +11,7 @@ lazy val isFipsRelease = { } lazy val snowparkName = s"snowpark${if (isFipsRelease) "-fips" else ""}" lazy val jdbcName = s"snowflake-jdbc${if (isFipsRelease) "-fips" else ""}" -lazy val snowparkVersion = "1.17.0-SNAPSHOT" +lazy val snowparkVersion = "1.17.0" lazy val Javadoc = config("genjavadoc") extend Compile diff --git a/scripts/deploy-fips.sh b/scripts/deploy-fips.sh new file mode 100755 index 00000000..85af6bda --- /dev/null +++ b/scripts/deploy-fips.sh @@ -0,0 +1,120 @@ +#!/bin/bash -ex +# +# Push Snowpark Java/Scala FIPS build to the public maven repository. +# This script needs to be executed by snowflake jenkins job. +# + +if [ -z "$GPG_KEY_ID" ]; then + export GPG_KEY_ID="Snowflake Computing" + echo "[WARN] GPG key ID not specified, using default: $GPG_KEY_ID." +fi + +if [ -z "$GPG_KEY_PASSPHRASE" ]; then + echo "[ERROR] GPG passphrase is not specified for $GPG_KEY_ID!" + exit 1 +fi + +if [ -z "$GPG_PRIVATE_KEY" ]; then + echo "[ERROR] GPG private key file is not specified!" + exit 1 +fi + +if [ -z "$sonatype_user" ]; then + echo "[ERROR] Jenkins sonatype user is not specified!" + exit 1 +fi + +if [ -z "$sonatype_password" ]; then + echo "[ERROR] Jenkins sonatype pwd is not specified!" + exit 1 +fi + +if [ -z "$PUBLISH" ]; then + echo "[ERROR] 'PUBLISH' is not specified!" + exit 1 +fi + +if [ -z "$github_version_tag" ]; then + echo "[ERROR] 'github_version_tag' is not specified!" + exit 1 +fi + +mkdir -p ~/.ivy2 + +STR=$'host=central.sonatype.com +user='$sonatype_user' +password='$sonatype_password'' + +echo "$STR" > ~/.ivy2/.credentials + +# import private key first +echo "[INFO] Importing PGP key." +if [ ! -z "$GPG_PRIVATE_KEY" ] && [ -f "$GPG_PRIVATE_KEY" ]; then + # First check if already imported private key + if ! gpg --list-secret-key | grep "$GPG_KEY_ID"; then + gpg --allow-secret-key-import --import "$GPG_PRIVATE_KEY" + fi +fi + +which sbt +if [ $? -ne 0 ] +then + pushd .. + echo "[INFO] sbt is not installed, downloading latest sbt for test and build." + curl -L -o sbt-1.11.4.zip https://github.com/sbt/sbt/releases/download/v1.11.4/sbt-1.11.4.zip + unzip sbt-1.11.4.zip + PATH=$PWD/sbt/bin:$PATH + popd +else + echo "[INFO] Using system installed sbt." +fi +which sbt +sbt version + +echo "[INFO] Checking out snowpark-java-scala @ tag: $github_version_tag." +git checkout $github_version_tag + +export SNOWPARK_FIPS="" + +if [ "$PUBLISH" = true ]; then + if [ -v SNOWPARK_FIPS ]; then + echo "[INFO] Packaging snowpark-fips @ tag: $github_version_tag." + else + echo "[INFO] Packaging snowpark @ tag: $github_version_tag." + fi + sbt +publishSigned + echo "[INFO] Staged packaged artifacts locally with PGP signing." + sbt sonaUpload + echo "[SUCCESS] Uploaded artifacts to central portal." + echo "[ACTION-REQUIRED] Please log in to Central Portal to publish artifacts: https://central.sonatype.com/" + # TODO: alternatively automate publishing fully +# sbt sonaRelease +# echo "[SUCCESS] Released Snowpark Java-Scala v$github_version_tag to Maven." +else + #release to s3 + echo "[INFO] Staging signed artifacts to local ivy2 repository." + rm -rf ~/.ivy2/local/ + sbt +publishLocalSigned + + # SBT will build FIPS version of Snowpark automatically if the environment variable exists. + if [ -v SNOWPARK_FIPS ]; then + S3_JENKINS_URL="s3://sfc-eng-jenkins/repository/snowparkclient-fips" + S3_DATA_URL="s3://sfc-eng-data/client/snowparkclient-fips/releases" + echo "[INFO] Uploading snowpark-fips artifacts to:" + else + S3_JENKINS_URL="s3://sfc-eng-jenkins/repository/snowparkclient" + S3_DATA_URL="s3://sfc-eng-data/client/snowparkclient/releases" + echo "[INFO] Uploading snowpark artifacts to:" + fi + echo "[INFO] - $S3_JENKINS_URL/$github_version_tag/" + echo "[INFO] - $S3_DATA_URL/$github_version_tag/" + + # Rename all produced artifacts to include version number (sbt doesn't by default when publishing to local ivy2 repository). + find ~/.ivy2/local -type f -name "*snowpark*" | while read file; do newfile=$(echo "$file" | sed "s/\(2\.1[23]\)\([-\.]\)/\1-${github_version_tag#v}\2/"); mv "$file" "$newfile"; done + + # Copy all files, flattening the nested structure of the ivy2 repository into the expected structure on s3. + find ~/.ivy2/local -type f -name "*snowpark*" -exec aws s3 cp \{\} $S3_JENKINS_URL/$github_version_tag/ \; + find ~/.ivy2/local -type f -name "*snowpark*" -exec aws s3 cp \{\} $S3_DATA_URL/$github_version_tag/ \; + + echo "[SUCCESS] Published Snowpark Java-Scala v$github_version_tag artifacts to S3." +fi diff --git a/scripts/deploy.sh b/scripts/deploy.sh index cdfa5705..0ecd000b 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -1,10 +1,7 @@ #!/bin/bash -ex # -# Push Snowpark Java/Scala to the public maven repository. +# Push Snowpark Java/Scala build to the public maven repository. # This script needs to be executed by snowflake jenkins job. -# If the SNOWPARK_FIPS environment variable exists when running -# the script, the fips build of the snowpark client will be -# published instead of the regular build. # if [ -z "$GPG_KEY_ID" ]; then @@ -75,7 +72,9 @@ which sbt sbt version echo "[INFO] Checking out snowpark-java-scala @ tag: $github_version_tag." -git checkout tags/$github_version_tag +git checkout $github_version_tag + +unset SNOWPARK_FIPS if [ "$PUBLISH" = true ]; then if [ -v SNOWPARK_FIPS ]; then diff --git a/scripts/utils.sh b/scripts/utils.sh index 942fde49..02c42cf3 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -69,7 +69,7 @@ run_test_suites() { '++ 2.12.20 OtherTests:testOnly * -- -l SampleDataTest' \ '++ 2.13.16 OtherTests:testOnly * -- -l SampleDataTest' \ '++ 2.12.20 UDFTests:testOnly * -- -l SampleDataTest' \ - '++ 2.13.16 UDFTests:testOnly * -- -l SampleDataTest -l com.snowflake.snowpark.UDFPackageTest' \ + '++ 2.13.16 UDFTests:testOnly * -- -l SampleDataTest -l com.snowflake.snowpark.UDFPackageTest' \ '++ 2.12.20 UDTFTests:testOnly * -- -l SampleDataTest' \ '++ 2.13.16 UDTFTests:testOnly * -- -l SampleDataTest -l com.snowflake.snowpark.UDFPackageTest' \ +SprocTests:test diff --git a/src/test/scala/com/snowflake/snowpark/UtilsSuite.scala b/src/test/scala/com/snowflake/snowpark/UtilsSuite.scala index 44027c6a..abb091f9 100644 --- a/src/test/scala/com/snowflake/snowpark/UtilsSuite.scala +++ b/src/test/scala/com/snowflake/snowpark/UtilsSuite.scala @@ -492,7 +492,7 @@ class UtilsSuite extends SNTestBase { } test("Utils.version matches sbt build") { - assert(Utils.Version == "1.17.0-SNAPSHOT") + assert(Utils.Version == "1.17.0") } test("Utils.retrySleepTimeInMS") { From 09f2fefbf48eaa5ffe4ebc8fdb11885e9aa4008f Mon Sep 17 00:00:00 2001 From: Fabian Gonzalez Mendez Date: Tue, 16 Sep 2025 11:52:37 -0600 Subject: [PATCH 02/10] SNOW-2246194 Add support for `functions.concat_ws_ignore_nulls` and `functions.array_flatten` (#238) --- .../snowflake/snowpark_java/Functions.java | 134 ++++++++++++++++++ .../com/snowflake/snowpark/functions.scala | 123 ++++++++++++++++ .../snowpark_test/JavaFunctionSuite.java | 116 +++++++++++++++ .../snowpark_test/FunctionSuite.scala | 79 +++++++++++ 4 files changed, 452 insertions(+) diff --git a/src/main/java/com/snowflake/snowpark_java/Functions.java b/src/main/java/com/snowflake/snowpark_java/Functions.java index 32259e2a..041b257c 100644 --- a/src/main/java/com/snowflake/snowpark_java/Functions.java +++ b/src/main/java/com/snowflake/snowpark_java/Functions.java @@ -1482,6 +1482,51 @@ public static Column concat_ws(Column separator, Column... exprs) { JavaUtils.columnArrayToSeq(Column.toScalaColumnArray(exprs)))); } + /** + * Concatenates two or more strings ignoring any null values. + * + *

Unlike {@link #concat_ws}, this function automatically filters out null values before + * concatenation. + * + *

Examples + * + *

{@code
+   * DataFrame df = session.createDataFrame(
+   *   new Row[] {
+   *     Row.create("Hello", "World", null),
+   *     Row.create(null, null, null),
+   *     Row.create("Hello", null, null)
+   *   },
+   *   StructType.create(
+   *     new StructField("A", DataTypes.StringType),
+   *     new StructField("B", DataTypes.StringType),
+   *     new StructField("C", DataTypes.StringType)
+   *   )
+   * );
+   *
+   * df.select(
+   *   concat_ws_ignore_nulls(" | ", col("A"), col("B"), col("C")).as("concat_ws_ignore_nulls")
+   * ).show();
+   * ----------------------------
+   * |"CONCAT_WS_IGNORE_NULLS"  |
+   * ----------------------------
+   * |Hello | World             |
+   * |                          |
+   * |Hello                     |
+   * ----------------------------
+   * }
+ * + * @param separator A string literal used as the separator between concatenated values. + * @param exprs The columns to be concatenated. + * @return A Column containing the concatenated values with null values filtered out. + * @since 1.17.0 + */ + public static Column concat_ws_ignore_nulls(String separator, Column... exprs) { + return new Column( + com.snowflake.snowpark.functions.concat_ws_ignore_nulls( + separator, JavaUtils.columnArrayToSeq(Column.toScalaColumnArray(exprs)))); + } + /** * Returns the input string with the first letter of each word in uppercase and the subsequent * letters in lowercase. @@ -3579,6 +3624,95 @@ public static Column array_contains(Column variant, Column array) { variant.toScalaColumn(), array.toScalaColumn())); } + /** + * Flattens an array of arrays into a single array, removing only one level of nesting. + * + *

Examples + * + *

Example 1: Flattens a two-level nested array into a single array of elements. + * + *

{@code
+   * DataFrame df = session.createDataFrame(
+   *   new Row[] {
+   *     Row.create((Object) new int[][] {{1, 2}, {3, 4}}),
+   *     Row.create((Object) new int[][] {{5, 6, 7}, {8}}),
+   *     Row.create((Object) new int[][] {{}, {9, 10}}),
+   *   },
+   *   StructType.create(new StructField(
+   *     "a",
+   *     DataTypes.createArrayType(DataTypes.createArrayType(DataTypes.IntegerType))
+   *   ))
+   * );
+   *
+   * df.select(array_flatten(col("a"))).show();
+   * --------------------------
+   * |"ARRAY_FLATTEN(""A"")"  |
+   * --------------------------
+   * |[                       |
+   * |  1,                    |
+   * |  2,                    |
+   * |  3,                    |
+   * |  4                     |
+   * |]                       |
+   * |[                       |
+   * |  5,                    |
+   * |  6,                    |
+   * |  7,                    |
+   * |  8                     |
+   * |]                       |
+   * |[                       |
+   * |  9,                    |
+   * |  10                    |
+   * |]                       |
+   * --------------------------
+   * }
+ * + *

Example 2: Flattens only one level of a three-level nested array. + * + *

{@code
+   * DataFrame df = session.createDataFrame(
+   *   new Row[] {
+   *     Row.create((Object) new int[][][] {{{1, 2}, {3}}, {{4, 5}}}),
+   *   },
+   *   StructType.create(new StructField(
+   *     "a",
+   *     DataTypes.createArrayType(DataTypes.createArrayType(DataTypes.createArrayType(DataTypes.IntegerType)))
+   *   ))
+   * );
+   *
+   * df.select(array_flatten(col("a"))).show();
+   * --------------------------
+   * |"ARRAY_FLATTEN(""A"")"  |
+   * --------------------------
+   * |[                       |
+   * |  [                     |
+   * |    1,                  |
+   * |    2                   |
+   * |  ],                    |
+   * |  [                     |
+   * |    3                   |
+   * |  ],                    |
+   * |  [                     |
+   * |    4,                  |
+   * |    5                   |
+   * |  ]                     |
+   * |]                       |
+   * --------------------------
+   * }
+ * + * @param array Column containing the array of arrays to flatten. + * + * + * @return A column containing the flattened array. + * @since 1.17.0 + */ + public static Column array_flatten(Column array) { + return new Column(com.snowflake.snowpark.functions.array_flatten(array.toScalaColumn())); + } + /** * Returns an ARRAY containing all elements from the source ARRAY as well as the new element. * diff --git a/src/main/scala/com/snowflake/snowpark/functions.scala b/src/main/scala/com/snowflake/snowpark/functions.scala index f1994fa7..71922b63 100644 --- a/src/main/scala/com/snowflake/snowpark/functions.scala +++ b/src/main/scala/com/snowflake/snowpark/functions.scala @@ -1307,6 +1307,54 @@ object functions { builtin("concat_ws")(args: _*) } + /** + * Concatenates two or more strings ignoring any null values. + * + * Unlike [[concat_ws]], this function automatically filters out null values before concatenation. + * + * '''Examples''' + * + * {{{ + * val df = session.createDataFrame( + * Seq( + * Row("Hello", "World", null), + * Row(null, null, null), + * Row("Hello", null, null), + * ), + * StructType( + * StructField("A", StringType), + * StructField("B", StringType), + * StructField("C", StringType), + * ) + * ) + * + * df.select( + * concat_ws_ignore_nulls(" | ", col("A"), col("B"), col("C")).as("concat_ws_ignore_nulls") + * ).show() + * ---------------------------- + * |"CONCAT_WS_IGNORE_NULLS" | + * ---------------------------- + * |Hello | World | + * | | + * |Hello | + * ---------------------------- + * }}} + * + * @param separator + * A string literal used as the separator between concatenated values. + * @param exprs + * The columns to be concatenated. + * @return + * A Column containing the concatenated values with null values filtered out. + * @group str_func + * @since 1.17.0 + */ + def concat_ws_ignore_nulls(separator: String, exprs: Column*): Column = { + val stringArrays = exprs.map(_.cast(ArrayType(StringType))) + val nonNullArray = array_compact(array_flatten(array_construct_compact(stringArrays: _*))) + array_to_string(nonNullArray, lit(separator)) + } + /** * Returns the input string with the first letter of each word in uppercase and the subsequent * letters in lowercase. @@ -2954,6 +3002,81 @@ object functions { builtin("array_contains")(variant, array) } + /** + * Flattens an array of arrays into a single array, removing only one level of nesting. + * + * '''Examples''' + * + * Example 1: Flattens a two-level nested array into a single array of elements. + * + * {{{ + * val df = Seq( + * Array(Array(1, 2), Array(3, 4)), + * Array(Array(5, 6, 7), Array(8)), + * Array(Array.empty[Int], Array(9, 10)), + * ).toDF("a") + * + * df.select(array_flatten(col("a"))).show() + * -------------------------- + * |"ARRAY_FLATTEN(""A"")" | + * -------------------------- + * |[ | + * | 1, | + * | 2, | + * | 3, | + * | 4 | + * |] | + * |[ | + * | 5, | + * | 6, | + * | 7, | + * | 8 | + * |] | + * |[ | + * | 9, | + * | 10 | + * |] | + * -------------------------- + * }}} + * + * Example 2: Flattens only one level of a three-level nested array. + * + * {{{ + * val df = Seq( + * Array(Array(Array(1, 2), Array(3)), Array(Array(4, 5))) + * ).toDF("a") + * + * df.select(array_flatten(col("a"))).show() + * -------------------------- + * |"ARRAY_FLATTEN(""A"")" | + * -------------------------- + * |[ | + * | [ | + * | 1, | + * | 2 | + * | ], | + * | [ | + * | 3 | + * | ], | + * | [ | + * | 4, | + * | 5 | + * | ] | + * |] | + * -------------------------- + * }}} + * + * @param array + * Column containing the array of arrays to flatten. + * - If any element of `array` is not an ARRAY, the function throws an error. + * - If `array` is NULL, the function returns NULL. + * @return + * A column containing the flattened array. + * @group semi_func + * @since 1.17.0 + */ + def array_flatten(array: Column): Column = builtin("array_flatten")(array) + /** * Returns an ARRAY containing all elements from the source ARRAY as well as the new element. * diff --git a/src/test/java/com/snowflake/snowpark_test/JavaFunctionSuite.java b/src/test/java/com/snowflake/snowpark_test/JavaFunctionSuite.java index 72e477b5..71c2c86e 100644 --- a/src/test/java/com/snowflake/snowpark_test/JavaFunctionSuite.java +++ b/src/test/java/com/snowflake/snowpark_test/JavaFunctionSuite.java @@ -1,9 +1,17 @@ package com.snowflake.snowpark_test; +import static org.junit.Assert.assertThrows; + import com.snowflake.snowpark_java.*; +import com.snowflake.snowpark_java.types.DataTypes; +import com.snowflake.snowpark_java.types.StructField; +import com.snowflake.snowpark_java.types.StructType; import java.sql.Date; import java.sql.Time; import java.sql.Timestamp; +import java.util.Arrays; +import net.snowflake.client.jdbc.SnowflakeSQLException; +import org.junit.Assert; import org.junit.Test; public class JavaFunctionSuite extends TestBase { @@ -828,6 +836,84 @@ public void concat_ws() { df.select(Functions.concat_ws(Functions.lit(","), df.col("a"), df.col("b"))), expected); } + @Test + public void concat_ws_ignore_nulls() { + DataFrame df = + getSession() + .createDataFrame( + new Row[] { + Row.create(new String[] {"a", "b"}, new String[] {"c"}, "d", "e", 1, 2), + Row.create( + new String[] {"Hello", null, "world"}, + new String[] {null, "!", null}, + "bye", + "world", + 3, + null), + Row.create(new String[] {null, null}, new String[] {"R", "H"}, null, "TD", 4, 5), + Row.create(null, new String[] {null}, null, null, null, null), + Row.create(null, null, null, null, null, null), + }, + StructType.create( + new StructField("arr1", DataTypes.createArrayType(DataTypes.StringType)), + new StructField("arr2", DataTypes.createArrayType(DataTypes.StringType)), + new StructField("str1", DataTypes.StringType), + new StructField("str2", DataTypes.StringType), + new StructField("int1", DataTypes.IntegerType), + new StructField("int2", DataTypes.IntegerType))); + + Column[] columns = + Arrays.stream(df.schema().fieldNames()).map(Functions::col).toArray(Column[]::new); + + // Single character delimiter + checkAnswer( + df.select(Functions.concat_ws_ignore_nulls(",", columns)), + new Row[] { + Row.create("a,b,c,d,e,1,2"), + Row.create("Hello,world,!,bye,world,3"), + Row.create("R,H,TD,4,5"), + Row.create(""), + Row.create("") + }); + + // Multi-character delimiter + checkAnswer( + df.select(Functions.concat_ws_ignore_nulls(" : ", columns)), + new Row[] { + Row.create("a : b : c : d : e : 1 : 2"), + Row.create("Hello : world : ! : bye : world : 3"), + Row.create("R : H : TD : 4 : 5"), + Row.create(""), + Row.create("") + }); + + DataFrame df2 = + getSession() + .createDataFrame( + new Row[] { + Row.create(Date.valueOf("2021-12-21")), Row.create(Date.valueOf("1969-12-31")) + }, + StructType.create(new StructField("YearMonth", DataTypes.DateType))); + + checkAnswer( + df2.select( + Functions.concat_ws_ignore_nulls( + "-", + Functions.year(Functions.col("YearMonth")), + Functions.month(Functions.col("YearMonth")))), + new Row[] {Row.create("2021-12"), Row.create("1969-12")}); + + // Resulting column should allow to define an alias + checkAnswer( + df2.select( + Functions.concat_ws_ignore_nulls( + "-", + Functions.year(Functions.col("YearMonth")), + Functions.month(Functions.col("YearMonth"))) + .alias("YEAR_MONTH")), + new Row[] {Row.create("2021-12"), Row.create("1969-12")}); + } + @Test public void initcap_length_lower_upper() { DataFrame df = getSession().sql("select * from values('asdFg'),('qqq'),('Qw') as T(a)"); @@ -2446,6 +2532,36 @@ public void array_slice() { df.select(Functions.array_slice(df.col("arr1"), df.col("d"), df.col("e"))), expected); } + @Test + public void array_flatten() { + // Flattening a 2D array + DataFrame df1 = + getSession().sql("SELECT [[1, 2, 3], [], [4], [5, NULL, PARSE_JSON('null')]] AS A"); + checkAnswer( + df1.select(Functions.array_flatten(Functions.col("A"))), + new Row[] {Row.create("[\n 1,\n 2,\n 3,\n 4,\n 5,\n undefined,\n null\n]")}); + + // Flattening a 3D array + DataFrame df2 = getSession().sql("SELECT [[[1, 2], [3]]] AS A"); + checkAnswer( + df2.select(Functions.array_flatten(Functions.col("A"))), + new Row[] {Row.create("[\n [\n 1,\n 2\n ],\n [\n 3\n ]\n]")}); + + // Flattening a null array + DataFrame df3 = getSession().sql("SELECT NULL::ARRAY AS A"); + checkAnswer( + df3.select(Functions.array_flatten(Functions.col("A"))), + new Row[] {Row.create((Object) null)}); + + // Flattening an array with non-array elements + DataFrame df4 = getSession().sql("SELECT [1, 2, 3] AS A"); + SnowflakeSQLException exception = + assertThrows( + SnowflakeSQLException.class, + () -> df4.select(Functions.array_flatten(Functions.col("A"))).collect()); + Assert.assertTrue(exception.getMessage().contains("not an array")); + } + @Test public void array_to_string() { DataFrame df = diff --git a/src/test/scala/com/snowflake/snowpark_test/FunctionSuite.scala b/src/test/scala/com/snowflake/snowpark_test/FunctionSuite.scala index cac76914..4e8e31ea 100644 --- a/src/test/scala/com/snowflake/snowpark_test/FunctionSuite.scala +++ b/src/test/scala/com/snowflake/snowpark_test/FunctionSuite.scala @@ -413,6 +413,60 @@ trait FunctionSuite extends TestData { Seq(Row("test1,a"), Row("test2,b"), Row("test3,c"))) } + test("concat_ws_ignore_nulls") { + val df = session.createDataFrame( + Seq( + Row(Array("a", "b"), Array("c"), "d", "e", 1, 2), + Row(Array("Hello", null, "world"), Array(null, "!", null), "bye", "world", 3, null), + Row(Array(null, null), Array("R", "H"), null, "TD", 4, 5), + Row(null, Array(null), null, null, null, null), + Row(null, null, null, null, null, null)), + StructType( + StructField("arr1", ArrayType(StringType)), + StructField("arr2", ArrayType(StringType)), + StructField("str1", StringType), + StructField("str2", StringType), + StructField("int1", IntegerType), + StructField("int2", IntegerType))) + + val columns = df.schema.map(field => col(field.name)) + + // Single character delimiter + checkAnswer( + df.select(concat_ws_ignore_nulls(",", columns: _*)), + Seq( + Row("a,b,c,d,e,1,2"), + Row("Hello,world,!,bye,world,3"), + Row("R,H,TD,4,5"), + Row(""), + Row(""))) + + // Multi-character delimiter + checkAnswer( + df.select(concat_ws_ignore_nulls(" : ", columns: _*)), + Seq( + Row("a : b : c : d : e : 1 : 2"), + Row("Hello : world : ! : bye : world : 3"), + Row("R : H : TD : 4 : 5"), + Row(""), + Row(""))) + + val df2 = session.createDataFrame( + Seq(Row(Date.valueOf("2021-12-21")), Row(Date.valueOf("1969-12-31"))), + StructType(StructField("YearMonth", DateType))) + + checkAnswer( + df2.select(concat_ws_ignore_nulls("-", year(col("YearMonth")), month(col("YearMonth")))), + Seq(Row("2021-12"), Row("1969-12"))) + + // Resulting column should allow to define an alias + checkAnswer( + df2.select( + concat_ws_ignore_nulls("-", year(col("YearMonth")), month(col("YearMonth"))) + .alias("YEAR_MONTH")), + Seq(Row("2021-12"), Row("1969-12"))) + } + test("initcap length lower upper") { checkAnswer( string2.select(initcap(col("A")), length(col("A")), lower(col("A")), upper(col("A"))), @@ -1579,6 +1633,31 @@ trait FunctionSuite extends TestData { Seq(Row("[\n 2\n]"), Row("[\n 5\n]"), Row("[\n 6,\n 7\n]"))) } + test("array_flatten") { + // Flattening a 2D array + val df1 = session.sql("SELECT [[1, 2, 3], [], [4], [5, NULL, PARSE_JSON('null')]] AS A") + checkAnswer( + df1.select(array_flatten(col("A"))), + Seq(Row("[\n 1,\n 2,\n 3,\n 4,\n 5,\n undefined,\n null\n]"))) + + // Flattening a 3D array + val df2 = session.sql("SELECT [[[1, 2], [3]]] AS A") + checkAnswer( + df2.select(array_flatten(col("A"))), + Seq(Row("[\n [\n 1,\n 2\n ],\n [\n 3\n ]\n]"))) + + // Flattening a null array + val df3 = session.sql("SELECT NULL::ARRAY AS A") + checkAnswer(df3.select(array_flatten(col("A"))), Seq(Row(null))) + + // Flattening an array with non-array elements + val df4 = session.sql("SELECT [1, 2, 3] AS A") + val exception = intercept[SnowflakeSQLException] { + df4.select(array_flatten(col("A"))).collect() + } + assert(exception.getMessage.contains("not an array")) + } + test("array_to_string") { checkAnswer( array3.select(array_to_string(col("arr1"), col("f"))), From 36734fd83cef8b56e81b0ff871a80131d65145c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Agust=C3=ADn=20Venegas?= Date: Tue, 16 Sep 2025 13:46:08 -0600 Subject: [PATCH 03/10] SNOW-2246196 | SNOW-2245985 Adding support for 'when' and 'otherwise' methods when Any value type (#241) * Adding support for and methods when Any value type * Adjusting docstring --- .../com/snowflake/snowpark_java/CaseExpr.java | 49 ++++++++++++++--- .../snowflake/snowpark_java/Functions.java | 23 ++++++-- .../scala/com/snowflake/snowpark/Column.scala | 55 +++++++++++++++++-- .../com/snowflake/snowpark/functions.scala | 36 +++++++++--- .../snowpark_test/JavaColumnSuite.java | 24 ++++++++ .../snowflake/snowpark_test/ColumnSuite.scala | 45 +++++++++++++++ 6 files changed, 204 insertions(+), 28 deletions(-) diff --git a/src/main/java/com/snowflake/snowpark_java/CaseExpr.java b/src/main/java/com/snowflake/snowpark_java/CaseExpr.java index 875ae02b..4be9fede 100644 --- a/src/main/java/com/snowflake/snowpark_java/CaseExpr.java +++ b/src/main/java/com/snowflake/snowpark_java/CaseExpr.java @@ -20,25 +20,56 @@ public class CaseExpr extends Column { } /** - * Appends one more WHEN condition to the CASE expression. + * Appends one more WHEN condition to the CASE expression. This method handles any literal value + * and converts it into a `Column` if applies. + * + *

Example: + * + *

{@code
+   * Column result = when(col("age").lt(lit(18)), "Minor")
+   * .when(col("age").lt(lit(65)), "Adult")
+   * .otherwise("Senior");
+   * }
* - * @since 0.12.0 * @param condition The case condition * @param value The result value in the given condition * @return The result case expression + * @since 0.12.0 */ - public CaseExpr when(Column condition, Column value) { - return new CaseExpr(caseExpr.when(condition.toScalaColumn(), value.toScalaColumn())); + public CaseExpr when(Column condition, Object value) { + return new CaseExpr(caseExpr.when(condition.toScalaColumn(), toExpr(value).toScalaColumn())); } /** - * Sets the default result for this CASE expression. + * Sets the default result for this CASE expression. This method handles any literal value and + * converts it into a `Column` if applies. + * + *

Example: + * + *

{@code
+   * Column result = when(col("state").equal(lit("CA")), lit(1000))
+   * .when(col("state").equal(lit("NY")), lit(2000))
+   * .otherwise(1000);
+   * }
* + * @param value The default value, which can be any literal (e.g., String, int, boolean) or a + * `Column`. + * @return The result column. * @since 0.12.0 - * @param value The default value - * @return The result column */ - public Column otherwise(Column value) { - return new Column(caseExpr.otherwise(value.toScalaColumn())); + public Column otherwise(Object value) { + return new Column(caseExpr.otherwise(toExpr(value).toScalaColumn())); + } + + /** + * Converts any value to an Expression. If the value is already a Column, uses its expression + * directly. Otherwise, wraps it with lit() to create a Column expression. + */ + private Column toExpr(Object exp) { + if (exp instanceof Column) { + return ((Column) exp); + } + + return Functions.lit(exp); } } diff --git a/src/main/java/com/snowflake/snowpark_java/Functions.java b/src/main/java/com/snowflake/snowpark_java/Functions.java index 041b257c..cfc88549 100644 --- a/src/main/java/com/snowflake/snowpark_java/Functions.java +++ b/src/main/java/com/snowflake/snowpark_java/Functions.java @@ -4325,19 +4325,20 @@ public static Column get_path(Column col, Column path) { *
{@code
    * import com.snowflake.snowpark_java.Functions;
    * df.select(Functions
-   *     .when(df.col("col").is_null, Functions.lit(1))
-   *     .when(df.col("col").equal_to(Functions.lit(1)), Functions.lit(6))
+   *     .when(df.col("col").is_null, 1)
+   *     .when(df.col("col").equal_to(Functions.lit(1)), 6)
    *     .otherwise(Functions.lit(7)));
    * }
* - * @since 0.12.0 * @param condition The condition * @param value The result value * @return The result column + * @since 0.12.0 */ - public static CaseExpr when(Column condition, Column value) { + public static CaseExpr when(Column condition, Object value) { return new CaseExpr( - com.snowflake.snowpark.functions.when(condition.toScalaColumn(), value.toScalaColumn())); + com.snowflake.snowpark.functions.when( + condition.toScalaColumn(), toExpr(value).toScalaColumn())); } /** @@ -6038,4 +6039,16 @@ private static UserDefinedFunction userDefinedFunction( String funcName, Supplier func) { return javaUDF("Functions", funcName, "", "", func); } + + /** + * Converts any value to an Expression. If the value is already a Column, uses its expression + * directly. Otherwise, wraps it with lit() to create a Column expression. + */ + private static Column toExpr(Object exp) { + if (exp instanceof Column) { + return ((Column) exp); + } + + return Functions.lit(exp); + } } diff --git a/src/main/scala/com/snowflake/snowpark/Column.scala b/src/main/scala/com/snowflake/snowpark/Column.scala index 64434330..2fe81efb 100644 --- a/src/main/scala/com/snowflake/snowpark/Column.scala +++ b/src/main/scala/com/snowflake/snowpark/Column.scala @@ -764,18 +764,54 @@ class CaseExpr private[snowpark] (branches: Seq[(Expression, Expression)]) /** * Appends one more WHEN condition to the CASE expression. * + * This method handles any literal value and converts it into a `Column`. + * + * ===Example=== + * {{{ + * val df = session.sql("SELECT * FROM values (10), (25), (65), (70) as T(age)") + * val result = df.select( + * when(col("age") < lit(18), "Minor") + * .when(col("age") < lit(65), lit("Adult")) + * .otherwise("Senior") + * ) + * // The second when condition will be "Adult" for rows where age >= 18 and age < 65 + * }}} + * + * @param condition + * The case condition. + * @param value + * The result value, which can be any literal (e.g., String, Int, Boolean) or a `Column`. + * @return + * The result case expression. * @since 0.2.0 */ - def when(condition: Column, value: Column): CaseExpr = - new CaseExpr(branches :+ ((condition.expr, value.expr))) + def when(condition: Column, value: Any): CaseExpr = + new CaseExpr(branches :+ (condition.expr, toExpr(value))) /** * Sets the default result for this CASE expression. * + * This method handles any literal value and converts it into a `Column` using `lit()`. + * + * ===Example=== + * {{{ + * val df = session.sql("SELECT * FROM values (10), (25), (65), (70) as T(age)") + * val result = df.select( + * when(col("age") < lit(18), "Minor") + * .when(col("age") < lit(65), lit("Adult")) + * .otherwise("Senior") + * ) + * // The age_category column will be "Senior" for rows where age >= 65 + * }}} + * + * @param value + * The default value, which can be any literal (e.g., String, Int, Boolean) or a `Column`. + * @return + * The result column. * @since 0.2.0 */ - def otherwise(value: Column): Column = withExpr { - CaseWhen(branches, Option(value.expr)) + def otherwise(value: Any): Column = withExpr { + CaseWhen(branches, Option(toExpr(value))) } /** @@ -783,5 +819,14 @@ class CaseExpr private[snowpark] (branches: Seq[(Expression, Expression)]) * * @since 0.2.0 */ - def `else`(value: Column): Column = otherwise(value) + def `else`(value: Any): Column = otherwise(value) + + /** + * Converts any value to an Expression. If the value is already a Column, uses its expression + * directly. Otherwise, wraps it with lit() to create a Column expression. + */ + private def toExpr(exp: Any) = exp match { + case c: Column => c.expr + case _ => lit(exp).expr + } } diff --git a/src/main/scala/com/snowflake/snowpark/functions.scala b/src/main/scala/com/snowflake/snowpark/functions.scala index 71922b63..8acf20eb 100644 --- a/src/main/scala/com/snowflake/snowpark/functions.scala +++ b/src/main/scala/com/snowflake/snowpark/functions.scala @@ -3594,21 +3594,31 @@ object functions { * Works like a cascading if-then-else statement. A series of conditions are evaluated in * sequence. When a condition evaluates to TRUE, the evaluation stops and the associated result * (after THEN) is returned. If none of the conditions evaluate to TRUE, then the result after the - * optional OTHERWISE is returned, if present; otherwise NULL is returned. For Example: + * optional OTHERWISE is returned, if present; otherwise NULL is returned. + * + * ===Example=== * {{{ - * import functions._ - * df.select( - * when(col("col").is_null, lit(1)) - * .when(col("col") === 1, lit(2)) - * .otherwise(lit(3)) - * ) + * import functions._ + * val df = session.sql("SELECT * FROM values (null, 5), (1, 10), (2, 15) as T(col, numeric_col)") + * val result = df.select( + * when(col("col").is_null, lit(1)) + * .when(col("col") === 1, lit(2)) + * .when(col("col") === 1, col("numeric_col") * 0.10) + * .otherwise(lit(3)) + * ) * }}} * + * @param condition + * The case condition. + * @param value + * The result value, which can be any literal (e.g., String, Int, Boolean) or a `Column`. + * @return + * The result case expression. * @group con_func * @since 0.2.0 */ - def when(condition: Column, value: Column): CaseExpr = - new CaseExpr(Seq((condition.expr, value.expr))) + def when(condition: Column, value: Any): CaseExpr = + new CaseExpr(Seq((condition.expr, toExpr(value)))) /** * Returns one of two specified expressions, depending on a condition. @@ -5672,4 +5682,12 @@ object functions { "")(func) } + /** + * Converts any value to an Expression. If the value is already a Column, uses its expression + * directly. Otherwise, wraps it with lit() to create a Column expression. + */ + private def toExpr(exp: Any) = exp match { + case c: Column => c.expr + case _ => lit(exp).expr + } } diff --git a/src/test/java/com/snowflake/snowpark_test/JavaColumnSuite.java b/src/test/java/com/snowflake/snowpark_test/JavaColumnSuite.java index 3aa04cfe..da7bf5ad 100644 --- a/src/test/java/com/snowflake/snowpark_test/JavaColumnSuite.java +++ b/src/test/java/com/snowflake/snowpark_test/JavaColumnSuite.java @@ -281,6 +281,30 @@ public void caseWhen() { Row.create((Object) null), Row.create(5) }); + + // Handling no column type values + checkAnswer( + df.select( + Functions.when(df.col("a").is_null(), 5) + .when(df.col("a").equal_to(Functions.lit(1)), 6) + .otherwise(7) + .as("a")), + new Row[] {Row.create(5), Row.create(7), Row.create(6), Row.create(7), Row.create(5)}); + + // Handling null values + checkAnswer( + df.select( + Functions.when(df.col("a").is_null(), null) + .when(df.col("a").equal_to(Functions.lit(1)), null) + .otherwise(null) + .as("a")), + new Row[] { + Row.create((Object) null), + Row.create((Object) null), + Row.create((Object) null), + Row.create((Object) null), + Row.create((Object) null) + }); } @Test diff --git a/src/test/scala/com/snowflake/snowpark_test/ColumnSuite.scala b/src/test/scala/com/snowflake/snowpark_test/ColumnSuite.scala index 63034dde..2e7bc2cf 100644 --- a/src/test/scala/com/snowflake/snowpark_test/ColumnSuite.scala +++ b/src/test/scala/com/snowflake/snowpark_test/ColumnSuite.scala @@ -513,6 +513,51 @@ class ColumnSuite extends TestData { .as("a")), Seq(Row(5), Row(7), Row(6), Row(7), Row(5))) + // no column typed value + checkAnswer( + nullData1.select( + functions + .when(col("a").is_null, lit(1)) + .when(col("a") === 1, col("a") / 2) + .when(col("a") === 2, col("a") * 2) + .when(col("a") === 3, pow(col("a"), 2)) + .as("a")), + Seq(Row(0.5), Row(1.0), Row(1.0), Row(4.0), Row(9.0))) + + checkAnswer( + nullData1.select( + functions + .when(col("a").is_null, "null_value") + .when(col("a") <= 2, "lower or equal than two") + .when(col("a") >= 3, "greater than two") + .as("a")), + Seq( + Row("greater than two"), + Row("lower or equal than two"), + Row("lower or equal than two"), + Row("null_value"), + Row("null_value"))) + + // No column otherwise + checkAnswer( + nullData1.select( + functions + .when(col("a").is_null, lit(5)) + .when(col("a") === 1, lit(6)) + .otherwise(7) + .as("a")), + Seq(Row(5), Row(7), Row(6), Row(7), Row(5))) + + // Handling nulls + checkAnswer( + nullData1.select( + functions + .when(col("a").is_null, null) + .when(col("a") === 1, null) + .otherwise(null) + .as("a")), + Seq(Row(null), Row(null), Row(null), Row(null), Row(null))) + // empty otherwise checkAnswer( nullData1.select( From d38b67d2ca836dac919e3dab4055a2437c6bc915 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Agust=C3=ADn=20Venegas?= Date: Tue, 16 Sep 2025 15:09:52 -0600 Subject: [PATCH 04/10] SNOW-2246205 Add support for DataFrame.show with truncate parameter (#240) * Add support for DataFrame.show with truncate parameter * Applying format to the code added * Adding some doc improvements * Solve comment --- .../snowflake/snowpark_java/DataFrame.java | 26 +++++++ .../com/snowflake/snowpark/DataFrame.scala | 33 ++++++++- .../snowpark_test/JavaOpenTelemetrySuite.java | 8 +++ .../snowpark_test/DataFrameSuite.scala | 70 +++++++++++++++++++ 4 files changed, 135 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/snowflake/snowpark_java/DataFrame.java b/src/main/java/com/snowflake/snowpark_java/DataFrame.java index c4644c21..d96d5194 100644 --- a/src/main/java/com/snowflake/snowpark_java/DataFrame.java +++ b/src/main/java/com/snowflake/snowpark_java/DataFrame.java @@ -903,6 +903,18 @@ public void show() { df.show(); } + /** + * Evaluates this DataFrame and prints out the first ten rows with configurable column truncation. + * + * @param truncate Whether to truncate long column values. If {@code true}, column values longer + * than 50 characters will be truncated with "...". If {@code false}, full column values will + * be displayed regardless of length. + * @since 1.17.0 + */ + public void show(boolean truncate) { + df.show(truncate); + } + /** * Evaluates this DataFrame and prints out the first `''n''` rows. * @@ -913,6 +925,20 @@ public void show(int n) { df.show(n); } + /** + * Evaluates this DataFrame and prints out the first {@code n} rows with configurable column + * truncation. + * + * @param n The number of rows to print out. + * @param truncate Whether to truncate long column values. If {@code true}, column values longer + * than 50 characters will be truncated with "...". If {@code false}, full column values will + * be displayed regardless of length. + * @since 1.17.0 + */ + public void show(int n, boolean truncate) { + df.show(n, truncate); + } + /** * Evaluates this DataFrame and prints out the first `''n''` rows with the specified maximum * number of characters per column. diff --git a/src/main/scala/com/snowflake/snowpark/DataFrame.scala b/src/main/scala/com/snowflake/snowpark/DataFrame.scala index 1e9ad03a..0c381f5c 100644 --- a/src/main/scala/com/snowflake/snowpark/DataFrame.scala +++ b/src/main/scala/com/snowflake/snowpark/DataFrame.scala @@ -2616,6 +2616,35 @@ class DataFrame private[snowpark] ( show(n, 50) } + /** + * Evaluates this DataFrame and prints out the first `n` rows. + * + * @param n + * The number of rows to print out. + * @param truncate + * Whether to truncate long column values. If `true`, column values longer than 50 characters + * will be truncated. If `false`, full column values will be displayed. + * @group actions + * @since 1.17.0 + */ + def show(n: Int, truncate: Boolean): Unit = action("show") { + val maxWidth = if (truncate) 50 else 0 + this.show(n, maxWidth) + } + + /** + * Evaluates this DataFrame and prints out the first 10 rows. + * + * @param truncate + * Whether to truncate long column values. If `true`, column values longer than 50 characters + * will be truncated. If `false`, full column values will be displayed. + * @group actions + * @since 1.17.0 + */ + def show(truncate: Boolean): Unit = action("show") { + this.show(10, truncate) + } + /** * Evaluates this DataFrame and prints out the first `''n''` rows with the specified maximum * number of characters per column. @@ -2714,7 +2743,7 @@ class DataFrame private[snowpark] ( if (colWidth(index) < str.length) { colWidth(index) = str.length } - if (colWidth(index) > maxWidth) { + if (maxWidth != 0 && colWidth(index) > maxWidth) { colWidth(index) = maxWidth } }) @@ -2746,7 +2775,7 @@ class DataFrame private[snowpark] ( row .zip(colWidth) .map { case (str, size) => - if (str.length > maxWidth) { + if (maxWidth != 0 && str.length > maxWidth) { // if truncated, add ... to the end (str.take(maxWidth - 3) + "...").padTo(size, " ").mkString } else { diff --git a/src/test/java/com/snowflake/snowpark_test/JavaOpenTelemetrySuite.java b/src/test/java/com/snowflake/snowpark_test/JavaOpenTelemetrySuite.java index 0acefee3..8d84c314 100644 --- a/src/test/java/com/snowflake/snowpark_test/JavaOpenTelemetrySuite.java +++ b/src/test/java/com/snowflake/snowpark_test/JavaOpenTelemetrySuite.java @@ -47,6 +47,14 @@ public void show() { checkSpan("snow.snowpark.DataFrame", "show"); df.show(1, 100); checkSpan("snow.snowpark.DataFrame", "show"); + df.show(true); + checkSpan("snow.snowpark.DataFrame", "show"); + df.show(false); + checkSpan("snow.snowpark.DataFrame", "show"); + df.show(1, true); + checkSpan("snow.snowpark.DataFrame", "show"); + df.show(1, false); + checkSpan("snow.snowpark.DataFrame", "show"); } @Test diff --git a/src/test/scala/com/snowflake/snowpark_test/DataFrameSuite.scala b/src/test/scala/com/snowflake/snowpark_test/DataFrameSuite.scala index 1a930903..20908b33 100644 --- a/src/test/scala/com/snowflake/snowpark_test/DataFrameSuite.scala +++ b/src/test/scala/com/snowflake/snowpark_test/DataFrameSuite.scala @@ -83,6 +83,76 @@ trait DataFrameSuite extends TestData with BeforeAndAfterEach { |""".stripMargin) } + test("show not truncated") { + // run show function, make sure no error reported + val df = Seq( + "Short Sample", + "Exceeding Maximum Characters Length Row Value To Evaluate Truncated Results").toDF("Column") + df.show(false) + + assert( + getShowString(df, 10, 0) == + """------------------------------------------------------------------------------- + ||"COLUMN" | + |------------------------------------------------------------------------------- + ||Short Sample | + ||Exceeding Maximum Characters Length Row Value To Evaluate Truncated Results | + |------------------------------------------------------------------------------- + |""".stripMargin) + } + + test("show truncated") { + // run show function, make sure no error reported + val df = Seq( + "Short Sample", + "Exceeding Maximum Characters Length Row Value To Evaluate Truncated Results").toDF("Column") + df.show(true) + + assert( + getShowString(df, 10) == + """------------------------------------------------------ + ||"COLUMN" | + |------------------------------------------------------ + ||Short Sample | + ||Exceeding Maximum Characters Length Row Value T... | + |------------------------------------------------------ + |""".stripMargin) + } + + test("show not truncated limited rows") { + // run show function, make sure no error reported + val df = Seq( + "Exceeding Maximum Characters Length Row Value To Evaluate Truncated Results", + "Short Sample").toDF("Column") + df.show(1, false) + + assert( + getShowString(df, 1, 0) == + """------------------------------------------------------------------------------- + ||"COLUMN" | + |------------------------------------------------------------------------------- + ||Exceeding Maximum Characters Length Row Value To Evaluate Truncated Results | + |------------------------------------------------------------------------------- + |""".stripMargin) + } + + test("show truncated limited rows") { + // run show function, make sure no error reported + val df = Seq( + "Exceeding Maximum Characters Length Row Value To Evaluate Truncated Results", + "Short Sample").toDF("Column") + df.show(1, true) + + assert( + getShowString(df, 1) == + """------------------------------------------------------ + ||"COLUMN" | + |------------------------------------------------------ + ||Exceeding Maximum Characters Length Row Value T... | + |------------------------------------------------------ + |""".stripMargin) + } + test("show with null data") { // run show function, make sure no error reported val df = Seq((1, null), (2, "NotNull")).toDF("a", "b") From 58919ec8a6a035a0fc1d0de256448031f61bc552 Mon Sep 17 00:00:00 2001 From: Hemit Shah Date: Tue, 16 Sep 2025 20:24:36 -0700 Subject: [PATCH 05/10] SNOW-2339275: fix FIPS tests (#244) --- scripts/deploy-fips.sh | 7 +++++++ scripts/utils.sh | 10 +++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/deploy-fips.sh b/scripts/deploy-fips.sh index 85af6bda..d8676aa6 100755 --- a/scripts/deploy-fips.sh +++ b/scripts/deploy-fips.sh @@ -74,10 +74,17 @@ sbt version echo "[INFO] Checking out snowpark-java-scala @ tag: $github_version_tag." git checkout $github_version_tag +<<<<<<< HEAD export SNOWPARK_FIPS="" if [ "$PUBLISH" = true ]; then if [ -v SNOWPARK_FIPS ]; then +======= +export SNOWPARK_FIPS="true" + +if [ "$PUBLISH" = true ]; then + if [ "$SNOWPARK_FIPS" = true ]; then +>>>>>>> 94c75ab (SNOW-2339275: fix FIPS tests (#244)) echo "[INFO] Packaging snowpark-fips @ tag: $github_version_tag." else echo "[INFO] Packaging snowpark @ tag: $github_version_tag." diff --git a/scripts/utils.sh b/scripts/utils.sh index 02c42cf3..720775a8 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -62,8 +62,16 @@ run_test_suites() { # Avoid failures in subsequent test runs due to an already closed stderr. export DISABLE_REDIRECT_STDERR="" + # Set JVM system property for FIPS test if SNOWPARK_FIPS is true. + if [ "$SNOWPARK_FIPS" = true ]; then + FIPS='-J-DFIPS_TEST=true' + echo "Passing $FIPS to sbt" + else + FIPS='' + fi + # test - sbt clean +compile \ + sbt $FIPS clean +compile \ +JavaAPITests:test \ +NonparallelTests:test \ '++ 2.12.20 OtherTests:testOnly * -- -l SampleDataTest' \ From 46fe0de9f4b0c21bf52cac4922bcc444deeee634 Mon Sep 17 00:00:00 2001 From: Bing Li Date: Tue, 16 Sep 2025 21:17:16 -0700 Subject: [PATCH 06/10] Fix deploy-fips.sh --- scripts/deploy-fips.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/scripts/deploy-fips.sh b/scripts/deploy-fips.sh index d8676aa6..0c0ee2a1 100755 --- a/scripts/deploy-fips.sh +++ b/scripts/deploy-fips.sh @@ -74,17 +74,10 @@ sbt version echo "[INFO] Checking out snowpark-java-scala @ tag: $github_version_tag." git checkout $github_version_tag -<<<<<<< HEAD -export SNOWPARK_FIPS="" - -if [ "$PUBLISH" = true ]; then - if [ -v SNOWPARK_FIPS ]; then -======= export SNOWPARK_FIPS="true" if [ "$PUBLISH" = true ]; then if [ "$SNOWPARK_FIPS" = true ]; then ->>>>>>> 94c75ab (SNOW-2339275: fix FIPS tests (#244)) echo "[INFO] Packaging snowpark-fips @ tag: $github_version_tag." else echo "[INFO] Packaging snowpark @ tag: $github_version_tag." From 003d57c9c20e6ca723fde2b8fc7716c4b17b5fca Mon Sep 17 00:00:00 2001 From: Hemit Shah Date: Wed, 17 Sep 2025 15:35:30 -0700 Subject: [PATCH 07/10] SNOW-2343819: consolidate common deploy functionality, generate sha256 checksum files in deploy scripts (#245) 1. What Jira ticket or GitHub issue is this PR addressing? Make sure that there is a ticket or issue accompanying your PR. Fixes SNOW-2343819. 2. Fill out the following pre-review checklist: - [ ] I am adding a new automated test(s) to verify correctness of my new code - [ ] I am adding new logging messages - [ ] I am adding a new telemetry message - [ ] I am adding new credentials - [ ] I am adding a new dependency 3. Please describe how your code solves the related issue. Need `.sha256` checksum files for released artifacts as well. Updating deploy scripts to do so. See - https://snowflake.slack.com/archives/C054J469AN4/p1758085517918489 - https://github.com/snowflakedb/snowflake/pull/337992 --- scripts/deploy-common.sh | 122 +++++++++++++++++++++++++++++++++++++++ scripts/deploy-fips.sh | 114 +----------------------------------- scripts/deploy.sh | 114 +----------------------------------- 3 files changed, 124 insertions(+), 226 deletions(-) create mode 100644 scripts/deploy-common.sh diff --git a/scripts/deploy-common.sh b/scripts/deploy-common.sh new file mode 100644 index 00000000..d5b23fcf --- /dev/null +++ b/scripts/deploy-common.sh @@ -0,0 +1,122 @@ +#!/bin/bash -ex +# +# DO NOT RUN DIRECTLY. +# Script must be sourced by deploy.sh or deploy-fips.sh +# after setting or unsetting `SNOWPARK_FIPS` environment variable. +# + +if [ -z "$GPG_KEY_ID" ]; then + export GPG_KEY_ID="Snowflake Computing" + echo "[WARN] GPG key ID not specified, using default: $GPG_KEY_ID." +fi + +if [ -z "$GPG_KEY_PASSPHRASE" ]; then + echo "[ERROR] GPG passphrase is not specified for $GPG_KEY_ID!" + exit 1 +fi + +if [ -z "$GPG_PRIVATE_KEY" ]; then + echo "[ERROR] GPG private key file is not specified!" + exit 1 +fi + +if [ -z "$sonatype_user" ]; then + echo "[ERROR] Jenkins sonatype user is not specified!" + exit 1 +fi + +if [ -z "$sonatype_password" ]; then + echo "[ERROR] Jenkins sonatype pwd is not specified!" + exit 1 +fi + +if [ -z "$PUBLISH" ]; then + echo "[ERROR] 'PUBLISH' is not specified!" + exit 1 +fi + +if [ -z "$github_version_tag" ]; then + echo "[ERROR] 'github_version_tag' is not specified!" + exit 1 +fi + +mkdir -p ~/.ivy2 + +STR=$'host=central.sonatype.com +user='$sonatype_user' +password='$sonatype_password'' + +echo "$STR" > ~/.ivy2/.credentials + +# import private key first +echo "[INFO] Importing PGP key." +if [ ! -z "$GPG_PRIVATE_KEY" ] && [ -f "$GPG_PRIVATE_KEY" ]; then + # First check if already imported private key + if ! gpg --list-secret-key | grep "$GPG_KEY_ID"; then + gpg --allow-secret-key-import --import "$GPG_PRIVATE_KEY" + fi +fi + +which sbt +if [ $? -ne 0 ] +then + pushd .. + echo "[INFO] sbt is not installed, downloading latest sbt for test and build." + curl -L -o sbt-1.11.4.zip https://github.com/sbt/sbt/releases/download/v1.11.4/sbt-1.11.4.zip + unzip sbt-1.11.4.zip + PATH=$PWD/sbt/bin:$PATH + popd +else + echo "[INFO] Using system installed sbt." +fi +which sbt +sbt version + +echo "[INFO] Checking out snowpark-java-scala @ tag: $github_version_tag." +git checkout $github_version_tag + +if [ "$PUBLISH" = true ]; then + if [ "$SNOWPARK_FIPS" = true ]; then + echo "[INFO] Packaging snowpark-fips @ tag: $github_version_tag." + else + echo "[INFO] Packaging snowpark @ tag: $github_version_tag." + fi + sbt +publishSigned + echo "[INFO] Staged packaged artifacts locally with PGP signing." + sbt sonaUpload + echo "[SUCCESS] Uploaded artifacts to central portal." + echo "[ACTION-REQUIRED] Please log in to Central Portal to publish artifacts: https://central.sonatype.com/" + # TODO: alternatively automate publishing fully +# sbt sonaRelease +# echo "[SUCCESS] Released Snowpark Java-Scala v$github_version_tag to Maven." +else + #release to s3 + echo "[INFO] Staging signed artifacts to local ivy2 repository." + rm -rf ~/.ivy2/local/ + sbt +publishLocalSigned + + # SBT will build FIPS version of Snowpark automatically if the environment variable exists. + if [ "$SNOWPARK_FIPS" = true ]; then + S3_JENKINS_URL="s3://sfc-eng-jenkins/repository/snowparkclient-fips" + S3_DATA_URL="s3://sfc-eng-data/client/snowparkclient-fips/releases" + echo "[INFO] Uploading snowpark-fips artifacts to:" + else + S3_JENKINS_URL="s3://sfc-eng-jenkins/repository/snowparkclient" + S3_DATA_URL="s3://sfc-eng-data/client/snowparkclient/releases" + echo "[INFO] Uploading snowpark artifacts to:" + fi + echo "[INFO] - $S3_JENKINS_URL/$github_version_tag/" + echo "[INFO] - $S3_DATA_URL/$github_version_tag/" + + # Rename all produced artifacts to include version number (sbt doesn't by default when publishing to local ivy2 repository). + find ~/.ivy2/local -type f -name '*snowpark*' | while read file; do newfile=$(echo "$file" | sed "s/\(2\.1[23]\)\([-\.]\)/\1-${github_version_tag#v}\2/"); mv "$file" "$newfile"; done + + # Generate sha256 checksums for all artifacts produced except .md5, .sha1, and existing .sha256 checksum files. + find ~/.ivy2/local -type f -name '*snowpark*' ! -name '*.md5' ! -name '*.sha1' ! -name '*.sha256' -exec sh -c 'for f; do sha256sum "$f" | awk '"'"'{printf "%s", $1}'"'"' > "$f.sha256"; done' _ {} + + + # Copy all files, flattening the nested structure of the ivy2 repository into the expected structure on s3. + find ~/.ivy2/local -type f -name '*snowpark*' -exec aws s3 cp \{\} $S3_JENKINS_URL/$github_version_tag/ \; + find ~/.ivy2/local -type f -name '*snowpark*' -exec aws s3 cp \{\} $S3_DATA_URL/$github_version_tag/ \; + + echo "[SUCCESS] Published Snowpark Java-Scala v$github_version_tag artifacts to S3." +fi diff --git a/scripts/deploy-fips.sh b/scripts/deploy-fips.sh index 0c0ee2a1..69d141fe 100755 --- a/scripts/deploy-fips.sh +++ b/scripts/deploy-fips.sh @@ -4,117 +4,5 @@ # This script needs to be executed by snowflake jenkins job. # -if [ -z "$GPG_KEY_ID" ]; then - export GPG_KEY_ID="Snowflake Computing" - echo "[WARN] GPG key ID not specified, using default: $GPG_KEY_ID." -fi - -if [ -z "$GPG_KEY_PASSPHRASE" ]; then - echo "[ERROR] GPG passphrase is not specified for $GPG_KEY_ID!" - exit 1 -fi - -if [ -z "$GPG_PRIVATE_KEY" ]; then - echo "[ERROR] GPG private key file is not specified!" - exit 1 -fi - -if [ -z "$sonatype_user" ]; then - echo "[ERROR] Jenkins sonatype user is not specified!" - exit 1 -fi - -if [ -z "$sonatype_password" ]; then - echo "[ERROR] Jenkins sonatype pwd is not specified!" - exit 1 -fi - -if [ -z "$PUBLISH" ]; then - echo "[ERROR] 'PUBLISH' is not specified!" - exit 1 -fi - -if [ -z "$github_version_tag" ]; then - echo "[ERROR] 'github_version_tag' is not specified!" - exit 1 -fi - -mkdir -p ~/.ivy2 - -STR=$'host=central.sonatype.com -user='$sonatype_user' -password='$sonatype_password'' - -echo "$STR" > ~/.ivy2/.credentials - -# import private key first -echo "[INFO] Importing PGP key." -if [ ! -z "$GPG_PRIVATE_KEY" ] && [ -f "$GPG_PRIVATE_KEY" ]; then - # First check if already imported private key - if ! gpg --list-secret-key | grep "$GPG_KEY_ID"; then - gpg --allow-secret-key-import --import "$GPG_PRIVATE_KEY" - fi -fi - -which sbt -if [ $? -ne 0 ] -then - pushd .. - echo "[INFO] sbt is not installed, downloading latest sbt for test and build." - curl -L -o sbt-1.11.4.zip https://github.com/sbt/sbt/releases/download/v1.11.4/sbt-1.11.4.zip - unzip sbt-1.11.4.zip - PATH=$PWD/sbt/bin:$PATH - popd -else - echo "[INFO] Using system installed sbt." -fi -which sbt -sbt version - -echo "[INFO] Checking out snowpark-java-scala @ tag: $github_version_tag." -git checkout $github_version_tag - export SNOWPARK_FIPS="true" - -if [ "$PUBLISH" = true ]; then - if [ "$SNOWPARK_FIPS" = true ]; then - echo "[INFO] Packaging snowpark-fips @ tag: $github_version_tag." - else - echo "[INFO] Packaging snowpark @ tag: $github_version_tag." - fi - sbt +publishSigned - echo "[INFO] Staged packaged artifacts locally with PGP signing." - sbt sonaUpload - echo "[SUCCESS] Uploaded artifacts to central portal." - echo "[ACTION-REQUIRED] Please log in to Central Portal to publish artifacts: https://central.sonatype.com/" - # TODO: alternatively automate publishing fully -# sbt sonaRelease -# echo "[SUCCESS] Released Snowpark Java-Scala v$github_version_tag to Maven." -else - #release to s3 - echo "[INFO] Staging signed artifacts to local ivy2 repository." - rm -rf ~/.ivy2/local/ - sbt +publishLocalSigned - - # SBT will build FIPS version of Snowpark automatically if the environment variable exists. - if [ -v SNOWPARK_FIPS ]; then - S3_JENKINS_URL="s3://sfc-eng-jenkins/repository/snowparkclient-fips" - S3_DATA_URL="s3://sfc-eng-data/client/snowparkclient-fips/releases" - echo "[INFO] Uploading snowpark-fips artifacts to:" - else - S3_JENKINS_URL="s3://sfc-eng-jenkins/repository/snowparkclient" - S3_DATA_URL="s3://sfc-eng-data/client/snowparkclient/releases" - echo "[INFO] Uploading snowpark artifacts to:" - fi - echo "[INFO] - $S3_JENKINS_URL/$github_version_tag/" - echo "[INFO] - $S3_DATA_URL/$github_version_tag/" - - # Rename all produced artifacts to include version number (sbt doesn't by default when publishing to local ivy2 repository). - find ~/.ivy2/local -type f -name "*snowpark*" | while read file; do newfile=$(echo "$file" | sed "s/\(2\.1[23]\)\([-\.]\)/\1-${github_version_tag#v}\2/"); mv "$file" "$newfile"; done - - # Copy all files, flattening the nested structure of the ivy2 repository into the expected structure on s3. - find ~/.ivy2/local -type f -name "*snowpark*" -exec aws s3 cp \{\} $S3_JENKINS_URL/$github_version_tag/ \; - find ~/.ivy2/local -type f -name "*snowpark*" -exec aws s3 cp \{\} $S3_DATA_URL/$github_version_tag/ \; - - echo "[SUCCESS] Published Snowpark Java-Scala v$github_version_tag artifacts to S3." -fi +source scripts/deploy-common.sh diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 0ecd000b..e4715f12 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -4,117 +4,5 @@ # This script needs to be executed by snowflake jenkins job. # -if [ -z "$GPG_KEY_ID" ]; then - export GPG_KEY_ID="Snowflake Computing" - echo "[WARN] GPG key ID not specified, using default: $GPG_KEY_ID." -fi - -if [ -z "$GPG_KEY_PASSPHRASE" ]; then - echo "[ERROR] GPG passphrase is not specified for $GPG_KEY_ID!" - exit 1 -fi - -if [ -z "$GPG_PRIVATE_KEY" ]; then - echo "[ERROR] GPG private key file is not specified!" - exit 1 -fi - -if [ -z "$sonatype_user" ]; then - echo "[ERROR] Jenkins sonatype user is not specified!" - exit 1 -fi - -if [ -z "$sonatype_password" ]; then - echo "[ERROR] Jenkins sonatype pwd is not specified!" - exit 1 -fi - -if [ -z "$PUBLISH" ]; then - echo "[ERROR] 'PUBLISH' is not specified!" - exit 1 -fi - -if [ -z "$github_version_tag" ]; then - echo "[ERROR] 'github_version_tag' is not specified!" - exit 1 -fi - -mkdir -p ~/.ivy2 - -STR=$'host=central.sonatype.com -user='$sonatype_user' -password='$sonatype_password'' - -echo "$STR" > ~/.ivy2/.credentials - -# import private key first -echo "[INFO] Importing PGP key." -if [ ! -z "$GPG_PRIVATE_KEY" ] && [ -f "$GPG_PRIVATE_KEY" ]; then - # First check if already imported private key - if ! gpg --list-secret-key | grep "$GPG_KEY_ID"; then - gpg --allow-secret-key-import --import "$GPG_PRIVATE_KEY" - fi -fi - -which sbt -if [ $? -ne 0 ] -then - pushd .. - echo "[INFO] sbt is not installed, downloading latest sbt for test and build." - curl -L -o sbt-1.11.4.zip https://github.com/sbt/sbt/releases/download/v1.11.4/sbt-1.11.4.zip - unzip sbt-1.11.4.zip - PATH=$PWD/sbt/bin:$PATH - popd -else - echo "[INFO] Using system installed sbt." -fi -which sbt -sbt version - -echo "[INFO] Checking out snowpark-java-scala @ tag: $github_version_tag." -git checkout $github_version_tag - unset SNOWPARK_FIPS - -if [ "$PUBLISH" = true ]; then - if [ -v SNOWPARK_FIPS ]; then - echo "[INFO] Packaging snowpark-fips @ tag: $github_version_tag." - else - echo "[INFO] Packaging snowpark @ tag: $github_version_tag." - fi - sbt +publishSigned - echo "[INFO] Staged packaged artifacts locally with PGP signing." - sbt sonaUpload - echo "[SUCCESS] Uploaded artifacts to central portal." - echo "[ACTION-REQUIRED] Please log in to Central Portal to publish artifacts: https://central.sonatype.com/" - # TODO: alternatively automate publishing fully -# sbt sonaRelease -# echo "[SUCCESS] Released Snowpark Java-Scala v$github_version_tag to Maven." -else - #release to s3 - echo "[INFO] Staging signed artifacts to local ivy2 repository." - rm -rf ~/.ivy2/local/ - sbt +publishLocalSigned - - # SBT will build FIPS version of Snowpark automatically if the environment variable exists. - if [ -v SNOWPARK_FIPS ]; then - S3_JENKINS_URL="s3://sfc-eng-jenkins/repository/snowparkclient-fips" - S3_DATA_URL="s3://sfc-eng-data/client/snowparkclient-fips/releases" - echo "[INFO] Uploading snowpark-fips artifacts to:" - else - S3_JENKINS_URL="s3://sfc-eng-jenkins/repository/snowparkclient" - S3_DATA_URL="s3://sfc-eng-data/client/snowparkclient/releases" - echo "[INFO] Uploading snowpark artifacts to:" - fi - echo "[INFO] - $S3_JENKINS_URL/$github_version_tag/" - echo "[INFO] - $S3_DATA_URL/$github_version_tag/" - - # Rename all produced artifacts to include version number (sbt doesn't by default when publishing to local ivy2 repository). - find ~/.ivy2/local -type f -name "*snowpark*" | while read file; do newfile=$(echo "$file" | sed "s/\(2\.1[23]\)\([-\.]\)/\1-${github_version_tag#v}\2/"); mv "$file" "$newfile"; done - - # Copy all files, flattening the nested structure of the ivy2 repository into the expected structure on s3. - find ~/.ivy2/local -type f -name "*snowpark*" -exec aws s3 cp \{\} $S3_JENKINS_URL/$github_version_tag/ \; - find ~/.ivy2/local -type f -name "*snowpark*" -exec aws s3 cp \{\} $S3_DATA_URL/$github_version_tag/ \; - - echo "[SUCCESS] Published Snowpark Java-Scala v$github_version_tag artifacts to S3." -fi +source scripts/deploy-common.sh From e0a98596878fd40d9617441b46a0af66427f6486 Mon Sep 17 00:00:00 2001 From: Hemit Shah Date: Thu, 18 Sep 2025 20:37:28 -0700 Subject: [PATCH 08/10] SNOW-2346738: resolve sonaUpload validation failure caused by fat test jar name (#246) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. What Jira ticket or GitHub issue is this PR addressing? Make sure that there is a ticket or issue accompanying your PR. Fixes SNOW-2346738 2. Fill out the following pre-review checklist: - [ ] I am adding a new automated test(s) to verify correctness of my new code - [ ] I am adding new logging messages - [ ] I am adding a new telemetry message - [ ] I am adding new credentials - [ ] I am adding a new dependency 3. Please describe how your code solves the related issue. Validation of packages to be release failed due to naming of fat test JAR containing compiled test sources and dependencies. See - https://es-ci-legacy-mainvalidation-001.jenkinsdev1.us-west-2.aws-dev.app.snowflake.com/job/MavenPushThunderSnowClient/47/console - https://snowflake.slack.com/archives/C054J469AN4/p1758170003398679?thread_ts=1758085517.918489&cid=C054J469AN4 We resolve this issue by changing the SBT artifact publishing commands to now do the following. - When publishing locally via`sbt +publishLocal`​ or `sbt +publishLocalSigned`​, and for uploading to s3 buckets (`$PUBLISH`​ environment variable must be `false`​ or not set​), the following artifacts are produced with their associated signature and checksum files: - `ivy.xml`​ - `snowpark_2.1[23].jar` - `snowpark_2.1[23].pom` - `snowpark_2.1[23]-javadoc.jar`​ - `snowpark_2.1[23]-scaladoc.jar` - `snowpark_2.1[23]-bundle.tar.gz` - `snowpark_2.1[23]-bundle.zip` - `snowpark_2.1[23]-with-dependencies.jar` - `fat-test-snowpark_2.1[23]-fat-test.jar` - When publishing to Maven central repository via `sbt +publishSigned`​ (`$PUBLISH`​ environment variable must be `true`​), only the following artifacts are produced with their associated signature and checksum files: - `ivy.xml`​ - `snowpark_2.1[23].jar` - `snowpark_2.1[23].pom` - `snowpark_2.1[23]-javadoc.jar`​ - `snowpark_2.1[23]-scaladoc.jar` --- build.sbt | 32 ++++++++++++++++++++++++-------- scripts/deploy-common.sh | 28 ++++++++++++++++++---------- 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/build.sbt b/build.sbt index e0e5f7c3..1658b3d1 100644 --- a/build.sbt +++ b/build.sbt @@ -9,6 +9,18 @@ lazy val isFipsRelease = { // scalastyle:on println result } +lazy val includeFatJarsAndBundles = { + // When the PUBLISH environment variable is true, we assume the caller is publishing to Maven, + // in which case we do not want to include fat JAR, ZIP, or Tarball bundle artifacts. + val result = !sys.env.getOrElse("PUBLISH", "false").toBoolean + // scalastyle:off println + println(s"Including Fat JARs and Bundles in published artifacts: $result") + // scalastyle:on println + result +} +def isFatJarOrBundle(c: String): Boolean = + c.contains("with-dependencies") || c.contains("fat-test") || c.contains("bundle") + lazy val snowparkName = s"snowpark${if (isFipsRelease) "-fips" else ""}" lazy val jdbcName = s"snowflake-jdbc${if (isFipsRelease) "-fips" else ""}" lazy val snowparkVersion = "1.17.0" @@ -165,13 +177,8 @@ lazy val root = (project in file(".")) // Release settings - // Release JAR including compiled test classes - Test / packageBin / publishArtifact := true, - // Also publish a test-sources JAR - Test / packageSrc / publishArtifact := true, - Test / packageSrc / artifact := - (Compile / packageSrc / artifact).value.withClassifier(Some("tests-sources")), - addArtifact(Test / packageSrc / artifact, Test / packageSrc), + // Disable publishing the source files JAR + Compile / packageSrc / publishArtifact := false, // Fat JAR settings assembly / assemblyJarName := @@ -247,6 +254,16 @@ lazy val root = (project in file(".")) Artifact(name = snowparkName, `type` = "bundle", extension = "tar.gz", classifier = "bundle"), Universal / packageZipTarball), + // Explicitly list checksum files to be generated for visibility + checksums := Seq("md5", "sha1"), + + // Filter out bundles and fat jars if publishing to maven + artifacts := artifacts.value filter ( + a => includeFatJarsAndBundles || !isFatJarOrBundle(a.classifier.getOrElse(""))), + packagedArtifacts := packagedArtifacts.value filter ( + af => includeFatJarsAndBundles || !isFatJarOrBundle(af._1.classifier.getOrElse(""))), + + // Signed publish settings credentials += Credentials(Path.userHome / ".ivy2" / ".credentials"), // Set up GPG key for release build from environment variable: GPG_HEX_CODE // Build jenkins job must have set it, otherwise, the release build will fail. @@ -256,7 +273,6 @@ lazy val root = (project in file(".")) Properties.envOrNone("GPG_HEX_CODE").getOrElse("Jenkins_build_not_set_GPG_HEX_CODE"), "ignored" // this field is ignored; passwords are supplied by pinentry ), - // usePgpKeyHex(Properties.envOrElse("GPG_SIGNATURE", "12345")), Global / pgpPassphrase := Properties.envOrNone("GPG_KEY_PASSPHRASE").map(_.toCharArray), publishMavenStyle := true, releaseCrossBuild := true, diff --git a/scripts/deploy-common.sh b/scripts/deploy-common.sh index d5b23fcf..3670cb33 100644 --- a/scripts/deploy-common.sh +++ b/scripts/deploy-common.sh @@ -88,7 +88,7 @@ if [ "$PUBLISH" = true ]; then echo "[ACTION-REQUIRED] Please log in to Central Portal to publish artifacts: https://central.sonatype.com/" # TODO: alternatively automate publishing fully # sbt sonaRelease -# echo "[SUCCESS] Released Snowpark Java-Scala v$github_version_tag to Maven." +# echo "[SUCCESS] Released Snowpark Java-Scala $github_version_tag to Maven." else #release to s3 echo "[INFO] Staging signed artifacts to local ivy2 repository." @@ -97,26 +97,34 @@ else # SBT will build FIPS version of Snowpark automatically if the environment variable exists. if [ "$SNOWPARK_FIPS" = true ]; then - S3_JENKINS_URL="s3://sfc-eng-jenkins/repository/snowparkclient-fips" - S3_DATA_URL="s3://sfc-eng-data/client/snowparkclient-fips/releases" + S3_JENKINS_URL="s3://sfc-eng-jenkins/repository/snowparkclient-fips/$github_version_tag/" + S3_DATA_URL="s3://sfc-eng-data/client/snowparkclient-fips/releases/$github_version_tag/" echo "[INFO] Uploading snowpark-fips artifacts to:" else - S3_JENKINS_URL="s3://sfc-eng-jenkins/repository/snowparkclient" - S3_DATA_URL="s3://sfc-eng-data/client/snowparkclient/releases" + S3_JENKINS_URL="s3://sfc-eng-jenkins/repository/snowparkclient/$github_version_tag/" + S3_DATA_URL="s3://sfc-eng-data/client/snowparkclient/releases/$github_version_tag/" echo "[INFO] Uploading snowpark artifacts to:" fi - echo "[INFO] - $S3_JENKINS_URL/$github_version_tag/" - echo "[INFO] - $S3_DATA_URL/$github_version_tag/" + echo "[INFO] - $S3_JENKINS_URL" + echo "[INFO] - $S3_DATA_URL" + + # Remove release folders in s3 for current release version if they already exist due to previously failed release pipeline runs. + echo "[INFO] Deleting $github_version_tag release folders in s3 if they already exist." + aws s3 rm "$S3_JENKINS_URL" --recursive + echo "[INFO] $S3_JENKINS_URL folder deleted if it exists." + aws s3 rm "$S3_DATA_URL" --recursive + echo "[INFO] $S3_DATA_URL folder deleted if it exists." # Rename all produced artifacts to include version number (sbt doesn't by default when publishing to local ivy2 repository). + # TODO: BEFORE SNOWPARK v2.12.0, fix the regex in the sed command to not match the 2.12.x or 2.13.x named folder under ~/.ivy2/local/com.snowflake/snowpark_2.1[23]/ find ~/.ivy2/local -type f -name '*snowpark*' | while read file; do newfile=$(echo "$file" | sed "s/\(2\.1[23]\)\([-\.]\)/\1-${github_version_tag#v}\2/"); mv "$file" "$newfile"; done # Generate sha256 checksums for all artifacts produced except .md5, .sha1, and existing .sha256 checksum files. find ~/.ivy2/local -type f -name '*snowpark*' ! -name '*.md5' ! -name '*.sha1' ! -name '*.sha256' -exec sh -c 'for f; do sha256sum "$f" | awk '"'"'{printf "%s", $1}'"'"' > "$f.sha256"; done' _ {} + # Copy all files, flattening the nested structure of the ivy2 repository into the expected structure on s3. - find ~/.ivy2/local -type f -name '*snowpark*' -exec aws s3 cp \{\} $S3_JENKINS_URL/$github_version_tag/ \; - find ~/.ivy2/local -type f -name '*snowpark*' -exec aws s3 cp \{\} $S3_DATA_URL/$github_version_tag/ \; + find ~/.ivy2/local -type f -name '*snowpark*' ! -name '*.sha1' -exec aws s3 cp \{\} $S3_JENKINS_URL \; + find ~/.ivy2/local -type f -name '*snowpark*' ! -name '*.sha1' -exec aws s3 cp \{\} $S3_DATA_URL \; - echo "[SUCCESS] Published Snowpark Java-Scala v$github_version_tag artifacts to S3." + echo "[SUCCESS] Published Snowpark Java-Scala $github_version_tag artifacts to S3." fi From 005e06c8b43012656c2c5c416e8ac15e42abb75d Mon Sep 17 00:00:00 2001 From: Hemit Shah Date: Mon, 22 Sep 2025 16:56:40 -0700 Subject: [PATCH 09/10] SNOW-2346738: resolve sonaUpload validation failure caused by missing sources jar (#249) 1. What Jira ticket or GitHub issue is this PR addressing? Make sure that there is a ticket or issue accompanying your PR. Fixes SNOW-2346738 2. Fill out the following pre-review checklist: - [ ] I am adding a new automated test(s) to verify correctness of my new code - [ ] I am adding new logging messages - [ ] I am adding a new telemetry message - [ ] I am adding new credentials - [ ] I am adding a new dependency 3. Please describe how your code solves the related issue. Include sources JAR when publishing to Maven to fulfill requirements: https://central.sonatype.org/publish/requirements/ --- build.sbt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index 1658b3d1..2b512ea1 100644 --- a/build.sbt +++ b/build.sbt @@ -177,8 +177,11 @@ lazy val root = (project in file(".")) // Release settings - // Disable publishing the source files JAR - Compile / packageSrc / publishArtifact := false, + // Disable publishing the source files JAR unless publishing to maven. + Compile / packageSrc / publishArtifact := !includeFatJarsAndBundles, + + // Disable publishing test source files in all scenarios. + Test / packageSrc / publishArtifact := false, // Fat JAR settings assembly / assemblyJarName := From 25eafc342f515cdb8f592edc3b04bc9a951e9522 Mon Sep 17 00:00:00 2001 From: Hemit Shah Date: Tue, 4 Nov 2025 15:38:33 -0800 Subject: [PATCH 10/10] NOSNOW: fix docs generation (#252) --- build.sbt | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sbt b/build.sbt index 2b512ea1..ecafc7bc 100644 --- a/build.sbt +++ b/build.sbt @@ -38,6 +38,7 @@ lazy val javadocSettings = inConfig(Javadoc)(Defaults.configSettings) ++ Seq( !(s.getParent.contains("internal") || s.getParent.contains("Internal"))), Javadoc / javacOptions := Seq( "--allow-script-in-comments", + "-use", "-windowtitle", s"Snowpark Java API Reference $snowparkVersion", "-doctitle", s"Snowpark Java API Reference $snowparkVersion", "-header", s"""