From c0e2a8908027504c8de619f95f527234b3c6cf56 Mon Sep 17 00:00:00 2001 From: Arkadiusz Komarzewski Date: Thu, 18 Apr 2019 13:52:53 +0200 Subject: [PATCH 1/5] Bump sbt to 1.2.8 --- project/build.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/build.properties b/project/build.properties index 5620cc50..c0bab049 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.2.1 +sbt.version=1.2.8 From 5b60f88aa60a6dbde6a3982c1f7ec23b0a394e9c Mon Sep 17 00:00:00 2001 From: Arkadiusz Komarzewski Date: Thu, 18 Apr 2019 13:51:21 +0200 Subject: [PATCH 2/5] Bump Spark version to 2.4 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index efac7020..896fff30 100644 --- a/build.sbt +++ b/build.sbt @@ -20,7 +20,7 @@ organization := "com.mozilla" scalaVersion in ThisBuild := "2.11.8" -val sparkVersion = "2.3.0" +val sparkVersion = "2.4.0" lazy val root = (project in file(".")). settings( From 7626179d0f57bf035649264203e9b139d66db1f5 Mon Sep 17 00:00:00 2001 From: Arkadiusz Komarzewski Date: Thu, 18 Apr 2019 13:59:57 +0200 Subject: [PATCH 3/5] Update Kafka connector artifact name --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 896fff30..7bb77a24 100644 --- a/build.sbt +++ b/build.sbt @@ -29,7 +29,7 @@ lazy val root = (project in file(".")). libraryDependencies += "org.apache.spark" %% "spark-core" % sparkVersion % "provided", libraryDependencies += "org.apache.spark" %% "spark-streaming" % sparkVersion % "provided", libraryDependencies += "org.apache.spark" %% "spark-sql" % sparkVersion % "provided", - libraryDependencies += "org.apache.spark" %% "spark-sql-kafka-0-10" % sparkVersion + libraryDependencies += "org.apache.spark" %% "spark-streaming-kafka-0-10" % sparkVersion exclude("net.jpountz.lz4", "lz4"), //conflicts with org.lz4:lz4-java:1.4.0 from spark-core libraryDependencies += "org.eclipse.jetty" % "jetty-servlet" % "9.3.20.v20170531" % Provided, // needed for metrics libraryDependencies += "org.rogach" %% "scallop" % "1.0.2", From 0ae8b89b454473cebc1f87527e58076c0f27b4ee Mon Sep 17 00:00:00 2001 From: Arkadiusz Komarzewski Date: Thu, 18 Apr 2019 14:54:14 +0200 Subject: [PATCH 4/5] Exclude transitive json4s dependency from moztelemetry --- build.sbt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 7bb77a24..3ceec0de 100644 --- a/build.sbt +++ b/build.sbt @@ -24,7 +24,8 @@ val sparkVersion = "2.4.0" lazy val root = (project in file(".")). settings( - libraryDependencies += "com.mozilla.telemetry" %% "moztelemetry" % "1.1-SNAPSHOT", + libraryDependencies += "com.mozilla.telemetry" %% "moztelemetry" % "1.1-SNAPSHOT" + exclude("org.json4s", "json4s-jackson_2.11"), libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.1" % Test, libraryDependencies += "org.apache.spark" %% "spark-core" % sparkVersion % "provided", libraryDependencies += "org.apache.spark" %% "spark-streaming" % sparkVersion % "provided", From bce345492c4308b6d26aaea2a59a74ec905ddad6 Mon Sep 17 00:00:00 2001 From: Arkadiusz Komarzewski Date: Thu, 18 Apr 2019 14:55:26 +0200 Subject: [PATCH 5/5] Revert "Update Kafka connector artifact name" This reverts commit 7626179d0f57bf035649264203e9b139d66db1f5. --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 3ceec0de..f474800f 100644 --- a/build.sbt +++ b/build.sbt @@ -30,7 +30,7 @@ lazy val root = (project in file(".")). libraryDependencies += "org.apache.spark" %% "spark-core" % sparkVersion % "provided", libraryDependencies += "org.apache.spark" %% "spark-streaming" % sparkVersion % "provided", libraryDependencies += "org.apache.spark" %% "spark-sql" % sparkVersion % "provided", - libraryDependencies += "org.apache.spark" %% "spark-streaming-kafka-0-10" % sparkVersion + libraryDependencies += "org.apache.spark" %% "spark-sql-kafka-0-10" % sparkVersion exclude("net.jpountz.lz4", "lz4"), //conflicts with org.lz4:lz4-java:1.4.0 from spark-core libraryDependencies += "org.eclipse.jetty" % "jetty-servlet" % "9.3.20.v20170531" % Provided, // needed for metrics libraryDependencies += "org.rogach" %% "scallop" % "1.0.2",