diff --git a/contrib/templates/scala-job/library/template_variables.tmpl b/contrib/templates/scala-job/library/template_variables.tmpl index 43628df2..3b666b1e 100644 --- a/contrib/templates/scala-job/library/template_variables.tmpl +++ b/contrib/templates/scala-job/library/template_variables.tmpl @@ -3,7 +3,7 @@ {{- end }} {{ define `dbr_version` -}} - 16.3 + 16.4 {{- end }} {{ define `scala_major_minor_version` -}} diff --git a/contrib/templates/scala-job/template/{{.project_name}}/src/main/scala/com/examples/Main.scala b/contrib/templates/scala-job/template/{{.project_name}}/src/main/scala/com/examples/Main.scala index a6648511..0b81582d 100644 --- a/contrib/templates/scala-job/template/{{.project_name}}/src/main/scala/com/examples/Main.scala +++ b/contrib/templates/scala-job/template/{{.project_name}}/src/main/scala/com/examples/Main.scala @@ -19,15 +19,7 @@ object Main { println("Showing nyctaxi trips ...") val nycTaxi = new NycTaxi(spark) val df = nycTaxi.trips() - - // Define a simple UDF that formats the passenger count as a string - val testudf = udf((count: String) => s"test: $count") - - // Apply the UDF to the passenger_count column - val transformedDF = df.withColumn("testresult", testudf(F.col("dropoff_zip"))) - - // Show the transformed DataFrame - transformedDF.show() + df.show() } def getSession(): SparkSession = { @@ -39,6 +31,7 @@ object Main { println("Running outside Databricks") DatabricksSession.builder() .serverless() + .validateSession(false) .addCompiledArtifacts(Main.getClass.getProtectionDomain.getCodeSource.getLocation.toURI) .getOrCreate() }