From ca5a72b4e947deff4e6447b2f83e226b0dd5127c Mon Sep 17 00:00:00 2001 From: Garland Zhang Date: Tue, 4 Nov 2025 17:07:35 +0000 Subject: [PATCH 1/7] Add default scala template --- contrib/templates/default-scala/README.md | 12 +++++++ .../databricks_template_schema.json | 13 +++++-- .../library/template_variables.tmpl | 0 .../template/{{.project_name}}/.gitignore | 0 .../template/{{.project_name}}/README.md.tmpl | 13 +++++-- .../template/{{.project_name}}/build.sbt.tmpl | 0 .../{{.project_name}}/databricks.yml.tmpl | 0 .../project/plugins.sbt.tmpl | 0 .../{{.project_name}}/resources/.gitkeep | 0 .../resources/{{.project_name}}.job.yml.tmpl | 36 +++++++++++++++++++ .../src/main/scala/com/examples/Main.scala | 0 .../src/main/scala/com/examples/NycTaxi.scala | 0 .../test/scala/com/examples/NycTaxiSpec.scala | 0 contrib/templates/scala-job/README.md | 10 ------ .../resources/{{.project_name}}.job.yml.tmpl | 17 --------- 15 files changed, 68 insertions(+), 33 deletions(-) create mode 100644 contrib/templates/default-scala/README.md rename contrib/templates/{scala-job => default-scala}/databricks_template_schema.json (50%) rename contrib/templates/{scala-job => default-scala}/library/template_variables.tmpl (100%) rename contrib/templates/{scala-job => default-scala}/template/{{.project_name}}/.gitignore (100%) rename contrib/templates/{scala-job => default-scala}/template/{{.project_name}}/README.md.tmpl (78%) rename contrib/templates/{scala-job => default-scala}/template/{{.project_name}}/build.sbt.tmpl (100%) rename contrib/templates/{scala-job => default-scala}/template/{{.project_name}}/databricks.yml.tmpl (100%) rename contrib/templates/{scala-job => default-scala}/template/{{.project_name}}/project/plugins.sbt.tmpl (100%) rename contrib/templates/{scala-job => default-scala}/template/{{.project_name}}/resources/.gitkeep (100%) create mode 100644 contrib/templates/default-scala/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl rename contrib/templates/{scala-job => default-scala}/template/{{.project_name}}/src/main/scala/com/examples/Main.scala (100%) rename contrib/templates/{scala-job => default-scala}/template/{{.project_name}}/src/main/scala/com/examples/NycTaxi.scala (100%) rename contrib/templates/{scala-job => default-scala}/template/{{.project_name}}/src/test/scala/com/examples/NycTaxiSpec.scala (100%) delete mode 100644 contrib/templates/scala-job/README.md delete mode 100644 contrib/templates/scala-job/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl diff --git a/contrib/templates/default-scala/README.md b/contrib/templates/default-scala/README.md new file mode 100644 index 00000000..5c6be785 --- /dev/null +++ b/contrib/templates/default-scala/README.md @@ -0,0 +1,12 @@ +# default-scala + +This template helps you create Scala projects with Databricks Asset Bundles. It uses sbt to compile and package Scala files, and can be used with Databricks Connect for local development. + +It supports two compute types: standard clusters and serverless compute. + +Run +``` +databricks bundle init --template-dir contrib/templates/default-scala https://github.com/databricks/bundle-examples +``` + +and follow the generated README.md to get started. \ No newline at end of file diff --git a/contrib/templates/scala-job/databricks_template_schema.json b/contrib/templates/default-scala/databricks_template_schema.json similarity index 50% rename from contrib/templates/scala-job/databricks_template_schema.json rename to contrib/templates/default-scala/databricks_template_schema.json index 3571e778..de28ab6f 100644 --- a/contrib/templates/scala-job/databricks_template_schema.json +++ b/contrib/templates/default-scala/databricks_template_schema.json @@ -1,5 +1,5 @@ { - "welcome_message": "\nWelcome to the scala-job template for Databricks Asset Bundles!\n\nA workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html.\nworkspace_host: {{workspace_host}}", + "welcome_message": "\nWelcome to the default-scala template for Databricks Asset Bundles!\n\nA workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html.\nworkspace_host: {{workspace_host}}", "properties": { "project_name": { "type": "string", @@ -8,10 +8,17 @@ "pattern": "^[A-Za-z_][A-Za-z0-9-_]+$", "pattern_match_failure_message": "Name must consist of letters, numbers, dashes, and underscores." }, + "compute_type": { + "type": "string", + "description": "\nPlease select the compute type.\ncompute_type", + "enum": ["standard cluster", "serverless"], + "default": "serverless", + "order": 2 + }, "artifacts_dest_path": { "type": "string", - "description": "\nPlease provide the Unity Catalog volume destination path in Databricks where the directory will be created containing the JAR and other artifacts to store. [example: /Volumes/abcdef1234567890]", - "order": 2, + "description": "\nPlease provide the Unity Catalog volume destination path in Databricks where the directory will be created containing the JAR and other artifacts to store. [example: /Volumes/abcdef1234567890]{{if eq .compute_type \"standard cluster\"}}\nNote: your admin must allowlist the volume JAR path you specify for your workspace (see https://docs.databricks.com/en/data-governance/unity-catalog/manage-privileges/allowlist.html){{end}}", + "order": 3, "pattern": "^/Volumes(?:/[a-z0-9_-]+)+/?$", "pattern_match_failure_message": "Please enter a valid path like /Volumes/my-folder/. Only lowercase letters, numbers, dashes, and underscores are allowed in folder names." } diff --git a/contrib/templates/scala-job/library/template_variables.tmpl b/contrib/templates/default-scala/library/template_variables.tmpl similarity index 100% rename from contrib/templates/scala-job/library/template_variables.tmpl rename to contrib/templates/default-scala/library/template_variables.tmpl diff --git a/contrib/templates/scala-job/template/{{.project_name}}/.gitignore b/contrib/templates/default-scala/template/{{.project_name}}/.gitignore similarity index 100% rename from contrib/templates/scala-job/template/{{.project_name}}/.gitignore rename to contrib/templates/default-scala/template/{{.project_name}}/.gitignore diff --git a/contrib/templates/scala-job/template/{{.project_name}}/README.md.tmpl b/contrib/templates/default-scala/template/{{.project_name}}/README.md.tmpl similarity index 78% rename from contrib/templates/scala-job/template/{{.project_name}}/README.md.tmpl rename to contrib/templates/default-scala/template/{{.project_name}}/README.md.tmpl index e0b3f48c..e42a5558 100644 --- a/contrib/templates/scala-job/template/{{.project_name}}/README.md.tmpl +++ b/contrib/templates/default-scala/template/{{.project_name}}/README.md.tmpl @@ -1,6 +1,6 @@ # {{.project_name}} -The '{{.project_name}}' project was generated by using the scala-job template. +The '{{.project_name}}' project was generated by using the default-scala template. ## Getting started @@ -90,7 +90,14 @@ by running `sbt test`. ## Customizations ### Job configuration -The bundles piggybacks off the same configurations used in APIs. If you want to use an existing cluster instead of spinning one up everytime, replace job_cluster_key in tasks with existing_cluster_id: +{{- if eq .compute_type "serverless"}} +This project uses serverless compute. The serverless environment automatically manages compute resources. -You can also change to an all-purpose (dedicated) cluster by removing the data_security_mode of the created cluster. +If you prefer to use standard compute with job clusters, you can modify the job YAML file to use `job_clusters` instead of the `environments` configuration. +{{- else }} +This project uses standard compute with a job cluster. Standard compute requires Unity Catalog volume JAR paths to be allowlisted by your workspace admin. +If you want to use an existing cluster instead, replace `job_cluster_key` in the task configuration with `existing_cluster_id: ` in the job YAML file. + +You can also customize the cluster configuration (node type, worker count, Spark version) in the `job_clusters` section of the job YAML file. +{{- end}} diff --git a/contrib/templates/scala-job/template/{{.project_name}}/build.sbt.tmpl b/contrib/templates/default-scala/template/{{.project_name}}/build.sbt.tmpl similarity index 100% rename from contrib/templates/scala-job/template/{{.project_name}}/build.sbt.tmpl rename to contrib/templates/default-scala/template/{{.project_name}}/build.sbt.tmpl diff --git a/contrib/templates/scala-job/template/{{.project_name}}/databricks.yml.tmpl b/contrib/templates/default-scala/template/{{.project_name}}/databricks.yml.tmpl similarity index 100% rename from contrib/templates/scala-job/template/{{.project_name}}/databricks.yml.tmpl rename to contrib/templates/default-scala/template/{{.project_name}}/databricks.yml.tmpl diff --git a/contrib/templates/scala-job/template/{{.project_name}}/project/plugins.sbt.tmpl b/contrib/templates/default-scala/template/{{.project_name}}/project/plugins.sbt.tmpl similarity index 100% rename from contrib/templates/scala-job/template/{{.project_name}}/project/plugins.sbt.tmpl rename to contrib/templates/default-scala/template/{{.project_name}}/project/plugins.sbt.tmpl diff --git a/contrib/templates/scala-job/template/{{.project_name}}/resources/.gitkeep b/contrib/templates/default-scala/template/{{.project_name}}/resources/.gitkeep similarity index 100% rename from contrib/templates/scala-job/template/{{.project_name}}/resources/.gitkeep rename to contrib/templates/default-scala/template/{{.project_name}}/resources/.gitkeep diff --git a/contrib/templates/default-scala/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl b/contrib/templates/default-scala/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl new file mode 100644 index 00000000..335a5fde --- /dev/null +++ b/contrib/templates/default-scala/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl @@ -0,0 +1,36 @@ +# The main job for {{.project_name}} + +resources: + jobs: + {{.project_name}}: + name: {{.project_name}} + tasks: + - task_key: main_task +{{- if eq .compute_type "standard cluster"}} + job_cluster_key: job_cluster +{{- end}} + spark_jar_task: + main_class_name: {{template `main_class_name` .}} +{{- if eq .compute_type "serverless"}} + environment_key: default +{{- else}} + libraries: + - jar: ${workspace.artifact_path}/.internal/{{.project_name}}-assembly-{{template `version` .}}.jar +{{- end}} +{{- if eq .compute_type "serverless"}} + environments: + - environment_key: default + spec: + environment_version: "4-scala-preview" + java_dependencies: + - ${workspace.artifact_path}/.internal/{{.project_name}}-assembly-{{template `version` .}}.jar +{{- else}} + job_clusters: + - job_cluster_key: job_cluster + new_cluster: + spark_version: 17.3.x-scala2.13 + node_type_id: i3.xlarge + autoscale: + min_workers: 1 + max_workers: 4 +{{- end}} \ No newline at end of file diff --git a/contrib/templates/scala-job/template/{{.project_name}}/src/main/scala/com/examples/Main.scala b/contrib/templates/default-scala/template/{{.project_name}}/src/main/scala/com/examples/Main.scala similarity index 100% rename from contrib/templates/scala-job/template/{{.project_name}}/src/main/scala/com/examples/Main.scala rename to contrib/templates/default-scala/template/{{.project_name}}/src/main/scala/com/examples/Main.scala diff --git a/contrib/templates/scala-job/template/{{.project_name}}/src/main/scala/com/examples/NycTaxi.scala b/contrib/templates/default-scala/template/{{.project_name}}/src/main/scala/com/examples/NycTaxi.scala similarity index 100% rename from contrib/templates/scala-job/template/{{.project_name}}/src/main/scala/com/examples/NycTaxi.scala rename to contrib/templates/default-scala/template/{{.project_name}}/src/main/scala/com/examples/NycTaxi.scala diff --git a/contrib/templates/scala-job/template/{{.project_name}}/src/test/scala/com/examples/NycTaxiSpec.scala b/contrib/templates/default-scala/template/{{.project_name}}/src/test/scala/com/examples/NycTaxiSpec.scala similarity index 100% rename from contrib/templates/scala-job/template/{{.project_name}}/src/test/scala/com/examples/NycTaxiSpec.scala rename to contrib/templates/default-scala/template/{{.project_name}}/src/test/scala/com/examples/NycTaxiSpec.scala diff --git a/contrib/templates/scala-job/README.md b/contrib/templates/scala-job/README.md deleted file mode 100644 index 2f476a07..00000000 --- a/contrib/templates/scala-job/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# scala-job - -This is an (experimental) template for creating using Scala with Databricks Asset Bundles. It uses sbt to compile and package Scala files, and can be used with Databricks Connect for local development. - -Run -``` -databricks bundle init --template-dir contrib/templates/scala-job https://github.com/databricks/bundle-examples -``` - -and follow the generated README.md to get started. \ No newline at end of file diff --git a/contrib/templates/scala-job/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl b/contrib/templates/scala-job/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl deleted file mode 100644 index 97f2f6e3..00000000 --- a/contrib/templates/scala-job/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl +++ /dev/null @@ -1,17 +0,0 @@ -# The main job for {{.project_name}} - -resources: - jobs: - {{.project_name}}: - name: {{.project_name}} - tasks: - - task_key: main_task - spark_jar_task: - main_class_name: {{template `main_class_name` .}} - environment_key: default - environments: - - environment_key: default - spec: - environment_version: "4-scala-preview" - java_dependencies: - - ${workspace.artifact_path}/.internal/{{.project_name}}-assembly-{{template `version` .}}.jar From 3261ac39f8d025b25c8f4f375ba1921ca34fd89f Mon Sep 17 00:00:00 2001 From: Garland Zhang Date: Tue, 4 Nov 2025 17:10:55 +0000 Subject: [PATCH 2/7] cleanup messaging --- .../default-scala/template/{{.project_name}}/README.md.tmpl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/contrib/templates/default-scala/template/{{.project_name}}/README.md.tmpl b/contrib/templates/default-scala/template/{{.project_name}}/README.md.tmpl index e42a5558..231bcce7 100644 --- a/contrib/templates/default-scala/template/{{.project_name}}/README.md.tmpl +++ b/contrib/templates/default-scala/template/{{.project_name}}/README.md.tmpl @@ -91,11 +91,10 @@ by running `sbt test`. ### Job configuration {{- if eq .compute_type "serverless"}} -This project uses serverless compute. The serverless environment automatically manages compute resources. +This project uses serverless compute. No cluster setup is required. -If you prefer to use standard compute with job clusters, you can modify the job YAML file to use `job_clusters` instead of the `environments` configuration. {{- else }} -This project uses standard compute with a job cluster. Standard compute requires Unity Catalog volume JAR paths to be allowlisted by your workspace admin. +This project uses standard compute for the job cluster. Standard compute requires Unity Catalog volume JAR paths to be allowlisted by your workspace admin. If you want to use an existing cluster instead, replace `job_cluster_key` in the task configuration with `existing_cluster_id: ` in the job YAML file. From 0f0a93d75c0166829c139bdd07d589f3297f5920 Mon Sep 17 00:00:00 2001 From: Garland Zhang Date: Wed, 5 Nov 2025 14:54:31 +0000 Subject: [PATCH 3/7] improvements --- .../databricks_template_schema.json | 18 +++++++++++++++++- .../library/template_variables.tmpl | 4 ++++ .../template/{{.project_name}}/README.md.tmpl | 2 +- .../{{.project_name}}/databricks.yml.tmpl | 12 ++++++++++++ .../resources/{{.project_name}}.job.yml.tmpl | 10 ++++++++++ .../src/main/scala/com/examples/Main.scala | 13 ++++++++++++- 6 files changed, 56 insertions(+), 3 deletions(-) diff --git a/contrib/templates/default-scala/databricks_template_schema.json b/contrib/templates/default-scala/databricks_template_schema.json index de28ab6f..4101f5eb 100644 --- a/contrib/templates/default-scala/databricks_template_schema.json +++ b/contrib/templates/default-scala/databricks_template_schema.json @@ -17,10 +17,26 @@ }, "artifacts_dest_path": { "type": "string", - "description": "\nPlease provide the Unity Catalog volume destination path in Databricks where the directory will be created containing the JAR and other artifacts to store. [example: /Volumes/abcdef1234567890]{{if eq .compute_type \"standard cluster\"}}\nNote: your admin must allowlist the volume JAR path you specify for your workspace (see https://docs.databricks.com/en/data-governance/unity-catalog/manage-privileges/allowlist.html){{end}}", + "description": "\nPlease provide the Unity Catalog volume destination path in Databricks where the directory will be created containing the JAR and other artifacts to store.{{if eq .compute_type \"standard cluster\"}}\nNote: your admin must allowlist the volume JAR path you specify for your workspace (see https://docs.databricks.com/en/data-governance/unity-catalog/manage-privileges/allowlist.html){{end}}\nartifacts_dest_path [example: /Volumes/abcdef1234567890]", "order": 3, "pattern": "^/Volumes(?:/[a-z0-9_-]+)+/?$", "pattern_match_failure_message": "Please enter a valid path like /Volumes/my-folder/. Only lowercase letters, numbers, dashes, and underscores are allowed in folder names." + }, + "default_catalog": { + "type": "string", + "default": "{{default_catalog}}", + "pattern": "^\\w*$", + "pattern_match_failure_message": "Invalid catalog name.", + "description": "\nPlease provide an initial catalog{{if eq (default_catalog) \"\"}} (leave blank when not using Unity Catalog){{end}}.\ndefault_catalog", + "order": 4 + }, + "default_schema": { + "type": "string", + "default": "default", + "pattern": "^\\w+$", + "pattern_match_failure_message": "Invalid schema name.", + "description": "\nPlease provide an initial schema.\ndefault_schema", + "order": 5 } } } diff --git a/contrib/templates/default-scala/library/template_variables.tmpl b/contrib/templates/default-scala/library/template_variables.tmpl index 94507b31..8363b0bd 100644 --- a/contrib/templates/default-scala/library/template_variables.tmpl +++ b/contrib/templates/default-scala/library/template_variables.tmpl @@ -2,6 +2,10 @@ 0.1 {{- end }} +{{ define `databricks_cli_version` -}} + 0.241.0 +{{- end }} + {{ define `dbr_version` -}} 17.0 {{- end }} diff --git a/contrib/templates/default-scala/template/{{.project_name}}/README.md.tmpl b/contrib/templates/default-scala/template/{{.project_name}}/README.md.tmpl index 231bcce7..3d4792ed 100644 --- a/contrib/templates/default-scala/template/{{.project_name}}/README.md.tmpl +++ b/contrib/templates/default-scala/template/{{.project_name}}/README.md.tmpl @@ -4,7 +4,7 @@ The '{{.project_name}}' project was generated by using the default-scala templat ## Getting started -1. Install the Databricks CLI from https://docs.databricks.com/dev-tools/cli/install.html. The version must be v0.241.0 or later. +1. Install the Databricks CLI from https://docs.databricks.com/dev-tools/cli/install.html. The version must be v{{template `databricks_cli_version` .}} or later. 2. Authenticate to your Databricks workspace (if you have not done so already): ``` diff --git a/contrib/templates/default-scala/template/{{.project_name}}/databricks.yml.tmpl b/contrib/templates/default-scala/template/{{.project_name}}/databricks.yml.tmpl index ffa39be3..77e71f8e 100644 --- a/contrib/templates/default-scala/template/{{.project_name}}/databricks.yml.tmpl +++ b/contrib/templates/default-scala/template/{{.project_name}}/databricks.yml.tmpl @@ -7,6 +7,12 @@ bundle: include: - resources/*.yml +variables: + catalog: + description: The catalog to use + schema: + description: The schema to use + workspace: host: {{workspace_host}} artifact_path: {{.artifacts_dest_path}}/${bundle.name}/${bundle.target}/${workspace.current_user.short_name} @@ -29,6 +35,9 @@ targets: default: true workspace: host: {{workspace_host}} + variables: + catalog: {{.default_catalog}} + schema: {{.default_schema}} prod: mode: production @@ -39,3 +48,6 @@ targets: permissions: - {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}} level: CAN_MANAGE + variables: + catalog: {{.default_catalog}} + schema: {{.default_schema}} diff --git a/contrib/templates/default-scala/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl b/contrib/templates/default-scala/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl index 335a5fde..c38f3e92 100644 --- a/contrib/templates/default-scala/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl +++ b/contrib/templates/default-scala/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl @@ -4,6 +4,11 @@ resources: jobs: {{.project_name}}: name: {{.project_name}} + parameters: + - name: catalog + default: ${var.catalog} + - name: schema + default: ${var.schema} tasks: - task_key: main_task {{- if eq .compute_type "standard cluster"}} @@ -11,6 +16,11 @@ resources: {{- end}} spark_jar_task: main_class_name: {{template `main_class_name` .}} + parameters: + - "--catalog" + - "{{{{job.parameters.catalog}}}}" + - "--schema" + - "{{{{job.parameters.schema}}}}" {{- if eq .compute_type "serverless"}} environment_key: default {{- else}} diff --git a/contrib/templates/default-scala/template/{{.project_name}}/src/main/scala/com/examples/Main.scala b/contrib/templates/default-scala/template/{{.project_name}}/src/main/scala/com/examples/Main.scala index 177bcd00..259c12a1 100644 --- a/contrib/templates/default-scala/template/{{.project_name}}/src/main/scala/com/examples/Main.scala +++ b/contrib/templates/default-scala/template/{{.project_name}}/src/main/scala/com/examples/Main.scala @@ -12,16 +12,27 @@ object Main { def main(args: Array[String]): Unit = { println("Hello, World!") + val catalog = getFromArgs(args, "catalog").getOrElse("samples") + val schema = getFromArgs(args, "schema").getOrElse("nyctaxi") + + println(s"Using catalog: $catalog, schema: $schema") + val spark = getSession() println("Showing range ...") spark.range(3).show() println("Showing nyctaxi trips ...") - val nycTaxi = new NycTaxi(spark) + val nycTaxi = new NycTaxi(spark, catalog, schema) val df = nycTaxi.trips() df.show() } + private def getFromArgs(args: Array[String], key: String): Option[String] = { + args.sliding(2, 2).collectFirst { + case Array(k, v) if k == s"--$key" => v + } + } + def getSession(): SparkSession = { // Get DATABRICKS_RUNTIME_VERSION environment variable if (sys.env.contains("DATABRICKS_RUNTIME_VERSION")) { From f28a7e98d15a77aa3da81415fa6928d9cdb25ea4 Mon Sep 17 00:00:00 2001 From: Garland Zhang Date: Wed, 5 Nov 2025 14:58:41 +0000 Subject: [PATCH 4/7] . --- .../resources/{{.project_name}}.job.yml.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/templates/default-scala/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl b/contrib/templates/default-scala/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl index c38f3e92..32c7066c 100644 --- a/contrib/templates/default-scala/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl +++ b/contrib/templates/default-scala/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl @@ -18,9 +18,9 @@ resources: main_class_name: {{template `main_class_name` .}} parameters: - "--catalog" - - "{{{{job.parameters.catalog}}}}" + - "{{`{{job.parameters.catalog}}`}}" - "--schema" - - "{{{{job.parameters.schema}}}}" + - "{{`{{job.parameters.schema}}`}}" {{- if eq .compute_type "serverless"}} environment_key: default {{- else}} From 0aec12a5cc37a5e0d37badfa5f1bb9d16d02130d Mon Sep 17 00:00:00 2001 From: Garland Zhang Date: Wed, 5 Nov 2025 15:07:09 +0000 Subject: [PATCH 5/7] clean --- .../{{.project_name}}/src/main/scala/com/examples/Main.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/templates/default-scala/template/{{.project_name}}/src/main/scala/com/examples/Main.scala b/contrib/templates/default-scala/template/{{.project_name}}/src/main/scala/com/examples/Main.scala index 259c12a1..27fd45a6 100644 --- a/contrib/templates/default-scala/template/{{.project_name}}/src/main/scala/com/examples/Main.scala +++ b/contrib/templates/default-scala/template/{{.project_name}}/src/main/scala/com/examples/Main.scala @@ -22,7 +22,7 @@ object Main { spark.range(3).show() println("Showing nyctaxi trips ...") - val nycTaxi = new NycTaxi(spark, catalog, schema) + val nycTaxi = new NycTaxi(spark) val df = nycTaxi.trips() df.show() } From c851817b68e3531c56abfed787308aba7bd51309 Mon Sep 17 00:00:00 2001 From: Garland Zhang Date: Thu, 6 Nov 2025 16:31:59 +0000 Subject: [PATCH 6/7] cleanup --- .../databricks_template_schema.json | 22 ++++++++++++++--- .../library/template_variables.tmpl | 1 + .../{{.project_name}}/databricks.yml.tmpl | 11 +++++++-- .../src/main/scala/com/examples/Main.scala | 24 ++++++++++++------- 4 files changed, 44 insertions(+), 14 deletions(-) diff --git a/contrib/templates/default-scala/databricks_template_schema.json b/contrib/templates/default-scala/databricks_template_schema.json index 4101f5eb..9d239768 100644 --- a/contrib/templates/default-scala/databricks_template_schema.json +++ b/contrib/templates/default-scala/databricks_template_schema.json @@ -30,13 +30,29 @@ "description": "\nPlease provide an initial catalog{{if eq (default_catalog) \"\"}} (leave blank when not using Unity Catalog){{end}}.\ndefault_catalog", "order": 4 }, - "default_schema": { + "personal_schemas": { + "type": "string", + "description": "\nWould you like to use a personal schema for each user working on this project? (e.g., 'catalog.{{short_name}}')\npersonal_schemas", + "enum": [ + "yes, use a schema based on the current user name during development", + "no, use a shared schema during development" + ], + "order": 5 + }, + "shared_schema": { + "skip_prompt_if": { + "properties": { + "personal_schemas": { + "const": "yes, use a schema based on the current user name during development" + } + } + }, "type": "string", "default": "default", "pattern": "^\\w+$", "pattern_match_failure_message": "Invalid schema name.", - "description": "\nPlease provide an initial schema.\ndefault_schema", - "order": 5 + "description": "\nPlease provide an initial schema during development.\ndefault_schema", + "order": 6 } } } diff --git a/contrib/templates/default-scala/library/template_variables.tmpl b/contrib/templates/default-scala/library/template_variables.tmpl index 8363b0bd..136630ed 100644 --- a/contrib/templates/default-scala/library/template_variables.tmpl +++ b/contrib/templates/default-scala/library/template_variables.tmpl @@ -1,3 +1,4 @@ +{{/* Project artifact version used in JAR naming */}} {{ define `version` -}} 0.1 {{- end }} diff --git a/contrib/templates/default-scala/template/{{.project_name}}/databricks.yml.tmpl b/contrib/templates/default-scala/template/{{.project_name}}/databricks.yml.tmpl index 77e71f8e..bfd50a10 100644 --- a/contrib/templates/default-scala/template/{{.project_name}}/databricks.yml.tmpl +++ b/contrib/templates/default-scala/template/{{.project_name}}/databricks.yml.tmpl @@ -13,6 +13,13 @@ variables: schema: description: The schema to use +{{- $dev_schema := .shared_schema }} +{{- $prod_schema := .shared_schema }} +{{- if (regexp "^yes").MatchString .personal_schemas}} + {{- $dev_schema = "${workspace.current_user.short_name}"}} + {{- $prod_schema = "default"}} +{{- end}} + workspace: host: {{workspace_host}} artifact_path: {{.artifacts_dest_path}}/${bundle.name}/${bundle.target}/${workspace.current_user.short_name} @@ -37,7 +44,7 @@ targets: host: {{workspace_host}} variables: catalog: {{.default_catalog}} - schema: {{.default_schema}} + schema: {{$dev_schema}} prod: mode: production @@ -50,4 +57,4 @@ targets: level: CAN_MANAGE variables: catalog: {{.default_catalog}} - schema: {{.default_schema}} + schema: {{$prod_schema}} diff --git a/contrib/templates/default-scala/template/{{.project_name}}/src/main/scala/com/examples/Main.scala b/contrib/templates/default-scala/template/{{.project_name}}/src/main/scala/com/examples/Main.scala index 27fd45a6..3d380e90 100644 --- a/contrib/templates/default-scala/template/{{.project_name}}/src/main/scala/com/examples/Main.scala +++ b/contrib/templates/default-scala/template/{{.project_name}}/src/main/scala/com/examples/Main.scala @@ -12,19 +12,25 @@ object Main { def main(args: Array[String]): Unit = { println("Hello, World!") - val catalog = getFromArgs(args, "catalog").getOrElse("samples") - val schema = getFromArgs(args, "schema").getOrElse("nyctaxi") - - println(s"Using catalog: $catalog, schema: $schema") - val spark = getSession() + + // Process command-line arguments and set catalog/schema if provided + setCatalogAndSchema(spark, args) + println("Showing range ...") spark.range(3).show() + } - println("Showing nyctaxi trips ...") - val nycTaxi = new NycTaxi(spark) - val df = nycTaxi.trips() - df.show() + private def setCatalogAndSchema(spark: SparkSession, args: Array[String]): Unit = { + getFromArgs(args, "catalog").foreach { catalog => + spark.sql(s"USE CATALOG $catalog") + println(s"Using catalog: $catalog") + } + + getFromArgs(args, "schema").foreach { schema => + spark.sql(s"USE SCHEMA $schema") + println(s"Using schema: $schema") + } } private def getFromArgs(args: Array[String], key: String): Option[String] = { From 5c664e6e8f54fb78d4410f3c61a83d7bad4e9d31 Mon Sep 17 00:00:00 2001 From: Garland Zhang Date: Thu, 6 Nov 2025 16:38:55 +0000 Subject: [PATCH 7/7] remove reedundant --- .../{{.project_name}}/src/main/scala/com/examples/Main.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/contrib/templates/default-scala/template/{{.project_name}}/src/main/scala/com/examples/Main.scala b/contrib/templates/default-scala/template/{{.project_name}}/src/main/scala/com/examples/Main.scala index 3d380e90..f2605645 100644 --- a/contrib/templates/default-scala/template/{{.project_name}}/src/main/scala/com/examples/Main.scala +++ b/contrib/templates/default-scala/template/{{.project_name}}/src/main/scala/com/examples/Main.scala @@ -14,9 +14,8 @@ object Main { val spark = getSession() - // Process command-line arguments and set catalog/schema if provided setCatalogAndSchema(spark, args) - +git println("Showing range ...") spark.range(3).show() }