diff --git a/docs/website-landing-copy.md b/docs/website-landing-copy.md new file mode 100644 index 0000000..e024e90 --- /dev/null +++ b/docs/website-landing-copy.md @@ -0,0 +1,105 @@ +# Landing page copy for [testbricks.netlify.app](https://testbricks.netlify.app/) + +Drop-in files for `engineeringmadness/testbricks-website` (this library repo cannot push that repository): + +- `docs/website/index.html` → website `index.html` +- `docs/website/App.tsx` → website `src/App.tsx` + +The live site still uses the old `SparkMock` name in examples. Marketing copy stays high-level; the code tabs show the current API. + +## Meta (`index.html`) + +**title:** `testbricks — Run Databricks workflows locally` + +**description:** + +> testbricks runs Databricks workflows on your laptop with a Spark proxy, a drop-in dbutils, and a workflow runner. + +**og:description:** + +> Run Databricks workflows on your laptop with SparkProxy, a dbutils mock, and a JSON workflow runner. + +## Hero + +**h1:** Run Databricks Workflows E2E / **in your Local Environment** (unchanged) + +**body:** + +> testbricks is a Python library with genuinely useful mocks — a Spark proxy that reads and writes tables as CSV, a drop-in `dbutils` replacement, and a runner that executes a whole workflow JSON in dependency order. No cluster. No waiting around. + +**install:** `pip install testbricks` + +**fine print:** `Python 3.10+ · works in notebooks, scripts and CI` + +## Features + +**SparkProxy** + +> A SparkSession stand-in that speaks the same API. Table reads and writes land as CSV files on disk — no cluster, no metastore, no waiting. + +**Drop-in dbutils** + +> Widgets, filesystem helpers, secrets, and more. Import it instead of the real thing and your notebook runs unchanged. + +**Workflow runner** + +> Parses a Databricks workflow JSON, builds the task graph, and runs notebooks in the right order on your machine. + +**Zero cluster time** + +> Iterate in seconds on your laptop. Debug with breakpoints, run it in CI, and keep your compute bill for the things that matter. + +## How it works + +1. **Swap in the mocks** — Replace the Spark session and dbutils object with the testbricks equivalents at the top of your notebook. +2. **Drop your data in a folder** — CSV files in a base directory stand in for your tables. Read, write and inspect them with any tool you like. +3. **Run the whole workflow** — Hand the runner your workflow JSON and it walks the dependency graph, notebook by notebook, right on your machine. + +## Code examples (`SNIPPETS` in `App.tsx`) + +First tab label: `SparkProxy` (not `SparkMock`). + +### SparkProxy + +**blurb:** Point it at a folder. Table reads and writes land as plain CSV files you can open anywhere. + +```python +from testbricks import SparkProxy + +spark = SparkProxy("./data") + +df = spark.read.option("header", "true").option("inferSchema", "true").table("bronze.customers") +df.write.mode("overwrite").saveAsTable("silver.customers_enriched") +``` + +### dbutils + +**blurb:** A drop-in replacement for the Databricks dbutils object. Notebooks the runner executes get it automatically. + +```python +from testbricks.dbutils import configure, dbutils + +configure("./data") # same catalog root as SparkProxy + +dbutils.widgets.text("filter_country", "USA") +country = dbutils.widgets.get("filter_country") + +for info in dbutils.fs.ls("/"): + print(info.name, info.size) +``` + +### LocalWorkflowRunner + +**blurb:** Feed it your exported workflow JSON. It resolves the task graph and runs every notebook in dependency order. + +```python +from testbricks import SparkProxy, LocalWorkflowRunner + +spark = SparkProxy("./data") +runner = LocalWorkflowRunner( + source_dir="./notebooks", + workflow_json_path="./workflow.json", + base_path="./data", +) +runner.run_workflow(extra_globals={"spark": spark}) +``` diff --git a/docs/website/App.tsx b/docs/website/App.tsx new file mode 100644 index 0000000..298d86f --- /dev/null +++ b/docs/website/App.tsx @@ -0,0 +1,464 @@ +import { useState } from "react"; +import { Highlight, Prism, type PrismTheme } from "prism-react-renderer"; +import { + Check, + Copy, + Database, + FileSpreadsheet, + Github, + Laptop, + Package, + Star, + Waves, + Workflow, +} from "lucide-react"; + +const GITHUB_URL = "https://github.com/engineeringmadness/testbricks"; +const PYPI_URL = "https://pypi.org/project/testbricks/"; + +/* ---------------------------------- data --------------------------------- */ + +type Snippet = { + id: string; + label: string; + blurb: string; + code: string; +}; + +const SNIPPETS: Snippet[] = [ + { + id: "spark", + label: "SparkProxy", + blurb: + "Point it at a folder. Table reads and writes land as plain CSV files you can open anywhere.", + code: `from testbricks import SparkProxy + +spark = SparkProxy("./data") + +df = spark.read.option("header", "true").option("inferSchema", "true").table("bronze.customers") +df.write.mode("overwrite").saveAsTable("silver.customers_enriched")`, + }, + { + id: "dbutils", + label: "dbutils", + blurb: + "A drop-in replacement for the Databricks dbutils object. Notebooks the runner executes get it automatically.", + code: `from testbricks.dbutils import configure, dbutils + +configure("./data") # same catalog root as SparkProxy + +dbutils.widgets.text("filter_country", "USA") +country = dbutils.widgets.get("filter_country") + +for info in dbutils.fs.ls("/"): + print(info.name, info.size)`, + }, + { + id: "runner", + label: "LocalWorkflowRunner", + blurb: + "Feed it your exported workflow JSON. It resolves the task graph and runs every notebook in dependency order.", + code: `from testbricks import SparkProxy, LocalWorkflowRunner + +spark = SparkProxy("./data") +runner = LocalWorkflowRunner( + source_dir="./notebooks", + workflow_json_path="./workflow.json", + base_path="./data", +) +runner.run_workflow(extra_globals={"spark": spark})`, + }, +]; + +const FEATURES = [ + { + icon: Database, + title: "SparkProxy", + body: "A SparkSession stand-in that speaks the same API. Table reads and writes land as CSV files on disk — no cluster, no metastore, no waiting.", + }, + { + icon: FileSpreadsheet, + title: "Drop-in dbutils", + body: "Widgets, filesystem helpers, secrets, and more. Import it instead of the real thing and your notebook runs unchanged.", + }, + { + icon: Workflow, + title: "Workflow runner", + body: "Parses a Databricks workflow JSON, builds the task graph, and runs notebooks in the right order on your machine.", + }, + { + icon: Laptop, + title: "Zero cluster time", + body: "Iterate in seconds on your laptop. Debug with breakpoints, run it in CI, and keep your compute bill for the things that matter.", + }, +]; + +const STEPS = [ + { + n: "01", + title: "Swap in the mocks", + body: "Replace the Spark session and dbutils object with the testbricks equivalents at the top of your notebook.", + }, + { + n: "02", + title: "Drop your data in a folder", + body: "CSV files in a base directory stand in for your tables. Read, write and inspect them with any tool you like.", + }, + { + n: "03", + title: "Run the whole workflow", + body: "Hand the runner your workflow JSON and it walks the dependency graph, notebook by notebook, right on your machine.", + }, +]; + +/* -------------------------------- helpers -------------------------------- */ + +function useCopy() { + const [copied, setCopied] = useState(null); + + function copy(value: string, key: string) { + void navigator.clipboard.writeText(value).then(() => { + setCopied(key); + window.setTimeout(() => setCopied(null), 1800); + }); + } + + return { copied, copy }; +} + +/* ------------------------------ code theme ------------------------------- */ + +// Prism's Python grammar only styles `def`/`class` names, so extend it to also +// highlight function/method calls (and constructor calls like `SparkProxy(...)`). +// Inserted after `keyword` so keywords such as `from`/`import` still take priority. +Prism.languages.insertBefore("python", "builtin", { + "function-call": { + pattern: /\b[A-Za-z_][A-Za-z0-9_]*(?=\s*\()/, + alias: "function", + }, +}); + +// Dark editor palette drawn from the Sandy Shore design tokens (teal, coral, sand). +const CODE_THEME: PrismTheme = { + plain: { + color: "#d7e5e2", + backgroundColor: "#0c1a1a", + }, + styles: [ + { + types: ["comment", "prolog", "cdata"], + style: { color: "#5f7a76", fontStyle: "italic" }, + }, + { + types: ["string", "char", "attr-value", "string-interpolation"], + style: { color: "#8fd6c5" }, + }, + { + types: ["keyword", "atrule"], + style: { color: "#f4a261" }, + }, + { + types: ["boolean", "constant"], + style: { color: "#ff9e7d" }, + }, + { + types: ["number"], + style: { color: "#ffb36b" }, + }, + { + types: ["function", "class-name", "builtin", "maybe-class-name"], + style: { color: "#ffd166" }, + }, + { + types: ["operator", "punctuation"], + style: { color: "#93a6a1" }, + }, + ], +}; + +function CodeBlock({ code }: { code: string }) { + return ( + + {({ style, tokens, getLineProps, getTokenProps }) => ( + 
+          {tokens.map((line, i) => (
+            

+              {line.map((token, key) => (
+                
+              ))}
+            

+          ))}
+
+ )} + + ); +} + +/* -------------------------------- sections ------------------------------- */ + +function Nav() { + return ( +
+ +
+ ); +} + +function Hero() { + const { copied, copy } = useCopy(); + const install = "pip install testbricks"; + + return ( +
+
+

+ Run Databricks Workflows E2E + in your Local Environment +

+ +

+ testbricks is a Python library with genuinely useful mocks — a Spark proxy that reads and + writes tables as CSV, a drop-in{" "} + dbutils replacement, and + a runner that executes a whole workflow JSON in dependency order. No cluster. No waiting + around. +

+ +
+ + + + See it in action + +
+ +

+ Python 3.10+ · works in notebooks, scripts and CI +

+
+
+ ); +} + +function Features() { + return ( +
+
+

+ Mocks that actually hold up +

+

+ Everything your notebook reaches for in a Databricks runtime, quietly reimplemented for a + machine that fits on your desk. +

+ +
+ {FEATURES.map((f) => ( +
+ + + +

{f.title}

+

{f.body}

+
+ ))} +
+
+
+ ); +} + +function CodeSection() { + const [active, setActive] = useState(SNIPPETS[0].id); + const { copied, copy } = useCopy(); + const snippet = SNIPPETS.find((s) => s.id === active) ?? SNIPPETS[0]; + + return ( +
+
+

+ Three imports, and you're local +

+

+ The same notebook code you'd ship to a job cluster, running on your machine. +

+ +
+
+ {SNIPPETS.map((s) => ( + + ))} +
+ +
+

{snippet.blurb}

+ +
+ + +
+
+
+ ); +} + +function HowItWorks() { + return ( +
+
+

How it works

+ +
    + {STEPS.map((s) => ( +
  1. + {s.n} +

    {s.title}

    +

    {s.body}

    +
    2. + ))} +
+ +
+ + + +

+ Love Testbricks? +

+
+ + + Star on GitHub + +
+
+
+
+ ); +} + +function Footer() { + return ( + + ); +} + +export default function App() { + return ( +
+
+ ); +} diff --git a/docs/website/index.html b/docs/website/index.html new file mode 100644 index 0000000..a8d4611 --- /dev/null +++ b/docs/website/index.html @@ -0,0 +1,25 @@ + + + + + + + testbricks — Run Databricks workflows locally + + + + + + + + +
+ + +