Add pyproject.toml as project definition format

airflow/airflow.go

@@ -90,6 +90,9 @@ var (
 
 	//go:embed include/airflow3/requirements-client.txt
 	Af3RequirementsTxtClient string
+
+	//go:embed include/airflow3/pyprojecttoml
+	Af3PyProjectTOML string
 )
 
 func initDirs(root string, dirs []string) error {
@@ -208,6 +211,35 @@ func Init(path, airflowImageName, airflowImageTag, template, clientImageTag stri
 	return nil
 }
 
+// InitPyProject scaffolds a new Airflow project using pyproject.toml as the
+// project definition instead of a Dockerfile. Only supported for Airflow 3.
+func InitPyProject(path, projectName, airflowVersion, runtimeVersion, pythonVersion string) error {
+	dirs := []string{"dags", "plugins", "include"}
+	if err := initDirs(path, dirs); err != nil {
+		return errors.Wrap(err, "failed to create project directories")
+	}
+
+	files := map[string]string{
+		"pyproject.toml":                       fmt.Sprintf(Af3PyProjectTOML, projectName, pythonVersion, airflowVersion, runtimeVersion),
+		".gitignore":                           Af3Gitignore,
+		".dockerignore":                        Af3Dockerignore,
+		".env":                                 "",
+		"airflow_settings.yaml":                Af3Settingsyml,
+		"packages.txt":                         "",
+		"requirements.txt":                     Af3RequirementsTxt,
+		"dags/exampledag.py":                   Af3ExampleDag,
+		"dags/.airflowignore":                  "",
+		"tests/dags/test_dag_example.py":       Af3DagExampleTest,
+		".astro/test_dag_integrity_default.py": Af3DagIntegrityTestDefault,
+		".astro/dag_integrity_exceptions.txt":  "# Add dag files to exempt from parse test below. ex: dags/<test-file>",
+	}
+	if err := initFiles(path, files); err != nil {
+		return errors.Wrap(err, "failed to create project files")
+	}
+
+	return nil
+}
+
 // repositoryName creates an airflow repository name
 func repositoryName(name string) string {
 	return fmt.Sprintf("%s/%s", name, componentName)

airflow/airflow_test.go

@@ -172,6 +172,72 @@ func (s *Suite) TestInitWithoutClientImageTag() {
 	}
 }
 
+func (s *Suite) TestInitPyProject() {
+	tmpDir, err := os.MkdirTemp("", "temp")
+	s.Require().NoError(err)
+	defer os.RemoveAll(tmpDir)
+
+	err = InitPyProject(tmpDir, "my-project", "3.0.1", "3.0-7", "3.12")
+	s.NoError(err)
+
+	// Files that SHOULD exist
+	expectedFiles := []string{
+		"pyproject.toml",
+		".gitignore",
+		".env",
+		"airflow_settings.yaml",
+		"dags/exampledag.py",
+		"dags/.airflowignore",
+		"tests/dags/test_dag_example.py",
+		".astro/test_dag_integrity_default.py",
+		".astro/dag_integrity_exceptions.txt",
+	}
+	for _, file := range expectedFiles {
+		exist, err := fileutil.Exists(filepath.Join(tmpDir, file), nil)
+		s.NoError(err)
+		s.True(exist, "Expected file %s to exist", file)
+	}
+
+	// Files that should NOT exist (Dockerfile-format-specific)
+	dockerFiles := []string{
+		"Dockerfile",
+		"README.md",
+	}
+	for _, file := range dockerFiles {
+		exist, err := fileutil.Exists(filepath.Join(tmpDir, file), nil)
+		s.NoError(err)
+		s.False(exist, "Expected file %s to NOT exist", file)
+	}
+
+	// Verify pyproject.toml content
+	content, err := os.ReadFile(filepath.Join(tmpDir, "pyproject.toml"))
+	s.NoError(err)
+	s.Contains(string(content), `name = "my-project"`)
+	s.Contains(string(content), `requires-python = ">=3.12"`)
+	s.Contains(string(content), `airflow-version = "3.0.1"`)
+	s.Contains(string(content), `runtime-version = "3.0-7"`)
+	s.NotContains(string(content), "Dockerfile")
+}
+
+func (s *Suite) TestInitPyProject_CanBeReadBack() {
+	tmpDir, err := os.MkdirTemp("", "temp")
+	s.Require().NoError(err)
+	defer os.RemoveAll(tmpDir)
+
+	err = InitPyProject(tmpDir, "roundtrip-test", "3.0.1", "3.0-7", "3.12")
+	s.NoError(err)
+
+	// Read it back with our parser from Phase 1
+	proj, err := ReadProject(tmpDir)
+	s.NoError(err)
+	s.Equal("roundtrip-test", proj.Name)
+	s.Equal(">=3.12", proj.RequiresPython)
+	s.Equal("3.0.1", proj.AirflowVersion)
+	s.Equal("3.0-7", proj.RuntimeVersion)
+	s.Equal("docker", proj.Mode) // default when not specified in template
+	s.Empty(proj.Dependencies)   // empty list in template
+}
+
 func (s *Suite) TestTemplateInitFail() {
 	ExtractTemplate = func(templateDir, destDir string) error {
 		err := errors.New("error extracting files")

airflow/docker.go

@@ -245,6 +245,12 @@ func (d *DockerCompose) Start(opts *airflowTypes.StartOptions) error {
 	envConns := opts.EnvConns
 	useProxy := !opts.NoProxy
 
+	// Resolve Dockerfile: for pyproject.toml projects, generate from project definition.
+	dockerfile, resolveErr := EnsureDockerfile(d.airflowHome, d.dockerfile)
+	if resolveErr != nil {
+		return fmt.Errorf("error resolving Dockerfile: %w", resolveErr)
+	}
+
 	// Build this project image
 	if imageName == "" {
 		if !config.CFG.DisableAstroRun.GetBool() {
@@ -254,7 +260,7 @@ func (d *DockerCompose) Start(opts *airflowTypes.StartOptions) error {
 				fmt.Printf("Adding 'astro-run-dag' package to requirements.txt unsuccessful: %s\nManually add package to requirements.txt", err.Error())
 			}
 		}
-		imageBuildErr := d.imageHandler.Build(d.dockerfile, buildSecretString, airflowTypes.ImageBuildConfig{Path: d.airflowHome, NoCache: noCache})
+		imageBuildErr := d.imageHandler.Build(dockerfile, buildSecretString, airflowTypes.ImageBuildConfig{Path: d.airflowHome, NoCache: noCache})
 		if !config.CFG.DisableAstroRun.GetBool() {
 			// remove astro-run-dag from requirments.txt
 			err := fileutil.RemoveLineFromFile("./requirements.txt", "astro-run-dag", " # This package is needed for the astro run command. It will be removed before a deploy")
@@ -1329,8 +1335,14 @@ func (d *DockerCompose) Build(customImageName, buildSecretString string, noCache
 		return d.imageHandler.TagLocalImage(customImageName)
 	}
 
-	// Build the image
-	return d.imageHandler.Build(d.dockerfile, buildSecretString, airflowTypes.ImageBuildConfig{
+	// Resolve Dockerfile: for pyproject.toml projects, generate from project definition.
+	// Done at build time (not init) so changes to pyproject.toml are always picked up.
+	dockerfile, err := EnsureDockerfile(d.airflowHome, d.dockerfile)
+	if err != nil {
+		return fmt.Errorf("error resolving Dockerfile: %w", err)
+	}
+
+	return d.imageHandler.Build(dockerfile, buildSecretString, airflowTypes.ImageBuildConfig{
 		Path:    d.airflowHome,
 		NoCache: noCache,
 	})

airflow/docker_image.go

@@ -99,7 +99,12 @@ func (d *DockerImage) Build(dockerfilePath, buildSecretString string, buildConfi
 		return err
 	}
 	if dockerfilePath == "" {
-		dockerfilePath = "Dockerfile"
+		// For pyproject.toml projects, generate a Dockerfile from the project definition.
+		resolved, resolveErr := EnsureDockerfile(buildConfig.Path, "Dockerfile")
+		if resolveErr != nil {
+			return fmt.Errorf("error resolving Dockerfile: %w", resolveErr)
+		}
+		dockerfilePath = resolved
 	}
 	args := []string{"build"}
 	addPullFlag, err := shouldAddPullFlag(dockerfilePath)

airflow/dockerfile_gen.go

@@ -0,0 +1,118 @@
+package airflow
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	airflowversions "github.com/astronomer/astro-cli/airflow_versions"
+)
+
+const (
+	generatedDockerfileComment = "# Auto-generated from pyproject.toml — do not edit.\n# Changes should be made in pyproject.toml.\n"
+	genDirPerm                 = os.FileMode(0o755) //nolint:mnd
+	genFilePerm                = os.FileMode(0o644) //nolint:mnd
+	generatedDockerfileName    = "Dockerfile.pyproject"
+)
+
+// validDebPkgRe matches valid Debian package names: starts with alnum, then alnum/./+/-
+var validDebPkgRe = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9.+\-]*$`)
+
+// GenerateDockerfile produces Dockerfile content from an AstroProject.
+// Returns an error if the project has invalid fields.
+func GenerateDockerfile(project *AstroProject) (string, error) {
+	if project.RuntimeVersion == "" {
+		return "", fmt.Errorf("runtime-version is required to generate a Dockerfile")
+	}
+
+	for _, pkg := range project.SystemPackages {
+		if !validDebPkgRe.MatchString(pkg) {
+			return "", fmt.Errorf("invalid system package name %q: must match Debian package naming rules", pkg)
+		}
+	}
+
+	var b strings.Builder
+
+	b.WriteString(generatedDockerfileComment)
+
+	imageName := AstroRuntimeAirflow3ImageName
+	registry := AstroImageRegistryBaseImageName
+	b.WriteString(fmt.Sprintf("FROM %s/%s:%s\n", registry, imageName, project.RuntimeVersion))
+
+	if len(project.SystemPackages) > 0 {
+		b.WriteString("USER root\n")
+		b.WriteString(fmt.Sprintf(
+			"RUN apt-get update && apt-get install -y --no-install-recommends %s && rm -rf /var/lib/apt/lists/*\n",
+			strings.Join(project.SystemPackages, " "),
+		))
+		b.WriteString("USER astro\n")
+	}
+
+	return b.String(), nil
+}
+
+// EnsureDockerfile checks if the project uses pyproject.toml and generates
+// a Dockerfile in .astro/ if needed. Returns the path to the Dockerfile to use
+// (either the generated one or the original).
+func EnsureDockerfile(airflowHome, originalDockerfile string) (string, error) {
+	proj, found, err := TryReadProject(airflowHome)
+	if !found {
+		return originalDockerfile, nil
+	}
+	if err != nil {
+		return "", fmt.Errorf("error reading pyproject.toml: %w", err)
+	}
+
+	// If a hand-written Dockerfile exists in the project root, use it instead
+	// of generating one. This supports users who need custom Docker steps
+	// beyond what pyproject.toml can express (the "eject" pattern).
+	dockerfilePath := filepath.Join(airflowHome, "Dockerfile")
+	if _, err := os.Stat(dockerfilePath); err == nil {
+		return "Dockerfile", nil
+	}
+
+	// Resolve runtime-version from airflow-version if missing, and pin it
+	if proj.RuntimeVersion == "" {
+		if proj.AirflowVersion == "" {
+			return "", fmt.Errorf("[tool.astro] requires airflow-version in pyproject.toml")
+		}
+		resolved := airflowversions.GetLatestRuntimeForAirflow(proj.AirflowVersion)
+		if resolved == "" {
+			return "", fmt.Errorf("could not resolve a runtime version for airflow-version %q", proj.AirflowVersion)
+		}
+		fmt.Printf("Resolved runtime-version %q for airflow-version %q (pinning to pyproject.toml)\n", resolved, proj.AirflowVersion)
+		fmt.Println("Note: one airflow-version may have multiple runtime versions. Pin runtime-version in pyproject.toml to avoid accidental upgrades on deploy.")
+		if pinErr := PinRuntimeVersion(airflowHome, resolved); pinErr != nil {
+			fmt.Printf("Warning: could not pin runtime-version to pyproject.toml: %s\n", pinErr)
+		}
+		proj.RuntimeVersion = resolved
+	}
+
+	// Validate airflow-version matches runtime-version
+	if proj.AirflowVersion != "" {
+		actual := airflowversions.GetAirflowVersionForRuntime(proj.RuntimeVersion)
+		if actual != "" && actual != proj.AirflowVersion {
+			fmt.Printf("Warning: airflow-version %q in pyproject.toml does not match runtime-version %q (which bundles Airflow %s). Consider updating airflow-version or runtime-version.\n",
+				proj.AirflowVersion, proj.RuntimeVersion, actual)
+		}
+	}
+
+	content, err := GenerateDockerfile(proj)
+	if err != nil {
+		return "", err
+	}
+
+	genDir := filepath.Join(airflowHome, ".astro")
+	if err := os.MkdirAll(genDir, genDirPerm); err != nil {
+		return "", fmt.Errorf("error creating .astro directory: %w", err)
+	}
+
+	genPath := filepath.Join(genDir, generatedDockerfileName)
+	if err := os.WriteFile(genPath, []byte(content), genFilePerm); err != nil {
+		return "", fmt.Errorf("error writing generated Dockerfile: %w", err)
+	}
+
+	return genPath, nil
+}