fix: add dag-folder input and fix root-folder path matching

.github/workflows/e2e/get_astro_env_info/action.yaml

@@ -43,32 +43,37 @@ runs:
       shell: bash
       id: get-info
       run: |
+        WORKSPACE_ID="${{ inputs.secret_workspace_id }}"
         if [ "${{ inputs.input_workspace_id }}" != "" ]; then
           echo "Using provided workspace_id"
-          echo "WORKSPACE_ID=${{ inputs.input_workspace_id }}" >> $GITHUB_OUTPUT
-        else
-          echo "WORKSPACE_ID=${{ inputs.secret_workspace_id }}" >> $GITHUB_OUTPUT
+          WORKSPACE_ID="${{ inputs.input_workspace_id }}"
         fi
 
+        ORGANIZATION_ID="${{ inputs.secret_organization_id }}"
         if [ "${{ inputs.input_organization_id }}" != "" ]; then
           echo "Using provided org_id"
-          echo "ORGANIZATION_ID=${{ inputs.input_organization_id }}" >> $GITHUB_OUTPUT
-        else
-          echo "ORGANIZATION_ID=${{ inputs.secret_organization_id }}" >> $GITHUB_OUTPUT
+          ORGANIZATION_ID="${{ inputs.input_organization_id }}"
         fi
 
+        ASTRONOMER_HOST="${{ inputs.secret_astronomer_host }}"
         if [ "${{ inputs.input_astronomer_host }}" != "" ]; then
           echo "Using provided astronomer_host"
-          echo "ASTRONOMER_HOST=${{ inputs.input_astronomer_host }}" >> $GITHUB_OUTPUT
-        else
-          echo "ASTRONOMER_HOST=${{ inputs.secret_astronomer_host }}" >> $GITHUB_OUTPUT
+          ASTRONOMER_HOST="${{ inputs.input_astronomer_host }}"
         fi
 
+        ASTRO_API_TOKEN="${{ inputs.secret_astro_api_token }}"
         if [ "${{ inputs.input_astro_api_token }}" != "" ]; then
           echo "Using provided token"
-          echo "ASTRO_API_TOKEN=${{ inputs.input_astro_api_token }}" >> $GITHUB_OUTPUT
-          echo "ASTRO_API_TOKEN=${{ inputs.input_astro_api_token }}" >> $GITHUB_ENV
-        else
-          echo "ASTRO_API_TOKEN=${{ inputs.secret_astro_api_token }}" >> $GITHUB_OUTPUT
-          echo "ASTRO_API_TOKEN=${{ inputs.secret_astro_api_token }}" >> $GITHUB_ENV
+          ASTRO_API_TOKEN="${{ inputs.input_astro_api_token }}"
         fi
+
+        if [ "$WORKSPACE_ID" = "" ] || [ "$ORGANIZATION_ID" = "" ] || [ "$ASTRONOMER_HOST" = "" ] || [ "$ASTRO_API_TOKEN" = "" ]; then
+          echo "Missing Astro test configuration. Provide workflow inputs or make WORKSPACE_ID, ORGANIZATION_ID, ASTRONOMER_HOST, and ASTRO_API_TOKEN available to this job." >&2
+          exit 1
+        fi
+
+        echo "WORKSPACE_ID=$WORKSPACE_ID" >> $GITHUB_OUTPUT
+        echo "ORGANIZATION_ID=$ORGANIZATION_ID" >> $GITHUB_OUTPUT
+        echo "ASTRONOMER_HOST=$ASTRONOMER_HOST" >> $GITHUB_OUTPUT
+        echo "ASTRO_API_TOKEN=$ASTRO_API_TOKEN" >> $GITHUB_OUTPUT
+        echo "ASTRO_API_TOKEN=$ASTRO_API_TOKEN" >> $GITHUB_ENV

.github/workflows/tests.yaml

@@ -29,11 +29,46 @@ jobs:
           TOKEN=$(jq -r '.inputs.token' $GITHUB_EVENT_PATH)
           echo ::add-mask::$TOKEN
 
+  check-astro-test-config:
+    name: Check Astro Test Config
+    runs-on: ubuntu-latest
+    environment: e2e-test
+    outputs:
+      has_astro_test_config: ${{ steps.check.outputs.has_astro_test_config }}
+    steps:
+      - name: Check Astro test configuration
+        id: check
+        env:
+          SECRET_WORKSPACE_ID: ${{ secrets.WORKSPACE_ID }}
+          SECRET_ORGANIZATION_ID: ${{ secrets.ORGANIZATION_ID }}
+          SECRET_ASTRONOMER_HOST: ${{ secrets.ASTRONOMER_HOST }}
+          SECRET_ASTRO_API_TOKEN: ${{ secrets.ASTRO_API_TOKEN }}
+        run: |
+          INPUT_WORKSPACE_ID=$(jq -r '.inputs.workspace_id // empty' "$GITHUB_EVENT_PATH")
+          INPUT_ORGANIZATION_ID=$(jq -r '.inputs.org_id // empty' "$GITHUB_EVENT_PATH")
+          INPUT_ASTRONOMER_HOST=$(jq -r '.inputs.astronomer_host // empty' "$GITHUB_EVENT_PATH")
+          INPUT_ASTRO_API_TOKEN=$(jq -r '.inputs.token // empty' "$GITHUB_EVENT_PATH")
+
+          WORKSPACE_ID="${INPUT_WORKSPACE_ID:-$SECRET_WORKSPACE_ID}"
+          ORGANIZATION_ID="${INPUT_ORGANIZATION_ID:-$SECRET_ORGANIZATION_ID}"
+          ASTRONOMER_HOST="${INPUT_ASTRONOMER_HOST:-$SECRET_ASTRONOMER_HOST}"
+          ASTRO_API_TOKEN="${INPUT_ASTRO_API_TOKEN:-$SECRET_ASTRO_API_TOKEN}"
+
+          if [[ -n "$WORKSPACE_ID" && -n "$ORGANIZATION_ID" && -n "$ASTRONOMER_HOST" && -n "$ASTRO_API_TOKEN" ]]; then
+            echo "Astro test configuration is available."
+            echo "has_astro_test_config=true" >> $GITHUB_OUTPUT
+          else
+            echo "Astro test configuration is unavailable for this run. Skipping e2e jobs."
+            echo "has_astro_test_config=false" >> $GITHUB_OUTPUT
+          fi
+
   # Create test deployments would use the template files and generate deployments with unique names
   create-test-deployments:
     name: Create Deployment
-    needs: redact-inputs
+    if: ${{ needs.check-astro-test-config.outputs.has_astro_test_config == 'true' }}
+    needs: [redact-inputs, check-astro-test-config]
     runs-on: ubuntu-latest
+    environment: e2e-test
     strategy:
       matrix:
         deployment: [deployment-hibernate.yaml, deployment.yaml]
@@ -59,11 +94,10 @@ jobs:
 
       - name: Install dependencies
         run: |
-
-          wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && chmod +x /usr/bin/yq
-
           mkdir -p "$HOME/.local/bin"
           echo "$HOME/.local/bin" >> $GITHUB_PATH
+          curl -fsSL https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -o "$HOME/.local/bin/yq"
+          chmod +x "$HOME/.local/bin/yq"
           curl -sSL https://raw.githubusercontent.com/astronomer/astro-cli/main/godownloader.sh | bash -s -- -b "$HOME/.local/bin"
           astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }}
           astro workspace switch ${{ steps.get-astro-env-info.outputs.workspace_id }}
@@ -88,6 +122,7 @@ jobs:
   default-deploy-tests:
     name: Default Deploy Test
     runs-on: ubuntu-latest
+    environment: e2e-test
     needs: [create-test-deployments]
     strategy:
       max-parallel: 1
@@ -183,6 +218,7 @@ jobs:
   dag-deploy-test:
     name: DAG Deploy Test
     runs-on: ubuntu-latest
+    environment: e2e-test
     needs: [default-deploy-tests, create-test-deployments]
     strategy:
       matrix:
@@ -264,10 +300,181 @@ jobs:
           image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }}
           hibernation_spec_after: ${{ steps.get-deployment-after.outputs.hibernation_spec }}
 
+  dag-folder-deploy-test:
+    name: DAG Folder Deploy Test
+    runs-on: ubuntu-latest
+    environment: e2e-test
+    needs: [dag-deploy-test, create-test-deployments]
+    strategy:
+      matrix:
+        deployment_id:
+          [
+            "${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}",
+          ]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Get Astro Environment Info
+        id: get-astro-env-info
+        uses: ./.github/workflows/e2e/get_astro_env_info
+        with:
+          input_workspace_id: ${{ github.event.inputs.workspace_id }}
+          input_organization_id: ${{ github.event.inputs.org_id }}
+          input_astronomer_host: ${{ github.event.inputs.astronomer_host }}
+          input_astro_api_token: ${{ github.event.inputs.token }}
+          secret_workspace_id: ${{ secrets.WORKSPACE_ID }}
+          secret_organization_id: ${{ secrets.ORGANIZATION_ID }}
+          secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }}
+          secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }}
+
+      - name: Mock git commands for dag-folder scenario
+        run: |
+          # Use mock that returns dags/ paths (outside root-folder) to simulate
+          # repos where DAGs live at a different location in version control
+          mv e2e-setup/mocks/dag-folder-git.sh /usr/local/bin/git
+          chmod +x /usr/local/bin/git
+
+      - name: Install dependencies
+        run: |
+
+          sudo apt-get install jq
+          # we need to pre-install the CLI to set the context
+          mkdir -p "$HOME/.local/bin"
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+          curl -sSL https://raw.githubusercontent.com/astronomer/astro-cli/main/godownloader.sh | bash -s -- -b "$HOME/.local/bin"
+
+      - name: Set CLI context
+        run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }}
+
+      - name: Get Deployment Info Before Test
+        id: get-deployment-before
+        uses: ./.github/workflows/e2e/get_deployment_info
+        with:
+          deployment_id: ${{ matrix.deployment_id }}
+          organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }}
+          astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }}
+          astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }}
+
+      - name: Deploy to Astro with dag-folder
+        uses: ./
+        with:
+          deployment-id: ${{ matrix.deployment_id }}
+          workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }}
+          root-folder: e2e-setup/astro-project
+          dag-folder: /dags/
+          parse: true
+
+      - name: Get Deployment Info After Test
+        id: get-deployment-after
+        uses: ./.github/workflows/e2e/get_deployment_info
+        with:
+          deployment_id: ${{ matrix.deployment_id }}
+          organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }}
+          astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }}
+          astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }}
+
+      - name: Validate Deploy Action
+        uses: ./.github/workflows/e2e/validate_deployment
+        with:
+          is_dag_only_deploy: true
+          dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }}
+          image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }}
+          hibernation_spec_before: ${{ steps.get-deployment-before.outputs.hibernation_spec }}
+          dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }}
+          image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }}
+          hibernation_spec_after: ${{ steps.get-deployment-after.outputs.hibernation_spec }}
+
+  dag-folder-prefix-no-deploy-test:
+    name: DAG Folder Prefix No Deploy Test
+    runs-on: ubuntu-latest
+    environment: e2e-test
+    needs: [dag-folder-deploy-test, create-test-deployments]
+    strategy:
+      matrix:
+        deployment_id:
+          [
+            "${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}",
+          ]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Get Astro Environment Info
+        id: get-astro-env-info
+        uses: ./.github/workflows/e2e/get_astro_env_info
+        with:
+          input_workspace_id: ${{ github.event.inputs.workspace_id }}
+          input_organization_id: ${{ github.event.inputs.org_id }}
+          input_astronomer_host: ${{ github.event.inputs.astronomer_host }}
+          input_astro_api_token: ${{ github.event.inputs.token }}
+          secret_workspace_id: ${{ secrets.WORKSPACE_ID }}
+          secret_organization_id: ${{ secrets.ORGANIZATION_ID }}
+          secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }}
+          secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }}
+
+      - name: Mock git commands for dag-folder prefix scenario
+        run: |
+          # Use mock that returns dags-old/ paths to ensure we only match the
+          # configured dags/ directory
+          mv e2e-setup/mocks/dag-folder-prefix-no-deploy-git.sh /usr/local/bin/git
+          chmod +x /usr/local/bin/git
+
+      - name: Install dependencies
+        run: |
+
+          sudo apt-get install jq
+          # we need to pre-install the CLI to set the context
+          mkdir -p "$HOME/.local/bin"
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+          curl -sSL https://raw.githubusercontent.com/astronomer/astro-cli/main/godownloader.sh | bash -s -- -b "$HOME/.local/bin"
+
+      - name: Set CLI context
+        run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }}
+
+      - name: Get Deployment Info Before Test
+        id: get-deployment-before
+        uses: ./.github/workflows/e2e/get_deployment_info
+        with:
+          deployment_id: ${{ matrix.deployment_id }}
+          organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }}
+          astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }}
+          astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }}
+
+      - name: Deploy to Astro with dag-folder prefix mismatch
+        uses: ./
+        with:
+          deployment-id: ${{ matrix.deployment_id }}
+          workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }}
+          root-folder: e2e-setup/astro-project
+          dag-folder: /dags/
+          parse: true
+
+      - name: Get Deployment Info After Test
+        id: get-deployment-after
+        uses: ./.github/workflows/e2e/get_deployment_info
+        with:
+          deployment_id: ${{ matrix.deployment_id }}
+          organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }}
+          astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }}
+          astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }}
+
+      - name: Validate no image or dag deploy happened
+        uses: ./.github/workflows/e2e/validate_deployment
+        with:
+          is_no_deploy: true
+          dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }}
+          image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }}
+          hibernation_spec_before: ${{ steps.get-deployment-before.outputs.hibernation_spec }}
+          dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }}
+          image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }}
+          hibernation_spec_after: ${{ steps.get-deployment-after.outputs.hibernation_spec }}
+
   pytests-test:
     name: Pytest Test
     runs-on: ubuntu-latest
-    needs: [dag-deploy-test, create-test-deployments]
+    environment: e2e-test
+    needs: [dag-folder-prefix-no-deploy-test, create-test-deployments]
     strategy:
       matrix:
         deployment_id:
@@ -355,6 +562,7 @@ jobs:
   infer-deploy:
     name: Infer Deploy Test
     runs-on: ubuntu-latest
+    environment: e2e-test
     needs: [pytests-test, create-test-deployments]
     strategy:
       max-parallel: 1
@@ -449,6 +657,7 @@ jobs:
   no-deploy-test:
     name: No Deploy Test
     runs-on: ubuntu-latest
+    environment: e2e-test
     needs: [infer-deploy, create-test-deployments]
     strategy:
       max-parallel: 1
@@ -566,6 +775,7 @@ jobs:
   custom-docker-image-test:
     name: Custom Docker Image Test
     runs-on: ubuntu-latest
+    environment: e2e-test
     needs: [no-deploy-test, create-test-deployments]
     strategy:
       matrix:
@@ -651,6 +861,7 @@ jobs:
   dbt-deploy-test:
     name: DBT Deploy Test
     runs-on: ubuntu-latest
+    environment: e2e-test
     needs: [custom-docker-image-test, create-test-deployments]
     strategy:
       matrix:
@@ -740,6 +951,7 @@ jobs:
   deployment-preview-test:
     name: Deployment Preview Test
     runs-on: ubuntu-latest
+    environment: e2e-test
     needs: [create-test-deployments]
     strategy:
       matrix:
@@ -990,8 +1202,10 @@ jobs:
   delete-test-deployments:
     name: Delete Deployment
     runs-on: ubuntu-latest
+    environment: e2e-test
     needs:
       [
+        check-astro-test-config,
         create-test-deployments,
         dag-deploy-test,
         pytests-test,
@@ -1000,7 +1214,7 @@ jobs:
         deployment-preview-test,
       ]
     # ensure that we always try delete the deployment even if any of the tests fail
-    if: always()
+    if: ${{ always() && needs.check-astro-test-config.outputs.has_astro_test_config == 'true' }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v6