[PolySwarm] May cleanup + full connector pair, mirroring upstream PR #6149

.circleci/config.yml

@@ -20,6 +20,50 @@ executors:
     docker:
       - image: "cimg/base:stable"
 
+commands:
+  resolve_release_ref:
+    description: Resolve RELEASE_REF from PR base branch, fallback to closest base branch script.
+    steps:
+      - run:
+          name: Resolve RELEASE_REF
+          command: |
+            PR_BASE_REF=""
+
+            if [ -n "${CIRCLE_PULL_REQUEST:-}" ]; then
+              PR_NUMBER="${CIRCLE_PULL_REQUEST##*/}"
+              API_URL="https://api.github.com/repos/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/pulls/${PR_NUMBER}"
+
+              if [ -n "${GITHUB_TOKEN:-}" ]; then
+                PR_BASE_REF="$(curl --silent --show-error --fail \
+                  -H "Accept: application/vnd.github+json" \
+                  -H "Authorization: Bearer ${GITHUB_TOKEN}" \
+                  "${API_URL}" \
+                | python3 -c 'import json,sys; print((json.load(sys.stdin).get("base") or {}).get("ref",""))')" || PR_BASE_REF=""
+              else
+                PR_BASE_REF="$(curl --silent --show-error --fail \
+                  -H "Accept: application/vnd.github+json" \
+                  "${API_URL}" \
+                | python3 -c 'import json,sys; print((json.load(sys.stdin).get("base") or {}).get("ref",""))')" || PR_BASE_REF=""
+              fi
+            fi
+
+            if [ -n "${PR_BASE_REF:-}" ]; then
+              case "${PR_BASE_REF}" in
+                master|release/6.9.x|lts/*)
+                  RESOLVED_RELEASE_REF="${PR_BASE_REF}"
+                  echo "Using PR base branch for RELEASE_REF: ${RESOLVED_RELEASE_REF}"
+                  ;;
+                *)
+                  RESOLVED_RELEASE_REF="$(bash ./shared/tools/ci/detect-base-branch.sh origin)"
+                  echo "PR base '${PR_BASE_REF}' is outside allowed refs, using fallback RELEASE_REF: ${RESOLVED_RELEASE_REF}"
+                  ;;
+              esac
+            else
+              RESOLVED_RELEASE_REF="$(bash ./shared/tools/ci/detect-base-branch.sh origin)"
+              echo "Using detected fallback RELEASE_REF: ${RESOLVED_RELEASE_REF}"
+            fi
+            echo "export RELEASE_REF=${RESOLVED_RELEASE_REF}" >> "$BASH_ENV"
+            echo "RELEASE_REF=${RESOLVED_RELEASE_REF}" >> "$BASH_ENV"
 
 jobs:
   ensure_formatting:
@@ -74,8 +118,15 @@ jobs:
     docker:
       - image: cimg/python:3.12
     working_directory: ~/repo
+    parameters:
+      ci_executor:
+        type: executor
+        description: Executor to use for this job.
+        default: ci_environment
+    executor: << parameters.ci_executor >>
     steps:
       - checkout
+      - resolve_release_ref
       - run:
           name: Build Connectors Schemas for XTM composer and Update Config Documentation
           command: |
@@ -87,7 +138,7 @@ jobs:
       - run:
           name: Commit Manifest if changed
           command: |
-            if [ "$CIRCLE_BRANCH" = "master" ]; then
+            if [ "$CIRCLE_BRANCH" = "$RELEASE_REF" ]; then
               echo "$GITHUB_GPG_SECRET_KEY" | base64 -d > /tmp/private.key
               gpg --import /tmp/private.key
               rm /tmp/private.key
@@ -106,13 +157,24 @@ jobs:
                 # Add custom flag to avoid triggering workflows
                 git commit -m "[Automation] Build and update manifest [ci build-manifest]"
                 # Push quietly to prevent showing the token in log
-                git push -q https://${GITHUB_TOKEN}@github.com/OpenCTI-Platform/connectors.git ${CIRCLE_BRANCH}   
+                if git push -q https://${GITHUB_TOKEN}@github.com/OpenCTI-Platform/connectors.git ${CIRCLE_BRANCH}; then
+                  echo "Manifest changes pushed successfully."
+                  exit 0
+                else
+                  echo "Push rejected, ${CIRCLE_BRANCH} probably moved. Trying rebase..."
+                  if ! git pull --rebase origin ${CIRCLE_BRANCH}; then
+                    echo "Rebase failed, aborting."
+                    git rebase --abort
+                    exit 1
+                  fi
+                  git push -q https://${GITHUB_TOKEN}@github.com/OpenCTI-Platform/connectors.git ${CIRCLE_BRANCH}
+                fi
               else
                 echo "Manifest not changed. Skipping commit."
                 exit 0
               fi
             else
-              echo "Skipping, not master"
+              echo "Skipping, not $RELEASE_REF branch."
             fi
       - run:
           name: Skip steps if flag detected
@@ -126,8 +188,15 @@ jobs:
       - image: cimg/python:3.11
     working_directory: ~/repo
     parallelism: 4
+    parameters:
+      ci_executor:
+        type: executor
+        description: Executor to use for this job.
+        default: ci_environment
+    executor: << parameters.ci_executor >>
     steps:
       - checkout
+      - resolve_release_ref
       - run: |
           sudo apt-get update -qq 
           sudo apt install -y \
@@ -145,7 +214,7 @@ jobs:
               | circleci tests run \
                   --split-by=timings \
                   --command="xargs bash run_test.sh"
-        # Save JUnit so CircleCI can show pass/fail per connector and build timings history
+      # Save JUnit so CircleCI can show pass/fail per connector and build timings history
       - store_test_results:
           path: test_outputs
 
@@ -161,6 +230,7 @@ jobs:
     executor: << parameters.ci_executor >>
     steps:
       - checkout
+      - resolve_release_ref
       - run:
           name: Install dependencies
           command: pip install -r .circleci/scripts/requirements.txt
@@ -224,7 +294,7 @@ workflows:
             - test
           filters:
             tags:
-              only: /[0-9]+(\.[0-9]+)+(\.[0-9]+)*/
+              only: /[0-9]+(\.[0-9]+)+(\.[0-9]+)*(-lts\.[0-9]*)?/
             branches:
               ignore: /.*/
       - build:
@@ -281,4 +351,4 @@ workflows:
             - build_prerelease
           filters:
             tags:
-              only: /.*/
+              only: /.*/

.circleci/scripts/generate_ci.py

@@ -119,6 +119,7 @@ def main():
         pycti=get_pycti(),
         tags=get_tags(),
         repo=REPOSITORY,
+        release_ref=os.getenv("RELEASE_REF", ""),
     )
     write_config(config)
 

.circleci/scripts/requirements.txt

@@ -1,3 +1,3 @@
 Jinja2==3.1.6
 PyYAML==6.0.3
-requests==2.32.5
+requests==2.33.0

.circleci/templates/dynamic.yml.j2

@@ -6,6 +6,8 @@ jobs:
   build_{{top_dir}}_{{sub_dir}}:
     docker:
       - image: cimg/base:current-24.04
+    environment:
+      RELEASE_REF: "{{ release_ref }}"
     steps:
       - checkout
       - setup_remote_docker
@@ -39,17 +41,17 @@ jobs:
 
             {% if pycti.replace -%}
                 {% if "prerelease" in tags %}
-                git+https://github.com/OpenCTI-Platform/opencti.git@master#subdirectory=client-python
+                git+https://github.com/OpenCTI-Platform/opencti.git@$RELEASE_REF#subdirectory=client-python
 
             find . -name requirements.txt -exec sed "s|^pycti==.*$|pycti @ git+https://github.com/OpenCTI-Platform/opencti.git@$CIRCLE_BRANCH#subdirectory=client-python|" -i {} \;
             find . -name pyproject.toml -exec sed "s|pycti==.*$|pycti @ git+https://github.com/OpenCTI-Platform/opencti.git@$CIRCLE_BRANCH#subdirectory=client-python\",|" -i {} \;
             find . -name requirements.txt -exec sed "s|^connectors-sdk.*$|connectors-sdk @ git+https://github.com/OpenCTI-Platform/connectors.git@$CIRCLE_BRANCH#subdirectory=connectors-sdk|" -i {} \;
             find . -name pyproject.toml -exec sed "s|connectors-sdk.*$|connectors-sdk @ git+https://github.com/OpenCTI-Platform/connectors.git@$CIRCLE_BRANCH#subdirectory=connectors-sdk\",|" -i {} \;
                 {% elif "rolling" in tags %}
-            find . -name requirements.txt -exec sed "s|^pycti==.*$|pycti @ git+https://github.com/OpenCTI-Platform/opencti.git@master#subdirectory=client-python|" -i {} \;
-            find . -name pyproject.toml -exec sed "s|pycti==.*$|pycti @ git+https://github.com/OpenCTI-Platform/opencti.git@master#subdirectory=client-python\",|" -i {} \;
-            find . -name requirements.txt -exec sed "s|^connectors-sdk.*$|connectors-sdk @ git+https://github.com/OpenCTI-Platform/connectors.git@master#subdirectory=connectors-sdk|" -i {} \;
-            find . -name pyproject.toml -exec sed "s|connectors-sdk.*$|connectors-sdk @ git+https://github.com/OpenCTI-Platform/connectors.git@master#subdirectory=connectors-sdk\",|" -i {} \;
+            find . -name requirements.txt -exec sed "s|^pycti==.*$|pycti @ git+https://github.com/OpenCTI-Platform/opencti.git@$RELEASE_REF#subdirectory=client-python|" -i {} \;
+            find . -name pyproject.toml -exec sed "s|pycti==.*$|pycti @ git+https://github.com/OpenCTI-Platform/opencti.git@$RELEASE_REF#subdirectory=client-python\",|" -i {} \;
+            find . -name requirements.txt -exec sed "s|^connectors-sdk.*$|connectors-sdk @ git+https://github.com/OpenCTI-Platform/connectors.git@$RELEASE_REF#subdirectory=connectors-sdk|" -i {} \;
+            find . -name pyproject.toml -exec sed "s|connectors-sdk.*$|connectors-sdk @ git+https://github.com/OpenCTI-Platform/connectors.git@$RELEASE_REF#subdirectory=connectors-sdk\",|" -i {} \;
                 {% endif %}
             {% endif -%}
             docker buildx build . \