Updated database workflows to centralize them in general_packages

.github/workflows/MCUsReleaseLive.yaml

@@ -35,6 +35,10 @@ jobs:
         with:
           app-id: ${{ vars.MIKROE_ACTIONS }}
           private-key: ${{ secrets.MIKROE_ACTIONS_KEY_AUTHORIZE }}
+          owner: MikroElektronika
+          repositories: |
+            core_packages
+            general_packages
 
       - name: Checkout code
         uses: actions/checkout@v6
@@ -70,7 +74,6 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install -r scripts/requirements/shared.txt
-          pip install -r scripts/requirements/databases.txt
           pip install -r scripts/requirements/support.txt
           sudo apt-get install p7zip-full
 
@@ -102,6 +105,17 @@ jobs:
           git commit -m "Updated changelog files with latest release info."
           git push
 
+
+      - name: Publish internal Core queries to latest release
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+        run: |
+          set -euo pipefail
+          rm -f core_queries.7z
+          7z a -t7z core_queries.7z ./resources/queries >/dev/null
+          tag="$(gh release view --repo MikroElektronika/core_packages --json tagName --jq .tagName)"
+          gh release upload "$tag" core_queries.7z --clobber --repo MikroElektronika/core_packages
+
       # Create a pull request using the GitHub API
       - name: Create Pull Request
         id: create_pr
@@ -143,25 +157,6 @@ jobs:
         if: failure()
         run: echo "::error::Merge conflict occurred. Please resolve manually."
 
-      - name: Update and upload new database
-        run: |
-          python -u scripts/reupload_databases.py ${{ steps.app-token.outputs.token }} ${{ github.repository }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} ${{ secrets.PROG_DEBUG_MIKROPROG }} ${{ secrets.PROG_DEBUG_JLINK }} ${{ github.event.inputs.release_version }} "latest" "Live"
-
-      - name: Commit and Push Changes
-        run: |
-          DB_NAME="necto_db.db"
-          STATUS=$(git status --short "$DB_NAME")
-          if [ -z "$STATUS" ]; then
-            echo "No changes made to $DB_NAME";
-          else
-            echo "Updating with new $DB_NAME";
-            git pull
-            git add $DB_NAME
-            git commit -m "Updated $DB_NAME with latest merged release."
-            git push
-          fi
-          curl --location --request POST '${{ secrets.ERP_DB_IMPORT_API }}' \
-          --header 'Authorization: Basic ${{ secrets.ERP_DB_IMPORT_KEY }}'
 
       - name: Run Index Script
         env:
@@ -172,7 +167,7 @@ jobs:
           ES_INDEX_LIVE: ${{ secrets.ES_INDEX_LIVE }}
         run: |
             echo "Indexing to Live."
-            python -u scripts/index.py ${{ github.repository }} ${{ steps.app-token.outputs.token }} ${{ secrets.ES_INDEX_LIVE }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} "False" ${{ github.event.inputs.release_version }} "False" "False"
+            python -u scripts/index.py ${{ github.repository }} ${{ steps.app-token.outputs.token }} ${{ secrets.ES_INDEX_LIVE }} "False" ${{ github.event.inputs.release_version }} "False"
 
       - name: Send notification to Mattermost
         if: ${{ github.event.inputs.notify_channel == 'true' }}
@@ -183,3 +178,19 @@ jobs:
           curl -X POST -H 'Content-Type: application/json' \
           --data "{\"text\": \"$MESSAGE\"}" \
           $MATTERMOST_WEBHOOK_URL
+
+      - name: Trigger centralized database update in general_packages
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+        run: |
+          gh api --method POST repos/MikroElektronika/general_packages/dispatches \
+            -f event_type=database-update \
+            -f 'client_payload[request_id]=core-${{ github.run_id }}' \
+            -f 'client_payload[source_repo]=${{ github.repository }}' \
+            -f 'client_payload[source_ref]=${{ github.ref_name }}' \
+            -f 'client_payload[channel]=live' \
+            -f 'client_payload[operation]=refresh' \
+            -f 'client_payload[core_release]=latest' \
+            -f 'client_payload[core_version]=${{ github.event.inputs.release_version }}' \
+            -f 'client_payload[sdk_version]=latest' \
+            -F 'client_payload[mcus_only]=true'

.github/workflows/MCUsReleaseTest.yaml

@@ -27,6 +27,10 @@ jobs:
         with:
           app-id: ${{ vars.MIKROE_ACTIONS }}
           private-key: ${{ secrets.MIKROE_ACTIONS_KEY_AUTHORIZE }}
+          owner: MikroElektronika
+          repositories: |
+            core_packages
+            general_packages
 
       - name: Checkout code
         uses: actions/checkout@v6
@@ -76,13 +80,29 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install -r scripts/requirements/shared.txt
-          pip install -r scripts/requirements/databases.txt
           pip install -r scripts/requirements/support.txt
           sudo apt-get install p7zip-full
+      - name: Download database_dev from general_packages for packaging
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+        run: |
+          set -euo pipefail
+          rm -rf .central-db
+          mkdir -p .central-db
+          gh release download --repo MikroElektronika/general_packages --pattern 'database_dev.7z' --dir .central-db
+          7z x -y -o.central-db/content .central-db/database_dev.7z >/dev/null
+          cp .central-db/content/necto_db.db necto_db_dev.db
 
-      - name: Update database for packaging
+
+      - name: Publish internal Core queries to latest release
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
         run: |
-          python -u scripts/reupload_databases.py ${{ steps.app-token.outputs.token }} ${{ github.repository }} ${{ secrets.PROG_DEBUG_CODEGRIP_TEST }} ${{ secrets.PROG_DEBUG_MIKROPROG }} ${{ secrets.PROG_DEBUG_JLINK }} ${{ github.event.inputs.release_version }} "latest" "Test" "--mcus_only" "True"
+          set -euo pipefail
+          rm -f core_queries.7z
+          7z a -t7z core_queries.7z ./resources/queries >/dev/null
+          tag="$(gh release view --repo MikroElektronika/core_packages --json tagName --jq .tagName)"
+          gh release upload "$tag" core_queries.7z --clobber --repo MikroElektronika/core_packages
 
       - name: Upload MCUs Asset
         env:
@@ -94,77 +114,35 @@ jobs:
           ES_INDEX_LIVE: ${{ secrets.ES_INDEX_LIVE }}
         run: python -u scripts/release_mcus.py ${{ steps.app-token.outputs.token }} ${{ github.repository }} ${{ github.event.inputs.release_version }}
 
-  update_and_upload_new_database:
+
+  trigger_general_database_update:
     runs-on: ubuntu-latest
     needs: upload_mcu_assets_test
-
     permissions:
-      pull-requests: write
-      contents: write
-      packages: write
-      actions: read
-
+      contents: read
     steps:
       - name: Authorize Mikroe Actions App
         uses: actions/create-github-app-token@v2
         id: app-token
         with:
           app-id: ${{ vars.MIKROE_ACTIONS }}
           private-key: ${{ secrets.MIKROE_ACTIONS_KEY_AUTHORIZE }}
+          owner: MikroElektronika
+          repositories: |
+            general_packages
 
-      - name: Checkout code
-        uses: actions/checkout@v6
-        with:
-          ref: ${{ github.event.inputs.release_branch }}
-          token: ${{ steps.app-token.outputs.token }}
-
-      - name: Add GitHub Actions credentials
-        run: |
-          git config user.name github-actions
-          git config user.email github-actions@github.com
-
-      - name: Set up Python
-        uses: actions/setup-python@v6
-        with:
-          python-version: '3.x'
-
-      - name: Cache Python packages
-        uses: actions/cache@v5
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-
-      - name: Install Dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r scripts/requirements/shared.txt
-          pip install -r scripts/requirements/databases.txt
-          pip install -r scripts/requirements/support.txt
-          sudo apt-get install p7zip-full
-
-      - name: Update and upload new database
-        run: |
-          python -u scripts/reupload_databases.py ${{ steps.app-token.outputs.token }} ${{ github.repository }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} ${{ secrets.PROG_DEBUG_MIKROPROG }} ${{ secrets.PROG_DEBUG_JLINK }} ${{ github.event.inputs.release_version }} "latest" "Test"
-
-      - name: Run Index Script
-        env:
-          ES_HOST: ${{ secrets.ES_HOST }}
-          ES_USER: ${{ secrets.ES_USER }}
-          ES_PASSWORD: ${{ secrets.ES_PASSWORD }}
-          ES_INDEX_TEST: ${{ secrets.ES_INDEX_TEST }}
-          ES_INDEX_LIVE: ${{ secrets.ES_INDEX_LIVE }}
-        run: |
-            echo "Indexing to Test."
-            python -u scripts/index.py ${{ github.repository }} ${{ steps.app-token.outputs.token }} ${{ secrets.ES_INDEX_TEST }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} "False" ${{ github.event.inputs.release_version }} "False" "False"
-
-      - name: Notify Mattermost - Test ready
+      - name: Trigger Development database update
         env:
-          MATTERMOST_WEBHOOK_URL_SDK: ${{ secrets.MATTERMOST_WEBHOOK_URL_SDK }}
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
         run: |
-          CURRENT_DATE=$(date +'%Y-%m-%d')  # Get current date in YYYY-MM-DD format
-          MESSAGE="**MCU package update ready to test for $CURRENT_DATE.**\n> Test using latest NECTO dev\n  + [LIN](https://software-update.mikroe.com/NECTOStudio7/development/necto/linux/NECTOInstaller.zip)\n  + [MAC](https://software-update.mikroe.com/NECTOStudio7/development/necto/macos/NECTOInstaller.dmg)\n  + [WIN](https://software-update.mikroe.com/NECTOStudio7/development/necto/win/NECTOInstaller.zip)"
-          curl -X POST -H 'Content-Type: application/json' \
-          --data "{\"text\": \"$MESSAGE\"}" \
-          $MATTERMOST_WEBHOOK_URL_SDK
+          gh api --method POST repos/MikroElektronika/general_packages/dispatches \
+            -f event_type=database-update \
+            -f 'client_payload[request_id]=core-${{ github.run_id }}' \
+            -f 'client_payload[source_repo]=${{ github.repository }}' \
+            -f 'client_payload[source_ref]=${{ github.event.inputs.release_branch }}' \
+            -f 'client_payload[channel]=development' \
+            -f 'client_payload[operation]=refresh' \
+            -f 'client_payload[core_release]=latest' \
+            -f 'client_payload[core_version]=${{ github.event.inputs.release_version }}' \
+            -f 'client_payload[sdk_version]=latest' \
+            -F 'client_payload[mcus_only]=true'

.github/workflows/checkIndexes.yaml

@@ -13,7 +13,7 @@ on:
       regex:
         type: string
         description: Regex to use when searching for indexed items
-        default: "arm_gcc_clang|arm_mikroc|clocks|database|dspic|^images$|mikroe_utils|avr|pic|preinit|riscv|schemas|unit_test_lib|.+[device|tool]_support$|^codegrip_pack"
+        default: "arm_gcc_clang|arm_mikroc|clocks|dspic|^images$|mikroe_utils|avr|pic|preinit|riscv|schemas|unit_test_lib|.+[device|tool]_support$"
       fix:
         type: boolean
         description: Fix the broken links with new ones?
@@ -32,7 +32,7 @@ on:
     # - cron: "0 10-16/2 * * 1-5"  # Every 2 hours from 11:00 to 17:00, Monday–Friday
 
 env:
-  GLOBAL_REGEX: "arm_gcc_clang|arm_mikroc|clocks|database|dspic|^images$|mikroe_utils|avr|pic|preinit|riscv|schemas|unit_test_lib|.+[device|tool]_support$|^codegrip_pack"
+  GLOBAL_REGEX: "arm_gcc_clang|arm_mikroc|clocks|dspic|^images$|mikroe_utils|avr|pic|preinit|riscv|schemas|unit_test_lib|.+[device|tool]_support$"
   ES_HOST_LEGACY: ${{ secrets.ES_HOST_LEGACY }}
   ES_USER_LEGACY: ${{ secrets.ES_USER_LEGACY }}
   ES_PASSWORD_LEGACY: ${{ secrets.ES_PASSWORD_LEGACY }}

.github/workflows/index.yaml

@@ -44,16 +44,6 @@ jobs:
           pip install -r scripts/requirements/shared.txt
           sudo apt-get install p7zip-full
 
-      - name: Update and upload new database
-        if: ${{ github.event.inputs.keep_dates == 'false' }}
-        run: |
-          if [[ ${{ github.event.inputs.select_index }} == "Live" ]]; then
-            echo "Updating Live DB."
-            python -u scripts/reupload_databases.py ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} ${{ secrets.PROG_DEBUG_MIKROPROG }} ${{ secrets.PROG_DEBUG_JLINK }} ${{ github.event.inputs.release_version }} "latest" ${{ github.event.inputs.select_index }}
-          else
-            echo "Updating Test DB."
-            python -u scripts/reupload_databases.py ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} ${{ secrets.PROG_DEBUG_CODEGRIP_TEST }} ${{ secrets.PROG_DEBUG_MIKROPROG }} ${{ secrets.PROG_DEBUG_JLINK }} ${{ github.event.inputs.release_version }} "latest" ${{ github.event.inputs.select_index }}
-          fi
 
       - name: Run Index Script
         env:
@@ -63,13 +53,13 @@ jobs:
         run: |
           if [[ ${{ github.event.inputs.select_index }} == "Live" ]]; then
             echo "Indexing to Live."
-            python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_LIVE }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} ${{ github.event.inputs.force_index }} ${{ github.event.inputs.release_version }} False ${{ github.event.inputs.set_as_latest }} "--keep_previous_dates" ${{ github.event.inputs.keep_dates }}
+            python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_LIVE }} ${{ github.event.inputs.force_index }} ${{ github.event.inputs.release_version }} ${{ github.event.inputs.set_as_latest }} "--keep_previous_dates" ${{ github.event.inputs.keep_dates }}
           else
             echo "Indexing to Test."
             if [[ ${{ github.event.inputs.set_as_latest }} ]]; then
               echo "Promote to latest requested, but ignored. Only available for LIVE updates."
             fi
-            python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_TEST }} ${{ secrets.PROG_DEBUG_CODEGRIP_TEST }} ${{ github.event.inputs.force_index }} ${{ github.event.inputs.release_version }} False False "--keep_previous_dates" ${{ github.event.inputs.keep_dates }}
+            python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_TEST }} ${{ github.event.inputs.force_index }} ${{ github.event.inputs.release_version }} False "--keep_previous_dates" ${{ github.event.inputs.keep_dates }}
           fi
 
       - name: Send notification to Mattermost

.github/workflows/publishCoreQueries.yaml

@@ -0,0 +1,48 @@
+name: Publish Internal Core Queries
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'resources/queries/**'
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Authorize Mikroe Actions App
+        uses: actions/create-github-app-token@v2
+        id: app-token
+        with:
+          app-id: ${{ vars.MIKROE_ACTIONS }}
+          private-key: ${{ secrets.MIKROE_ACTIONS_KEY_AUTHORIZE }}
+
+      - name: Checkout code
+        uses: actions/checkout@v6
+        with:
+          token: ${{ steps.app-token.outputs.token }}
+
+      - name: Install 7-Zip
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y p7zip-full
+
+      - name: Pack internal Core queries
+        run: |
+          set -euo pipefail
+          rm -f core_queries.7z
+          7z a -t7z core_queries.7z ./resources/queries >/dev/null
+
+      - name: Upload to latest Core release
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+        run: |
+          set -euo pipefail
+          tag="$(gh release view --json tagName --jq .tagName)"
+          gh release upload "$tag" core_queries.7z --clobber
+          echo "Published core_queries.7z to $tag."
+          echo "This asset is internal-only and is intentionally absent from metadata.json."