diff --git a/.github/workflows/MCUsReleaseLive.yaml b/.github/workflows/MCUsReleaseLive.yaml index 244046250e..45959bba8f 100644 --- a/.github/workflows/MCUsReleaseLive.yaml +++ b/.github/workflows/MCUsReleaseLive.yaml @@ -35,6 +35,10 @@ jobs: with: app-id: ${{ vars.MIKROE_ACTIONS }} private-key: ${{ secrets.MIKROE_ACTIONS_KEY_AUTHORIZE }} + owner: MikroElektronika + repositories: | + core_packages + general_packages - name: Checkout code uses: actions/checkout@v6 @@ -70,7 +74,6 @@ jobs: run: | python -m pip install --upgrade pip pip install -r scripts/requirements/shared.txt - pip install -r scripts/requirements/databases.txt pip install -r scripts/requirements/support.txt sudo apt-get install p7zip-full @@ -102,6 +105,17 @@ jobs: git commit -m "Updated changelog files with latest release info." git push + + - name: Publish internal Core queries to latest release + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + set -euo pipefail + rm -f core_queries.7z + 7z a -t7z core_queries.7z ./resources/queries >/dev/null + tag="$(gh release view --repo MikroElektronika/core_packages --json tagName --jq .tagName)" + gh release upload "$tag" core_queries.7z --clobber --repo MikroElektronika/core_packages + # Create a pull request using the GitHub API - name: Create Pull Request id: create_pr @@ -143,25 +157,6 @@ jobs: if: failure() run: echo "::error::Merge conflict occurred. Please resolve manually." - - name: Update and upload new database - run: | - python -u scripts/reupload_databases.py ${{ steps.app-token.outputs.token }} ${{ github.repository }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} ${{ secrets.PROG_DEBUG_MIKROPROG }} ${{ secrets.PROG_DEBUG_JLINK }} ${{ github.event.inputs.release_version }} "latest" "Live" - - - name: Commit and Push Changes - run: | - DB_NAME="necto_db.db" - STATUS=$(git status --short "$DB_NAME") - if [ -z "$STATUS" ]; then - echo "No changes made to $DB_NAME"; - else - echo "Updating with new $DB_NAME"; - git pull - git add $DB_NAME - git commit -m "Updated $DB_NAME with latest merged release." - git push - fi - curl --location --request POST '${{ secrets.ERP_DB_IMPORT_API }}' \ - --header 'Authorization: Basic ${{ secrets.ERP_DB_IMPORT_KEY }}' - name: Run Index Script env: @@ -172,7 +167,7 @@ jobs: ES_INDEX_LIVE: ${{ secrets.ES_INDEX_LIVE }} run: | echo "Indexing to Live." - python -u scripts/index.py ${{ github.repository }} ${{ steps.app-token.outputs.token }} ${{ secrets.ES_INDEX_LIVE }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} "False" ${{ github.event.inputs.release_version }} "False" "False" + python -u scripts/index.py ${{ github.repository }} ${{ steps.app-token.outputs.token }} ${{ secrets.ES_INDEX_LIVE }} "False" ${{ github.event.inputs.release_version }} "False" - name: Send notification to Mattermost if: ${{ github.event.inputs.notify_channel == 'true' }} @@ -183,3 +178,19 @@ jobs: curl -X POST -H 'Content-Type: application/json' \ --data "{\"text\": \"$MESSAGE\"}" \ $MATTERMOST_WEBHOOK_URL + + - name: Trigger centralized database update in general_packages + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + gh api --method POST repos/MikroElektronika/general_packages/dispatches \ + -f event_type=database-update \ + -f 'client_payload[request_id]=core-${{ github.run_id }}' \ + -f 'client_payload[source_repo]=${{ github.repository }}' \ + -f 'client_payload[source_ref]=${{ github.ref_name }}' \ + -f 'client_payload[channel]=live' \ + -f 'client_payload[operation]=refresh' \ + -f 'client_payload[core_release]=latest' \ + -f 'client_payload[core_version]=${{ github.event.inputs.release_version }}' \ + -f 'client_payload[sdk_version]=latest' \ + -F 'client_payload[mcus_only]=true' diff --git a/.github/workflows/MCUsReleaseTest.yaml b/.github/workflows/MCUsReleaseTest.yaml index 184bedbebf..8a1e02f311 100644 --- a/.github/workflows/MCUsReleaseTest.yaml +++ b/.github/workflows/MCUsReleaseTest.yaml @@ -27,6 +27,10 @@ jobs: with: app-id: ${{ vars.MIKROE_ACTIONS }} private-key: ${{ secrets.MIKROE_ACTIONS_KEY_AUTHORIZE }} + owner: MikroElektronika + repositories: | + core_packages + general_packages - name: Checkout code uses: actions/checkout@v6 @@ -76,13 +80,29 @@ jobs: run: | python -m pip install --upgrade pip pip install -r scripts/requirements/shared.txt - pip install -r scripts/requirements/databases.txt pip install -r scripts/requirements/support.txt sudo apt-get install p7zip-full + - name: Download database_dev from general_packages for packaging + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + set -euo pipefail + rm -rf .central-db + mkdir -p .central-db + gh release download --repo MikroElektronika/general_packages --pattern 'database_dev.7z' --dir .central-db + 7z x -y -o.central-db/content .central-db/database_dev.7z >/dev/null + cp .central-db/content/necto_db.db necto_db_dev.db - - name: Update database for packaging + + - name: Publish internal Core queries to latest release + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} run: | - python -u scripts/reupload_databases.py ${{ steps.app-token.outputs.token }} ${{ github.repository }} ${{ secrets.PROG_DEBUG_CODEGRIP_TEST }} ${{ secrets.PROG_DEBUG_MIKROPROG }} ${{ secrets.PROG_DEBUG_JLINK }} ${{ github.event.inputs.release_version }} "latest" "Test" "--mcus_only" "True" + set -euo pipefail + rm -f core_queries.7z + 7z a -t7z core_queries.7z ./resources/queries >/dev/null + tag="$(gh release view --repo MikroElektronika/core_packages --json tagName --jq .tagName)" + gh release upload "$tag" core_queries.7z --clobber --repo MikroElektronika/core_packages - name: Upload MCUs Asset env: @@ -94,16 +114,12 @@ jobs: ES_INDEX_LIVE: ${{ secrets.ES_INDEX_LIVE }} run: python -u scripts/release_mcus.py ${{ steps.app-token.outputs.token }} ${{ github.repository }} ${{ github.event.inputs.release_version }} - update_and_upload_new_database: + + trigger_general_database_update: runs-on: ubuntu-latest needs: upload_mcu_assets_test - permissions: - pull-requests: write - contents: write - packages: write - actions: read - + contents: read steps: - name: Authorize Mikroe Actions App uses: actions/create-github-app-token@v2 @@ -111,60 +127,22 @@ jobs: with: app-id: ${{ vars.MIKROE_ACTIONS }} private-key: ${{ secrets.MIKROE_ACTIONS_KEY_AUTHORIZE }} + owner: MikroElektronika + repositories: | + general_packages - - name: Checkout code - uses: actions/checkout@v6 - with: - ref: ${{ github.event.inputs.release_branch }} - token: ${{ steps.app-token.outputs.token }} - - - name: Add GitHub Actions credentials - run: | - git config user.name github-actions - git config user.email github-actions@github.com - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '3.x' - - - name: Cache Python packages - uses: actions/cache@v5 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r scripts/requirements/shared.txt - pip install -r scripts/requirements/databases.txt - pip install -r scripts/requirements/support.txt - sudo apt-get install p7zip-full - - - name: Update and upload new database - run: | - python -u scripts/reupload_databases.py ${{ steps.app-token.outputs.token }} ${{ github.repository }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} ${{ secrets.PROG_DEBUG_MIKROPROG }} ${{ secrets.PROG_DEBUG_JLINK }} ${{ github.event.inputs.release_version }} "latest" "Test" - - - name: Run Index Script - env: - ES_HOST: ${{ secrets.ES_HOST }} - ES_USER: ${{ secrets.ES_USER }} - ES_PASSWORD: ${{ secrets.ES_PASSWORD }} - ES_INDEX_TEST: ${{ secrets.ES_INDEX_TEST }} - ES_INDEX_LIVE: ${{ secrets.ES_INDEX_LIVE }} - run: | - echo "Indexing to Test." - python -u scripts/index.py ${{ github.repository }} ${{ steps.app-token.outputs.token }} ${{ secrets.ES_INDEX_TEST }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} "False" ${{ github.event.inputs.release_version }} "False" "False" - - - name: Notify Mattermost - Test ready + - name: Trigger Development database update env: - MATTERMOST_WEBHOOK_URL_SDK: ${{ secrets.MATTERMOST_WEBHOOK_URL_SDK }} + GH_TOKEN: ${{ steps.app-token.outputs.token }} run: | - CURRENT_DATE=$(date +'%Y-%m-%d') # Get current date in YYYY-MM-DD format - MESSAGE="**MCU package update ready to test for $CURRENT_DATE.**\n> Test using latest NECTO dev\n + [LIN](https://software-update.mikroe.com/NECTOStudio7/development/necto/linux/NECTOInstaller.zip)\n + [MAC](https://software-update.mikroe.com/NECTOStudio7/development/necto/macos/NECTOInstaller.dmg)\n + [WIN](https://software-update.mikroe.com/NECTOStudio7/development/necto/win/NECTOInstaller.zip)" - curl -X POST -H 'Content-Type: application/json' \ - --data "{\"text\": \"$MESSAGE\"}" \ - $MATTERMOST_WEBHOOK_URL_SDK + gh api --method POST repos/MikroElektronika/general_packages/dispatches \ + -f event_type=database-update \ + -f 'client_payload[request_id]=core-${{ github.run_id }}' \ + -f 'client_payload[source_repo]=${{ github.repository }}' \ + -f 'client_payload[source_ref]=${{ github.event.inputs.release_branch }}' \ + -f 'client_payload[channel]=development' \ + -f 'client_payload[operation]=refresh' \ + -f 'client_payload[core_release]=latest' \ + -f 'client_payload[core_version]=${{ github.event.inputs.release_version }}' \ + -f 'client_payload[sdk_version]=latest' \ + -F 'client_payload[mcus_only]=true' diff --git a/.github/workflows/checkIndexes.yaml b/.github/workflows/checkIndexes.yaml index 017f82ac64..4e0e7cedb8 100644 --- a/.github/workflows/checkIndexes.yaml +++ b/.github/workflows/checkIndexes.yaml @@ -13,7 +13,7 @@ on: regex: type: string description: Regex to use when searching for indexed items - default: "arm_gcc_clang|arm_mikroc|clocks|database|dspic|^images$|mikroe_utils|avr|pic|preinit|riscv|schemas|unit_test_lib|.+[device|tool]_support$|^codegrip_pack" + default: "arm_gcc_clang|arm_mikroc|clocks|dspic|^images$|mikroe_utils|avr|pic|preinit|riscv|schemas|unit_test_lib|.+[device|tool]_support$" fix: type: boolean description: Fix the broken links with new ones? @@ -32,7 +32,7 @@ on: # - cron: "0 10-16/2 * * 1-5" # Every 2 hours from 11:00 to 17:00, Monday–Friday env: - GLOBAL_REGEX: "arm_gcc_clang|arm_mikroc|clocks|database|dspic|^images$|mikroe_utils|avr|pic|preinit|riscv|schemas|unit_test_lib|.+[device|tool]_support$|^codegrip_pack" + GLOBAL_REGEX: "arm_gcc_clang|arm_mikroc|clocks|dspic|^images$|mikroe_utils|avr|pic|preinit|riscv|schemas|unit_test_lib|.+[device|tool]_support$" ES_HOST_LEGACY: ${{ secrets.ES_HOST_LEGACY }} ES_USER_LEGACY: ${{ secrets.ES_USER_LEGACY }} ES_PASSWORD_LEGACY: ${{ secrets.ES_PASSWORD_LEGACY }} diff --git a/.github/workflows/index.yaml b/.github/workflows/index.yaml index 21af0f0981..5d76b6d4da 100644 --- a/.github/workflows/index.yaml +++ b/.github/workflows/index.yaml @@ -44,16 +44,6 @@ jobs: pip install -r scripts/requirements/shared.txt sudo apt-get install p7zip-full - - name: Update and upload new database - if: ${{ github.event.inputs.keep_dates == 'false' }} - run: | - if [[ ${{ github.event.inputs.select_index }} == "Live" ]]; then - echo "Updating Live DB." - python -u scripts/reupload_databases.py ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} ${{ secrets.PROG_DEBUG_MIKROPROG }} ${{ secrets.PROG_DEBUG_JLINK }} ${{ github.event.inputs.release_version }} "latest" ${{ github.event.inputs.select_index }} - else - echo "Updating Test DB." - python -u scripts/reupload_databases.py ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} ${{ secrets.PROG_DEBUG_CODEGRIP_TEST }} ${{ secrets.PROG_DEBUG_MIKROPROG }} ${{ secrets.PROG_DEBUG_JLINK }} ${{ github.event.inputs.release_version }} "latest" ${{ github.event.inputs.select_index }} - fi - name: Run Index Script env: @@ -63,13 +53,13 @@ jobs: run: | if [[ ${{ github.event.inputs.select_index }} == "Live" ]]; then echo "Indexing to Live." - python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_LIVE }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} ${{ github.event.inputs.force_index }} ${{ github.event.inputs.release_version }} False ${{ github.event.inputs.set_as_latest }} "--keep_previous_dates" ${{ github.event.inputs.keep_dates }} + python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_LIVE }} ${{ github.event.inputs.force_index }} ${{ github.event.inputs.release_version }} ${{ github.event.inputs.set_as_latest }} "--keep_previous_dates" ${{ github.event.inputs.keep_dates }} else echo "Indexing to Test." if [[ ${{ github.event.inputs.set_as_latest }} ]]; then echo "Promote to latest requested, but ignored. Only available for LIVE updates." fi - python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_TEST }} ${{ secrets.PROG_DEBUG_CODEGRIP_TEST }} ${{ github.event.inputs.force_index }} ${{ github.event.inputs.release_version }} False False "--keep_previous_dates" ${{ github.event.inputs.keep_dates }} + python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_TEST }} ${{ github.event.inputs.force_index }} ${{ github.event.inputs.release_version }} False "--keep_previous_dates" ${{ github.event.inputs.keep_dates }} fi - name: Send notification to Mattermost diff --git a/.github/workflows/notify_codegrip.yaml b/.github/workflows/notify_codegrip.yaml deleted file mode 100644 index d29b23e8a6..0000000000 --- a/.github/workflows/notify_codegrip.yaml +++ /dev/null @@ -1,56 +0,0 @@ -name: Notify Codegrip Release Mattermost - -on: - workflow_dispatch: - -jobs: - notify: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v6 - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '3.x' - - - name: Cache Python packages - uses: actions/cache@v5 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install pytz - pip install py7zr - pip install aiohttp - pip install chardet - pip install aiofiles - pip install requests - pip install packaging - pip install elasticsearch==7.13.4 - sudo apt-get install p7zip-full - - - name: Build Message with Python - id: build_message - env: - ES_HOST: ${{ secrets.ES_HOST }} - ES_USER: ${{ secrets.ES_USER }} - ES_PASSWORD: ${{ secrets.ES_PASSWORD }} - run: | - python -u scripts/build_message_codegrip.py "NECTO DAILY UPDATE" ${{ secrets.RELEASES_SPREADSHEET }} ${{ secrets.ES_INDEX_LIVE }} - - - name: Send notification to Mattermost - Product Release - if: steps.build_message.outputs.has_packages == 'true' - env: - MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK_URL }} - run: | - MESSAGE=$(cat message.txt) - curl -X POST -H 'Content-Type: application/json' \ - --data "{\"text\": \"$MESSAGE\"}" \ - $MATTERMOST_WEBHOOK_URL diff --git a/.github/workflows/publishCoreQueries.yaml b/.github/workflows/publishCoreQueries.yaml new file mode 100644 index 0000000000..b48d99e8b5 --- /dev/null +++ b/.github/workflows/publishCoreQueries.yaml @@ -0,0 +1,48 @@ +name: Publish Internal Core Queries + +on: + push: + branches: [main] + paths: + - 'resources/queries/**' + workflow_dispatch: + +permissions: + contents: write + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - name: Authorize Mikroe Actions App + uses: actions/create-github-app-token@v2 + id: app-token + with: + app-id: ${{ vars.MIKROE_ACTIONS }} + private-key: ${{ secrets.MIKROE_ACTIONS_KEY_AUTHORIZE }} + + - name: Checkout code + uses: actions/checkout@v6 + with: + token: ${{ steps.app-token.outputs.token }} + + - name: Install 7-Zip + run: | + sudo apt-get update + sudo apt-get install -y p7zip-full + + - name: Pack internal Core queries + run: | + set -euo pipefail + rm -f core_queries.7z + 7z a -t7z core_queries.7z ./resources/queries >/dev/null + + - name: Upload to latest Core release + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + set -euo pipefail + tag="$(gh release view --json tagName --jq .tagName)" + gh release upload "$tag" core_queries.7z --clobber + echo "Published core_queries.7z to $tag." + echo "This asset is internal-only and is intentionally absent from metadata.json." diff --git a/.github/workflows/reReleasePackagesKeepDates.yaml b/.github/workflows/reReleasePackagesKeepDates.yaml index bfaec8b858..5a6261f3e8 100644 --- a/.github/workflows/reReleasePackagesKeepDates.yaml +++ b/.github/workflows/reReleasePackagesKeepDates.yaml @@ -18,6 +18,10 @@ jobs: with: app-id: ${{ vars.MIKROE_ACTIONS }} private-key: ${{ secrets.MIKROE_ACTIONS_KEY_AUTHORIZE }} + owner: MikroElektronika + repositories: | + core_packages + general_packages - name: Checkout code uses: actions/checkout@v6 @@ -46,7 +50,18 @@ jobs: sudo apt-get install p7zip-full pip install -r scripts/requirements/shared.txt pip install -r scripts/requirements/release.txt - pip install -r scripts/requirements/databases.txt + + + - name: Download database from general_packages for read-only packaging + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + set -euo pipefail + rm -rf .central-db + mkdir -p .central-db + gh release download --repo MikroElektronika/general_packages --pattern 'database.7z' --dir .central-db + 7z x -y -o.central-db/content .central-db/database.7z >/dev/null + cp .central-db/content/necto_db.db necto_db.db - name: Run Package Script env: @@ -57,16 +72,3 @@ jobs: ES_INDEX_TEST: ${{ secrets.ES_INDEX_TEST }} ES_INDEX_LIVE: ${{ secrets.ES_INDEX_LIVE }} run: python -u scripts/package.py ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} "latest" "--releases_to_update" "${{ github.event.inputs.release_names }}" - - ## Left commented out for potential future use. - # - name: Run Index Script - # env: - # ES_HOST: ${{ secrets.ES_HOST }} - # ES_USER: ${{ secrets.ES_USER }} - # ES_PASSWORD: ${{ secrets.ES_PASSWORD }} - # run: | - # echo "Indexing to Test first." - # python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_TEST }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} False latest False False "--es_host" ${{ secrets.ES_HOST }} "--es_user" "${{ secrets.ES_USER }}" "--es_password" "${{ secrets.ES_PASSWORD }}" "--keep_previous_dates" "True" - # echo "Indexing to Live next." - # python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_LIVE }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} False latest False False "--es_host" ${{ secrets.ES_HOST }} "--es_user" "${{ secrets.ES_USER }}" "--es_password" "${{ secrets.ES_PASSWORD }}" "--keep_previous_dates" "True" - # fi \ No newline at end of file diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 5551ad7016..7f26720baf 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -15,6 +15,10 @@ jobs: with: app-id: ${{ vars.MIKROE_ACTIONS }} private-key: ${{ secrets.MIKROE_ACTIONS_KEY_AUTHORIZE }} + owner: MikroElektronika + repositories: | + core_packages + general_packages - name: Checkout code uses: actions/checkout@v6 @@ -34,11 +38,31 @@ jobs: python -m pip install --upgrade pip pip install -r scripts/requirements/shared.txt pip install -r scripts/requirements/release.txt - pip install -r scripts/requirements/databases.txt + sudo apt-get update + sudo apt-get install -y p7zip-full - - name: Update database for packaging + + - name: Download database from general_packages for packaging + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} run: | - python -u scripts/reupload_databases.py ${{ steps.app-token.outputs.token }} ${{ github.repository }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} ${{ secrets.PROG_DEBUG_MIKROPROG }} ${{ secrets.PROG_DEBUG_JLINK }} "latest" "latest" "Live" "--mcus_only" "True" + set -euo pipefail + rm -rf .central-db + mkdir -p .central-db + gh release download --repo MikroElektronika/general_packages --pattern 'database.7z' --dir .central-db + 7z x -y -o.central-db/content .central-db/database.7z >/dev/null + cp .central-db/content/necto_db.db necto_db.db + + + - name: Publish internal Core queries to latest release + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + set -euo pipefail + rm -f core_queries.7z + 7z a -t7z core_queries.7z ./resources/queries >/dev/null + tag="$(gh release view --repo MikroElektronika/core_packages --json tagName --jq .tagName)" + gh release upload "$tag" core_queries.7z --clobber --repo MikroElektronika/core_packages - name: Run Release Script env: @@ -49,3 +73,19 @@ jobs: ES_INDEX: ${{ secrets.ES_INDEX_LIVE }} ES_INDEX_LIVE: ${{ secrets.ES_INDEX_LIVE }} run: python -u scripts/package.py ${{ steps.app-token.outputs.token }} ${{ github.repository }} ${{ github.ref_name }} + + - name: Trigger database update in general_packages + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + gh api --method POST repos/MikroElektronika/general_packages/dispatches \ + -f event_type=database-update \ + -f 'client_payload[request_id]=core-${{ github.run_id }}' \ + -f 'client_payload[source_repo]=${{ github.repository }}' \ + -f 'client_payload[source_ref]=${{ github.ref_name }}' \ + -f 'client_payload[channel]=live' \ + -f 'client_payload[operation]=refresh' \ + -f 'client_payload[core_release]=latest' \ + -f 'client_payload[core_version]=${{ github.ref_name }}' \ + -f 'client_payload[sdk_version]=latest' \ + -F 'client_payload[mcus_only]=true' diff --git a/.github/workflows/releaseCodegripPackages.yaml b/.github/workflows/releaseCodegripPackages.yaml deleted file mode 100644 index ff52d07228..0000000000 --- a/.github/workflows/releaseCodegripPackages.yaml +++ /dev/null @@ -1,110 +0,0 @@ -name: Release Codegrip Packages Separately - -on: - workflow_dispatch: - inputs: - select_index: - type: choice - description: Index as test or live - options: - - Test - - Live - start_date: - type: string - description: First date for codegrip releases to release (better use csv, not spreadsheet to see the date) - default: "2025-06-12" - end_date: - type: string - description: Last date for codegrip releases (better use csv, not spreadsheet to see the date) - default: "2025-06-13" - -jobs: - release-codegrip-packages: - runs-on: ubuntu-latest - steps: - - name: Authorize Mikroe Actions App - uses: actions/create-github-app-token@v2 - id: app-token - with: - app-id: ${{ vars.MIKROE_ACTIONS }} - private-key: ${{ secrets.MIKROE_ACTIONS_KEY_AUTHORIZE }} - - - name: Checkout code - uses: actions/checkout@v6 - with: - token: ${{ steps.app-token.outputs.token }} - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '3.x' - - - name: Cache Python packages - uses: actions/cache@v5 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r scripts/requirements/shared.txt - sudo apt-get install p7zip-full - - - name: Update database for packaging - run: | - python -u scripts/reupload_databases.py ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} ${{ secrets.PROG_DEBUG_MIKROPROG }} ${{ secrets.PROG_DEBUG_JLINK }} "latest" "latest" ${{ github.event.inputs.select_index }} "--mcus_only" "False" - - - name: Add GitHub Actions credentials - run: | - git config user.name github-actions - git config user.email github-actions@github.com - - - name: Commit and Push Changes - run: | - if [[ ${{ github.event.inputs.select_index }} == "Live" ]]; then - DB_NAME="necto_db.db" - else - DB_NAME="necto_db_dev.db" - fi - STATUS=$(git status --short "$DB_NAME") - if [ -z "$STATUS" ]; then - echo "No changes made to $DB_NAME"; - else - echo "Updating with new $DB_NAME"; - echo "test" - git pull - git add $DB_NAME - git commit -m "Updated $DB_NAME with latest merged release." - git push - if [[ ${{ github.event.inputs.select_index }} == "Live" ]]; then - curl --location --request POST '${{ secrets.ERP_DB_IMPORT_API }}' \ - --header 'Authorization: Basic ${{ secrets.ERP_DB_IMPORT_KEY }}' - fi - fi - - - name: Run Index Script - env: - ES_HOST: ${{ secrets.ES_HOST }} - ES_USER: ${{ secrets.ES_USER }} - ES_PASSWORD: ${{ secrets.ES_PASSWORD }} - run: | - if [[ ${{ github.event.inputs.select_index }} == "Test" ]]; then - echo "Indexing to Test." - python -u scripts/index_codegrip_packages.py ${{ secrets.ES_INDEX_TEST }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} ${{ github.event.inputs.start_date }} ${{ github.event.inputs.end_date }} > message.txt - else - echo "Indexing to Live." - python -u scripts/index_codegrip_packages.py ${{ secrets.ES_INDEX_LIVE }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} ${{ github.event.inputs.start_date }} ${{ github.event.inputs.end_date }} > message.txt - fi - - - name: Send notification to Mattermost - SDK Team - env: - MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK_URL_SDK }} - run: | - cat message.txt - MESSAGE=$(cat message.txt) - curl -X POST -H 'Content-Type: application/json' \ - --data "{\"text\": \"$MESSAGE\"}" \ - $MATTERMOST_WEBHOOK_URL \ No newline at end of file diff --git a/.github/workflows/updateDb.yaml b/.github/workflows/updateDb.yaml deleted file mode 100644 index 3133cfcf39..0000000000 --- a/.github/workflows/updateDb.yaml +++ /dev/null @@ -1,83 +0,0 @@ -name: Update Latest Database - -on: - workflow_dispatch: - inputs: - release_version: - type: string - description: Which release version to update the database for (type v1.0.6 for example) - default: "latest" - select_index: - type: choice - description: Index to test or live - options: - - Test - - Live - -jobs: - Update-Database: - runs-on: ubuntu-latest - steps: - - name: Authorize Mikroe Actions App - uses: actions/create-github-app-token@v2 - id: app-token - with: - app-id: ${{ vars.MIKROE_ACTIONS }} - private-key: ${{ secrets.MIKROE_ACTIONS_KEY }} - - - name: Checkout code - uses: actions/checkout@v6 - with: - token: ${{ steps.app-token.outputs.token }} - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '3.x' - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r scripts/requirements/shared.txt - pip install -r scripts/requirements/databases.txt - sudo apt-get install p7zip-full - - - name: Upload new database asset - run: | - if [[ ${{ github.event.inputs.release_version }} == "Live" ]]; then - python -u scripts/reupload_databases.py ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} ${{ secrets.PROG_DEBUG_MIKROPROG }} ${{ secrets.PROG_DEBUG_JLINK }} "latest" "latest" ${{ secrets.ES_INDEX_LIVE }} - else - python -u scripts/reupload_databases.py ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} ${{ secrets.PROG_DEBUG_CODEGRIP_TEST }} ${{ secrets.PROG_DEBUG_MIKROPROG }} ${{ secrets.PROG_DEBUG_JLINK }} "latest" "latest" ${{ secrets.ES_INDEX_TEST }} - fi - - - name: Add GitHub Actions credentials - run: | - git config user.name github-actions - git config user.email github-actions@github.com - - - name: Commit and Push Changes - run: | - if [ -n "$(git status --porcelain)" ]; then - echo "Updating with new CHANGELOG.md"; - git add necto_db.db - git commit -m "Updated necto database with latest merged release." - git push - curl --location --request POST '${{ secrets.ERP_DB_IMPORT_API }}' \ - --header 'Authorization: Basic ${{ secrets.ERP_DB_IMPORT_KEY }}' - else - echo "No changes made to necto_db.db"; - fi - - - name: Run Index Script - env: - ES_HOST: ${{ secrets.ES_HOST }} - ES_USER: ${{ secrets.ES_USER }} - ES_PASSWORD: ${{ secrets.ES_PASSWORD }} - run: | - if [[ ${{ github.event.inputs.release_version }} == "Live" ]]; then - echo "Indexing database to LIVE." - python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_LIVE }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} False ${{ github.event.inputs.release_version }} True False - else - echo "Indexing database to TEST." - python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_TEST }} ${{ secrets.PROG_DEBUG_CODEGRIP_TEST }} False ${{ github.event.inputs.release_version }} True False - fi diff --git a/.github/workflows/updateDbDevices.yaml b/.github/workflows/updateDbDevices.yaml deleted file mode 100644 index cf1200c3ef..0000000000 --- a/.github/workflows/updateDbDevices.yaml +++ /dev/null @@ -1,331 +0,0 @@ -name: Remove/Update Devices in the Database - -on: - repository_dispatch: - types: [trigger-workflow-update-database-devices] - workflow_dispatch: - inputs: - select_index: - type: choice - description: Index as test or live - options: - - Test - - Live - default: "Test" - select_action: - type: choice - description: Select what you want to do - options: - - Set sdk_support - - Remove Devices - default: "Set SDKToDevice" - regex: - type: string - description: Regex of Device uids to use (type "Spreadsheet Regex" to use spreadsheet regex) - default: "PIC32MZ2048EFH144|MCU_CARD_10_FOR_KINETIS_MK60DN512VLQ10" - delete_device: - type: boolean - description: Remove Device completely? (only used for "Remove Devices" option, clears only sdk_support if not checked) - default: false - xc8_specific: - type: boolean - description: Set XC8 support? (only used for "Set sdk_support" option) - default: false - AI_generated_sdk: - type: boolean - description: AI generated SDK? (only used for "Set sdk_support" option) - default: false - -jobs: - Update_Devices_in_Dev_Database: - if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.select_index == 'Test' }} - runs-on: ubuntu-latest - steps: - - name: Authorize Mikroe Actions App - uses: actions/create-github-app-token@v2 - id: app-token - with: - app-id: ${{ vars.MIKROE_ACTIONS }} - private-key: ${{ secrets.MIKROE_ACTIONS_KEY_AUTHORIZE }} - - - name: Checkout code - uses: actions/checkout@v6 - with: - token: ${{ steps.app-token.outputs.token }} - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '3.x' - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r scripts/requirements/shared.txt - pip install -r scripts/requirements/databases.txt - sudo apt-get install p7zip-full - - - name: Upload new database asset - run: | - python -u scripts/update_devices_db.py ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} "Test" "${{ github.event.inputs.select_action }}" "${{ github.event.inputs.regex }}" ${{ github.event.inputs.delete_device }} ${{ github.event.inputs.xc8_specific }} "--ai_sdk" ${{ github.event.inputs.AI_generated_sdk }} "--spreadsheet_link" "${{ secrets.RELEASES_SPREADSHEET }}" - - - name: Add GitHub Actions credentials - run: | - git config user.name github-actions - git config user.email github-actions@github.com - - - name: Commit and Push Changes - run: | - if [ -n "$(git status --porcelain)" ]; then - echo "Updating with new dev database"; - git add necto_db_dev.db - git commit -m "Updated necto dev database with latest merged release." - git push - else - echo "No changes made to necto_db_dev.db"; - fi - - - name: Run Index Script - env: - ES_HOST: ${{ secrets.ES_HOST }} - ES_USER: ${{ secrets.ES_USER }} - ES_PASSWORD: ${{ secrets.ES_PASSWORD }} - run: | - echo "Indexing database to TEST." - python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_TEST }} ${{ secrets.PROG_DEBUG_CODEGRIP_TEST }} "True" "latest" "True" "False" - - Update_Devices_in_Live_Database: - if: > - ((github.event_name == 'workflow_dispatch' && github.event.inputs.select_index == 'Live') || - github.event_name == 'schedule') - runs-on: ubuntu-latest - steps: - - name: Authorize Mikroe Actions App - uses: actions/create-github-app-token@v2 - id: app-token - with: - app-id: ${{ vars.MIKROE_ACTIONS }} - private-key: ${{ secrets.MIKROE_ACTIONS_KEY_AUTHORIZE }} - - - name: Checkout code - uses: actions/checkout@v6 - with: - token: ${{ steps.app-token.outputs.token }} - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '3.x' - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r scripts/requirements/shared.txt - pip install -r scripts/requirements/databases.txt - sudo apt-get install p7zip-full - - - name: Upload new database asset - Scheduled - if: ${{ github.event_name == 'schedule' }} - run: | - python -u scripts/update_devices_db.py ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} "Live" "Set sdk_support" "Spreadsheet Regex" "False" "False" "--spreadsheet_link" "${{ secrets.RELEASES_SPREADSHEET }}" - - - name: Upload new database asset - Manual Run - if: ${{ github.event_name == 'workflow_dispatch' }} - run: | - python -u scripts/update_devices_db.py ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} "Live" "${{ github.event.inputs.select_action }}" "${{ github.event.inputs.regex }}" ${{ github.event.inputs.delete_device }} ${{ github.event.inputs.xc8_specific }} "--ai_sdk" ${{ github.event.inputs.AI_generated_sdk }} "--spreadsheet_link" "${{ secrets.RELEASES_SPREADSHEET }}" - - - name: Add GitHub Actions credentials - run: | - git config user.name github-actions - git config user.email github-actions@github.com - - - name: Commit and Push Changes - run: | - if [ -n "$(git status --porcelain)" ]; then - echo "Updating with new live database"; - git add necto_db.db - git commit -m "Updated necto live database with latest merged release." - git push - curl --location --request POST '${{ secrets.ERP_DB_IMPORT_API }}' \ - --header 'Authorization: Basic ${{ secrets.ERP_DB_IMPORT_KEY }}' - else - echo "No changes made to necto_db.db"; - fi - - - name: Run Index Script - env: - ES_HOST: ${{ secrets.ES_HOST }} - ES_USER: ${{ secrets.ES_USER }} - ES_PASSWORD: ${{ secrets.ES_PASSWORD }} - run: | - echo "Indexing database to LIVE." - python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_LIVE }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} "True" "latest" "True" "False" - - Update_Devices_from_SDK: - if: ${{ github.event_name == 'repository_dispatch' }} - runs-on: ubuntu-latest - steps: - - name: Authorize Mikroe Actions App - uses: actions/create-github-app-token@v2 - id: app-token - with: - app-id: ${{ vars.MIKROE_ACTIONS }} - private-key: ${{ secrets.MIKROE_ACTIONS_KEY_AUTHORIZE }} - - - name: Checkout code - uses: actions/checkout@v6 - with: - token: ${{ steps.app-token.outputs.token }} - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '3.x' - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r scripts/requirements/shared.txt - pip install -r scripts/requirements/databases.txt - sudo apt-get install p7zip-full - - - name: Extract payload version - id: mikrosdk_payload - run: | - echo "mcu_regex=${{ github.event.client_payload.mcu_regex }}" >> $GITHUB_OUTPUT - echo "index=${{ github.event.client_payload.index }}" >> $GITHUB_OUTPUT - - - name: Upload new database asset - run: | - python -u scripts/update_devices_db.py ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} ${{ steps.mikrosdk_payload.outputs.index }} "Set sdk_support" "${{ steps.mikrosdk_payload.outputs.mcu_regex }}" "false" "false" - - - name: Add GitHub Actions credentials - run: | - git config user.name github-actions - git config user.email github-actions@github.com - - - name: Commit and Push Changes - run: | - if [[ ${{ steps.mikrosdk_payload.outputs.index }} == "Live" ]]; then - DB_NAME="necto_db.db" - else - DB_NAME="necto_db_dev.db" - fi - STATUS=$(git status --short "$DB_NAME") - if [ -z "$STATUS" ]; then - echo "No changes made to $DB_NAME"; - else - echo "Updating with new $DB_NAME"; - echo "test" - git pull - git add $DB_NAME - git commit -m "Updated $DB_NAME with latest merged release." - git push - if [[ ${{ steps.mikrosdk_payload.outputs.index }} == "Live" ]]; then - curl --location --request POST '${{ secrets.ERP_DB_IMPORT_API }}' \ - --header 'Authorization: Basic ${{ secrets.ERP_DB_IMPORT_KEY }}' - fi - fi - - - name: Run Index Script - env: - ES_HOST: ${{ secrets.ES_HOST }} - ES_USER: ${{ secrets.ES_USER }} - ES_PASSWORD: ${{ secrets.ES_PASSWORD }} - run: | - if [[ ${{ steps.mikrosdk_payload.outputs.index }} == "Test" ]]; then - echo "Indexing database to TEST." - python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_TEST }} ${{ secrets.PROG_DEBUG_CODEGRIP_TEST }} "True" "latest" "True" "False" - fi - if [[ ${{ steps.mikrosdk_payload.outputs.index }} == "Live" ]]; then - echo "Indexing database to LIVE." - python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_LIVE }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} "True" "latest" "True" "False" - fi - - Notify_Database_Update: - name: Notify Database Update Summary - needs: - - Update_Devices_in_Dev_Database - - Update_Devices_in_Live_Database - - Update_Devices_from_SDK - if: always() - runs-on: ubuntu-latest - - steps: - - name: Resolve context - id: context - run: | - # ----------------------------- - # Detect which job ran - # ----------------------------- - if [[ "${{ needs.Update_Devices_in_Live_Database.result }}" == "success" ]]; then - DB_TYPE="Live" - elif [[ "${{ needs.Update_Devices_in_Dev_Database.result }}" == "success" ]]; then - DB_TYPE="Dev" - elif [[ "${{ needs.Update_Devices_from_SDK.result }}" == "success" ]]; then - DB_TYPE="${{ github.event.client_payload.index }}" - else - echo "No database update job ran successfully." - exit 0 - fi - - # ----------------------------- - # Detect trigger type - # ----------------------------- - case "${{ github.event_name }}" in - schedule) - TRIGGER="Scheduled" - ;; - workflow_dispatch) - TRIGGER="Manual" - ;; - repository_dispatch) - TRIGGER="Repository dispatch" - ;; - *) - TRIGGER="Unknown" - ;; - esac - - # ----------------------------- - # Determine update type - # ----------------------------- - if [[ "${{ github.event_name }}" == "repository_dispatch" ]]; then - UPDATE_TYPE="Set SDK Support" - REGEX="${{ github.event.client_payload.mcu_regex }}" - else - UPDATE_TYPE="${{ github.event.inputs.select_action }}" - REGEX="${{ github.event.inputs.regex }}" - fi - - # Normalize wording - if [[ "$UPDATE_TYPE" == "Set sdk_support" ]]; then - UPDATE_TYPE="Set SDK Support" - elif [[ "$UPDATE_TYPE" == "Remove Devices" ]]; then - UPDATE_TYPE="Remove Devices" - fi - - # Spreadsheet regex fallback (scheduled runs) - if [[ "${{ github.event_name }}" == "schedule" ]]; then - REGEX="Spreadsheet Regex" - fi - - # Export - echo "db_type=$DB_TYPE" >> $GITHUB_OUTPUT - echo "trigger=$TRIGGER" >> $GITHUB_OUTPUT - echo "update_type=$UPDATE_TYPE" >> $GITHUB_OUTPUT - echo "regex=$REGEX" >> $GITHUB_OUTPUT - - - name: Notify Mattermost - Database Update - env: - MATTERMOST_WEBHOOK_URL_SDK: ${{ secrets.MATTERMOST_WEBHOOK_URL_SDK }} - run: | - MESSAGE="**NECTO Devices Database Update Completed**\n\n\ - **Database type:** ${{ steps.context.outputs.db_type }}\n\ - **GitHub trigger:** ${{ steps.context.outputs.trigger }}\n\ - **Update type:** ${{ steps.context.outputs.update_type }}\n\ - **Regex:** \`${{ steps.context.outputs.regex }}\`\n" - - curl -X POST -H 'Content-Type: application/json' \ - --data "{\"text\": \"$MESSAGE\"}" \ - $MATTERMOST_WEBHOOK_URL_SDK \ No newline at end of file diff --git a/.github/workflows/updateDbFromSdk.yaml b/.github/workflows/updateDbFromSdk.yaml deleted file mode 100644 index a6ac768171..0000000000 --- a/.github/workflows/updateDbFromSdk.yaml +++ /dev/null @@ -1,125 +0,0 @@ -name: Update Latest Database From mikroSDK - -on: - repository_dispatch: - types: [trigger-workflow-update-database-from-sdk] - -jobs: - Update-Database-From-mikroSDK: - runs-on: ubuntu-latest - steps: - - name: Authorize Mikroe Actions App - uses: actions/create-github-app-token@v2 - id: app-token - with: - app-id: ${{ vars.MIKROE_ACTIONS }} - private-key: ${{ secrets.MIKROE_ACTIONS_KEY_AUTHORIZE }} - - - name: Checkout code - uses: actions/checkout@v6 - with: - ref: ${{ github.event.client_payload.branch }} - token: ${{ steps.app-token.outputs.token }} - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '3.x' - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r scripts/requirements/shared.txt - pip install -r scripts/requirements/databases.txt - sudo apt-get install p7zip-full - - - name: Extract payload version - id: mikrosdk_payload - run: | - echo "tag_version=${{ github.event.client_payload.version }}" >> $GITHUB_OUTPUT - echo "index=${{ github.event.client_payload.index }}" >> $GITHUB_OUTPUT - - - name: Upload new database asset - run: | - if [[ ${{ steps.mikrosdk_payload.outputs.index }} == "Live" ]]; then - python -u scripts/reupload_databases.py ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} ${{ secrets.PROG_DEBUG_MIKROPROG }} ${{ secrets.PROG_DEBUG_JLINK }} "latest" ${{ steps.mikrosdk_payload.outputs.tag_version }} ${{ steps.mikrosdk_payload.outputs.index }} - else - python -u scripts/reupload_databases.py ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} ${{ secrets.PROG_DEBUG_CODEGRIP_TEST }} ${{ secrets.PROG_DEBUG_MIKROPROG }} ${{ secrets.PROG_DEBUG_JLINK }} "latest" ${{ steps.mikrosdk_payload.outputs.tag_version }} ${{ steps.mikrosdk_payload.outputs.index }} - fi - - - name: Add GitHub Actions credentials - run: | - git config user.name github-actions - git config user.email github-actions@github.com - - - name: Commit and Push Changes - run: | - if [[ ${{ steps.mikrosdk_payload.outputs.index }} == "Live" ]]; then - DB_NAME="necto_db.db" - else - DB_NAME="necto_db_dev.db" - fi - STATUS=$(git status --short "$DB_NAME") - if [ -z "$STATUS" ]; then - echo "No changes made to $DB_NAME"; - else - echo "Updating with new $DB_NAME"; - echo "test" - git pull - git add $DB_NAME - git commit -m "Updated $DB_NAME with latest merged release." - git push - if [[ ${{ steps.mikrosdk_payload.outputs.index }} == "Live" ]]; then - curl --location --request POST '${{ secrets.ERP_DB_IMPORT_API }}' \ - --header 'Authorization: Basic ${{ secrets.ERP_DB_IMPORT_KEY }}' - fi - fi - - - name: Upload and re-index Schemas - env: - ES_HOST: ${{ secrets.ES_HOST }} - ES_USER: ${{ secrets.ES_USER }} - ES_PASSWORD: ${{ secrets.ES_PASSWORD }} - ES_INDEX_TEST: ${{ secrets.ES_INDEX_TEST }} - ES_INDEX_LIVE: ${{ secrets.ES_INDEX_LIVE }} - run: | - if [[ ${{ steps.mikrosdk_payload.outputs.index }} == "Live" ]]; then - echo "Uploading and indexing to Live." - python -u scripts/update_schemas.py ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} "latest" ${{ secrets.ES_INDEX_LIVE }} "False" - else - echo "Uploading and indexing to Test and Live." - python -u scripts/update_schemas.py ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} "latest" ${{ secrets.ES_INDEX_TEST }} "False" - fi - - # Create a pull request using the GitHub API - - name: Create Pull Request - id: create_pr - if: ${{ github.event.client_payload.branch != 'main' && steps.mikrosdk_payload.outputs.index == 'Live' }} - run: | - PR_RESPONSE=$(curl -s -X POST \ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github+json" \ - https://api.github.com/repos/${{ github.repository }}/pulls \ - -d "{\"title\":\"Merge branch ${{ github.event.client_payload.branch }} into main\",\"head\":\"${{ github.event.client_payload.branch }}\",\"base\":\"main\",\"body\":\"Automatically created pull request to merge branch ${{ github.event.client_payload.branch }} into main\"}") - - echo "$PR_RESPONSE" > pr_response.json - PR_NUMBER=$(jq '.number' pr_response.json) - echo "PR NUMBER IS: ${PR_NUMBER}" - rm pr_response.json - echo "Pull request number is $PR_NUMBER" - echo "pull_request_number=$PR_NUMBER" >> $GITHUB_OUTPUT - - - name: Run Index Script - env: - ES_HOST: ${{ secrets.ES_HOST }} - ES_USER: ${{ secrets.ES_USER }} - ES_PASSWORD: ${{ secrets.ES_PASSWORD }} - run: | - if [[ ${{ steps.mikrosdk_payload.outputs.index }} == "Test" ]]; then - echo "Indexing database to TEST." - python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_TEST }} ${{ secrets.PROG_DEBUG_CODEGRIP_TEST }} "True" "latest" "True" "False" - fi - if [[ ${{ steps.mikrosdk_payload.outputs.index }} == "Live" ]]; then - echo "Indexing database to LIVE." - python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_LIVE }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} "True" "latest" "True" "False" - fi diff --git a/.github/workflows/updateSchemasAndClocks.yaml b/.github/workflows/updateSchemasAndClocks.yaml index 85d7ef0742..a61a257506 100644 --- a/.github/workflows/updateSchemasAndClocks.yaml +++ b/.github/workflows/updateSchemasAndClocks.yaml @@ -47,7 +47,6 @@ jobs: run: | python -m pip install --upgrade pip pip install -r scripts/requirements/shared.txt - pip install -r scripts/requirements/databases.txt sudo apt-get install p7zip-full - name: Determine trigger (manual/merge) diff --git a/scripts/addSdkVersion.py b/scripts/addSdkVersion.py deleted file mode 100644 index 8eaa3a2487..0000000000 --- a/scripts/addSdkVersion.py +++ /dev/null @@ -1,197 +0,0 @@ -import os, re, sys, \ - shutil, argparse, \ - sqlite3 - -from pathlib import Path - -## Import utility modules -## Append to system path -sys.path.append(str(Path(os.path.dirname(__file__)).parent.parent.absolute())) -sys.path.append(str(Path(os.path.dirname(__file__)).absolute())) - -import enums as enums -import support as utility - -def functionRegex(value, pattern): - reg = re.compile(value) - return reg.search(pattern) is not None - -def read_data_from_db(db, sql_query): - ## Open the database / connect to it - con = sqlite3.connect(db) - cur = con.cursor() - - ## Create the REGEXP function to be used in DB - con.create_function("REGEXP", 2, functionRegex) - - ## Execute the desired query - results = cur.execute(sql_query).fetchall() - # results = cur.fetchall() - - ## Close the connection - cur.close() - con.close() - - ## Return query results - return len(results), results - -def insertIntoTable(db, tableName, values, columns): - import sqlite3 - - conn = sqlite3.connect(db) - cur = conn.cursor() - numOfItems = '' - for itemCount in range(1, len(values) + 1): - numOfItems += '?,' - cur.execute(f'INSERT OR IGNORE INTO {tableName} ({columns}) VALUES ({numOfItems[:-1]})', values) - conn.commit() - conn.close() - -## Download databases or fetch from disk -def downloadDb(downloadLink, overwrite=True): - dbPath = None - if 'http' in downloadLink: - if '.7z' in downloadLink: - dbPath = os.path.join(os.path.dirname(__file__), "dbGithub.db") - if overwrite or not os.path.isfile(dbPath): - utility.extract_archive_from_url( - downloadLink, os.path.join(os.path.dirname(__file__), "dbGithub") - ) - shutil.move( - os.path.join(os.path.dirname(__file__), "dbGithub/necto_db.db"), - os.path.join(os.path.dirname(__file__), "dbGithub.db") - ) - if os.path.exists(os.path.join(os.path.dirname(__file__), "dbGithub")): - shutil.rmtree(os.path.join(os.path.dirname(__file__), "dbGithub")) - else: - dbPath = downloadLink ## Assume it is a local literal path - - return dbPath - -def filter_versions(versions): - # Filter out versions that contain non-numeric characters (e.g., words or suffixes) - filtered_versions = [v for v in versions if all(part.isdigit() for part in v.split('.'))] - return filtered_versions - -def get_highest_and_second_highest(versions): - from packaging import version - # Parse the version strings to version objects for comparison - version_objects = [version.parse(v) for v in versions] - - # Sort the versions in descending order - sorted_versions = sorted(version_objects, reverse=True) - - # Get the highest and second-highest versions - highest_version = str(sorted_versions[0]) - second_highest_version = str(sorted_versions[1]) if len(sorted_versions) > 1 else None - - return highest_version, second_highest_version - -def addSdkVersion(database, sdkVersion): - sdkVersionUid = None - sdkVersionUidPrevious = None - isSdkVersionPresent = read_data_from_db( - database, f'SELECT DISTINCT uid, version FROM SDKs WHERE version = "{sdkVersion}"' - ) - if not isSdkVersionPresent[enums.dbSync.COUNT.value]: - SDKsCollumns = 'uid, sdk_development_kit, name, legacy, icon, version, installed' - print(f'\033[33mAdding {sdkVersion} to the SDKs table for {database}.\033[0m') - insertIntoTable( - database, - 'SDKs', - [ - f'mikrosdk_v{sdkVersion.replace('.','')}', - 0, - 'mikroSDK', - 0, - 'images/mikrosdk.png', - sdkVersion, - 0 - ], - SDKsCollumns - ) - sdkVersionUid = f'mikrosdk_v{sdkVersion.replace('.','')}' - - sdkVersions = read_data_from_db( - database, f'SELECT DISTINCT version FROM SDKs WHERE name IS "mikroSDK"' - ) - - versionList = [] - for eachSdkVersion in sdkVersions[enums.dbSync.ELEMENTS.value]: - versionList.append(eachSdkVersion[0]) - currentVersion, previousVersion = get_highest_and_second_highest(filter_versions(versionList)) - sdkVersionUidPrevious = read_data_from_db( - database, f'SELECT uid FROM SDKs WHERE version = "{previousVersion}"' - )[enums.dbSync.ELEMENTS.value][0][0] - - return sdkVersionUid, sdkVersionUidPrevious - -def insertIntoSdk(database, tableName, tableCollumn, sdkUidPrevious, sdkUidNew): - for eachTable, eachUid in zip(tableName, tableCollumn): - print(f'\033[33mUpdating {eachTable} with new {sdkUidNew}.\033[0m') - allFoundValues = read_data_from_db( - database, f'SELECT * FROM {eachTable} WHERE sdk_uid = "{sdkUidPrevious}"' - ) - for eachValue in allFoundValues[enums.dbSync.ELEMENTS.value]: - formattedMessage = 'Inserted %s into %s table.\n' % (eachValue[enums.dbSync.ELEMENTS.value],eachTable) - # TODO - uncomment for debug purposes - # print(formattedMessage) - insertIntoTable( - database, - eachTable, - [ - sdkUidNew, - eachValue[enums.dbSync.ELEMENTS.value] - ], - f'sdk_uid, {eachUid}' - ) - -## Main runner -if __name__ == "__main__": - # First, check for arguments passed - parser = argparse.ArgumentParser(description='') - parser.add_argument( - '--sdkVersionByTag', - type=str, - default='', - help='GITHUB tag name used for SDK version.' - ) - - ## Parse the arguments - args = parser.parse_args() - - ## Step 1 - if links passed, download the database first - database = downloadDb( - ## Always download database from latest release - 'https://github.com/MikroElektronika/core_packages/releases/latest/download/database.7z', - False - ) - - ## Step 2 - add new sdk version - sdkVersionUidNew, sdkVersionUidPrevious = addSdkVersion(database, args.sdkVersionByTag) - ## Make sure to check if it exists already - if not sdkVersionUidNew: - raise ValueError('mikroSDK %s already exists in current database!' % args.sdkVersionByTag) - - ## Step 3 - add data to tables - insertIntoSdk( - database, - [ - 'SDKToBoard', - 'SDKToBuildSystem', - 'SDKToCompiler', - 'SDKToDevice', - 'SDKToDisplay' - ], - [ - 'board_uid', - 'build_system_uid', - 'compiler_uid', - 'device_uid', - 'display_uid' - ], - sdkVersionUidPrevious, - sdkVersionUidNew - ) - ## ------------------------------------------------------------------------------------ ## -## EOF Main runner diff --git a/scripts/build_message_codegrip.py b/scripts/build_message_codegrip.py deleted file mode 100644 index b24f76c358..0000000000 --- a/scripts/build_message_codegrip.py +++ /dev/null @@ -1,135 +0,0 @@ -import os, time, argparse -import json, pytz, sys -from elasticsearch import Elasticsearch -from datetime import datetime -import classes.class_generate_events_json as calendar_events - -import support as support - -def fetch_current_indexed_cg_packs(es : Elasticsearch, index_name): - # Search query to use - query_search = { - "size": 5000, - "query": { - "match_all": {} - } - } - - # Search the base with provided query - num_of_retries = 1 - while num_of_retries <= 10: - try: - response = es.search(index=index_name, body=query_search) - if not response['timed_out']: - break - except: - print("Executing search query - retry number %i" % num_of_retries) - num_of_retries += 1 - - all_packages = [] - for eachHit in response['hits']['hits']: - if not 'name' in eachHit['_source']: - continue - if '_type' in eachHit: - if '_doc' == eachHit['_type'] and 'codegrip_pack' in eachHit['_source']['name']: - if False == eachHit['_source']['hidden']: - all_packages.append(eachHit['_source']) - - # Sort all_packages alphabetically by the 'name' field - all_packages.sort(key=lambda x: x['name']) - - return all_packages - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Create the release post message for Web.") - parser.add_argument("title", help="Event title for calendar.") - parser.add_argument("doc_link", help="Spreadsheet table with release details - link.") - parser.add_argument("index", help="SDK packages index.") - - ## Parse the arguments - args = parser.parse_args() - - # Elasticsearch instance used for getting indexing info - num_of_retries = 1 - print("Trying to connect to ES.") - while True: - es = Elasticsearch([os.environ['ES_HOST']], http_auth=(os.environ['ES_USER'], os.environ['ES_PASSWORD'])) - if es.ping(): - break - # Wait 1 second and try again if connection fails - if 10 == num_of_retries: - # Exit if it fails 10 times, something is wrong with the server - raise ValueError("Connection to ES failed!") - print(f"Connection retry: {num_of_retries}") - num_of_retries += 1 - - time.sleep(1) - - current_date = datetime.now().strftime("%Y-%m-%d") - - # Get all indexed sdk packages - all_cg_packs = fetch_current_indexed_cg_packs(es, args.index) - - ## Update release calendar values - release_calendar = calendar_events.events_json(args.doc_link, args.title) - release_calendar.fetch_data() - ## Then generate the input file for teamup API - release_calendar.generate_file(os.path.join(os.path.dirname(__file__), 'releases.json')) - - with open(os.path.join(os.path.dirname(__file__), 'releases.json'), 'r') as file: - data = json.load(file) - - release_spreadsheet_data = '' - update_present = 0 - timezone = pytz.timezone('Europe/Belgrade') - date_message = datetime.now(timezone).strftime("%a %b %d %H:%M:%S %Z %Y") - header = f'Codegrip Packages Release for {date_message}:\n\n' - todays_release = '+ New\n' - todays_update = '\n+ Updated\n' - - for event in data["NECTO DAILY UPDATE"]["events"]: - if event['end_dt'].startswith(current_date) and event['released']: - release_spreadsheet_data += event['notes'] - - # Add newly added SDK packages to the release message - for cg_file in all_cg_packs: - # First check if this package was published during latest releases - if 'published_at' in cg_file: - # Find newly published SDK packages - if cg_file['published_at'].startswith(current_date): - # Check if it is a newly released package that is listed in release spreadsheet - if cg_file['display_name'] in release_spreadsheet_data: - # Separate Codegrip Packs to display them at the beginning of the list - todays_release += f' + {cg_file['display_name']}\n' - for mcu in cg_file['mcus']: - todays_release += f' + {mcu}\n' - # If it is not newly released package - add it to UPDATED section - else: - todays_update += f' + {cg_file['display_name']}\n' - update_present = 1 - - # Case for the days when we only updated existing Codegrip Packs, but don't release new - if todays_release == '+ New\n': - todays_release = '' - - # If there were any updates today - add them - if update_present: - todays_release += todays_update - - # Only create a Mattermost message when there are CODEGRIP packages for today's release - has_packages = bool(todays_release.strip()) - - github_output = os.environ.get('GITHUB_OUTPUT') - if github_output: - with open(github_output, 'a') as file: - file.write(f'has_packages={str(has_packages).lower()}\n') - - if has_packages: - message = header + todays_release + '\n---' - - with open(os.path.join(os.getcwd(), 'message.txt'), 'w') as file: - file.write(message) - - print(message) - else: - print("No CODEGRIP packages found for today's release. Mattermost notification will be skipped.") diff --git a/scripts/check_indexes.py b/scripts/check_indexes.py index 2faaa6c57d..a10122babf 100644 --- a/scripts/check_indexes.py +++ b/scripts/check_indexes.py @@ -4,7 +4,7 @@ import classes.class_es as es # Legacy packages for NECTO version 7.0.4 and lower -legacy_packages = ["clocks", "schemas", "database", "images", "images_sdk"] +legacy_packages = ["clocks", "schemas", "images", "images_sdk"] # Thirdparty authors for packages that are used in NECTO thirdparty_authors = ["Microchip"] @@ -116,8 +116,6 @@ def str2bool(v): package_name = f'{indexed_item['source']['name']}.7z' else: package_name = f'{indexed_item['source']['name']}.json' - if indexed_item['source']['name'] == 'database' and 'test' in args.es_index: - package_name = 'database_dev.7z' # Set gh_package_name only for github assets if 'gh_package_name' not in indexed_item['source'] and 'Device Pack' not in indexed_item['source']['category'] and indexed_item['source']['author'] not in thirdparty_authors: indexed_item['source'].update({"gh_package_name": package_name}) diff --git a/scripts/classes/release_per_vendor.py b/scripts/classes/release_per_vendor.py index f0a5c9cad1..89721a91e0 100644 --- a/scripts/classes/release_per_vendor.py +++ b/scripts/classes/release_per_vendor.py @@ -9,14 +9,8 @@ ## Special files, i.e. non vendor specific SPECIAL_RELEASE_FILENAMES = { "clocks.json", - "database.7z", - "database_dev.7z", - "database_experimental.7z", "docs.7z", - "erp_db.db", "metadata.json", - "necto_db.db", - "necto_db_dev.db", "schemas.json", } diff --git a/scripts/enums.py b/scripts/enums.py deleted file mode 100644 index af64143dd8..0000000000 --- a/scripts/enums.py +++ /dev/null @@ -1,20 +0,0 @@ -from enum import Enum - -## Class used for error handling. -## TODO - add new ones if needed. -class errType(Enum): - SUCCESS = 0 - FAIL = 1 - -## Enums used for database manipulation -class dbSync(Enum): - COUNT = 0 - ELEMENTS = 1 - BOARDTODEVICEBOARD = 0 - BOARDTODEVICEDEVICE = 1 - BOARDTODEVICEPACKAGES = 2 - DEVICETOPACKAGEUID = 0 - DEVICETOPACKAGEDEF = 1 - PROGRAMMERSPROGRAMMER = 0 - PROGRAMMERTODEVICEPROGRAMMER = 0 - PROGRAMMERTODEVICEDEVICE = 1 diff --git a/scripts/index.py b/scripts/index.py index 19d730d4b0..5d056d81dc 100644 --- a/scripts/index.py +++ b/scripts/index.py @@ -5,7 +5,6 @@ import support as support import read_microchip_index as MCHP -import read_codegrip_index as CODEGRIP from packaging.version import Version @@ -108,7 +107,6 @@ def remove_duplicate_indexed_files(es : Elasticsearch, index_name): # All package types to check for typeCheck = [ 'mcu', - 'database', 'mcu_clocks', 'mcu_schemas' ] @@ -125,13 +123,10 @@ def remove_duplicate_indexed_files(es : Elasticsearch, index_name): num_of_retries += 1 checkDict = {} - db_version = None for eachHit in response['hits']['hits']: if not 'name' in eachHit['_source']: continue name = eachHit['_source']['name'] - if name == 'database': - db_version = eachHit['_source']['version'] if '_type' in eachHit: type = eachHit['_type'] id = eachHit['_id'] @@ -149,7 +144,6 @@ def remove_duplicate_indexed_files(es : Elasticsearch, index_name): print("Removed %s/%s" % (eachId[1], eachId[0])) response = es.delete(index=index_name, id=eachId[0], doc_type=None) - return db_version def resolve_publish_date(es: Elasticsearch, index_name, package_name): # Search query to use @@ -337,7 +331,7 @@ def check_version_and_hash(es: Elasticsearch, index_name, metadata_content, toke return uploaded_asset_hash, index_hash, (uploaded_asset_hash != index_hash), new_version, existed, indexed_version # Function to index release details into Elasticsearch -def index_release_to_elasticsearch(es : Elasticsearch, index_name, release_details, token, repo, force, update_database=False, db_version=None, keep_previous_date=False): +def index_release_to_elasticsearch(es : Elasticsearch, index_name, release_details, token, repo, force, keep_previous_date=False): # Get all currently indexed items indexed_items = fetch_current_indexed_packages(es, index_name) # Iterate over each asset in the release and previous release @@ -368,8 +362,8 @@ def index_release_to_elasticsearch(es : Elasticsearch, index_name, release_detai release_details = fetch_release_details(repo, token, release_tag) print(f'\033[33mProcessing assets for: {release_details[0]['name']}\033[0m') for asset in release_details[0].get('assets', []): - # Do not index metadata or docs - if asset['name'] == 'metadata.json' or asset['name'] == 'docs.7z': + # Do not index metadata, docs or queries + if asset['name'] in {'metadata.json', 'docs.7z', 'core_queries.7z'}: continue update_package = True @@ -379,13 +373,8 @@ def index_release_to_elasticsearch(es : Elasticsearch, index_name, release_detai always_index = [ 'clocks', 'schemas', - 'database', - 'database_dev' ] - if update_database: - if name_without_extension not in always_index: - continue doc = None if name_without_extension == "clocks": @@ -443,14 +432,6 @@ def index_release_to_elasticsearch(es : Elasticsearch, index_name, release_detai update_package = True package_name = name_without_extension - if 'database' in name_without_extension: - package_name = 'database' - if ('dev' in name_without_extension) and ('test' in index_name): - print("Database test version.") - elif ('dev' not in name_without_extension) and ('live' in index_name): - print("Database live version.") - else: - continue current_hash, index_hash, check_version, new_version, existed, previous_version = check_version_and_hash(es, index_name, metadata_content[0], token, name_without_extension) @@ -501,12 +482,6 @@ def index_release_to_elasticsearch(es : Elasticsearch, index_name, release_detai } ) - # Always update the database version based on the version in elasticsearch - if 'package_name' in locals(): - if ('database' == package_name): - if doc: - doc['version'] = increase_version(previous_version, part="patch") - # Index the document if doc: # If requested to keep previous date, only update the hash value @@ -520,20 +495,14 @@ def index_release_to_elasticsearch(es : Elasticsearch, index_name, release_detai # Kibana v8 requires _type to be in body in order to have doc_type defined doc['_type'] = '_doc' if re.search(r'^.+\.(json|7z)$', asset['name']) and (update_package or force or (name_without_extension in always_index)) or keep_previous_date: - if update_database: - if name_without_extension in always_index: - resp = es.index(index=index_name, doc_type=None, id=name_without_extension, body=doc) - print(f"{resp["result"]} {resp['_id']}") - else: - resp = es.index(index=index_name, doc_type=None, id=name_without_extension, body=doc) - print(f"{resp["result"]} {resp['_id']}") - # Database is indexed as separate ID for both indexes, so skip it in this step - if (name_without_extension in always_index) and ('database' not in name_without_extension): - if ('ES_INDEX_TEST' in os.environ) and ('ES_INDEX_LIVE' in os.environ): - if index_name == os.environ['ES_INDEX_TEST']: - resp = es.index(index=os.environ['ES_INDEX_LIVE'], doc_type=None, id=name_without_extension, body=doc) - print(f"Indexed to LIVE as well.") - print(f"{resp["result"]} {resp['_id']}") + resp = es.index(index=index_name, doc_type=None, id=name_without_extension, body=doc) + print(f"{resp["result"]} {resp['_id']}") + if name_without_extension in always_index: + if ('ES_INDEX_TEST' in os.environ) and ('ES_INDEX_LIVE' in os.environ): + if index_name == os.environ['ES_INDEX_TEST']: + resp = es.index(index=os.environ['ES_INDEX_LIVE'], doc_type=None, id=name_without_extension, body=doc) + print("Indexed to LIVE as well.") + print(f"{resp["result"]} {resp['_id']}") else: print(f'\033[34mNothing to update for {name_without_extension}\033[0m') @@ -625,50 +594,6 @@ def index_microchip_packs(es: Elasticsearch, index_name: str): resp = es.index(index=index_name, doc_type=None, id=eachItem['name'], body=eachItem) print(f"{resp["result"]} {resp['_id']}") -def index_codegrip_packs(es: Elasticsearch, index_name, doc_codegrip): - package_items = CODEGRIP.convert_item_to_json(doc_codegrip, True) - - # Get the current time in UTC - current_time = datetime.now(timezone.utc).replace(microsecond=0) - # If you specifically want the 'Z' at the end instead of the offset - published_at = current_time.isoformat().replace('+00:00', 'Z') - # Get the current date and time in UTC - current_date = datetime.now().date() - - for package in package_items: - package_release_date = datetime.strptime(package_items[package]['release_date'], "%Y-%m-%dT%H:%M:%SZ").date() - package_release_date_time = datetime.strptime(package_items[package]['release_date'], "%Y-%m-%dT%H:%M:%SZ") - # Release only for packages with release date lower or equal than current date - if package_release_date <= current_date: - previous_version, new_version, mcus_to_index = CODEGRIP.get_version(es, index_name, package_items[package]['package_name'], package_items[package]['mcus'], package_items[package]['package_version']) - if previous_version != new_version and len(mcus_to_index): - doc = { - "name": package_items[package]['package_name'], - "display_name": package_items[package]['display_name'], - "author": "MIKROE", - "hidden": False, - "type": "programmer_dfp", - "version": new_version, - "package_version": package_items[package]['package_version'], - "published_at": package_release_date_time.isoformat().replace('+00:00', 'Z'), - "category": "CODEGRIP Device Pack", - "download_link": package_items[package]['download_link'], - "package_changed": True, - "install_location": package_items[package]['install_location'], - "dependencies": json.loads(package_items[package]['dependencies']), - "mcus": mcus_to_index - } - - if previous_version: - doc["published_at"] = published_at - - # Kibana v8 requires _type to be in body in order to have doc_type defined - doc['_type'] = '_doc' - resp = es.index(index=index_name, doc_type=None, id=package_items[package]['package_name'], body=doc) - - print(f"{resp["result"]} {resp['_id']}") - print(f"\033[95mVersion for asset {package_items[package]['package_name']} has been updated from {previous_version} to {new_version}") - if __name__ == '__main__': # First, check for arguments passed def str2bool(v): @@ -686,10 +611,8 @@ def str2bool(v): parser.add_argument("repo", help="Repository name, e.g., 'username/repo'") parser.add_argument("token", help="GitHub Token") parser.add_argument("select_index", help="Provided index name") - parser.add_argument('doc_codegrip', type=str, help='Spreadsheet table download link.') parser.add_argument("force_index", help="If true will update packages even if hash is the same", type=str2bool) - parser.add_argument("release_version", help="Selected release version to index to current database", type=str) - parser.add_argument("update_database", help="If true will update database.7z", type=str2bool) + parser.add_argument("release_version", help="Selected Core release version to index", type=str) parser.add_argument("promote_release_to_latest", help="Sets current release as latest", type=str2bool, default=False) parser.add_argument("--es_host", help="Elasticsearch host value", default="") parser.add_argument("--es_user", help="Elasticsearch username value", default="") @@ -724,21 +647,16 @@ def str2bool(v): time.sleep(1) # Remove any previous multiple indexes, if any - db_version = remove_duplicate_indexed_files( - es, args.select_index - ) + remove_duplicate_indexed_files(es, args.select_index) # Index microchip device family packs index_microchip_packs(es, args.select_index) - index_codegrip_packs(es, args.select_index, args.doc_codegrip) # Now index the new release index_release_to_elasticsearch( es, args.select_index, fetch_release_details(args.repo, args.token, args.release_version), args.token, args.repo, args.force_index, - args.update_database, - db_version, args.keep_previous_dates ) diff --git a/scripts/index_codegrip_packages.py b/scripts/index_codegrip_packages.py deleted file mode 100644 index d39116f497..0000000000 --- a/scripts/index_codegrip_packages.py +++ /dev/null @@ -1,113 +0,0 @@ -import os, time, argparse, json -from elasticsearch import Elasticsearch -from datetime import datetime, timezone - -import support as support -import read_codegrip_index as CODEGRIP - -def index_codegrip_packs(es: Elasticsearch, index_name, doc_codegrip, start_date, end_date): - package_items = CODEGRIP.convert_item_to_json(doc_codegrip, True) - - # Get the current time in UTC - current_time = datetime.now(timezone.utc).replace(microsecond=0) - # If you specifically want the 'Z' at the end instead of the offset - current_date = current_time.isoformat().replace('+00:00', 'Z') - - print(f"Codegrip Packages Release action has been triggered for {start_date} - {end_date} dates!") - if 'test' in index_name: - print(f"Here are the changes (for Dev NECTO):") - else: - print(f"Here are the changes (for Live NECTO):") - - for package in package_items: - package_release_date = datetime.strptime(package_items[package]['release_date'], "%Y-%m-%dT%H:%M:%SZ").date() - # Release only for packages with release date in between of requested days - if package_release_date.strftime("%Y-%m-%d") >= start_date and package_release_date.strftime("%Y-%m-%d") <= end_date: - previous_version, new_version, mcus_to_index = CODEGRIP.get_version(es, index_name, package_items[package]['package_name'], package_items[package]['mcus'], package_items[package]['package_version']) - if previous_version != new_version and len(mcus_to_index): - - if package_items[package]['release_date'].split('T')[0] == current_date.split('T')[0]: - published_at_date = current_date - else: - published_at_date = "2023-11-16T06:00:00Z" - - doc = { - "name": package_items[package]['package_name'], - "display_name": package_items[package]['display_name'], - "author": "MIKROE", - "hidden": True, - "type": "programmer_dfp", - "version": new_version, - "package_version": package_items[package]['package_version'], - # Index it to the date that is not visible in NECTO. - # After script is finished make sure to run - # https://github.com/MikroElektronika/mikrosdk_v2/actions/workflows/updateReleaseIndexDate.yaml - # with the settings set according to release spreadsheet. - "published_at": published_at_date, - "category": "CODEGRIP Device Pack", - "download_link": package_items[package]['download_link'], - "package_changed": True, - "install_location": package_items[package]['install_location'], - "dependencies": json.loads(package_items[package]['dependencies']), - "mcus": mcus_to_index - } - - # Kibana v8 requires _type to be in body in order to have doc_type defined - doc['_type'] = '_doc' - resp = es.index(index=index_name, doc_type=None, id=package_items[package]['package_name'], body=doc) - - if ('created' == resp['result'] or 'updated' == resp['result']): - print(f"- {package_items[package]['package_name']}") - print(f' - "version": {previous_version} -> {new_version}') - print(f' - "published_at": {published_at_date.split('T')[0]} instead of {package_release_date.strftime("%Y-%m-%d")}') - -if __name__ == '__main__': - # First, check for arguments passed - def str2bool(v): - if isinstance(v, bool): - return v - if v.lower() in ('yes', 'true', 't', 'y', '1'): - return True - elif v.lower() in ('no', 'false', 'f', 'n', '0'): - return False - else: - raise argparse.ArgumentTypeError('Boolean value expected.') - - # Get arguments - parser = argparse.ArgumentParser(description="Index Codegrip Packages.") - parser.add_argument("select_index", help="Provided index name") - parser.add_argument('doc_codegrip', type=str, help='Spreadsheet table download link.') - parser.add_argument('start_date', type=str, help='First date for Codegrip Packages release.') - parser.add_argument('end_date', type=str, help='Last date for Codegrip Packages release.') - parser.add_argument("--es_host", help="Elasticsearch host value", default="") - parser.add_argument("--es_user", help="Elasticsearch username value", default="") - parser.add_argument("--es_password", help="Elasticsearch password value", default="") - args = parser.parse_args() - - # For local debug purposes - if args.es_host and args.es_user and args.es_password: - es_host = args.es_host - es_user = args.es_user - es_password = args.es_password - else: - es_host = os.environ['ES_HOST'] - es_user = os.environ['ES_USER'] - es_password = os.environ['ES_PASSWORD'] - - # Elasticsearch instance used for indexing - num_of_retries = 1 - print("Trying to connect to ES.") - while True: - es = Elasticsearch([es_host], http_auth=(es_user, es_password)) - if es.ping(): - break - # Wait 1 second and try again if connection fails - if 10 == num_of_retries: - # Exit if it fails 10 times, something is wrong with the server - raise ValueError("Connection to ES failed!") - print(f"Connection retry: {num_of_retries}") - num_of_retries += 1 - - time.sleep(1) - - index_codegrip_packs(es, args.select_index, args.doc_codegrip, args.start_date, args.end_date) diff --git a/scripts/package.py b/scripts/package.py index 46b18b5b37..b20b0ab683 100644 --- a/scripts/package.py +++ b/scripts/package.py @@ -446,64 +446,7 @@ def read_data_from_db(db, sql_query): ## Return query results return len(results), results -def updateTable(db, query, newFieldValue): - try: - # Connect to existing SQLite database - conn = sqlite3.connect(db) - cur = conn.cursor() - # Update fields in the table with data - cur.execute(query, (newFieldValue,)) - # Commit changes - conn.commit() - except sqlite3.Error as e: - print("SQLite error:", e) - finally: - # Close connection - conn.close() - -def update_database(package_name, mcus, db_path): - installer_package_column = 15 - for each_pack in mcus: - for each_mcu in mcus[each_pack]['mcu_names']: - ## Replace for MCUs which have different json file names and UID in database - each_mcu = re.sub('_', '-', each_mcu) - is_present, read_data_compiler = read_data_from_db(db_path, f'SELECT compiler_uid FROM CompilerToDevice WHERE device_uid IS "{each_mcu.replace('dsPIC', 'DSPIC')}"') - is_present, read_data = read_data_from_db(db_path, f'SELECT * FROM Devices WHERE uid IS "{each_mcu.upper()}"') - counter = 0 - data_as_list_joined = [] - while counter != len(read_data): - existing_packages = {} - if read_data[counter][installer_package_column]: - if 'compiler_flags' not in read_data[counter][installer_package_column]: - existing_packages = json.loads(read_data[counter][installer_package_column]) - else: - if read_data[counter][installer_package_column - 1]: - existing_packages = json.loads(read_data[counter][installer_package_column - 1]) - data_as_list = list(read_data[counter]) - for each_compiler in list(read_data_compiler): - if 'mchp_xc' in each_compiler[0] and '_xc' in package_name: - existing_packages[each_compiler[0]] = package_name - else: - if not re.search('xc(8|16|32)', package_name): - for each_split_check in package_name.split('_')[1:]: - if re.search(each_split_check, each_compiler[0]): - existing_packages[each_compiler[0]] = package_name - data_as_list[installer_package_column] = existing_packages - data_as_list_joined.append(data_as_list) - counter += 1 - for each_list in data_as_list_joined: - if is_present: - updateTable( - db_path, - f'''UPDATE Devices SET installer_package = ? WHERE uid = "{each_mcu.upper()}"''', - json.dumps(each_list[installer_package_column]) - ) - else: - raise ValueError("%s does not exist in database!" % each_mcu) - - return - -async def package_asset(source_dir, output_dir, arch, entry_name, tag_name, packages, current_metadata, db_paths): +async def package_asset(source_dir, output_dir, arch, entry_name, tag_name, packages, current_metadata): """ Package and upload an asset as a release to GitHub """ cmake_files = find_cmake_files(os.path.join(source_dir, "cmake")) file_paths = parse_files_for_paths(cmake_files, source_dir, True) @@ -576,11 +519,8 @@ async def package_asset(source_dir, output_dir, arch, entry_name, tag_name, pack vendor = gh_uploader.resolve_mcu_vendor(data['cmake_file_path']) - packages.append({"name" : name_without_extension, "display_name": displayName, 'compilers': compilers, "version" : version, "hash" :archiveHash, "vendor" : "MIKROE", "type" : "mcu", "category": "MCU Package", "hidden" : False, 'install_location': install_location, 'vendor': vendor}) - - # Mark package for appropriate device and toolchain - for each_db in db_paths: - update_database(name_without_extension, mcuNames, each_db) + mcu_full_list = sorted({mcu for item in mcuNames.values() for mcu in item["mcu_names"]}) + packages.append({"name": name_without_extension, "display_name": displayName, 'compilers': compilers, "version": version, "hash": archiveHash, "vendor": vendor, "type": "mcu", "category": "MCU Package", "hidden": False, 'install_location': install_location, "mcus": mcu_full_list}) mcu_check = None mcu_full_list = [] @@ -734,46 +674,6 @@ def get_release_id(repo, tag_name, token): else: return response.json()['id'] -def fetch_elasticsearch_data(index_name): - # Elasticsearch instance used for indexing - num_of_retries = 1 - print("Trying to connect to ES.") - while True: - es = Elasticsearch([os.environ['ES_HOST']], http_auth=(os.environ['ES_USER'], os.environ['ES_PASSWORD'])) - if es.ping(): - break - # Wait 1 second and try again if connection fails - if 10 == num_of_retries: - # Exit if it fails 10 times, something is wrong with the server - raise ValueError("Connection to ES failed!") - print(f"Connection retry: {num_of_retries}") - num_of_retries += 1 - - time.sleep(1) - - # Search query to use - query_search = { - "size": 5000, - "query": { - "match_all": {} - } - } - - # Search the base with provided query - num_of_retries = 1 - while num_of_retries <= 10: - try: - response = es.search(index=index_name, body=query_search) - if not response['timed_out']: - break - except: - print("Executing search query - retry number %i" % num_of_retries) - num_of_retries += 1 - - for eachHit in response['hits']['hits']: - if 'database' in eachHit['_id']: - return eachHit['_source']['version'] - return None def update_metadata(new_files, version): """ Update the metadata with the new files """ @@ -781,18 +681,7 @@ def update_metadata(new_files, version): print(f"Updating metadata objects version to {version}.") for new_file in new_files: - if 'database' == new_file['name']: - db_version = fetch_elasticsearch_data(os.environ['ES_INDEX_LIVE']) - if not db_version: - db_version = version - new_file['version'] = db_version - elif 'database_dev' == new_file['name']: - db_version = fetch_elasticsearch_data(os.environ['ES_INDEX_TEST']) - if not db_version: - db_version = version - new_file['version'] = db_version - else: - new_file['version'] = version + new_file['version'] = version updated_metadata.append(new_file) @@ -805,8 +694,6 @@ def append_package(packages, package, display_name, version, install=None, categ else: install_location = f'packages/{os.path.basename(package.lower())[:-3]}' package_type = f"{os.path.basename(package.lower())[:-3]}" - if os.path.basename(package.lower()) == 'database_dev.7z': - package_type = 'database' packages.append({ "name": f"{os.path.basename(package.lower())[:-3]}", "display_name": display_name, @@ -822,7 +709,6 @@ def append_package(packages, package, display_name, version, install=None, categ async def main(token, repo, tag_name, releases_to_update): """ Main function to orchestrate packaging and uploading assets """ architectures = ["ARM", "RISCV", "PIC32", "PIC", "dsPIC", "AVR", "RL78", "RX"] - db_paths = ['necto_db.db', 'necto_db_dev.db'] current_metadata = fetch_current_metadata(repo, token) @@ -853,7 +739,7 @@ async def main(token, repo, tag_name, releases_to_update): print(f"\033[34mProcessing {source_directory} to {output_directory}\033[0m") await package_asset( source_directory, output_directory, arch, entry.name, - tag_name, packages, current_metadata, db_paths + tag_name, packages, current_metadata ) with open('mcu_packages.json', 'w') as file: json.dump(packages, file) @@ -863,9 +749,6 @@ async def main(token, repo, tag_name, releases_to_update): payload = uploader.build_release_payload_from_packages(packages, 'output') - for each_db in db_paths: - gh_uploader.append_to_payload(payload, each_db, os.path.join(parent_dir, each_db)) - # Generate clocks.json input_directory = "./" output_file = "./output/docs/clocks.json" @@ -880,24 +763,6 @@ async def main(token, repo, tag_name, releases_to_update): schemaGenerator.generate() gh_uploader.append_to_payload(payload, 'schemas.json', Path(output_file).resolve()) - # Generate database packages - for each_db in db_paths: - shutil.copy(f'./{each_db}', './utils/databases/necto_db.db') - package_suffix = '' - if 'dev' in each_db: - package_suffix = '_dev' - archive_path = compress_directory_7z(os.path.join('./utils', 'databases'), f'database{package_suffix}.7z') - append_package( - packages, archive_path, - "NECTO Database", - get_version_based_on_hash( - f'databases{package_suffix}', tag_name.replace("v", ""), - hash_directory_contents(archive_path), current_metadata - ), - f'databases' - ) - gh_uploader.append_to_payload(payload, f'database{package_suffix}.7z', Path(str(archive_path)).resolve()) - # Generate document files asset archive_path = compress_directory_7z(os.path.join('./output', 'docs'), 'docs.7z') gh_uploader.append_to_payload(payload, 'docs.7z', Path(str(archive_path)).resolve()) diff --git a/scripts/read_codegrip_index.py b/scripts/read_codegrip_index.py deleted file mode 100644 index d19470655e..0000000000 --- a/scripts/read_codegrip_index.py +++ /dev/null @@ -1,157 +0,0 @@ -import os, sqlite3, re -import support as support -from elasticsearch import Elasticsearch - -def functionRegex(value, pattern): - reg = re.compile(value) - return reg.search(pattern) is not None - -def read_data_from_db(db, sql_query): - ## Open the database / connect to it - con = sqlite3.connect(db) - cur = con.cursor() - - ## Create the REGEXP function to be used in DB - con.create_function("REGEXP", 2, functionRegex) - - ## Execute the desired query - cur.execute(sql_query) - results = [] - for row in cur.fetchall(): - results.append(row[0].lower()) - - ## Close the connection - cur.close() - con.close() - - ## Return query results - return results - -def filter_versions(versions): - # Filter out versions that contain non-numeric characters (e.g., words or suffixes) - filtered_versions = [v for v in versions if all(part.isdigit() for part in v.split('.'))] - return filtered_versions - -def get_devices_from_latest_sdk(index_name): - if 'test' in index_name: - db = 'necto_db_dev.db' - else: - db = 'necto_db.db' - sdkVersions = read_data_from_db(db, 'SELECT DISTINCT version FROM SDKs WHERE name IS "mikroSDK"') - versions = filter_versions(list(v for v in sdkVersions)) - max_version = f'mikrosdk_v{max(versions, key=lambda v: tuple(map(int, v.split('.')))).replace('.','')}' - # Get all the MCUs that are supported in NECTO - query = f''' - SELECT DISTINCT uid FROM Devices - INNER JOIN SDKToDevice ON SDKToDevice.device_uid = Devices.uid - WHERE SDKToDevice.sdk_uid REGEXP '{max_version}|mikroc\.legacy.+' - ''' - mcu_list = read_data_from_db(db, query) - - return mcu_list - -def increment_version(version): - major, minor, patch = map(int, version.split('.')) - return f"{major}.{minor}.{patch + 1}" - -def get_version(es: Elasticsearch, index_name, asset, csv_package_mcus, package_version): - # Search query to use - query_search = { - "size": 5000, - "query": { - "match_all": {} - } - } - - # Search the base with provided query - search_es_name = asset - num_of_retries = 1 - while num_of_retries <= 10: - try: - response = es.search(index=index_name, body=query_search) - if not response['timed_out']: - break - except: - print("Executing search query - retry number %i" % num_of_retries) - num_of_retries += 1 - - indexed_version = None - indexed_package_version = None - for eachHit in response['hits']['hits']: - if not 'name' in eachHit['_source']: - continue - name = eachHit['_source']['name'] - if name == search_es_name: - if 'version' in eachHit['_source']: - indexed_version = eachHit['_source']['version'] - indexed_package_version = eachHit['_source']['package_version'] - indexed_mcus = eachHit['_source']['mcus'] - - mcu_list = get_devices_from_latest_sdk(index_name) - - mcus_to_index = [] - for mcu in csv_package_mcus: - if mcu.lower() in mcu_list: - mcus_to_index.append(mcu) - - new_version = package_version - if indexed_version: - if mcus_to_index != indexed_mcus or indexed_package_version != package_version: - new_version = increment_version(indexed_version) - - return indexed_version, new_version, mcus_to_index - -def convert_item_to_json(docLink, saveToFile=False): - import urllib.request - - with urllib.request.urlopen(docLink) as f: - html = f.read().decode('utf-8') - with open(os.path.join(os.path.dirname(__file__), 'devices.txt'), 'w') as devices: - devices.write(html) - devices.close() - - import pandas as pd - import numpy as np - - # Read the CSV file - df = pd.read_csv(os.path.join(os.path.dirname(__file__), "devices.txt")) - - # Replace NaN with False for all other columns except 'package_name' - df.replace({np.nan: False}, inplace=True) - - # Drop rows where `package_name` is NaN or invalid - df = df[df['package_name'] != False] - - # Group by `package_name` and restructure data - grouped_programmer_data = {} - - for package_name, group in df.groupby("package_name"): - # Get the first row of the group to extract common package details - package_details = group.iloc[0][[ - "vendor", - "programmers", - "debuggers", - "category", - "package_name", - "package_version", - "display_name", - "install_location", - "download_link", - "dependencies", - "release_date" - ]].to_dict() - - # Add the list of MCU names under the key "mcus" - package_details["mcus"] = group["name"].tolist() - - # Store the result by package_name - grouped_programmer_data[package_name] = package_details - if os.path.exists(os.path.join(os.path.dirname(__file__), "devices.txt")): - os.remove(os.path.join(os.path.dirname(__file__), "devices.txt")) - - if saveToFile: - import json - with open(os.path.join(os.path.dirname(__file__), 'devices.json'), 'w') as json_file: - json_file.write(json.dumps(grouped_programmer_data, indent=4)) - - return grouped_programmer_data diff --git a/scripts/recursive_build.py b/scripts/recursive_build.py deleted file mode 100644 index e08133c9af..0000000000 --- a/scripts/recursive_build.py +++ /dev/null @@ -1,694 +0,0 @@ -import os, re, subprocess, shutil, json, sqlite3 - -from packaging import version - -# Global variable for local_app_data_path -local_app_data_path = '/home/runner/.MIKROE/NECTOStudio7' - -# Path for storing artifacts. -testPath = '/home/runner/test_results' - -# Path to sdk_build_automation tool. -toolPath = '/home/runner/MikroElektronika/NECTOStudio/bin/sdk_build_automation' - -# Global variable to trace failed tests. -build_failed = False - -# Supported compilers list for each architecture. -compiler_list = { - 'ARM': ['gcc_arm_none_eabi', 'clang-llvm'], - 'RISCV': ['xpack-riscv-none-embed-gcc', 'clang-llvm-riscv'], - 'PIC': ['mchp_xc8'], - 'DSPIC': ['mchp_xc16'], - 'PIC32': ['mchp_xc32'] -} - -# Define a REGEXP function for SQLite. -def regexp(expr, item): - # Handle the case where item is None - if item is None: - return False - - # Compile the regular expression and search the item - reg = re.compile(expr) - return reg.search(item) is not None - -# Extracts the SDK version from the manifest.json file. -def get_sdk_version(): - sdk_list = [] - conn = sqlite3.connect(os.path.join(local_app_data_path, 'databases', 'necto_db.db')) - - # Create REGEXP function for python script. - conn.create_function("REGEXP", 2, regexp) - cursor = conn.cursor() - - cursor.execute(f""" - SELECT uid - FROM SDKs - WHERE uid REGEXP "mikrosdk_v"; - """) - rows = cursor.fetchall() - if rows: - sdk_list.extend([row[0] for row in rows]) - - # Function to extract the numeric version from a string - def extract_version(version_string): - match = re.search(r'mikrosdk_v(\d+)', version_string) - if match: - return int(match.group(1)) # Return the version as an integer - return -1 # Return a default value if no match is found - - return sorted(sdk_list, key=extract_version)[-1] - -# Runs the bash command. -def run_cmd(cmd, changes_dict, status_key): - global build_failed - # Blue color for build tool command command. - print(f"\033[94m{cmd}\033[0m") - - try: - # Store all the output lines to print only important ones. - output = subprocess.check_output(cmd, shell=True, text=True) - for line in output.splitlines(): - if line.startswith("Building:"): - changes_dict['build_status'][status_key] = 'UNDEFINED' - # White color for the current setup build. - print(line) - elif "Build success!" in line: - changes_dict['build_status'][status_key] = 'SUCCESS' - # Green color for success. - print("\033[92m{}\033[0m".format(line)) - elif "Build failed" in line: - changes_dict['build_status'][status_key] = 'FAIL' - # Red color for failure. - print("\033[91m{}\033[0m".format(line)) - build_failed = True - - # Error handling for failed builds not to fail the job. - except subprocess.CalledProcessError as e: - for line in e.output.splitlines(): - if line.startswith("Building:"): - changes_dict['build_status'][status_key] = 'UNDEFINED' - # White color for the current setup build. - print(line) - elif "Build success!" in line: - changes_dict['build_status'][status_key] = 'SUCCESS' - # Green color for success. - print(f"\033[92m{line}\033[0m") # Green color for success - elif "Build failed" in line: - changes_dict['build_status'][status_key] = 'FAIL' - # Red color for failure. - print(f"\033[91m{line}\033[0m") # Red color for failure - build_failed = True - -# Runs the build commands for each member of mcu_list, board_list, and mcu_card_list. -def run_builds(changes_dict): - # Get the SDK version from manifest.json file. - sdk_version = get_sdk_version() - - # Run build for all MCUs from mcu_list. - print(f"\033[93mRunning build for {len(changes_dict['mcu_list'])} MCUs\033[0m") - for mcu in changes_dict['mcu_list']: - # Get the necessary compiler for the current MCU build. - compilers = get_compilers(mcu, is_mcu=True) - for compiler in compilers: - cmd = f'xvfb-run --auto-servernum --server-num=1 {toolPath} --isBareMetal "1" --compiler "{compiler}" --sdk "{sdk_version}" --board "GENERIC_ARM_BOARD" --mcu "{mcu}" --installPrefix "{testPath}/mcu_build/{compiler}"' - run_cmd(cmd, changes_dict, mcu + ' ' + compiler) - -# Returns the list of compilers based on the given name and type. -def get_compilers(name, is_mcu=True): - if is_mcu: - if any(substring in name for substring in ["SAM", "STM", "TM4C", "MK"]): - return compiler_list["ARM"] - elif any(substring in name for substring in ["GD32", "RISC"]): - return compiler_list["RISCV"] - elif "PIC32" in name: - return compiler_list["PIC32"] - elif any(substring in name for substring in ["DSPIC", "PIC24", "dsPIC"]): - return compiler_list["DSPIC"] - elif any(substring in name for substring in ["PIC18", "PIC16", "PIC12", "PIC10"]): - return compiler_list["PIC"] - elif "AT" in name and "ATSAM" not in name: - return compiler_list["AVR"] - -def functionRegex(value, pattern): - c_pattern = re.compile(r"\b" + pattern.lower() + r"\b") - return c_pattern.search(value) is not None - -def read_data_from_db(db, sql_query): - ## Open the database / connect to it - con = sqlite3.connect(db) - cur = con.cursor() - - ## Create the REGEXP function to be used in DB - con.create_function("REGEXP", 2, functionRegex) - - ## Execute the desired query - results = cur.execute(sql_query).fetchall() - - ## Close the connection - cur.close() - con.close() - - ## Return query results - return len(results), results - -def get_changed_files(branch='main'): - try: - # Run the git diff command to get the list of changed files - result = subprocess.run( - ['git', 'diff', '--name-only', branch], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - check=True - ) - - # The output is a string of file paths separated by newlines - changed_files = result.stdout.splitlines() - - return changed_files - except subprocess.CalledProcessError as e: - print(f"Error running git command: {e.stderr}") - return [] - - -def find_cmake_files(path): - files = get_changed_files('main') - cmake_files = [] - for file in files: - if 'cmake/' in file and 'delays/' not in file and file not in cmake_files: - cmake_files.append(file) - return cmake_files - -def parse_files_for_paths(cmake_files, source_dir, isGCC=None): - """ Parse cmake files to extract paths, directory contents, and regex for folder names inside if blocks relative to the source_directory """ - path_pattern = re.compile(r'set\((\w+)\s+"([^"]+)"\)') - regex_pattern = re.compile(r'if.*MATCHES\s+"([^"]+)"') # Regex to capture MATCHES condition - regex_pattern_or = re.compile(r'.*MATCHES\s+"([^"]+)"') # Regex to capture MATCHES condition - paths = {} - for file in cmake_files: - file_name = os.path.splitext(os.path.basename(file))[0] # Use filename without extension - paths[file_name] = {'files': set(), 'regex': None, 'cmake_file_path': file} # Initialize a set for paths and a regex entry per cmake file - inside_if = False - vendor = os.path.basename(os.path.dirname(file)) - with open(file, 'r') as f: - regex_array = [] - current_regex = None - for line in f: - if isGCC and 'list(APPEND local_list_include' in line: - - systemPath = line.split()[-1][:-1].replace("${vendor}", vendor) - if 'doc_ds' in systemPath or ('sam' in systemPath and re.search('^(at)?sam.+$', file_name)): - systemPath = os.path.dirname(systemPath) - systemPath = os.path.join(source_dir, systemPath) - paths[file_name]['files'].add(systemPath) - else: - if 'if(' in line and '${MCU_NAME} MATCHES' in line: - regex_match = regex_pattern.search(line) - if 'OR ${MCU_NAME} MATCHES' in regex_match.string: - check_split = regex_match.string.split('OR') - for each_split in check_split: - regex_array.append(regex_pattern_or.search(each_split).group(1)) - current_regex = '|'.join(regex_array) - else: - if regex_match: - current_regex = regex_match.group(1) # Capture the regex when it appears - regex_array.append(current_regex) - inside_if = True - elif inside_if and line.strip() == 'endif()': - inside_if = False - if current_regex: - paths[file_name]['regex'] = '|'.join(regex_array) # Save the regex before resetting - current_regex = None - elif inside_if: - matches = path_pattern.search(line) - if matches: - key, value = matches.groups() - full_path = os.path.join(source_dir, value) - if key == "SYSTEM_LIB_INCLUDE_DIR" and value: - if os.path.isdir(full_path): - for root, dirs, files in os.walk(full_path): - for file in files: - paths[file_name]['files'].add(os.path.join(root, file)) - continue - paths[file_name]['files'].add(full_path) - return paths - -def copy_files(files, output_dir, source_dir): - """ Copy individual files from source_paths to output_dir keeping the folder structure """ - for full_source_path in files: - if os.path.exists(full_source_path): - relative_path = os.path.relpath(full_source_path, start=source_dir) - full_dest_path = os.path.join(output_dir, relative_path) - os.makedirs(os.path.dirname(full_dest_path), exist_ok=True) - if os.path.isdir(full_source_path): - shutil.copytree(full_source_path, full_dest_path, dirs_exist_ok=True) - else: - shutil.copy(full_source_path, full_dest_path) - else: - print(f"File not found: {full_source_path}") - -def copy_interrupt_files(source_dir, output_dir): - """ Copy specific interrupt files that should be included in every package """ - # Copy interrupts.h to interrupts/include - source_file_h = os.path.join(source_dir, 'interrupts/include/interrupts.h') - dest_dir_h = os.path.join(output_dir, 'interrupts/include') - os.makedirs(dest_dir_h, exist_ok=True) - shutil.copy(source_file_h, dest_dir_h) - - # Copy interrupts.c to interrupts - source_file_c = os.path.join(source_dir, 'interrupts/interrupts.c') - dest_dir_c = os.path.join(output_dir, 'interrupts') - os.makedirs(dest_dir_c, exist_ok=True) - shutil.copy(source_file_c, dest_dir_c) - -def extract_mcu_names(file_name, source_dir, output_dir, regex): - """ - Copy files from a specific subdirectory in source_dir that match a regex to a corresponding subdirectory in output_dir, - maintaining the folder structure. - """ - mcus = {} - mcus[file_name] = {'mcu_names': set(), 'cores': set()} - source_subdir = os.path.join(source_dir, 'def') - - if regex: - regex_pattern = re.compile(regex, re.IGNORECASE) - - for root, dirs, files in os.walk(source_subdir): - for file in files: - if file.endswith('.json'): - mcu_name = os.path.splitext(file)[0] - if regex_pattern.match(mcu_name): - mcus[file_name]['mcu_names'].add(mcu_name) - - return mcus - -def find_first_matching_mcu_name(source_dir, regex): - """ - Find and return the filename of the first file that matches a given regex in the specified 'def' directory. - """ - if not regex: - return None - - def_dir = os.path.join(source_dir, 'def') # Define the path to the 'def' directory - regex_pattern = re.compile(regex) # Compile the regex pattern for efficiency - - # Walk through the directory - for root, dirs, files in os.walk(def_dir): - for file in files: - if file.endswith('.json'): - if regex_pattern.search(os.path.splitext(file)[0]): # Check if the file matches the regex - filename_without_extension = os.path.splitext(file)[0] # Remove the extension from the filename - return filename_without_extension # Return just the filename without its path or extension - - return None # Return None if no matching file is found - -def extract_regex_from_cmake(cmake_file): - """ Extract regex patterns from a given .cmake file, supporting complex logical conditions. """ - # Extended regex pattern to capture multiple regex conditions separated by 'OR' - regex_pattern = re.compile(r'\$\{MCU_NAME\}\s*MATCHES\s*"(.*?)"') - # Use a set to avoid duplicates if the same regex appears more than once - regexes = set() - - with open(cmake_file, 'r') as file: - content = file.read() - - # Finding all occurrences of MCU_NAME MATCHES conditions - if_conditions = re.findall(r'if\s*\((.*?)\)\s*endif', content, re.DOTALL) - for condition in if_conditions: - # Extract all regex patterns within the condition - matches = regex_pattern.findall(condition) - regexes.update(matches) - - return list(regexes) - -def copy_files_based_on_regex(source_dir, dest_dir, check_string): - if not check_string: - return - - fileCopied = False - for root, dirs, files in os.walk(source_dir): - for file in files: - if file.endswith(".cmake"): - full_path = os.path.join(root, file) - regexes = extract_regex_from_cmake(full_path) - for regex in regexes: - if re.match(regex, check_string) and not fileCopied: - # If check_string matches the regex, copy the file - fileCopied = True - dest_file_path = os.path.join(dest_dir, file) - os.makedirs(os.path.dirname(dest_file_path), exist_ok=True) - shutil.copy(full_path, dest_file_path) - if not fileCopied and 'STM32' in check_string and 'gcc_clang' in source_dir: - if check_string[6] == '7': ## in special grouped case for M7 not covered by single files - os.makedirs(dest_dir, exist_ok=True) - shutil.copy(os.path.join(root, 'm7.cmake'), os.path.join(dest_dir, 'm7.cmake')) - elif check_string[6] == '0': ## in special grouped case for M0 not covered by single files - os.makedirs(dest_dir, exist_ok=True) - shutil.copy(os.path.join(root, 'm0.cmake'), os.path.join(dest_dir, 'm0.cmake')) - - -def copy_cmake_files(cmake_file, source_dir, output_dir, regex): - relative_path = os.path.relpath(cmake_file, start=source_dir) - destination_path = os.path.join(output_dir, relative_path) - os.makedirs(os.path.dirname(destination_path), exist_ok=True) - shutil.copy(cmake_file, destination_path) - - # Copy coreUtils.cmake - shutil.copy(os.path.join(source_dir, "cmake/coreUtils.cmake"), os.path.join(output_dir, "cmake")) - - # Copy either mikroeExportConfig.cmake.in or ExportConfig.cmake.in depending on which exists - export_config_path = os.path.join(source_dir, "cmake/mikroeExportConfig.cmake.in") - fallback_config_path = os.path.join(source_dir, "cmake/ExportConfig.cmake.in") - install_headers_path = os.path.join(source_dir, "cmake/InstallHeaders.cmake.in") - if os.path.exists(export_config_path): - shutil.copy(export_config_path, os.path.join(output_dir, "cmake")) - elif os.path.exists(fallback_config_path): - shutil.copy(fallback_config_path, os.path.join(output_dir, "cmake")) - if os.path.exists(install_headers_path): - shutil.copy(install_headers_path, os.path.join(output_dir, "cmake")) - if 'gcc_clang' in source_dir: - mcuFileName = find_first_matching_mcu_name(source_dir, regex) - delays_cmake_dir = os.path.join(os.path.dirname(cmake_file), 'delays') - delay_relative_path = os.path.relpath(delays_cmake_dir, start=source_dir) - delay_destination_path = os.path.join(output_dir, delay_relative_path) - copy_files_based_on_regex(delays_cmake_dir, delay_destination_path, mcuFileName) - -def copy_schemas(mcus, source_dir, output_dir, base_path): - - for mcu in mcus: - schemas_dir = os.path.join(source_dir, 'schemas', mcu) - dest_path = os.path.join(base_path, 'schemas', mcu) - if os.path.exists(schemas_dir): - os.makedirs(dest_path, exist_ok=True) - shutil.copytree(schemas_dir, dest_path, dirs_exist_ok=True) - -def copy_interrupts(mcus, source_dir, output_dir, base_path): - - for mcu in mcus: - interrupts_dir = os.path.join(source_dir, 'interrupts', 'include', 'interrupts_mcu', mcu.lower()) - dest_path = os.path.join(base_path, 'interrupts', 'include', 'interrupts_mcu', mcu.lower()) - if os.path.exists(interrupts_dir): - os.makedirs(dest_path, exist_ok=True) - shutil.copytree(interrupts_dir, dest_path, dirs_exist_ok=True) - - copy_interrupt_files(source_dir, base_path) - -def copy_files_from_dir(mcus, source_dir, output_dir, base_path, subdirectory): - - source_subdir = os.path.join(source_dir, subdirectory) - output_subdir = os.path.join(base_path, subdirectory) - - for root, dirs, files in os.walk(source_subdir): - for file in files: - if os.path.basename(file) == 'mcu.h': - # Special rule for 'mcu.h' files, ensure the directory matches the regex - if 'XC16' in root: - mcuCheck = os.path.basename(root) - else: - mcuCheck = os.path.basename(root).upper() - if mcuCheck in mcus: - relative_path = os.path.relpath(root, start=source_subdir) - full_dest_path = os.path.join(output_subdir, relative_path) - shutil.copytree(root, full_dest_path, dirs_exist_ok=True) - if os.path.splitext(os.path.basename(file))[0].upper().replace('DSPIC', 'dsPIC') in mcus: - if 'gcc_clang' in source_subdir and re.match('^MKV?.+XXX.+$', file): - continue - else: - full_source_path = os.path.join(root, file) - relative_path = os.path.relpath(full_source_path, start=source_subdir) - full_dest_path = os.path.join(output_subdir, relative_path) - os.makedirs(os.path.dirname(full_dest_path), exist_ok=True) - shutil.copy(full_source_path, full_dest_path) - -def copy_delays(cores, source_dir, output_dir, base_path): - for core in cores: - delays_dir = os.path.join(source_dir, 'delays', core.lower()) - dest_path = os.path.join(base_path, 'delays', core.lower()) - if os.path.exists(delays_dir): - os.makedirs(dest_path, exist_ok=True) - shutil.copytree(delays_dir, dest_path, dirs_exist_ok=True) - -# Function to replace placeholders in a single file -def replace_placeholders_in_file(source_file, dest_file, replacements): - with open(source_file, 'r') as file: - file_content = file.read() - - # Replace placeholders - for placeholder, replacement in replacements.items(): - if replacement is not None: - file_content = file_content.replace(placeholder, replacement) - - if not os.path.exists(os.path.dirname(dest_file)): - os.makedirs(os.path.dirname(dest_file)) - - # Write the updated content to the destination file - with open(dest_file, 'w') as file: - file.write(file_content) - -def get_core_from_def(file_path): - # Check if the file exists - if os.path.exists(file_path): - # Open the JSON file and load its contents - with open(file_path, 'r') as file: - data = json.load(file) - - # Check if the "core" key exists in the JSON data - if 'core' in data: - core = data['core'] - if core == 'M7EF': - core = 'M7' - else: - print(f'Warning: "core" key not found in {file_path}') - else: - print(f'Error: File {file_path} does not exist.') - - return core - -def get_core(mcuNames, package_name, cmake_file, source_dir, changes_dict): - for mcu_name in mcuNames[cmake_file]['mcu_names']: - core = get_core_from_def(os.path.join(source_dir, "def", f"{mcu_name}.json")) - mcuNames[cmake_file]['cores'].add(core) - changes_dict['mcu_list'].append(mcu_name) - -def filter_versions(versions): - # Filter out versions that contain non-numeric characters (e.g., words or suffixes) - filtered_versions = [v for v in versions if all(part.isdigit() for part in v.split('.'))] - return filtered_versions - -def insertIntoTable(db, tableName, values, columns): - import sqlite3 - - conn = sqlite3.connect(db) - cur = conn.cursor() - numOfItems = '' - for itemCount in range(1, len(values) + 1): - numOfItems += '?,' - cur.execute(f'INSERT OR IGNORE INTO {tableName} ({columns}) VALUES ({numOfItems[:-1]})', values) - conn.commit() - conn.close() - -def updateDevicesFromCore(dbs, queries): - allDevicesDirs = os.listdir(queries) - for eachDeviceDir in allDevicesDirs: - currentDeviceDir = os.path.join(queries, eachDeviceDir) - currentDeviceFiles = os.listdir(currentDeviceDir) - - for eachDb in dbs: - if eachDb: - if 'Devices.json' in currentDeviceFiles: - with open(os.path.join(currentDeviceDir, 'Devices.json'), 'r') as file: - device = json.load(file) - file.close() - values = [] - collumns = [] - for eachKey in device.keys(): - collumns.append(eachKey) - values.append(device[eachKey]) - insertIntoTable( - eachDb, - 'Devices', - values, - ','.join(collumns) - ) - - if 'LinkerTables.json' in currentDeviceFiles: - with open(os.path.join(currentDeviceDir, 'LinkerTables.json'), 'r') as file: - linkerTables = json.load(file) - file.close() - table_keys = [list(table.keys())[0] for table in linkerTables['tables']] - for eachTableKey in table_keys: - collumns = ['device_uid'] - values = [linkerTables['device_uid']] - for eachKey in linkerTables['tables']: - if eachTableKey in eachKey: - collumns.append(list(eachKey[eachTableKey].keys())[0]) - if 'SDKToDevice' == eachTableKey: - sdkVersions = read_data_from_db(eachDb, 'SELECT DISTINCT version FROM SDKs WHERE name IS "mikroSDK"') - versions = filter_versions(list(v[0] for v in sdkVersions[1])) - threshold_version = version.parse(eachKey[eachTableKey][collumns[1]][:-1]) - filtered_versions = [f'mikrosdk_v{v.replace('.','')}' for v in versions if version.parse(v) >= threshold_version] - values.append(filtered_versions) - # Add Packages if they are not present in the database - elif 'DeviceToPackage' == eachTableKey: - package_uids = linkerTables['tables'][2]['DeviceToPackage']['package_uid'] - for package_uid in package_uids: - pin_count = package_uid.split('/')[0] - package_name = package_uid.split('/')[1] - insertIntoTable( - eachDb, - 'Packages', - [ - pin_count, - package_uid, - package_uid, - "", - '{"_MSDK_PACKAGE_NAME_":"' + package_name + '","_MSDK_DIP_SOCKET_TYPE_":""}' - ], - 'pin_count,name,uid,stm_sdk_config,sdk_config' - ) - values.append(eachKey[eachTableKey][collumns[1]]) - else: - values.append(eachKey[eachTableKey][collumns[1]]) - break - if list == type(values[1]): - for eachValue in values[1]: - insertIntoTable( - eachDb, - eachTableKey, - [ - values[0], - eachValue - ], - ','.join(collumns) - ) - else: - insertIntoTable( - eachDb, - eachTableKey, - values, - ','.join(collumns) - ) - - return - -def package_asset(source_dir, output_dir, arch, entry_name, changes_dict): - cmake_files = find_cmake_files(source_dir) - file_paths = parse_files_for_paths(cmake_files, source_dir, True) - for cmake_file, data in file_paths.items(): - base_output_dir = os.path.join(output_dir, f"{arch.lower()}_{entry_name.lower()}_{cmake_file}") # Subdirectory for this .cmake file - # Copy the .cmake file into the package directory - copy_cmake_files(data['cmake_file_path'], source_dir, base_output_dir, data['regex']) - - mcuNames = extract_mcu_names(cmake_file, source_dir, output_dir, data['regex']) - - # Copy individual files - copy_files(data['files'], base_output_dir, source_dir) - # Copy schema directories - copy_schemas(mcuNames[cmake_file]['mcu_names'], source_dir, output_dir, base_output_dir) - copy_interrupts(mcuNames[cmake_file]['mcu_names'], source_dir, output_dir, base_output_dir) - # Copy defs - copy_files_from_dir(mcuNames[cmake_file]['mcu_names'], source_dir, output_dir, base_output_dir, 'def') - # Copy startups - copy_files_from_dir(mcuNames[cmake_file]['mcu_names'], source_dir, output_dir, base_output_dir, 'startup') - # Copy linker scirpts - copy_files_from_dir(mcuNames[cmake_file]['mcu_names'], source_dir, output_dir, base_output_dir, 'linker_scripts') - - get_core(mcuNames, f"{arch.lower()}_{entry_name.lower()}_{cmake_file}", cmake_file, source_dir, changes_dict) - coreQueriesPath = os.path.join(os.getcwd(), 'resources/queries') - if os.path.exists(os.path.join(coreQueriesPath, 'mcus')): - updateDevicesFromCore([f"{local_app_data_path}/databases/necto_db.db"], os.path.join(coreQueriesPath, 'mcus')) - - # Copy delay files - copy_delays(mcuNames[cmake_file]['cores'], source_dir, output_dir, base_output_dir) - # Copy std_library to every package - std_library_path = os.path.join(source_dir, 'std_library') - if os.path.exists(std_library_path): - shutil.copytree(std_library_path, os.path.join(base_output_dir, "std_library"), dirs_exist_ok=True) - # Copy include to every package - include_path = os.path.join(source_dir, 'include') - if os.path.exists(include_path): - shutil.copytree(include_path, os.path.join(base_output_dir, "include"), dirs_exist_ok=True) - # Copy cstdio to every package - cstdio_path = os.path.join(source_dir, 'cstdio') - if os.path.exists(cstdio_path): - shutil.copytree(cstdio_path, os.path.join(base_output_dir, "cstdio"), dirs_exist_ok=True) - - # Copy common to every package - shutil.copytree(os.path.join(source_dir, 'common'), os.path.join(base_output_dir, "common"), dirs_exist_ok=True) - # Copy base CMakeLists.txt to every package - shutil.copy(os.path.join(source_dir, "CMakeLists.txt"), base_output_dir) - - # Finally copy everthing to AppData location - shutil.copytree(base_output_dir, os.path.join(local_app_data_path, "packages", "core", arch, entry_name, f"{arch.lower()}_{entry_name.lower()}_{cmake_file}")) - - # Copy packages to artifacts as well - shutil.copytree(base_output_dir, os.path.join(testPath, "packages", f"{arch.lower()}_{entry_name.lower()}_{cmake_file}")) - -# Writes the result dictionary to a JSON file and ensures testPath exists. -def write_results_to_file(changes_dict): - os.makedirs(testPath, exist_ok=True) - - with open(f'{testPath}/built_changes.json', 'w+') as json_file: - json.dump(changes_dict, json_file, indent=4) - - print(f"All the data for build has been written to {testPath}/built_changes.json") - -def main(): - files = get_changed_files('main') - archs = [] - architectures = ["ARM", "RISCV", "PIC32", "PIC", "dsPIC", "AVR"] - valid_entries = ["gcc_clang", "XC32", "XC16", "XC8"] - for file in files: - for architecture in architectures: - if architecture in file and architecture not in archs: - archs.append(architecture) - changes_dict = { - 'mcu_list': [], - 'build_status': {} - } - - for arch in archs: - root_source_directory = f"./{arch}" - root_output_directory = f"./output/{arch}" - # List directories directly under the root source directory - try: - with os.scandir(root_source_directory) as entries: - print(entries) - for entry in entries: - # Don't process packaging for the MikroC entries - if entry.name not in valid_entries: - continue - print(root_source_directory) - print(entry) - if entry.is_dir(): - source_directory = os.path.join(root_source_directory, entry.name) - output_directory = os.path.join(root_output_directory, entry.name) - - print(f"Processing {source_directory} to {output_directory}") - package_asset(source_directory, output_directory, arch, entry.name, changes_dict) - except Exception as e: - print(f"Failed to process directories in {root_source_directory}: {e}") - print("\033[93mSomething went wrong while configuring the packages, chack manually.\033[0m") - - print("\033[93mAll requested core packages have been generated successfully.\033[0m") - run_builds(changes_dict) - - # Write all the used info for building to artifact folder. - write_results_to_file(changes_dict) - - shutil.copyfile(os.path.join(local_app_data_path, 'databases', 'necto_db.db'), os.path.join(testPath, 'necto_db.db')) - - if build_failed == True: - # Red text for failure. - print("\033[91mRecursive Build Failed!\033[0m") - # Fail the job as well. - exit(1) - else: - # Green text for success. - print("\033[92mRecursive Build Success!\033[0m") - - -if __name__ == "__main__": - main() diff --git a/scripts/release_mcus.py b/scripts/release_mcus.py index 9d73ceb2c3..f82e72766e 100644 --- a/scripts/release_mcus.py +++ b/scripts/release_mcus.py @@ -477,124 +477,6 @@ def read_data_from_db(db, sql_query): ## Return query results return len(results), results -def insertIntoTable(db, tableName, values, columns): - conn = sqlite3.connect(db) - cur = conn.cursor() - numOfItems = '' - for itemCount in range(1, len(values) + 1): - numOfItems += '?,' - cur.execute(f'INSERT OR IGNORE INTO {tableName} ({columns}) VALUES ({numOfItems[:-1]})', values) - conn.commit() - conn.close() - -def updateTable(db, query, newFieldValue): - try: - # Connect to existing SQLite database - conn = sqlite3.connect(db) - cur = conn.cursor() - # Update fields in the table with data - cur.execute(query, (newFieldValue,)) - # Commit changes - conn.commit() - except sqlite3.Error as e: - print("SQLite error:", e) - finally: - # Close connection - conn.close() - -def deleteFromTable(db, sql_query): - try: - sqliteConnection = sqlite3.connect(db) - cursor = sqliteConnection.cursor() - - # Deleting single record now - cursor.execute(sql_query) - sqliteConnection.commit() - cursor.close() - except sqlite3.Error as error: - print("Failed to delete record from sqlite table", error) - finally: - if sqliteConnection: - sqliteConnection.close() - -def update_database(package_name, mcus, db_path): - installer_package_column = 15 - for each_pack in mcus: - for each_mcu in mcus[each_pack]['mcu_names']: - ## Replace for MCUs which have different json file names and UID in database - each_mcu = re.sub('_', '-', each_mcu) - is_present, read_data_compiler = read_data_from_db(db_path, f'SELECT compiler_uid FROM CompilerToDevice WHERE device_uid IS "{each_mcu.replace('dsPIC', 'DSPIC')}"') - is_present, read_data = read_data_from_db(db_path, f'SELECT * FROM Devices WHERE uid IS "{each_mcu.upper()}"') - counter = 0 - data_as_list_joined = [] - while counter != len(read_data): - existing_packages = {} - if read_data[counter][installer_package_column]: - if 'compiler_flags' not in read_data[counter][installer_package_column]: - existing_packages = json.loads(read_data[counter][installer_package_column]) - else: - if read_data[counter][installer_package_column - 1]: - existing_packages = json.loads(read_data[counter][installer_package_column - 1]) - data_as_list = list(read_data[counter]) - for each_compiler in list(read_data_compiler): - if 'mchp_xc' in each_compiler[0] and '_xc' in package_name: - existing_packages[each_compiler[0]] = package_name - else: - if not re.search('xc(8|16|32)', package_name): - for each_split_check in package_name.split('_')[1:]: - if re.search(each_split_check, each_compiler[0]): - existing_packages[each_compiler[0]] = package_name - data_as_list[installer_package_column] = existing_packages - data_as_list_joined.append(data_as_list) - counter += 1 - for each_list in data_as_list_joined: - if is_present: - updateTable( - db_path, - f'''UPDATE Devices SET installer_package = ? WHERE uid = "{each_mcu.upper()}"''', - json.dumps(each_list[installer_package_column]) - ) - else: - raise ValueError("%s does not exist in database!" % each_mcu) - - return - -def fetch_existing_asset_names(release): - return [asset['name'] for asset in release['assets']] - -def fetch_current_indexed_packages(es : Elasticsearch, index_name): - # Search query to use - query_search = { - "size": 5000, - "query": { - "match_all": {} - } - } - - # Search the base with provided query - num_of_retries = 1 - while num_of_retries <= 10: - try: - response = es.search(index=index_name, body=query_search) - if not response['timed_out']: - break - except: - print("Executing search query - retry number %i" % num_of_retries) - num_of_retries += 1 - - all_packages = [] - for eachHit in response['hits']['hits']: - if not 'name' in eachHit['_source']: - continue - if '_type' in eachHit: - if '_doc' == eachHit['_type']: - all_packages.append(eachHit['_source']) - - # Sort all_packages alphabetically by the 'name' field - all_packages.sort(key=lambda x: x['name']) - - return all_packages - async def package_asset(source_dir, output_dir, arch, entry_name, packages, current_metadata, db_paths, latest_release=None): """ Package and upload an asset as a release to GitHub """ cmake_files = find_cmake_files(os.path.join(source_dir, "cmake")) @@ -668,10 +550,6 @@ async def package_asset(source_dir, output_dir, arch, entry_name, packages, curr packages.append({"name" : name_without_extension, "display_name": displayName, 'compilers': compilers, "version" : version, "hash" :archiveHash, "vendor" : "MIKROE", "type" : "mcu", "category": "MCU Package", "hidden" : False, 'install_location': install_location, 'vendor': vendor}) - # Mark package for appropriate device and toolchain - for each_db in db_paths: - update_database(name_without_extension, mcuNames, each_db) - mcu_check = None mcu_full_list = [] for each_pack in mcuNames: @@ -774,48 +652,6 @@ def get_version_based_on_hash(package_name, version, hash_value, current_metadat # If the package is not found or the hash doesn't match, return the provided version return version -def fetch_elasticsearch_data(index_name): - # Elasticsearch instance used for indexing - num_of_retries = 1 - print("Trying to connect to ES.") - while True: - es = Elasticsearch([os.environ['ES_HOST']], http_auth=(os.environ['ES_USER'], os.environ['ES_PASSWORD'])) - if es.ping(): - break - # Wait 1 second and try again if connection fails - if 10 == num_of_retries: - # Exit if it fails 10 times, something is wrong with the server - raise ValueError("Connection to ES failed!") - print(f"Connection retry: {num_of_retries}") - num_of_retries += 1 - - time.sleep(1) - - # Search query to use - query_search = { - "size": 5000, - "query": { - "match_all": {} - } - } - - # Search the base with provided query - num_of_retries = 1 - while num_of_retries <= 10: - try: - response = es.search(index=index_name, body=query_search) - if not response['timed_out']: - break - except: - print("Executing search query - retry number %i" % num_of_retries) - num_of_retries += 1 - - for eachHit in response['hits']['hits']: - if eachHit['_id'] == 'database': - if eachHit['_source']['name'] == 'database': - return eachHit['_source']['version'] - - return None def update_metadata(current_metadata, new_files, version): """ Update the metadata with the new files """ @@ -824,13 +660,7 @@ def update_metadata(current_metadata, new_files, version): print(f"Updating metadata objects version to {version}.") for new_file in new_files: - if 'database' == new_file['name']: - db_version = fetch_elasticsearch_data(os.environ['ES_INDEX_LIVE']) - if not db_version: - db_version = version - new_file['version'] = db_version - else: - new_file['version'] = version + new_file['version'] = version updated_metadata.append(new_file) @@ -847,8 +677,6 @@ def append_package(packages, package, display_name, version, install=None, categ hash_value = hash else: hash_value = hash_directory_contents(package[:-3]) - if os.path.basename(package.lower()) == 'database_dev.7z': - package_type = 'database' packages.append({ "name": f"{os.path.basename(package.lower())[:-3]}", "display_name": display_name, @@ -881,8 +709,6 @@ async def main(token, repo, tag_name, live=False): """ Main function to orchestrate packaging and uploading assets """ architectures = ["ARM", "RISCV", "PIC32", "PIC", "dsPIC", "AVR", "RL78", "RX"] - db_paths = ['necto_db_dev.db'] - current_metadata = fetch_current_metadata(repo, token) latest_release = fetch_latest_release_version(repo, token) @@ -931,7 +757,7 @@ async def main(token, repo, tag_name, live=False): print(f"\033[34mProcessing {source_directory} to {output_directory}\033[0m") await package_asset( source_directory, output_directory, arch, entry.name, - packages, current_metadata, db_paths, latest_release + packages, current_metadata, latest_release ) with open('mcu_packages.json', 'w') as file: json.dump(packages, file) @@ -941,9 +767,6 @@ async def main(token, repo, tag_name, live=False): payload = uploader.build_release_payload_from_packages(packages, 'output') - for each_db in db_paths: - gh_uploader.append_to_payload(payload, each_db, os.path.join(parent_dir, each_db)) - # Generate clocks.json if not live: input_directory = "./" @@ -960,26 +783,6 @@ async def main(token, repo, tag_name, live=False): schemaGenerator.generate() gh_uploader.append_to_payload(payload, 'schemas.json', Path(output_file).resolve()) - # Generate database packages - for each_db in db_paths: - shutil.copy(f'./{each_db}', './utils/databases/necto_db.db') - package_suffix = '' - if 'dev' in each_db: - package_suffix = '_dev' - archive_path = compress_directory_7z(os.path.join('./utils', 'databases'), f'database{package_suffix}.7z') - current_db_hash = hash_directory_contents(os.path.join('./utils', 'databases')) - append_package( - packages, archive_path, - "NECTO Database", - get_version_based_on_hash( - f'database{package_suffix}', (latest_release['tag_name']).replace("v", ""), - current_db_hash, current_metadata - ), - f'databases', - hash=current_db_hash - ) - gh_uploader.append_to_payload(payload, f'database{package_suffix}.7z', Path(str(archive_path)).resolve()) - # Generate document files asset if not live: archive_path = compress_directory_7z(os.path.join('./output', 'docs'), 'docs.7z') @@ -999,7 +802,7 @@ async def main(token, repo, tag_name, live=False): parser.add_argument("token", help="GitHub Token") parser.add_argument("repo", help="Repository name, e.g., 'username/repo'") parser.add_argument("tag_name", help="Tag name from the release") - parser.add_argument("--live", help="Upload only database?", type=bool, default=False) + parser.add_argument("--live", help="Upload MCU packages only", type=bool, default=False) args = parser.parse_args() print("Starting the upload process...") asyncio.run(main(args.token, args.repo, args.tag_name, args.live)) diff --git a/scripts/requirements/databases.txt b/scripts/requirements/databases.txt deleted file mode 100644 index 70d5ed75c1..0000000000 --- a/scripts/requirements/databases.txt +++ /dev/null @@ -1,3 +0,0 @@ -openpyxl -pandas -packaging diff --git a/scripts/reupload_databases.py b/scripts/reupload_databases.py deleted file mode 100644 index b819035996..0000000000 --- a/scripts/reupload_databases.py +++ /dev/null @@ -1,1767 +0,0 @@ -import os, re, sys, \ - shutil, argparse, \ - sqlite3, json, \ - asyncio, aiohttp, \ - subprocess, aiofiles, \ - requests, hashlib, \ - xmltodict, time - - -from pathlib import Path -from packaging import version -from urllib.parse import urlparse -from collections import defaultdict -from packaging.version import Version - -import xml.etree.ElementTree as ET - -## Import utility modules -## Append to system path -sys.path.append(str(Path(os.path.dirname(__file__)).parent.parent.absolute())) -sys.path.append(str(Path(os.path.dirname(__file__)).absolute())) - -import enums as enums -import support as utility -import addSdkVersion as sdk -import read_microchip_index as MCHP - -entranceCheckProg = True -entranceCheckDebug = True - -START_TIME = time.perf_counter() - -mcuCardCheckList = [ - 'CARD', 'SIBRAIN', 'MICROMOD', 'PIM' -] - -def functionRegex(value, pattern): - reg = re.compile(value) - return reg.search(pattern) is not None - -def read_data_from_db(db, sql_query): - ## Open the database / connect to it - con = sqlite3.connect(db) - cur = con.cursor() - - ## Create the REGEXP function to be used in DB - con.create_function("REGEXP", 2, functionRegex) - - ## Execute the desired query - results = cur.execute(sql_query).fetchall() - # results = cur.fetchall() - - ## Close the connection - cur.close() - con.close() - - ## Return query results - return len(results), results - -def column_exists(db, table_name, column_name): - try: - with sqlite3.connect(db) as conn: - cursor = conn.cursor() - cursor.execute(f"PRAGMA table_info({table_name})") - columns = [row[1] for row in cursor.fetchall()] - return column_name in columns - except sqlite3.Error: - return False - -def addCollumnToTable(db, tableName, collumnName, collumnType, defaultValue='NoDefault'): - import sqlite3 - - conn = sqlite3.connect(db) - cur = conn.cursor() - if defaultValue == 'NoDefault': - cur.execute(f'ALTER TABLE {tableName} ADD COLUMN {collumnName} {collumnType};') - else: - cur.execute(f'ALTER TABLE {tableName} ADD COLUMN {collumnName} {collumnType} default {defaultValue};') - conn.commit() - conn.close() - -def insertIntoTable(db, tableName, values, columns): - import sqlite3 - - conn = sqlite3.connect(db) - cur = conn.cursor() - numOfItems = '' - for itemCount in range(1, len(values) + 1): - numOfItems += '?,' - cur.execute(f'INSERT OR IGNORE INTO {tableName} ({columns}) VALUES ({numOfItems[:-1]})', values) - conn.commit() - conn.close() - -# MIKROE boards regex -MIKROE_REGEX_BOARDS = re.compile( - r'^(FUSION_FOR.+|MIKROMEDIA_.+|UNI_CLICKER|EASYPIC.+|EASYMX_.+|CLICKER|' - r'FLIP_AND_CLICK_.+|PICPLC16|6LOWPAN|HEXIWEAR|FLOWPAW|QUAIL|EASY.+|UNI_DS|MINI_32)$' -) - -def update_vendor(db, uid, vendor): - # Do not update GENERIC boards - if uid.startswith('GENERIC'): - return - - # Override vendor to MIKROE if uid matches - # (for MIKROE boards only) - if MIKROE_REGEX_BOARDS.match(str(uid)): - vendor = "MIKROE" - - conn = sqlite3.connect(db) - cur = conn.cursor() - - cur.execute( - "UPDATE Boards SET Vendor = ? WHERE uid = ?", - (vendor, uid) - ) - - conn.commit() - conn.close() - -def deleteFromTable(db, sql_query): - try: - sqliteConnection = sqlite3.connect(db) - cursor = sqliteConnection.cursor() - - # Deleting single record now - cursor.execute(sql_query) - sqliteConnection.commit() - cursor.close() - except sqlite3.Error as error: - print("Failed to delete record from sqlite table", error) - finally: - if sqliteConnection: - sqliteConnection.close() - -def updateTableCollumn(db, table, collumn, setNewValue, collumnIf, collumnIfValue, customQuery=None): - import sqlite3 - - conn = sqlite3.connect(db) - cur = conn.cursor() - if customQuery: - cur.execute(customQuery) - else: - cur.execute(f'UPDATE {table} SET {collumn} = "{setNewValue}" WHERE {collumnIf} = "{collumnIfValue}"') - conn.commit() - conn.close() - -def filter_versions(versions): - # Filter out versions that contain non-numeric characters (e.g., words or suffixes) - filtered_versions = [v for v in versions if all(part.isdigit() for part in v.split('.'))] - return filtered_versions - -def get_highest_and_second_highest(versions): - from packaging import version - # Parse the version strings to version objects for comparison - version_objects = [version.parse(v) for v in versions] - - # Sort the versions in descending order - sorted_versions = sorted(version_objects, reverse=True) - - # Get the highest and second-highest versions - highest_version = str(sorted_versions[0]) - second_highest_version = str(sorted_versions[1]) if len(sorted_versions) > 1 else None - - return highest_version, second_highest_version - -def find_and_convert_xml_files(base_path): - device_dict = defaultdict(list) - - for root, _, files in os.walk(base_path): - if 'device_support.xml' in files: - xml_path = os.path.join(root, 'device_support.xml') - root_folder = os.path.basename(root) # Get the root folder name - try: - tree = ET.parse(xml_path) - root_element = tree.getroot() - extract_devices(root_element, root_folder, device_dict) - except ET.ParseError as e: - print(f"Error parsing {xml_path}: {e}") - - return dict(device_dict) - -def extract_devices(root, root_folder, device_dict, namespace="{http://crownking/mplab}"): - for family in root.findall(f"{namespace}family"): - for device in family.findall(f"{namespace}device"): - device_name = device.attrib.get(f"{{http://crownking/mplab}}name", "Unknown") - support = device.find(f"{namespace}support") - support_attributes = support.attrib if support is not None else {} - - device_dict[device_name].append({ - "root_folder": root_folder, - "support": support_attributes - }) - -def filter_releases_by_version(json_data): - def get_highest_version(versions): - return max(versions, key=Version) - - def fetch_latest_release(releases, version): - if isinstance(releases, (list, tuple)): - for release in releases: - if release['@version'] == version: - return release['atmel:devices']['atmel:device'] - else: - return releases['atmel:devices']['atmel:device'] - - # Extract the pdsc items from the JSON data - pdsc_items = json_data.get('idx', {}).get('pdsc', []) - - # Then filter out only the TP packs - dfp_tp_packs = [pdsc_item for pdsc_item in pdsc_items if re.search('TP\.pdsc', pdsc_item['@name'])] - - # Iterate through each pdsc item - dfp_tp_link_list = [] - for dfp_tp_pack in dfp_tp_packs: - releases = dfp_tp_pack.get('atmel:releases', {}).get('atmel:release', []) - if isinstance(releases, (list, tuple)): - max_version = get_highest_version([release['@version'] for release in releases]) - else: - max_version = releases['@version'] - - if '@version' in releases: - dfp_tp_link_list.append(f'https://{dfp_tp_pack['@url']}/{utility.drop_extension(dfp_tp_pack['@name'])}.{dfp_tp_pack['@version']}.atpack') - else: - for release in releases: - if release['@version'] == max_version: - dfp_tp_link_list.append(f'https://{dfp_tp_pack['@url']}/{utility.drop_extension(dfp_tp_pack['@name'])}.{dfp_tp_pack['@version']}.atpack') - - return dfp_tp_link_list - -def fetch_latest_package_links(xml_content): - # Form download links to latest packages - return filter_releases_by_version(xml_content) - -## Download databases or fetch from disk -def downloadDb(downloadLink, overwrite=True): - dbPath1 = None - dbPath2 = None - if 'http' in downloadLink: - if '.7z' in downloadLink: - dbPath1 = os.path.join(os.path.dirname(__file__), "databases/necto_db.db") - if overwrite or not os.path.isfile(dbPath1): - utility.extract_archive_from_url( - downloadLink, os.path.join(os.path.dirname(__file__), "databases") - ) - if 'database_dev' not in downloadLink: - dbPath2 = os.path.join(os.path.dirname(__file__), "erp_db.db") - if overwrite or not os.path.isfile(dbPath2): - shutil.copyfile(dbPath1, dbPath2) - else: - dbPath1 = downloadLink ## Assume it is a local literal path - - return dbPath1, dbPath2 - -def checkDeviceDetails(db, allDevicesGithub): - if allDevicesGithub[enums.dbSync.COUNT.value]: - deviceDetailsColumns = 'uid, graphic_mcu, notes, datasheet_url, is_mcu_card, device_uid' - for eachDevice in allDevicesGithub[enums.dbSync.ELEMENTS.value]: - currentDevice = read_data_from_db( - db, f'SELECT * FROM DeviceDetails WHERE uid IS "{eachDevice[enums.dbSync.DEVICETOPACKAGEUID.value]}"' - ) - # If '_' character is met - it is MCU card uid - if '_' in eachDevice[0]: - mcu = eachDevice[1].replace(".json", "") - datasheet_url = f'https://download.mikroe.com/documents/datasheets/erp/{eachDevice[1].replace(".json", "")}.pdf' - device_uid = mcu - else: - datasheet_url = f'https://download.mikroe.com/documents/datasheets/erp/{eachDevice[0]}.pdf' - device_uid = '' - if not currentDevice[0]: - insertIntoTable( - db, - 'DeviceDetails', - [ - eachDevice[enums.dbSync.DEVICETOPACKAGEUID.value], ## uid - False, ## graphic_mcu - '', ## notes - datasheet_url, ## datasheet_url - any(element in eachDevice[enums.dbSync.DEVICETOPACKAGEUID.value] for element in mcuCardCheckList), - device_uid ## device_uid - ], - deviceDetailsColumns - ) - print("Added %s to database DeviceDetails table.\n" % eachDevice[enums.dbSync.DEVICETOPACKAGEUID.value]) - return - -def checkDevicePackages(database, allDevicesGithub): - packageString = None - if allDevicesGithub[enums.dbSync.COUNT.value]: - boardToDeviceColumns = 'board_uid, device_uid, package_uid' - for eachDevice in allDevicesGithub[enums.dbSync.ELEMENTS.value]: - devicePackages = read_data_from_db( - database, f'SELECT * FROM DeviceToPackage WHERE device_uid IS "{eachDevice[enums.dbSync.DEVICETOPACKAGEUID.value]}"' - ) - if devicePackages[enums.dbSync.COUNT.value]: - packageString = ','.join([pkg[enums.dbSync.ELEMENTS.value] for pkg in devicePackages[enums.dbSync.ELEMENTS.value]]) - boardToDeviceUid = read_data_from_db( - database, f'SELECT * FROM BoardToDevice WHERE device_uid IS "{eachDevice[enums.dbSync.DEVICETOPACKAGEUID.value]}"' - ) - if boardToDeviceUid[enums.dbSync.COUNT.value]: - for boardValues in boardToDeviceUid[enums.dbSync.ELEMENTS.value]: - if boardValues[enums.dbSync.BOARDTODEVICEPACKAGES.value] != packageString: - deleteFromTable( - database, - f'DELETE FROM BoardToDevice WHERE (board_uid="{boardValues[enums.dbSync.BOARDTODEVICEBOARD.value]}" AND device_uid="{boardValues[enums.dbSync.BOARDTODEVICEDEVICE.value]}")' - ) - insertIntoTable( - database, - 'BoardToDevice', - [ - boardValues[enums.dbSync.BOARDTODEVICEBOARD.value], ## board_uid - boardValues[enums.dbSync.BOARDTODEVICEDEVICE.value], ## device_uid - packageString ## package_uid - ], - boardToDeviceColumns - ) - # TODO - uncomment for testing purposes - # print("Added %s/%s/%s to database BoardToDevice table.\n" % ((boardValues[enums.dbSync.BOARDTODEVICEBOARD.value], boardValues[enums.dbSync.BOARDTODEVICEDEVICE.value], packageString))) - return - -def clearDevicePackages(database): - deleteFromTable( - database, - f'DELETE FROM BoardToDevice WHERE (board_uid IS NULL OR device_uid IS NULL)' - ) - -def getProgDbgAsJson(docLink, saveToFile=False): - import urllib.request - - with urllib.request.urlopen(docLink) as f: - html = f.read().decode('utf-8') - # For JLink we need only devices, so remove Flash and RAM information - # that are causing issues while parsing the csv file - # Each Flash and RAM element starts with ", {", so just trim it and - # adjust tabulation between data set members - if 'JLink' in docLink: - html = 'Programmers, Debuggers, ' + html - html_lines = html.replace('\r', '').split('\n') - html = '' - for line in html_lines: - if 'Programmers, ' in line: - html += line.split(', {')[0].replace('"', '').replace(', ', ',') + '\r\n' - elif line.strip() != '': - html += 'Segger J-Link,Segger J-Link,' + line.split(', {')[0].replace('"', '').replace(', ', ',') + '\r\n' - with open(os.path.join(os.path.dirname(__file__), 'devices.txt'), 'w') as devices: - devices.write(html) - devices.close() - - import pandas as pd - import numpy as np - - df = pd.read_csv(os.path.join(os.path.dirname(__file__), "devices.txt")) - df.replace({np.nan: False}, inplace=True) - if 'amazonaws' in docLink: - if 'Codegrip' in docLink: - data_dict = df.set_index('name').to_dict(orient='index') - else: - data_dict = df.set_index('Device').to_dict(orient='index') - else: - data_dict = df.set_index('Name').to_dict(orient='index') - formatted_dict = {mcu.lower(): data for mcu, data in data_dict.items()} - if os.path.exists(os.path.join(os.path.dirname(__file__), "devices.txt")): - os.remove(os.path.join(os.path.dirname(__file__), "devices.txt")) - - if saveToFile: - import json - with open(os.path.join(os.path.dirname(__file__), 'devices.json'), 'w') as json_file: - json_file.write(json.dumps(formatted_dict, indent=4)) - - return formatted_dict - -def checkProgrammerToDevice(database, devices, progDbgInfo, addGeneral=False): - ProgrammerToDeviceColumns = 'programer_uid, device_uid, device_support_package' - - progUidList = [ - progUid[enums.dbSync.PROGRAMMERSPROGRAMMER.value] for progUid in - read_data_from_db( - database, 'SELECT DISTINCT uid FROM Programmers' - )[enums.dbSync.ELEMENTS.value] - ] - - global entranceCheckProg - if entranceCheckProg: - entranceCheckProg = False - for eachProgUid in progUidList: - deleteFromTable( - database, - f'DELETE FROM ProgrammerToDevice WHERE programer_uid="{eachProgUid}"' - ) - # TODO - uncomment for testing purposes - # print("Removed %s from database ProgrammerToDevice table.\n" % eachProgUid) - - for eachDevice in devices[enums.dbSync.ELEMENTS.value]: - if eachDevice[enums.dbSync.DEVICETOPACKAGEDEF.value].replace('.json', '').lower() in progDbgInfo: - for eachProgCheckKey in progDbgInfo[eachDevice[enums.dbSync.DEVICETOPACKAGEDEF.value].replace('.json', '').lower()].keys(): - if re.search('Programmers', eachProgCheckKey, re.IGNORECASE): - if progDbgInfo[eachDevice[enums.dbSync.DEVICETOPACKAGEDEF.value].replace('.json', '').lower()][eachProgCheckKey]: - splitProgsDebuggers = progDbgInfo[eachDevice[enums.dbSync.DEVICETOPACKAGEDEF.value].replace('.json', '').lower()][eachProgCheckKey].split('/') - for eachProgDebug in splitProgsDebuggers: - progDebugUid = read_data_from_db( - database, - f'SELECT uid FROM Programmers WHERE name IS "{eachProgDebug}"' - ) - if 'package_name' in progDbgInfo[eachDevice[enums.dbSync.ELEMENTS.value].lower().replace('.json', '')]: - device_support_package = f'["{progDbgInfo[eachDevice[enums.dbSync.ELEMENTS.value].lower().replace('.json', '')]['package_name']}"]' - # If there is no Debugger support for Codegrip in csv file, add it but without any codegrip package - if device_support_package == '["False"]': - device_support_package = '[""]' - else: - device_support_package = '' - if progDebugUid[enums.dbSync.COUNT.value]: - insertIntoTable( - database, - 'ProgrammerToDevice', - [ - progDebugUid[enums.dbSync.ELEMENTS.value][0][enums.dbSync.PROGRAMMERTODEVICEPROGRAMMER.value], ## programer_uid - eachDevice[enums.dbSync.DEVICETOPACKAGEUID.value], ## device_uid - device_support_package - ], - ProgrammerToDeviceColumns - ) - # TODO - uncomment for testing purposes - # print( - # "Added %s/%s/%s to database ProgrammerToDevice table.\n" % - # ( - # progDebugUid[enums.dbSync.ELEMENTS.value][0][enums.dbSync.PROGRAMMERTODEVICEPROGRAMMER.value], - # eachDevice[enums.dbSync.DEVICETOPACKAGEUID.value], - # device_support_package - # ) - # ) - else: - # Workaround for SEGGER - it has support per device family, not device name - # so we need to see if there is any device matching the family - for key in progDbgInfo: - if eachDevice[enums.dbSync.DEVICETOPACKAGEDEF.value].replace('.json', '').lower().startswith(key): - for eachProgCheckKey in progDbgInfo[key].keys(): - if re.search('Programmers', eachProgCheckKey, re.IGNORECASE): - if progDbgInfo[key][eachProgCheckKey]: - splitProgsDebuggers = progDbgInfo[key][eachProgCheckKey].split('/') - for eachProgDebug in splitProgsDebuggers: - progDebugUid = read_data_from_db( - database, - f'SELECT uid FROM Programmers WHERE name IS "{eachProgDebug}"' - ) - if 'package_name' in progDbgInfo[key]: - device_support_package = f'["{progDbgInfo[key]['package_name']}"]' - # If there is no Debugger support for Codegrip in csv file, add it but without any codegrip package - if device_support_package == '["False"]': - device_support_package = '[""]' - else: - device_support_package = '' - if progDebugUid[enums.dbSync.COUNT.value]: - insertIntoTable( - database, - 'ProgrammerToDevice', - [ - progDebugUid[enums.dbSync.ELEMENTS.value][0][enums.dbSync.PROGRAMMERTODEVICEPROGRAMMER.value], ## programer_uid - eachDevice[enums.dbSync.DEVICETOPACKAGEUID.value], ## device_uid - device_support_package - ], - ProgrammerToDeviceColumns - ) - # Always add gdb_general - if addGeneral: - insertIntoTable( - database, - 'ProgrammerToDevice', - [ - 'gdb_general', ## programer_uid - eachDevice[enums.dbSync.DEVICETOPACKAGEUID.value], ## device_uid - '' ## device_support_package - ], - ProgrammerToDeviceColumns - ) - # TODO - uncomment for testing purposes - # print( - # "Added gdb_general/%s to database ProgrammerToDevice table.\n" % - # ( - # eachDevice[enums.dbSync.DEVICETOPACKAGEUID.value] - # ) - # ) - - return - -def checkDebuggerToDevice(database, devices, progDbgInfo, addGeneral=False): - DebuggerToDeviceColumns = 'debugger_uid, device_uid' - - progUidList = [ - progUid[enums.dbSync.PROGRAMMERSPROGRAMMER.value] for progUid in - read_data_from_db( - database, 'SELECT DISTINCT uid FROM Debuggers' - )[enums.dbSync.ELEMENTS.value] - ] - - global entranceCheckDebug - if entranceCheckDebug: - entranceCheckDebug = False - for eachProgUid in progUidList: - deleteFromTable( - database, - f'DELETE FROM DebuggerToDevice WHERE debugger_uid="{eachProgUid}"' - ) - # TODO - uncomment for testing purposes - # print("Removed %s from database DebuggerToDevice table.\n" % eachProgUid) - - for eachDevice in devices[enums.dbSync.ELEMENTS.value]: - if eachDevice[enums.dbSync.DEVICETOPACKAGEDEF.value].replace('.json', '').lower() in progDbgInfo: - for eachProgCheckKey in progDbgInfo[eachDevice[enums.dbSync.DEVICETOPACKAGEDEF.value].replace('.json', '').lower()].keys(): - if re.search('Debuggers',eachProgCheckKey, re.IGNORECASE): - if progDbgInfo[eachDevice[enums.dbSync.DEVICETOPACKAGEDEF.value].replace('.json', '').lower()][eachProgCheckKey]: - splitProgsDebuggers = progDbgInfo[eachDevice[enums.dbSync.DEVICETOPACKAGEDEF.value].replace('.json', '').lower()][eachProgCheckKey].split('/') - for eachProgDebug in splitProgsDebuggers: - progDebugUid = read_data_from_db( - database, - f'SELECT uid FROM Debuggers WHERE name IS "{eachProgDebug}"' - ) - if progDebugUid[enums.dbSync.COUNT.value]: - insertIntoTable( - database, - 'DebuggerToDevice', - [ - progDebugUid[enums.dbSync.ELEMENTS.value][0][enums.dbSync.PROGRAMMERTODEVICEPROGRAMMER.value], ## debugger_uid - eachDevice[enums.dbSync.DEVICETOPACKAGEUID.value] ## device_uid - ], - DebuggerToDeviceColumns - ) - # TODO - uncomment for testing purposes - # print( - # "Added %s/%s to database DebuggerToDevice table.\n" % - # ( - # progDebugUid[enums.dbSync.ELEMENTS.value][0][enums.dbSync.PROGRAMMERTODEVICEPROGRAMMER.value], - # eachDevice[enums.dbSync.DEVICETOPACKAGEUID.value] - # ) - # ) - else: - # Workaround for SEGGER - it has support per device family, not device name - # so we need to see if there is any device matching the family - for key in progDbgInfo: - if eachDevice[enums.dbSync.DEVICETOPACKAGEDEF.value].replace('.json', '').lower().startswith(key): - for eachProgCheckKey in progDbgInfo[key].keys(): - if re.search('Debuggers',eachProgCheckKey, re.IGNORECASE): - if progDbgInfo[key][eachProgCheckKey]: - splitProgsDebuggers = progDbgInfo[key][eachProgCheckKey].split('/') - for eachProgDebug in splitProgsDebuggers: - progDebugUid = read_data_from_db( - database, - f'SELECT uid FROM Debuggers WHERE name IS "{eachProgDebug}"' - ) - if progDebugUid[enums.dbSync.COUNT.value]: - insertIntoTable( - database, - 'DebuggerToDevice', - [ - progDebugUid[enums.dbSync.ELEMENTS.value][0][enums.dbSync.PROGRAMMERTODEVICEPROGRAMMER.value], ## debugger_uid - eachDevice[enums.dbSync.DEVICETOPACKAGEUID.value] ## device_uid - ], - DebuggerToDeviceColumns - ) - ## Always add gdb_general? - if addGeneral: - insertIntoTable( - database, - 'ProgrammerToDevice', - [ - 'gdb_general', ## programer_uid - eachDevice[enums.dbSync.DEVICETOPACKAGEUID.value] ## device_uid - ], - DebuggerToDeviceColumns - ) - # TODO - uncomment for testing purposes - # print("Added gdb_general/%s to database ProgrammerToDevice table.\n" % eachDevice[enums.dbSync.DEVICETOPACKAGEUID.value]) - return - -def addCollumnsToTable(db, collumns, table, types, defaultValues=None): - for eachCollumn, eachType, defaultValue in zip(collumns, types, defaultValues): - checkCollumn = read_data_from_db( - db, f'SELECT COUNT(*) AS CNTREC FROM pragma_table_info("{table}") WHERE name="{eachCollumn}"' - ) - if not checkCollumn[enums.dbSync.ELEMENTS.value] \ - [enums.dbSync.COUNT.value] \ - [enums.dbSync.COUNT.value]: - addCollumnToTable(db, table, eachCollumn, eachType, defaultValue) - # TODO - uncomment for testing purposes - # print("Added %s collumn (type %s) to %s table. (Default value - %s)\n" % (eachCollumn, eachType, table, defaultValue)) - return - -def compress_directory_7z(base_output_dir, entry_name, arch=None): - """ - Compresses the given directory into a 7z archive using the 7z command line tool. - - Args: - source_dir (str): Path to the directory to be compressed. - output_file (str): Path where the output .7z file should be saved. - - Returns: - bool: True if compression was successful, False otherwise. - """ - # Construct the command to compress the directory - command = [ - '7z', 'a', # 'a' stands for adding to an archive - '-t7z', # Specify 7z archive type - '-mx3', - '-mtc=off' # Do not store timestamps - ] - - # Check if the source directory exists - if arch: - archive_name = base_output_dir + ".7z" - else: - archive_name = os.path.join(os.path.dirname(base_output_dir), entry_name) - - command.append(archive_name) # Path to the output .7z file - command.append(os.path.join(base_output_dir, '*')) # Path to the source directory content - - if not os.path.isdir(base_output_dir): - print(f"The specified directory does not exist: {base_output_dir}") - return False - - # Execute the command - try: - subprocess.run(command, check=True) - print(f"Archive created successfully: {archive_name}") - return archive_name - except subprocess.CalledProcessError as e: - print(f"An error occurred while creating the archive: {e}") - return None - -async def get_all_assets(session, token, repo, release_id): - """ Retrieve all assets for a given release, handling pagination """ - headers = {'Authorization': f'token {token}'} - assets = [] - page = 1 - - while True: - # Fetch assets with pagination - assets_url = f"https://api.github.com/repos/{repo}/releases/{release_id}/assets?page={page}&per_page=100" - async with session.get(assets_url, headers=headers) as response: - page_assets = await response.json() - - # If no more assets, break the loop - if not page_assets: - break - - assets.extend(page_assets) - page += 1 - - return assets - -async def upload_release_asset(session, token, repo, asset_path, release_version=None): - """ Upload a release asset to GitHub """ - print(f"Preparing to upload asset: {os.path.basename(asset_path)}...") - headers = {'Authorization': f'token {token}', 'Content-Type': 'application/octet-stream'} - release_url = f"https://api.github.com/repos/{repo}/releases/latest" - if release_version: - if len(release_version) and ('latest' != release_version): - release_url = f"https://api.github.com/repos/{repo}/releases/tags/{release_version}" - async with session.get(release_url, headers=headers) as response: - response_data = await response.json() - release_id = response_data['id'] - - # Get all assets for the release - assets = await get_all_assets(session, token, repo, release_id) - # Then, filter out the one needed - existing_asset = next((asset for asset in assets if asset['name'] == os.path.basename(asset_path)), None) - - # If the asset exists, delete it - if existing_asset: - delete_url = existing_asset['url'] - async with session.delete(delete_url, headers=headers) as response: - if response.status == 204: - print(f"Deleted asset: {os.path.basename(asset_path)}.") - else: - print(f"Failed to delete asset: {os.path.basename(asset_path)}. Status code: {response.status}") - return False - else: - print(f"Asset {os.path.basename(asset_path)} not found. Nothing to delete.") - - upload_url = f"https://uploads.github.com/repos/{repo}/releases/{release_id}/assets?name={os.path.basename(asset_path)}" - async with aiofiles.open(asset_path, 'rb') as f: - data = await f.read() - async with session.post(upload_url, headers=headers, data=data) as response: - result = await response.json() - print(f"Upload completed for: {os.path.basename(asset_path)}.") - return result - -# Gets latest release headers from repository -def get_headers(api, token): - if api: - return { - 'Authorization': f'token {token}' - } - else: - return { - 'Authorization': f'Bearer {token}', - 'Accept': 'application/octet-stream' - } - -def fetch_all_releases(repo, token, api_headers): - api_headers = get_headers(True, token) - url = f"https://api.github.com/repos/{repo}/releases" - - releases = [] - params = { - "per_page": 100, - "page": 1, - } - - while True: - response = requests.get(url, headers=api_headers, params=params) - response.raise_for_status() - - page_releases = response.json() - if not page_releases: - break - - releases.extend(page_releases) - - if len(page_releases) < params["per_page"]: - break - - params["page"] += 1 - - return releases - -# Function to fetch release details from GitHub -def fetch_release_details(repo, token, release_version): - api_headers = get_headers(True, token) - - if "latest" == release_version: - return utility.get_latest_release(repo, api_headers) - else: - url = f'https://api.github.com/repos/{repo}/releases' - responce_acquired = False - - # First: 5 fast attempts (10s timeout) - for attempt in range(1, 6): - try: - print(f'GitHub API attempt {attempt}/5 (timeout=10s)') - # Get all releases with pagination - all_releases = fetch_all_releases(repo, token, api_headers) - responce_acquired = True - break - - except requests.exceptions.RequestException as e: - last_exception = e - print(f'\033[93mAttempt {attempt} failed:\033[0m {e}') - - if not responce_acquired: - # Final fallback attempt (600s timeout) - try: - print('Final attempt with extended timeout (600s)') - # Get all releases with pagination - all_releases = fetch_all_releases(repo, token, api_headers) - - except requests.exceptions.RequestException as e: - print('\033[91mFinal attempt failed too\033[0m') - raise last_exception from e - - release_check = None - release_check = utility.get_specified_release(all_releases, release_version) - if release_check: - return release_check - else: - ## Always fallback to latest release - print("WARNING: Falling back to LATEST release.") - return utility.get_latest_release(repo, api_headers) - -def formRegexQuery(collumn, regexes): - finalQuery = '' - if 'like' in regexes: - like_patterns = regexes["like"] - like_conditions = " OR ".join([f"{collumn} LIKE '%{pattern}%'" for pattern in like_patterns]) - finalQuery = f'({like_conditions})' - if 'not_like' in regexes: - not_like_patterns = regexes["not_like"] - not_like_conditions = " AND ".join([f"{collumn} NOT LIKE '%{pattern}%'" for pattern in not_like_patterns]) - finalQuery += f' AND ({not_like_conditions})' - - return f"{finalQuery};" - -def updateBoardsFromSdk(dbs, queries): - allBoardDirs = os.listdir(queries) - for eachBoardDir in allBoardDirs: - currentBoardDir = os.path.join(queries, eachBoardDir) - currentBoardFiles = os.listdir(currentBoardDir) - - for eachDb in dbs: - if eachDb: - if 'Boards.json' in currentBoardFiles: - with open(os.path.join(currentBoardDir, 'Boards.json'), 'r') as file: - board = json.load(file) - file.close() - values = [] - collumns = [] - for eachKey in board.keys(): - collumns.append(eachKey) - values.append(board[eachKey]) - insertIntoTable( - eachDb, - 'Boards', - values, - ','.join(collumns) - ) - - if 'LinkerTables.json' in currentBoardFiles: - with open(os.path.join(currentBoardDir, 'LinkerTables.json'), 'r') as file: - linkerTables = json.load(file) - file.close() - for eachTable in linkerTables['tables']: - if 'BoardToSocket' in eachTable: - for eachSocket in eachTable['BoardToSocket']['socket_uid']: - checkSocket = read_data_from_db(eachDb, f'SELECT uid FROM Sockets WHERE uid IS "{eachSocket}"') - if not checkSocket[enums.dbSync.COUNT.value]: - insertIntoTable( - eachDb, - 'Sockets', - eachSocket, - 'uid' - ) - insertIntoTable( - eachDb, - 'BoardToSocket', - [ - linkerTables['board_uid'], - eachSocket - ], - 'board_uid, socket_uid' - ) - - if 'SDKToBoard' in eachTable: - sdkVersions = read_data_from_db(eachDb, 'SELECT DISTINCT version FROM SDKs WHERE name IS "mikroSDK"') - versions = filter_versions(list(v[0] for v in sdkVersions[enums.dbSync.ELEMENTS.value])) - threshold_version = version.parse(eachTable['SDKToBoard']['sdk_uid'][:-1]) - filtered_versions = [f'mikrosdk_v{v.replace('.','')}' for v in versions if version.parse(v) >= threshold_version] - for eachVersion in filtered_versions: - insertIntoTable( - eachDb, - 'SDKToBoard', - [ - eachVersion, - linkerTables['board_uid'] - ], - 'sdk_uid, board_uid' - ) - - if 'BoardToDevice' in eachTable: - if 'regexes' in eachTable['BoardToDevice']['device_uid']: - formedRegex = formRegexQuery('uid', eachTable['BoardToDevice']['device_uid']['regexes']) - currentDeviceUids = read_data_from_db( - eachDb, f'SELECT uid FROM Devices WHERE {formedRegex}' - ) - if currentDeviceUids[enums.dbSync.COUNT.value]: - for eachDeviceUid in currentDeviceUids[enums.dbSync.ELEMENTS.value]: - insertIntoTable( - eachDb, - 'BoardToDevice', - [ - linkerTables['board_uid'], - eachDeviceUid[0] - ], - 'board_uid, device_uid' - ) - else: - if list == type(eachTable['BoardToDevice']['device_uid']): - for eachDevice in eachTable['BoardToDevice']['device_uid']: - insertIntoTable( - eachDb, - 'BoardToDevice', - [ - linkerTables['board_uid'], - eachDevice - ], - 'board_uid, device_uid' - ) - else: - insertIntoTable( - eachDb, - 'BoardToDevice', - [ - linkerTables['board_uid'], - eachTable['BoardToDevice']['device_uid'] - ], - 'board_uid, device_uid' - ) - - return - -def updateDevicesFromSdk(dbs, queries): - allDevicesDirs = os.listdir(queries) - for eachDeviceDir in allDevicesDirs: - currentDeviceDir = os.path.join(queries, eachDeviceDir) - currentDeviceFiles = os.listdir(currentDeviceDir) - - for eachDb in dbs: - if eachDb: - if 'Devices.json' in currentDeviceFiles: - with open(os.path.join(currentDeviceDir, 'Devices.json'), 'r') as file: - device = json.load(file) - file.close() - values = [] - collumns = [] - for eachKey in device.keys(): - collumns.append(eachKey) - values.append(device[eachKey]) - insertIntoTable( - eachDb, - 'Devices', - values, - ','.join(collumns) - ) - - if 'LinkerTables.json' in currentDeviceFiles: - with open(os.path.join(currentDeviceDir, 'LinkerTables.json'), 'r') as file: - linkerTables = json.load(file) - file.close() - table_keys = [list(table.keys())[0] for table in linkerTables['tables']] - for eachTableKey in table_keys: - collumns = ['device_uid'] - values = [linkerTables['device_uid']] - for eachKey in linkerTables['tables']: - if eachTableKey in eachKey: - collumns.append(list(eachKey[eachTableKey].keys())[0]) - if 'SDKToDevice' == eachTableKey: - sdkVersions = read_data_from_db(eachDb, 'SELECT DISTINCT version FROM SDKs WHERE name IS "mikroSDK"') - versions = filter_versions(list(v[0] for v in sdkVersions[enums.dbSync.ELEMENTS.value])) - threshold_version = version.parse(eachKey[eachTableKey][collumns[1]][:-1]) - filtered_versions = [f'mikrosdk_v{v.replace('.','')}' for v in versions if version.parse(v) >= threshold_version] - values.append(filtered_versions) - else: - values.append(eachKey[eachTableKey][collumns[1]]) - break - if list == type(values[1]): - for eachValue in values[1]: - insertIntoTable( - eachDb, - eachTableKey, - [ - values[0], - eachValue - ], - ','.join(collumns) - ) - else: - insertIntoTable( - eachDb, - eachTableKey, - values, - ','.join(collumns) - ) - - return - -def createErpDbpSyncInfo(db, table): - currentData = read_data_from_db(db, f'SELECT DISTINCT name FROM {table};') - if not column_exists(db, table, 'dbp_uid'): - addCollumnToTable(db, table, 'dbp_uid', 'TEXT', 'NoDefault') - for name in currentData[1]: - dbp_uid = name[0] - dbp_uid = re.sub(r"\+", "_PLUS", dbp_uid) - dbp_uid = re.sub(r"\s+", "_", dbp_uid) - if table == 'DeviceArchitectures': - dbp_uid = re.sub(r"\-", "_", dbp_uid) - updateTableCollumn(db, table, 'dbp_uid', dbp_uid.upper(), 'name', name[0]) - -def createErpDbInfo(device): - core_name = None - try: - data = json.loads(device['sdk_config']) - core_name = data.get("CORE_NAME", "") - except (json.JSONDecodeError, TypeError): - core_name = "" - if core_name == '': - try: - data = json.loads(device['core_info']) - core_name = data[0].get("core_name_define", "") - except (json.JSONDecodeError, TypeError): - core_name = "" - - # Normalize architecture names - if core_name.startswith('M') and 'MIPS' not in core_name and 'MICROAPTIV' not in core_name: - core_name = 'ARM Cortex-' + core_name.replace('DSP', '').replace('EF', '') - elif 'MIPS' in core_name or 'MICROAPTIV' in core_name or '32' in core_name: - core_name = 'PIC32' - elif '16' in core_name or '18' in core_name: - core_name = 'PIC' - elif '24' in core_name or '33' in core_name or 'DSPIC' in core_name: - core_name = 'dsPIC' - elif '64K' in core_name: - core_name = 'AVR' - elif 'RISCV' in core_name: - core_name = 'RISC-V' - - new_family_uid = ( - device['vendor'].upper() + '_' + - core_name.upper().replace(' ', '_').replace('-', '_').replace('+', '_PLUS') + '_' + - device['family_uid'].upper().replace('+', '_PLUS').replace(' ', '_') - ) - - return new_family_uid, device['vendor'], core_name - -def updateDevicesFromCore(dbs, queries): - allDevicesDirs = os.listdir(queries) - for eachDeviceDir in allDevicesDirs: - currentDeviceDir = os.path.join(queries, eachDeviceDir) - currentDeviceFiles = os.listdir(currentDeviceDir) - - for eachDb in dbs: - if eachDb: - if 'Devices.json' in currentDeviceFiles: - with open(os.path.join(currentDeviceDir, 'Devices.json'), 'r') as file: - device = json.load(file) - file.close() - values = [] - collumns = [] - for eachKey in device.keys(): - collumns.append(eachKey) - if eachKey == 'family_uid' and 'erp_db' in eachDb: - device[eachKey], _, _ = createErpDbInfo(device) - values.append(device[eachKey]) - insertIntoTable( - eachDb, - 'Devices', - values, - ','.join(collumns) - ) - - if 'LinkerTables.json' in currentDeviceFiles: - with open(os.path.join(currentDeviceDir, 'LinkerTables.json'), 'r') as file: - linkerTables = json.load(file) - file.close() - table_keys = [list(table.keys())[0] for table in linkerTables['tables']] - for eachTableKey in table_keys: - collumns = ['device_uid'] - values = [linkerTables['device_uid']] - for eachKey in linkerTables['tables']: - if eachTableKey in eachKey: - collumns.append(list(eachKey[eachTableKey].keys())[0]) - if 'SDKToDevice' == eachTableKey: - sdkVersions = read_data_from_db(eachDb, 'SELECT DISTINCT version FROM SDKs WHERE name IS "mikroSDK"') - versions = filter_versions(list(v[0] for v in sdkVersions[enums.dbSync.ELEMENTS.value])) - threshold_version = version.parse(eachKey[eachTableKey][collumns[1]][:-1]) - filtered_versions = [f'mikrosdk_v{v.replace('.','')}' for v in versions if version.parse(v) >= threshold_version] - values.append(filtered_versions) - # Add Packages if they are not present in the database - elif 'DeviceToPackage' == eachTableKey: - package_uids = linkerTables['tables'][enums.dbSync.BOARDTODEVICEPACKAGES.value]['DeviceToPackage']['package_uid'] - for package_uid in package_uids: - pin_count = package_uid.split('/')[0] - package_name = package_uid.split('/')[1] - insertIntoTable( - eachDb, - 'Packages', - [ - pin_count, - package_uid, - package_uid, - "", - '{"_MSDK_PACKAGE_NAME_":"' + package_name + '","_MSDK_DIP_SOCKET_TYPE_":""}' - ], - 'pin_count,name,uid,stm_sdk_config,sdk_config' - ) - values.append(eachKey[eachTableKey][collumns[1]]) - else: - values.append(eachKey[eachTableKey][collumns[1]]) - break - if list == type(values[1]): - for eachValue in values[1]: - insertIntoTable( - eachDb, - eachTableKey, - [ - values[0], - eachValue - ], - ','.join(collumns) - ) - else: - insertIntoTable( - eachDb, - eachTableKey, - values, - ','.join(collumns) - ) - - return - -def updateMCHPProgrammers(eachDb, converted_data, json_data_list): - programmersColumns = 'uid,hidden,name,icon,installed,description,installer_package' - debuggersColumns = 'uid,hidden,name,icon,description' - progToDeviceColumns = 'programer_uid,device_uid,device_support_package' - debuggerToDeviceColumns = 'debugger_uid,device_uid' - - ## Add all tools found in microchip index file to programmers table - counter = 1 - for prog_item in converted_data: - print("%sProg item number %s/%s : %s" % (utility.Colors.OKGREEN, counter, len(converted_data), prog_item['display_name'])) - time.sleep(3) - counter += 1 - # TODO: uncomment for testing purposes - # print("%sInserting %s into Programmers table" % (utility.Colors.OKCYAN, prog_item['uid'])) - dfpsMap = json.loads(prog_item['dfps']) - insertIntoTable( - eachDb, - 'Programmers', - [ - prog_item['uid'], - prog_item['hidden'], - prog_item['display_name'], - prog_item['icon'], - prog_item['installed'], - prog_item['description'], - prog_item['installer_package'] - ], - programmersColumns - ) - # TODO: uncomment for testing purposes - # print(f"Inserting {prog_item['uid']} into Debuggers table") - dfpsMap = json.loads(prog_item['dfps']) - insertIntoTable( - eachDb, - 'Debuggers', - [ - prog_item['uid'], - prog_item['hidden'], - prog_item['display_name'], - prog_item['icon'], - prog_item['description'] - ], - debuggersColumns - ) - ## Add MCU to Programmer mapping found in microchip index file - missingMcuDfp = [] - for mcu in prog_item['mcus']: - - ## DebuggerToDevice Section - has_debug = False - element_found = False - if mcu in json_data_list: - for each_sub_element in json_data_list[mcu]: - if re.search(prog_item['uid'], each_sub_element['root_folder'], re.IGNORECASE): - for each_support in each_sub_element['support']: - if each_support.endswith('d'): - element_found = True - if each_sub_element['support'][each_support].lower() != 'no': - has_debug = True - break - if element_found: - break - ## EOF DebuggerToDevice Section - - # TODO: uncomment for testing purposes - # print(f"Inserting {mcu.upper()}:{prog_item['uid']} into ProgrammerToDevice table") - if mcu in dfpsMap: - exists, uid_list = read_data_from_db(eachDb, f"SELECT uid FROM Devices WHERE def_file = \"{mcu.upper()}.json\"") - if not exists: - exists, uid_list = read_data_from_db(eachDb, f"SELECT uid FROM Devices WHERE def_file = \"{mcu}.json\"") - if exists: - for mcu_uid in uid_list: - insertIntoTable( - eachDb, - 'ProgrammerToDevice', - [ - prog_item['uid'], - mcu_uid[0], - json.dumps(dfpsMap[mcu]) - ], - progToDeviceColumns - ) - if has_debug: - # TODO: uncomment for testing purposes - # print(f"Inserting {mcu.upper()}:{prog_item['uid']} into DebuggerToDevice table") - insertIntoTable( - eachDb, - 'DebuggerToDevice', - [ - prog_item['uid'], - mcu_uid[0] - ], - debuggerToDeviceColumns - ) - else: - missingMcuDfp.append(mcu) - print("%sFollowing MCUs do not have DFP: %s" % (utility.Colors.WARNING, missingMcuDfp)) - -def update_erp_info(erpDb, nectoDb): - def normalize_uid(value): - return value.upper().replace(' ', '_').replace('-', '_').replace('+', '_PLUS') - - # Rows for DeviceVendors table - device_vendors = [] - vendors_seen = set() - # Rows for DeviceArchitectures table - device_architectures = [] - core_seen = set() - # Rows for DeviceFamilies table - device_families = [] - families_seen = set() - # Rows for Devices table in ERP database - erp_devices_families = [] - - # Fetch info about all MCUs in the database - sql = """SELECT DISTINCT vendor, sdk_config, family_uid, core_info, uid FROM Devices - WHERE uid NOT LIKE '%\\_%' ESCAPE '\\'""" - _, results = read_data_from_db(nectoDb, sql) - - for vendor, sdk_config, family_uid, core_info, uid in results: - necto_device_info = { - 'vendor': vendor, - 'sdk_config': sdk_config, - 'family_uid': family_uid, - 'core_info': core_info - } - - # Fetch data needed for ERP database - family_uid, vendor_name, core_name = createErpDbInfo(necto_device_info) - erp_devices_families.append({ - 'uid': uid, - 'family_uid': family_uid - }) - - vendor_uid = normalize_uid(vendor_name) - core_uid = f"{vendor_uid}_{normalize_uid(core_name)}" - - # Data for DeviceVendors table - if vendor_uid not in vendors_seen: - vendors_seen.add(vendor_uid) - device_vendors.append({ - 'uid': vendor_uid, - 'name': vendor_name - }) - - # Data for DeviceArchitectures table - if core_uid not in core_seen: - core_seen.add(core_uid) - device_architectures.append({ - 'uid': core_uid, - 'name': core_name, - 'vendor_uid': vendor_uid - }) - - # Data for DeviceFamilies table - if family_uid not in families_seen: - families_seen.add(family_uid) - device_families.append({ - 'uid': family_uid, - 'name': necto_device_info['family_uid'], - 'architecture_uid': core_uid - }) - - # Insert data into ERP tables - for database in [erpDb, nectoDb]: - for row in device_vendors: - insertIntoTable( - database, 'DeviceVendors', - [row['uid'], row['name']], - 'uid,name' - ) - for row in device_architectures: - insertIntoTable( - database, 'DeviceArchitectures', - [row['uid'], row['name'], row['vendor_uid']], - 'uid,name,vendor_uid' - ) - for row in device_families: - insertIntoTable( - database, 'DeviceFamilies', - [row['uid'], row['name'], row['architecture_uid']], - 'uid,name,architecture_uid' - ) - - # Condition for ERP database - as we take all the info from necto_db.db - # family_uid there isn't applicable for ERP system, so we need to overwrite it. - for row in erp_devices_families: - updateTableCollumn( - erpDb, - 'Devices', - 'family_uid', - row['family_uid'], - 'uid', - row['uid'] - ) - -def update_legacy_sdk_support(database): - # Get the list of all Legacy sdk_uid values - sql = """ - SELECT uid FROM SDKs - WHERE uid LIKE "%legacy%" - """ - numOfElements, results = read_data_from_db(database, sql) - - sdk_uids = [row[0] for row in results] - - # Iterate through legacy SDK uids - for sdk_uid in sdk_uids: - # Get all MCUs that have current legacy SDK support as a list - sql = f""" - SELECT DISTINCT device_uid FROM SDKToDevice - WHERE sdk_uid == '{sdk_uid}' - AND device_uid NOT LIKE "%\\_%" ESCAPE '\\'; - """ - numOfElements, results = read_data_from_db(database, sql) - - current_legacy_sdk_device_uids = [row[0] for row in results] - - # Get all Cards that don't have current legacy SDK support as a list - sql = f""" - SELECT DISTINCT device_uid FROM SDKToDevice - WHERE sdk_uid != '{sdk_uid}' - AND device_uid LIKE "%\\_%" ESCAPE '\\'; - """ - numOfElements, results = read_data_from_db(database, sql) - - current_card_device_uids = [row[0] for row in results] - - # Add current legacy SDK support for the Cards which MCUs have this legacy SDK support - for device_uid in current_legacy_sdk_device_uids: - for card_device_uid in current_card_device_uids: - if device_uid.lower() in card_device_uid.lower(): - insertIntoTable(database, 'SDKToDevice', [card_device_uid, sdk_uid], 'device_uid, sdk_uid') - # TODO - uncomment for testing purposes - # print(f"Added {sdk_uid} support for {card_device_uid}") - - # Get all Boards that don't have current legacy SDK support, - # but that have Devices with this legacy SDK support - sql = f""" - SELECT DISTINCT SDKToBoard.board_uid, '{sdk_uid}' - FROM SDKToBoard - INNER JOIN BoardToDevice - ON BoardToDevice.board_uid = SDKToBoard.board_uid - INNER JOIN SDKToDevice - ON SDKToDevice.device_uid = BoardToDevice.device_uid - WHERE SDKToDevice.sdk_uid = '{sdk_uid}' - AND SDKToBoard.sdk_uid != '{sdk_uid}'; - """ - numOfElements, results = read_data_from_db(database, sql) - board_uids = [row[0] for row in results] - for board_uid in board_uids: - insertIntoTable(database, 'SDKToBoard', [board_uid, sdk_uid], 'board_uid, sdk_uid') - # TODO - uncomment for testing purposes - # print(f"Added {sdk_uid} support for {board_uid}") - -def hash_file(filename): - """Generate MD5 hash of a file.""" - hash_md5 = hashlib.md5() - with open(filename, "rb") as f: - for chunk in iter(lambda: f.read(4096), b""): - hash_md5.update(chunk) - return hash_md5.hexdigest() - -def hash_directory_contents(directory): - """Generate a hash for the contents of a directory.""" - all_hashes = [] - for root, dirs, files in os.walk(directory): - dirs.sort() # Ensure directory traversal is in a consistent order - files.sort() # Ensure file traversal is in a consistent order - for filename in files: - file_path = os.path.join(root, filename) - file_hash = hash_file(file_path) - all_hashes.append(file_hash) - - # Combine all file hashes into one hash - combined_hash = hashlib.md5("".join(all_hashes).encode()).hexdigest() - return combined_hash - -def compare_hashes(dir1, dir2): - hash_dir1 = hash_directory_contents(dir1) - hash_dir2 = hash_directory_contents(dir2) - return hash_dir1 == hash_dir2 - -def copy_folder_contents(source_folder, destination_folder): - # Ensure the source folder exists - if not os.path.exists(source_folder): - print(f"The source folder '{source_folder}' does not exist.") - return - - # Ensure the destination folder exists, create it if it doesn't - if not os.path.exists(destination_folder): - os.makedirs(destination_folder) - - # Copy the contents of the source folder to the destination folder - for item in os.listdir(source_folder): - source_path = os.path.join(source_folder, item) - destination_path = os.path.join(destination_folder, item) - - if os.path.isdir(source_path): - shutil.copytree(source_path, destination_path) - else: - shutil.copy2(source_path, destination_path) - - print(f"Contents of '{source_folder}' have been copied to '{destination_folder}'.") - -def fix_icon_names(db, tableName): - if db: - numElements, elements = read_data_from_db(db, f'SELECT * FROM {tableName} WHERE icon NOT REGEXP "^images/boards/board-.+|images/boards/board.png$|images/displays/no_display.png$|images/displays/display-.+"') - if numElements: - for eachElement in elements: - newString = eachElement[2].replace(f"boards/", "boards/board-") - if 'displays' in eachElement[2]: - newString = eachElement[2].replace(f"displays/", "displays/display-") - updateTableCollumn( - db, - tableName, - "icon", - newString, - "uid", - eachElement[0] - ) - -def log_step(message): - elapsed = time.perf_counter() - START_TIME - total = int(elapsed) - minutes = total // 60 - seconds = total % 60 - print(f'\033[0m[{minutes:02d}:{seconds:02d}] {message}') - -## Main runner -async def main( - token, repo, doc_codegrip, doc_mikroprog, doc_jlink, - release_version="", release_version_sdk="", index="Test", mcus_only=True -): - start = time.perf_counter() - global entranceCheckProg - global entranceCheckDebug - ## Step 1 - download the database first - ## Always use latest release - dbName = 'necto_db_dev' - dbPackageName = 'database_dev' - if 'Live' == index: - dbName = 'necto_db' - dbPackageName = 'database' - log_step('\033[96mStep 1: Downloading the database.\033[0m') - databaseNecto, databaseErp = downloadDb( - ## Always download database from latest release - f'https://github.com/MikroElektronika/core_packages/releases/latest/download/{dbPackageName}.7z', - False - ) - - ## Step 2 - Update database with new SDK if needed - ## Add new sdk version - if 'latest' == release_version_sdk: - release_version_sdk = fetch_release_details('MikroElektronika/mikrosdk_v2', token, release_version_sdk)['tag_name'] - - for eachDb in [databaseNecto, databaseErp]: - if eachDb: - log_step(f'\033[96mStep 2: Checking if {release_version_sdk} is present in {eachDb}.\033[0m') - sdkVersionUidNew, sdkVersionUidPrevious = sdk.addSdkVersion(eachDb, release_version_sdk.replace('mikroSDK-', '')) - ## Make sure to check if it exists already, so as not to add again - if sdkVersionUidNew: - ## Add data to tables - for eachDb in [databaseNecto, databaseErp]: - if eachDb: - sdk.insertIntoSdk( - eachDb, - [ - 'SDKToBoard', - 'SDKToBuildSystem', - 'SDKToCompiler', - 'SDKToDevice', - 'SDKToDisplay' - ], - [ - 'board_uid', - 'build_system_uid', - 'compiler_uid', - 'device_uid', - 'display_uid' - ], - sdkVersionUidPrevious, - sdkVersionUidNew - ) - ## EOF Step 2 - - ## Step 3 - Update database with mikroSDK settings - if release_version_sdk: - if not mcus_only: - sdkQueriesPath = os.path.join(os.path.dirname(__file__), 'tmp/queries') - sdkMetadataPath = os.path.join(os.path.dirname(__file__), 'tmp/metadata.json') - ghPath = f'download/{release_version_sdk}' - if "latest" == release_version_sdk: - ghPath = 'latest/download' - if not os.path.exists(sdkQueriesPath): - utility.extract_archive_from_url( - f'https://github.com/MikroElektronika/mikrosdk_v2/releases/{ghPath}/queries.7z', - sdkQueriesPath, token - ) - if not os.path.isfile(sdkMetadataPath): - utility.download_file_from_link( - f'https://github.com/MikroElektronika/mikrosdk_v2/releases/{ghPath}/metadata.json', - sdkMetadataPath, token - ) - if os.path.exists(os.path.join(sdkQueriesPath, 'boards')): - updateBoardsFromSdk([databaseErp, databaseNecto], os.path.join(sdkQueriesPath, 'boards')) ## If any new boards were added - if os.path.exists(os.path.join(sdkQueriesPath, 'cards')): - updateDevicesFromSdk([databaseErp, databaseNecto], os.path.join(sdkQueriesPath, 'cards')) ## If any new mcu cards were added - - ## This part adds package dependencies for each board present in mikroSDK - jsonFile = json.load(open(sdkMetadataPath, 'r'))['packages'] - for eachDb in [databaseErp, databaseNecto]: - if eachDb: - log_step(f'\033[96mStep 3.1: Adding info for new Boards into {eachDb}.\033[0m') - addCollumnsToTable( - eachDb, ['installer_package'], 'Boards', ['Text'], ['NoDefault'] - ) - for eachBoard in jsonFile: - updateTableCollumn( - eachDb, None, None, None, None, None, jsonFile[eachBoard]['db_query'] - ) - - ## Always add MCU information stored in CORE repo - coreQueriesPath = os.path.join(os.getcwd(), 'resources/queries') - if os.path.exists(os.path.join(coreQueriesPath, 'mcus')): - log_step(f'\033[96mStep 3.2: Adding info for new Devices into {[databaseErp, databaseNecto]}.\033[0m') - updateDevicesFromCore([databaseErp, databaseNecto], os.path.join(coreQueriesPath, 'mcus')) - ## EOF Step 3 - - ## Step 4 - add missing collumns to tables - if not mcus_only: - if databaseErp: - log_step('\033[96mStep 4: Adding extra columns for ERP database.\033[0m') - addCollumnsToTable( - databaseErp, ['pid'], 'Boards', ['VARCHAR(50)'], ['NoDefault'] - ) - addCollumnsToTable( - databaseErp, ['package_uid'], 'BoardToDevice', ['TEXT'], ['NoDefault'] - ) - addCollumnsToTable( - databaseErp, ['pid', 'graphic_tool'], 'Compilers', ['VARCHAR(50)', 'BOOLEAN'], ['NoDefault', 0] - ) - ## EOF Step 4 - - ## Step 5 - select all unique devices from github database - if not mcus_only: - log_step('\033[96mStep 5: Fetching all unique devices from the database.\033[0m') - allDevicesGithub = read_data_from_db( - databaseNecto, 'SELECT DISTINCT uid, def_file FROM Devices' - ) - ## EOF Step 5 - - ## Step 6 - add any missing MCU device details - if not mcus_only: - for eachDb in [databaseNecto, databaseErp]: - if eachDb: - log_step(f'\033[96mStep 6: Adding missing DeviceDetails rows to {eachDb}.\033[0m') - checkDeviceDetails(eachDb, allDevicesGithub) - ## EOF Step 6 - - ## Step 7 - add any missing package_uid to BoardToDevice - if not mcus_only: - if databaseErp: - log_step(f'\033[96mStep 7: Adding missing BoardToDevice rows to {databaseErp}.\033[0m') - checkDevicePackages(databaseErp, allDevicesGithub) - ## EOF Step 7 - - ## Step 8 - clear any empty rows from BoardToDevice - if not mcus_only: - if databaseErp: - log_step(f'\033[96mStep 8: Clearing empty BoardToDevice rows in {databaseErp}.\033[0m') - clearDevicePackages(databaseErp) - ## EOF Step 8 - - ## Step 9 - synchronize programmers for all devices - CODEGRIP first - if not mcus_only: - progDbgAsJson = getProgDbgAsJson( - doc_codegrip, - True - ) - if databaseErp: - log_step(f'\033[96mStep 9.1: Adding CODEGRIP packs information into ProgrammerToDevice for {databaseErp}.\033[0m') - checkProgrammerToDevice(databaseErp, allDevicesGithub, progDbgAsJson, True) - log_step(f'\033[96mStep 9.2: Adding CODEGRIP packs information into DebuggerToDevice for {databaseErp}.\033[0m') - checkDebuggerToDevice(databaseErp, allDevicesGithub, progDbgAsJson, False) - entranceCheckProg, entranceCheckDebug = True, True - log_step(f'\033[96mStep 9.1: Adding CODEGRIP packs information into ProgrammerToDevice for {databaseNecto}.\033[0m') - checkProgrammerToDevice(databaseNecto, allDevicesGithub, progDbgAsJson, True) - log_step(f'\033[96mStep 9.2: Adding CODEGRIP packs information into DebuggerToDevice for {databaseNecto}.\033[0m') - checkDebuggerToDevice(databaseNecto, allDevicesGithub, progDbgAsJson, False) - ## EOF Step 9 - - ## Step 10 - syncronize programmers for all devices - mikroProg next - if not mcus_only: - progDbgAsJson = getProgDbgAsJson( - f'https://docs.google.com/spreadsheets/d/{doc_mikroprog}/export?format=csv', - True - ) - if databaseErp: - log_step(f'\033[96mStep 10.1: Adding MikroProg packs information into ProgrammerToDevice for {databaseErp}.\033[0m') - checkProgrammerToDevice(databaseErp, allDevicesGithub, progDbgAsJson, True) - log_step(f'\033[96mStep 10.2: Adding MikroProg packs information into DebuggerToDevice for {databaseErp}.\033[0m') - checkDebuggerToDevice(databaseErp, allDevicesGithub, progDbgAsJson, False) - log_step(f'\033[96mStep 10.1: Adding MikroProg packs information into ProgrammerToDevice for {databaseNecto}.\033[0m') - checkProgrammerToDevice(databaseNecto, allDevicesGithub, progDbgAsJson, True) - log_step(f'\033[96mStep 10.2: Adding MikroProg packs information into DebuggerToDevice for {databaseNecto}.\033[0m') - checkDebuggerToDevice(databaseNecto, allDevicesGithub, progDbgAsJson, False) - ## EOF Step 10 - - ## Step 11 - syncronize programmers for all devices - jlink last - if not mcus_only: - progDbgAsJson = getProgDbgAsJson( - doc_jlink, - True - ) - log_step(f'\033[96mStep 11.1: Adding JLink packs information into ProgrammerToDevice for {databaseNecto}.\033[0m') - checkProgrammerToDevice(databaseNecto, allDevicesGithub, progDbgAsJson, True) - log_step(f'\033[96mStep 11.2: Adding JLink packs information into DebuggerToDevice for {databaseNecto}.\033[0m') - checkDebuggerToDevice(databaseNecto, allDevicesGithub, progDbgAsJson, False) - ## EOF Step 11 - - ## Step 12 add microchip info to programmers table - custom_link = 'https://packs.download.microchip.com/index.idx' - if not mcus_only: - # Download the index file - xml_content = MCHP.download_index_file(custom_link) - converted_data, item_list_unused = MCHP.convert_idx_to_json(xml_content) - - ## Fetch all DFP TP packs from Microchips website - dfp_links = fetch_latest_package_links(xmltodict.parse(xml_content)) - dfp_file_path = os.path.join(os.path.dirname(__file__), 'tmp/dfp_packs') - os.makedirs(dfp_file_path, exist_ok=True) - ## Download and extract all found tool packs - for link in dfp_links: - url = urlparse(link) - pack_name = os.path.basename(url.path) - pack_path=os.path.join(dfp_file_path, utility.drop_extension(pack_name)) - if not os.path.exists(pack_path): - utility.extract_archive_from_url( - url=link, - destination=pack_path - ) - ## Gather all 'device_support.xml' content into one dictionary - json_data_list = find_and_convert_xml_files(dfp_file_path) - - for eachDb in [databaseErp, databaseNecto]: - if eachDb: - log_step(f'\033[96mStep 12: Adding MCHP packs information into {eachDb}.\033[0m') - ## Add missing columns to programmer table - addCollumnsToTable( - eachDb, ['installer_package'], 'Programmers', ['Text'], ['NoDefault'] - ) - addCollumnsToTable( - eachDb, ['device_support_package'], 'ProgrammerToDevice', ['Text'], ['NoDefault'] - ) - updateMCHPProgrammers(eachDb, converted_data, json_data_list) - ## EOF Step 12 - - ## Step 13 - add legacy SDK support for Boards and Cards that should have it - if not mcus_only: - for eachDb in [databaseErp, databaseNecto]: - if eachDb: - log_step(f'\033[96mStep 13: Adding legacy SDK support into {eachDb}.\033[0m') - update_legacy_sdk_support(eachDb) - ## EOF Step 13 - - ## Step 14 - update families - if not mcus_only: - if databaseErp: - ## Add information into ERP db needed for the Web Site - log_step(f'\033[96mStep 14: Adding ERP-applicable info into databases.\033[0m') - update_erp_info(databaseErp, databaseNecto) - ## EOF Step 14 - - ## Step 15 - update the icon names - if not mcus_only: - for eachDb in [databaseErp, databaseNecto]: - log_step(f'\033[96mStep 15: Checking image names in {eachDb}.\033[0m') - fix_icon_names(eachDb, "Boards") - fix_icon_names(eachDb, "Displays") - ## EOF Step 15 - - ## Step 16 - add vendors for all Boards - ## Add new vendor column for NECTO filtering - if databaseNecto: - ## NECTO database only - log_step('\033[96mStep 16: Adding vendors for all Boards.\033[0m') - addCollumnsToTable( - databaseNecto, ['vendor'], 'Boards', ['VARCHAR(50)'], ['NoDefault'] - ) - allBoardUids = read_data_from_db( - databaseNecto, 'SELECT DISTINCT uid FROM Boards' - ) - for boardUid in allBoardUids[enums.dbSync.ELEMENTS.value]: - currentBoardDevice = read_data_from_db( - databaseNecto, f'SELECT device_uid FROM BoardToDevice WHERE board_uid IS "{boardUid[enums.dbSync.BOARDTODEVICEBOARD.value]}"' - ) - for device in currentBoardDevice[1]: - vendor_list = (read_data_from_db(databaseNecto, f'SELECT vendor FROM Devices WHERE uid=="{device[enums.dbSync.BOARDTODEVICEBOARD.value]}";'))[enums.dbSync.ELEMENTS.value] - if len(vendor_list): - vendor = vendor_list[0] - break - update_vendor(databaseNecto, boardUid[0], vendor[0]) - ## EOF Step 16 - - ## Step 17 - if queries are different, add them to new file - if not mcus_only: - log_step('\033[96mStep 17: Checking if there are any changes to queries.\033[0m') - if not compare_hashes( - os.path.join(os.path.dirname(__file__), 'databases/queries'), - os.path.join(os.path.dirname(os.getcwd()), 'utils/databases/queries') - ): - ## Hashes are different, so copy new files here - copy_folder_contents( - os.path.join(os.getcwd(), 'utils/databases/queries'), - os.path.join(os.path.dirname(__file__), 'databases/queries') - ) - ## EOF Step 17 - - ## STEP 18 - Add dbp_uid field values to ERP db - sync with DBP - log_step(f'\033[96mStep 18: Updating ERP database for DBP sync if needed.\033[0m') - if databaseErp: - createErpDbpSyncInfo(db=databaseErp, table='DeviceVendors') - createErpDbpSyncInfo(db=databaseErp, table='DeviceFamilies') - createErpDbpSyncInfo(db=databaseErp, table='DeviceArchitectures') - ## EOF Step 18 - - ## Step 19 - re-upload over existing assets - log_step('\033[96mStep 19: Uploading database archive.\033[0m') - archive_path = compress_directory_7z(os.path.join(os.path.dirname(__file__), 'databases'), f'{dbPackageName}.7z') - async with aiohttp.ClientSession() as session: - upload_result = await upload_release_asset(session, token, repo, archive_path, release_version) - if databaseErp: - log_step('\033[96mStep 19: Uploading ERP database file.\033[0m') - async with aiohttp.ClientSession() as session: - upload_result = await upload_release_asset(session, token, repo, databaseErp, release_version) - ## EOF Step 19 - - ## Step 20 - overwrite the existing necto_db.db in root with newly generated one - log_step(f'\033[96mStep 20: Overwriting {dbName} file.\033[0m') - shutil.copy2(databaseNecto, os.path.join(os.getcwd(), f'{dbName}.db')) - ## EOF Step 20 - ## ------------------------------------------------------------------------------------ ## -## EOF Main runner - -if __name__ == "__main__": - # First, check for arguments passed - def str2bool(v): - if isinstance(v, bool): - return v - if v.lower() in ('yes', 'true', 't', 'y', '1'): - return True - elif v.lower() in ('no', 'false', 'f', 'n', '0'): - return False - else: - raise argparse.ArgumentTypeError('Boolean value expected.') - - # Then, check for arguments passed - parser = argparse.ArgumentParser(description='') - parser.add_argument("token", help="GitHub Token") - parser.add_argument("repo", help="Repository name, e.g., 'username/repo'") - parser.add_argument('doc_codegrip', type=str, help='CODEGRIP spreadsheet table download link.') - parser.add_argument('doc_mikroprog', type=str, help='MikroPROG spreadsheet table download link.') - parser.add_argument('doc_jlink', type=str, help='JLink spreadsheet table download link.') - parser.add_argument('specific_tag', type=str, help='Specific release tag for database update.', default="") - parser.add_argument('specific_tag_mikrosdk', type=str, help='Specific release tag from mikrosdk for database update.', default="") - parser.add_argument('index', type=str, help='Index selection - Live/Test.', default="Test") - parser.add_argument('--mcus_only', type=str2bool, help='If True - will upload asset.', default=False) - - ## Parse the arguments - args = parser.parse_args() - - ## Run the main code - asyncio.run( - main( - args.token, args.repo, - args.doc_codegrip, args.doc_mikroprog, args.doc_jlink, - args.specific_tag, args.specific_tag_mikrosdk, - args.index, args.mcus_only - ) - ) diff --git a/scripts/update_devices_db.py b/scripts/update_devices_db.py deleted file mode 100644 index 0bbef53901..0000000000 --- a/scripts/update_devices_db.py +++ /dev/null @@ -1,340 +0,0 @@ -import os, re, sys, \ - shutil, argparse, \ - sqlite3, json, \ - asyncio, aiohttp, \ - subprocess, aiofiles -from datetime import datetime -from pathlib import Path -from packaging import version - -import classes.class_generate_events_json as calendar_events - -## Import utility modules -## Append to system path -sys.path.append(str(Path(os.path.dirname(__file__)).parent.parent.absolute())) -sys.path.append(str(Path(os.path.dirname(__file__)).absolute())) - -import enums as enums -import support as utility - -def functionRegex(value, pattern): - reg = re.compile(value) - return reg.search(pattern) is not None - -def read_data_from_db(db, sql_query): - ## Open the database / connect to it - con = sqlite3.connect(db) - cur = con.cursor() - - ## Create the REGEXP function to be used in DB - con.create_function("REGEXP", 2, functionRegex) - - ## Execute the desired query - results = cur.execute(sql_query).fetchall() - # results = cur.fetchall() - - ## Close the connection - cur.close() - con.close() - - ## Return query results - return len(results), results - -def deleteFromTable(db, sql_query): - try: - sqliteConnection = sqlite3.connect(db) - cursor = sqliteConnection.cursor() - - ## Create the REGEXP function to be used in DB - sqliteConnection.create_function("REGEXP", 2, functionRegex) - - # Deleting single record now - cursor.execute(sql_query) - sqliteConnection.commit() - cursor.close() - except sqlite3.Error as error: - print("Failed to delete record from sqlite table", error) - finally: - if sqliteConnection: - sqliteConnection.close() - -def updateTableCollumn(db, table, collumn, setNewValue, collumnIf, collumnIfValue, customQuery=None): - import sqlite3 - - conn = sqlite3.connect(db) - cur = conn.cursor() - - ## Create the REGEXP function to be used in DB - conn.create_function("REGEXP", 2, functionRegex) - - if customQuery: - cur.execute(customQuery) - else: - cur.execute(f'UPDATE {table} SET {collumn} = "{setNewValue}" WHERE {collumnIf} REGEXP "{collumnIfValue}"') - conn.commit() - conn.close() - -## Download databases or fetch from disk -def downloadDb(downloadLink, overwrite=True): - dbPath1 = None - dbPath2 = None - if 'http' in downloadLink: - if '.7z' in downloadLink: - dbPath1 = os.path.join(os.path.dirname(__file__), "databases/necto_db.db") - if overwrite or not os.path.isfile(dbPath1): - utility.extract_archive_from_url( - downloadLink, os.path.join(os.path.dirname(__file__), "databases") - ) - if 'database_dev' not in downloadLink: - dbPath2 = os.path.join(os.path.dirname(__file__), "erp_db.db") - if overwrite or not os.path.isfile(dbPath2): - shutil.copyfile(dbPath1, dbPath2) - else: - dbPath1 = downloadLink ## Assume it is a local literal path - - return dbPath1, dbPath2 - -def compress_directory_7z(base_output_dir, entry_name, arch=None): - """ - Compresses the given directory into a 7z archive using the 7z command line tool. - - Args: - source_dir (str): Path to the directory to be compressed. - output_file (str): Path where the output .7z file should be saved. - - Returns: - bool: True if compression was successful, False otherwise. - """ - # Construct the command to compress the directory - command = [ - '7z', 'a', # 'a' stands for adding to an archive - '-t7z', # Specify 7z archive type - '-mx3', - '-mtc=off' # Do not store timestamps - ] - - # Check if the source directory exists - if arch: - archive_name = base_output_dir + ".7z" - else: - archive_name = os.path.join(os.path.dirname(base_output_dir), entry_name) - - command.append(archive_name) # Path to the output .7z file - command.append(os.path.join(base_output_dir, '*')) # Path to the source directory content - - if not os.path.isdir(base_output_dir): - print(f"The specified directory does not exist: {base_output_dir}") - return False - - # Execute the command - try: - subprocess.run(command, check=True) - print(f"Archive created successfully: {archive_name}") - return archive_name - except subprocess.CalledProcessError as e: - print(f"An error occurred while creating the archive: {e}") - return None - -async def get_all_assets(session, token, repo, release_id): - """ Retrieve all assets for a given release, handling pagination """ - headers = {'Authorization': f'token {token}'} - assets = [] - page = 1 - - while True: - # Fetch assets with pagination - assets_url = f"https://api.github.com/repos/{repo}/releases/{release_id}/assets?page={page}&per_page=100" - async with session.get(assets_url, headers=headers) as response: - page_assets = await response.json() - - # If no more assets, break the loop - if not page_assets: - break - - assets.extend(page_assets) - page += 1 - - return assets - -async def upload_release_asset(session, token, repo, asset_path, release_version=None): - """ Upload a release asset to GitHub """ - print(f"Preparing to upload asset: {os.path.basename(asset_path)}...") - headers = {'Authorization': f'token {token}', 'Content-Type': 'application/octet-stream'} - release_url = f"https://api.github.com/repos/{repo}/releases/latest" - if release_version: - if len(release_version) and ('latest' != release_version): - release_url = f"https://api.github.com/repos/{repo}/releases/tags/{release_version}" - async with session.get(release_url, headers=headers) as response: - response_data = await response.json() - release_id = response_data['id'] - - # Get all assets for the release - assets = await get_all_assets(session, token, repo, release_id) - # Then, filter out the one needed - existing_asset = next((asset for asset in assets if asset['name'] == os.path.basename(asset_path)), None) - - # If the asset exists, delete it - if existing_asset: - delete_url = existing_asset['url'] - async with session.delete(delete_url, headers=headers) as response: - if response.status == 204: - print(f"Deleted asset: {os.path.basename(asset_path)}.") - else: - print(f"Failed to delete asset: {os.path.basename(asset_path)}. Status code: {response.status}") - return False - else: - print(f"Asset {os.path.basename(asset_path)} not found. Nothing to delete.") - - upload_url = f"https://uploads.github.com/repos/{repo}/releases/{release_id}/assets?name={os.path.basename(asset_path)}" - async with aiofiles.open(asset_path, 'rb') as f: - data = await f.read() - async with session.post(upload_url, headers=headers, data=data) as response: - result = await response.json() - print(f"Upload completed for: {os.path.basename(asset_path)}.") - return result - -def setSDKSupport(dbs, regex, ai_sdk, xc8_specific): - for eachDb in dbs: - if eachDb: - updateTableCollumn( - eachDb, - "Devices", - "sdk_support", - 1, - "uid", - regex - ) - if xc8_specific: - query = '''UPDATE Devices SET necto_config = '{\"XC8_SUPPORTED\":\"TRUE\"}' WHERE uid REGEXP "''' + regex + '"' - updateTableCollumn( - eachDb, - None, - None, - None, - None, - None, - query - ) - if ai_sdk: - query = '''UPDATE Devices SET sdk_config = REPLACE(sdk_config, '}', ',"AI_GENERATED_SDK":"True"}') WHERE uid REGEXP "''' + regex + '"' - updateTableCollumn( - eachDb, - None, - None, - None, - None, - None, - query - ) - - - return - -def removeDeviceFromDb(dbs, regex, delete_device): - for eachDb in dbs: - if eachDb: - if delete_device: - deleteFromTable( - eachDb, - f''' - DELETE FROM Devices - WHERE uid REGEXP "{regex}"; - ''' - ) - else: - updateTableCollumn( - eachDb, - "Devices", - "sdk_support", - 0, - "uid", - regex - ) - - return - -## Main runner -async def main(token, repo, index, action, regex, delete_device, xc8_specific, ai_sdk=False, spreadsheet_link=""): - date_to_update = datetime.now().strftime("%Y-%m-%d") - - ## If this is a scheduled run - check in the daily release spreadsheet which regex should be used - if regex == 'Spreadsheet Regex': - release_spreadsheet = calendar_events.events_json(spreadsheet_link, "NECTO DAILY UPDATE") - release_spreadsheet.fetch_data() - release_spreadsheet.generate_file(os.path.join(os.path.dirname(__file__), 'releases.json')) - with open(os.path.join(os.path.dirname(__file__), 'releases.json'), 'r') as file: - data = json.load(file) - for release_candidate in data['NECTO DAILY UPDATE']['events']: - if date_to_update in release_candidate['start_dt']: - ## If match was found, get the sdk_support regex from the spreadsheet - regex = release_candidate['regex'] - - ## Download the database first - ## Always use latest release - dbName = 'necto_db_dev' - dbPackageName = 'database_dev' - if "Live" == index: - dbName = 'necto_db' - dbPackageName = 'database' - databaseNecto, databaseErp = downloadDb( - ## Always download database from latest release - f'https://github.com/MikroElektronika/core_packages/releases/latest/download/{dbPackageName}.7z', - False - ) - - ## Update database with requested settings - if "Set sdk_support" == action: - setSDKSupport([databaseErp, databaseNecto], regex, ai_sdk, xc8_specific) - else: - removeDeviceFromDb([databaseErp, databaseNecto], regex, delete_device) - - ## Reupload databases over existing assets - archive_path = compress_directory_7z(os.path.join(os.path.dirname(__file__), 'databases'), f'{dbPackageName}.7z') - async with aiohttp.ClientSession() as session: - upload_result = await upload_release_asset(session, token, repo, archive_path, None) - if databaseErp: - async with aiohttp.ClientSession() as session: - upload_result = await upload_release_asset(session, token, repo, databaseErp, None) - - ## Overwrite the existing necto_db.db in root with newly generated one - shutil.copy2(databaseNecto, os.path.join(os.getcwd(), f'{dbName}.db')) - ## ------------------------------------------------------------------------------------ ## -## EOF Main runner - -if __name__ == "__main__": - # First, check for arguments passed - def str2bool(v): - if isinstance(v, bool): - return v - if v.lower() in ('yes', 'true', 't', 'y', '1'): - return True - elif v.lower() in ('no', 'false', 'f', 'n', '0'): - return False - else: - raise argparse.ArgumentTypeError('Boolean value expected.') - - # First, check for arguments passed - parser = argparse.ArgumentParser(description='') - parser.add_argument("token", help="GitHub Token") - parser.add_argument("repo", help="Repository name, e.g., 'username/repo'") - parser.add_argument('index', type=str, help='Index selection - Live/Test.', default="Test") - parser.add_argument('action', type=str, help='Action selection - Remove/Update Devices.', default="Set sdk_support") - parser.add_argument('regex', type=str, help='Regex for Devices that need to be updated.', default="") - parser.add_argument('delete_device', type=str2bool, help='If True - will remove device from DB completely.', default=False) - parser.add_argument('xc8_specific', type=str2bool, help='If True - will add {"XC8_SUPPORTED":"TRUE"} to necto_config.', default=False) - parser.add_argument('--ai_sdk', type=str2bool, help='If True - will add AI_GENERATED field to sdk_config.', default=False) - parser.add_argument('--spreadsheet_link', type=str, help='Link to the daily release spreadsheet.', default="") - - ## Parse the arguments - args = parser.parse_args() - - ## Run the main code - asyncio.run( - main( - args.token, args.repo, - args.index, args.action, - args.regex, args.delete_device, - args.xc8_specific, args.ai_sdk, - args.spreadsheet_link - ) - )