Merge forked changes

.circleci/config.yml

@@ -2,59 +2,144 @@ version: 2.1
 orbs:
   slack: circleci/slack@3.4.2
 
+executors:
+  docker-executor:
+    docker:
+      - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/circle-ci:stitch-tap-tester-uv
+
 jobs:
   build:
-    docker:
-      - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/circle-ci:stitch-tap-tester
+    executor: docker-executor
+    steps:
+      - run: echo "CI Done"
+
+  ensure_env:
+    executor: docker-executor
     steps:
       - checkout
       - run:
           name: 'Setup virtual env'
           command: |
-            python3 -mvenv /usr/local/share/virtualenvs/tap-shopify
+            uv venv --python 3.12 /usr/local/share/virtualenvs/tap-shopify
             source /usr/local/share/virtualenvs/tap-shopify/bin/activate
-            pip install -U 'pip<19.2' 'setuptools<51.0.0'
-            pip install .[dev]
+            uv pip install -U pip setuptools
+            uv pip install .[dev]
+            uv pip install --upgrade awscli
+      - persist_to_workspace:
+          root: /
+          paths:
+            - root/.local/share/uv/python
+            - usr/local/share/virtualenvs/tap-shopify
+
+  run_pylint:
+    executor: docker-executor
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /
       - run:
           name: 'pylint'
           command: |
             source /usr/local/share/virtualenvs/tap-shopify/bin/activate
-            make test
+            pylint tap_shopify -d missing-docstring,too-many-branches,consider-using-f-string,consider-using-generator,consider-using-dict-items,unnecessary-dunder-call,duplicate-code,too-many-lines
+  json_validator:
+    executor: docker-executor
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /
       - run:
           name: 'JSON Validator'
           command: |
             source /usr/local/share/virtualenvs/tap-tester/bin/activate
             stitch-validate-json tap_shopify/schemas/*.json
+  run_unit_tests:
+    executor: docker-executor
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /
       - run:
           name: 'Unit Tests'
           command: |
             source /usr/local/share/virtualenvs/tap-shopify/bin/activate
-            pip install nose coverage
-            nosetests --with-coverage --cover-erase --cover-package=tap_shopify --cover-html-dir=htmlcov tests/unittests
+            pip install pytest coverage parameterized nose2[coverage_plugin]>=0.6.5
+            coverage run -m pytest tests/unittests
             coverage html
       - store_test_results:
           path: test_output/report.xml
       - store_artifacts:
           path: htmlcov
-      - add_ssh_keys
+  run_integration_tests:
+    executor: docker-executor
+    parallelism: 2
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /
       - run:
           name: 'Integration Tests'
           command: |
+            source /usr/local/share/virtualenvs/tap-shopify/bin/activate
             aws s3 cp s3://com-stitchdata-dev-deployment-assets/environments/tap-tester/tap_tester_sandbox dev_env.sh
             source dev_env.sh
+            unset USE_STITCH_BACKEND
+            mkdir /tmp/${CIRCLE_PROJECT_REPONAME}
+            export STITCH_CONFIG_DIR=/tmp/${CIRCLE_PROJECT_REPONAME}
             source /usr/local/share/virtualenvs/tap-tester/bin/activate
-            run-test --tap=tap-shopify tests
+            circleci tests glob "tests/test_*.py" | circleci tests split > ./tests-to-run
+            if [ -s ./tests-to-run ]; then
+              for test_file in $(cat ./tests-to-run)
+              do
+                echo $test_file > $STITCH_CONFIG_DIR/tap_test.txt
+                run-test --tap=${CIRCLE_PROJECT_REPONAME} $test_file
+              done
+            fi
       - slack/notify-on-failure:
           only_for_branches: master
+      - store_artifacts:
+          path: /tmp/tap-shopify
 
 workflows:
   version: 2
   commit: &commit_jobs
     jobs:
+      - ensure_env:
+          context:
+            - circleci-user
+            - tier-1-tap-user
+      - run_pylint:
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          requires:
+            - ensure_env
+      - json_validator:
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          requires:
+            - ensure_env
+      - run_unit_tests:
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          requires:
+            - ensure_env
+      - run_integration_tests:
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          requires:
+            - ensure_env
       - build:
           context:
             - circleci-user
             - tier-1-tap-user
+          requires:
+            - run_pylint
+            - run_unit_tests
+            - run_integration_tests
   build_daily:
     <<: *commit_jobs
     triggers:

.github/pull_request_template.md

@@ -9,3 +9,7 @@
 
 # Rollback steps
  - revert this branch
+
+#### AI generated code
+https://internal.qlik.dev/general/ways-of-working/code-reviews/#guidelines-for-ai-generated-code
+- [ ] this PR has been written with the help of GitHub Copilot or another generative AI tool

CHANGELOG.md

@@ -1,7 +1,147 @@
 # Changelog
 
+### 3.12.0
+  * Re-stringify JSON values in transform_object to prevent SQL error 22P02 for QTC. [#253](https://github.com/singer-io/tap-shopify/pull/253)
+  * Fix metafield schema SQL type conflicts.
+
+### 3.11.1
+  * Add token refresh and retry logic for 401 errors during bulk operation polling [#252](https://github.com/singer-io/tap-shopify/pull/252)
+
+### 3.11.0
+  * Filter GraphQL field pruning to top-level record selection sets [#250](https://github.com/singer-io/tap-shopify/pull/250)
+
+### 3.10.0
+  * Add support of client_cred grant type auth [#249](https://github.com/singer-io/tap-shopify/pull/249)
+  * Fix current bookmark value for no data time window [#251](https://github.com/singer-io/tap-shopify/pull/251)
+
+### 3.9.0
+  * Add pagination support for fulfillment line items [#248](https://github.com/singer-io/tap-shopify/pull/248)
+
+### 3.8.1
+  * Update the bookmark value even if the record is not retrieved [#247](https://github.com/singer-io/tap-shopify/pull/247)
+
+### 3.8.0
+  * Exponential backoff for Shopify bulk operations in progress [#246](https://github.com/singer-io/tap-shopify/pull/246)
+
+### 3.7.3
+  * Cleanup state file for failed Bulk operations [#244](https://github.com/singer-io/tap-shopify/pull/244)
+
+### 3.7.2
+  * New automatic_keys Support and Bulk State Persistence Logic [#237](https://github.com/singer-io/tap-shopify/pull/237)
+
+### 3.7.1
+  * Query and extract multiple pages of products for each collection [#227](https://github.com/singer-io/tap-shopify/pull/231)
+
+### 3.7.0
+  * Refactor Orders Stream: standard GraphQL → Bulk API Migration [#227](https://github.com/singer-io/tap-shopify/pull/227)
+
+### 3.6.2
+  * Set max_size of pagination to 30 for fullfilment_orders [#230](https://github.com/singer-io/tap-shopify/pull/230)
+
+## 3.6.1
+  * Dependency upgrades [#228](https://github.com/singer-io/tap-shopify/pull/228)
+
+## 3.6.0
+  * Introduce new stream fulfillment_orders. [#222](https://github.com/singer-io/tap-shopify/pull/222)
+
+## 3.5.0
+  * Add fields in orders stream - retailLocation and location ID (within fulfillments) [#219](https://github.com/singer-io/tap-shopify/pull/219)
+  * Skip the author field on missing read_users scope [#223](https://github.com/singer-io/tap-shopify/pull/223)
+
+## 3.4.0
+  * Introduce new stream order_shipping_lines. [#213](https://github.com/singer-io/tap-shopify/pull/213)
+  * shippingLine is a nested object within the orders stream and it represents individual shipping methods.
+
+## 3.3.2
+  * Fix date_window bug[#221](https://github.com/singer-io/tap-shopify/pull/221)
+
+## 3.3.1
+  * Fix transformation bug for abandoned checkouts [#218](https://github.com/singer-io/tap-shopify/pull/218)
+
+## 3.3.0
+  * Dynamically generate graphql query [#214](https://github.com/singer-io/tap-shopify/pull/214)
+
+## 3.2.1
+  * Refactor refund and transactions sync logic to rely on updated parent objects [#212](https://github.com/singer-io/tap-shopify/pull/212)
+
+## 3.2.0
+  * Enhance tap with additional fields across streams [#209](https://github.com/singer-io/tap-shopify/pull/209)
+  * Bookmarks are now updated after fetching records within the specified date range.
+
+## 3.1.0
+  * Add missing fields into the schema of orders and order_refunds stream [#208](https://github.com/singer-io/tap-shopify/pull/208)
+
+## 3.0.0
+  * Migrate Remaining Shopify Streams from REST API to GraphQL API [#201](https://github.com/singer-io/tap-shopify/pull/201)
+  * Introduce new streams - `collections`, `metafields_collections`, `metafields_customers`, `metafields_orders`, `metafields_products`
+  * Delete streams - `collects`, `custom_collections` and `metafields`
+
+## 2.1.0
+  * Optimize Shopify Metafields Sync Performance [#200](https://github.com/singer-io/tap-shopify/pull/200)
+  * Include the new field (= inventoryItem) in the product variants schema.
+  * Add retry for the interruptible sync error from the server side.
+
+## 2.0.2
+  * Update bookmark logic for transactions and order_refunds stream [#197](https://github.com/singer-io/tap-shopify/pull/197)
+
+## 2.0.1
+  * Fixed error handling for GraphQL client [#195](https://github.com/singer-io/tap-shopify/pull/195)
+
+## 2.0.0
+  * Deprecated REST Admin API for products
+  * GraphQL support added for deprecated streams
+  * New stream `Product Variants` added
+  * More details here: [#193](https://github.com/singer-io/tap-shopify/pull/193)
+
+## 1.10.0
+  * Updates the Shopify SDK to 12.3.0
+  * Updates API version used to 2024-01
+  * Incarporates schema changes [#187](https://github.com/singer-io/tap-shopify/pull/187)
+
+## 1.9.0
+  * Updates to run on python 3.11 [#186](https://github.com/singer-io/tap-shopify/pull/186)
+
+## 1.8.0
+  * Updates the Shopify SDK to 12.3.0
+  * Updates API version used to 2023_04
+  * Adds and removes fields per Shopify API changelog for versions 2022_10, 2023_01, 2023_04 [#178](https://github.com/singer-io/tap-shopify/pull/178)
+
+## 1.7.6
+  * Add backoff for 404 error code [#159](https://github.com/singer-io/tap-shopify/pull/159)
+
+## 1.7.5
+  * Add backoff for ConnectionResetError [#169](https://github.com/singer-io/tap-shopify/pull/169)
+## 1.7.4
+  * Add backoff for IncompleteRead [#144](https://github.com/singer-io/tap-shopify/pull/144)
+
+## 1.7.3
+  * Update interrupted sync bookmark strategy [#166](https://github.com/singer-io/tap-shopify/pull/166)
+
+## 1.7.2
+  * Add URLError (connection reset by peer) to retry logic [#165](https://github.com/singer-io/tap-shopify/pull/165)
+
+## 1.7.1
+  * Update bookmarking logic [#143](https://github.com/singer-io/tap-shopify/pull/143)
+
+## 1.7.0
+  From [#157](https://github.com/singer-io/tap-shopify/pull/157):
+  * API/SDK Upgrade to v12.0.1
+  * New Field Additions to Schema
+  * Fields removal from the schema
+
+## 1.6.2
+  * Add canonicalization of transaction receipts to OrderRefunds [#156] (https://github.com/singer-io/tap-shopify/pull/156)
+
+## 1.6.1
+  * Fixing Tranformation Issues [#149] (https://github.com/singer-io/tap-shopify/pull/149)
+
+## 1.6.0
+  * API/SDK Upgrade to v10.0.0 [#135] (https://github.com/singer-io/tap-shopify/pull/135)
+  * New Field Additions to Schema [#140] (https://github.com/singer-io/tap-shopify/pull/140)
+
 ## 1.5.1
   * Request Timeout Implementation [#129](https://github.com/singer-io/tap-shopify/pull/129)
+
 ## 1.5.0
   * Adds `events` stream [#127](https://github.com/singer-io/tap-shopify/pull/127)
 

README.md

@@ -6,22 +6,51 @@ spec](https://github.com/singer-io/getting-started/blob/master/SPEC.md).
 
 This tap:
 
-- Pulls raw data from [Shopify](https://help.shopify.com/en/api/reference)
+- Pulls raw data from [Shopify Graphql Admin API](https://shopify.dev/docs/api/admin-graphql/latest)
 - Extracts the following resources:
-  - [Abandoned Checkouts](https://help.shopify.com/en/api/reference/orders/abandoned_checkouts)
-  - [Collects](https://help.shopify.com/en/api/reference/products/collect)
-  - [Custom Collections](https://help.shopify.com/en/api/reference/products/customcollection)
-  - [Customers](https://help.shopify.com/en/api/reference/customers)
-  - [Metafields](https://help.shopify.com/en/api/reference/metafield)
-  - [Orders](https://help.shopify.com/en/api/reference/orders)
-  - [Products](https://help.shopify.com/en/api/reference/products)
-  - [Transactions](https://help.shopify.com/en/api/reference/orders/transaction)
-  - [Locations](https://help.shopify.com/en/api/reference/inventory/location)
-  - [Inventory Levels](https://help.shopify.com/en/api/reference/inventory/inventorylevel)
-  - [Inventory Item](https://help.shopify.com/en/api/reference/inventory/inventoryitem)
+  - [Abandoned Checkouts](https://shopify.dev/docs/api/admin-graphql/latest/queries/abandonedcheckouts)
+  - [Collections](https://shopify.dev/docs/api/admin-graphql/latest/queries/collections)
+  - [Customers](https://shopify.dev/docs/api/admin-graphql/latest/queries/customers)
+  - [Metafields Collections](https://shopify.dev/docs/api/admin-graphql/latest/queries/collections)
+  - [Metafields Customers](https://shopify.dev/docs/api/admin-graphql/latest/queries/customers)
+  - [Metafields Orders](https://shopify.dev/docs/api/admin-graphql/latest/queries/orders)
+  - [Metafields Products](https://shopify.dev/docs/api/admin-graphql/latest/queries/products)
+  - [Orders](https://shopify.dev/docs/api/admin-graphql/latest/queries/orders)
+  - [OrderShippingLines](https://shopify.dev/docs/api/admin-graphql/latest/objects/ShippingLine)
+  - [FulfillmentOrders](https://shopify.dev/docs/api/admin-graphql/latest/queries/fulfillmentorders)
+  - [Products](https://shopify.dev/docs/api/admin-graphql/latest/queries/products)
+  - [Product Variants](https://shopify.dev/docs/api/admin-graphql/latest/queries/productVariants)
+  - [Transactions](https://shopify.dev/docs/api/admin-graphql/latest/queries/orders)
+  - [Locations](https://shopify.dev/docs/api/admin-graphql/latest/queries/locations)
+  - [Inventory Levels](https://shopify.dev/docs/api/admin-graphql/latest/queries/inventorylevel)
+  - [Inventory Item](https://shopify.dev/docs/api/admin-graphql/latest/queries/inventoryitems)
 - Outputs the schema for each resource
 - Incrementally pulls data based on the input state
-- When Metafields are selected, this tap will sync the Shopify store's top-level Metafields and any additional Metafields for selected tables that also have them (ie: Orders, Products, Customers)
+
+## Stream Details
+
+| Stream Name            | Replication Key | Key Properties |
+|------------------------|----------------|---------------|
+| abandoned_checkouts    | updatedAt      | id            |
+| collections            | updatedAt      | id            |
+| customers              | updatedAt      | id            |
+| events                 | createdAt      | id            |
+| inventory_items        | updatedAt      | id            |
+| inventory_levels       | updatedAt      | id            |
+| locations              | createdAt      | id            |
+| metafields_collections  | updatedAt      | id            |
+| metafields_customers   | updatedAt      | id            |
+| metafields_orders      | updatedAt      | id            |
+| metafields_products    | updatedAt      | id            |
+| order_refunds         | updatedAt      | id            |
+| orders                 | updatedAt      | id            |
+| order_shipping_lines   | updatedAt      | id            |
+| fulfillment_orders     | updatedAt      | id            |
+| product_variants      | updatedAt      | id            |
+| products               | updatedAt      | id            |
+| transactions           | createdAt      | id            |
+
+Currently, `locations` graphql endpoint doesn't support querying on the `updatedAt`, therefore, `createdAt` is made the replication key.
 
 ## Quick Start
 
@@ -64,4 +93,4 @@ This tap:
 
 ---
 
-Copyright © 2019 Stitch
+Copyright © 2025 Stitch