diff --git a/.circleci/config.yml b/.circleci/config.yml index 3b6ba989..b66734ac 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,59 +2,144 @@ version: 2.1 orbs: slack: circleci/slack@3.4.2 +executors: + docker-executor: + docker: + - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/circle-ci:stitch-tap-tester-uv + jobs: build: - docker: - - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/circle-ci:stitch-tap-tester + executor: docker-executor + steps: + - run: echo "CI Done" + + ensure_env: + executor: docker-executor steps: - checkout - run: name: 'Setup virtual env' command: | - python3 -mvenv /usr/local/share/virtualenvs/tap-shopify + uv venv --python 3.12 /usr/local/share/virtualenvs/tap-shopify source /usr/local/share/virtualenvs/tap-shopify/bin/activate - pip install -U 'pip<19.2' 'setuptools<51.0.0' - pip install .[dev] + uv pip install -U pip setuptools + uv pip install .[dev] + uv pip install --upgrade awscli + - persist_to_workspace: + root: / + paths: + - root/.local/share/uv/python + - usr/local/share/virtualenvs/tap-shopify + + run_pylint: + executor: docker-executor + steps: + - checkout + - attach_workspace: + at: / - run: name: 'pylint' command: | source /usr/local/share/virtualenvs/tap-shopify/bin/activate - make test + pylint tap_shopify -d missing-docstring,too-many-branches,consider-using-f-string,consider-using-generator,consider-using-dict-items,unnecessary-dunder-call,duplicate-code,too-many-lines + json_validator: + executor: docker-executor + steps: + - checkout + - attach_workspace: + at: / - run: name: 'JSON Validator' command: | source /usr/local/share/virtualenvs/tap-tester/bin/activate stitch-validate-json tap_shopify/schemas/*.json + run_unit_tests: + executor: docker-executor + steps: + - checkout + - attach_workspace: + at: / - run: name: 'Unit Tests' command: | source /usr/local/share/virtualenvs/tap-shopify/bin/activate - pip install nose coverage - nosetests --with-coverage --cover-erase --cover-package=tap_shopify --cover-html-dir=htmlcov tests/unittests + pip install pytest coverage parameterized nose2[coverage_plugin]>=0.6.5 + coverage run -m pytest tests/unittests coverage html - store_test_results: path: test_output/report.xml - store_artifacts: path: htmlcov - - add_ssh_keys + run_integration_tests: + executor: docker-executor + parallelism: 2 + steps: + - checkout + - attach_workspace: + at: / - run: name: 'Integration Tests' command: | + source /usr/local/share/virtualenvs/tap-shopify/bin/activate aws s3 cp s3://com-stitchdata-dev-deployment-assets/environments/tap-tester/tap_tester_sandbox dev_env.sh source dev_env.sh + unset USE_STITCH_BACKEND + mkdir /tmp/${CIRCLE_PROJECT_REPONAME} + export STITCH_CONFIG_DIR=/tmp/${CIRCLE_PROJECT_REPONAME} source /usr/local/share/virtualenvs/tap-tester/bin/activate - run-test --tap=tap-shopify tests + circleci tests glob "tests/test_*.py" | circleci tests split > ./tests-to-run + if [ -s ./tests-to-run ]; then + for test_file in $(cat ./tests-to-run) + do + echo $test_file > $STITCH_CONFIG_DIR/tap_test.txt + run-test --tap=${CIRCLE_PROJECT_REPONAME} $test_file + done + fi - slack/notify-on-failure: only_for_branches: master + - store_artifacts: + path: /tmp/tap-shopify workflows: version: 2 commit: &commit_jobs jobs: + - ensure_env: + context: + - circleci-user + - tier-1-tap-user + - run_pylint: + context: + - circleci-user + - tier-1-tap-user + requires: + - ensure_env + - json_validator: + context: + - circleci-user + - tier-1-tap-user + requires: + - ensure_env + - run_unit_tests: + context: + - circleci-user + - tier-1-tap-user + requires: + - ensure_env + - run_integration_tests: + context: + - circleci-user + - tier-1-tap-user + requires: + - ensure_env - build: context: - circleci-user - tier-1-tap-user + requires: + - run_pylint + - run_unit_tests + - run_integration_tests build_daily: <<: *commit_jobs triggers: diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index c71d3b31..3ac0252c 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -9,3 +9,7 @@ # Rollback steps - revert this branch + +#### AI generated code +https://internal.qlik.dev/general/ways-of-working/code-reviews/#guidelines-for-ai-generated-code +- [ ] this PR has been written with the help of GitHub Copilot or another generative AI tool diff --git a/CHANGELOG.md b/CHANGELOG.md index a69eaf56..4d2539a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,147 @@ # Changelog +### 3.12.0 + * Re-stringify JSON values in transform_object to prevent SQL error 22P02 for QTC. [#253](https://github.com/singer-io/tap-shopify/pull/253) + * Fix metafield schema SQL type conflicts. + +### 3.11.1 + * Add token refresh and retry logic for 401 errors during bulk operation polling [#252](https://github.com/singer-io/tap-shopify/pull/252) + +### 3.11.0 + * Filter GraphQL field pruning to top-level record selection sets [#250](https://github.com/singer-io/tap-shopify/pull/250) + +### 3.10.0 + * Add support of client_cred grant type auth [#249](https://github.com/singer-io/tap-shopify/pull/249) + * Fix current bookmark value for no data time window [#251](https://github.com/singer-io/tap-shopify/pull/251) + +### 3.9.0 + * Add pagination support for fulfillment line items [#248](https://github.com/singer-io/tap-shopify/pull/248) + +### 3.8.1 + * Update the bookmark value even if the record is not retrieved [#247](https://github.com/singer-io/tap-shopify/pull/247) + +### 3.8.0 + * Exponential backoff for Shopify bulk operations in progress [#246](https://github.com/singer-io/tap-shopify/pull/246) + +### 3.7.3 + * Cleanup state file for failed Bulk operations [#244](https://github.com/singer-io/tap-shopify/pull/244) + +### 3.7.2 + * New automatic_keys Support and Bulk State Persistence Logic [#237](https://github.com/singer-io/tap-shopify/pull/237) + +### 3.7.1 + * Query and extract multiple pages of products for each collection [#227](https://github.com/singer-io/tap-shopify/pull/231) + +### 3.7.0 + * Refactor Orders Stream: standard GraphQL → Bulk API Migration [#227](https://github.com/singer-io/tap-shopify/pull/227) + +### 3.6.2 + * Set max_size of pagination to 30 for fullfilment_orders [#230](https://github.com/singer-io/tap-shopify/pull/230) + +## 3.6.1 + * Dependency upgrades [#228](https://github.com/singer-io/tap-shopify/pull/228) + +## 3.6.0 + * Introduce new stream fulfillment_orders. [#222](https://github.com/singer-io/tap-shopify/pull/222) + +## 3.5.0 + * Add fields in orders stream - retailLocation and location ID (within fulfillments) [#219](https://github.com/singer-io/tap-shopify/pull/219) + * Skip the author field on missing read_users scope [#223](https://github.com/singer-io/tap-shopify/pull/223) + +## 3.4.0 + * Introduce new stream order_shipping_lines. [#213](https://github.com/singer-io/tap-shopify/pull/213) + * shippingLine is a nested object within the orders stream and it represents individual shipping methods. + +## 3.3.2 + * Fix date_window bug[#221](https://github.com/singer-io/tap-shopify/pull/221) + +## 3.3.1 + * Fix transformation bug for abandoned checkouts [#218](https://github.com/singer-io/tap-shopify/pull/218) + +## 3.3.0 + * Dynamically generate graphql query [#214](https://github.com/singer-io/tap-shopify/pull/214) + +## 3.2.1 + * Refactor refund and transactions sync logic to rely on updated parent objects [#212](https://github.com/singer-io/tap-shopify/pull/212) + +## 3.2.0 + * Enhance tap with additional fields across streams [#209](https://github.com/singer-io/tap-shopify/pull/209) + * Bookmarks are now updated after fetching records within the specified date range. + +## 3.1.0 + * Add missing fields into the schema of orders and order_refunds stream [#208](https://github.com/singer-io/tap-shopify/pull/208) + +## 3.0.0 + * Migrate Remaining Shopify Streams from REST API to GraphQL API [#201](https://github.com/singer-io/tap-shopify/pull/201) + * Introduce new streams - `collections`, `metafields_collections`, `metafields_customers`, `metafields_orders`, `metafields_products` + * Delete streams - `collects`, `custom_collections` and `metafields` + +## 2.1.0 + * Optimize Shopify Metafields Sync Performance [#200](https://github.com/singer-io/tap-shopify/pull/200) + * Include the new field (= inventoryItem) in the product variants schema. + * Add retry for the interruptible sync error from the server side. + +## 2.0.2 + * Update bookmark logic for transactions and order_refunds stream [#197](https://github.com/singer-io/tap-shopify/pull/197) + +## 2.0.1 + * Fixed error handling for GraphQL client [#195](https://github.com/singer-io/tap-shopify/pull/195) + +## 2.0.0 + * Deprecated REST Admin API for products + * GraphQL support added for deprecated streams + * New stream `Product Variants` added + * More details here: [#193](https://github.com/singer-io/tap-shopify/pull/193) + +## 1.10.0 + * Updates the Shopify SDK to 12.3.0 + * Updates API version used to 2024-01 + * Incarporates schema changes [#187](https://github.com/singer-io/tap-shopify/pull/187) + +## 1.9.0 + * Updates to run on python 3.11 [#186](https://github.com/singer-io/tap-shopify/pull/186) + +## 1.8.0 + * Updates the Shopify SDK to 12.3.0 + * Updates API version used to 2023_04 + * Adds and removes fields per Shopify API changelog for versions 2022_10, 2023_01, 2023_04 [#178](https://github.com/singer-io/tap-shopify/pull/178) + +## 1.7.6 + * Add backoff for 404 error code [#159](https://github.com/singer-io/tap-shopify/pull/159) + +## 1.7.5 + * Add backoff for ConnectionResetError [#169](https://github.com/singer-io/tap-shopify/pull/169) +## 1.7.4 + * Add backoff for IncompleteRead [#144](https://github.com/singer-io/tap-shopify/pull/144) + +## 1.7.3 + * Update interrupted sync bookmark strategy [#166](https://github.com/singer-io/tap-shopify/pull/166) + +## 1.7.2 + * Add URLError (connection reset by peer) to retry logic [#165](https://github.com/singer-io/tap-shopify/pull/165) + +## 1.7.1 + * Update bookmarking logic [#143](https://github.com/singer-io/tap-shopify/pull/143) + +## 1.7.0 + From [#157](https://github.com/singer-io/tap-shopify/pull/157): + * API/SDK Upgrade to v12.0.1 + * New Field Additions to Schema + * Fields removal from the schema + +## 1.6.2 + * Add canonicalization of transaction receipts to OrderRefunds [#156] (https://github.com/singer-io/tap-shopify/pull/156) + +## 1.6.1 + * Fixing Tranformation Issues [#149] (https://github.com/singer-io/tap-shopify/pull/149) + +## 1.6.0 + * API/SDK Upgrade to v10.0.0 [#135] (https://github.com/singer-io/tap-shopify/pull/135) + * New Field Additions to Schema [#140] (https://github.com/singer-io/tap-shopify/pull/140) + ## 1.5.1 * Request Timeout Implementation [#129](https://github.com/singer-io/tap-shopify/pull/129) + ## 1.5.0 * Adds `events` stream [#127](https://github.com/singer-io/tap-shopify/pull/127) diff --git a/Makefile b/Makefile deleted file mode 100644 index 1bd8b48f..00000000 --- a/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -.DEFAULT_GOAL := test - -test: - pylint tap_shopify -d missing-docstring,too-many-branches - nosetests tests/unittests diff --git a/README.md b/README.md index 15d57ca9..f44dc09a 100644 --- a/README.md +++ b/README.md @@ -6,22 +6,51 @@ spec](https://github.com/singer-io/getting-started/blob/master/SPEC.md). This tap: -- Pulls raw data from [Shopify](https://help.shopify.com/en/api/reference) +- Pulls raw data from [Shopify Graphql Admin API](https://shopify.dev/docs/api/admin-graphql/latest) - Extracts the following resources: - - [Abandoned Checkouts](https://help.shopify.com/en/api/reference/orders/abandoned_checkouts) - - [Collects](https://help.shopify.com/en/api/reference/products/collect) - - [Custom Collections](https://help.shopify.com/en/api/reference/products/customcollection) - - [Customers](https://help.shopify.com/en/api/reference/customers) - - [Metafields](https://help.shopify.com/en/api/reference/metafield) - - [Orders](https://help.shopify.com/en/api/reference/orders) - - [Products](https://help.shopify.com/en/api/reference/products) - - [Transactions](https://help.shopify.com/en/api/reference/orders/transaction) - - [Locations](https://help.shopify.com/en/api/reference/inventory/location) - - [Inventory Levels](https://help.shopify.com/en/api/reference/inventory/inventorylevel) - - [Inventory Item](https://help.shopify.com/en/api/reference/inventory/inventoryitem) + - [Abandoned Checkouts](https://shopify.dev/docs/api/admin-graphql/latest/queries/abandonedcheckouts) + - [Collections](https://shopify.dev/docs/api/admin-graphql/latest/queries/collections) + - [Customers](https://shopify.dev/docs/api/admin-graphql/latest/queries/customers) + - [Metafields Collections](https://shopify.dev/docs/api/admin-graphql/latest/queries/collections) + - [Metafields Customers](https://shopify.dev/docs/api/admin-graphql/latest/queries/customers) + - [Metafields Orders](https://shopify.dev/docs/api/admin-graphql/latest/queries/orders) + - [Metafields Products](https://shopify.dev/docs/api/admin-graphql/latest/queries/products) + - [Orders](https://shopify.dev/docs/api/admin-graphql/latest/queries/orders) + - [OrderShippingLines](https://shopify.dev/docs/api/admin-graphql/latest/objects/ShippingLine) + - [FulfillmentOrders](https://shopify.dev/docs/api/admin-graphql/latest/queries/fulfillmentorders) + - [Products](https://shopify.dev/docs/api/admin-graphql/latest/queries/products) + - [Product Variants](https://shopify.dev/docs/api/admin-graphql/latest/queries/productVariants) + - [Transactions](https://shopify.dev/docs/api/admin-graphql/latest/queries/orders) + - [Locations](https://shopify.dev/docs/api/admin-graphql/latest/queries/locations) + - [Inventory Levels](https://shopify.dev/docs/api/admin-graphql/latest/queries/inventorylevel) + - [Inventory Item](https://shopify.dev/docs/api/admin-graphql/latest/queries/inventoryitems) - Outputs the schema for each resource - Incrementally pulls data based on the input state -- When Metafields are selected, this tap will sync the Shopify store's top-level Metafields and any additional Metafields for selected tables that also have them (ie: Orders, Products, Customers) + +## Stream Details + +| Stream Name | Replication Key | Key Properties | +|------------------------|----------------|---------------| +| abandoned_checkouts | updatedAt | id | +| collections | updatedAt | id | +| customers | updatedAt | id | +| events | createdAt | id | +| inventory_items | updatedAt | id | +| inventory_levels | updatedAt | id | +| locations | createdAt | id | +| metafields_collections | updatedAt | id | +| metafields_customers | updatedAt | id | +| metafields_orders | updatedAt | id | +| metafields_products | updatedAt | id | +| order_refunds | updatedAt | id | +| orders | updatedAt | id | +| order_shipping_lines | updatedAt | id | +| fulfillment_orders | updatedAt | id | +| product_variants | updatedAt | id | +| products | updatedAt | id | +| transactions | createdAt | id | + +Currently, `locations` graphql endpoint doesn't support querying on the `updatedAt`, therefore, `createdAt` is made the replication key. ## Quick Start @@ -64,4 +93,4 @@ This tap: --- -Copyright © 2019 Stitch +Copyright © 2025 Stitch diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index b88034e4..00000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -description-file = README.md diff --git a/setup.py b/setup.py index ac1458ad..ab907973 100755 --- a/setup.py +++ b/setup.py @@ -1,9 +1,10 @@ #!/usr/bin/env python from setuptools import setup +from setuptools import find_packages setup( name="tap-shopify", - version="1.5.1", + version="3.12.0", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", @@ -11,14 +12,17 @@ python_requires='>=3.5.2', py_modules=["tap_shopify"], install_requires=[ - "ShopifyAPI==8.4.1", - "singer-python==5.12.1", + # Important: review the monkey-patched method in the GraphQL client when upgrading this dependency. + "ShopifyAPI==12.7.0", + "singer-python==6.8.0", + "graphql-core==3.2.6", + 'requests==2.34.2', ], extras_require={ 'dev': [ - 'pylint==2.7.4', + 'pylint==3.3.6', 'ipdb', - 'requests==2.20.0', + 'requests==2.34.2', 'nose', ] }, @@ -26,7 +30,7 @@ [console_scripts] tap-shopify=tap_shopify:main """, - packages=["tap_shopify"], + packages=find_packages(), package_data = { "schemas": ["tap_shopify/schemas/*.json"] }, diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index e0299bcb..1ff82b34 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -13,19 +13,20 @@ from singer import metadata from singer import Transformer from tap_shopify.context import Context -from tap_shopify.exceptions import ShopifyError +from tap_shopify.client import ShopifyClient +from tap_shopify.exceptions import ShopifyError, ShopifyAPIError, ShopifyUnauthorizedError from tap_shopify.streams.base import shopify_error_handling, get_request_timeout -import tap_shopify.streams # Load stream objects into Context -REQUIRED_CONFIG_KEYS = ["shop", "api_key"] +REQUIRED_CONFIG_KEYS = ["shop"] LOGGER = singer.get_logger() SDC_KEYS = {'id': 'integer', 'name': 'string', 'myshopify_domain': 'string'} +UNSUPPORTED_FIELDS = {"author"} @shopify_error_handling def initialize_shopify_client(): - api_key = Context.config['api_key'] + api_key = Context.config.get('access_token') or Context.config.get('api_key') shop = Context.config['shop'] - version = '2021-04' + version = '2025-07' session = shopify.Session(shop, version, api_key) shopify.ShopifyResource.activate_session(session) @@ -33,7 +34,34 @@ def initialize_shopify_client(): shopify.Shop.set_timeout(get_request_timeout()) # Shop.current() makes a call for shop details with provided shop and api_key - return shopify.Shop.current().attributes + try: + return shopify.Shop.current().attributes + except pyactiveresource.connection.UnauthorizedAccess as exc: + raise ShopifyUnauthorizedError(exc, "Invalid access token") from exc + +# Add helper +def fetch_app_scopes(): + query = """ + query { + currentAppInstallation { + accessScopes { + handle + } + } + } + """ + data = json.loads(shopify.GraphQL().execute(query)) + return {s["handle"] for s in data["data"]["currentAppInstallation"]["accessScopes"]} + +def has_read_users_access(): + # If the app does not have the 'read_users' scope, return False + if 'read_users' not in fetch_app_scopes(): + LOGGER.warning( + "Skipping '%s' field: 'read_users' scope is not granted for public apps.", + ", ".join(UNSUPPORTED_FIELDS) + ) + return False + return True def get_abs_path(path): return os.path.join(os.path.dirname(os.path.realpath(__file__)), path) @@ -44,10 +72,10 @@ def load_schemas(): # This schema represents many of the currency values as JSON schema # 'number's, which may result in lost precision. - for filename in os.listdir(get_abs_path('schemas')): + for filename in sorted(os.listdir(get_abs_path('schemas'))): path = get_abs_path('schemas') + '/' + filename schema_name = filename.replace('.json', '') - with open(path) as file: + with open(path, encoding='UTF-8') as file: schemas[schema_name] = json.load(file) return schemas @@ -62,23 +90,16 @@ def get_discovery_metadata(stream, schema): mdata = metadata.write(mdata, (), 'valid-replication-keys', [stream.replication_key]) for field_name in schema['properties'].keys(): - if field_name in stream.key_properties or field_name == stream.replication_key: + if field_name in stream.key_properties or field_name == stream.replication_key \ + or field_name in stream.automatic_keys: mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'automatic') + elif field_name in UNSUPPORTED_FIELDS and not has_read_users_access(): + mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'unsupported') else: mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'available') return metadata.to_list(mdata) -def load_schema_references(): - shared_schema_file = "definitions.json" - shared_schema_path = get_abs_path('schemas/') - - refs = {} - with open(os.path.join(shared_schema_path, shared_schema_file)) as data_file: - refs[shared_schema_file] = json.load(data_file) - - return refs - def add_synthetic_key_to_schema(schema): for k in SDC_KEYS: schema['properties']['_sdc_shop_' + k] = {'type': ["null", SDC_KEYS[k]]} @@ -89,21 +110,24 @@ def discover(): raw_schemas = load_schemas() streams = [] + user_agent = Context.config.get("user_agent") - refs = load_schema_references() for schema_name, schema in raw_schemas.items(): if schema_name not in Context.stream_objects: continue stream = Context.stream_objects[schema_name]() - - # resolve_schema_references() is changing value of passed refs. - # Customer is a stream and it's a nested field of orders and abandoned_checkouts streams - # and those 3 _sdc fields are also added inside nested field customer for above 2 stream - # so create a copy of refs before passing it to resolve_schema_references(). - refs_copy = copy.deepcopy(refs) - catalog_schema = add_synthetic_key_to_schema( - singer.resolve_schema_references(schema, refs_copy)) + catalog_schema = add_synthetic_key_to_schema(schema) + + # For metafield streams, remove 'integer' and 'object' from the value field's + # type array to avoid SQL type conflicts in targets + if user_agent and schema_name.startswith('metafields_'): + value_prop = catalog_schema.get('properties', {}).get('value', {}) + if isinstance(value_prop.get('type'), list): + value_prop['type'] = [t for t in value_prop['type'] + if t not in ('integer', 'object')] + if 'properties' in value_prop and value_prop['properties'] == {}: + del value_prop['properties'] # create and add catalog entry catalog_entry = { @@ -136,6 +160,13 @@ def shuffle_streams(stream_name): def sync(): shop_attributes = initialize_shopify_client() sdc_fields = {"_sdc_shop_" + x: shop_attributes[x] for x in SDC_KEYS} + require_reauth = False + + # If there is a currently syncing stream bookmark, shuffle the + # stream order so it gets sync'd first + currently_sync_stream_name = Context.state.get('bookmarks', {}).get('currently_sync_stream') + if currently_sync_stream_name: + shuffle_streams(currently_sync_stream_name) # Emit all schemas first so we have them for child streams for stream in Context.catalog["streams"]: @@ -146,12 +177,6 @@ def sync(): bookmark_properties=stream["replication_key"]) Context.counts[stream["tap_stream_id"]] = 0 - # If there is a currently syncing stream bookmark, shuffle the - # stream order so it gets sync'd first - currently_sync_stream_name = Context.state.get('bookmarks', {}).get('currently_sync_stream') - if currently_sync_stream_name: - shuffle_streams(currently_sync_stream_name) - # Loop over streams in catalog for catalog_entry in Context.catalog['streams']: stream_id = catalog_entry['tap_stream_id'] @@ -166,19 +191,27 @@ def sync(): if not Context.state.get('bookmarks'): Context.state['bookmarks'] = {} Context.state['bookmarks']['currently_sync_stream'] = stream_id + singer.write_state(Context.state) - with Transformer() as transformer: - for rec in stream.sync(): - extraction_time = singer.utils.now() - record_schema = catalog_entry['schema'] - record_metadata = metadata.to_map(catalog_entry['metadata']) - rec = transformer.transform({**rec, **sdc_fields}, - record_schema, - record_metadata) - singer.write_record(stream_id, - rec, - time_extracted=extraction_time) - Context.counts[stream_id] += 1 + try: + # some fields have epoch-time as date, hence transform into UTC date + with Transformer(singer.UNIX_SECONDS_INTEGER_DATETIME_PARSING) as transformer: + for rec in stream.sync(): + extraction_time = singer.utils.now() + record_schema = catalog_entry['schema'] + record_metadata = metadata.to_map(catalog_entry['metadata']) + rec = transformer.transform({**rec, **sdc_fields}, + record_schema, + record_metadata) + singer.write_record(stream_id, + rec, + time_extracted=extraction_time) + Context.counts[stream_id] += 1 + except ShopifyAPIError as e: + if stream_id == 'fulfillment_orders' and 'Access denied' in str(e.__cause__): + require_reauth = True + continue + raise e Context.state['bookmarks'].pop('currently_sync_stream') singer.write_state(Context.state) @@ -188,6 +221,10 @@ def sync(): LOGGER.info('%s: %d', stream_id, stream_count) LOGGER.info('----------------------') + if require_reauth: + raise ShopifyAPIError("Required scopes are missing for the `fulfillment_orders` stream. " \ + "Please re-authorize the connection to sync this stream.") + @utils.handle_top_exception(LOGGER) def main(): try: @@ -197,6 +234,14 @@ def main(): Context.config = args.config Context.state = args.state + if 'client_id' in Context.config: + # Initialize the ShopifyClient for token management (client credentials mode) + Context.client = ShopifyClient( + config_path=args.config_path, + config=Context.config + ) + Context.config['access_token'] = Context.client.access_token + # If discover flag was passed, run discovery mode and dump output to stdout if args.discover: catalog = discover() @@ -223,6 +268,12 @@ def main(): msg = body.get('errors') finally: raise ShopifyError(exc, msg) from exc + except ShopifyUnauthorizedError as error: + raise error + except ShopifyError as error: + raise error + except ShopifyAPIError as error: + raise error except Exception as exc: raise ShopifyError(exc) from exc diff --git a/tap_shopify/client.py b/tap_shopify/client.py new file mode 100644 index 00000000..fadca6e6 --- /dev/null +++ b/tap_shopify/client.py @@ -0,0 +1,110 @@ +import json +import urllib.error +import backoff +import requests +import shopify +import singer +from tap_shopify.streams.base import get_request_timeout +from tap_shopify.exceptions import ShopifyError + +LOGGER = singer.get_logger() + +SHOPIFY_API_VERSION = '2025-07' + +class ShopifyClient: + """ + Handles Shopify authentication via client credentials grant type. + + - Fetches an access token on startup if one is not already present + - Re-fetches the token when the API returns a 401 (token expired or revoked) + - Saves refreshed tokens back to the config file + """ + + def __init__(self, config_path, config): + self.config_path = config_path + self.config = config + self.access_token = config.get('access_token') + + if not self.access_token: + self._refresh_access_token() + + + # pylint: disable=broad-exception-caught + @backoff.on_exception(backoff.expo, + requests.exceptions.RequestException, + max_tries=3, + factor=2) + def _refresh_access_token(self): + """Generate a new access token using client credentials grant type.""" + shop = self.config['shop'] + client_id = self.config['client_id'] + client_secret = self.config['client_secret'] + + token_url = f"https://{shop}.myshopify.com/admin/oauth/access_token" + payload = { + "client_id": client_id, + "client_secret": client_secret, + "grant_type": "client_credentials" + } + + LOGGER.info("Requesting new access token via client credentials grant") + response = requests.post(token_url, json=payload, + headers={"Accept": "application/json"}, + timeout=30) + + if response.status_code != 200: + try: + error_data = response.json() + error_detail = ( + error_data.get('error_description') + or error_data.get('error') + or response.text + ) + except Exception: + error_detail = response.text + + raise ShopifyError( + urllib.error.HTTPError( + url=token_url, + code=response.status_code, + msg=error_detail, + hdrs={}, + fp=None + ), + f"Failed to obtain access token. " + f"Status: {response.status_code}. {error_detail}" + ) + + token_data = response.json() + self.access_token = token_data['access_token'] + + # Update config in memory + self.config['access_token'] = self.access_token + + # Write back to config file + self._write_config() + + def _write_config(self): + """Save updated config (with new token) back to config file.""" + LOGGER.info("Saving credentials back to config") + + with open(self.config_path, 'r', encoding='utf-8') as f: + config = json.load(f) + + config['access_token'] = self.config['access_token'] + + with open(self.config_path, 'w', encoding='utf-8') as f: + json.dump(config, f, indent=2) + + def refresh_token(self): + """Force a token refresh. Called when the API returns a 401 (token expired or revoked).""" + self._refresh_access_token() + + def reinitialize_session(self): + """Reinitialize the Shopify session with the current access token.""" + session = shopify.Session(self.config['shop'], SHOPIFY_API_VERSION, self.access_token) + shopify.ShopifyResource.activate_session(session) + + # set request timeout + shopify.Shop.set_timeout(get_request_timeout()) + LOGGER.info("Shopify session reinitialized with refreshed token") diff --git a/tap_shopify/context.py b/tap_shopify/context.py index 3408bb8f..a2b72cc0 100644 --- a/tap_shopify/context.py +++ b/tap_shopify/context.py @@ -10,6 +10,7 @@ class Context(): stream_map = {} stream_objects = {} counts = {} + client = None # ShopifyClient instance for token management @classmethod def get_catalog_entry(cls, stream_name): @@ -23,6 +24,31 @@ def is_selected(cls, stream_name): stream_metadata = metadata.to_map(stream['metadata']) return metadata.get(stream_metadata, (), 'selected') + @classmethod + def get_unselected_fields(cls, stream_name): + stream = cls.get_catalog_entry(stream_name) + stream_metadata = metadata.to_map(stream['metadata']) + # All fields defined in the schema + all_fields = set(stream["schema"]["properties"].keys()) + + # Selected fields from metadata + selected_fields = set() + for breadcrumb, data in stream_metadata.items(): + if len(breadcrumb) == 2: + if data.get('inclusion') == 'unsupported': + continue + if data.get('selected') or data.get('inclusion') == 'automatic': + selected_fields.add(breadcrumb[1]) + + return list(all_fields - selected_fields) + + @classmethod + def get_all_fields(cls, stream_name): + stream = cls.get_catalog_entry(stream_name) + # All fields defined in the schema + all_fields = set(stream["schema"]["properties"].keys()) + return all_fields + @classmethod def get_results_per_page(cls, default_results_per_page): results_per_page = default_results_per_page diff --git a/tap_shopify/exceptions.py b/tap_shopify/exceptions.py index 94fa62dd..0c244212 100644 --- a/tap_shopify/exceptions.py +++ b/tap_shopify/exceptions.py @@ -1,3 +1,16 @@ class ShopifyError(Exception): def __init__(self, error, msg=''): super().__init__('{}\n{}'.format(error.__class__.__name__, msg)) + +class ShopifyUnauthorizedError(Exception): + def __init__(self, error, msg=''): + super().__init__('{}\n{}'.format(error.__class__.__name__, msg)) + +class ShopifyAPIError(Exception): + """Raised for any unexpected api error without a valid status code""" + +class BulkOperationInProgressError(Exception): + """Raised when a bulk operation is already in progress""" + def __init__(self, message, bulk_op_id=None): + super().__init__(message) + self.bulk_op_id = bulk_op_id diff --git a/tap_shopify/schemas/abandoned_checkouts.json b/tap_shopify/schemas/abandoned_checkouts.json index ee59cfdd..040b555c 100644 --- a/tap_shopify/schemas/abandoned_checkouts.json +++ b/tap_shopify/schemas/abandoned_checkouts.json @@ -1,64 +1,20 @@ { "type": "object", "properties": { - "note_attributes": { - "type": [ - "null", - "array" - ], - "items": { - "type": [ - "null", - "object" - ], - "properties": { - "name": { - "type": [ - "null", - "string" - ] - }, - "value": { - "type": [ - "null", - "string" - ] - } - } - } - }, - "location_id": { - "type": [ - "null", - "integer" - ] - }, - "buyer_accepts_marketing": { - "type": [ - "null", - "boolean" - ] - }, - "currency": { + "note": { "type": [ "null", "string" ] }, - "completed_at": { + "completedAt": { "type": [ "null", "string" ], "format": "date-time" }, - "token": { - "type": [ - "null", - "string" - ] - }, - "billing_address": { + "billingAddress": { "type": [ "null", "object" @@ -76,7 +32,7 @@ "string" ] }, - "first_name": { + "firstName": { "type": [ "null", "string" @@ -100,7 +56,7 @@ "string" ] }, - "last_name": { + "lastName": { "type": [ "null", "string" @@ -124,7 +80,7 @@ "string" ] }, - "country_code": { + "countryCodeV2": { "type": [ "null", "string" @@ -142,7 +98,7 @@ "string" ] }, - "province_code": { + "provinceCode": { "type": [ "null", "string" @@ -153,315 +109,267 @@ "null", "number" ] + }, + "coordinatesValidated": { + "type": [ + "null", + "boolean" + ] + }, + "formattedArea": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "timeZone": { + "type": [ + "null", + "string" + ] + }, + "validationResultSummary": { + "type": [ + "null", + "string" + ] } } }, - "email": { - "type": [ - "null", - "string" - ] - }, - "discount_codes": { - "type": [ - "null", - "array" - ], + "discountCodes": { + "type": "array", "items": { "type": [ "null", - "object" - ], - "properties": { - "type": { - "type": [ - "null", - "string" - ] - }, - "amount": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "code": { - "type": [ - "null", - "string" - ] - } - } + "string" + ] } }, - "customer_locale": { - "type": [ - "null", - "string" - ] - }, - "created_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "updated_at": { + "createdAt": { "type": [ "null", "string" ], "format": "date-time" }, - "gateway": { - "type": [ - "null", - "string" - ] - }, - "referring_site": { - "type": [ - "null", - "string" - ] - }, - "source_identifier": { - "type": [ - "null", - "string" - ] - }, - "total_weight": { - "type": [ - "null", - "integer" - ] - }, - "tax_lines": { - "$ref": "definitions.json#/tax_lines" - }, - "total_line_items_price": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "closed_at": { + "updatedAt": { "type": [ "null", "string" ], "format": "date-time" }, - "device_id": { - "type": [ - "null", - "integer" - ] - }, - "phone": { - "type": [ - "null", - "string" - ] - }, - "source_name": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "total_tax": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "subtotal_price": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "line_items": { - "$ref": "definitions.json#/line_items" - }, - "source_url": { - "type": [ - "null", - "string" - ] - }, - "total_discounts": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "note": { - "type": [ - "null", - "string" - ] - }, - "presentment_currency": { - "type": [ - "null", - "string" - ] - }, - "shipping_lines": { - "type": [ - "null", - "array" - ], + "taxLines": { + "type": "array", "items": { "type": [ "null", "object" ], "properties": { - "applied_discounts": { - "type": [ - "null", - "array" - ], - "items": { - "type": [ - "null", - "object" - ] - } - }, - "custom_tax_lines": { + "priceSet": { "type": [ "null", - "array" + "object" ], - "items": { - "type": [ - "null", - "object" - ] + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } } }, - "phone": { - "type": [ - "null", - "string" - ] - }, - "validation_context": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "string" - ] - }, - "carrier_identifier": { - "type": [ - "null", - "string" - ] - }, - "api_client_id": { - "type": [ - "null", - "integer" - ] - }, - "price": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "requested_fulfillment_service_id": { - "type": [ - "null", - "string" - ] - }, "title": { "type": [ "null", "string" ] }, - "code": { - "type": [ - "null", - "string" - ] - }, - "tax_lines": { - "$ref": "definitions.json#/tax_lines" - }, - "carrier_service_id": { + "rate": { "type": [ "null", - "integer" + "number" ] }, - "delivery_category": { + "source": { "type": [ "null", "string" ] }, - "markup": { + "channelLiable": { "type": [ "null", - "string" + "boolean" ] }, - "source": { + "ratePercentage": { "type": [ "null", - "string" + "number" ] } } } }, - "user_id": { + "totalLineItemsPriceSet": { "type": [ "null", - "integer" - ] - }, - "source": { - "type": [ - "null", - "string" - ] - }, - "shipping_address": { - "type": [ + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "totalTaxSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "shippingAddress": { + "type": [ "null", "object" ], @@ -478,7 +386,7 @@ "string" ] }, - "first_name": { + "firstName": { "type": [ "null", "string" @@ -493,8 +401,9 @@ "latitude": { "type": [ "null", - "number" - ] + "string" + ], + "format": "singer.decimal" }, "zip": { "type": [ @@ -502,7 +411,7 @@ "string" ] }, - "last_name": { + "lastName": { "type": [ "null", "string" @@ -526,7 +435,7 @@ "string" ] }, - "country_code": { + "countryCodeV2": { "type": [ "null", "string" @@ -544,7 +453,7 @@ "string" ] }, - "province_code": { + "provinceCode": { "type": [ "null", "string" @@ -553,44 +462,670 @@ "longitude": { "type": [ "null", - "number" + "string" + ], + "format": "singer.decimal" + }, + "coordinatesValidated": { + "type": [ + "null", + "boolean" + ] + }, + "formattedArea": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "timeZone": { + "type": [ + "null", + "string" + ] + }, + "validationResultSummary": { + "type": [ + "null", + "string" ] } } }, - "abandoned_checkout_url": { + "abandonedCheckoutUrl": { "type": [ "null", "string" ] }, - "landing_site": { + "totalDiscountSet": { "type": [ "null", - "string" - ] + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } }, - "customer": { - "$ref": "definitions.json#/customer" + "taxesIncluded": { + "type": "boolean" }, - "total_price": { + "totalDutiesSet": { "type": [ "null", - "string" + "object" ], - "format": "singer.decimal" + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } }, - "cart_token": { + "totalPriceSet": { "type": [ "null", - "string" - ] + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } }, - "taxes_included": { + "lineItems": { "type": [ "null", - "boolean" - ] + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "quantity": { + "type": [ + "null", + "integer" + ] + }, + "sku": { + "type": [ + "null", + "string" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "variantTitle": { + "type": [ + "null", + "string" + ] + }, + "variant": { + "type": [ + "null", + "object" + ], + "properties": { + "title": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + } + } + }, + "discountedTotalPriceSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "components": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "quantity": { + "type": [ + "null", + "integer" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "variantTitle": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "customAttributes": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "key": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "product": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + }, + "discountedUnitPriceSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "discountedUnitPriceWithCodeDiscount": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "originalTotalPriceSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "originalUnitPriceSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + } + } + } + }, + "subtotalPriceSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "customer": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "lastOrder": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + } + } } } } diff --git a/tap_shopify/schemas/collections.json b/tap_shopify/schemas/collections.json new file mode 100644 index 00000000..d7adf254 --- /dev/null +++ b/tap_shopify/schemas/collections.json @@ -0,0 +1,154 @@ +{ + "type": "object", + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "handle": { + "type": [ + "null", + "string" + ] + }, + "collectionType": { + "type": [ + "null", + "string" + ] + }, + "updatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "productsCount": { + "type": [ + "null", + "object" + ], + "properties": { + "count": { + "type": [ + "null", + "integer" + ] + }, + "precision": { + "type": [ + "null", + "string" + ] + } + } + }, + "sortOrder": { + "type": [ + "null", + "string" + ] + }, + "ruleSet": { + "type": [ + "null", + "object" + ], + "properties": { + "appliedDisjunctively": { + "type": [ + "null", + "boolean" + ] + }, + "rules": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "column": { + "type": [ + "null", + "string" + ] + }, + "condition": { + "type": [ + "null", + "string" + ] + }, + "relation": { + "type": [ + "null", + "string" + ] + } + } + } + } + } + }, + "seo": { + "type": [ + "null", + "object" + ], + "properties": { + "description": { + "type": [ + "null", + "string" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + } + } + }, + "feedback": { + "type": [ + "null", + "object" + ], + "properties": { + "summary": { + "type": [ + "null", + "string" + ] + } + } + }, + "products": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "string" + ] + } + } + } +} diff --git a/tap_shopify/schemas/collects.json b/tap_shopify/schemas/collects.json deleted file mode 100644 index acaab19c..00000000 --- a/tap_shopify/schemas/collects.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": [ - "null", - "integer" - ] - }, - "collection_id": { - "type": [ - "null", - "integer" - ] - }, - "created_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "position": { - "type": [ - "null", - "integer" - ] - }, - "product_id": { - "type": [ - "null", - "integer" - ] - }, - "sort_value": { - "type": [ - "null", - "string" - ] - }, - "updated_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - } - } -} diff --git a/tap_shopify/schemas/customers.json b/tap_shopify/schemas/customers.json index 00fb3d0b..8cc77457 100644 --- a/tap_shopify/schemas/customers.json +++ b/tap_shopify/schemas/customers.json @@ -1,3 +1,495 @@ { - "$ref": "definitions.json#/customer" + "type": "object", + "properties": { + "email": { + "type": [ + "null", + "string" + ] + }, + "multipassIdentifier": { + "type": [ + "null", + "string" + ] + }, + "defaultAddress": { + "type": [ + "null", + "object" + ], + "properties": { + "city": { + "type": [ + "null", + "string" + ] + }, + "address1": { + "type": [ + "null", + "string" + ] + }, + "zip": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "province": { + "type": [ + "null", + "string" + ] + }, + "phone": { + "type": [ + "null", + "string" + ] + }, + "country": { + "type": [ + "null", + "string" + ] + }, + "firstName": { + "type": [ + "null", + "string" + ] + }, + "lastName": { + "type": [ + "null", + "string" + ] + }, + "countryCodeV2": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "provinceCode": { + "type": [ + "null", + "string" + ] + }, + "address2": { + "type": [ + "null", + "string" + ] + }, + "company": { + "type": [ + "null", + "string" + ] + }, + "timeZone": { + "type": [ + "null", + "string" + ] + }, + "validationResultSummary": { + "type": [ + "null", + "string" + ] + }, + "latitude": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "longitude": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "coordinatesValidated": { + "type": [ + "null", + "boolean" + ] + }, + "formattedArea": { + "type": [ + "null", + "string" + ] + } + } + }, + "numberOfOrders": { + "type": [ + "null", + "string" + ] + }, + "state": { + "type": [ + "null", + "string" + ] + }, + "verifiedEmail": { + "type": [ + "null", + "boolean" + ] + }, + "firstName": { + "type": [ + "null", + "string" + ] + }, + "updatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "note": { + "type": [ + "null", + "string" + ] + }, + "phone": { + "type": [ + "null", + "string" + ] + }, + "addresses": { + "type": "array", + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "city": { + "type": [ + "null", + "string" + ] + }, + "address1": { + "type": [ + "null", + "string" + ] + }, + "zip": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "province": { + "type": [ + "null", + "string" + ] + }, + "phone": { + "type": [ + "null", + "string" + ] + }, + "country": { + "type": [ + "null", + "string" + ] + }, + "firstName": { + "type": [ + "null", + "string" + ] + }, + "lastName": { + "type": [ + "null", + "string" + ] + }, + "countryCodeV2": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "provinceCode": { + "type": [ + "null", + "string" + ] + }, + "address2": { + "type": [ + "null", + "string" + ] + }, + "company": { + "type": [ + "null", + "string" + ] + }, + "timeZone": { + "type": [ + "null", + "string" + ] + }, + "validationResultSummary": { + "type": [ + "null", + "string" + ] + }, + "latitude": { + "type": [ + "number", + "null" + ] + }, + "longitude": { + "type": [ + "number", + "null" + ] + }, + "coordinatesValidated": { + "type": [ + "null", + "boolean" + ] + }, + "formattedArea": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "lastName": { + "type": [ + "null", + "string" + ] + }, + "tags": { + "type": "array", + "items": { + "type": [ + "null", + "string" + ] + } + }, + "taxExempt": { + "type": [ + "null", + "boolean" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "createdAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "taxExemptions": { + "type": "array", + "items": { + "type": [ + "null", + "string" + ] + } + }, + "emailMarketingConsent": { + "type": [ + "null", + "object" + ], + "properties": { + "consentUpdatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "marketingOptInLevel": { + "type": [ + "null", + "string" + ] + }, + "marketingState": { + "type": [ + "null", + "string" + ] + } + } + }, + "smsMarketingConsent": { + "type": [ + "object", + "null" + ], + "properties": { + "consentCollectedFrom": { + "type": [ + "null", + "string" + ] + }, + "consentUpdatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "marketingOptInLevel": { + "type": [ + "null", + "string" + ] + }, + "marketingState": { + "type": [ + "null", + "string" + ] + } + } + }, + "validEmailAddress": { + "type": [ + "null", + "boolean" + ] + }, + "productSubscriberStatus": { + "type": [ + "null", + "string" + ] + }, + "amountSpent": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "dataSaleOptOut": { + "type": [ + "null", + "boolean" + ] + }, + "displayName": { + "type": [ + "null", + "string" + ] + }, + "locale": { + "type": [ + "null", + "string" + ] + }, + "lifetimeDuration": { + "type": [ + "null", + "string" + ] + }, + "lastOrder": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + } + } } diff --git a/tap_shopify/schemas/definitions.json b/tap_shopify/schemas/definitions.json deleted file mode 100644 index f2c8fe13..00000000 --- a/tap_shopify/schemas/definitions.json +++ /dev/null @@ -1,961 +0,0 @@ -{ - "order_adjustments": { - "items": { - "properties": { - "order_id": { - "type": [ - "null", - "integer" - ] - }, - "tax_amount": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "refund_id": { - "type": [ - "null", - "integer" - ] - }, - "amount": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "kind": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "reason": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "customer": { - "type": [ - "null", - "object" - ], - "properties": { - "last_order_name": { - "type": [ - "null", - "string" - ] - }, - "currency": { - "type": [ - "null", - "string" - ] - }, - "email": { - "type": [ - "null", - "string" - ] - }, - "multipass_identifier": { - "type": [ - "null", - "string" - ] - }, - "default_address": { - "type": [ - "null", - "object" - ], - "properties": { - "city": { - "type": [ - "null", - "string" - ] - }, - "address1": { - "type": [ - "null", - "string" - ] - }, - "zip": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "country_name": { - "type": [ - "null", - "string" - ] - }, - "province": { - "type": [ - "null", - "string" - ] - }, - "phone": { - "type": [ - "null", - "string" - ] - }, - "country": { - "type": [ - "null", - "string" - ] - }, - "first_name": { - "type": [ - "null", - "string" - ] - }, - "customer_id": { - "type": [ - "null", - "integer" - ] - }, - "default": { - "type": [ - "null", - "boolean" - ] - }, - "last_name": { - "type": [ - "null", - "string" - ] - }, - "country_code": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "province_code": { - "type": [ - "null", - "string" - ] - }, - "address2": { - "type": [ - "null", - "string" - ] - }, - "company": { - "type": [ - "null", - "string" - ] - } - } - }, - "orders_count": { - "type": [ - "null", - "integer" - ] - }, - "state": { - "type": [ - "null", - "string" - ] - }, - "verified_email": { - "type": [ - "null", - "boolean" - ] - }, - "total_spent": { - "type": [ - "null", - "string" - ] - }, - "last_order_id": { - "type": [ - "null", - "integer" - ] - }, - "first_name": { - "type": [ - "null", - "string" - ] - }, - "updated_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "note": { - "type": [ - "null", - "string" - ] - }, - "phone": { - "type": [ - "null", - "string" - ] - }, - "admin_graphql_api_id": { - "type": [ - "null", - "string" - ] - }, - "addresses": { - "type": [ - "null", - "array" - ], - "items": { - "type": [ - "null", - "object" - ], - "properties": { - "city": { - "type": [ - "null", - "string" - ] - }, - "address1": { - "type": [ - "null", - "string" - ] - }, - "zip": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "country_name": { - "type": [ - "null", - "string" - ] - }, - "province": { - "type": [ - "null", - "string" - ] - }, - "phone": { - "type": [ - "null", - "string" - ] - }, - "country": { - "type": [ - "null", - "string" - ] - }, - "first_name": { - "type": [ - "null", - "string" - ] - }, - "customer_id": { - "type": [ - "null", - "integer" - ] - }, - "default": { - "type": [ - "null", - "boolean" - ] - }, - "last_name": { - "type": [ - "null", - "string" - ] - }, - "country_code": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "province_code": { - "type": [ - "null", - "string" - ] - }, - "address2": { - "type": [ - "null", - "string" - ] - }, - "company": { - "type": [ - "null", - "string" - ] - } - } - } - }, - "last_name": { - "type": [ - "null", - "string" - ] - }, - "tags": { - "type": [ - "null", - "string" - ] - }, - "tax_exempt": { - "type": [ - "null", - "boolean" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "accepts_marketing": { - "type": [ - "null", - "boolean" - ] - }, - "accepts_marketing_updated_at": { - "anyOf": [ - { - "type": "string" , - "format": "date-time" - }, - { - "type": "string" - }, - { - "type": "null" - } - ] - }, - "created_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - } - } - }, - "location": { - "properties": { - "country_code": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "address1": { - "type": [ - "null", - "string" - ] - }, - "city": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "address2": { - "type": [ - "null", - "string" - ] - }, - "province_code": { - "type": [ - "null", - "string" - ] - }, - "zip": { - "type": [ - "null", - "string" - ] - }, - "localized_province_name": { - "type": [ - "null", - "string" - ] - }, - "localized_country_name": { - "type": [ - "null", - "string" - ] - }, - "updated_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "province": { - "type": [ - "null", - "string" - ] - }, - "phone": { - "type": [ - "null", - "string" - ] - }, - "legacy": { - "type": [ - "null", - "boolean"] - }, - "created_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "country": { - "type": [ - "null", - "string" - ] - }, - "active": { - "type": [ - "null", - "boolean"] - }, - "admin_graphql_api_id": { - "type": [ - "null", - "string"] - }, - "country_name": { - "type": [ - "null", - "string"] - } - }, - "type": [ - "null", - "object" - ] - }, - "line_item": { - "properties": { - "applied_discounts": { - "items": { - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "total_discount_set": {}, - "pre_tax_price_set": {}, - "price_set": {}, - "grams": { - "type": [ - "null", - "integer" - ] - }, - "compare_at_price": { - "type": [ - "null", - "string" - ] - }, - "destination_location_id": { - "type": [ - "null", - "integer" - ] - }, - "key": { - "type": [ - "null", - "string" - ] - }, - "line_price": { - "type": [ - "null", - "string" - ] - }, - "origin_location_id": { - "type": [ - "null", - "integer" - ] - }, - "applied_discount": { - "type": [ - "null", - "integer" - ] - }, - "fulfillable_quantity": { - "type": [ - "null", - "integer" - ] - }, - "variant_title": { - "type": [ - "null", - "string" - ] - }, - "properties": { - "anyOf": [ - { - "items": { - "properties": { - "name": { - "type": [ - "null", - "string" - ] - }, - "value": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - { - "properties": {}, - "type": ["null", "object"] - } - ] - }, - "tax_code": { - "type": [ - "null", - "string" - ] - }, - "discount_allocations": { - "items": { - "properties": { - "discount_application_index": { - "type": [ - "null", - "integer" - ] - }, - "amount_set": {}, - "amount": { - "type": [ - "null", - "number" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "admin_graphql_api_id": { - "type": [ - "null", - "string" - ] - }, - "pre_tax_price": { - "type": [ - "null", - "number" - ] - }, - "sku": { - "type": [ - "null", - "string" - ] - }, - "product_exists": { - "type": [ - "null", - "boolean" - ] - }, - "total_discount": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "fulfillment_status": { - "type": [ - "null", - "string" - ] - }, - "gift_card": { - "type": [ - "null", - "boolean" - ] - }, - "id": { - "type": ["null", "integer", "string"] - }, - "taxable": { - "type": [ - "null", - "boolean" - ] - }, - "vendor": { - "type": [ - "null", - "string" - ] - }, - "tax_lines": { - "$ref": "definitions.json#/tax_lines" - }, - "origin_location": { - "$ref": "definitions.json#/location" - }, - "price": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "requires_shipping": { - "type": [ - "null", - "boolean" - ] - }, - "fulfillment_service": { - "type": [ - "null", - "string" - ] - }, - "variant_inventory_management": { - "type": [ - "null", - "string" - ] - }, - "title": { - "type": [ - "null", - "string" - ] - }, - "destination_location": { - "$ref": "definitions.json#/location" - }, - "quantity": { - "type": [ - "null", - "integer" - ] - }, - "product_id": { - "type": [ - "null", - "integer" - ] - }, - "variant_id": { - "type": [ - "null", - "integer" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "line_items": { - "items": { - "$ref": "definitions.json#/line_item" - }, - "type": [ - "null", - "array" - ] - }, - "image": { - "properties": { - "updated_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "created_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "variant_ids": { - "type": [ - "null", - "array" - ], - "items": { - "type": [ - "null", - "integer" - ] - } - }, - "height": { - "type": [ - "null", - "integer" - ] - }, - "alt": { - "type": [ - "null", - "string" - ] - }, - "src": { - "type": [ - "null", - "string" - ] - }, - "position": { - "type": [ - "null", - "integer" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "admin_graphql_api_id": { - "type": [ - "null", - "string" - ] - }, - "width": { - "type": [ - "null", - "integer" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "tax_lines": { - "items": { - "properties": { - "price_set": { - }, - "price": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "title": { - "type": [ - "null", - "string" - ] - }, - "rate": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "compare_at": { - "type": [ - "null", - "string" - ] - }, - "position": { - "type": [ - "null", - "integer" - ] - }, - "source": { - "type": [ - "null", - "string" - ] - }, - "zone": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - } -} diff --git a/tap_shopify/schemas/events.json b/tap_shopify/schemas/events.json index fc52b5a2..6dfdc6f1 100644 --- a/tap_shopify/schemas/events.json +++ b/tap_shopify/schemas/events.json @@ -4,53 +4,174 @@ "id": { "type": [ "null", - "integer" + "string" ] }, - "created_at": { + "createdAt": { "type": [ "null", "string" ], "format": "date-time" }, - "body": { + "action": { "type": [ "null", "string" ] }, - "path": { + "appTitle": { "type": [ "null", "string" ] }, + "attributeToApp": { + "type": [ + "null", + "boolean" + ] + }, + "attributeToUser": { + "type": [ + "null", + "boolean" + ] + }, + "criticalAlert": { + "type": [ + "null", + "boolean" + ] + }, "message": { "type": [ "null", "string" ] }, - "subject_id": { + "subjectId": { "type": [ "null", - "integer" + "string" ] }, - "subject_type": { + "subjectType": { "type": [ "null", "string" ] }, - "verb": { + "additionalContent": { + "type": [ + "null", + "string" + ] + }, + "additionalData": { + "type": [ + "null", + "string" + ] + }, + "arguments": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "string" + ] + } + }, + "hasAdditionalContent": { + "type": [ + "null", + "boolean" + ] + }, + "secondaryMessage": { "type": [ "null", "string" ] }, + "attachments": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "fileExtension": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "size": { + "type": [ + "null", + "integer" + ] + }, + "url": { + "type": [ + "null", + "string" + ] + } + } + } + }, "author": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + }, + "canDelete": { + "type": [ + "null", + "boolean" + ] + }, + "canEdit": { + "type": [ + "null", + "boolean" + ] + }, + "edited": { + "type": [ + "null", + "boolean" + ] + }, + "rawMessage": { "type": [ "null", "string" diff --git a/tap_shopify/schemas/fulfillment_orders.json b/tap_shopify/schemas/fulfillment_orders.json new file mode 100644 index 00000000..59fda378 --- /dev/null +++ b/tap_shopify/schemas/fulfillment_orders.json @@ -0,0 +1,979 @@ +{ + "type": "object", + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "orderId": { + "type": [ + "null", + "string" + ] + }, + "updatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "supportedActions": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "action": { + "type": [ + "null", + "string" + ] + }, + "externalUrl": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "status": { + "type": [ + "null", + "string" + ] + }, + "requestStatus": { + "type": [ + "null", + "string" + ] + }, + "orderProcessedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "orderName": { + "type": [ + "null", + "string" + ] + }, + "channelId": { + "type": [ + "null", + "string" + ] + }, + "fulfillAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "fulfillBy": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "createdAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "destination": { + "type": [ + "null", + "object" + ], + "properties": { + "address1": { + "type": [ + "null", + "string" + ] + }, + "address2": { + "type": [ + "null", + "string" + ] + }, + "city": { + "type": [ + "null", + "string" + ] + }, + "province": { + "type": [ + "null", + "string" + ] + }, + "zip": { + "type": [ + "null", + "string" + ] + }, + "countryCode": { + "type": [ + "null", + "string" + ] + }, + "company": { + "type": [ + "null", + "string" + ] + }, + "email": { + "type": [ + "null", + "string" + ] + }, + "firstName": { + "type": [ + "null", + "string" + ] + }, + "lastName": { + "type": [ + "null", + "string" + ] + }, + "phone": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "location": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "fulfillmentHolds": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "displayReason": { + "type": [ + "null", + "string" + ] + }, + "handle": { + "type": [ + "null", + "string" + ] + }, + "reason": { + "type": [ + "null", + "string" + ] + }, + "reasonNotes": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "heldByRequestingApp": { + "type": [ + "null", + "boolean" + ] + } + } + } + }, + "internationalDuties": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "incoterm": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "deliveryMethod": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "methodType": { + "type": [ + "null", + "string" + ] + }, + "presentedName": { + "type": [ + "null", + "string" + ] + }, + "serviceCode": { + "type": [ + "null", + "string" + ] + }, + "sourceReference": { + "type": [ + "null", + "string" + ] + }, + "minDeliveryDateTime": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "maxDeliveryDateTime": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "brandedPromise": { + "type": [ + "null", + "object" + ], + "properties": { + "handle": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + } + } + }, + "additionalInformation": { + "type": [ + "null", + "object" + ], + "properties": { + "instructions": { + "type": [ + "null", + "string" + ] + }, + "phone": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "assignedLocation": { + "type": [ + "null", + "object" + ], + "properties": { + "address1": { + "type": [ + "null", + "string" + ] + }, + "address2": { + "type": [ + "null", + "string" + ] + }, + "city": { + "type": [ + "null", + "string" + ] + }, + "province": { + "type": [ + "null", + "string" + ] + }, + "zip": { + "type": [ + "null", + "string" + ] + }, + "countryCode": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "phone": { + "type": [ + "null", + "string" + ] + }, + "location": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "merchantRequests": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "kind": { + "type": [ + "null", + "string" + ] + }, + "message": { + "type": [ + "null", + "string" + ] + }, + "requestOptions": { + "type": [ + "null", + "object" + ] + }, + "responseData": { + "type": [ + "null", + "object" + ] + }, + "sentAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "fulfillmentOrder": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + } + } + } + }, + "locationsForMove": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "message": { + "type": [ + "null", + "string" + ] + }, + "movable": { + "type": [ + "null", + "boolean" + ] + }, + "unavailableLineItemsCount": { + "type": [ + "null", + "object" + ], + "properties": { + "count": { + "type": [ + "null", + "integer" + ] + }, + "precision": { + "type": [ + "null", + "string" + ] + } + } + }, + "availableLineItemsCount": { + "type": [ + "null", + "object" + ], + "properties": { + "count": { + "type": [ + "null", + "integer" + ] + }, + "precision": { + "type": [ + "null", + "string" + ] + } + } + }, + "location": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + }, + "availableLineItems": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "action": { + "id": [ + "null", + "string" + ] + } + } + } + }, + "unavailableLineItems": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + } + } + } + } + }, + "fulfillmentOrdersForMerge": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "fulfillments": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "status": { + "type": [ + "null", + "string" + ] + }, + "displayStatus": { + "type": [ + "null", + "string" + ] + }, + "requiresShipping": { + "type": [ + "null", + "boolean" + ] + }, + "totalQuantity": { + "type": [ + "null", + "integer" + ] + }, + "createdAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "updatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "inTransitAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "deliveredAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "estimatedDeliveryAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "originAddress": { + "type": [ + "null", + "object" + ], + "properties": { + "address1": { + "type": [ + "null", + "string" + ] + }, + "address2": { + "type": [ + "null", + "string" + ] + }, + "city": { + "type": [ + "null", + "string" + ] + }, + "provinceCode": { + "type": [ + "null", + "string" + ] + }, + "zip": { + "type": [ + "null", + "string" + ] + }, + "countryCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "trackingInfo": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "company": { + "type": [ + "null", + "string" + ] + }, + "number": { + "type": [ + "null", + "string" + ] + }, + "url": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "service": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + }, + "location": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + }, + "fulfillmentOrders": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "fulfillmentLineItems": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "quantity": { + "type": [ + "null", + "number" + ] + }, + "originalTotalSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + } + } + }, + "discountedTotalSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + } + } + }, + "lineItem": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + } + } + } + }, + "legacyResourceId": { + "type": [ + "null", + "string" + ] + }, + "order": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + }, + "events": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/tap_shopify/schemas/inventory_items.json b/tap_shopify/schemas/inventory_items.json index f6ce0838..47828eaa 100644 --- a/tap_shopify/schemas/inventory_items.json +++ b/tap_shopify/schemas/inventory_items.json @@ -4,7 +4,7 @@ "id": { "type": [ "null", - "integer" + "string" ] }, "sku": { @@ -13,49 +13,57 @@ "string" ] }, - "created_at": { + "createdAt": { "type": [ "null", "string" ], "format": "date-time" }, - "updated_at": { + "updatedAt": { "type": [ "null", "string" ], "format": "date-time" }, - "requires_shipping": { + "requiresShipping": { "type": [ "null", "boolean" ] }, - "cost": { + "unitCost": { "type": [ "null", - "string" + "object" ], - "format": "singer.decimal" + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + } + } }, - "country_code_of_origin": { + "countryCodeOfOrigin": { "type": [ "null", "string" ] }, - "province_code_of_origin": { + "provinceCodeOfOrigin": { "type": [ "null", "string" ] }, - "harmonized_system_code": { + "harmonizedSystemCode": { "type": [ "null", - "integer" + "string" ] }, "tracked": { @@ -64,7 +72,7 @@ "boolean" ] }, - "country_harmonized_system_codes": { + "countryHarmonizedSystemCodes": { "type": [ "null", "array" @@ -75,13 +83,13 @@ "object" ], "properties": { - "harmonized_system_code": { + "countryCode": { "type": [ "null", "string" ] }, - "country_code": { + "harmonizedSystemCode": { "type": [ "null", "string" @@ -89,12 +97,6 @@ } } } - }, - "admin_graphql_api_id": { - "type": [ - "null", - "string" - ] } } -} \ No newline at end of file +} diff --git a/tap_shopify/schemas/inventory_levels.json b/tap_shopify/schemas/inventory_levels.json index fb9e6f10..c365da36 100644 --- a/tap_shopify/schemas/inventory_levels.json +++ b/tap_shopify/schemas/inventory_levels.json @@ -1,21 +1,67 @@ { + "type": "object", "properties": { - "available": { - "type": ["null", "integer"] + "id": { + "type": ["null", "string"] }, - "inventory_item_id": { - "type": ["null", "integer"] + "canDeactivate": { + "type": ["null", "boolean"] }, - "updated_at": { + "createdAt": { "type": ["null", "string"], "format": "date-time" }, - "location_id": { - "type": ["null", "integer"] + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" }, - "admin_graphql_api_id": { + "deactivationAlert": { "type": ["null", "string"] + }, + "item": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "variant": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + } + } + } + } + }, + "location": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + } + } + }, + "quantities": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + } + } + } } - }, - "type": "object" + } } diff --git a/tap_shopify/schemas/locations.json b/tap_shopify/schemas/locations.json index 577e6917..b8d55375 100644 --- a/tap_shopify/schemas/locations.json +++ b/tap_shopify/schemas/locations.json @@ -1,3 +1,44 @@ { - "$ref": "definitions.json#/location" + "type": "object", + "properties": { + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "updatedAt": { "type": ["null", "string"], "format": "date-time" }, + "createdAt": { "type": ["null", "string"], "format": "date-time" }, + "isActive": { "type": ["null", "boolean"] }, + "addressVerified": { "type": ["null", "boolean"] }, + "deactivatable": { "type": ["null", "boolean"] }, + "deactivatedAt": { "type": ["null", "string"], "format": "date-time" }, + "deletable": { "type": ["null", "boolean"] }, + "fulfillsOnlineOrders": { "type": ["null", "boolean"] }, + "hasActiveInventory": { "type": ["null", "boolean"] }, + "hasUnfulfilledOrders": { "type": ["null", "boolean"] }, + "isFulfillmentService": { "type": ["null", "boolean"] }, + "legacyResourceId": { "type": ["null", "string"] }, + "shipsInventory": { "type": ["null", "boolean"] }, + "address": { + "type": ["null", "object"], + "properties": { + "countryCode": { "type": ["null", "string"] }, + "address1": { "type": ["null", "string"] }, + "city": { "type": ["null", "string"] }, + "address2": { "type": ["null", "string"] }, + "provinceCode": { "type": ["null", "string"] }, + "zip": { "type": ["null", "string"] }, + "province": { "type": ["null", "string"] }, + "phone": { "type": ["null", "string"] }, + "country": { "type": ["null", "string"] }, + "formatted": { "type": ["null", "array"], "items": { "type": ["null", "string"] } }, + "latitude": { "type": ["null", "string"], "format": "singer.decimal" }, + "longitude": { "type": ["null", "string"], "format": "singer.decimal" } + } + }, + "localPickupSettingsV2": { + "type": ["null", "object"], + "properties": { + "instructions": { "type": ["null", "string"] }, + "pickupTime": { "type": ["null", "string"] } + } + } + } } diff --git a/tap_shopify/schemas/metafields.json b/tap_shopify/schemas/metafields_collections.json similarity index 76% rename from tap_shopify/schemas/metafields.json rename to tap_shopify/schemas/metafields_collections.json index ba597731..5226a246 100644 --- a/tap_shopify/schemas/metafields.json +++ b/tap_shopify/schemas/metafields_collections.json @@ -1,18 +1,21 @@ { + "type": "object", "properties": { - "owner_id": { + "owner": { "type": [ "null", - "integer" - ] - }, - "admin_graphql_api_id": { - "type": [ - "null", - "string" - ] + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } }, - "owner_resource": { + "ownerType": { "type": [ "null", "string" @@ -30,7 +33,7 @@ "string" ] }, - "created_at": { + "createdAt": { "type": [ "null", "string" @@ -40,7 +43,7 @@ "id": { "type": [ "null", - "integer" + "string" ] }, "namespace": { @@ -64,13 +67,18 @@ ], "properties": {} }, - "updated_at": { + "updatedAt": { "type": [ "null", "string" ], "format": "date-time" + }, + "type": { + "type": [ + "null", + "string" + ] } - }, - "type": "object" + } } diff --git a/tap_shopify/schemas/custom_collections.json b/tap_shopify/schemas/metafields_customers.json similarity index 52% rename from tap_shopify/schemas/custom_collections.json rename to tap_shopify/schemas/metafields_customers.json index 4698ef45..5226a246 100644 --- a/tap_shopify/schemas/custom_collections.json +++ b/tap_shopify/schemas/metafields_customers.json @@ -1,103 +1,84 @@ { + "type": "object", "properties": { - "handle": { + "owner": { "type": [ "null", - "string" - ] + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } }, - "sort_order": { + "ownerType": { "type": [ "null", "string" ] }, - "body_html": { + "value_type": { "type": [ "null", "string" ] }, - "title": { + "key": { "type": [ "null", "string" ] }, - "id": { + "createdAt": { "type": [ "null", - "integer" - ] + "string" + ], + "format": "date-time" }, - "published_scope": { + "id": { "type": [ "null", "string" ] }, - "admin_graphql_api_id": { + "namespace": { "type": [ "null", "string" ] }, - "updated_at": { + "description": { "type": [ "null", "string" ] }, - "image": { - "properties": { - "alt": { - "type": [ - "null", - "string" - ] - }, - "src": { - "type": [ - "null", - "string" - ] - }, - "width": { - "type": [ - "null", - "integer" - ] - }, - "created_at": { - "type": [ - "null", - "string" - ] - }, - "height": { - "type": [ - "null", - "integer" - ] - } - }, + "value": { "type": [ "null", - "object" - ] + "integer", + "object", + "string" + ], + "properties": {} }, - "published_at": { + "updatedAt": { "type": [ "null", "string" - ] + ], + "format": "date-time" }, - "template_suffix": { + "type": { "type": [ "null", "string" ] } - }, - "type": "object" + } } diff --git a/tap_shopify/schemas/metafields_orders.json b/tap_shopify/schemas/metafields_orders.json new file mode 100644 index 00000000..5226a246 --- /dev/null +++ b/tap_shopify/schemas/metafields_orders.json @@ -0,0 +1,84 @@ +{ + "type": "object", + "properties": { + "owner": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + }, + "ownerType": { + "type": [ + "null", + "string" + ] + }, + "value_type": { + "type": [ + "null", + "string" + ] + }, + "key": { + "type": [ + "null", + "string" + ] + }, + "createdAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "namespace": { + "type": [ + "null", + "string" + ] + }, + "description": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "integer", + "object", + "string" + ], + "properties": {} + }, + "updatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "type": { + "type": [ + "null", + "string" + ] + } + } +} diff --git a/tap_shopify/schemas/metafields_products.json b/tap_shopify/schemas/metafields_products.json new file mode 100644 index 00000000..5226a246 --- /dev/null +++ b/tap_shopify/schemas/metafields_products.json @@ -0,0 +1,84 @@ +{ + "type": "object", + "properties": { + "owner": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + }, + "ownerType": { + "type": [ + "null", + "string" + ] + }, + "value_type": { + "type": [ + "null", + "string" + ] + }, + "key": { + "type": [ + "null", + "string" + ] + }, + "createdAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "namespace": { + "type": [ + "null", + "string" + ] + }, + "description": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "integer", + "object", + "string" + ], + "properties": {} + }, + "updatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "type": { + "type": [ + "null", + "string" + ] + } + } +} diff --git a/tap_shopify/schemas/order_refunds.json b/tap_shopify/schemas/order_refunds.json index d05a153d..a08d0b63 100644 --- a/tap_shopify/schemas/order_refunds.json +++ b/tap_shopify/schemas/order_refunds.json @@ -1,31 +1,23 @@ { "type": "object", "properties": { - "order_id": { - "type": [ - "null", - "integer" - ] - }, - "restock": { + "id": { "type": [ "null", - "boolean" + "string" ] }, - "order_adjustments": { - "$ref": "definitions.json#/order_adjustments" - }, - "processed_at": { + "createdAt": { "type": [ "null", "string" - ] + ], + "format": "date-time" }, - "user_id": { + "legacyResourceId": { "type": [ "null", - "integer" + "string" ] }, "note": { @@ -34,328 +26,420 @@ "string" ] }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "created_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "admin_graphql_api_id": { + "order": { "type": [ "null", - "string" - ] + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } }, - "refund_line_items": { + "refundLineItems": { "type": [ "null", "array" ], "items": { + "type": [ + "null", + "object" + ], "properties": { - "location_id": { + "id": { + "type": [ + "null", + "string" + ] + }, + "quantity": { "type": [ "null", "integer" ] }, - "subtotal_set": { + "priceSet": { + "type": [ + "null", + "object" + ], "properties": { - "shop_money": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], "properties": { - "currency_code": { + "amount": { "type": [ "null", "string" - ] + ], + "format": "singer.decimal" }, - "amount": { + "currencyCode": { "type": [ "null", "string" ] } - }, + } + }, + "shopMoney": { "type": [ "null", "object" - ] - }, - "presentment_money": { + ], "properties": { - "currency_code": { + "amount": { "type": [ "null", "string" - ] + ], + "format": "singer.decimal" }, - "amount": { + "currencyCode": { "type": [ "null", "string" ] } - }, + } + } + } + }, + "restockType": { + "type": [ + "null", + "string" + ] + }, + "restocked": { + "type": [ + "null", + "boolean" + ] + }, + "location": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { "type": [ "null", - "object" + "string" ] } - }, + } + }, + "subtotalSet": { "type": [ "null", "object" - ] - }, - "total_tax_set": { + ], "properties": { - "shop_money": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], "properties": { - "currency_code": { + "amount": { "type": [ "null", "string" - ] + ], + "format": "singer.decimal" }, - "amount": { + "currencyCode": { "type": [ "null", "string" ] } - }, + } + }, + "shopMoney": { "type": [ "null", "object" - ] - }, - "presentment_money": { + ], "properties": { - "currency_code": { + "amount": { "type": [ "null", "string" - ] + ], + "format": "singer.decimal" }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "totalTaxSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { "type": [ "null", "string" ] } - }, + } + }, + "shopMoney": { "type": [ "null", "object" - ] + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } } - }, - "type": [ - "null", - "object" - ] - }, - "line_item_id": { - "type": [ - "null", - "integer" - ] - }, - "total_tax": { - "type": [ - "null", - "number" - ] + } }, - "quantity": { - "type": [ - "null", - "integer" - ] - }, - "id": { + "lineItem": { "type": [ "null", - "integer" - ] - }, - "line_item": { + "object" + ], "properties": { - "gift_card": { + "id": { "type": [ "null", - "boolean" + "string" + ] + }, + "vendor": { + "type": [ + "null", + "string" + ] + }, + "quantity": { + "type": [ + "null", + "integer" ] }, - "price": { + "title": { "type": [ "null", "string" ] }, - "tax_lines": { + "requiresShipping": { + "type": [ + "null", + "boolean" + ] + }, + "originalTotalSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + } + } + }, + "taxLines": { "type": [ "null", "array" ], "items": { + "type": [ + "null", + "object" + ], "properties": { - "price_set": { + "priceSet": { + "type": [ + "null", + "object" + ], "properties": { - "shop_money": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], "properties": { - "currency_code": { + "amount": { "type": [ "null", - "string" + "number" ] }, - "amount": { + "currencyCode": { "type": [ "null", "string" ] } - }, + } + }, + "shopMoney": { "type": [ "null", "object" - ] - }, - "presentment_money": { + ], "properties": { - "currency_code": { + "amount": { "type": [ "null", - "string" + "number" ] }, - "amount": { + "currencyCode": { "type": [ "null", "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "rate": { "type": [ "null", - "object" + "number" ] }, - "price": { + "title": { "type": [ "null", "string" ] }, - "title": { + "source": { "type": [ "null", "string" ] }, - "rate": { + "channelLiable": { "type": [ "null", - "number" + "boolean" ] } - }, - "type": [ - "null", - "object" - ] + } } }, - "fulfillment_service": { + "taxable": { "type": [ "null", - "string" + "boolean" ] }, - "sku": { + "isGiftCard": { "type": [ "null", - "string" + "boolean" ] }, - "fulfillment_status": { + "name": { "type": [ "null", "string" ] }, - "properties": { + "discountedTotalSet": { "type": [ "null", - "array" + "object" ], - "items": { - "properties": { - "name": { - "type": [ - "null", - "string" - ] - }, - "value": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - } - }, - "quantity": { - "type": [ - "null", - "integer" - ] - }, - "variant_id": { - "type": [ - "null", - "integer" - ] - }, - "grams": { - "type": [ - "null", - "integer" - ] - }, - "requires_shipping": { - "type": [ - "null", - "boolean" - ] - }, - "vendor": { - "type": [ - "null", - "string" - ] - }, - "price_set": { "properties": { - "shop_money": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], "properties": { - "currency_code": { + "currencyCode": { "type": [ "null", "string" @@ -364,18 +448,18 @@ "amount": { "type": [ "null", - "string" + "number" ] } - }, + } + }, + "shopMoney": { "type": [ "null", "object" - ] - }, - "presentment_money": { + ], "properties": { - "currency_code": { + "currencyCode": { "type": [ "null", "string" @@ -384,166 +468,198 @@ "amount": { "type": [ "null", - "string" + "number" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, - "type": [ - "null", - "object" - ] - }, - "variant_inventory_management": { - "type": [ - "null", - "string" - ] + } }, - "pre_tax_price": { + "sku": { "type": [ "null", "string" ] }, - "variant_title": { + "product": { "type": [ "null", - "string" - ] - }, - "total_discount_set": { + "object" + ], "properties": { - "shop_money": { - "properties": { - "currency_code": { - "type": [ - "null", - "string" - ] - }, - "amount": { - "type": [ - "null", - "string" - ] - } - }, + "id": { "type": [ "null", - "object" - ] - }, - "presentment_money": { - "properties": { - "currency_code": { - "type": [ - "null", - "string" - ] - }, - "amount": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" + "string" ] } - }, - "type": [ - "null", - "object" - ] + } }, - "discount_allocations": { + "discountAllocations": { "type": [ "null", "array" ], "items": { + "type": [ + "null", + "object" + ], "properties": { - "amount": { + "allocatedAmountSet": { "type": [ "null", - "string" - ] - }, - "amount_set": { + "object" + ], "properties": { - "shop_money": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], "properties": { - "currency_code": { + "amount": { "type": [ "null", - "string" + "number" ] }, - "amount": { + "currencyCode": { "type": [ "null", "string" ] } - }, + } + }, + "shopMoney": { "type": [ "null", "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "number" + ] + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "discountApplication": { + "type": [ + "null", + "object" + ], + "properties": { + "index": { + "type": [ + "null", + "integer" ] }, - "presentment_money": { + "targetType": { + "type": [ + "null", + "string" + ] + }, + "targetSelection": { + "type": [ + "null", + "string" + ] + }, + "allocationMethod": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "object" + ], "properties": { - "currency_code": { + "__typename": { "type": [ "null", "string" ] }, "amount": { + "type": [ + "null", + "number" + ] + }, + "currencyCode": { "type": [ "null", "string" ] + }, + "percentage": { + "type": [ + "null", + "number" + ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + } + } + } + }, + "customAttributes": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "key": { "type": [ "null", - "object" + "string" ] }, - "discount_application_index": { + "value": { "type": [ "null", - "integer" + "string" ] } - }, - "type": [ - "null", - "object" - ] + } } }, - "pre_tax_price_set": { + "totalDiscountSet": { + "type": [ + "null", + "object" + ], "properties": { - "shop_money": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], "properties": { - "currency_code": { + "currencyCode": { "type": [ "null", "string" @@ -552,18 +668,18 @@ "amount": { "type": [ "null", - "string" + "number" ] } - }, + } + }, + "shopMoney": { "type": [ "null", "object" - ] - }, - "presentment_money": { + ], "properties": { - "currency_code": { + "currencyCode": { "type": [ "null", "string" @@ -572,99 +688,316 @@ "amount": { "type": [ "null", - "string" + "number" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, - "type": [ - "null", - "object" - ] - }, - "fulfillable_quantity": { - "type": [ - "null", - "integer" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "admin_graphql_api_id": { - "type": [ - "null", - "string" - ] - }, - "total_discount": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": [ - "null", - "string" - ] + } }, - "product_exists": { + "duties": { "type": [ "null", - "boolean" - ] + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "harmonizedSystemCode": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "taxLines": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "rate": { + "type": [ + "null", + "number" + ] + }, + "source": { + "type": [ + "null", + "string" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "channelLiable": { + "type": [ + "null", + "boolean" + ] + }, + "priceSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "number" + ] + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "number" + ] + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + } + } + } + }, + "countryCodeOfOrigin": { + "type": [ + "null", + "string" + ] + } + } + } }, - "taxable": { + "discountedUnitPriceSet": { "type": [ "null", - "boolean" - ] - }, - "product_id": { + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + } + } + } + } + } + } + } + }, + "orderAdjustments": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "amountSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { "type": [ "null", - "integer" - ] + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } }, - "title": { + "shopMoney": { "type": [ "null", - "string" - ] + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } } - }, + } + }, + "id": { "type": [ "null", - "object" + "string" ] }, - "subtotal": { + "reason": { "type": [ "null", - "number" + "string" ] }, - "restock_type": { + "taxAmountSet": { "type": [ "null", - "string" - ] + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } } - }, - "type": [ - "null", - "object" - ] + } } + }, + "updatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" } } } diff --git a/tap_shopify/schemas/order_shipping_lines.json b/tap_shopify/schemas/order_shipping_lines.json new file mode 100644 index 00000000..43425e5d --- /dev/null +++ b/tap_shopify/schemas/order_shipping_lines.json @@ -0,0 +1,475 @@ +{ + "type": "object", + "properties": { + "carrierIdentifier": { + "type": [ + "null", + "string" + ] + }, + "code": { + "type": [ + "null", + "string" + ] + }, + "orderId": { + "type": [ + "null", + "string" + ] + }, + "currentDiscountedPriceSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "custom": { + "type": [ + "null", + "boolean" + ] + }, + "deliveryCategory": { + "type": [ + "null", + "string" + ] + }, + "discountAllocations": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "allocatedAmountSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "discountApplication": { + "type": [ + "null", + "object" + ], + "properties": { + "__typename": { + "type": [ + "null", + "string" + ] + }, + "allocationMethod": { + "type": [ + "null", + "string" + ] + }, + "index": { + "type": [ + "null", + "integer" + ] + }, + "targetSelection": { + "type": [ + "null", + "string" + ] + }, + "targetType": { + "type": [ + "null", + "string" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "code": { + "type": [ + "null", + "string" + ] + }, + "description": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "object" + ], + "properties": { + "__typename": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "percentage": { + "type": [ + "null", + "number" + ] + } + } + } + } + } + } + } + }, + "discountedPriceSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "isRemoved": { + "type": [ + "null", + "boolean" + ] + }, + "originalPriceSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "phone": { + "type": [ + "null", + "string" + ] + }, + "shippingRateHandle": { + "type": [ + "null", + "string" + ] + }, + "source": { + "type": [ + "null", + "string" + ] + }, + "taxLines": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "channelLiable": { + "type": [ + "null", + "boolean" + ] + }, + "priceSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "rate": { + "type": [ + "null", + "number" + ] + }, + "ratePercentage": { + "type": [ + "null", + "string" + ] + }, + "source": { + "type": [ + "null", + "string" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "updatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + } + } +} \ No newline at end of file diff --git a/tap_shopify/schemas/orders.json b/tap_shopify/schemas/orders.json index dc87ea7e..eea4bb33 100644 --- a/tap_shopify/schemas/orders.json +++ b/tap_shopify/schemas/orders.json @@ -1,1115 +1,2421 @@ { + "type": "object", "properties": { - "presentment_currency": { - "type": [ - "null", - "string" - ] - }, - "subtotal_price_set": {}, - "total_discounts_set": {}, - "total_line_items_price_set": {}, - "total_price_set": {}, - "total_shipping_price_set": {}, - "total_tax_set": {}, - "total_price": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "line_items": { - "$ref": "definitions.json#/line_items" - }, - "processing_method": { - "type": [ - "null", - "string" - ] - }, - "order_number": { - "type": [ - "null", - "integer" - ] - }, - "confirmed": { - "type": [ - "null", - "boolean" - ] - }, - "total_discounts": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "total_line_items_price": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "order_adjustments": { - "$ref": "definitions.json#/order_adjustments" - }, - "shipping_lines": { + "additionalFees": { + "type": "array", "items": { + "type": "object", "properties": { - "tax_lines": { - "$ref": "definitions.json#/tax_lines" - }, - "phone": { - "type": [ - "null", - "string" - ] - }, - "discounted_price_set": {}, - "price_set": {}, + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, "price": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "title": { - "type": [ - "null", - "string" - ] - }, - "discount_allocations": { - "items": { - "properties": { - "discount_application_index": { - "type": [ - "null", - "integer" - ] - }, - "amount": { - "type": [ - "null", - "number" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "delivery_category": { - "type": [ - "null", - "string" - ] - }, - "discounted_price": { - "type": [ - "null", - "number" - ] - }, - "code": { - "type": [ - "null", - "string" - ] - }, - "requested_fulfillment_service_id": { - "type": [ - "null", - "string" - ] - }, - "carrier_identifier": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "source": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "admin_graphql_api_id": { - "type": [ - "null", - "string" - ] - }, - "device_id": { - "type": [ - "null", - "integer" - ] - }, - "cancel_reason": { - "type": [ - "null", - "string" - ] - }, - "currency": { - "type": [ - "null", - "string" - ] - }, - "payment_gateway_names": { - "items": { - "type": [ - "null", - "string" - ] - }, - "type": [ - "null", - "array" - ] - }, - "source_identifier": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "processed_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "referring_site": { - "type": [ - "null", - "string" - ] - }, - "contact_email": { - "type": [ - "null", - "string" - ] - }, - "location_id": { - "type": [ - "null", - "integer" - ] - }, - "fulfillments": { - "items": { - "properties": { - "location_id": { - "type": [ - "null", - "integer" - ] - }, - "receipt": { "type": ["null", "object"], "properties": { - "testcase": { - "type": ["null", "boolean"] + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } }, - "authorization": { - "type": ["null", "string"] + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } } } }, - "tracking_number": { - "type": [ - "null", - "string" - ] - }, - "created_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "shipment_status": { - "type": [ - "null", - "string" - ] - }, - "line_items": { - "$ref": "definitions.json#/line_items" - }, - "tracking_url": { - "type": [ - "null", - "string" - ] - }, - "service": { - "type": [ - "null", - "string" - ] - }, - "status": { - "type": [ - "null", - "string" - ] - }, - "admin_graphql_api_id": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "tracking_urls": { - "items": { - "type": [ - "null", - "string" - ] - }, - "type": [ - "null", - "array" - ] - }, - "tracking_numbers": { + "taxLines": { + "type": ["null", "array"], "items": { - "type": [ - "null", - "string" - ] - }, - "type": [ - "null", - "array" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "tracking_company": { - "type": [ - "null", - "string" - ] - }, - "updated_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" + "type": "object", + "properties": { + "channelLiable": { "type": ["null", "boolean"] }, + "priceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "rate": { "type": ["null", "number"] }, + "ratePercentage": { "type": ["null", "number"] }, + "source": { "type": ["null", "string"] }, + "title": { "type": ["null", "string"] } + } + } } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "customer": { - "$ref": "definitions.json#/customer" + } + } }, - "test": { - "type": [ - "null", - "boolean" - ] + "app": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "icon": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "string"] } + } + } + } }, - "total_tax": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" + "billingAddress": { + "type": ["null", "object"], + "properties": { + "address1": { "type": ["null", "string"] }, + "address2": { "type": ["null", "string"] }, + "city": { "type": ["null", "string"] }, + "company": { "type": ["null", "string"] }, + "coordinatesValidated": { "type": ["null", "boolean"] }, + "country": { "type": ["null", "string"] }, + "countryCodeV2": { "type": ["null", "string"] }, + "firstName": { "type": ["null", "string"] }, + "formattedArea": { "type": ["null", "string"] }, + "id": { "type": ["null", "string"] }, + "lastName": { "type": ["null", "string"] }, + "latitude": { "type": ["null", "number"] }, + "longitude": { "type": ["null", "number"] }, + "name": { "type": ["null", "string"] }, + "phone": { "type": ["null", "string"] }, + "province": { "type": ["null", "string"] }, + "provinceCode": { "type": ["null", "string"] }, + "timeZone": { "type": ["null", "string"] }, + "validationResultSummary": { "type": ["null", "string"] }, + "zip": { "type": ["null", "string"] } + } + }, + "billingAddressMatchesShippingAddress": { "type": ["null", "boolean"] }, + "canMarkAsPaid": { "type": ["null", "boolean"] }, + "canNotifyCustomer": { "type": ["null", "boolean"] }, + "cancelReason": { "type": ["null", "string"] }, + "cancellation": { + "type": ["null", "object"], + "properties": { + "staffNote": { "type": ["null", "string"] } + } }, - "payment_details": { + "cancelledAt": { "type": ["null", "string"], "format": "date-time" }, + "capturable": { "type": ["null", "boolean"] }, + "cartDiscountAmountSet": { + "type": ["null", "object"], "properties": { - "avs_result_code": { - "type": [ - "null", - "string" - ] - }, - "credit_card_company": { - "type": [ - "null", - "string" - ] - }, - "cvv_result_code": { - "type": [ - "null", - "string" - ] - }, - "credit_card_bin": { - "type": [ - "null", - "string" - ] + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } }, - "credit_card_number": { - "type": [ - "null", - "string" - ] + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } } - }, - "type": [ - "null", - "object" - ] - }, - "number": { - "type": [ - "null", - "integer" - ] - }, - "email": { - "type": [ - "null", - "string" - ] - }, - "source_name": { - "type": [ - "null", - "string" - ] - }, - "landing_site_ref": { - "type": [ - "null", - "string" - ] + } }, - "shipping_address": { + "channelInformation": { + "type": ["null", "object"], "properties": { - "phone": { - "type": [ - "null", - "string" - ] - }, - "country": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "address1": { - "type": [ - "null", - "string" - ] - }, - "longitude": { - "type": [ - "null", - "number" - ] - }, - "address2": { - "type": [ - "null", - "string" - ] - }, - "last_name": { - "type": [ - "null", - "string" - ] - }, - "first_name": { - "type": [ - "null", - "string" - ] + "id": { "type": ["null", "string"] }, + "channelId": { "type": ["null", "string"] } + } + }, + "clientIp": { "type": ["null", "string"] }, + "closed": { "type": ["null", "boolean"] }, + "closedAt": { "type": ["null", "string"], "format": "date-time" }, + "confirmationNumber": { "type": ["null", "string"] }, + "confirmed": { "type": ["null", "boolean"] }, + "createdAt": { "type": ["null", "string"], "format": "date-time" }, + "currencyCode": { "type": ["null", "string"] }, + "currentCartDiscountAmountSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } }, - "province": { - "type": [ - "null", - "string" - ] + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "currentShippingPriceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } }, - "city": { - "type": [ - "null", - "string" - ] + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "currentSubtotalLineItemsQuantity": { "type": ["null", "integer"] }, + "currentSubtotalPriceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } }, - "company": { - "type": [ - "null", - "string" - ] + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "currentTotalAdditionalFeesSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } }, - "latitude": { - "type": [ - "null", - "number" - ] + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "currentTotalDiscountsSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } }, - "country_code": { - "type": [ - "null", - "string" - ] + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "currentTotalDutiesSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } }, - "province_code": { - "type": [ - "null", - "string" - ] + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "currentTotalPriceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } }, - "zip": { - "type": [ - "null", - "string" - ] + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } } - }, - "type": [ - "null", - "object" - ] + } }, - "total_price_usd": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" + "currentTotalTaxSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } }, - "closed_at": { + "currentTotalWeight": { "type": ["null", "number"] }, + "customAttributes": { "type": [ "null", - "string" + "array" ], - "format": "date-time" - }, - "discount_applications": { "items": { - "properties": { - "target_type": { - "type": [ - "null", - "string" - ] - }, - "code": { - "type": [ - "null", - "string" - ] - }, - "description": { - "type": [ - "null", - "string" - ] - }, - "type": { - "type": [ - "null", - "string" - ] - }, - "target_selection": { - "type": [ - "null", - "string" - ] - }, - "allocation_method": { - "type": [ - "null", - "string" - ] - }, - "title": { - "type": [ - "null", - "string" - ] - }, - "value_type": { - "type": [ - "null", - "string" - ] - }, - "value": { - "type": [ - "null", - "number" - ] - } - }, "type": [ "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "note": { - "type": [ - "null", - "string" - ] - }, - "user_id": { - "type": [ - "null", - "integer" - ] - }, - "source_url": { - "type": [ - "null", - "string" - ] + "object" + ], + "properties": { + "key": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "string" + ] + } + } + } }, - "subtotal_price": { + "customer": { "type": [ "null", - "string" + "object" ], - "format": "singer.decimal" - }, - "billing_address": { "properties": { - "phone": { + "id": { + "type": [ + "null", + "string" + ] + }, + "email": { "type": [ "null", "string" ] }, - "country": { + "firstName": { "type": [ "null", "string" ] }, - "name": { + "lastName": { "type": [ "null", "string" ] }, - "address1": { + "state": { "type": [ "null", "string" ] }, - "longitude": { + "verifiedEmail": { "type": [ "null", - "number" + "boolean" ] }, - "address2": { + "lastOrder": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + }, + "updatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "createdAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "note": { "type": [ "null", "string" ] }, - "last_name": { + "multipassIdentifier": { "type": [ "null", "string" ] }, - "first_name": { + "tags": { "type": [ "null", "string" ] }, - "province": { + "taxExempt": { "type": [ "null", "string" ] }, - "city": { + "taxExemptions": { "type": [ "null", "string" ] }, - "company": { + "defaultAddress": { + "type": [ + "null", + "object" + ], + "properties": { + "address1": { + "type": [ + "null", + "string" + ] + }, + "address2": { + "type": [ + "null", + "string" + ] + }, + "city": { + "type": [ + "null", + "string" + ] + }, + "countryCodeV2": { + "type": [ + "null", + "string" + ] + }, + "province": { + "type": [ + "null", + "string" + ] + }, + "zip": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "country": { + "type": [ + "null", + "string" + ] + }, + "lastName": { + "type": [ + "null", + "string" + ] + }, + "phone": { + "type": [ + "null", + "string" + ] + }, + "firstName": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "provinceCode": { + "type": [ + "null", + "string" + ] + }, + "company": { + "type": [ + "null", + "string" + ] + } + } + }, + "addresses": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "address1": { + "type": [ + "null", + "string" + ] + }, + "address2": { + "type": [ + "null", + "string" + ] + }, + "city": { + "type": [ + "null", + "string" + ] + }, + "countryCodeV2": { + "type": [ + "null", + "string" + ] + }, + "province": { + "type": [ + "null", + "string" + ] + }, + "zip": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "country": { + "type": [ + "null", + "string" + ] + }, + "lastName": { + "type": [ + "null", + "string" + ] + }, + "phone": { + "type": [ + "null", + "string" + ] + }, + "firstName": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "provinceCode": { + "type": [ + "null", + "string" + ] + }, + "company": { + "type": [ + "null", + "string" + ] + } + } + } + } + } + }, + "customerJourneySummary": { + "type": [ + "null", + "object" + ], + "properties": { + "lastVisit": { + "type": [ + "null", + "object" + ], + "properties": { + "landingPage": { + "type": [ + "null", + "string" + ] + }, + "referrerUrl": { + "type": [ + "null", + "string" + ] + } + } + }, + "utmParameters": { + "type": [ + "null", + "object" + ], + "properties": { + "campaign": { + "type": [ + "null", + "string" + ] + }, + "content": { + "type": [ + "null", + "string" + ] + }, + "medium": { + "type": [ + "null", + "string" + ] + }, + "source": { + "type": [ + "null", + "string" + ] + }, + "term": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "merchantOfRecordApp": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { "type": [ "null", "string" ] + } + } + }, + "customerAcceptsMarketing": { "type": ["null", "boolean"] }, + "customerLocale": { "type": ["null", "string"] }, + "discountCodes": {"type": ["null", "array"], "items": { "type": "string" } }, + "discountCode": { "type": ["null", "string"] }, + "displayFinancialStatus": { "type": ["null", "string"] }, + "displayFulfillmentStatus": { "type": ["null", "string"] }, + "disputes": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "id": { "type": ["null", "string"] }, + "initiatedAs": { "type": ["null", "string"] }, + "status": { "type": ["null", "string"] } + } + } + }, + "dutiesIncluded": { "type": ["null", "boolean"] }, + "email": { "type": ["null", "string"] }, + "edited": { "type": ["null", "boolean"] }, + "estimatedTaxes": { "type": ["null", "boolean"] }, + "fulfillable": { "type": ["null", "boolean"] }, + "fullyPaid": { "type": ["null", "boolean"] }, + "hasTimelineComment": { "type": ["null", "boolean"] }, + "fulfillmentsCount": { + "type": ["null", "object"], + "properties": { + "count": { "type": ["null", "integer"] }, + "precision": { "type": ["null", "string"] } + } + }, + "id": { "type": "string" }, + "legacyResourceId": { "type": ["null", "string"] }, + "merchantBusinessEntity": { + "type": ["null", "object"], + "properties": { + "address": { + "type": ["null", "object"], + "properties": { + "address1": { "type": ["null", "string"] }, + "address2": { "type": ["null", "string"] }, + "city": { "type": ["null", "string"] }, + "countryCode": { "type": ["null", "string"] }, + "province": { "type": ["null", "string"] }, + "zip": { "type": ["null", "string"] } + } + }, + "companyName": { "type": ["null", "string"] }, + "displayName": { "type": ["null", "string"] }, + "id": { "type": ["null", "string"] }, + "primary": { "type": ["null", "boolean"] } + } + }, + "name": { "type": ["null", "string"] }, + "note": { "type": ["null", "string"] }, + "number": { "type": ["null", "number"] }, + "netPaymentSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "originalTotalAdditionalFeesSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "originalTotalDutiesSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "originalTotalPriceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "paymentGatewayNames": {"type": ["null", "array"], "items": { "type": "string" } }, + "phone": { "type": ["null", "string"] }, + "poNumber": { "type": ["null", "string"] }, + "presentmentCurrencyCode": { "type": ["null", "string"] }, + "processedAt": { "type": ["null", "string"], "format": "date-time" }, + "refundable": { "type": ["null", "boolean"] }, + "refundDiscrepancySet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "registeredSourceUrl": { "type": ["null", "string"] }, + "requiresShipping": { "type": ["null", "boolean"] }, + "restockable": { "type": ["null", "boolean"] }, + "returnStatus": { "type": ["null", "string"] }, + "shippingAddress": { + "type": ["null", "object"], + "properties": { + "address1": { "type": ["null", "string"] }, + "address2": { "type": ["null", "string"] }, + "city": { "type": ["null", "string"] }, + "company": { "type": ["null", "string"] }, + "coordinatesValidated": { "type": ["null", "boolean"] }, + "country": { "type": ["null", "string"] }, + "countryCodeV2": { "type": ["null", "string"] }, + "firstName": { "type": ["null", "string"] }, + "formattedArea": { "type": ["null", "string"] }, + "id": { "type": ["null", "string"] }, + "lastName": { "type": ["null", "string"] }, + "latitude": { "type": ["null", "number"] }, + "longitude": { "type": ["null", "number"] }, + "name": { "type": ["null", "string"] }, + "phone": { "type": ["null", "string"] }, + "province": { "type": ["null", "string"] }, + "provinceCode": { "type": ["null", "string"] }, + "timeZone": { "type": ["null", "string"] }, + "validationResultSummary": { "type": ["null", "string"] }, + "zip": { "type": ["null", "string"] } + } + }, + "shopifyProtect": { + "type": ["null", "object"], + "properties": { + "eligibility": { + "type": ["null", "object"], + "properties": { + "status": { "type": ["null", "string"] } + } + }, + "status": { "type": ["null", "string"] } + } + }, + "sourceIdentifier": { "type": ["null", "string"] }, + "sourceName": { "type": ["null", "string"] }, + "statusPageUrl": { "type": ["null", "string"] }, + "subtotalLineItemsQuantity": { "type": ["null", "integer"] }, + "subtotalPriceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "tags": {"type": ["null", "array"], "items": { "type": "string" } }, + "taxExempt": { "type": ["null", "boolean"] }, + "taxLines": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "channelLiable": { "type": ["null", "boolean"] }, + "priceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "rate": { "type": ["null", "number"] }, + "ratePercentage": { "type": ["null", "number"] }, + "source": { "type": ["null", "string"] }, + "title": { "type": ["null", "string"] } + } + } + }, + "taxesIncluded": { "type": ["null", "boolean"] }, + "test": { "type": ["null", "boolean"] }, + "totalCapturableSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "totalCashRoundingAdjustment": { + "type": ["null", "object"], + "properties": { + "paymentSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "refundSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + } + } + }, + "totalDiscountsSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } }, - "latitude": { - "type": [ - "null", - "number" - ] + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "totalOutstandingSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } }, - "country_code": { - "type": [ - "null", - "string" - ] + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "totalPriceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } }, - "province_code": { - "type": [ - "null", - "string" - ] + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "totalReceivedSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } }, - "zip": { - "type": [ - "null", - "string" - ] + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } } - }, - "type": [ - "null", - "object" - ] + } }, - "landing_site": { - "type": [ - "null", - "string" - ] + "totalRefundedSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } }, - "taxes_included": { - "type": [ - "null", - "boolean" - ] + "totalRefundedShippingSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } }, - "token": { - "type": [ - "null", - "string" - ] + "totalShippingPriceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } }, - "app_id": { - "type": [ - "null", - "integer" - ] + "totalTaxSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } }, - "total_tip_received": { - "type": [ - "null", - "string" - ] + "totalTipReceivedSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } }, - "browser_ip": { - "type": [ - "null", - "string" - ] + "totalWeight": { "type": ["null", "number"] }, + "transactionsCount": { + "type": ["null", "object"], + "properties": { + "count": { "type": ["null", "integer"] }, + "precision": { "type": ["null", "string"] } + } }, - "discount_codes": { + "unpaid": { "type": ["null", "boolean"] }, + "updatedAt": { "type": ["null", "string"], "format": "date-time" }, + "fulfillments": { + "type": ["null", "array"], "items": { + "type": "object", "properties": { - "code": { + "id": { "type": "string" }, + "name": { "type": ["null", "string"] }, + "status": { "type": ["null", "string"] }, + "totalQuantity": { "type": ["null", "number"] }, + "updatedAt": { "type": ["null", "string"], "format": "date-time" }, + "createdAt": { "type": ["null", "string"], "format": "date-time" }, + "deliveredAt": { "type": ["null", "string"], "format": "date-time" }, + "estimatedDeliveryAt": { "type": ["null", "string"], "format": "date-time" }, + "requiresShipping": { "type": ["null", "boolean"] }, + "inTransitAt" : { "type": ["null", "string"], "format": "date-time" }, + "trackingInfo": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "number": { "type": ["null", "string"] }, + "company": { "type": ["null", "string"] }, + "url": { "type": ["null", "string"] } + } + } + }, + "service": { + "type": ["null", "object"], + "properties": { + "serviceName": { "type": ["null", "string"] }, + "id": { "type": ["null", "string"] }, + "handle": { "type": ["null", "string"] }, + "trackingSupport": { "type": ["null", "boolean"] }, + "type": { "type": ["null", "string"] }, + "permitsSkuSharing": { "type": ["null", "boolean"] }, + "inventoryManagement": { "type": ["null", "boolean"] } + } + }, + "location": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "string"] } + } + }, + "location": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "string"] } + } + } + } + } + }, + "lineItems":{ + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { "type": [ "null", "string" ] }, - "amount": { + "vendor": { "type": [ "null", "string" - ], - "format": "singer.decimal" + ] }, - "type": { + "quantity": { "type": [ "null", - "string" + "number" ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "tax_lines": { - "$ref": "definitions.json#/tax_lines" - }, - "phone": { - "type": [ - "null", - "string" - ] - }, - "note_attributes": { - "items": { - "properties": { - "name": { + }, + "title": { "type": [ "null", "string" ] }, - "value": { + "requiresShipping": { "type": [ "null", - "string" + "boolean" ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "fulfillment_status": { - "type": [ - "null", - "string" - ] - }, - "order_status_url": { - "type": [ - "null", - "string" - ] - }, - "client_details": { - "properties": { - "session_hash": { - "type": [ - "null", - "string" - ] - }, - "accept_language": { - "type": [ - "null", - "string" - ] - }, - "browser_width": { - "type": [ - "null", - "integer" - ] - }, - "user_agent": { - "type": [ - "null", - "string" - ] - }, - "browser_ip": { - "type": [ - "null", - "string" - ] - }, - "browser_height": { - "type": [ - "null", - "integer" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "buyer_accepts_marketing": { - "type": [ - "null", - "boolean" - ] - }, - "checkout_token": { - "type": [ - "null", - "string" - ] - }, - "tags": { - "type": [ - "null", - "string" - ] - }, - "financial_status": { - "type": [ - "null", - "string" - ] - }, - "customer_locale": { - "type": [ - "null", - "string" - ] - }, - "checkout_id": { - "type": [ - "null", - "integer" - ] - }, - "total_weight": { - "type": [ - "null", - "integer" - ] - }, - "gateway": { - "type": [ - "null", - "string" - ] - }, - "cart_token": { - "type": [ - "null", - "string" - ] - }, - "cancelled_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "refunds": { - "items": { - "properties": { - "admin_graphql_api_id": { + }, + "originalTotalSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + } + } + }, + "taxLines": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "priceSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "number" + ] + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "number" + ] + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "rate": { + "type": [ + "null", + "number" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "source": { + "type": [ + "null", + "string" + ] + }, + "channelLiable": { + "type": [ + "null", + "boolean" + ] + } + } + } + }, + "taxable": { + "type": [ + "null", + "boolean" + ] + }, + "isGiftCard": { + "type": [ + "null", + "boolean" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "discountedTotalSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + } + } + }, + "sku": { "type": [ "null", "string" ] }, - "refund_line_items": { + "product": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + }, + "discountAllocations": { + "type": [ + "null", + "array" + ], "items": { + "type": [ + "null", + "object" + ], "properties": { - "line_item": { - "$ref": "definitions.json#/line_item" - }, - "location_id": { + "allocatedAmountSet": { "type": [ "null", - "integer" - ] + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "number" + ] + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "number" + ] + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } }, - "line_item_id": { + "discountApplication": { "type": [ "null", - "integer" - ] - }, - "quantity": { + "object" + ], + "properties": { + "index": { + "type": [ + "null", + "integer" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "targetType": { + "type": [ + "null", + "string" + ] + }, + "targetSelection": { + "type": [ + "null", + "string" + ] + }, + "allocationMethod": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "object" + ], + "properties": { + "__typename": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "percentage": { + "type": [ + "null", + "number" + ] + } + } + } + } + } + } + } + }, + "customAttributes": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "key": { "type": [ "null", - "integer" + "string" ] }, - "id": { + "value": { "type": [ "null", - "integer" + "string" ] - }, - "total_tax": { + } + } + } + }, + "totalDiscountSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + } + } + }, + "duties": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "harmonizedSystemCode": { "type": [ "null", - "number" + "string" ] }, - "restock_type": { + "id": { "type": [ "null", "string" ] }, - "subtotal": { + "taxLines": { "type": [ "null", - "number" + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "rate": { + "type": [ + "null", + "number" + ] + }, + "source": { + "type": [ + "null", + "string" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "channelLiable": { + "type": [ + "null", + "boolean" + ] + }, + "priceSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "number" + ] + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "number" + ] + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + } + } + } + }, + "countryCodeOfOrigin": { + "type": [ + "null", + "string" ] } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] + } + } }, - "restock": { + "discountedUnitPriceSet": { "type": [ "null", - "boolean" - ] + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + } + } }, - "note": { + "originalUnitPriceSet": { "type": [ "null", - "string" - ] + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + } + } }, - "id": { + "unfulfilledDiscountedTotalSet": { "type": [ "null", - "integer" - ] + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + } + } }, - "user_id": { + "unfulfilledOriginalTotalSet": { "type": [ "null", - "integer" - ] + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + } + } }, - "created_at": { + "variant": { "type": [ "null", - "string" + "object" ], - "format": "date-time" + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } }, - "processed_at": { + "lineItemGroup": { "type": [ - "null", - "string" + "null", + "object" ], - "format": "date-time" - }, - "order_adjustments": { - "$ref": "definitions.json#/order_adjustments" + "properties": { + "customAttributes": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "key": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "quantity": { + "type": [ + "null", + "integer" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "variantId": { + "type": [ + "null", + "string" + ] + }, + "variantSku": { + "type": [ + "null", + "string" + ] + } + } } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] + } + } }, - "created_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" + "shippingLine": { + "type": ["null", "object"], + "properties": { + "carrierIdentifier": { "type": ["null", "string"] }, + "code": { "type": ["null", "string"] }, + "currentDiscountedPriceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "custom": { "type": ["null", "boolean"] }, + "deliveryCategory": { "type": ["null", "string"] }, + "discountAllocations": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "allocatedAmountSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "discountApplication": { + "type": ["null", "object"], + "properties": { + "__typename": { "type": ["null", "string"] }, + "allocationMethod": { "type": ["null", "string"] }, + "index": { "type": ["null", "integer"] }, + "targetSelection": { "type": ["null", "string"] }, + "targetType": { "type": ["null", "string"] }, + "title": { "type": ["null", "string"] }, + "code": { "type": ["null", "string"] }, + "description": { "type": ["null", "string"] }, + "value": { + "type": ["null", "object"], + "properties": { + "__typename": { "type": ["null", "string"] }, + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] }, + "percentage": { "type": ["null", "number"] } + } + } + } + } + } + } + }, + "discountedPriceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "id": { "type": ["null", "string"] }, + "isRemoved": { "type": ["null", "boolean"] }, + "originalPriceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "phone": { "type": ["null", "string"] }, + "shippingRateHandle": { "type": ["null", "string"] }, + "source": { "type": ["null", "string"] }, + "taxLines": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "channelLiable": { "type": ["null", "boolean"] }, + "priceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "rate": { "type": ["null", "number"] }, + "ratePercentage": { "type": ["null", "string"] }, + "source": { "type": ["null", "string"] }, + "title": { "type": ["null", "string"] } + } + } + }, + "title": { "type": ["null", "string"] } + } }, - "updated_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" + "retailLocation": { + "type": ["null", "object"], + "properties": { + "activatable": { "type": ["null", "boolean"] }, + "address": { + "type": ["null", "object"], + "properties": { + "address1": { "type": ["null", "string"] }, + "address2": { "type": ["null", "string"] }, + "city": { "type": ["null", "string"] }, + "country": { "type": ["null", "string"] }, + "countryCode": { "type": ["null", "string"] }, + "formatted": { "type": ["null", "string"] }, + "latitude": { "type": ["null", "number"] }, + "longitude": { "type": ["null", "number"] }, + "phone": { "type": ["null", "string"] }, + "province": { "type": ["null", "string"] }, + "provinceCode": { "type": ["null", "string"] }, + "zip": { "type": ["null", "string"] } + } + }, + "addressVerified": { "type": ["null", "boolean"] }, + "createdAt": { "type": ["null", "string"], "format": "date-time" }, + "deactivatable": { "type": ["null", "boolean"] }, + "deactivatedAt": { "type": ["null", "string"], "format": "date-time" }, + "deletable": { "type": ["null", "boolean"] }, + "fulfillmentService": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "string"] } + } + }, + "fulfillsOnlineOrders": { "type": ["null", "boolean"] }, + "hasActiveInventory": { "type": ["null", "boolean"] }, + "hasUnfulfilledOrders": { "type": ["null", "boolean"] }, + "id": { "type": ["null", "string"] }, + "isActive": { "type": ["null", "boolean"] }, + "isFulfillmentService": { "type": ["null", "boolean"] }, + "legacyResourceId": { "type": ["null", "string"] }, + "localPickupSettingsV2": { + "type": ["null", "object"], + "properties": { + "instructions": { "type": ["null", "string"] }, + "pickupTime": { "type": ["null", "string"] } + } + }, + "name": { "type": ["null", "string"] }, + "shipsInventory": { "type": ["null", "boolean"] }, + "updatedAt": { "type": ["null", "string"], "format": "date-time" }, + "suggestedAddresses": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "address1": { "type": ["null", "string"] }, + "address2": { "type": ["null", "string"] }, + "city": { "type": ["null", "string"] }, + "country": { "type": ["null", "string"] }, + "countryCode": { "type": ["null", "string"] }, + "formatted": { "type": ["null", "string"] }, + "province": { "type": ["null", "string"] }, + "provinceCode": { "type": ["null", "string"] }, + "zip": { "type": ["null", "string"] } + } + } + } + } }, - "reference": { - "type": [ + "discountApplications": { + "type": [ "null", - "string" - ] - } - }, - "type": "object" + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "allocationMethod": { + "type": [ + "null", + "string" + ] + }, + "index": { + "type": [ + "null", + "integer" + ] + }, + "targetSelection": { + "type": [ + "null", + "string" + ] + }, + "targetType": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "object" + ], + "properties": { + "__typename": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "percentage": { + "type": [ + "null", + "number" + ] + } + } + }, + "__typename": { + "type": [ + "null", + "string" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "description": { + "type": [ + "null", + "string" + ] + }, + "code": { + "type": [ + "null", + "string" + ] + } + } + } + } + } } diff --git a/tap_shopify/schemas/product_variants.json b/tap_shopify/schemas/product_variants.json new file mode 100644 index 00000000..e9518bf8 --- /dev/null +++ b/tap_shopify/schemas/product_variants.json @@ -0,0 +1,207 @@ +{ + "type": "object", + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "createdAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "barcode": { + "type": [ + "null", + "string" + ] + }, + "availableForSale": { + "type": [ + "null", + "boolean" + ] + }, + "compareAtPrice": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "displayName": { + "type": [ + "null", + "string" + ] + }, + "image": { + "type": [ + "null", + "object" + ], + "properties": { + "altText": { + "type": [ + "null", + "string" + ] + }, + "height": { + "type": [ + "null", + "integer" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "url": { + "type": [ + "null", + "string" + ], + "format": "uri" + }, + "width": { + "type": [ + "null", + "integer" + ] + } + } + }, + "inventoryPolicy": { + "type": [ + "null", + "string" + ] + }, + "inventoryQuantity": { + "type": [ + "null", + "integer" + ] + }, + "position": { + "type": [ + "null", + "integer" + ] + }, + "price": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "requiresComponents": { + "type": [ + "null", + "boolean" + ] + }, + "sellableOnlineQuantity": { + "type": [ + "null", + "integer" + ] + }, + "sku": { + "type": [ + "null", + "string" + ] + }, + "taxCode": { + "type": [ + "null", + "string" + ] + }, + "taxable": { + "type": [ + "null", + "boolean" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "updatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "product": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + }, + "inventoryItem": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "measurement": { + "type": [ + "null", + "object" + ], + "properties": { + "weight": { + "type": [ + "null", + "object" + ], + "properties": { + "unit": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + } + } + } + } + } + } + } + } +} diff --git a/tap_shopify/schemas/products.json b/tap_shopify/schemas/products.json index 09f9a4ae..04d87ce0 100644 --- a/tap_shopify/schemas/products.json +++ b/tap_shopify/schemas/products.json @@ -1,4 +1,5 @@ { + "type": "object", "properties": { "status": { "type": [ @@ -6,46 +7,34 @@ "string" ] }, - "published_at": { + "publishedAt": { "type": [ "null", "string" ], "format": "date-time" }, - "created_at": { + "createdAt": { "type": [ "null", "string" ], "format": "date-time" }, - "published_scope": { - "type": [ - "null", - "string" - ] - }, "vendor": { "type": [ "null", "string" ] }, - "updated_at": { + "updatedAt": { "type": [ "null", "string" ], "format": "date-time" }, - "body_html": { - "type": [ - "null", - "string" - ] - }, - "product_type": { + "productType": { "type": [ "null", "string" @@ -70,12 +59,6 @@ "string" ] }, - "product_id": { - "type": [ - "null", - "integer" - ] - }, "values": { "type": [ "null", @@ -91,7 +74,7 @@ "id": { "type": [ "null", - "integer" + "string" ] }, "position": { @@ -107,222 +90,357 @@ ] } }, - "image": { - "$ref": "definitions.json#/image" - }, "handle": { "type": [ "null", "string" ] }, - "images": { + "templateSuffix": { "type": [ "null", - "array" - ], - "items": { - "$ref": "definitions.json#/image" - } + "string" + ] }, - "template_suffix": { + "title": { "type": [ "null", "string" ] }, - "title": { + "id": { "type": [ "null", "string" ] }, - "variants": { + "giftCardTemplateSuffix": { "type": [ "null", - "array" - ], - "items": { + "string" + ] + }, + "hasOnlyDefaultVariant": { + "type": [ + "null", + "string" + ] + }, + "hasOutOfStockVariants": { + "type": [ + "null", + "boolean" + ] + }, + "hasVariantsThatRequiresComponents": { + "type": [ + "null", + "boolean" + ] + }, + "isGiftCard": { + "type": [ + "null", + "boolean" + ] + }, + "description": { + "type": [ + "null", + "string" + ] + }, + "descriptionHtml": { + "type": [ + "null", + "string" + ] + }, + "compareAtPriceRange": { + "maxVariantCompareAtPrice": { + "type": [ + "null", + "object" + ], "properties": { - "barcode": { - "type": [ - "null", - "string" - ] - }, - "tax_code": { + "amount": { "type": [ "null", - "string" - ] - }, - "created_at": { - "type": [ - "null", - "string" + "string" ], - "format": "date-time" + "format": "singer.decimal" }, - "weight_unit": { + "currencyCode": { "type": [ "null", "string" ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "position": { - "type": [ - "null", - "integer" - ] - }, - "price": { + } + } + }, + "minVariantCompareAtPrice": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { "type": [ "null", - "string" + "string" ], "format": "singer.decimal" }, - "image_id": { - "type": [ - "null", - "integer" - ] - }, - "inventory_policy": { + "currencyCode": { "type": [ "null", "string" ] - }, - "sku": { + } + } + } + }, + "category": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + }, + "featuredMedia": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "mediaContentType": { + "type": [ + "null", + "string" + ] + }, + "status": { + "type": [ + "null", + "string" + ] + } + } + }, + "requiresSellingPlan": { + "type": [ + "null", + "boolean" + ] + }, + "totalInventory": { + "type": [ + "null", + "integer" + ] + }, + "tracksInventory": { + "type": [ + "null", + "boolean" + ] + }, + "media": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { "type": [ "null", "string" ] }, - "inventory_item_id": { - "type": [ - "null", - "integer" - ] - }, - "fulfillment_service": { + "alt": { "type": [ "null", "string" ] }, - "title": { + "status": { "type": [ "null", "string" ] }, - "weight": { - "type": [ - "null", - "number" - ] - }, - "inventory_management": { + "mediaContentType": { "type": [ "null", "string" ] }, - "taxable": { + "embedUrl": { "type": [ "null", - "boolean" + "string" ] }, - "admin_graphql_api_id": { + "mimeType": { "type": [ "null", "string" ] }, - "option1": { + "filename": { "type": [ "null", "string" ] }, - "compare_at_price": { + "image": { "type": [ "null", - "string" + "object" ], - "format": "singer.decimal" + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "url": { + "type": [ + "null", + "string" + ] + }, + "width": { + "type": [ + "null", + "string" + ] + }, + "height": { + "type": [ + "null", + "string" + ] + } + } }, - "updated_at": { + "sources": { "type": [ "null", - "string" + "array" ], - "format": "date-time" - }, - "option2": { - "type": [ - "null", - "string" - ] - }, - "old_inventory_quantity": { - "type": [ - "null", - "integer" - ] - }, - "requires_shipping": { - "type": [ - "null", - "boolean" - ] - }, - "inventory_quantity": { - "type": [ - "null", - "integer" - ] + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "url": { + "type": [ + "null", + "string" + ] + }, + "format": { + "type": [ + "null", + "string" + ] + }, + "mimeType": { + "type": [ + "null", + "string" + ] + }, + "fileSize": { + "type": [ + "null", + "string" + ] + } + } + } }, - "grams": { + "mediaWarnings": { "type": [ "null", - "integer" - ] + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "code": { + "type": [ + "null", + "string" + ] + }, + "message": { + "type": [ + "null", + "string" + ] + } + } + } }, - "option3": { + "mediaErrors": { "type": [ "null", - "string" - ] + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "code": { + "type": [ + "null", + "string" + ] + }, + "details": { + "type": [ + "null", + "string" + ] + }, + "message": { + "type": [ + "null", + "string" + ] + } + } + } } - }, - "type": [ - "null", - "object" - ] + } } - }, - "admin_graphql_api_id": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] } - }, - "type": "object" + } } diff --git a/tap_shopify/schemas/transactions.json b/tap_shopify/schemas/transactions.json index bd9d321b..53ebd02d 100644 --- a/tap_shopify/schemas/transactions.json +++ b/tap_shopify/schemas/transactions.json @@ -1,182 +1,614 @@ { + "type": "object", "properties": { - "error_code": { + "accountNumber": { "type": [ "null", "string" ] }, - "device_id": { + "amountRoundingSet": { "type": [ "null", - "integer" - ] + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } }, - "user_id": { + "amountSet": { "type": [ "null", - "integer" - ] + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } }, - "parent_id": { + "authorizationCode": { "type": [ "null", - "integer" + "string" ] }, - "test": { + "authorizationExpiresAt": { "type": [ "null", - "boolean" - ] + "string" + ], + "format": "date-time" }, - "kind": { + "createdAt": { "type": [ "null", "string" - ] + ], + "format": "date-time" }, - "order_id": { + "errorCode": { "type": [ "null", - "integer" + "string" ] }, - "amount": { + "formattedGateway": { "type": [ "null", "string" - ], - "format": "singer.decimal" + ] }, - "authorization": { + "gateway": { "type": [ "null", "string" ] }, - "currency": { + "id": { "type": [ "null", "string" ] }, - "source_name": { + "kind": { "type": [ "null", "string" ] }, - "message": { + "manualPaymentGateway": { "type": [ "null", - "string" + "boolean" ] }, - "id": { + "maximumRefundableV2": { "type": [ "null", - "integer" + "object" ] }, - "created_at": { + "multiCapturable": { "type": [ "null", - "string" + "boolean" ] }, - "status": { + "order": { "type": [ "null", - "string" - ] + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } }, - "payment_details": { + "parentTransaction": { + "type": [ + "null", + "object" + ], "properties": { - "cvv_result_code": { + "accountNumber": { "type": [ "null", "string" ] }, - "credit_card_bin": { + "createdAt": { "type": [ "null", "string" - ] + ], + "format": "date-time" }, - "credit_card_company": { + "id": { "type": [ "null", "string" ] }, - "credit_card_number": { + "status": { "type": [ "null", "string" ] }, - "avs_result_code": { + "paymentId": { "type": [ "null", "string" ] + }, + "processedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "amountSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } } - }, + } + }, + "paymentId": { "type": [ "null", - "object" + "string" ] }, - "gateway": { + "processedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "receiptJson": { "type": [ "null", "string" ] }, - "admin_graphql_api_id": { + "settlementCurrency": { "type": [ "null", "string" ] }, - "receipt": { + "settlementCurrencyRate": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "shopifyPaymentsSet": { "type": [ "null", "object" ], "properties": { - "fee_amount": { + "extendedAuthorizationSet": { "type": [ "null", - "string" + "object" ], - "format": "singer.decimal" + "properties": { + "extendedAuthorizationExpiresAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "standardAuthorizationExpiresAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + } + } }, - "gross_amount": { + "refundSet": { "type": [ "null", - "string" + "object" ], - "format": "singer.decimal" + "properties": { + "acquirerReferenceNumber": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "status": { + "type": [ + "null", + "string" + ] + }, + "test": { + "type": [ + "null", + "boolean" + ] + }, + "totalUnsettledSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } }, - "tax_amount": { + "shopMoney": { "type": [ "null", - "string" + "object" ], - "format": "singer.decimal" + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "fees": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "rate": { + "type": [ + "null", + "number" + ] + }, + "rateName": { + "type": [ + "null", + "string" + ] + }, + "taxAmount": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "type": { + "type": [ + "null", + "string" + ] + }, + "flatFeeName": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "flatFee": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } } - }, - "patternProperties": {".+": {}} + } }, - "location_id": { + "manuallyCapturable": { "type": [ "null", - "integer" + "boolean" ] + }, + "paymentDetails": { + "type": [ + "null", + "object" + ], + "properties": { + "avsResultCode": { + "type": [ + "null", + "string" + ] + }, + "bin": { + "type": [ + "null", + "string" + ] + }, + "company": { + "type": [ + "null", + "string" + ] + }, + "cvvResultCode": { + "type": [ + "null", + "string" + ] + }, + "expirationMonth": { + "type": [ + "null", + "integer" + ] + }, + "expirationYear": { + "type": [ + "null", + "integer" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "number": { + "type": [ + "null", + "string" + ] + }, + "paymentMethodName": { + "type": [ + "null", + "string" + ] + }, + "wallet": { + "type": [ + "null", + "string" + ] + }, + "paymentDescriptor": { + "type": [ + "null", + "string" + ] + } + } } - }, - "type": "object" + } } diff --git a/tap_shopify/streams/__init__.py b/tap_shopify/streams/__init__.py index 27a03e90..b647db5c 100644 --- a/tap_shopify/streams/__init__.py +++ b/tap_shopify/streams/__init__.py @@ -2,12 +2,17 @@ import tap_shopify.streams.customers import tap_shopify.streams.orders import tap_shopify.streams.order_refunds -import tap_shopify.streams.metafields +import tap_shopify.streams.metafields_products +import tap_shopify.streams.metafields_collections +import tap_shopify.streams.metafields_customers +import tap_shopify.streams.metafields_orders import tap_shopify.streams.transactions import tap_shopify.streams.products -import tap_shopify.streams.collects -import tap_shopify.streams.custom_collections import tap_shopify.streams.locations import tap_shopify.streams.inventory_levels import tap_shopify.streams.inventory_items +import tap_shopify.streams.product_variants import tap_shopify.streams.events +import tap_shopify.streams.collections +import tap_shopify.streams.fulfillment_orders +import tap_shopify.streams.order_shipping_lines diff --git a/tap_shopify/streams/abandoned_checkouts.py b/tap_shopify/streams/abandoned_checkouts.py index e3887f37..a1712ff8 100644 --- a/tap_shopify/streams/abandoned_checkouts.py +++ b/tap_shopify/streams/abandoned_checkouts.py @@ -1,9 +1,284 @@ -import shopify from tap_shopify.context import Context from tap_shopify.streams.base import Stream + class AbandonedCheckouts(Stream): - name = 'abandoned_checkouts' - replication_object = shopify.Checkout + """Stream class for Abandoned Checkouts in Shopify.""" + name = "abandoned_checkouts" + data_key = "abandonedCheckouts" + replication_key = "updatedAt" + + @classmethod + def process_sub_entities(cls, data, entity_name): + """ + Processes sub-entities from the response data. + + Args: + data (dict): Response data. + entity_name (str): Name of the entity to process. + + Returns: + list: List of processed sub-entities. + """ + sub_entities = [] + for item in data[entity_name]["edges"]: + if node := item.get("node"): + sub_entities.append(node) + return sub_entities + + def transform_object(self, obj): + """ + Transforms the object by processing its sub-entities. + + Args: + obj (dict): Object to transform. + + Returns: + dict: Transformed object. + """ + if obj.get("lineItems"): + obj["lineItems"] = self.process_sub_entities(obj, entity_name="lineItems") + return obj + + def get_query(self): + """ + Returns the GraphQL query for fetching abandoned checkouts. + + Returns: + str: GraphQL query string. + """ + return """ + query abandonedcheckouts($first: Int!, $after: String, $query: String) { + abandonedCheckouts(first: $first, after: $after, query: $query) { + edges { + node { + note + completedAt + billingAddress { + phone + country + firstName + name + latitude + zip + lastName + province + address2 + address1 + countryCodeV2 + city + company + provinceCode + longitude + coordinatesValidated + formattedArea + id + timeZone + validationResultSummary + } + discountCodes + createdAt + updatedAt + taxLines { + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + title + rate + source + channelLiable + ratePercentage + } + totalLineItemsPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + id + name + totalTaxSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + shippingAddress { + phone + country + firstName + name + latitude + zip + lastName + province + address2 + address1 + countryCodeV2 + city + company + provinceCode + longitude + coordinatesValidated + formattedArea + id + timeZone + validationResultSummary + } + abandonedCheckoutUrl + totalDiscountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + taxesIncluded + totalDutiesSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + lineItems(first: 250) { + edges { + node { + id + quantity + sku + title + variantTitle + variant { + title + id + } + discountedTotalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + components { + id + quantity + title + variantTitle + } + customAttributes { + key + value + } + product { + id + } + discountedUnitPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + discountedUnitPriceWithCodeDiscount { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + originalTotalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + originalUnitPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + subtotalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + customer { + id + lastOrder { + id + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + """ + -Context.stream_objects['abandoned_checkouts'] = AbandonedCheckouts +Context.stream_objects["abandoned_checkouts"] = AbandonedCheckouts diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index 724df755..631fa758 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -1,30 +1,36 @@ -import datetime +from datetime import timedelta import functools -import math -import sys +import json +import re import socket +import urllib +from urllib.error import URLError +import http import backoff import pyactiveresource import pyactiveresource.formats +import shopify import simplejson import singer from singer import metrics, utils +from graphql import parse, print_ast, visit +from graphql.language import Visitor, FieldNode, SelectionSetNode, OperationDefinitionNode, NameNode from tap_shopify.context import Context +from tap_shopify.exceptions import ShopifyError, ShopifyAPIError, ShopifyUnauthorizedError LOGGER = singer.get_logger() -RESULTS_PER_PAGE = 175 +RESULTS_PER_PAGE = 250 # set default timeout of 300 seconds REQUEST_TIMEOUT = 300 -# We've observed 500 errors returned if this is too large (30 days was too -# large for a customer) -DATE_WINDOW_SIZE = 1 +DEFAULT_DATE_WINDOW = 30 # We will retry a 500 error a maximum of 5 times before giving up MAX_RETRIES = 5 + # function to return request timeout def get_request_timeout(): @@ -37,6 +43,27 @@ def get_request_timeout(): return request_timeout +def execute_gql(self, query, variables=None, operation_name=None, timeout=None): + """ + This overrides the `execute` method from ShopifyAPI(v12.6.0) to remove the print statement + and also to explicitly pass the timeout value to the urlopen method. + Ensure to check the original impl before making any changes or upgrading the SDK version, + as this modification may affect future updates + """ + default_headers = {"Accept": "application/json", "Content-Type": "application/json"} + headers = self.merge_headers(default_headers, self.headers) + data = {"query": query, "variables": variables, "operationName": operation_name} + + req = urllib.request.Request(self.endpoint, json.dumps(data).encode("utf-8"), headers) + + try: + with urllib.request.urlopen(req, timeout=timeout) as response: + return response.read().decode("utf-8") + except urllib.error.HTTPError as http_error: + raise http_error + +shopify.GraphQL.execute = execute_gql + def is_not_status_code_fn(status_code): def gen_fn(exc): if getattr(exc, 'code', None) and exc.code not in status_code: @@ -49,22 +76,18 @@ def leaky_bucket_handler(details): LOGGER.info("Received 429 -- sleeping for %s seconds", details['wait']) +def retry_401_handler(_details): + if Context.client: + LOGGER.info("Received 401 Unauthorized - attempting token refresh and retry") + Context.client.refresh_token() + Context.client.reinitialize_session() + else: + LOGGER.info("Received 401 Unauthorized - no client available for token refresh") + def retry_handler(details): LOGGER.info("Received 500 or retryable error -- Retry %s/%s", details['tries'], MAX_RETRIES) -#pylint: disable=unused-argument -def retry_after_wait_gen(**kwargs): - # This is called in an except block so we can retrieve the exception - # and check it. - exc_info = sys.exc_info() - resp = exc_info[1].response - # Retry-After is an undocumented header. But honoring - # it was proven to work in our spikes. - # It's been observed to come through as lowercase, so fallback if not present - sleep_time_str = resp.headers.get('Retry-After', resp.headers.get('retry-after')) - yield math.floor(float(sleep_time_str)) - # boolean function to check if the error is 'timeout' error or not def is_timeout_error(error_raised): """ @@ -77,6 +100,11 @@ def is_timeout_error(error_raised): return True def shopify_error_handling(fnc): + @backoff.on_exception(backoff.expo, + (http.client.IncompleteRead, ConnectionResetError, + ShopifyAPIError, ShopifyError), + max_tries=MAX_RETRIES, + factor=2) @backoff.on_exception(backoff.expo, # timeout error raise by Shopify (pyactiveresource.connection.Error, socket.timeout), giveup=is_timeout_error, @@ -85,11 +113,21 @@ def shopify_error_handling(fnc): @backoff.on_exception(backoff.expo, (pyactiveresource.connection.ServerError, pyactiveresource.formats.Error, - simplejson.scanner.JSONDecodeError), + simplejson.scanner.JSONDecodeError, + URLError), giveup=is_not_status_code_fn(range(500, 599)), on_backoff=retry_handler, max_tries=MAX_RETRIES) - @backoff.on_exception(retry_after_wait_gen, + @backoff.on_exception(backoff.expo, + pyactiveresource.connection.ResourceNotFound, + giveup=is_not_status_code_fn([404]), + on_backoff=retry_handler, + max_tries=MAX_RETRIES) + @backoff.on_exception(backoff.expo, + ShopifyUnauthorizedError, + on_backoff=retry_401_handler, + max_tries=2) # Only retry once on 401 for refreshing token + @backoff.on_exception(backoff.expo, pyactiveresource.connection.ClientError, giveup=is_not_status_code_fn([429]), on_backoff=leaky_bucket_handler, @@ -100,31 +138,67 @@ def wrapper(*args, **kwargs): return fnc(*args, **kwargs) return wrapper -class Error(Exception): - """Base exception for the API interaction module""" - -class OutOfOrderIdsError(Error): - """Raised if our expectation of ordering by ID is violated""" - class Stream(): # Used for bookmarking and stream identification. Is overridden by # subclasses to change the bookmark key. name = None replication_method = 'INCREMENTAL' - replication_key = 'updated_at' + replication_key = 'updatedAt' + automatic_keys = [] key_properties = ['id'] - # Controls which SDK object we use to call the API by default. - replication_object = None - # Status parameter override option - status_key = None + date_window_size = None + data_key = None results_per_page = None + def _get_record_node_path(self): + """ + Returns the exact tuple of FieldNode names leading from the query root + to the selection set that holds the stream's top-level record fields. + + The default covers the standard ``{ stream { edges { node { FIELDS } } } }`` + shape; subclasses whose records sit at a different depth must override. + """ + return (self.data_key, "edges", "node") + def __init__(self): self.results_per_page = Context.get_results_per_page(RESULTS_PER_PAGE) + self.date_window_size = float(Context.config.get("date_window_size") or + DEFAULT_DATE_WINDOW) or DEFAULT_DATE_WINDOW # set request timeout self.request_timeout = get_request_timeout() + def get_query(self): + """ + Provides GraphQL query + """ + raise NotImplementedError("Function Not Implemented") + + def transform_object(self, obj): + """ + Modify this to perform custom transformation on each object + """ + return obj + + @classmethod + def camel_to_snake(cls, name): + """ + Convert camelCase to snake_case + + Args: + name (str): Input string in camelCase + + Returns: + str: Converted string in snake_case + """ + # Handle special cases + if not name: + return name + + # Use regex to insert underscore before capital letters + pattern = re.compile(r'(? str: + ast = parse(self.get_query()) + used_variable_names = set() + # O(1) membership checks. + fields_to_remove_set = set(fields_to_remove) + # Exact full path from the query root to the record-level selection set. + # Exact matching (not suffix) avoids false positives when a nested + # connection (e.g. lineItems.edges.node) ends with the same field names. + record_path = self._get_record_node_path() + + class FieldRemover(Visitor): + def __init__(self): + super().__init__() + # Incremental stack of FieldNode names maintained via + # enter_field / leave_field — O(1) push/pop per visit. + self._field_stack = [] + + def enter_field(self, node, *_): + self._field_stack.append(node.name.value) + + def leave_field(self, _node, *_): + self._field_stack.pop() + + def enter_selection_set(self, node, *_): + # Exact match against the full path: only the one selection set + # that corresponds to the stream's record node is pruned. + at_record_node = tuple(self._field_stack) == record_path + + new_selections = [] + for selection in node.selections: + if isinstance(selection, FieldNode): + if at_record_node and selection.name.value in fields_to_remove_set: + continue + # Track variable names used in field arguments. + for arg in selection.arguments or []: + if hasattr(arg.value, "name") and isinstance(arg.value.name, NameNode): + used_variable_names.add(arg.value.name.value) + new_selections.append(selection) + return SelectionSetNode(selections=new_selections) + + def leave_operation_definition(self, node, *_): + # Keep only variable definitions that are used + new_var_defs = [ + var_def for var_def in node.variable_definitions or [] + if var_def.variable.name.value in used_variable_names + ] + return OperationDefinitionNode( + operation=node.operation, + name=node.name, + variable_definitions=new_var_defs, + directives=node.directives, + selection_set=node.selection_set + ) + + # Start visiting the AST and dynamically gather used variable names + modified_ast = visit(ast, FieldRemover()) + return print_ast(modified_ast) # This function can be overridden by subclasses for specialized API # interactions. If you override it you need to remember to decorate it # with shopify_error_handling to get 429 and 500 handling. + # pylint: disable=E1123 @shopify_error_handling - def call_api(self, query_params): - # set timeout - self.replication_object.set_timeout(self.request_timeout) - return self.replication_object.find(**query_params) - - def get_query_params(self, since_id, status_key, updated_at_min, updated_at_max): - return { - "since_id": since_id, - "updated_at_min": updated_at_min, - "updated_at_max": updated_at_max, - "limit": self.results_per_page, - status_key: "any" + def call_api(self, query_params, query=None, data_key=None): + """ + - Modifies the default call API implementation to support GraphQL + - Returns response Object dict + """ + try: + query = query or self.get_query() + data_key = data_key or self.data_key + LOGGER.info("Fetching %s %s", self.name, query_params) + response = shopify.GraphQL().execute( + query=query, + variables=query_params, + timeout=self.request_timeout + ) + response = json.loads(response) + if "errors" in response.keys(): + raise ShopifyAPIError(response["errors"]) + + data = response.get("data", {}).get(data_key, {}) + return data + + except ShopifyAPIError as gql_error: + LOGGER.error("GraphQL Error: %s", gql_error) + raise ShopifyAPIError("An error occurred with the GraphQL API.") from gql_error + + except urllib.error.HTTPError as http_error: + # Extract X-Request-ID from the error response headers + request_id = http_error.headers.get("X-Request-ID") + error_body = http_error.read().decode("utf-8") if http_error.fp else None + error_message = ( + f"{http_error.reason} - {error_body}" + if error_body + else http_error.reason + ) + if http_error.code == 401: + raise ShopifyUnauthorizedError(http_error, + f"Unauthorized access - token may have expired with status {http_error.code} " + f"and X-Request-ID '{request_id or 'N/A'}', Reason: {error_message}." + ) from http_error + + raise ShopifyError(http_error, + f"GraphQL request failed for stream '{self.name}' with status {http_error.code} " + f"and X-Request-ID '{request_id or 'N/A'}', Reason: {error_message}." + ) from http_error + + except Exception as exc: + LOGGER.error("Unexpected error occurred.") + raise exc + + # pylint: disable=W0221 + def get_query_params(self, updated_at_min, updated_at_max, cursor=None): + """ + Construct query parameters for GraphQL requests. + + Args: + updated_at_min (str): Minimum updated_at timestamp. + updated_at_max (str): Maximum updated_at timestamp. + cursor (str): Pagination cursor, if any. + + Returns: + dict: Dictionary of query parameters. + """ + rkey = self.camel_to_snake(self.replication_key) + params = { + "query": f"{rkey}:>='{updated_at_min}' AND {rkey}:<'{updated_at_max}'", + "first": self.results_per_page, } + if cursor: + params["after"] = cursor + return params + + # pylint: disable=too-many-locals def get_objects(self): - updated_at_min = self.get_bookmark() - - stop_time = singer.utils.now().replace(microsecond=0) - date_window_size = float(Context.config.get("date_window_size", DATE_WINDOW_SIZE)) - - # Page through till the end of the resultset - while updated_at_min < stop_time: - # Bookmarking can also occur on the since_id - since_id = self.get_since_id() or 1 - - if since_id != 1: - LOGGER.info("Resuming sync from since_id %d", since_id) - - # It's important that `updated_at_min` has microseconds - # truncated. Why has been lost to the mists of time but we - # think it has something to do with how the API treats - # microseconds on its date windows. Maybe it's possible to - # drop data due to rounding errors or something like that? - updated_at_max = updated_at_min + datetime.timedelta(days=date_window_size) - if updated_at_max > stop_time: - updated_at_max = stop_time - while True: - status_key = self.status_key or "status" - query_params = self.get_query_params(since_id, - status_key, - updated_at_min, - updated_at_max) + """ + Returns: + - Yields list of objects for the stream + Performs: + - Pagination & Filtering of stream + - Transformation and bookmarking + """ + + last_updated_at = self.get_bookmark() + current_bookmark = last_updated_at + sync_start = utils.now().replace(microsecond=0) + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) + LOGGER.info("GraphQL query for stream '%s': %s", self.name, ' '.join(query.split())) + + while last_updated_at < sync_start: + date_window_end = last_updated_at + timedelta(days=self.date_window_size) + query_end = min(sync_start, date_window_end) + has_next_page, cursor = True, None + + while has_next_page: + query_params = self.get_query_params(last_updated_at, query_end, cursor) with metrics.http_request_timer(self.name): - objects = self.call_api(query_params) - - for obj in objects: - if obj.id < since_id: - # This verifies the api behavior expectation we - # have that all results actually honor the - # since_id parameter. - raise OutOfOrderIdsError("obj.id < since_id: {} < {}".format( - obj.id, since_id)) + data = self.call_api(query_params, query=query) + + for edge in data.get("edges"): + obj = self.transform_object(edge.get("node")) + replication_value = utils.strptime_to_utc(obj[self.replication_key]) + current_bookmark = max(current_bookmark, replication_value) yield obj - # You know you're at the end when the current page has - # less than the request size limits you set. - if len(objects) < self.results_per_page: - # Save the updated_at_max as our bookmark as we've synced all rows up in our - # window and can move forward. Also remove the since_id because we want to - # restart at 1. - Context.state.get('bookmarks', {}).get(self.name, {}).pop('since_id', None) - self.update_bookmark(utils.strftime(updated_at_max)) - break - - if objects[-1].id != max([o.id for o in objects]): - # This verifies the api behavior expectation we have - # that all pages are internally ordered by the - # `since_id`. - raise OutOfOrderIdsError("{} is not the max id in objects ({})".format( - objects[-1].id, max([o.id for o in objects]))) - since_id = objects[-1].id - - # Put since_id into the state. - self.update_bookmark(since_id, bookmark_key='since_id') - - updated_at_min = updated_at_max + page_info = data.get("pageInfo") + cursor , has_next_page = page_info.get("endCursor"), page_info.get("hasNextPage") - def sync(self): - """Yield's processed SDK object dicts to the caller. + last_updated_at = query_end + # Update bookmark to the latest value, but not beyond sync start time + max_bookmark_value = min(sync_start, current_bookmark) + self.update_bookmark(utils.strftime(max_bookmark_value)) - This is the default implementation. Get's all of self's objects - and calls to_dict on them with no further processing. + def sync(self): + """ + Default implementation for sync method """ - for obj in self.get_objects(): - yield obj.to_dict() + yield from self.get_objects() diff --git a/tap_shopify/streams/collections.py b/tap_shopify/streams/collections.py new file mode 100644 index 00000000..dc584207 --- /dev/null +++ b/tap_shopify/streams/collections.py @@ -0,0 +1,123 @@ +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream + + +class Collections(Stream): + """Stream class for Shopify collections.""" + name = "collections" + data_key = "collections" + replication_key = "updatedAt" + + def transform_products(self, data): + """ + Transforms the products data by extracting product IDs and handling pagination. + + Args: + data (dict): Product data. + + Returns: + list: List of product IDs. + """ + # Extract product IDs from the first page + product_ids = [ + node["id"] + for item in data["products"]["edges"] + if (node := item.get("node")) and "id" in node + ] + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) + + # Handle pagination + page_info = data["products"].get("pageInfo", {}) + while page_info.get("hasNextPage"): + params = { + "first": self.results_per_page, + "query": f"id:{data['id'].split('/')[-1]}", + "childafter": page_info.get("endCursor"), + } + + # Fetch the next page of data + response = self.call_api(params, query=query) + products_data = response.get("edges", [{}])[0].get("node", {}).get("products", {}) + product_ids.extend( + node["id"] + for item in products_data.get("edges", []) + if (node := item.get("node")) and "id" in node + ) + page_info = products_data.get("pageInfo", {}) + + return product_ids + + def transform_object(self, obj): + """ + Transforms a collection object. + + Args: + obj (dict): Collection object. + + Returns: + dict: Transformed collection object. + """ + obj["collectionType"] = "SMART" if obj.get("ruleSet") else "MANUAL" + if obj.get("products"): + obj["products"] = self.transform_products(obj) + return obj + + def get_query(self): + """ + Returns the GraphQL query for fetching collections. + + Returns: + str: GraphQL query string. + """ + return """ + query Collections($first: Int!, $after: String, $query: String, $childafter: String) { + collections(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { + edges { + node { + id + title + handle + updatedAt + productsCount { + count + precision + } + sortOrder + ruleSet { + appliedDisjunctively + rules { + column + condition + relation + } + } + seo { + description + title + } + feedback { + summary + } + products(first: 250, sortKey: ID, after: $childafter) { + edges { + node { + id + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + + +Context.stream_objects["collections"] = Collections diff --git a/tap_shopify/streams/collects.py b/tap_shopify/streams/collects.py deleted file mode 100644 index 83342ee5..00000000 --- a/tap_shopify/streams/collects.py +++ /dev/null @@ -1,50 +0,0 @@ -import shopify -import singer -from singer import utils -from tap_shopify.streams.base import (Stream, - OutOfOrderIdsError) -from tap_shopify.context import Context - -LOGGER = singer.get_logger() - -class Collects(Stream): - name = 'collects' - replication_object = shopify.Collect - replication_key = 'updated_at' - - def get_objects(self): - since_id = 1 - bookmark = self.get_bookmark() - max_bookmark = utils.strftime(utils.now()) - while True: - query_params = { - "since_id": since_id, - "limit": self.results_per_page, - } - - objects = self.call_api(query_params) - - for obj in objects: - # Syncing Collects is a full sync every time but emitting - # records that have an updated_date greater than the - # bookmark - if not obj.updated_at and obj.id: - LOGGER.info('Collect with id: %d does not have an updated_at, syncing it!', - obj.id) - if not obj.updated_at or utils.strptime_with_tz(obj.updated_at) > bookmark: - if obj.id < since_id: - raise OutOfOrderIdsError("obj.id < since_id: {} < {}".format( - obj.id, since_id)) - yield obj - - if len(objects) < self.results_per_page: - # Update the bookmark at the end of the last page - self.update_bookmark(max_bookmark) - break - if objects[-1].id != max([o.id for o in objects]): - raise OutOfOrderIdsError("{} is not the max id in objects ({})".format( - objects[-1].id, max([o.id for o in objects]))) - since_id = objects[-1].id - - -Context.stream_objects['collects'] = Collects diff --git a/tap_shopify/streams/custom_collections.py b/tap_shopify/streams/custom_collections.py deleted file mode 100644 index 5e5d0f49..00000000 --- a/tap_shopify/streams/custom_collections.py +++ /dev/null @@ -1,11 +0,0 @@ -import shopify - -from tap_shopify.streams.base import Stream -from tap_shopify.context import Context - - -class CustomCollections(Stream): - name = 'custom_collections' - replication_object = shopify.CustomCollection - -Context.stream_objects['custom_collections'] = CustomCollections diff --git a/tap_shopify/streams/customers.py b/tap_shopify/streams/customers.py index 8afab28c..f589abd6 100644 --- a/tap_shopify/streams/customers.py +++ b/tap_shopify/streams/customers.py @@ -1,11 +1,116 @@ -import shopify - -from tap_shopify.streams.base import Stream from tap_shopify.context import Context +from tap_shopify.streams.base import Stream class Customers(Stream): - name = 'customers' - replication_object = shopify.Customer + """Stream class for Shopify Customers.""" + name = "customers" + data_key = "customers" + replication_key = "updatedAt" + + def get_query(self): + """ + Returns the GraphQL query for fetching customers. + + Returns: + str: GraphQL query string. + """ + return """ + query Customers($first: Int!, $after: String, $query: String) { + customers(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { + edges { + node { + email + multipassIdentifier + defaultAddress { + city + address1 + zip + id + province + phone + country + firstName + lastName + countryCodeV2 + name + provinceCode + address2 + company + timeZone + validationResultSummary + latitude + longitude + coordinatesValidated + formattedArea + } + numberOfOrders + state + verifiedEmail + firstName + updatedAt + note + phone + addresses(first: 250) { + city + address1 + zip + id + province + phone + country + firstName + lastName + countryCodeV2 + name + provinceCode + address2 + company + timeZone + validationResultSummary + latitude + longitude + coordinatesValidated + formattedArea + } + lastName + tags + taxExempt + id + createdAt + taxExemptions + emailMarketingConsent { + consentUpdatedAt + marketingOptInLevel + marketingState + } + smsMarketingConsent { + consentCollectedFrom + consentUpdatedAt + marketingOptInLevel + marketingState + } + validEmailAddress + productSubscriberStatus + amountSpent { + amount + currencyCode + } + dataSaleOptOut + displayName + locale + lifetimeDuration + lastOrder { + id + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ -Context.stream_objects['customers'] = Customers +Context.stream_objects["customers"] = Customers diff --git a/tap_shopify/streams/events.py b/tap_shopify/streams/events.py index bd59aff2..16f6b70c 100644 --- a/tap_shopify/streams/events.py +++ b/tap_shopify/streams/events.py @@ -1,21 +1,84 @@ -import shopify - -from tap_shopify.streams.base import Stream from tap_shopify.context import Context +from tap_shopify.streams.base import Stream class Events(Stream): - name = 'events' - replication_object = shopify.Event - replication_key = "created_at" + """Stream class for Shopify Events.""" + + name = "events" + data_key = "events" + replication_key = "createdAt" + + def get_query(self): + """ + Returns the GraphQL query for fetching events. + + Returns: + str: GraphQL query string. + """ + return """ + query GetEvents($first: Int!, $after: String, $query: String) { + events(first: $first, after: $after, query: $query, sortKey: CREATED_AT) { + edges { + node { + id + createdAt + action + appTitle + attributeToApp + attributeToUser + criticalAlert + message + ... on BasicEvent { + id + subjectId + subjectType + action + additionalContent + additionalData + appTitle + arguments + attributeToApp + attributeToUser + createdAt + criticalAlert + hasAdditionalContent + message + secondaryMessage + } + ... on CommentEvent { + id + action + appTitle + attachments { + fileExtension + id + name + size + url + } + attributeToApp + attributeToUser + author { + id + } + canDelete + canEdit + createdAt + criticalAlert + edited + message + rawMessage + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ - def get_query_params(self, since_id, status_key, updated_at_min, updated_at_max): - return { - "since_id": since_id, - "created_at_min": updated_at_min, - "created_at_max": updated_at_max, - "limit": self.results_per_page, - status_key: "any" - } -Context.stream_objects['events'] = Events +Context.stream_objects["events"] = Events diff --git a/tap_shopify/streams/fulfillment_orders.py b/tap_shopify/streams/fulfillment_orders.py new file mode 100644 index 00000000..4032ddb3 --- /dev/null +++ b/tap_shopify/streams/fulfillment_orders.py @@ -0,0 +1,421 @@ +import singer +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream + +LOGGER = singer.get_logger() + + +class FulfillmentOrders(Stream): + """Stream class for Shopify fulfillment_orders.""" + + name = "fulfillment_orders" + data_key = "fulfillmentOrders" + replication_key = "updatedAt" + + # pylint: disable=W0221,fixme + def get_query_params(self, updated_at_min, updated_at_max, cursor=None): + """ + Construct query parameters for GraphQL requests. + + Args: + updated_at_min (str): Minimum updated_at timestamp. + updated_at_max (str): Maximum updated_at timestamp. + cursor (str): Pagination cursor, if any. + + Returns: + dict: Dictionary of query parameters. + """ + rkey = self.camel_to_snake(self.replication_key) + + params = { + "query": f"{rkey}:>='{updated_at_min}' AND {rkey}:<'{updated_at_max}'", + "first": self.results_per_page if self.results_per_page <= 30 else 30, + } + + if cursor: + params["after"] = cursor + return params + + def transform_childitems(self, data, parent_id, key, next_page_key): + """ + Paginate child items. + """ + child_records = [ + node for item in data["edges"] + if (node := item.get("node")) + ] + # Handle pagination + page_info = data.get("pageInfo", {}) + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) + while page_info.get("hasNextPage"): + params = { + "first": self.results_per_page, + "query": f"id:{parent_id.split('/')[-1]}", + next_page_key: page_info.get("endCursor"), + } + + # Fetch the next page of data + response = self.call_api(params, query=query) + node = response.get("edges", [])[0].get("node", {}) + child_records.extend( + node for item in node[key]["edges"] + if (node := item.get("node")) + ) + page_info = node.get("pageInfo", {}) + + return child_records + + def get_fulfillment_line_items(self, fulfillment_id, next_page=None): + """ + Fetch all fulfillment line items for a given fulfillment ID. + """ + fulfillment_line_items = [] + query = """ + query FulfillmentLineItems($fulfillmentId: ID!, $next_page: String) { + fulfillment(id: $fulfillmentId) { + fulfillmentLineItems(first: 100, after: $next_page) { + pageInfo { + hasNextPage + endCursor + } + nodes { + id + quantity + originalTotalSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + discountedTotalSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + lineItem { + id + } + } + } + } + } + """ + + params = { + "fulfillmentId": fulfillment_id, + } + if next_page: + params["next_page"] = next_page + + while True: + response = self.call_api(params, query=query, data_key="fulfillment") + fulfillment_line_items.extend( + response.get("fulfillmentLineItems", {}).get("nodes", []) + ) + + page_info = response.get("fulfillmentLineItems", {}).get("pageInfo", {}) + if not page_info.get("hasNextPage"): + break + + params["next_page"] = page_info.get("endCursor") + + return fulfillment_line_items + + def transform_object(self, obj): + """ + Transforms a collection object. + Args: + obj (dict): Collection object. + Returns: + dict: Transformed collection object. + """ + if obj.get("merchantRequests"): + obj["merchantRequests"] = self.transform_childitems( + obj.get("merchantRequests"), + obj["id"], "merchantRequests", + "merchant_request_after" + ) + + if obj.get("locationsForMove"): + obj["locationsForMove"] = self.transform_childitems( + obj.get("locationsForMove"), obj["id"], + "locationsForMove", + "locations_move_after" + ) + for item in obj["locationsForMove"]: + item["availableLineItems"] = item["availableLineItems"]["nodes"] + item["unavailableLineItems"] = item["unavailableLineItems"]["nodes"] + + if obj.get("fulfillments"): + obj["fulfillments"] = self.transform_childitems( + obj.get("fulfillments"), obj["id"], + "fulfillments", + "fulfillments_after" + ) + for item in obj["fulfillments"]: + item["fulfillmentOrders"] = item["fulfillmentOrders"]["nodes"] + item["events"] = item["events"]["nodes"] + initial_nodes = item["fulfillmentLineItems"]["nodes"] + if item["fulfillmentLineItems"]["pageInfo"]["hasNextPage"]: + more_nodes = self.get_fulfillment_line_items( + fulfillment_id=item["id"], + next_page=item["fulfillmentLineItems"]["pageInfo"]["endCursor"] + ) + item["fulfillmentLineItems"] = initial_nodes + more_nodes + else: + item["fulfillmentLineItems"] = initial_nodes + + if obj.get("fulfillmentOrdersForMerge"): + obj["fulfillmentOrdersForMerge"] = obj["fulfillmentOrdersForMerge"]["nodes"] + + return obj + + def get_query(self): + """ + Returns the GraphQL query for fetching fulfillmentOrders. + Returns: + str: GraphQL query string. + """ + return """ + query fulfillmentOrders($first: Int!, $after: String, $query: String, $merchant_request_after: String, $locations_move_after: String, $fulfillments_after: String) { + fulfillmentOrders(first: $first, after: $after, query: $query, includeClosed: true, sortKey: UPDATED_AT) { + edges { + node { + id + orderId + updatedAt + supportedActions { + action + externalUrl + } + status + requestStatus + orderProcessedAt + orderName + channelId + fulfillAt + fulfillBy + createdAt + destination { + address1 + address2 + city + countryCode + company + email + firstName + id + lastName + province + phone + zip + location { + id + } + } + fulfillmentHolds { + displayReason + handle + heldByRequestingApp + id + reason + reasonNotes + } + internationalDuties { + incoterm + } + deliveryMethod { + id + maxDeliveryDateTime + methodType + minDeliveryDateTime + presentedName + serviceCode + sourceReference + brandedPromise { + handle + name + } + additionalInformation { + instructions + phone + } + } + assignedLocation { + address1 + address2 + city + countryCode + name + phone + province + zip + location { + id + } + } + merchantRequests(first: 3, after: $merchant_request_after) { + edges { + node { + id + kind + message + requestOptions + responseData + sentAt + fulfillmentOrder { + id + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + locationsForMove(first: 3, after: $locations_move_after) { + edges { + node { + message + movable + location { + id + } + unavailableLineItemsCount { + count + precision + } + availableLineItemsCount { + count + precision + } + availableLineItems(first: 250) { + nodes { + id + } + } + unavailableLineItems(first:250) { + nodes { + id + } + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + fulfillmentOrdersForMerge(first: 250) { + nodes { + id + } + } + fulfillments(first: 3, after: $fulfillments_after) { + edges { + node { + createdAt + deliveredAt + displayStatus + estimatedDeliveryAt + id + inTransitAt + name + requiresShipping + status + totalQuantity + updatedAt + originAddress { + address1 + address2 + city + countryCode + provinceCode + zip + } + trackingInfo { + company + number + url + } + service { + id + } + location { + id + } + fulfillmentOrders(first: 250) { + nodes { + id + } + } + fulfillmentLineItems(first: 3) { + pageInfo { + hasNextPage + endCursor + } + nodes { + id + quantity + originalTotalSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + discountedTotalSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + currencyCode + amount + } + } + lineItem { + id + } + } + } + legacyResourceId + order { + id + } + events(first: 250) { + nodes { + id + } + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + + +Context.stream_objects["fulfillment_orders"] = FulfillmentOrders diff --git a/tap_shopify/streams/inventory_items.py b/tap_shopify/streams/inventory_items.py index 8b2dc676..dde493b4 100644 --- a/tap_shopify/streams/inventory_items.py +++ b/tap_shopify/streams/inventory_items.py @@ -1,56 +1,75 @@ -import singer -import shopify -from singer.utils import strftime,strptime_to_utc -from tap_shopify.streams.base import (Stream, shopify_error_handling) from tap_shopify.context import Context +from tap_shopify.streams.base import Stream -LOGGER = singer.get_logger() - -RESULTS_PER_PAGE = 250 class InventoryItems(Stream): - name = 'inventory_items' - replication_object = shopify.InventoryItem - - @shopify_error_handling - def get_inventory_items(self, inventory_items_ids): - # set timeout - self.replication_object.set_timeout(self.request_timeout) - return self.replication_object.find( - ids=inventory_items_ids, - limit=RESULTS_PER_PAGE) - - def get_objects(self): - - selected_parent = Context.stream_objects['products']() - selected_parent.name = "product_variants" + """Stream class for inventory items.""" - # Page through all `products`, bookmarking at `product_variants` - for parent_object in selected_parent.get_objects(): + name = "inventory_items" + data_key = "inventoryItems" + replication_key = "updatedAt" - product_variants = parent_object.variants - inventory_items_ids = ",".join( - [str(product_variant.inventory_item_id) for product_variant in product_variants]) + def transform_object(self, obj): + """ + Transforms the object by extracting country harmonized system codes. - # Max limit of IDs is 100 and Max limit of product_variants in one product is also 100 - # hence we can directly pass all inventory_items_ids - inventory_items = self.get_inventory_items(inventory_items_ids) + Args: + obj (dict): The object to transform. - for inventory_item in inventory_items: - yield inventory_item + Returns: + dict: Transformed object. + """ + hsc = obj.get("countryHarmonizedSystemCodes") + hsc_list = [] + if hsc and "edges" in hsc: + for edge in hsc.get("edges"): + node = edge.get("node") + if node: + hsc_list.append(node) + obj["countryHarmonizedSystemCodes"] = hsc_list + return obj - def sync(self): - bookmark = self.get_bookmark() - max_bookmark = bookmark - for inventory_item in self.get_objects(): - inventory_item_dict = inventory_item.to_dict() - replication_value = strptime_to_utc(inventory_item_dict[self.replication_key]) - if replication_value >= bookmark: - yield inventory_item_dict + def get_query(self): + """ + Returns GraphQL query to get all inventory items. - if replication_value > max_bookmark: - max_bookmark = replication_value + Returns: + str: GraphQL query string. + """ + return """ + query GetinventoryItems($first: Int!, $after: String, $query: String) { + inventoryItems(first: $first, after: $after, query: $query) { + edges { + node { + id + createdAt + sku + updatedAt + requiresShipping + countryCodeOfOrigin + provinceCodeOfOrigin + harmonizedSystemCode + tracked + unitCost { + amount + } + countryHarmonizedSystemCodes(first: 175) { + edges { + node { + countryCode + harmonizedSystemCode + } + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + """ - self.update_bookmark(strftime(max_bookmark)) -Context.stream_objects['inventory_items'] = InventoryItems +Context.stream_objects["inventory_items"] = InventoryItems diff --git a/tap_shopify/streams/inventory_levels.py b/tap_shopify/streams/inventory_levels.py index 8cc6efda..44a79862 100644 --- a/tap_shopify/streams/inventory_levels.py +++ b/tap_shopify/streams/inventory_levels.py @@ -1,60 +1,166 @@ -import shopify -from singer.utils import strftime, strptime_to_utc -from tap_shopify.streams.base import (Stream, - RESULTS_PER_PAGE, - shopify_error_handling) +from datetime import timedelta +from singer import metrics, utils from tap_shopify.context import Context +from tap_shopify.streams.base import Stream + class InventoryLevels(Stream): - name = 'inventory_levels' - replication_key = 'updated_at' - key_properties = ['location_id', 'inventory_item_id'] - replication_object = shopify.InventoryLevel - # Added decorator over functions of shopify SDK - replication_object.find = shopify_error_handling(replication_object.find) - - def api_call_for_inventory_levels(self, parent_object_id, bookmark): - # set timeout - self.replication_object.set_timeout(self.request_timeout) - return self.replication_object.find( - updated_at_min = bookmark, - limit = RESULTS_PER_PAGE, - location_ids=parent_object_id - ) - - def get_inventory_levels(self, parent_object, bookmark): - inventory_page = self.api_call_for_inventory_levels(parent_object, bookmark) - yield from inventory_page - - while inventory_page.has_next_page(): - inventory_page = inventory_page.next_page() - yield from inventory_page + """Stream class for inventory levels.""" + + name = "inventory_levels" + data_key = "locations" + child_data_key = "inventoryLevels" + replication_key = "updatedAt" + + def _get_record_node_path(self): + # Inventory-level records live at + # locations.edges.node.inventoryLevels.edges.node { FIELDS }. + return ("locations", "edges", "node", "inventoryLevels", "edges", "node") + + def get_next_page_child(self, parent_id, cursor, child_query): + """ + Gets all child objects efficiently with pagination. + + Args: + parent_id (str): The ID of the parent object. + cursor (str): The cursor for pagination. + child_query (str): The query for child objects. + + Yields: + dict: The child object. + """ + has_next_page = True + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) + while has_next_page: + child_query_params = { + "first": self.results_per_page, + "parentquery": f"id:{parent_id.split('/')[-1]}", + "query": child_query, + "childafter": cursor, + } + data = self.call_api(child_query_params, query=query) + for edge in data.get("edges", []): + node = edge.get("node", {}) + child_data = node.get(self.child_data_key, {}) + child_edges = child_data.get("edges", []) + yield from child_edges + + page_info = child_data.get("pageInfo", {}) + cursor = page_info.get("endCursor") + has_next_page = page_info.get("hasNextPage", False) + # pylint: disable=too-many-locals def get_objects(self): - bookmark = self.get_bookmark() + """ + Retrieves objects in paginated batches. + + Yields: + dict: The transformed object. + """ + last_updated_at = self.get_bookmark() + current_bookmark = last_updated_at + sync_start = utils.now().replace(microsecond=0) + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) + + while last_updated_at < sync_start: + date_window_end = last_updated_at + timedelta(days=self.date_window_size) + query_end = min(sync_start, date_window_end) + has_next_page, cursor = True, None + + while has_next_page: + query_params = self.get_query_params(last_updated_at, query_end, cursor) + with metrics.http_request_timer(self.name): + data = self.call_api(query_params, query=query) + + # Process parent objects + for edge in data.get("edges", []): + node = edge.get("node", {}) + + # Handle already fetched child objects + child_edges = node.get(self.child_data_key, {}).get("edges", []) + for child_obj in child_edges: + obj = self.transform_object(child_obj.get("node")) + replication_value = utils.strptime_to_utc(obj[self.replication_key]) + current_bookmark = max(current_bookmark, replication_value) + yield obj + + # Check if more child pages are needed + child_page_info = node.get(self.child_data_key, {}).get("pageInfo", {}) + if child_page_info.get("hasNextPage", False): + parent_id = node.get("id") + child_cursor = child_page_info.get("endCursor") + + # Get remaining child pages + for child_obj in self.get_next_page_child( + parent_id, child_cursor, query_params["query"] + ): + transformed_obj = self.transform_object(child_obj.get("node")) + replication_value = utils.strptime_to_utc( + transformed_obj[self.replication_key] + ) + current_bookmark = max(current_bookmark, replication_value) + yield transformed_obj - selected_parent = Context.stream_objects['locations']() - selected_parent.name = "inventory_level_locations" + page_info = data.get("pageInfo", {}) + cursor = page_info.get("endCursor") + has_next_page = page_info.get("hasNextPage") - # Get all locations data as location id is used for Inventory Level - # If we get locations updated after a bookmark - # then there is possibility of data loss for Inventory Level - # because location is not updated when any Inventory Level is updated inside it. - for parent_object in selected_parent.get_locations_data(): - yield from self.get_inventory_levels(parent_object.id, bookmark) + last_updated_at = query_end + # Update bookmark to the latest value, but not beyond sync start time + max_bookmark_value = min(sync_start, current_bookmark) + self.update_bookmark(utils.strftime(max_bookmark_value)) - def sync(self): - bookmark = self.get_bookmark() - max_bookmark = bookmark - for inventory_level in self.get_objects(): - inventory_level_dict = inventory_level.to_dict() - replication_value = strptime_to_utc(inventory_level_dict[self.replication_key]) - if replication_value >= bookmark: - yield inventory_level_dict + # pylint: disable=C0301 + def get_query(self): + """ + Returns the GraphQL query for inventory levels. - if replication_value > max_bookmark: - max_bookmark = replication_value + Returns: + str: The GraphQL query string. + """ + return """query GetInventoryLevels($first: Int!, $after: String, $query: String, $childafter: String, $parentquery: String) { + locations(first: $first, after: $after, query: $parentquery, sortKey: ID, includeInactive: true, includeLegacy: true) { + edges { + node { + inventoryLevels(first: $first, query: $query, after: $childafter) { + edges { + node { + canDeactivate + createdAt + deactivationAlert + id + location { + id + } + updatedAt + item { + id + variant { + id + } + } + quantities(names: ["available", "committed", "damaged", "incoming", "on_hand", "quality_control", "reserved", "safety_stock"]) { + id + name + quantity + updatedAt + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + id + } + } + pageInfo { + endCursor + hasNextPage + } + } + }""" - self.update_bookmark(strftime(max_bookmark)) -Context.stream_objects['inventory_levels'] = InventoryLevels +Context.stream_objects["inventory_levels"] = InventoryLevels diff --git a/tap_shopify/streams/locations.py b/tap_shopify/streams/locations.py index c3bd3c6c..67b60ec1 100644 --- a/tap_shopify/streams/locations.py +++ b/tap_shopify/streams/locations.py @@ -1,40 +1,69 @@ -import shopify -from singer import utils -from tap_shopify.streams.base import (Stream, shopify_error_handling) from tap_shopify.context import Context +from tap_shopify.streams.base import Stream class Locations(Stream): - name = 'locations' - replication_object = shopify.Location - # Added decorator over functions of shopify SDK - replication_object.find = shopify_error_handling(replication_object.find) + """Stream class for Shopify Locations""" - def get_locations_data(self): - # set timeout - self.replication_object.set_timeout(self.request_timeout) - location_page = self.replication_object.find() - yield from location_page + name = "locations" + data_key = "locations" + # Currently, the replication key is set to 'createdAt' because the Shopify + # locations graphql endpoint doesn't allow the filter on 'updatedAt' field. + replication_key = "createdAt" - while location_page.has_next_page(): - location_page = location_page.next_page() - yield from location_page + def get_query(self): + """ + Returns the GraphQL query for fetching locations. - def sync(self): - bookmark = self.get_bookmark() - max_bookmark = bookmark + Returns: + str: GraphQL query string. + """ + return """ + query GetLocations($first: Int!, $after: String, $query: String) { + locations(first: $first, after: $after, query: $query, sortKey: ID) { + edges { + node { + address { + countryCode + address1 + city + address2 + provinceCode + zip + province + phone + country + formatted + latitude + longitude + } + name + id + updatedAt + createdAt + isActive + addressVerified + deactivatable + deactivatedAt + deletable + fulfillsOnlineOrders + hasActiveInventory + hasUnfulfilledOrders + isFulfillmentService + legacyResourceId + localPickupSettingsV2 { + instructions + pickupTime + } + shipsInventory + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ - for location in self.get_locations_data(): - location_dict = location.to_dict() - replication_value = utils.strptime_to_utc(location_dict[self.replication_key]) - - if replication_value >= bookmark: - yield location_dict - - # update max bookmark if "replication_value" of current location is greater - if replication_value > max_bookmark: - max_bookmark = replication_value - - self.update_bookmark(utils.strftime(max_bookmark)) - -Context.stream_objects['locations'] = Locations +Context.stream_objects["locations"] = Locations diff --git a/tap_shopify/streams/metafields.py b/tap_shopify/streams/metafields.py index 5db1eecc..e620cd80 100644 --- a/tap_shopify/streams/metafields.py +++ b/tap_shopify/streams/metafields.py @@ -1,76 +1,143 @@ +from abc import ABC, abstractmethod +from datetime import timedelta import json -import shopify -import singer +from singer import utils, get_logger, metrics from tap_shopify.context import Context -from tap_shopify.streams.base import (Stream, - shopify_error_handling, - RESULTS_PER_PAGE, - OutOfOrderIdsError) - -LOGGER = singer.get_logger() - -def get_selected_parents(): - for parent_stream in ['orders', 'customers', 'products', 'custom_collections']: - if Context.is_selected(parent_stream): - yield Context.stream_objects[parent_stream]() - -@shopify_error_handling -def get_metafields(parent_object, since_id, parent_replication_object, timeout): - # set timeout - parent_replication_object.set_timeout(timeout) - # This call results in an HTTP request - the parent object never has a - # cache of this data so we have to issue that request. - return parent_object.metafields( - limit=Context.get_results_per_page(RESULTS_PER_PAGE), - since_id=since_id) - -class Metafields(Stream): - name = 'metafields' - replication_object = shopify.Metafield +from tap_shopify.streams.base import Stream +LOGGER = get_logger() + + +class Metafields(Stream, ABC): + """Stream class for Shopify Metafields""" + + name = None + data_key = None + child_data_key = "metafields" + replication_key = "updatedAt" + + def _get_record_node_path(self): + # Metafield records live at {data_key}.edges.node.metafields.edges.node. + # data_key is defined by each concrete subclass. + return (self.data_key, "edges", "node", "metafields", "edges", "node") + + @abstractmethod + def get_query(self): + """Placeholder for get_query method.""" + + def transform_object(self, obj): + """ + Transforms a metafield object for output. + """ + user_agent = Context.config.get("user_agent") + obj["value_type"] = obj.get("type") or None + obj["updated_at"] = obj.get("updatedAt") + if user_agent: + if isinstance(obj.get("value"), (dict, list)): + obj["value"] = json.dumps(obj["value"]) + elif obj["value_type"] in ["json", "weight", "volume", "dimension", "rating"]: + value = obj.get("value") + try: + obj["value"] = json.loads(value) if value is not None else value + except json.decoder.JSONDecodeError: + LOGGER.info("Failed to decode JSON value for obj %s", obj.get("id")) + return obj + + def fetch_paginated_child_data(self, initial_child_data, parent_id): + """ + Fetches all pages of child data by handling pagination. + """ + # Extract the numeric ID from the full path + numeric_id = parent_id.split('/')[-1] + page_info = initial_child_data.get("pageInfo", {}) + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) + + while page_info.get("hasNextPage"): + query_params = { + "first": self.results_per_page, + "query": f"id:{numeric_id}", + "childafter": page_info.get("endCursor"), + } + + response = self.call_api(query_params, query=query) + response_edges = response.get("edges", []) + if not response_edges: + break + + first_edge = response_edges[0] + child_data = first_edge.get("node", {}).get(self.child_data_key, {}) + + yield from child_data.get("edges", []) + + page_info = child_data.get("pageInfo", {}) + + # pylint: disable=too-many-locals, too-many-nested-blocks def get_objects(self): - # Get top-level shop metafields - yield from super().get_objects() - # Get parent objects, bookmarking at `metafield_` - for selected_parent in get_selected_parents(): - # The name member controls many things, but most importantly - # the bookmark key. This switches us over to the - # `metafield_` bookmark. We track that separately - # to make resetting individual streams easier. - selected_parent.name = "metafield_{}".format(selected_parent.name) - for parent_object in selected_parent.get_objects(): - since_id = 1 - while True: - metafields = get_metafields(parent_object, - since_id, - selected_parent.replication_object, - self.request_timeout) - for metafield in metafields: - if metafield.id < since_id: - raise OutOfOrderIdsError("metafield.id < since_id: {} < {}".format( - metafield.id, since_id)) - yield metafield - if len(metafields) < self.results_per_page: - break - if metafields[-1].id != max([o.id for o in metafields]): - raise OutOfOrderIdsError("{} is not the max id in metafields ({})".format( - metafields[-1].id, max([o.id for o in metafields]))) - since_id += metafields[-1].id + """ + Main iterator to yield metafield objects. + """ + sync_start = utils.now().replace(microsecond=0) + + # Set the initial last updated time to the bookmark minus one minute + # to ensure we don't miss any updates as it was observed shopify + # updates the parent object initially and then the child objects + last_updated_at = self.get_bookmark() - timedelta(minutes=1) + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) + LOGGER.info("GraphQL query for stream '%s': %s", self.name, ' '.join(query.split())) + + while last_updated_at < sync_start: + date_window_end = last_updated_at + timedelta(days=self.date_window_size) + query_end = min(sync_start, date_window_end) + + has_next_page = True + cursor = None + + while has_next_page: + query_params = self.get_query_params(last_updated_at, query_end, cursor) + with metrics.http_request_timer(self.name): + data = self.call_api(query_params, query=query) + + # Process parent objects + for edge in data.get("edges", []): + node = edge.get("node", {}) + + # First handle the already fetched child objects + child_edges = node.get(self.child_data_key).get("edges", []) + for child_obj in child_edges: + obj = self.transform_object(child_obj.get("node")) + yield obj + + # Check if we need to get more child pages + child_page_info = node.get(self.child_data_key, {}).get("pageInfo", {}) + if child_page_info.get("hasNextPage", False): + parent_id = node.get("id") + for child_obj in self.fetch_paginated_child_data( + node.get(self.child_data_key), parent_id + ): + transformed_obj = self.transform_object(child_obj.get("node")) + yield transformed_obj + + page_info = data.get("pageInfo", {}) + cursor, has_next_page = page_info.get("endCursor"), page_info.get("hasNextPage") + + last_updated_at = query_end def sync(self): - # Shop metafields - for metafield in self.get_objects(): - metafield = metafield.to_dict() - value_type = metafield.get("value_type") - if value_type and value_type == "json_string": - value = metafield.get("value") - try: - metafield["value"] = json.loads(value) if value is not None else value - except json.decoder.JSONDecodeError: - LOGGER.info("Failed to decode JSON value for metafield %s", metafield.get('id')) - metafield["value"] = value - - yield metafield - -Context.stream_objects['metafields'] = Metafields + """ + Performs pseudo incremental sync. + """ + start_time = utils.now().replace(microsecond=0) + max_bookmark_value = current_bookmark_value = self.get_bookmark() + + for obj in self.get_objects(): + replication_value = utils.strptime_to_utc(obj[self.replication_key]) + + max_bookmark_value = max(max_bookmark_value, replication_value) + + if replication_value >= current_bookmark_value: + yield obj + + # Update bookmark to the latest value, but not beyond sync start time + max_bookmark_value = min(start_time, max_bookmark_value) + self.update_bookmark(utils.strftime(max_bookmark_value)) diff --git a/tap_shopify/streams/metafields_collections.py b/tap_shopify/streams/metafields_collections.py new file mode 100644 index 00000000..1b94cd4e --- /dev/null +++ b/tap_shopify/streams/metafields_collections.py @@ -0,0 +1,58 @@ +"""MetafieldsCollections stream for Shopify tap.""" + +from tap_shopify.context import Context +from tap_shopify.streams.metafields import Metafields + + +class MetafieldsCollections(Metafields): + """Stream class for metafields associated with collections.""" + + name = "metafields_collections" + data_key = "collections" + + def get_query(self): + """Return the GraphQL query for fetching collection metafields.""" + return """ + query getCollectionsMetafields( + $first: Int!, $after: String, $query: String, $childafter: String + ) { + collections(first: $first, after: $after, query: $query) { + edges { + node { + metafields(first: $first, after: $childafter) { + edges { + node { + id + ownerType + value + type + key + createdAt + namespace + description + updatedAt + owner { + ... on Collection { + id + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + id + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + + +Context.stream_objects["metafields_collections"] = MetafieldsCollections diff --git a/tap_shopify/streams/metafields_customers.py b/tap_shopify/streams/metafields_customers.py new file mode 100644 index 00000000..e3290c78 --- /dev/null +++ b/tap_shopify/streams/metafields_customers.py @@ -0,0 +1,57 @@ +"""MetafieldsCustomers stream for Shopify tap.""" + +from tap_shopify.context import Context +from tap_shopify.streams.metafields import Metafields + + +class MetafieldsCustomers(Metafields): + """Stream class for metafields associated with customers.""" + + name = "metafields_customers" + data_key = "customers" + + def get_query(self): + """Return the GraphQL query for fetching customer metafields.""" + return """ + query getCustomerMetafields( + $first: Int!, $after: String, $query: String, $childafter: String + ) { + customers(first: $first, after: $after, query: $query) { + edges { + node { + metafields(first: $first, after: $childafter) { + edges { + node { + id + ownerType + value + type + key + createdAt + namespace + description + updatedAt + owner { + ... on Customer { + id + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + id + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + +Context.stream_objects["metafields_customers"] = MetafieldsCustomers diff --git a/tap_shopify/streams/metafields_orders.py b/tap_shopify/streams/metafields_orders.py new file mode 100644 index 00000000..2c7f0d36 --- /dev/null +++ b/tap_shopify/streams/metafields_orders.py @@ -0,0 +1,57 @@ +"""MetafieldsOrders stream for Shopify tap.""" + +from tap_shopify.context import Context +from tap_shopify.streams.metafields import Metafields + + +class MetafieldsOrders(Metafields): + """Stream class for metafields associated with orders.""" + name = "metafields_orders" + data_key = "orders" + + def get_query(self): + """Returns the GraphQL query for fetching order metafields.""" + return """ + query getOrderMetafields( + $first: Int!, $after: String, $query: String, $childafter: String + ) { + orders(first: $first, after: $after, query: $query) { + edges { + node { + metafields(first: $first, after: $childafter) { + edges { + node { + id + ownerType + value + type + key + createdAt + namespace + description + updatedAt + owner { + ... on Order { + id + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + id + } + } + pageInfo { + endCursor + hasNextPage + } + } + }""" + + +# Register the stream object in the context +Context.stream_objects["metafields_orders"] = MetafieldsOrders diff --git a/tap_shopify/streams/metafields_products.py b/tap_shopify/streams/metafields_products.py new file mode 100644 index 00000000..c4c5e6e3 --- /dev/null +++ b/tap_shopify/streams/metafields_products.py @@ -0,0 +1,57 @@ +"""MetafieldsProducts stream for Shopify tap.""" + +from tap_shopify.context import Context +from tap_shopify.streams.metafields import Metafields + + +class MetafieldsProducts(Metafields): + """Stream class for product metafields.""" + name = "metafields_products" + data_key = "products" + + def get_query(self): + """Return the GraphQL query for product metafields.""" + query = """ + query getProductMetafields( + $first: Int!, $after: String, $query: String, $childafter: String + ) { + products(first: $first, after: $after, query: $query) { + edges { + node { + metafields(first: $first, after: $childafter) { + edges { + node { + id + ownerType + value + type + key + createdAt + namespace + description + updatedAt + owner { + ... on Product { + id + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + id + } + } + pageInfo { + endCursor + hasNextPage + } + } + }""" + return query + + +Context.stream_objects["metafields_products"] = MetafieldsProducts diff --git a/tap_shopify/streams/order_refunds.py b/tap_shopify/streams/order_refunds.py index 2b168146..b4d34106 100644 --- a/tap_shopify/streams/order_refunds.py +++ b/tap_shopify/streams/order_refunds.py @@ -1,59 +1,407 @@ -import shopify -from singer.utils import strftime, strptime_to_utc +from datetime import timedelta +from singer import metrics, utils from tap_shopify.context import Context -from tap_shopify.streams.base import (Stream, - shopify_error_handling, - OutOfOrderIdsError) +from tap_shopify.streams.base import Stream + class OrderRefunds(Stream): - name = 'order_refunds' - replication_object = shopify.Refund - replication_key = 'created_at' - - @shopify_error_handling - def get_refunds(self, parent_object, since_id): - # set timeout - self.replication_object.set_timeout(self.request_timeout) - return self.replication_object.find( - order_id=parent_object.id, - limit=self.results_per_page, - since_id=since_id, - order='id asc') + """Stream class for fetching order refunds from Shopify""" + + name = "order_refunds" + data_key = "orders" + child_data_key = "refunds" + replication_key = "updatedAt" + automatic_keys = ["order"] + def _get_record_node_path(self): + # Refund records live at orders.edges.node.refunds { FIELDS }, + # not at the standard edges.node depth. + return ("orders", "edges", "node", "refunds") + + # pylint: disable=too-many-locals def get_objects(self): - selected_parent = Context.stream_objects['orders']() - selected_parent.name = "refund_orders" + """ + Fetch order refund objects within date windows, yielding each refund individually. + + Yields: + dict: Transformed refund object. + """ + + # Set the initial last updated time to the bookmark minus one minute + # to ensure we don't miss any updates as its observed shopify updates + # the parent object initially and then the child objects + last_updated_at = self.get_bookmark() - timedelta(minutes=1) + initial_bookmark_time = current_bookmark = self.get_bookmark() + sync_start = utils.now().replace(microsecond=0) + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) + + # Process each date window + while last_updated_at < sync_start: + date_window_end = last_updated_at + timedelta(days=self.date_window_size) + query_end = min(sync_start, date_window_end) + cursor = None - # Page through all `orders`, bookmarking at `refund_orders` - for parent_object in selected_parent.get_objects(): - since_id = 1 while True: - refunds = self.get_refunds(parent_object, since_id) - for refund in refunds: - if refund.id < since_id: - raise OutOfOrderIdsError("refund.id < since_id: {} < {}".format( - refund.id, since_id)) - yield refund - if len(refunds) < self.results_per_page: + query_params = self.get_query_params(last_updated_at, query_end, cursor) + + with metrics.http_request_timer(self.name): + data = self.call_api(query_params, query=query) + + # Process parent objects and their refunds + edges = data.get("edges", []) + for edge in edges: + node = edge.get("node", {}) + child_edges = node.get(self.child_data_key, []) + + # Yield each transformed refund + for child_obj in child_edges: + replication_value = utils.strptime_with_tz(child_obj[self.replication_key]) + current_bookmark = max(current_bookmark, replication_value) + # Perform the pseudo sync for the child objects + if replication_value >= initial_bookmark_time: + yield self.transform_object(child_obj) + + # Handle pagination + page_info = data.get("pageInfo", {}) + cursor = page_info.get("endCursor") + if not page_info.get("hasNextPage", False): break - if refunds[-1].id != max([o.id for o in refunds]): - raise OutOfOrderIdsError("{} is not the max id in refunds ({})".format( - refunds[-1].id, max([o.id for o in refunds]))) - since_id = refunds[-1].id - def sync(self): - bookmark = self.get_bookmark() - max_bookmark = bookmark - for refund in self.get_objects(): - refund_dict = refund.to_dict() - replication_value = strptime_to_utc(refund_dict[self.replication_key]) - if replication_value >= bookmark: - yield refund_dict + last_updated_at = query_end + # Update bookmark to the latest value, but not beyond sync start time + max_bookmark_value = min(sync_start, current_bookmark) + self.update_bookmark(utils.strftime(max_bookmark_value)) + + def transform_lineitems(self, data): + """ + Transforms the order lineitems data by extracting order IDs and handling pagination. + + Args: + data (dict): Order data. + + Returns: + list: List of refunds with lineitems. + """ + + lineitems = [ + node for item in data["refundLineItems"]["edges"] + if (node := item.get("node")) + ] + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) + + # Handle pagination + page_info = data["refundLineItems"].get("pageInfo", {}) + order_parent = data.get("order").get("id") + while page_info.get("hasNextPage"): + params = { + "first": self.results_per_page, + "query": f"id:{order_parent.split('/')[-1]}", + "childafter": page_info.get("endCursor"), + } + + # Fetch the next page of data + response = self.call_api(params, query=query) + nodes = response.get("edges", [])[0].get("node", {}) + lineitems_data = nodes.get("refunds")[0] + lineitems.extend( + node for item in lineitems_data["refundLineItems"]["edges"] + if (node := item.get("node")) + ) + page_info = lineitems_data.get("pageInfo", {}) + + return lineitems + + def transform_orderadjustments(self, data): + """ + Transforms the order adjustments data by extracting order IDs and handling pagination. + + Args: + data (dict): Order data. + + Returns: + list: List of adjustments. + """ + + orderadjustments = [ + node for item in data["orderAdjustments"]["edges"] + if (node := item.get("node")) + ] + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) + + # Handle pagination + page_info = data["orderAdjustments"].get("pageInfo", {}) + order_parent = data.get("order").get("id") + while page_info.get("hasNextPage"): + params = { + "first": self.results_per_page, + "query": f"id:{order_parent.split('/')[-1]}", + "orderadjustments_after": page_info.get("endCursor"), + } + + # Fetch the next page of data + response = self.call_api(params, query=query) + nodes = response.get("edges", [])[0].get("node", {}) + refunds_data = nodes.get("refunds")[0] + orderadjustments.extend( + node for item in refunds_data["orderAdjustments"]["edges"] + if (node := item.get("node")) + ) + page_info = refunds_data.get("pageInfo", {}) + + return orderadjustments + + def transform_object(self, obj): + """ + Transform refund objects by extracting refund line items from edges. + + Args: + obj (dict): Refund object. + + Returns: + dict: Transformed refund object. + """ + + if obj.get("refundLineItems"): + obj["refundLineItems"] = self.transform_lineitems(obj) + + if obj.get("orderAdjustments"): + obj["orderAdjustments"] = self.transform_orderadjustments(obj) + return obj + - if replication_value > max_bookmark: - max_bookmark = replication_value + def get_query(self): + """ + Returns the GraphQL query for fetching order refunds. - self.update_bookmark(strftime(max_bookmark)) + Returns: + str: GraphQL query string. + """ + # pylint: disable=line-too-long + return """query GetOrderRefunds($first: Int!, $after: String, $query: String, $childafter: String, $orderadjustments_after: String) { + orders(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { + edges { + node { + refunds(first: 250) { + id + createdAt + legacyResourceId + note + order { + id + } + orderAdjustments(first: 100, after: $orderadjustments_after) { + edges { + node { + amountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + id + reason + taxAmountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + refundLineItems(first: 50, after: $childafter) { + edges { + node { + id + quantity + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + restockType + restocked + location { + id + } + subtotalSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalTaxSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + lineItem { + id + vendor + quantity + title + requiresShipping + originalTotalSet { + presentmentMoney { + currencyCode + amount + } + shopMoney { + amount + currencyCode + } + } + taxLines(first: 250) { + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + rate + title + source + channelLiable + } + taxable + isGiftCard + name + discountedTotalSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + sku + product { + id + } + discountAllocations { + allocatedAmountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + discountApplication { + index + targetType + targetSelection + allocationMethod + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + } + customAttributes { + key + value + } + totalDiscountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + duties { + harmonizedSystemCode + id + taxLines { + rate + source + title + channelLiable + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + } + countryCodeOfOrigin + } + discountedUnitPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + updatedAt + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + }""" -Context.stream_objects['order_refunds'] = OrderRefunds +Context.stream_objects["order_refunds"] = OrderRefunds diff --git a/tap_shopify/streams/order_shipping_lines.py b/tap_shopify/streams/order_shipping_lines.py new file mode 100644 index 00000000..8b50b50b --- /dev/null +++ b/tap_shopify/streams/order_shipping_lines.py @@ -0,0 +1,310 @@ +from datetime import timedelta +from singer import metrics, utils +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream + + +class OrderShippingLines(Stream): + """Stream class for fetching order shippingLines from Shopify""" + + name = "order_shipping_lines" + data_key = "orders" + child_data_key = "shippingLines" + replication_key = "updatedAt" + + def _get_record_node_path(self): + # Shipping-line records live at + # orders.edges.node.shippingLines.edges.node { FIELDS }. + return ("orders", "edges", "node", "shippingLines", "edges", "node") + + # pylint: disable=too-many-locals + def get_objects(self): + """ + Fetch order shipping lines objects within date windows, yielding each shipping line. + + Yields: + dict: Transformed shipping line object. + """ + + # Set the initial last updated time to the bookmark minus one minute + # to ensure we don't miss any updates as its observed shopify updates + # the parent object initially and then the child objects + last_updated_at = self.get_bookmark() - timedelta(minutes=1) + current_bookmark = self.get_bookmark() + sync_start = utils.now().replace(microsecond=0) + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) + + # Process each date window + while last_updated_at < sync_start: + date_window_end = last_updated_at + timedelta(days=self.date_window_size) + query_end = min(sync_start, date_window_end) + cursor = None + + while True: + query_params = self.get_query_params(last_updated_at, query_end, cursor) + + with metrics.http_request_timer(self.name): + data = self.call_api(query_params, query=query) + + # Process parent objects and their shippinglines + edges = data.get("edges", []) + for edge in edges: + node = edge.get("node", {}) + + for shipping_line in self.paginate_shipping_lines(node, query): + shipping_line["orderId"] = node["id"].split("/")[-1] + shipping_line["updatedAt"] = node["updatedAt"] + + replication_value = utils.strptime_with_tz( + shipping_line[self.replication_key] + ) + current_bookmark = max(current_bookmark, replication_value) + yield self.transform_object(shipping_line) + + # Handle pagination + page_info = data.get("pageInfo", {}) + cursor = page_info.get("endCursor") + if not page_info.get("hasNextPage", False): + break + + last_updated_at = query_end + # Update bookmark to the latest value, but not beyond sync start time + max_bookmark_value = min(sync_start, current_bookmark) + self.update_bookmark(utils.strftime(max_bookmark_value)) + + def paginate_shipping_lines(self, data, query): + """ + Transforms the shippingLines data by handling pagination. + + Args: + data (dict): Order data. + query (str): Pruned GraphQL query string. + + Returns: + list: List of shippingLines. + """ + + for item in data["shippingLines"]["edges"]: + node = item.get("node") + if node: + yield node + + # Handle pagination + page_info = data.get("pageInfo", {}) + order_parent = data["id"] + while page_info.get("hasNextPage"): + params = { + "first": self.results_per_page, + "query": f"id:{order_parent.split('/')[-1]}", + "childafter": page_info.get("endCursor"), + } + + # Fetch the next page of data + response = self.call_api(params, query=query) + node = response.get("edges", [])[0].get("node", {}) + shipping_lines_data = node.get("shippingLines") + for item in shipping_lines_data["edges"]: + node = item.get("node") + if node: + yield node + + page_info = shipping_lines_data.get("pageInfo", {}) + + def get_query(self): + """ + Returns the GraphQL query for fetching order shipping lines. + + Returns: + str: GraphQL query string. + """ + return """ + query GetShippingLines($first: Int!, $after: String, $query: String, $childafter: String) { + orders(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { + edges { + node { + id + updatedAt + shippingLines(first: $first, after: $childafter) { + edges { + node { + carrierIdentifier + code + currentDiscountedPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + custom + deliveryCategory + discountAllocations { + allocatedAmountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + discountApplication { + allocationMethod + index + targetSelection + targetType + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + ... on AutomaticDiscountApplication { + __typename + allocationMethod + index + targetSelection + targetType + title + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + ... on DiscountCodeApplication { + __typename + allocationMethod + code + index + targetSelection + targetType + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + ... on ManualDiscountApplication { + description + allocationMethod + index + targetSelection + targetType + title + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + ... on ScriptDiscountApplication { + __typename + allocationMethod + index + targetSelection + targetType + title + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + } + } + discountedPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + id + isRemoved + originalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + phone + shippingRateHandle + source + taxLines { + channelLiable + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + rate + ratePercentage + source + title + } + title + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + + +Context.stream_objects["order_shipping_lines"] = OrderShippingLines diff --git a/tap_shopify/streams/orders.py b/tap_shopify/streams/orders.py index bb360204..afdf7a1e 100644 --- a/tap_shopify/streams/orders.py +++ b/tap_shopify/streams/orders.py @@ -1,10 +1,1374 @@ +from datetime import timedelta +import json +import time +import re +import urllib.error +import backoff +import requests import shopify - +import singer +from singer import metrics, utils from tap_shopify.context import Context from tap_shopify.streams.base import Stream +from tap_shopify.exceptions import ShopifyAPIError, BulkOperationInProgressError + +LOGGER = singer.get_logger() class Orders(Stream): - name = 'orders' - replication_object = shopify.Order + name = "orders" + data_key = "orders" + replication_key = "updatedAt" + + def get_query(self): + """ + Returns the GraphQL query string for the bulk operation. + The date filters will be injected via the bulk operation variables. + """ + return """ + { + orders(query: "%s") { + edges { + node { + additionalFees { + id + name + price { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + taxLines { + channelLiable + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + rate + ratePercentage + source + title + } + } + app { + id + name + icon { + id + } + } + billingAddress { + address1 + address2 + city + company + coordinatesValidated + country + countryCodeV2 + firstName + formattedArea + id + lastName + latitude + longitude + name + phone + province + provinceCode + timeZone + validationResultSummary + zip + } + billingAddressMatchesShippingAddress + canMarkAsPaid + canNotifyCustomer + cancelReason + cancellation { + staffNote + } + cancelledAt + capturable + cartDiscountAmountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + channelInformation { + id + channelId + } + clientIp + closed + closedAt + confirmationNumber + confirmed + createdAt + currencyCode + currentCartDiscountAmountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentShippingPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentSubtotalLineItemsQuantity + currentSubtotalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentTotalAdditionalFeesSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentTotalDiscountsSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentTotalDutiesSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentTotalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentTotalTaxSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentTotalWeight + customAttributes { + key + value + } + customer { + id + email + firstName + lastName + addresses { + address1 + address2 + city + countryCodeV2 + country + company + firstName + lastName + id + name + phone + province + provinceCode + zip + } + state + verifiedEmail + updatedAt + taxExempt + tags + taxExemptions + note + multipassIdentifier + createdAt + defaultAddress { + address1 + address2 + city + company + country + countryCodeV2 + firstName + id + lastName + name + province + phone + provinceCode + zip + } + } + customerJourneySummary { + lastVisit { + landingPage + referrerUrl + } + } + merchantOfRecordApp { + id + } + customerAcceptsMarketing + customerLocale + discountCodes + discountCode + displayFinancialStatus + displayFulfillmentStatus + disputes { + id + initiatedAs + status + } + dutiesIncluded + email + edited + estimatedTaxes + fulfillable + fullyPaid + hasTimelineComment + fulfillmentsCount { + count + precision + } + id + legacyResourceId + merchantBusinessEntity { + address { + address1 + address2 + city + countryCode + province + zip + } + companyName + displayName + id + primary + } + name + note + netPaymentSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + number + originalTotalAdditionalFeesSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + originalTotalDutiesSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + originalTotalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + paymentGatewayNames + phone + poNumber + presentmentCurrencyCode + processedAt + refundable + refundDiscrepancySet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + registeredSourceUrl + requiresShipping + restockable + returnStatus + shippingAddress { + address1 + address2 + city + company + coordinatesValidated + country + countryCodeV2 + firstName + formattedArea + id + lastName + latitude + longitude + name + phone + province + provinceCode + timeZone + validationResultSummary + zip + } + shopifyProtect { + eligibility { + status + } + status + } + sourceIdentifier + sourceName + statusPageUrl + subtotalLineItemsQuantity + subtotalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + tags + taxExempt + taxLines { + channelLiable + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + rate + ratePercentage + source + title + } + taxesIncluded + test + totalCapturableSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalCashRoundingAdjustment { + paymentSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + refundSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + } + totalDiscountsSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalOutstandingSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalReceivedSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalRefundedSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalRefundedShippingSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalShippingPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalTaxSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalTipReceivedSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalWeight + transactionsCount { + count + precision + } + unpaid + updatedAt + fulfillments { + id + name + status + totalQuantity + updatedAt + createdAt + deliveredAt + estimatedDeliveryAt + requiresShipping + inTransitAt + trackingInfo { + number + company + url + } + service { + serviceName + id + handle + trackingSupport + type + permitsSkuSharing + inventoryManagement + } + location { + id + } + } + lineItems { + edges { + node { + id + vendor + quantity + title + requiresShipping + originalTotalSet { + presentmentMoney { + currencyCode + amount + } + shopMoney { + amount + currencyCode + } + } + taxLines { + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + rate + title + source + channelLiable + } + taxable + isGiftCard + name + discountedTotalSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + sku + product { + id + } + discountAllocations { + allocatedAmountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + discountApplication { + index + targetType + targetSelection + allocationMethod + ... on AutomaticDiscountApplication { + title + } + ... on ManualDiscountApplication { + title + } + ... on ScriptDiscountApplication { + title + } + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + } + customAttributes { + key + value + } + totalDiscountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + duties { + harmonizedSystemCode + id + taxLines { + rate + source + title + channelLiable + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + } + countryCodeOfOrigin + } + discountedUnitPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + originalUnitPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + unfulfilledDiscountedTotalSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + unfulfilledOriginalTotalSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + variant { + id + } + lineItemGroup { + customAttributes { + key + value + } + id + quantity + title + variantId + variantSku + } + } + } + } + shippingLine { + carrierIdentifier + code + currentDiscountedPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + custom + deliveryCategory + discountAllocations { + allocatedAmountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + discountApplication { + allocationMethod + index + targetSelection + targetType + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + ... on AutomaticDiscountApplication { + __typename + allocationMethod + index + targetSelection + targetType + title + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + ... on DiscountCodeApplication { + __typename + allocationMethod + code + index + targetSelection + targetType + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + ... on ManualDiscountApplication { + description + allocationMethod + index + targetSelection + targetType + title + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + ... on ScriptDiscountApplication { + __typename + allocationMethod + index + targetSelection + targetType + title + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + } + } + discountedPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + id + isRemoved + originalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + phone + shippingRateHandle + source + taxLines { + channelLiable + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + rate + ratePercentage + source + title + } + title + } + retailLocation { + activatable + address { + address1 + address2 + city + country + countryCode + formatted + latitude + longitude + phone + province + provinceCode + zip + } + addressVerified + createdAt + deactivatable + deactivatedAt + deletable + fulfillmentService { + id + } + fulfillsOnlineOrders + hasActiveInventory + hasUnfulfilledOrders + id + isActive + isFulfillmentService + legacyResourceId + localPickupSettingsV2 { + instructions + pickupTime + } + name + shipsInventory + updatedAt + suggestedAddresses { + address1 + address2 + city + country + countryCode + formatted + province + provinceCode + zip + } + } + discountApplications { + edges { + node { + allocationMethod + index + targetSelection + targetType + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + ... on AutomaticDiscountApplication { + __typename + title + } + ... on DiscountCodeApplication { + __typename + code + } + ... on ManualDiscountApplication { + __typename + title + description + } + ... on ScriptDiscountApplication { + __typename + title + } + } + } + } + } + } + } + } + """ + + def is_discount_application(self, rec): + if '__typename' in rec and rec['__typename'] in ['AutomaticDiscountApplication', + 'DiscountCodeApplication', + 'ManualDiscountApplication', + 'ScriptDiscountApplication']: + return True + return False + + def update_bookmark(self, bookmark_value, bookmark_key=None, bulk_op_metadata=None): + # Standard Singer bookmark + singer.write_bookmark( + Context.state, + self.name, + bookmark_key or self.replication_key, + bookmark_value + ) + + # Store under orders -> bulk_operation + if bulk_op_metadata: + orders_bookmark = Context.state.setdefault("bookmarks", {}).setdefault("orders", {}) + orders_bookmark["bulk_operation"] = bulk_op_metadata + + singer.write_state(Context.state) + + def build_query_filter(self, updated_at_min, updated_at_max): + return f"updated_at:>='{updated_at_min}' AND updated_at:<'{updated_at_max}'" + + def submit_bulk_query(self, query_string): + url = f"https://{Context.config.get('shop')}.myshopify.com/admin/api/2025-07/graphql.json" + headers = { + "Content-Type": "application/json", + "X-Shopify-Access-Token": ( + Context.config.get("access_token") + or Context.config.get("api_key") + ), + } + operation = { + "query": """ + mutation bulkOperationRunQuery($query: String!) { + bulkOperationRunQuery(query: $query) { + bulkOperation { + id + status + createdAt + } + userErrors { + field + message + } + } + } + """, + "variables": { + "query": query_string + } + } + response = requests.post(url, headers=headers, json=operation, timeout=300) + LOGGER.info("X-request-ID for the bulk operation: %s", response.headers.get("X-Request-ID")) + + return response.json() + + def poll_bulk_completion(self, current_bookmark, bulk_op_id, timeout=82800): + def fetch_bulk_operation(op_id): + query = f""" + {{ + node(id: "{op_id}") {{ + ... on BulkOperation {{ + id + status + errorCode + createdAt + completedAt + objectCount + fileSize + url + }} + }} + }} + """ + # pylint: disable=unexpected-keyword-arg + # execute() is monkey-patched in base.py (execute_gql) to accept timeout + try: + response = json.loads(shopify.GraphQL().execute( + query=query, timeout=self.request_timeout)) + except urllib.error.HTTPError as http_error: + if http_error.code == 401: + LOGGER.warning("Received 401 Unauthorized during bulk operation polling.") + if Context.client: + Context.client.refresh_token() + Context.client.reinitialize_session() + Context.config['access_token'] = Context.client.access_token + else: + raise ShopifyAPIError( + "Received 401 Unauthorized during bulk operation polling " + "but no client is available to refresh the token." + ) from http_error + # Retry once with the refreshed token + response = json.loads(shopify.GraphQL().execute( + query=query, timeout=self.request_timeout)) + else: + raise + # pylint: enable=unexpected-keyword-arg + if not isinstance(response, dict): + raise ShopifyAPIError(f"Unexpected GraphQL response: {response}") + return response.get("data", {}).get("node") + + start = time.time() + last_status = None + + while time.time() - start < timeout: + op = fetch_bulk_operation(bulk_op_id) + + if not op: + LOGGER.warning("Bulk operation not found: %s", bulk_op_id) + return None + if not isinstance(op, dict): + raise ShopifyAPIError(f"Unexpected bulk operation format: {op}") + + current_status = op.get("status") + + if current_status != last_status: + LOGGER.info( + "Bulk operation - %s, status: %s, created at - %s, completed at - %s", + op.get("id"), + current_status, + op.get("createdAt"), + op.get("completedAt") or "N/A" + ) + last_status = current_status + + if current_status == "COMPLETED": + LOGGER.info("Bulk operation completed. File size: %s bytes", op.get("fileSize")) + self.update_bookmark( + bookmark_value=utils.strftime(current_bookmark), + bulk_op_metadata={ + "bulk_operation_id": op.get("id"), + "status": current_status, + "created_at": op.get("createdAt"), + "last_date_window": self.date_window_size, + } + ) + return op.get("url") + + if current_status in ["FAILED", "CANCELED"]: + self.clear_bulk_operation_state() + raise ShopifyAPIError(f"Bulk operation failed: {op.get('errorCode')}") + + time.sleep(60) + + # Save bookmark if timeout occurs + self.update_bookmark( + bookmark_value=utils.strftime(current_bookmark), + bulk_op_metadata={ + "bulk_operation_id": op.get("id"), + "status": op.get("status"), + "created_at": op.get("createdAt"), + "last_date_window": self.date_window_size, + } + ) + + elapsed = int(time.time() - start) + raise ShopifyAPIError( + f"Bulk operation id - {op.get('id') or 'UNKNOWN'} did not complete " + f"within {elapsed} seconds. " + "Please contact Shopify support with the operation ID for assistance." + ) + + # pylint: disable=unsupported-assignment-operation + def parse_bulk_jsonl(self, url): + """ + Streams and yields one order at a time, with its associated line items, + without holding all orders/line_items in memory. + """ + resp = requests.get(url, stream=True, timeout=60) + current_order = None + current_line_items = [] + current_discount_applications = [] + + for line in resp.iter_lines(): + if not line: + continue + rec = json.loads(line) + if not isinstance(rec, dict): + LOGGER.warning("Skipping unexpected JSONL line (not a dict): %s", rec) + continue + # Detect line item (child) or order (parent) + if '__parentId' in rec: + if self.is_discount_application(rec): + # It's a discount application belonging to current_order + current_discount_applications.append(rec) + else: + # It's a line item belonging to current_order + current_line_items.append(rec) + else: + if current_order: + current_order["lineItems"] = current_line_items + current_order["discountApplications"] = current_discount_applications + yield current_order + # Start tracking new parent group + current_order = rec + current_line_items = [] + current_discount_applications = [] + # Yield the last parent group (if exists) + if current_order: + current_order["lineItems"] = current_line_items + current_order["discountApplications"] = current_discount_applications + yield current_order + + def transform_object(self, obj): + if obj.get("lineItems", {}).get("edges"): + obj["lineItems"] = [item["node"] for item in obj["lineItems"]["edges"]] + return obj + + def clear_bulk_operation_state(self): + orders_bookmark = Context.state.get("bookmarks", {}).get("orders", {}) + if "bulk_operation" in orders_bookmark: + del orders_bookmark["bulk_operation"] + singer.write_state(Context.state) + + # pylint: disable=too-many-locals,too-many-statements + def get_objects(self): + last_updated_at = self.get_bookmark() + current_bookmark = last_updated_at + sync_start = utils.now().replace(microsecond=0) + query_template = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) + LOGGER.info( + "GraphQL query for stream '%s': %s", + self.name, + ' '.join(query_template.split()) + ) + + bulk_op = Context.state.get("bookmarks", {}).get("orders", {}).get("bulk_operation") + op_id = None + existing_url = None + + if bulk_op: + if bulk_op.get("last_date_window") != self.date_window_size: + LOGGER.info( + "Clearing existing bulk operation state due to date " + "window size change from %s to %s", + bulk_op.get("last_date_window"), + self.date_window_size + ) + self.clear_bulk_operation_state() + + else: + op_id = bulk_op.get("bulk_operation_id") + op_status = bulk_op.get("status") + + if op_status in ["RUNNING", "COMPLETED"]: + LOGGER.info("Resuming polling for existing bulk operation ID: %s", op_id) + existing_url = self.poll_bulk_completion(current_bookmark, op_id) + else: + self.clear_bulk_operation_state() + + while last_updated_at < sync_start: + date_window_end = last_updated_at + timedelta(days=self.date_window_size) + query_end = min(sync_start, date_window_end) + + if not existing_url: + existing_url = self.submit_and_poll_bulk_query( + query_template, + last_updated_at, + query_end, + current_bookmark + ) + if existing_url: + for obj in self.parse_bulk_jsonl(existing_url): + replication_value = utils.strptime_to_utc(obj[self.replication_key]) + current_bookmark = max(current_bookmark, replication_value) + + yield obj + else: + LOGGER.info("No data returned for the date range: %s to %s", + last_updated_at, query_end) + + self.clear_bulk_operation_state() + last_updated_at = query_end + max_bookmark_value = min(sync_start, current_bookmark) + self.update_bookmark(utils.strftime(max_bookmark_value)) + existing_url = None + + @backoff.on_exception( + backoff.expo, + BulkOperationInProgressError, + max_tries=7, + factor=10, + jitter=None, + on_backoff=lambda details: LOGGER.warning( + "Bulk operation already in progress (ID: %s). " + "Retry attempt %d after %.2f seconds. Total elapsed: %.2f seconds.", + getattr(details['exception'], 'bulk_op_id', 'UNKNOWN'), + details['tries'], + details['wait'], + details['elapsed'] + ) + ) + def submit_and_poll_bulk_query( + self, query_template, last_updated_at, query_end, current_bookmark + ): + """Submit bulk query and poll for completion with automatic retry on conflicts""" + with metrics.http_request_timer(self.name): + query_filter = self.build_query_filter( + utils.strftime(last_updated_at), + utils.strftime(query_end) + ) + query = query_template % query_filter + LOGGER.info("Fetching records in date range: %s", query_filter) + + bulk_op_data = self.submit_bulk_query(query) + + user_errors = ( + bulk_op_data.get("data", {}) + .get("bulkOperationRunQuery", {}) + .get("userErrors") + ) + + if user_errors: + for error in user_errors: + message = error.get("message", "") + if ( + "bulk query operation for this app and shop is already in progress" + in message + ): + # Extract BulkOperation ID using regex + match = re.search(r"gid://shopify/BulkOperation/\d+", message) + bulk_op_id = match.group(0) if match else None + + LOGGER.info("Detected concurrent bulk operation (ID: %s)", bulk_op_id) + raise BulkOperationInProgressError( + f"Bulk operation already in progress: {bulk_op_id}", + bulk_op_id=bulk_op_id + ) + + # Handle other user errors + raise ShopifyAPIError("Bulk query error: {}".format(user_errors)) + + bulk_operation = ( + bulk_op_data.get("data", {}) + .get("bulkOperationRunQuery", {}) + .get("bulkOperation") + ) + bulk_op_id = bulk_operation.get("id") if bulk_operation else None + if not bulk_op_id: + raise ShopifyAPIError("Invalid bulk operation response: {}".format(bulk_op_data)) + + return self.poll_bulk_completion(current_bookmark, bulk_op_id) + -Context.stream_objects['orders'] = Orders +Context.stream_objects["orders"] = Orders diff --git a/tap_shopify/streams/product_variants.py b/tap_shopify/streams/product_variants.py new file mode 100644 index 00000000..4f18cbe8 --- /dev/null +++ b/tap_shopify/streams/product_variants.py @@ -0,0 +1,82 @@ +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream + + +class ProductVariants(Stream): + """Stream class for Product Variants in Shopify.""" + name = "product_variants" + data_key = "productVariants" + replication_key = "updatedAt" + + def transform_object(self, obj): + """ + Transforms the object if needed. + + Args: + obj (dict): The object to transform. + + Returns: + dict: The transformed object. + """ + return obj + + def get_query(self): + """ + Returns the GraphQL query to get all product variants. + + Returns: + str: The GraphQL query string. + """ + return """ + query GetProductVariants($first: Int!, $after: String, $query: String) { + productVariants(first: $first, after: $after, query: $query) { + edges { + node { + id + createdAt + barcode + availableForSale + compareAtPrice + displayName + image { + altText + height + id + url + width + } + inventoryPolicy + inventoryQuantity + position + price + requiresComponents + sellableOnlineQuantity + sku + taxCode + taxable + title + updatedAt + product { + id + } + inventoryItem { + id + measurement { + weight { + unit + value + } + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + """ + + +Context.stream_objects["product_variants"] = ProductVariants diff --git a/tap_shopify/streams/products.py b/tap_shopify/streams/products.py index fbfcb6a0..e8c564c2 100644 --- a/tap_shopify/streams/products.py +++ b/tap_shopify/streams/products.py @@ -1,12 +1,157 @@ -import shopify - -from tap_shopify.streams.base import Stream from tap_shopify.context import Context +from tap_shopify.streams.base import Stream class Products(Stream): - name = 'products' - replication_object = shopify.Product - status_key = "published_status" + """Stream class for Shopify Products""" + + name = "products" + data_key = "products" + replication_key = "updatedAt" + + def transform_object(self, obj): + """ + Transforms the product object by extracting media information. + + Args: + obj (dict): Product object. + + Returns: + dict: Transformed product object. + """ + media = obj.get("media") + media_list = [] + if media and "edges" in media: + for edge in media.get("edges"): + node = edge.get("node") + if node: + media_list.append(node) + obj["media"] = media_list + return obj + + def get_query(self): + """ + Returns the GraphQL query to fetch all products. + + Returns: + str: GraphQL query string. + """ + return """ + query GetProducts($first: Int!, $after: String, $query: String) { + products(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { + edges { + node { + id + title + descriptionHtml + vendor + category { + id + } + tags + handle + publishedAt + createdAt + updatedAt + templateSuffix + status + productType + options { + id + name + position + values + } + giftCardTemplateSuffix + hasOnlyDefaultVariant + hasOutOfStockVariants + hasVariantsThatRequiresComponents + isGiftCard + description + compareAtPriceRange { + maxVariantCompareAtPrice { + amount + currencyCode + } + minVariantCompareAtPrice { + amount + currencyCode + } + } + featuredMedia { + id + mediaContentType + status + } + requiresSellingPlan + totalInventory + tracksInventory + media(first: 250) { + edges { + node { + id + alt + status + mediaContentType + mediaWarnings { + code + message + } + mediaErrors { + code + details + message + } + ... on ExternalVideo { + id + embedUrl + } + ... on MediaImage { + id + updatedAt + createdAt + mimeType + image { + url + width + height + id + } + } + ... on Model3d { + id + filename + sources { + url + format + mimeType + filesize + } + } + ... on Video { + id + updatedAt + createdAt + filename + sources { + url + format + mimeType + fileSize + } + } + } + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + """ + -Context.stream_objects['products'] = Products +Context.stream_objects["products"] = Products diff --git a/tap_shopify/streams/transactions.py b/tap_shopify/streams/transactions.py index fc9b2a55..ef59593f 100644 --- a/tap_shopify/streams/transactions.py +++ b/tap_shopify/streams/transactions.py @@ -1,125 +1,249 @@ -import shopify -import singer -from singer.utils import strftime, strptime_to_utc +from datetime import timedelta +from singer import metrics, utils from tap_shopify.context import Context -from tap_shopify.streams.base import (Stream, - shopify_error_handling) - -LOGGER = singer.get_logger() - -# https://help.shopify.com/en/api/reference/orders/transaction An -# order can have no more than 100 transactions associated with it. -TRANSACTIONS_RESULTS_PER_PAGE = 100 - -# We have observed transactions with receipt objects that contain both: -# - `token` and `Token` -# - `version` and `Version` -# - `ack` and `Ack` -# keys transactions where PayPal is the payment type. We reached out to -# PayPal support and they told us the values should be the same, so one -# can be safely ignored since its a duplicate. Example: The logic is to -# prefer `token` if both are present and equal, convert `Token` -> `token` -# if only `Token` is present, and throw an error if both are present and -# their values are not equal -def canonicalize(transaction_dict, field_name): - field_name_upper = field_name.capitalize() - # Not all Shopify transactions have receipts. Facebook has been shown - # to push a null receipt through the transaction - receipt = transaction_dict.get('receipt', {}) - if receipt: - value_lower = receipt.get(field_name) - value_upper = receipt.get(field_name_upper) - if value_lower and value_upper: - if value_lower == value_upper: - LOGGER.info(( - "Transaction (id=%d) contains a receipt " - "that has `%s` and `%s` keys with the same " - "value. Removing the `%s` key."), - transaction_dict['id'], - field_name, - field_name_upper, - field_name_upper) - transaction_dict['receipt'].pop(field_name_upper) - else: - raise ValueError(( - "Found Transaction (id={}) with a receipt that has " - "`{}` and `{}` keys with the different " - "values. Contact Shopify/PayPal support.").format( - transaction_dict['id'], - field_name_upper, - field_name)) - elif value_upper: - # pylint: disable=line-too-long - transaction_dict["receipt"][field_name] = transaction_dict['receipt'].pop(field_name_upper) +from tap_shopify.streams.base import Stream class Transactions(Stream): - name = 'transactions' - replication_key = 'created_at' - replication_object = shopify.Transaction - # Added decorator over functions of shopify SDK - replication_object.find = shopify_error_handling(replication_object.find) - # Transactions have no updated_at property. Therefore we have - # nothing to set the `replication_method` member to. - # https://help.shopify.com/en/api/reference/orders/transaction#properties - - def call_api_for_transactions(self, parent_object): - # set timeout - self.replication_object.set_timeout(self.request_timeout) - return self.replication_object.find( - limit=TRANSACTIONS_RESULTS_PER_PAGE, - order_id=parent_object.id, + """Stream class for Shopify transactions.""" + + name = "transactions" + data_key = "orders" + child_data_key = "transactions" + replication_key = "createdAt" + + def _get_record_node_path(self): + # Transaction records live at orders.edges.node.transactions { FIELDS }, + # not at the standard edges.node depth. + return ("orders", "edges", "node", "transactions") + + # pylint: disable=W0221 + def get_query_params(self, updated_at_min, updated_at_max, cursor=None): + """ + Construct query parameters for GraphQL requests. + + Args: + updated_at_min (str): Minimum updated_at timestamp. + updated_at_max (str): Maximum updated_at timestamp. + cursor (str): Pagination cursor, if any. + + Returns: + dict: Dictionary of query parameters. + """ + parent_filter_key = "updated_at" + query = ( + f"{parent_filter_key}:>='{updated_at_min}' " + f"AND {parent_filter_key}:<'{updated_at_max}'" ) + params = { + "query": query, + "first": self.results_per_page, + } - def get_transactions(self, parent_object): - # We do not need to support paging on this substream. If that - # were to become untrue, reference Metafields. - # - # We do not user the `transactions` method of the order object - # like in metafield because they overrode it here to not - # support limit overrides. - # - # https://github.com/Shopify/shopify_python_api/blob/e8c475ccc84b1516912b37f691d00ecd24921e9b/shopify/resources/order.py#L17-L18 - - page = self.call_api_for_transactions(parent_object) - yield from page - - while page.has_next_page(): - page = page.next_page() - yield from page + if cursor: + params["after"] = cursor + return params + # pylint: disable=too-many-locals def get_objects(self): - # Right now, it's ok for the user to select 'transactions' but not - # 'orders'. This data may not be all that useful but we're taking - # the less opinionated approach to begin with to favor simplicity. - # This is where you would need to add the behavior for enforcing - # that 'orders' is selected if we want to go that route in the - # future. - - # Get transactions, bookmarking at `transaction_orders` - selected_parent = Context.stream_objects['orders']() - selected_parent.name = "transaction_orders" - - # Page through all `orders`, bookmarking at `transaction_orders` - for parent_object in selected_parent.get_objects(): - transactions = self.get_transactions(parent_object) - for transaction in transactions: - yield transaction - - def sync(self): - bookmark = self.get_bookmark() - max_bookmark = bookmark - for transaction in self.get_objects(): - transaction_dict = transaction.to_dict() - replication_value = strptime_to_utc(transaction_dict[self.replication_key]) - if replication_value >= bookmark: - for field_name in ['token', 'version', 'ack', 'timestamp', 'build']: - canonicalize(transaction_dict, field_name) - yield transaction_dict - - if replication_value > max_bookmark: - max_bookmark = replication_value - - self.update_bookmark(strftime(max_bookmark)) - -Context.stream_objects['transactions'] = Transactions + """ + Fetch transaction objects within date windows, yielding each transaction individually. + + Yields: + dict: Transformed transaction object. + """ + # Set the initial last updated time to the bookmark minus one minute + # to ensure we don't miss any updates as its observed shopify updates + # the parent object initially and then the child objects + last_updated_at = self.get_bookmark() - timedelta(minutes=1) + initial_bookmark_time = current_bookmark = self.get_bookmark() + sync_start = utils.now().replace(microsecond=0) + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) + + while last_updated_at < sync_start: + date_window_end = last_updated_at + timedelta(days=self.date_window_size) + query_end = min(sync_start, date_window_end) + cursor = None + + while True: + query_params = self.get_query_params(last_updated_at, query_end, cursor) + + with metrics.http_request_timer(self.name): + data = self.call_api(query_params, query=query) + + edges = data.get("edges", []) + for edge in edges: + node = edge.get("node", {}) + child_edges = node.get(self.child_data_key, []) + + # Yield each transformed transaction object + for child_obj in child_edges: + replication_value = utils.strptime_with_tz(child_obj[self.replication_key]) + current_bookmark = max(current_bookmark, replication_value) + # Perform the pseudo sync for the child objects + if replication_value >= initial_bookmark_time: + yield self.transform_object(child_obj) + + page_info = data.get("pageInfo", {}) + cursor = page_info.get("endCursor") + if not page_info.get("hasNextPage", False): + break + + last_updated_at = query_end + # Update bookmark to the latest value, but not beyond sync start time + max_bookmark_value = min(sync_start, current_bookmark) + self.update_bookmark(utils.strftime(max_bookmark_value)) + + def get_query(self): + """ + Returns query for fetching transactions. + + Note: + Shopify has a limit of 100 transactions per order as per shopify support. + To be on the safer side, we are limiting the transactions to 250. + + Returns: + str: GraphQL query string. + """ + return """ + query GetTransactions($first: Int!, $after: String, $query: String) { + orders(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { + edges { + node { + transactions(first: 250) { + accountNumber + amountRoundingSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + amountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + authorizationCode + authorizationExpiresAt + createdAt + errorCode + formattedGateway + gateway + id + kind + manualPaymentGateway + maximumRefundableV2 { + amount + currencyCode + } + multiCapturable + order { + id + } + parentTransaction { + accountNumber + createdAt + id + status + paymentId + processedAt + amountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + } + paymentId + processedAt + receiptJson + settlementCurrency + settlementCurrencyRate + shopifyPaymentsSet { + extendedAuthorizationSet { + extendedAuthorizationExpiresAt + standardAuthorizationExpiresAt + } + refundSet { + acquirerReferenceNumber + } + } + status + test + totalUnsettledSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + fees { + id + rate + rateName + taxAmount { + amount + currencyCode + } + type + flatFeeName + amount { + amount + currencyCode + } + flatFee { + amount + currencyCode + } + } + manuallyCapturable + paymentDetails { + ... on CardPaymentDetails { + avsResultCode + bin + company + cvvResultCode + expirationMonth + expirationYear + name + number + paymentMethodName + wallet + } + ... on LocalPaymentMethodsPaymentDetails { + paymentDescriptor + paymentMethodName + } + ... on ShopPayInstallmentsPaymentDetails { + paymentMethodName + } + } + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + + +Context.stream_objects["transactions"] = Transactions diff --git a/tests/base.py b/tests/base.py index 2d45ba7b..c7e6d800 100644 --- a/tests/base.py +++ b/tests/base.py @@ -2,17 +2,17 @@ Setup expectations for test sub classes Run discovery for as a prerequisite for most tests """ -import unittest -import os -from datetime import datetime as dt -from datetime import timezone as tz import dateutil.parser +import os import pytz +from datetime import datetime as dt from datetime import timedelta -from tap_tester import connections, menagerie, runner +from datetime import timezone as tz +from tap_tester import connections, menagerie, runner +from tap_tester.base_case import BaseCase -class BaseTapTest(unittest.TestCase): +class BaseTapTest(BaseCase): """ Setup expectations for test sub classes Run discovery for as a prerequisite for most tests @@ -27,7 +27,7 @@ class BaseTapTest(unittest.TestCase): FULL = "FULL_TABLE" START_DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" BOOKMARK_COMPARISON_FORMAT = "%Y-%m-%dT00:00:00+00:00" - DEFAULT_RESULTS_PER_PAGE = 175 + DEFAULT_RESULTS_PER_PAGE = 250 @staticmethod def tap_name(): @@ -37,16 +37,16 @@ def tap_name(): @staticmethod def get_type(): """the expected url route ending""" - return "platform.shopify" + # return "platform.shopify-tba" # moved to alpha Oct 13, 2023 + # return "platform.shopify" # new connections after Oct 13th + return "platform.shopify-byoa" def get_properties(self, original: bool = True): """Configuration properties required for the tap.""" return_value = { 'start_date': '2017-07-01T00:00:00Z', 'shop': 'stitchdatawearhouse', - 'date_window_size': 30, - # BUG: https://jira.talendforge.org/browse/TDL-13180 - # 'results_per_page': '50' + 'date_window_size': 180 } if original: @@ -60,70 +60,73 @@ def get_properties(self, original: bool = True): return return_value @staticmethod - def get_credentials(original_credentials: bool = True): + def get_credentials(original_credentials=True): """Authentication information for the test account""" - if original_credentials: - return { - 'api_key': os.getenv('TAP_SHOPIFY_API_KEY_STITCHDATAWEARHOUSE') - } - return { - 'api_key': os.getenv('TAP_SHOPIFY_API_KEY_TALENDDATAWEARHOUSE') + 'client_id': os.getenv('TAP_SHOPIFY_CLIENT_ID'), + 'client_secret': os.getenv('TAP_SHOPIFY_CLIENT_SECRET') } def expected_metadata(self): """The expected streams and metadata about the streams""" default = { - self.REPLICATION_KEYS: {"updated_at"}, + self.REPLICATION_KEYS: {"updatedAt"}, self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.API_LIMIT: self.DEFAULT_RESULTS_PER_PAGE} meta = default.copy() - meta.update({self.FOREIGN_KEYS: {"owner_id", "owner_resource"}}) + meta[self.REPLICATION_KEYS] = {"updatedAt"} + meta[self.API_LIMIT] = 30 + meta.update({self.FOREIGN_KEYS: {"owner", "ownerType"}}) return { "abandoned_checkouts": { - self.REPLICATION_KEYS: {"updated_at"}, + self.REPLICATION_KEYS: {"updatedAt"}, self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.INCREMENTAL, - # BUG: https://jira.talendforge.org/browse/TDL-13180 + # BUG: https://qlik-dev.atlassian.net/browse/TDL-13180 self.API_LIMIT: 50}, - "collects": default, - "custom_collections": default, + "collections": default, "customers": default, "orders": default, - "order_refunds": { - self.REPLICATION_KEYS: {"created_at"}, + "fulfillment_orders": { + self.REPLICATION_KEYS: {"updatedAt"}, self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.INCREMENTAL, - self.API_LIMIT: self.DEFAULT_RESULTS_PER_PAGE}, - "products": default, - "inventory_items": {self.REPLICATION_KEYS: {"updated_at"}, + self.API_LIMIT: 30 + }, + "order_shipping_lines": { + self.REPLICATION_KEYS: {"updatedAt"}, self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.INCREMENTAL, - self.API_LIMIT: 250}, - "metafields": meta, + self.API_LIMIT: 15 + }, + "order_refunds": default, + "products": default, + "product_variants": default, + "inventory_items": default, + "metafields_collections": meta, + "metafields_customers": meta, + "metafields_orders": meta, + "metafields_products": meta, "transactions": { - self.REPLICATION_KEYS: {"created_at"}, + self.REPLICATION_KEYS: {"createdAt"}, self.PRIMARY_KEYS: {"id"}, - self.FOREIGN_KEYS: {"order_id"}, + self.FOREIGN_KEYS: {"order"}, self.REPLICATION_METHOD: self.INCREMENTAL, - self.API_LIMIT: self.DEFAULT_RESULTS_PER_PAGE}, + self.API_LIMIT: 150}, "locations": { - self.REPLICATION_KEYS: {"updated_at"}, + self.REPLICATION_KEYS: {"createdAt"}, self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.INCREMENTAL, - self.API_LIMIT: 0}, - "inventory_levels": { - self.REPLICATION_KEYS: {"updated_at"}, - self.PRIMARY_KEYS: {"location_id", "inventory_item_id"}, - self.REPLICATION_METHOD: self.INCREMENTAL, - self.API_LIMIT: self.DEFAULT_RESULTS_PER_PAGE}, + self.API_LIMIT: 2 + }, + "inventory_levels": default, "events": { - self.REPLICATION_KEYS: {"created_at"}, + self.REPLICATION_KEYS: {"createdAt"}, self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.API_LIMIT: 50 @@ -132,7 +135,12 @@ def expected_metadata(self): def expected_streams(self): """A set of expected stream names""" - return set(self.expected_metadata().keys()) + # removed "abandoned_checkouts", as per the Doc: + # https://help.shopify.com/en/manual/orders/abandoned-checkouts?st_source=admin&st_campaign=abandoned_checkouts_footer&utm_source=admin&utm_campaign=abandoned_checkouts_footer#review-your-abandoned-checkouts + # abandoned checkouts are saved in the Shopify admin for three months. + # Every Monday, abandoned checkouts that are older than three months are removed from your admin. + # Also no POST call is available for this endpoint: https://shopify.dev/api/admin-rest/2022-01/resources/abandoned-checkouts + return set(self.expected_metadata().keys()) - {"abandoned_checkouts"} def child_streams(self): """ @@ -300,8 +308,13 @@ def select_all_streams_and_fields(conn_id, catalogs, select_all_fields: bool = T def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.start_date = self.get_properties().get("start_date") - self.store_1_streams = {'custom_collections', 'orders', 'products', 'customers', 'locations', 'inventory_levels', 'inventory_items', 'events'} - self.store_2_streams = {'abandoned_checkouts', 'collects', 'metafields', 'transactions', 'order_refunds', 'products', 'locations', 'inventory_levels', 'inventory_items', 'events'} + self.store_1_streams = {'collections', 'orders', 'products', 'customers', 'locations', 'inventory_levels', 'inventory_items', 'fulfillment_orders'} + # removed 'abandoned_checkouts' from store 2 streams, as per the Doc: + # https://help.shopify.com/en/manual/orders/abandoned-checkouts?st_source=admin&st_campaign=abandoned_checkouts_footer&utm_source=admin&utm_campaign=abandoned_checkouts_footer#review-your-abandoned-checkouts + # abandoned checkouts are saved in the Shopify admin for three months. + # Every Monday, abandoned checkouts that are older than three months are removed from your admin. + # Also no POST call is available for this endpoint: https://shopify.dev/api/admin-rest/2022-01/resources/abandoned-checkouts + self.store_2_streams = {'metafields_products', 'transactions', 'order_refunds', 'products', 'locations', 'inventory_levels', 'inventory_items', 'order_shipping_lines'} #modified this method to accommodate replication key in the current_state def calculated_states_by_stream(self, current_state): @@ -353,3 +366,37 @@ def timedelta_formatted(self, dtime, days=0): except ValueError: return Exception("Datetime object is not of the format: {}".format(self.START_DATE_FORMAT)) + + @staticmethod + def parse_date(date_value): + """ + Pass in string-formatted-datetime, parse the value + return it as an un-formatted datetime object. + """ + date_formats = { + "%Y-%m-%dT%H:%M:%S.%fZ", + "%Y-%m-%dT%H:%M:%S%z", + "%Y-%m-%dT%H:%M:%SZ", + "%Y-%m-%dT%H:%M:%S.%f+00:00", + "%Y-%m-%dT%H:%M:%S+00:00", + "%Y-%m-%d" + } + for date_format in date_formats: + try: + date_stripped = dt.strptime(date_value, date_format) + return date_stripped + except ValueError: + pass + + raise NotImplementedError(f"Tests do not account for dates of this format: {date_value}") + + # function for verifying the date format + def is_expected_date_format(self, date): + try: + # parse date + dt.strptime(date, "%Y-%m-%dT%H:%M:%S.%fZ") + except ValueError: + # return False if date is in not expected format + return False + # return True in case of no error + return True diff --git a/tests/sync_rows.py b/tests/sync_rows.py index d7d7675d..ad497ffc 100644 --- a/tests/sync_rows.py +++ b/tests/sync_rows.py @@ -11,8 +11,6 @@ from functools import reduce from singer import metadata -LOGGER = singer.get_logger() - # The token used to authenticate our requests was generated on # [2018-09-18](https://github.com/stitchdata/environments/commit/82609cef972fd631c628b8eb733f37eea8f5d4f4). # If it ever expires, you'll need to login to Shopify via the 1Password diff --git a/tests/test_all_fields.py b/tests/test_all_fields.py new file mode 100644 index 00000000..2c6d1de3 --- /dev/null +++ b/tests/test_all_fields.py @@ -0,0 +1,100 @@ +import os + +from tap_tester import runner, menagerie +from base import BaseTapTest + + +KNOWN_MISSING_FIELDS = { + 'events': { + 'attachments', + 'edited', + 'author', + 'canDelete', + 'rawMessage', + 'canEdit', + } +} + +class AllFieldsTest(BaseTapTest): + + @staticmethod + def name(): + return "tap_tester_shopify_all_fields_test" + + def get_properties(self, original: bool = True): + """Configuration properties required for the tap.""" + return_value = { + 'start_date': '2025-01-01T00:00:00Z', + 'shop': 'talenddatawearhouse', + 'date_window_size': 30, + 'results_per_page': '30' + } + + return return_value + + def test_run(self): + """ + Ensure running the tap with all streams and fields selected results in the + replication of all fields. + - Verify no unexpected streams were replicated + - Verify that more than just the automatic fields are replicated for each stream + """ + + expected_streams = self.expected_streams() + + # instantiate connection + conn_id = self.create_connection() + + # run check mode + found_catalogs = menagerie.get_catalogs(conn_id) + + # table and field selection + test_catalogs_all_fields = [catalog for catalog in found_catalogs + if catalog.get('stream_name') in expected_streams] + self.select_all_streams_and_fields(conn_id, test_catalogs_all_fields, select_all_fields=True) + + # grab metadata after performing table-and-field selection to set expectations + stream_to_all_catalog_fields = dict() # used for asserting all fields are replicated + for catalog in test_catalogs_all_fields: + stream_id, stream_name = catalog['stream_id'], catalog['stream_name'] + catalog_entry = menagerie.get_annotated_schema(conn_id, stream_id) + fields_from_field_level_md = [md_entry['breadcrumb'][1] + for md_entry in catalog_entry['metadata'] + if md_entry['breadcrumb'] != []] + stream_to_all_catalog_fields[stream_name] = set(fields_from_field_level_md) + + # run initial sync + record_count_by_stream = self.run_sync(conn_id) + synced_records = runner.get_records_from_target_output() + + # Verify no unexpected streams were replicated + synced_stream_names = set(synced_records.keys()) + self.assertSetEqual(expected_streams, synced_stream_names) + + for stream in expected_streams: + with self.subTest(stream=stream): + + # expected values + expected_automatic_keys = self.expected_primary_keys().get(stream, set()) | self.expected_replication_keys().get(stream, set()) + # get all expected keys + expected_all_keys = stream_to_all_catalog_fields[stream] + + # collect actual values + messages = synced_records.get(stream) + + actual_all_keys = set() + # collect actual values + for message in messages['messages']: + if message['action'] == 'upsert': + actual_all_keys.update(message['data'].keys()) + + # Verify that you get some records for each stream + self.assertGreater(record_count_by_stream.get(stream, -1), 0) + + # verify all fields for a stream were replicated + self.assertGreater(len(expected_all_keys), len(expected_automatic_keys)) + self.assertTrue(expected_automatic_keys.issubset(expected_all_keys), + msg=f'{expected_automatic_keys-expected_all_keys} is not in "expected_all_keys"') + + expected_all_keys = expected_all_keys - KNOWN_MISSING_FIELDS.get(stream, set()) + self.assertSetEqual(expected_all_keys, actual_all_keys) diff --git a/tests/test_automatic_fields.py b/tests/test_automatic_fields.py index 76158e29..ad8756cb 100644 --- a/tests/test_automatic_fields.py +++ b/tests/test_automatic_fields.py @@ -21,7 +21,7 @@ def __init__(self, *args, **kwargs): def test_run(self): with self.subTest(store="store_1"): conn_id = self.create_connection(original_credentials=True) - self.automatic_test(conn_id, self.store_1_streams) + self.automatic_test(conn_id, self.store_1_streams - {"orders"}) with self.subTest(store="store_2"): conn_id = self.create_connection(original_properties=False, original_credentials=False) @@ -31,6 +31,7 @@ def automatic_test(self, conn_id, testable_streams): """ Verify that for each stream you can get multiple pages of data when no fields are selected and only the automatic fields are replicated. + Verify that all replicated records have unique primary key values. PREREQUISITE For EACH stream add enough data that you surpass the limit of a single @@ -52,6 +53,7 @@ def automatic_test(self, conn_id, testable_streams): record_count_by_stream = self.run_sync(conn_id) actual_fields_by_stream = runner.examine_target_output_for_fields() + synced_records = runner.get_records_from_target_output() for stream in incremental_streams: with self.subTest(stream=stream): @@ -60,6 +62,13 @@ def automatic_test(self, conn_id, testable_streams): # SKIP THIS ASSERTION FOR STREAMS WHERE YOU CANNOT GET # MORE THAN 1 PAGE OF DATA IN THE TEST ACCOUNT stream_metadata = self.expected_metadata().get(stream, {}) + expected_primary_keys = self.expected_primary_keys().get(stream, set()) + + extra_automatic_keys = {"order"} if stream == "order_refunds" else set() + + # collect records + messages = synced_records.get(stream) + minimum_record_count = stream_metadata.get( self.API_LIMIT, self.get_properties().get('result_per_page', self.DEFAULT_RESULTS_PER_PAGE) @@ -72,7 +81,15 @@ def automatic_test(self, conn_id, testable_streams): # verify that only the automatic fields are sent to the target self.assertEqual( actual_fields_by_stream.get(stream, set()), - self.expected_primary_keys().get(stream, set()) | + expected_primary_keys | extra_automatic_keys | self.expected_replication_keys().get(stream, set()), msg="The fields sent to the target are not the automatic fields" ) + + # Verify that all replicated records have unique primary key values. + records_pks_set = {tuple([message.get('data').get(primary_key) for primary_key in expected_primary_keys]) + for message in messages.get('messages')} + records_pks_list = [tuple([message.get('data').get(primary_key) for primary_key in expected_primary_keys]) + for message in messages.get('messages')] + self.assertCountEqual(records_pks_set, records_pks_list, + msg="We have duplicate records for {}".format(stream)) diff --git a/tests/test_bookmarks.py b/tests/test_bookmarks.py index a6a1f135..a28c3dd7 100644 --- a/tests/test_bookmarks.py +++ b/tests/test_bookmarks.py @@ -5,7 +5,7 @@ from dateutil.parser import parse -from tap_tester import menagerie, runner +from tap_tester import menagerie, runner, LOGGER from base import BaseTapTest @@ -19,6 +19,34 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.start_date = '2021-04-01T00:00:00Z' + def max_bookmarks_by_stream(self, sync_records): + """ + Return the maximum value for the replication key for each stream + which is the bookmark expected value. + + Comparisons are based on the class of the bookmark value. Dates will be + string compared which works for ISO date-time strings + """ + max_bookmarks = {} + for stream, batch in sync_records.items(): + upsert_messages = [m for m in batch.get('messages') if m['action'] == 'upsert'] + stream_bookmark_key = self.expected_replication_keys().get(stream, set()) + assert len(stream_bookmark_key) == 1 # There shouldn't be a compound replication key + stream_bookmark_key = stream_bookmark_key.pop() + + bk_values = [message["data"].get(stream_bookmark_key) for message in upsert_messages] + max_bookmarks[stream] = {stream_bookmark_key: None} + for bk_value in bk_values: + if bk_value is None: + continue + + if max_bookmarks[stream][stream_bookmark_key] is None: + max_bookmarks[stream][stream_bookmark_key] = bk_value + + if bk_value > max_bookmarks[stream][stream_bookmark_key]: + max_bookmarks[stream][stream_bookmark_key] = bk_value + return max_bookmarks + def test_run(self): with self.subTest(store="store_1"): conn_id = self.create_connection(original_credentials=True) @@ -50,6 +78,7 @@ def bookmarks_test(self, conn_id, testable_streams): found_catalogs = menagerie.get_catalogs(conn_id) incremental_streams = {key for key, value in self.expected_replication_method().items() if value == self.INCREMENTAL and key in testable_streams} + incremental_streams = incremental_streams # Our test data sets for Shopify do not have any abandoned_checkouts our_catalogs = [catalog for catalog in found_catalogs if @@ -130,7 +159,7 @@ def bookmarks_test(self, conn_id, testable_streams): dt.utcfromtimestamp(target_min_value)) except (OverflowError, ValueError, TypeError): - print("bookmarks cannot be converted to dates, comparing values directly") + LOGGER.warn("bookmarks cannot be converted to dates, comparing values directly") # verify that there is data with different bookmark values - setup necessary self.assertGreaterEqual(target_value, target_min_value, @@ -157,4 +186,4 @@ def bookmarks_test(self, conn_id, testable_streams): target_value = self.local_to_utc(dt.utcfromtimestamp(target_value)) except (OverflowError, ValueError, TypeError): - print("bookmarks cannot be converted to dates, comparing values directly") + LOGGER.warn("bookmarks cannot be converted to dates, comparing values directly") diff --git a/tests/test_bookmarks_updated.py b/tests/test_bookmarks_updated.py index c39ce1a3..b61dbe2a 100644 --- a/tests/test_bookmarks_updated.py +++ b/tests/test_bookmarks_updated.py @@ -13,7 +13,7 @@ class BookmarkTest(BaseTapTest): """Test tap sets a bookmark and respects it for the next sync of a stream""" @staticmethod def name(): - return "tap_tester_shopify_bookmark_test" + return "tap_tester_shopify_bookmark_update_test" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -26,7 +26,12 @@ def __init__(self, *args, **kwargs): # creating this global variable for store 2 which is required only for this test, all the other test are referencing from base global store_2_streams - store_2_streams = {'abandoned_checkouts', 'collects', 'metafields', 'transactions', 'order_refunds', 'products', 'locations', 'inventory_items', 'events', 'customers', 'custom_collections', 'orders'} + # removed 'abandoned_checkouts' from store 2 streams, as per the Doc: + # https://help.shopify.com/en/manual/orders/abandoned-checkouts?st_source=admin&st_campaign=abandoned_checkouts_footer&utm_source=admin&utm_campaign=abandoned_checkouts_footer#review-your-abandoned-checkouts + # abandoned checkouts are saved in the Shopify admin for three months. + # Every Monday, abandoned checkouts that are older than three months are removed from your admin. + # Also no POST call is available for this endpoint: https://shopify.dev/api/admin-rest/2022-01/resources/abandoned-checkouts + store_2_streams = {'metafields_products', 'transactions', 'order_refunds', 'products', 'locations', 'inventory_items', 'customers', 'collections', 'order_shipping_lines'} def test_run_store_2(self): with self.subTest(store="store_2"): @@ -72,7 +77,20 @@ def bookmarks_test(self, conn_id, testable_streams): #simulated_states = self.calculated_states_by_stream(first_sync_bookmark) # We are hardcoding the updated state to ensure that we get atleast 1 record in second sync. These values have been provided after reviewing the max bookmark value for each of the streams - simulated_states = {'products': {'updated_at': '2021-12-20T05:10:05.000000Z'}, 'collects': {'updated_at': '2021-09-01T09:08:28.000000Z'}, 'abandoned_checkouts': {'updated_at': '2021-10-28T12:43:14.000000Z'}, 'inventory_levels': {'updated_at': '2021-12-20T05:09:34.000000Z'}, 'locations': {'updated_at': '2021-07-20T09:00:22.000000Z'}, 'events': {'created_at': '2021-12-20T05:09:01.000000Z'}, 'inventory_items': {'updated_at': '2021-09-15T19:44:11.000000Z'}, 'transactions': {'created_at': '2021-12-20T00:08:52-05:00'}, 'metafields': {'updated_at': '2021-09-07T21:18:05.000000Z'}, 'order_refunds': {'created_at': '2021-05-01T17:41:18.000000Z'}, 'customers': {'updated_at': '2021-12-20T05:08:17.000000Z'}, 'orders': {'updated_at': '2021-12-20T05:09:01.000000Z'}, 'custom_collections': {'updated_at': '2021-12-20T17:41:18.000000Z'}} + simulated_states = { + 'products': {'updatedAt': '2025-01-23T14:08:21.000000Z'}, + 'abandoned_checkouts': {'updatedAt': '2025-01-20T06:56:01.000000Z'}, + 'inventory_levels': {'updatedAt': '2024-12-05T09:26:47.000000Z'}, + 'locations': {'createdAt': '2021-07-29T08:38:44.000000Z'}, + 'inventory_items': {'updatedAt': '2021-09-15T19:44:11.000000Z'}, + 'transactions': {'createdAt': '2024-12-05T09:58:40.000000Z'}, + 'metafields_customer': '2025-01-21T13:28:24.000000Z', + 'order_refunds': {'updatedAt': '2024-12-05T09:58:40.000000Z'}, + 'customers': {'updatedAt': '2025-01-19T20:55:07.000000Z'}, + 'collections': {'updatedAt': '2025-01-21T13:29:06.000000Z'}, + 'order_shipping_lines': {'updatedAt': '2021-09-30T01:02:21.000000Z'}, + 'metafields_products': {'updatedAt': '2025-01-21T03:11:54.000000Z'} + } for stream, updated_state in simulated_states.items(): new_state['bookmarks'][stream] = updated_state @@ -95,10 +113,13 @@ def bookmarks_test(self, conn_id, testable_streams): # information required for assertions from sync 1 and 2 based on expected values first_sync_count = first_sync_record_count.get(stream, 0) second_sync_count = second_sync_record_count.get(stream, 0) + first_sync_messages = [record.get('data') for record in first_sync_records.get(stream, {}).get('messages', []) - if record.get('action') == 'upsert'] + if record.get('action') == 'upsert'] + second_sync_messages = [record.get('data') for record in second_sync_records.get(stream, {}).get('messages', []) if record.get('action') == 'upsert'] + first_bookmark_value = first_sync_bookmark.get('bookmarks', {stream: None}).get(stream) first_bookmark_value = list(first_bookmark_value.values())[0] second_bookmark_value = second_sync_bookmark.get('bookmarks', {stream: None}).get(stream) @@ -112,7 +133,9 @@ def bookmarks_test(self, conn_id, testable_streams): # verify the syncs sets a bookmark of the expected form self.assertIsNotNone(first_bookmark_value) + self.assertTrue(self.is_expected_date_format(first_bookmark_value)) self.assertIsNotNone(second_bookmark_value) + self.assertTrue(self.is_expected_date_format(second_bookmark_value)) # verify the 2nd bookmark is equal to 1st sync bookmark #NOT A BUG (IS the expected behaviour for shopify as they are using date windowing : TDL-17096 : 2nd bookmark value is getting assigned from the execution time rather than the actual bookmark time. This is an invalid assertion for shopify @@ -126,16 +149,13 @@ def bookmarks_test(self, conn_id, testable_streams): for record in second_sync_messages: replication_key_value = record.get(replication_key) # verify the 2nd sync replication key value is greater or equal to the 1st sync bookmarks - self.assertGreaterEqual(replication_key_value, simulated_bookmark_value, msg="Second sync records do not respect the previous bookmark") + self.assertGreaterEqual(self.convert_state_to_utc(replication_key_value), simulated_bookmark_value, msg="Second sync records do not respect the previous bookmark") # verify the 2nd sync bookmark value is the max replication key value for a given stream self.assertLessEqual(replication_key_value, second_bookmark_value_utc, msg="Second sync bookmark was set incorrectly, a record with a greater replication key value was synced") # verify that we get less data in the 2nd sync - # collects has all the records with the same value of replication key, so we are removing from this assertion - if stream not in ('collects'): - self.assertLess(second_sync_count, first_sync_count, - msg="Second sync does not have less records, bookmark usage not verified") + self.assertLess(second_sync_count, first_sync_count, + msg="Second sync does not have less records, bookmark usage not verified") # verify that we get atleast 1 record in the second sync - if stream not in ('collects'): - self.assertGreater(second_sync_count, 0, msg="Second sync did not yield any records") + self.assertGreater(second_sync_count, 0, msg="Second sync did not yield any records") diff --git a/tests/test_discovery.py b/tests/test_discovery.py index 5a2377dd..89aa4a49 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -33,23 +33,24 @@ def test_run(self): • verify that all other fields have inclusion of available (metadata and schema) """ conn_id = self.create_connection() + expected_streams = self.expected_streams() | {"abandoned_checkouts"} # Verify number of actual streams discovered match expected found_catalogs = menagerie.get_catalogs(conn_id) self.assertGreater(len(found_catalogs), 0, msg="unable to locate schemas for connection {}".format(conn_id)) self.assertEqual(len(found_catalogs), - len(self.expected_streams()), + len(expected_streams), msg="Expected {} streams, actual was {} for connection {}," " actual {}".format( - len(self.expected_streams()), + len(expected_streams), len(found_catalogs), found_catalogs, conn_id)) # Verify the stream names discovered were what we expect found_catalog_names = {c['tap_stream_id'] for c in found_catalogs} - self.assertEqual(set(self.expected_streams()), + self.assertEqual(set(expected_streams), set(found_catalog_names), msg="Expected streams don't match actual streams") @@ -58,7 +59,7 @@ def test_run(self): self.assertTrue(all([re.fullmatch(r"[a-z_]+", name) for name in found_catalog_names]), msg="One or more streams don't follow standard naming") - for stream in self.expected_streams(): + for stream in expected_streams: with self.subTest(stream=stream): catalog = next(iter([catalog for catalog in found_catalogs if catalog["stream_name"] == stream])) @@ -74,6 +75,14 @@ def test_run(self): self.assertTrue(len(stream_properties) == 1, msg="There is more than one top level breadcrumb") + # collect fields + actual_fields = [] + for md_entry in metadata: + if md_entry['breadcrumb'] != []: + actual_fields.append(md_entry['breadcrumb'][1]) + # Verify there are no duplicate/conflicting metadata entries. + self.assertEqual(len(actual_fields), len(set(actual_fields)), msg="There are duplicate entries in the fields of '{}' stream".format(stream)) + # verify replication key(s) self.assertEqual( set(stream_properties[0].get( @@ -120,6 +129,10 @@ def test_run(self): expected_primary_keys = self.expected_primary_keys()[stream] expected_replication_keys = self.expected_replication_keys()[stream] expected_automatic_fields = expected_primary_keys | expected_replication_keys + + # In the order_refunds stream, order field is explicitly marked as automatic + if stream == "order_refunds": + expected_automatic_fields = expected_automatic_fields | {"order"} # verify that primary, replication and foreign keys # are given the inclusion of automatic in annotated schema. @@ -131,13 +144,18 @@ def test_run(self): expected_automatic_fields, actual_automatic_fields)) - # verify that all other fields have inclusion of available - # This assumes there are no unsupported fields for SaaS sources + # verify that all other fields have inclusion of available or unsupported self.assertTrue( - all({value.get("inclusion") == "available" for key, value - in schema["properties"].items() - if key not in actual_automatic_fields}), - msg="Not all non key properties are set to available in annotated schema") + all( + ( + value.get("inclusion") == "available" + or value.get("inclusion") == "unsupported" + ) + for key, value in schema["properties"].items() + if key not in actual_automatic_fields + ), + msg="Not all non key properties are set to available in annotated schema" + ) # verify that primary, replication and foreign keys # are given the inclusion of automatic in metadata. @@ -151,10 +169,9 @@ def test_run(self): expected_automatic_fields, actual_automatic_fields)) - # verify that all other fields have inclusion of available - # This assumes there are no unsupported fields for SaaS sources + # verify that all other fields have inclusion of available or unsupported self.assertTrue( - all({item.get("metadata").get("inclusion") == "available" + all({(item.get("metadata").get("inclusion") == "available" or item.get("metadata").get("inclusion") == "unsupported") for item in metadata if item.get("breadcrumb", []) != [] and item.get("breadcrumb", ["properties", None])[1] diff --git a/tests/test_interrupted_sync.py b/tests/test_interrupted_sync.py new file mode 100644 index 00000000..3ab6473f --- /dev/null +++ b/tests/test_interrupted_sync.py @@ -0,0 +1,237 @@ +""" +Test tap successfully resumes after interrupted sync without missing any records +""" +import random + +from datetime import datetime as dt + +from tap_tester import menagerie, connections, runner, LOGGER +from base import BaseTapTest + + +class InterruptedSyncTest(BaseTapTest): + """Test tap sets a bookmark and respects it for the next sync of a stream""" + @staticmethod + def name(): + return "tap_tester_shopify_int_sync_test" + + def group_streams(self, sync_order, currently_syncing): + self.assertIn(currently_syncing, sync_order, + msg="Currently sycning stream not found in sync order") + index = len(sync_order) + for i, stream in enumerate(sync_order): + if stream == currently_syncing: + index = i + break + return { + "completed": sync_order[:index], + "yet_to_be_synced": sync_order[(index + 1):], + } + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.start_date = '2022-03-01T00:00:00Z' + + def test_run(self): + + conn_id = self.create_connection(original_properties=False, original_credentials=False) + + expected_streams = {'customers', + 'collections', + 'orders', + 'products', + 'transactions'} + + # Select all streams and no fields within streams + found_catalogs = menagerie.get_catalogs(conn_id) + + # Our test data sets for Shopify do not have any abandoned_checkouts + our_catalogs = [catalog for catalog in found_catalogs if + catalog.get('tap_stream_id') in expected_streams] + self.select_all_streams_and_fields(conn_id, our_catalogs, select_all_fields=True) + + ################################# + # Run first sync + ################################# + + first_sync_record_count = self.run_sync(conn_id) + first_sync_state = menagerie.get_state(conn_id) + first_sync_records = runner.get_records_from_target_output() + first_sync_order = runner.get_stream_sync_order_from_target() + + # verify that the sync only sent records to the target for selected streams (catalogs) + self.assertSetEqual(set(first_sync_record_count.keys()), expected_streams) + + # BUG:TDL-17087 : State has additional values which are not streams + # Need to remove additional values from bookmark value + extra_stuff = {'transaction_orders', + 'metafield_products', + 'refund_orders', + 'product_variants'} + + for keys in list(first_sync_state['bookmarks'].keys()): + if keys in extra_stuff: + first_sync_state['bookmarks'].pop(keys) + + ################################ + # Update State between Syncs + ################################ + + # hardcoding the updated state to ensure atleast 1 record in resuming (2nd) sync. + # values have been provided after reviewing the max bookmark value for each of the streams + currently_syncing_stream = random.choice(list(expected_streams)) + LOGGER.info("Randomly selected currently syncing stream: %s", currently_syncing_stream) + + stream_groups = self.group_streams(first_sync_order, currently_syncing_stream) + completed_streams = stream_groups.get('completed') + yet_to_be_synced_streams = stream_groups.get('yet_to_be_synced') + + base_state = {'bookmarks': + {'currently_sync_stream': currently_syncing_stream, + 'customers': first_sync_state.get('bookmarks').get('customers'), + 'orders': first_sync_state.get('bookmarks').get('orders'), + 'collections': first_sync_state.get('bookmarks').get('collections'), + 'products': first_sync_state.get('bookmarks').get('products'), + 'transactions': first_sync_state.get('bookmarks').get('transactions') + }} + + # remove yet to be synced streams from base state and then set new state + new_state = { + 'bookmarks': { + key: val + for key, val in base_state['bookmarks'].items() + if key not in yet_to_be_synced_streams + } + } + + menagerie.set_state(conn_id, new_state) + + ################################ + # Run Resuming (2nd) Sync + ################################ + + resuming_sync_record_count = self.run_sync(conn_id) + resuming_sync_records = runner.get_records_from_target_output() + resuming_sync_state = menagerie.get_state(conn_id) + resuming_sync_order = runner.get_stream_sync_order_from_target() + + LOGGER.info("First sync stream order: %s", first_sync_order) + LOGGER.info("currently syncing stream: %s", currently_syncing_stream) + LOGGER.info("yet to be synced streams: %s", yet_to_be_synced_streams) + LOGGER.info("completed streams: %s", completed_streams) + LOGGER.info("Resuming sync stream order: %s", resuming_sync_order) + + # tap level assertions + self.assertTrue(first_sync_state.get('bookmarks')) + self.assertTrue(resuming_sync_state.get('bookmarks')) + self.assertIsNone(first_sync_state.get('bookmarks', {}).get('currently_sync_stream')) + self.assertIsNone(resuming_sync_state.get('bookmarks', {}).get('currently_sync_stream')) + + # verify streams are shuffled so the resuming sync starts with currently_syncing_stream + self.assertEqual(resuming_sync_order[0], currently_syncing_stream) + + expected_resuming_sync_order = ( + [currently_syncing_stream] + yet_to_be_synced_streams + completed_streams + ) + self.assertListEqual(expected_resuming_sync_order, resuming_sync_order) + + for stream in expected_streams: + with self.subTest(stream=stream): + + # expected values (rep method = incremental for all shopify streams as of Jul-2023) + expected_replication_keys = self.expected_replication_keys() + # information required for assertions from sync 1 and 2 based on expected values + first_sync_count = first_sync_record_count.get(stream, 0) + resuming_sync_count = resuming_sync_record_count.get(stream, 0) + + first_sync_messages = [ + record.get('data') for record + in first_sync_records.get(stream, {}).get('messages', []) + if record.get('action') == 'upsert'] + + resuming_sync_messages = [ + record.get('data') for record + in resuming_sync_records.get(stream, {}).get('messages', []) + if record.get('action') == 'upsert'] + + replication_key = next(iter(expected_replication_keys[stream])) + first_bookmark_stream = first_sync_state.get('bookmarks', {}).get(stream, {}) + first_bookmark_value = first_bookmark_stream.get(replication_key) + resuming_bookmark_stream = resuming_sync_state.get('bookmarks', {}).get(stream, {}) + resuming_bookmark_value = resuming_bookmark_stream.get(replication_key) + resuming_bookmark_value_utc = self.convert_state_to_utc(resuming_bookmark_value) + + if stream in new_state['bookmarks'].keys(): + simulated_bookmark = new_state['bookmarks'][stream] + simulated_bookmark_value = simulated_bookmark[replication_key] + + youngest_first_sync_date = max( + self.parse_date(record.get(replication_key)) + for record in first_sync_messages) + + # verify the syncs sets a bookmark of the expected form + self.assertIsNotNone(first_bookmark_value) + self.assertTrue(self.is_expected_date_format(first_bookmark_value)) + self.assertIsNotNone(resuming_bookmark_value) + self.assertTrue(self.is_expected_date_format(resuming_bookmark_value)) + + # verify the resuming bookmark is greater or equal than 1st sync bookmark + self.assertGreaterEqual(resuming_bookmark_value, first_bookmark_value) + + # verify oldest record from resuming sync respects bookmark from previous sync + if stream in new_state['bookmarks'].keys() and resuming_sync_messages: + # if metafields owner_resource != 'shop' resuming_sync_messages can be empty + actual_oldest_resuming_replication_date = min( + self.parse_date(record.get(replication_key)) + for record in resuming_sync_messages) + + self.assertEqual(actual_oldest_resuming_replication_date, + self.parse_date(simulated_bookmark_value), + msg="Oldest resuming sync record not respecting bookmark") + + # all interrupted recs are in full recs, interrupted rec counts verified + first_sync_records_after_bookmark = [ + record for record in first_sync_messages + if self.parse_date(record[replication_key]) >= + self.parse_date(simulated_bookmark_value)] + # remove any records that got added after the first sync + filtered_resuming_records = [ + record for record in resuming_sync_messages + if self.parse_date(record[replication_key]) <= + youngest_first_sync_date] + + self.assertEqual(first_sync_records_after_bookmark, filtered_resuming_records, + msg="Incorrect data in the resuming sync") + + for record in resuming_sync_messages: + replication_key_value = record.get(replication_key) + # this assertion is only for completed and interrupted streams + if stream in new_state['bookmarks'].keys(): + # verify 2nd sync rep key value is greater or equal to 1st sync bookmarks + msg = "Resuming sync records do not respect the previous bookmark" + self.assertGreaterEqual(replication_key_value, simulated_bookmark_value, + msg=msg) + # verify the 2nd sync bookmark value is the max rep key value for given stream + msg = ("Resuming sync bookmark was set incorrectly, a record with a greater" + " replication key value was synced") + self.assertLessEqual(replication_key_value, resuming_bookmark_value_utc, + msg=msg) + + # verify less data in 2nd sync for streams that started or completed + if stream in new_state['bookmarks'].keys(): + self.assertLess(resuming_sync_count, first_sync_count, + msg="Resuming sync record count greater than expected") + + # verify yet to be sync'd streams have equal oldest record + if stream in yet_to_be_synced_streams: + oldest_first_sync_replication_date = min( + self.parse_date(record.get(replication_key)) + for record in first_sync_messages) + oldest_resuming_sync_replication_date = min( + self.parse_date(record.get(replication_key)) + for record in resuming_sync_messages) + self.assertEqual(oldest_resuming_sync_replication_date, + oldest_first_sync_replication_date) + + # verify that we get at least 1 record in the resuming sync + self.assertGreater(resuming_sync_count, 0, msg="Resuming sync yielded 0 recs") diff --git a/tests/test_pagination.py b/tests/test_pagination.py index b7b619e7..f4b8ab2e 100644 --- a/tests/test_pagination.py +++ b/tests/test_pagination.py @@ -13,12 +13,13 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.start_date = '2021-04-01T00:00:00Z' - def name(self): + @staticmethod + def name(): return "tap_tester_shopify_pagination_test" def get_properties(self, *args, **kwargs): props = super().get_properties(*args, **kwargs) - props['results_per_page'] = '50' + props['results_per_page'] = '30' return props def test_run(self): @@ -27,9 +28,11 @@ def test_run(self): # limit of records returned in 1 page # Documentation: https://help.shopify.com/en/manual/locations/setting-up-your-locations # 'inventory_items': - # As it can call for max 100 product_variants and + # As it can call for max 100 product_variants and # we can generate only one inventory_item for one product_variants - excepted_streams = {'locations', 'inventory_items'} + # 'orders': + # Pagination is not supported for orders BULK API + excepted_streams = {'locations', 'inventory_items', 'orders'} with self.subTest(store="store_1"): conn_id = self.create_connection(original_credentials=True) @@ -39,7 +42,6 @@ def test_run(self): conn_id = self.create_connection(original_properties=False, original_credentials=False) self.pagination_test(conn_id, self.store_2_streams - excepted_streams) - def pagination_test(self, conn_id, testable_streams): """ Verify that for each stream you can get multiple pages of data diff --git a/tests/test_shop_info_fields.py b/tests/test_shop_info_fields.py index ee188569..1aeb7102 100644 --- a/tests/test_shop_info_fields.py +++ b/tests/test_shop_info_fields.py @@ -13,12 +13,17 @@ class ShopInfoFieldsTest(BaseTapTest): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.start_date = '2021-04-01T00:00:00Z' + self.start_date = '2024-12-01T00:00:00Z' @staticmethod def name(): return "tap_tester_shopify_shop_info_fields_test" + def get_properties(self, *args, **kwargs): + props = super().get_properties(*args, **kwargs) + props['results_per_page'] = '30' + return props + def test_run(self): """ Verify shop information fields are present in catalog for every streams. diff --git a/tests/test_start_date.py b/tests/test_start_date.py index 52635141..7ecad59b 100644 --- a/tests/test_start_date.py +++ b/tests/test_start_date.py @@ -8,7 +8,7 @@ from dateutil.parser import parse -from tap_tester import menagerie, runner +from tap_tester import menagerie, runner, LOGGER from base import BaseTapTest @@ -31,21 +31,15 @@ def get_properties(self, original: bool = True): 'shop': 'talenddatawearhouse', 'date_window_size': 30, # BUG: https://jira.talendforge.org/browse/TDL-13180 - 'results_per_page': '50' + 'results_per_page': '30' } if original: return return_value - return_value["start_date"] = '2021-04-21T00:00:00Z' + return_value["start_date"] = '2021-09-15T00:00:00Z' return return_value - @staticmethod - def get_credentials(original_credentials: bool = True): - return { - 'api_key': os.getenv('TAP_SHOPIFY_API_KEY_TALENDDATAWEARHOUSE') - } - @staticmethod def name(): return "tap_tester_shopify_start_date_test" @@ -56,7 +50,15 @@ def test_run(self): # Select all streams and all fields within streams found_catalogs = menagerie.get_catalogs(conn_id) - incremental_streams = {key for key, value in self.expected_replication_method().items() + # removed 'abandoned_checkouts', as per the Doc: + # https://help.shopify.com/en/manual/orders/abandoned-checkouts?st_source=admin&st_campaign=abandoned_checkouts_footer&utm_source=admin&utm_campaign=abandoned_checkouts_footer#review-your-abandoned-checkouts + # abandoned checkouts are saved in the Shopify admin for three months. + # Every Monday, abandoned checkouts that are older than three months are removed from your admin. + # Also no POST call is available for this endpoint: https://shopify.dev/api/admin-rest/2022-01/resources/abandoned-checkouts + expected_replication_method = self.expected_replication_method() + expected_replication_method.pop("abandoned_checkouts") + expected_replication_method.pop("orders") + incremental_streams = {key for key, value in expected_replication_method.items() if value == self.INCREMENTAL} # IF THERE ARE STREAMS THAT SHOULD NOT BE TESTED @@ -72,6 +74,7 @@ def test_run(self): # Count actual rows synced first_sync_records = runner.get_records_from_target_output() + first_min_bookmarks = self.min_bookmarks_by_stream(first_sync_records) # set the start date for a new connection based off bookmarks largest value first_max_bookmarks = self.max_bookmarks_by_stream(first_sync_records) @@ -112,27 +115,49 @@ def test_run(self): for stream in incremental_streams: with self.subTest(stream=stream): + # get primary key values for both sync records + expected_primary_keys = self.expected_primary_keys()[stream] + primary_keys_list_1 = [tuple(message.get('data').get(expected_pk) for expected_pk in expected_primary_keys) + for message in first_sync_records.get(stream).get('messages') + if message.get('action') == 'upsert'] + primary_keys_list_2 = [tuple(message.get('data').get(expected_pk) for expected_pk in expected_primary_keys) + for message in second_sync_records.get(stream).get('messages') + if message.get('action') == 'upsert'] + primary_keys_sync_1 = set(primary_keys_list_1) + primary_keys_sync_2 = set(primary_keys_list_2) + # verify that each stream has less records than the first connection sync self.assertGreaterEqual( first_sync_record_count.get(stream, 0), second_sync_record_count.get(stream, 0), msg="second had more records, start_date usage not verified") - # verify all data from 2nd sync >= start_date - target_mark = second_min_bookmarks.get(stream, {"mark": None}) - target_value = next(iter(target_mark.values())) # there should be only one + # Verify by primary key values, that all records of the 2nd sync are included in the 1st sync since 2nd sync has a later start date. + self.assertTrue(primary_keys_sync_2.issubset(primary_keys_sync_1)) + + # verify all data from both syncs >= start_date + first_sync_target_mark = first_min_bookmarks.get(stream, {"mark": None}) + second_sync_target_mark = second_min_bookmarks.get(stream, {"mark": None}) + + # get start dates for both syncs + first_sync_start_date = self.get_properties()["start_date"] + second_sync_start_date = self.start_date + + for start_date, target_mark in zip((first_sync_start_date, second_sync_start_date), (first_sync_target_mark, second_sync_target_mark)): + target_value = next(iter(target_mark.values())) # there should be only one - if target_value: + if target_value: - # it's okay if there isn't target data for a stream - try: - target_value = self.local_to_utc(parse(target_value)) + # it's okay if there isn't target data for a stream + try: + target_value = self.local_to_utc(parse(target_value)) - # verify that the minimum bookmark sent to the target for the second sync - # is greater than or equal to the start date - self.assertGreaterEqual(target_value, - self.local_to_utc(parse(self.start_date))) + # verify that the minimum bookmark sent to the target for the second sync + # is greater than or equal to the start date + self.assertGreaterEqual(target_value, + self.local_to_utc(parse(start_date))) - except (OverflowError, ValueError, TypeError): - print("bookmarks cannot be converted to dates, " - "can't test start_date for {}".format(stream)) + except (OverflowError, ValueError, TypeError): + LOGGER.warn( + "bookmarks cannot be converted to dates, can't test start_date for %s", stream + ) diff --git a/tests/unittests/test_base.py b/tests/unittests/test_base.py new file mode 100644 index 00000000..203a762f --- /dev/null +++ b/tests/unittests/test_base.py @@ -0,0 +1,138 @@ +import json +import unittest +from unittest.mock import patch, MagicMock +from datetime import datetime +from dateutil.tz import tzlocal +from itertools import cycle +from tap_shopify.streams.base import Stream, ShopifyAPIError +from tap_shopify.context import Context + +class TestStream(unittest.TestCase): + + def mock_query(): + return """ + { + products(first: 10, after: "cursor") { + edges { + node { + id + updatedAt + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + def setUp(self): + self.stream = Stream() + self.stream.data_key = "products" + self.stream.name = "products" + # Mock the Context.config to include start_date + self.original_config = Context.config + Context.config = { + "start_date": "2025-01-01T00:00:00Z", + "date_window_size": 30 + } + Context.catalog = { + "streams": [ + { + "tap_stream_id": "products", + "schema": { + "properties": { + "id": {"type": "string"}, + "updatedAt": {"type": "string"} + } + }, + "metadata": [] + } + ] + } + + def tearDown(self): + # Reset Context.config to its original state + Context.config = self.original_config + + @patch('shopify.GraphQL') + @patch.object(Stream, 'get_query', return_value='mocked_query') + def test_call_api_success(self, mock_get_query, mock_graphql): + """Test successful GraphQL query execution.""" + # Mock the response from Shopify GraphQL API + mock_response = { + "data": { + "products": { + "edges": [{"node": {"id": "mocked_id", "updated_at": "2025-01-01T00:00:00Z"}}], + "pageInfo": {"endCursor": "cursor_123", "hasNextPage": False}, + } + } + } + mock_graphql.return_value.execute.return_value = json.dumps(mock_response) + + query_params = self.stream.get_query_params("2025-01-01T00:00:00Z", "2025-01-02T00:00:00Z") + result = self.stream.call_api(query_params) + + self.assertEqual(result, mock_response["data"]["products"]) + + @patch('shopify.GraphQL') + @patch.object(Stream, 'get_query', return_value='mocked_query') + def test_call_api_error(self, mock_get_query, mock_graphql): + """Test GraphQL query execution with an error.""" + # Mock an error response from Shopify GraphQL API + mock_graphql.return_value.execute.side_effect = ShopifyAPIError("GraphQL error") + + query_params = self.stream.get_query_params("2025-01-01T00:00:00Z", "2025-01-02T00:00:00Z") + + with self.assertRaises(ShopifyAPIError): + self.stream.call_api(query_params) + + @patch('shopify.GraphQL') + @patch.object(Stream, 'get_query', return_value='mocked_query') + def test_call_api_empty_response(self, mock_get_query, mock_graphql): + """Test GraphQL query execution with an empty response.""" + # Mock an empty response from Shopify GraphQL API + mock_response = {} + mock_graphql.return_value.execute.return_value = json.dumps(mock_response) + + query_params = self.stream.get_query_params("2025-01-01T00:00:00Z", "2025-01-02T00:00:00Z") + result = self.stream.call_api(query_params) + + self.assertEqual(result, {}) + + @patch('shopify.GraphQL') + @patch.object(Stream, 'get_query', return_value=mock_query()) + @patch.object(Stream, 'transform_object', side_effect=lambda x: x) + @patch('tap_shopify.streams.base.utils.now', return_value=datetime(2025, 2, 1, 0, 0, tzinfo=tzlocal())) + def test_get_objects(self, mock_now, mock_transform_object, mock_get_query, mock_graphql): + """Test get_objects with pagination and bookmarking.""" + # Mock the response from Shopify GraphQL API + mock_response_page_1 = { + "data": { + "products": { + "edges": [{"node": {"id": "mocked_id_1", "updatedAt": "2025-01-01T00:00:00Z"}}], + "pageInfo": {"endCursor": "cursor_123", "hasNextPage": True}, + } + } + } + mock_response_page_2 = { + "data": { + "products": { + "edges": [{"node": {"id": "mocked_id_2", "updatedAt": "2025-01-01T00:00:00Z"}}], + "pageInfo": {"endCursor": "cursor_456", "hasNextPage": False}, + } + } + } + alternating_responses = cycle([ + json.dumps(mock_response_page_1), + json.dumps(mock_response_page_2) + ]) + + # Set side_effect to use the infinite alternating cycle + mock_graphql.return_value.execute.side_effect = lambda *args, **kwargs: next(alternating_responses) + + objects = list(self.stream.get_objects()) + + self.assertEqual(len(objects), 4) + self.assertEqual(objects[0], {"id": "mocked_id_1", "updatedAt": "2025-01-01T00:00:00Z"}) + self.assertEqual(objects[1], {"id": "mocked_id_2", "updatedAt": "2025-01-01T00:00:00Z"}) diff --git a/tests/unittests/test_client.py b/tests/unittests/test_client.py new file mode 100644 index 00000000..a24b68b1 --- /dev/null +++ b/tests/unittests/test_client.py @@ -0,0 +1,703 @@ +import json +import os +import tempfile +import unittest +from unittest.mock import patch, MagicMock + +import requests + +from tap_shopify.client import ( + ShopifyClient, + SHOPIFY_API_VERSION, +) +from tap_shopify.context import Context +from tap_shopify.exceptions import ShopifyError, ShopifyUnauthorizedError + +class TestShopifyClientInit(unittest.TestCase): + """Tests for ShopifyClient initialization.""" + + def _make_config(self, **overrides): + base = { + "shop": "test-shop", + "client_id": "cid", + "client_secret": "csecret", + "access_token": "existing_token", + "start_date": "2025-01-01T00:00:00Z", + } + base.update(overrides) + return base + + def _write_config_file(self, config): + """Write config dict to a temp file and return its path.""" + tmp = tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) + json.dump(config, tmp) + tmp.close() + return tmp.name + + @patch('tap_shopify.client.requests.post') + def test_init_uses_existing_token_without_refresh(self, mock_post): + """When an access_token is already in config, no refresh should be triggered.""" + config = self._make_config() # has access_token='existing_token' + path = self._write_config_file(config) + try: + client = ShopifyClient(path, config) + mock_post.assert_not_called() + self.assertEqual(client.access_token, "existing_token") + finally: + os.unlink(path) + + @patch('tap_shopify.client.requests.post') + def test_init_fetches_token_when_missing(self, mock_post): + """First run: no access_token in config, should fetch one via client credentials.""" + config = self._make_config() + del config['access_token'] + path = self._write_config_file(config) + try: + mock_post.return_value = MagicMock( + status_code=200, + json=MagicMock(return_value={"access_token": "fetched_token"}), + ) + client = ShopifyClient(path, config) + mock_post.assert_called_once() + self.assertEqual(client.access_token, "fetched_token") + self.assertEqual(config['access_token'], "fetched_token") + finally: + os.unlink(path) + + + +class TestRefreshAccessToken(unittest.TestCase): + """Tests for ShopifyClient._refresh_access_token.""" + + def _create_client_skip_init(self, config, config_path="/tmp/dummy.json"): + client = object.__new__(ShopifyClient) + client.config = config + client.config_path = config_path + return client + + @patch('tap_shopify.client.requests.post') + def test_successful_refresh(self, mock_post): + """Successful token refresh should update config and access_token.""" + config = { + "shop": "test-shop", + "client_id": "cid", + "client_secret": "csecret", + } + tmp = tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) + json.dump(config, tmp) + tmp.close() + + try: + client = self._create_client_skip_init(config, tmp.name) + mock_post.return_value = MagicMock( + status_code=200, + json=MagicMock(return_value={ + "access_token": "refreshed_token", + "expires_in": 86400, + }), + ) + client._refresh_access_token() + + self.assertEqual(client.access_token, "refreshed_token") + self.assertEqual(config['access_token'], "refreshed_token") + + # Verify the correct endpoint was called (shop + .myshopify.com) + mock_post.assert_called_once_with( + "https://test-shop.myshopify.com/admin/oauth/access_token", + json={ + "client_id": "cid", + "client_secret": "csecret", + "grant_type": "client_credentials", + }, + headers={"Accept": "application/json"}, + timeout=30, + ) + + # Verify config file was updated (access_token only, no expiry fields) + with open(tmp.name, 'r') as f: + saved = json.load(f) + self.assertEqual(saved['access_token'], "refreshed_token") + finally: + os.unlink(tmp.name) + + @patch('tap_shopify.client.requests.post') + def test_refresh_failure_raises(self, mock_post): + """Non-200 response from token endpoint should raise.""" + config = { + "shop": "test-shop", + "client_id": "cid", + "client_secret": "csecret", + } + tmp = tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) + json.dump(config, tmp) + tmp.close() + + try: + client = self._create_client_skip_init(config, tmp.name) + mock_post.return_value = MagicMock( + status_code=401, + text="Unauthorized", + ) + with self.assertRaises(ShopifyError) as ctx: + client._refresh_access_token() + self.assertIn("Failed to obtain access token", str(ctx.exception)) + finally: + os.unlink(tmp.name) + + +class TestWriteConfig(unittest.TestCase): + """Tests for ShopifyClient._write_config.""" + + def test_write_config_updates_file(self): + """Config file should be updated with new access_token (token_expires_at not persisted).""" + original = { + "shop": "test-shop", + "client_id": "cid", + "client_secret": "csecret", + "start_date": "2025-01-01T00:00:00Z", + } + tmp = tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) + json.dump(original, tmp) + tmp.close() + + try: + config = { + **original, + "access_token": "new_token", + } + client = object.__new__(ShopifyClient) + client.config = config + client.config_path = tmp.name + + client._write_config() + + with open(tmp.name, 'r') as f: + saved = json.load(f) + + self.assertEqual(saved['access_token'], "new_token") + self.assertNotIn('token_expires_at', saved) + # Original keys preserved + self.assertEqual(saved['shop'], "test-shop") + self.assertEqual(saved['client_id'], "cid") + finally: + os.unlink(tmp.name) + + def test_write_config_preserves_extra_keys(self): + """Extra keys in the config file should not be removed.""" + original = { + "shop": "test-shop", + "client_id": "cid", + "client_secret": "csecret", + "custom_setting": "keep_me", + } + tmp = tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) + json.dump(original, tmp) + tmp.close() + + try: + config = { + **original, + "access_token": "tok", + } + client = object.__new__(ShopifyClient) + client.config = config + client.config_path = tmp.name + + client._write_config() + + with open(tmp.name, 'r') as f: + saved = json.load(f) + self.assertEqual(saved['custom_setting'], "keep_me") + finally: + os.unlink(tmp.name) + + + +class TestRefreshToken(unittest.TestCase): + """Tests for ShopifyClient.refresh_token (reactive 401 refresh).""" + + @patch.object(ShopifyClient, '_refresh_access_token') + def test_refresh_token_calls_underlying_refresh(self, mock_refresh): + """refresh_token should always call _refresh_access_token.""" + client = object.__new__(ShopifyClient) + client.config = {"shop": "test-shop", "access_token": "tok"} + client.config_path = "/tmp/dummy.json" + client.access_token = "tok" + client.refresh_token() + mock_refresh.assert_called_once() + + +class TestReinitializeSession(unittest.TestCase): + """Tests for ShopifyClient.reinitialize_session.""" + + @patch('tap_shopify.client.shopify.Shop.set_timeout') + @patch('tap_shopify.client.shopify.ShopifyResource.activate_session') + @patch('tap_shopify.client.shopify.Session') + def test_reinitialize_session(self, mock_session_cls, mock_activate, mock_set_timeout): + """reinitialize_session should create a new session, activate it, and set timeout.""" + client = object.__new__(ShopifyClient) + client.config = {"shop": "test-shop"} + client.access_token = "my_token" + client.config_path = "/tmp/dummy.json" + + mock_session_instance = MagicMock() + mock_session_cls.return_value = mock_session_instance + + client.reinitialize_session() + + mock_session_cls.assert_called_once_with( + "test-shop", + SHOPIFY_API_VERSION, + "my_token", + ) + mock_activate.assert_called_once_with(mock_session_instance) + mock_set_timeout.assert_called_once() + + +class TestRetry401Handler(unittest.TestCase): + """Tests for the retry_401_handler function used in backoff decorator.""" + + def setUp(self): + self.original_client = Context.client + + def tearDown(self): + Context.client = self.original_client + + def test_retry_401_handler_refreshes_and_reinitializes(self): + """retry_401_handler should call refresh_token and reinitialize_session.""" + from tap_shopify.streams.base import retry_401_handler + + mock_client = MagicMock() + Context.client = mock_client + + retry_401_handler({'wait': 1, 'tries': 1}) + + mock_client.refresh_token.assert_called_once() + mock_client.reinitialize_session.assert_called_once() + + def test_retry_401_handler_no_client(self): + """retry_401_handler should do nothing if Context.client is None.""" + from tap_shopify.streams.base import retry_401_handler + + Context.client = None + # Should not raise + retry_401_handler({'wait': 1, 'tries': 1}) + + @patch('tap_shopify.client.requests.post') + def test_retry_401_handler_updates_context_config_access_token(self, mock_post): + """After retry_401_handler fires, Context.config['access_token'] must reflect + the newly fetched token. + + This relies on ShopifyClient.config being the *same dict object* as + Context.config (passed by reference in main()), so that + _refresh_access_token()'s `self.config['access_token'] = ...` + is immediately visible via Context.config['access_token']. + """ + import tempfile, os + from tap_shopify.streams.base import retry_401_handler + + shared_config = { + "shop": "test-shop", + "client_id": "cid", + "client_secret": "csecret", + "access_token": "old_token", + "start_date": "2025-01-01T00:00:00Z", + } + + tmp = tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) + json.dump(shared_config, tmp) + tmp.close() + + try: + # Wire up a real ShopifyClient sharing the same config dict as Context + client = object.__new__(ShopifyClient) + client.config = shared_config # same object as Context.config below + client.config_path = tmp.name + client.access_token = "old_token" + + Context.config = shared_config # same dict reference + Context.client = client + + mock_post.return_value = MagicMock( + status_code=200, + json=MagicMock(return_value={"access_token": "new_token"}), + ) + + with patch('shopify.Session'), \ + patch('shopify.ShopifyResource.activate_session'), \ + patch('shopify.Shop.set_timeout'): + retry_401_handler({'wait': 1, 'tries': 1}) + + # The token update in ShopifyClient must be visible through Context.config + self.assertEqual(Context.config['access_token'], "new_token") + self.assertEqual(client.access_token, "new_token") + finally: + os.unlink(tmp.name) + + +class TestCallApiWithTokenRefresh(unittest.TestCase): + """Tests for call_api behaviour regarding token handling.""" + + def setUp(self): + self.original_config = Context.config + self.original_client = Context.client + Context.config = { + "start_date": "2025-01-01T00:00:00Z", + "date_window_size": 30, + } + Context.catalog = { + "streams": [{ + "tap_stream_id": "products", + "schema": {"properties": {"id": {"type": "string"}, "updatedAt": {"type": "string"}}}, + "metadata": [], + }], + } + + def tearDown(self): + Context.config = self.original_config + Context.client = self.original_client + + def _make_stream(self): + from tap_shopify.streams.base import Stream + stream = Stream() + stream.name = "products" + stream.data_key = "products" + return stream + + @patch('shopify.GraphQL') + @patch('tap_shopify.streams.base.Stream.get_query', return_value='{ products { edges { node { id } } } }') + def test_call_api_success(self, mock_get_query, mock_graphql): + """call_api should return data on a successful GraphQL response.""" + Context.client = None + + mock_response = { + "data": {"products": {"edges": [], "pageInfo": {"endCursor": None, "hasNextPage": False}}} + } + mock_graphql.return_value.execute.return_value = json.dumps(mock_response) + + stream = self._make_stream() + result = stream.call_api({"query": "test", "first": 10}) + self.assertEqual(result, mock_response["data"]["products"]) + + @patch('shopify.GraphQL') + @patch('tap_shopify.streams.base.Stream.get_query', return_value='{ products { edges { node { id } } } }') + def test_call_api_without_client(self, mock_get_query, mock_graphql): + """call_api should work even if Context.client is None (backward compat).""" + Context.client = None + + mock_response = { + "data": {"products": {"edges": [], "pageInfo": {"endCursor": None, "hasNextPage": False}}} + } + mock_graphql.return_value.execute.return_value = json.dumps(mock_response) + + stream = self._make_stream() + result = stream.call_api({"query": "test", "first": 10}) + self.assertEqual(result, mock_response["data"]["products"]) + + @patch('shopify.GraphQL') + @patch('tap_shopify.streams.base.Stream.get_query', return_value='{ products { edges { node { id } } } }') + def test_non_401_http_error_raises_shopify_error(self, mock_get_query, mock_graphql): + """Non-401 HTTPError should raise ShopifyError.""" + import urllib.error + from tap_shopify.exceptions import ShopifyError + + Context.client = None + + http_error = urllib.error.HTTPError( + url="https://test-shop.myshopify.com/admin/api/graphql.json", + code=500, + msg="Internal Server Error", + hdrs=MagicMock(**{"get.return_value": "req-456"}), + fp=None, + ) + mock_graphql.return_value.execute.side_effect = http_error + + stream = self._make_stream() + + with self.assertRaises(ShopifyError): + stream.call_api({"query": "test", "first": 10}) + + @patch('shopify.GraphQL') + @patch('tap_shopify.streams.base.Stream.get_query', return_value='{ products { edges { node { id } } } }') + def test_401_http_error_raises_shopify_unauthorized_error(self, mock_get_query, mock_graphql): + """401 HTTPError should raise ShopifyUnauthorizedError (not ShopifyError).""" + import urllib.error + from tap_shopify.exceptions import ShopifyUnauthorizedError + + # Must provide a mock client so the retry_401_handler doesn't blow up + mock_client = MagicMock() + mock_client.refresh_token.return_value = False + Context.client = mock_client + + http_error = urllib.error.HTTPError( + url="https://test-shop.myshopify.com/admin/api/graphql.json", + code=401, + msg="Unauthorized", + hdrs=MagicMock(**{"get.return_value": "req-789"}), + fp=None, + ) + mock_graphql.return_value.execute.side_effect = http_error + + stream = self._make_stream() + + with self.assertRaises(ShopifyUnauthorizedError): + stream.call_api({"query": "test", "first": 10}) + + +class TestMainClient(unittest.TestCase): + """Tests for ShopifyClient wiring in main().""" + + @patch('tap_shopify.ShopifyClient') + @patch('tap_shopify.discover') + @patch('singer.utils.parse_args') + def test_main_stores_client_in_context(self, mock_parse_args, mock_discover, mock_client_cls): + """main() should store the ShopifyClient instance in Context.client.""" + from tap_shopify import main + + mock_args = MagicMock() + mock_args.config = { + "shop": "test-shop", + "client_id": "cid", + "client_secret": "csecret", + "access_token": "tok", + "start_date": "2025-01-01T00:00:00Z", + } + mock_args.state = {} + mock_args.dev = False + mock_args.discover = True + mock_args.config_path = "/tmp/config.json" + mock_args.catalog = None + mock_parse_args.return_value = mock_args + + mock_client_instance = MagicMock() + mock_client_instance.access_token = "tok" + mock_client_cls.return_value = mock_client_instance + mock_discover.return_value = {"streams": []} + + try: + main() + except SystemExit: + pass + + self.assertEqual(Context.client, mock_client_instance) + + @patch('tap_shopify.discover') + @patch('singer.utils.parse_args') + def test_main_no_client_when_api_key_present(self, mock_parse_args, mock_discover): + """main() should NOT create ShopifyClient when api_key is in config (legacy auth).""" + from tap_shopify import main + + mock_args = MagicMock() + mock_args.config = { + "shop": "test-shop", + "api_key": "legacy_key", + "start_date": "2025-01-01T00:00:00Z", + } + mock_args.state = {} + mock_args.discover = True + mock_args.config_path = "/tmp/config.json" + mock_args.catalog = None + mock_parse_args.return_value = mock_args + + mock_discover.return_value = {"streams": []} + + original_client = Context.client + Context.client = None + try: + main() + except SystemExit: + pass + + self.assertIsNone(Context.client) + Context.client = original_client + + @patch('tap_shopify.ShopifyClient') + @patch('tap_shopify.discover') + @patch('singer.utils.parse_args') + def test_main_updates_config_access_token(self, mock_parse_args, mock_discover, mock_client_cls): + """main() should update Context.config['access_token'] from client.access_token.""" + from tap_shopify import main + + mock_args = MagicMock() + mock_args.config = { + "shop": "test-shop", + "client_id": "cid", + "client_secret": "csecret", + "access_token": "old_tok", + "start_date": "2025-01-01T00:00:00Z", + } + mock_args.state = {} + mock_args.dev = False + mock_args.discover = True + mock_args.config_path = "/tmp/config.json" + mock_args.catalog = None + mock_parse_args.return_value = mock_args + + mock_client_instance = MagicMock() + mock_client_instance.access_token = "refreshed_tok" + mock_client_cls.return_value = mock_client_instance + mock_discover.return_value = {"streams": []} + + try: + main() + except SystemExit: + pass + + self.assertEqual(Context.config.get('access_token'), "refreshed_tok") + + def _make_discover_args(self, extra_config=None): + mock_args = MagicMock() + mock_args.config = { + "shop": "test-shop", + "client_id": "cid", + "client_secret": "csecret", + "access_token": "tok", + "start_date": "2025-01-01T00:00:00Z", + } + if extra_config: + mock_args.config.update(extra_config) + mock_args.state = {} + mock_args.dev = False + mock_args.discover = True + mock_args.config_path = "/tmp/config.json" + mock_args.catalog = None + return mock_args + + @patch('tap_shopify.ShopifyClient') + @patch('tap_shopify.discover') + @patch('singer.utils.parse_args') + def test_main_propagates_shopify_unauthorized_error( + self, mock_parse_args, mock_discover, mock_client_cls + ): + """main() must re-raise ShopifyUnauthorizedError as-is, not wrap it in ShopifyError. + + Before the fix, ShopifyUnauthorizedError fell into `except Exception` and was + re-raised as ShopifyError(exc) with an empty message, losing the original context. + After the fix, a dedicated `except ShopifyUnauthorizedError` branch re-raises it + directly so callers receive the correct type and message. + """ + from tap_shopify import main + + mock_parse_args.return_value = self._make_discover_args() + + mock_client_instance = MagicMock() + mock_client_instance.access_token = "tok" + mock_client_cls.return_value = mock_client_instance + + original_error = ShopifyUnauthorizedError( + Exception("UnauthorizedAccess"), "Invalid access token" + ) + mock_discover.side_effect = original_error + + with self.assertRaises(ShopifyUnauthorizedError) as ctx: + main() + + # The exception must be the original instance (not a wrapped ShopifyError) + self.assertIsInstance(ctx.exception, ShopifyUnauthorizedError) + self.assertNotIsInstance(ctx.exception, ShopifyError) + self.assertIn("Invalid access token", str(ctx.exception)) + + @patch('tap_shopify.ShopifyClient') + @patch('tap_shopify.discover') + @patch('singer.utils.parse_args') + def test_main_unauthorized_error_message_preserved( + self, mock_parse_args, mock_discover, mock_client_cls + ): + """ShopifyUnauthorizedError message must survive propagation through main(). + + Previously the message was lost because the error was caught by the generic + `except Exception` handler and re-raised as ShopifyError(exc, msg=''). + """ + from tap_shopify import main + + mock_parse_args.return_value = self._make_discover_args() + + mock_client_instance = MagicMock() + mock_client_instance.access_token = "tok" + mock_client_cls.return_value = mock_client_instance + + expected_message = "Invalid access token" + mock_discover.side_effect = ShopifyUnauthorizedError( + Exception("UnauthorizedAccess"), expected_message + ) + + try: + main() + self.fail("Expected ShopifyUnauthorizedError to be raised") + except ShopifyUnauthorizedError as exc: + self.assertIn(expected_message, str(exc)) + except ShopifyError: + self.fail( + "ShopifyUnauthorizedError was incorrectly wrapped as ShopifyError, " + "losing the original message" + ) + + +class TestBackoffOnRefresh(unittest.TestCase): + """Tests for backoff/retry on token refresh failures.""" + + @patch('tap_shopify.client.requests.post') + def test_refresh_retries_on_connection_error(self, mock_post): + """_refresh_access_token should retry on RequestException.""" + config = { + "shop": "test-shop", + "client_id": "cid", + "client_secret": "csecret", + } + tmp = tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) + json.dump(config, tmp) + tmp.close() + + try: + client = object.__new__(ShopifyClient) + client.config = config + client.config_path = tmp.name + + # Fail twice then succeed + mock_post.side_effect = [ + requests.exceptions.ConnectionError("Connection refused"), + requests.exceptions.ConnectionError("Connection refused"), + MagicMock( + status_code=200, + json=MagicMock(return_value={ + "access_token": "recovered_token", + "expires_in": 86400, + }), + ), + ] + client._refresh_access_token() + self.assertEqual(client.access_token, "recovered_token") + self.assertEqual(mock_post.call_count, 3) # backoff retried twice then succeeded + finally: + os.unlink(tmp.name) + + @patch('tap_shopify.client.requests.post') + def test_refresh_gives_up_after_max_retries(self, mock_post): + """_refresh_access_token should give up after max retries.""" + config = { + "shop": "test-shop", + "client_id": "cid", + "client_secret": "csecret", + } + tmp = tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) + json.dump(config, tmp) + tmp.close() + + try: + client = object.__new__(ShopifyClient) + client.config = config + client.config_path = tmp.name + + mock_post.side_effect = requests.exceptions.ConnectionError("Connection refused") + + with self.assertRaises(requests.exceptions.ConnectionError): + client._refresh_access_token() + + # backoff max_tries=3 + self.assertEqual(mock_post.call_count, 3) + finally: + os.unlink(tmp.name) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/unittests/test_date_window.py b/tests/unittests/test_date_window.py new file mode 100644 index 00000000..402dc8d7 --- /dev/null +++ b/tests/unittests/test_date_window.py @@ -0,0 +1,58 @@ +import unittest +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream + +class TestShopifyDateWindowHandling(unittest.TestCase): + + def test_no_date_window_value(self): + """Test that no value for date_window is handled correctly.""" + Context.config = { + "start_date": "2025-01-01T00:00:00Z", + } + streams = Stream() + self.assertEqual(streams.date_window_size, 30) + + def test_valid_int_date_window_value(self): + """Test that valid integer value for date_window is handled correctly.""" + Context.config = { + "start_date": "2025-01-01T00:00:00Z", + "date_window_size": 10 + } + streams = Stream() + self.assertEqual(streams.date_window_size, 10) + + def test_valid_str_date_window_value(self): + """Test that valid str value for date_window is handled correctly..""" + Context.config = { + "start_date": "2025-01-01T00:00:00Z", + "date_window_size": "10" + } + streams = Stream() + self.assertEqual(streams.date_window_size, 10) + + def test_valid_float_date_window_value(self): + """Test that valid float value for date_window is handled correctly.""" + Context.config = { + "start_date": "2025-01-01T00:00:00Z", + "date_window_size": "11.00" + } + streams = Stream() + self.assertEqual(streams.date_window_size, 11) + + def test_zero_str_date_window_value(self): + """Test that valid zero string value for date_window is handled correctly.""" + Context.config = { + "start_date": "2025-01-01T00:00:00Z", + "date_window_size": "0" + } + streams = Stream() + self.assertEqual(streams.date_window_size, 30) + + def test_zero_int_date_window_value(self): + """Test that valid zero integer value for date_window is handled correctly.""" + Context.config = { + "start_date": "2025-01-01T00:00:00Z", + "date_window_size": 0 + } + streams = Stream() + self.assertEqual(streams.date_window_size, 30) diff --git a/tests/unittests/test_error_handling.py b/tests/unittests/test_error_handling.py new file mode 100644 index 00000000..57f901bd --- /dev/null +++ b/tests/unittests/test_error_handling.py @@ -0,0 +1,49 @@ +import unittest +import http.client +import socket +from parameterized import parameterized +import pyactiveresource +import simplejson +from urllib.error import URLError +from unittest.mock import patch, MagicMock +import itertools +from tap_shopify.streams.products import Products +from tap_shopify.streams.base import ShopifyAPIError + +class MockResponse: + def __init__(self, msg, url, code): + self.msg = msg + self.url = url + self.code = code + +class TestShopifyErrorHandling(unittest.TestCase): + + @parameterized.expand([ + ["http_incompleteread_error", lambda: http.client.IncompleteRead(10), http.client.IncompleteRead], + ["connection_reset_error", lambda: ConnectionResetError("Connection reset by peer"), ConnectionResetError], + ["shopify_api_error", lambda: ShopifyAPIError("Shopify API error"), ShopifyAPIError], + ["pyactiveresource_connection_error", lambda: pyactiveresource.connection.Error("Resource connection error with timed out"), pyactiveresource.connection.Error], + ["socket_timeout_error", lambda: socket.timeout("The read operation timed out"), socket.timeout], + ["server_error", lambda: pyactiveresource.connection.ServerError(MockResponse("Server error", "https://shopify.com", 500)), pyactiveresource.connection.ServerError], + ["formats_error", lambda: pyactiveresource.formats.Error("Format error"), pyactiveresource.formats.Error], + ["json_decode_error", lambda: simplejson.scanner.JSONDecodeError("JSON decode error", "doc", 0), simplejson.scanner.JSONDecodeError], + ["url_error", lambda: URLError("URL error"), URLError] + ]) + @patch("shopify.GraphQL") + def test_api_errors(self, name, error_fn, expected_exception, mock_graphql): + """Test handling of different API errors""" + + side_effect = error_fn() # Call the lambda to create the exception instance + + with self.subTest(name=name, side_effect=side_effect, expected_exception=expected_exception): + # Mock GraphQL execute method to simulate errors + mock_graphql_instance = mock_graphql.return_value + mock_graphql_instance.execute = MagicMock(side_effect=itertools.repeat(side_effect, 5)) + + obj = Products() + obj.data_key = "products" + + with self.assertRaises(expected_exception): + obj.call_api(query_params={}) + + self.assertEqual(mock_graphql_instance.execute.call_count, 5) diff --git a/tests/unittests/test_inventory_items.py b/tests/unittests/test_inventory_items.py deleted file mode 100644 index 76721c44..00000000 --- a/tests/unittests/test_inventory_items.py +++ /dev/null @@ -1,92 +0,0 @@ -import unittest -from unittest import mock -from singer.utils import strptime_to_utc -from tap_shopify.context import Context - -INVENTORY_ITEM_OBJECT = Context.stream_objects['inventory_items']() - -class Product(): - def __init__(self, id, variants): - self.id = id - self.variants = variants - -class ProductVariant(): - def __init__(self, id, inventory_item_id): - self.id = id - self.inventory_item_id = inventory_item_id - -class InventoryItems(): - def __init__(self, id, updated_at): - self.id = id - self.updated_at = updated_at - - def to_dict(self): - return {"id": self.id, "updated_at": self.updated_at} - -ITEM_1 = InventoryItems("i11", "2021-08-11T01:57:05-04:00") -ITEM_2 = InventoryItems("i12", "2021-08-12T01:57:05-04:00") -ITEM_3 = InventoryItems("i21", "2021-08-13T01:57:05-04:00") -ITEM_4 = InventoryItems("i22", "2021-08-14T01:57:05-04:00") - -class TestInventoryItems(unittest.TestCase): - - @mock.patch("tap_shopify.streams.base.Stream.get_objects") - @mock.patch("tap_shopify.streams.inventory_items.InventoryItems.get_inventory_items") - def test_get_objects_with_product_variant(self, mock_get_inventory_items, mock_parent_object): - - expected_inventory_items = [ITEM_1, ITEM_2, ITEM_3, ITEM_4] - product1 = Product("p1", [ProductVariant("v11", "i11"), ProductVariant("v21", "i21")]) - product2 = Product("p2", [ProductVariant("v12", "i12"), ProductVariant("v22", "i22")]) - - mock_get_inventory_items.side_effect = [[ITEM_1, ITEM_2], [ITEM_3, ITEM_4]] - mock_parent_object.return_value = [product1, product2] - - actual_inventory_items = list(INVENTORY_ITEM_OBJECT.get_objects()) - - #Verify that it returns inventory_item of all product variant - self.assertEqual(actual_inventory_items, expected_inventory_items) - - - @mock.patch("tap_shopify.streams.base.Stream.get_objects") - @mock.patch("tap_shopify.streams.inventory_items.InventoryItems.get_inventory_items") - def test_get_objects_with_product_but_no_variant(self, mock_get_inventory_items, mock_parent_object): - - expected_inventory_items = [ITEM_3, ITEM_4] - - #Product1 contain no variant - product1 = Product("p1", []) - - product2 = Product("p2", [ProductVariant("v12", "i12"), ProductVariant("v22", "i22")]) - mock_parent_object.return_value = [product1, product2] - - mock_get_inventory_items.side_effect = [[], [ITEM_3, ITEM_4]] - - actual_inventory_items = list(INVENTORY_ITEM_OBJECT.get_objects()) - #Verify that it returns inventory_item of existing product variant - self.assertEqual(actual_inventory_items, expected_inventory_items) - - - @mock.patch("tap_shopify.streams.base.Stream.get_objects") - @mock.patch("tap_shopify.streams.inventory_items.InventoryItems.get_inventory_items") - def test_get_objects_with_no_product(self, mock_get_inventory_items, mock_parent_object): - - #No product exist - mock_parent_object.return_value = [] - expected_inventory_items = [] - - actual_inventory_items = list(INVENTORY_ITEM_OBJECT.get_objects()) - self.assertEqual(actual_inventory_items, expected_inventory_items) - - @mock.patch("tap_shopify.streams.base.Stream.get_bookmark") - @mock.patch("tap_shopify.streams.inventory_items.InventoryItems.get_objects") - def test_sync(self, mock_get_objects, mock_get_bookmark): - - expected_sync = [ITEM_3.to_dict(), ITEM_4.to_dict()] - mock_get_objects.return_value = [ITEM_1, ITEM_2, ITEM_3, ITEM_4] - - mock_get_bookmark.return_value = strptime_to_utc("2021-08-13T01:05:05-04:00") - - actual_sync = list(INVENTORY_ITEM_OBJECT.sync()) - - #Verify that only 2 record syncs - self.assertEqual(actual_sync, expected_sync) \ No newline at end of file diff --git a/tests/unittests/test_inventory_levels.py b/tests/unittests/test_inventory_levels.py deleted file mode 100644 index cf2992a8..00000000 --- a/tests/unittests/test_inventory_levels.py +++ /dev/null @@ -1,76 +0,0 @@ -import unittest -from unittest import mock -from singer.utils import strptime_to_utc -from tap_shopify.context import Context - -INVENTORY_LEVEL_OBJECT = Context.stream_objects['inventory_levels']() - -class Location(): - def __init__(self, id): - self.id = id - -class InventoryLevels(): - def __init__(self, id, updated_at): - self.id = id - self.updated_at = updated_at - - def to_dict(self): - return {"id": self.id, "updated_at": self.updated_at} - -LEVEL_1 = InventoryLevels("inv_level1", "2021-08-11T01:57:05-04:00") -LEVEL_2 = InventoryLevels("inv_level2", "2021-08-12T01:57:05-04:00") -LEVEL_3 = InventoryLevels("inv_level3", "2021-08-13T01:57:05-04:00") -LEVEL_4 = InventoryLevels("inv_level4", "2021-08-14T01:57:05-04:00") - -@mock.patch("tap_shopify.streams.base.Stream.get_bookmark") -class TestInventoryItems(unittest.TestCase): - - @mock.patch("tap_shopify.streams.locations.Locations.get_locations_data") - @mock.patch("tap_shopify.streams.inventory_levels.InventoryLevels.get_inventory_levels") - def test_get_objects_with_locations(self, mock_get_inventory_levels, mock_parent_object, mock_get_bookmark): - ''' - Verify that expected data should be emitted for inventory_levels if locations found. - ''' - expected_inventory_levels = [LEVEL_1, LEVEL_2, LEVEL_3, LEVEL_4] - location1 = Location("location1") - location2 = Location("location2") - - mock_get_inventory_levels.side_effect = [[LEVEL_1, LEVEL_2], [LEVEL_3, LEVEL_4]] - mock_parent_object.return_value = [location1, location2] - - actual_inventory_levels = list(INVENTORY_LEVEL_OBJECT.get_objects()) - - #Verify that it returns inventory_levels for all locations - self.assertEqual(actual_inventory_levels, expected_inventory_levels) - - @mock.patch("tap_shopify.streams.locations.Locations.get_locations_data") - @mock.patch("tap_shopify.streams.inventory_levels.InventoryLevels.get_inventory_levels") - def test_get_objects_with_no_locations(self, mock_get_inventory_levels, mock_parent_object, mock_get_bookmark): - ''' - Verify that no data should be emitted for inventory_levels if no locations found. - ''' - # No data for parent stream location - mock_parent_object.return_value = [] - expected_inventory_levels = [] - - actual_inventory_levels = list(INVENTORY_LEVEL_OBJECT.get_objects()) - - # No get_inventory_levels should be called and no data should be returned - self.assertEqual(actual_inventory_levels, expected_inventory_levels) - self.assertEqual(mock_get_inventory_levels.call_count, 0) - - @mock.patch("tap_shopify.streams.inventory_levels.InventoryLevels.get_objects") - def test_sync(self, mock_get_objects, mock_get_bookmark): - ''' - Verify that only data updated after specific bookmark are yielded from sync. - ''' - - expected_sync = [LEVEL_3.to_dict(), LEVEL_4.to_dict()] - mock_get_objects.return_value = [LEVEL_1, LEVEL_2, LEVEL_3, LEVEL_4] - - mock_get_bookmark.return_value = strptime_to_utc("2021-08-13T01:05:05-04:00") - - actual_sync = list(INVENTORY_LEVEL_OBJECT.sync()) - - #Verify that only 2 record syncs - self.assertEqual(actual_sync, expected_sync) diff --git a/tests/unittests/test_locations.py b/tests/unittests/test_locations.py deleted file mode 100644 index 6f398a4f..00000000 --- a/tests/unittests/test_locations.py +++ /dev/null @@ -1,43 +0,0 @@ -import unittest -from unittest import mock -from singer import utils -from singer.utils import strptime_to_utc, strftime -from tap_shopify.context import Context - -LOCATIONS_OBJECT = Context.stream_objects['locations']() - - -class Locations(): - def __init__(self, id, updated_at): - self.id = id - self.updated_at = updated_at - - def to_dict(self): - return {"id": self.id, "updated_at": self.updated_at} - - -LOCATION_1 = Locations("i11", "2021-08-11T01:57:05-04:00") -LOCATION_2 = Locations("i12", "2021-08-12T01:57:05-04:00") -LOCATION_3 = Locations("i21", "2021-08-13T01:57:05-04:00") -LOCATION_4 = Locations("i22", "2021-08-14T01:57:05-04:00") - - -class TestLocations(unittest.TestCase): - @mock.patch("tap_shopify.streams.base.Stream.update_bookmark") - @mock.patch("tap_shopify.streams.base.Stream.get_bookmark") - @mock.patch("tap_shopify.streams.locations.Locations.get_locations_data") - def test_sync(self, mock_get_locations_data, mock_get_bookmark, mock_update_bookmark): - - expected_sync = [LOCATION_3.to_dict(), LOCATION_4.to_dict()] - mock_get_locations_data.return_value = [LOCATION_1, LOCATION_2, LOCATION_3, LOCATION_4] - - mock_get_bookmark.return_value = strptime_to_utc("2021-08-13T01:05:05-04:00") - - actual_sync = list(LOCATIONS_OBJECT.sync()) - - # Verify that only 2 record syncs - self.assertEqual(actual_sync, expected_sync) - max_bookmark = strptime_to_utc("2021-08-14T01:57:05-04:00") - - # Verify that maximum replication key of all keys is updated as bookmark - mock_update_bookmark.assert_called_with(utils.strftime(max_bookmark)) diff --git a/tests/unittests/test_orders_poll_bulk_completion.py b/tests/unittests/test_orders_poll_bulk_completion.py new file mode 100644 index 00000000..c0f86fab --- /dev/null +++ b/tests/unittests/test_orders_poll_bulk_completion.py @@ -0,0 +1,308 @@ +""" +Unit tests for the 401 token-refresh-and-retry logic in +Orders.poll_bulk_completion (tap_shopify/streams/orders.py). + +Scenario under test +------------------- +During a long-running GraphQL Bulk Operation poll the Shopify access token can +expire. When that happens shopify.GraphQL().execute() raises a +urllib.error.HTTPError with status 401. The connector must: + + 1. Refresh the access token via Context.client. + 2. Reinitialize the Shopify session so subsequent calls use the new token. + 3. Update Context.config['access_token'] with the refreshed token. + 4. Retry the *same* status-check request (no bulk-operation re-submission). + +If no Context.client is present (non-OAuth flow) a ShopifyAPIError is raised. +Non-401 HTTP errors must propagate unchanged. +""" + +import json +import unittest +import urllib.error +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch + +from tap_shopify.context import Context +from tap_shopify.exceptions import ShopifyAPIError +from tap_shopify.streams.orders import Orders + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_BULK_OP_ID = "gid://shopify/BulkOperation/123" +_RESULT_URL = "https://storage.example.com/bulk_result.jsonl" + + +def _op_response(status, url=_RESULT_URL): + """Build a JSON-encoded fake bulk-operation GraphQL response.""" + return json.dumps({ + "data": { + "node": { + "id": _BULK_OP_ID, + "status": status, + "errorCode": None, + "createdAt": "2026-01-01T00:00:00Z", + "completedAt": "2026-01-01T01:00:00Z" if status == "COMPLETED" else None, + "objectCount": "10", + "fileSize": "1024", + "url": url if status == "COMPLETED" else None, + } + } + }) + + +def _http_error(code): + """Create a urllib.error.HTTPError with the given HTTP status code.""" + return urllib.error.HTTPError( + url="https://test.myshopify.com/admin/api/2025-07/graphql.json", + code=code, + msg="Unauthorized" if code == 401 else "Server Error", + hdrs=MagicMock(get=MagicMock(return_value=None)), + fp=None, + ) + + +def _make_stream(): + """Return an Orders instance with bookmarking helpers pre-mocked.""" + stream = Orders() + stream.date_window_size = 30 + stream.update_bookmark = MagicMock() + stream.clear_bulk_operation_state = MagicMock() + return stream + + +_BOOKMARK = datetime(2026, 1, 1, tzinfo=timezone.utc) + + +def _mock_client(new_token="refreshed_token"): + """Return a mock ShopifyClient whose access_token is *new_token*.""" + client = MagicMock() + client.access_token = new_token + return client + + +# --------------------------------------------------------------------------- +# Test class +# --------------------------------------------------------------------------- + +class TestPollBulkCompletion401Handling(unittest.TestCase): + + def setUp(self): + self._orig_config = Context.config + self._orig_client = Context.client + Context.config = { + "access_token": "original_token", + "start_date": "2025-01-01T00:00:00Z", + } + Context.client = None + + def tearDown(self): + Context.config = self._orig_config + Context.client = self._orig_client + + # ------------------------------------------------------------------ + # Happy-path baseline + # ------------------------------------------------------------------ + + @patch("tap_shopify.streams.orders.time") + @patch("shopify.GraphQL") + def test_completed_status_returns_result_url(self, mock_graphql, mock_time): + """Immediate COMPLETED response returns the bulk result URL.""" + mock_time.time.side_effect = [0, 1] + mock_graphql.return_value.execute.return_value = _op_response("COMPLETED") + + stream = _make_stream() + result = stream.poll_bulk_completion( + current_bookmark=_BOOKMARK, + bulk_op_id=_BULK_OP_ID, + ) + + self.assertEqual(result, _RESULT_URL) + stream.update_bookmark.assert_called_once() + + @patch("tap_shopify.streams.orders.time") + @patch("shopify.GraphQL") + def test_running_then_completed_polls_same_operation(self, mock_graphql, mock_time): + """Connector loops through RUNNING before COMPLETED, calling sleep between iterations.""" + mock_time.time.side_effect = [0, 1, 62] + mock_graphql.return_value.execute.side_effect = [ + _op_response("RUNNING"), + _op_response("COMPLETED"), + ] + + stream = _make_stream() + result = stream.poll_bulk_completion( + current_bookmark=_BOOKMARK, + bulk_op_id=_BULK_OP_ID, + ) + + self.assertEqual(result, _RESULT_URL) + self.assertEqual(mock_graphql.return_value.execute.call_count, 2) + mock_time.sleep.assert_called_once_with(60) + + @patch("tap_shopify.streams.orders.time") + @patch("shopify.GraphQL") + def test_failed_status_raises_shopify_api_error(self, mock_graphql, mock_time): + """A FAILED bulk operation raises ShopifyAPIError and clears state.""" + mock_time.time.side_effect = [0, 1] + mock_graphql.return_value.execute.return_value = _op_response("FAILED") + + stream = _make_stream() + with self.assertRaises(ShopifyAPIError): + stream.poll_bulk_completion( + current_bookmark=_BOOKMARK, + bulk_op_id=_BULK_OP_ID, + ) + + stream.clear_bulk_operation_state.assert_called_once() + + # ------------------------------------------------------------------ + # 401 handling — core feature under test + # ------------------------------------------------------------------ + + @patch("tap_shopify.streams.orders.time") + @patch("shopify.GraphQL") + def test_401_refreshes_token_and_retries_same_bulk_op(self, mock_graphql, mock_time): + """ + On a 401 the connector refreshes the token, reinitializes the session, + updates Context.config, and retries the status check for the SAME + bulk operation — no re-submission of the bulk query. + """ + mock_time.time.side_effect = [0, 1] + mock_graphql.return_value.execute.side_effect = [ + _http_error(401), # first call → 401 + _op_response("COMPLETED"), # retry → success + ] + + client = _mock_client("refreshed_token") + Context.client = client + + stream = _make_stream() + result = stream.poll_bulk_completion( + current_bookmark=_BOOKMARK, + bulk_op_id=_BULK_OP_ID, + ) + + # Token refresh and session reinit were called exactly once + client.refresh_token.assert_called_once() + client.reinitialize_session.assert_called_once() + + # Context.config carries the new token for subsequent calls + self.assertEqual(Context.config["access_token"], "refreshed_token") + + # Result URL returned — polling continued on the existing bulk op + self.assertEqual(result, _RESULT_URL) + + # execute called twice: 401 probe + successful retry + self.assertEqual(mock_graphql.return_value.execute.call_count, 2) + + @patch("tap_shopify.streams.orders.time") + @patch("shopify.GraphQL") + def test_401_mid_poll_loop_continues_polling_after_refresh(self, mock_graphql, mock_time): + """ + 401 can appear in any poll iteration. After refresh the loop continues + and eventually reaches COMPLETED. + """ + mock_time.time.side_effect = [0, 1, 62] + mock_graphql.return_value.execute.side_effect = [ + _op_response("RUNNING"), # first poll — ok + _http_error(401), # second poll — token expired + _op_response("COMPLETED"), # retry of second poll — success + ] + + client = _mock_client("refreshed_token") + Context.client = client + + stream = _make_stream() + result = stream.poll_bulk_completion( + current_bookmark=_BOOKMARK, + bulk_op_id=_BULK_OP_ID, + ) + + client.refresh_token.assert_called_once() + client.reinitialize_session.assert_called_once() + self.assertEqual(Context.config["access_token"], "refreshed_token") + self.assertEqual(result, _RESULT_URL) + self.assertEqual(mock_graphql.return_value.execute.call_count, 3) + mock_time.sleep.assert_called_once_with(60) + + @patch("tap_shopify.streams.orders.time") + @patch("shopify.GraphQL") + def test_401_without_client_raises_shopify_api_error(self, mock_graphql, mock_time): + """ + If Context.client is None (non-OAuth / api_key flow) and a 401 is + received, ShopifyAPIError is raised with a descriptive message. + """ + mock_time.time.side_effect = [0, 1] + mock_graphql.return_value.execute.side_effect = _http_error(401) + Context.client = None # no client available + + stream = _make_stream() + with self.assertRaises(ShopifyAPIError) as ctx: + stream.poll_bulk_completion( + current_bookmark=_BOOKMARK, + bulk_op_id=_BULK_OP_ID, + ) + + self.assertIn("no client is available", str(ctx.exception)) + + @patch("tap_shopify.streams.orders.time") + @patch("shopify.GraphQL") + def test_401_persists_after_retry_propagates_http_error(self, mock_graphql, mock_time): + """ + If the retry after token refresh also returns 401, the HTTPError + propagates to the caller — no infinite retry loop. + """ + mock_time.time.side_effect = [0, 1] + mock_graphql.return_value.execute.side_effect = [ + _http_error(401), # initial call + _http_error(401), # retry — still 401 + ] + + client = _mock_client("refreshed_token") + Context.client = client + + stream = _make_stream() + with self.assertRaises(urllib.error.HTTPError) as ctx: + stream.poll_bulk_completion( + current_bookmark=_BOOKMARK, + bulk_op_id=_BULK_OP_ID, + ) + + self.assertEqual(ctx.exception.code, 401) + # Refresh was still attempted once + client.refresh_token.assert_called_once() + client.reinitialize_session.assert_called_once() + # execute called twice: original + one retry + self.assertEqual(mock_graphql.return_value.execute.call_count, 2) + + # ------------------------------------------------------------------ + # Non-401 HTTP errors + # ------------------------------------------------------------------ + + @patch("tap_shopify.streams.orders.time") + @patch("shopify.GraphQL") + def test_non_401_http_error_propagates_without_token_refresh(self, mock_graphql, mock_time): + """HTTP errors other than 401 (e.g. 500) are re-raised without any token refresh.""" + mock_time.time.side_effect = [0, 1] + mock_graphql.return_value.execute.side_effect = _http_error(500) + + client = _mock_client() + Context.client = client + + stream = _make_stream() + with self.assertRaises(urllib.error.HTTPError) as ctx: + stream.poll_bulk_completion( + current_bookmark=_BOOKMARK, + bulk_op_id=_BULK_OP_ID, + ) + + self.assertEqual(ctx.exception.code, 500) + client.refresh_token.assert_not_called() + client.reinitialize_session.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unittests/test_schema_validation.py b/tests/unittests/test_schema_validation.py new file mode 100644 index 00000000..ab5384d6 --- /dev/null +++ b/tests/unittests/test_schema_validation.py @@ -0,0 +1,457 @@ +import unittest +from unittest.mock import patch +import json +from graphql import parse, FieldNode, SelectionSetNode +from pathlib import Path +from tap_shopify.streams.orders import Orders +from tap_shopify.streams.transactions import Transactions +from tap_shopify.streams.inventory_levels import InventoryLevels +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream +from textwrap import dedent + +SCHEMA_PATH = Path("tap_shopify/schemas/orders.json") + +def load_schema_fields(schema, prefix=""): + """Recursively extract schema field paths.""" + fields = set() + for key, value in schema.get("properties", {}).items(): + full_key = f"{prefix}.{key}" if prefix else key + fields.add(full_key) + return fields + + +def extract_top_level_fields_from_node(selection_set: SelectionSetNode, path=["orders", "edges", "node"]): + """Extract top-level field names from a selection set under the given field path.""" + + def find_selection(selections, name): + for sel in selections: + if isinstance(sel, FieldNode) and sel.name.value == name: + return sel + return None + + current_set = selection_set + for key in path: + node = find_selection(current_set.selections, key) + if node is None or node.selection_set is None: + return set() + current_set = node.selection_set + + return { + field.name.value + for field in current_set.selections + if isinstance(field, FieldNode) + } + +class TestGraphQLSchemaMatch(unittest.TestCase): + def test_schema_matches_graphql_query(self): + # Load and parse schema + schema = json.loads(SCHEMA_PATH.read_text()) + schema_fields = load_schema_fields(schema) + + # Load and parse GraphQL query + query_str = Orders().get_query() + ast = parse(query_str) + query_fields = set() + + for definition in ast.definitions: + if hasattr(definition, "selection_set") and isinstance(definition.selection_set, SelectionSetNode): + query_fields.update(extract_top_level_fields_from_node(definition.selection_set)) + + # Compare fields + missing_in_query = schema_fields - query_fields + missing_in_schema = query_fields - schema_fields + + self.assertFalse(missing_in_query, f"Schema fields missing in query: {missing_in_query}") + self.assertFalse(missing_in_schema, f"Query fields missing in schema: {missing_in_schema}") + + @patch('singer.metadata.to_map') + @patch.object(Context, 'get_catalog_entry') + def test_get_unselected_fields(self, mock_get_catalog_entry, mock_to_map): + # Mock schema + mock_schema = { + "properties": { + "id": {"type": "string"}, + "name": {"type": "string"}, + "email": {"type": "string"}, + "created_at": {"type": "string"}, + } + } + + # Mock metadata (breadcrumb: metadata dict) + mock_metadata_map = { + (): {"inclusion": "available"}, # Root - should be skipped + ("properties", "id"): {"selected": True}, + ("properties", "name"): {"inclusion": "automatic"}, + ("properties", "email"): {"selected": False}, # Not selected + ("properties", "created_at"): {"selected": False}, # Not selected + } + + # Mock return values + mock_get_catalog_entry.return_value = { + "schema": mock_schema, + "metadata": "dummy_metadata" + } + mock_to_map.return_value = mock_metadata_map + + # Call the method + result = Context.get_unselected_fields("test_stream") + + # Expecting unselected fields + expected = ["email", "created_at"] + self.assertCountEqual(result, expected) + + def setUp(self): + class TestClass(Stream): + data_key = "products" + + def get_query(self): + return dedent(""" + query GetProducts($first: Int!, $after: String, $query: String) { + products(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { + edges { + node { + id + title + descriptionHtml + vendor + category { + id + } + tags + handle + publishedAt + createdAt + updatedAt + templateSuffix + status + productType + giftCardTemplateSuffix + hasOnlyDefaultVariant + hasOutOfStockVariants + hasVariantsThatRequiresComponents + isGiftCard + description + requiresSellingPlan + totalInventory + media(first: 250) { + edges { + node { + id + alt + status + mediaContentType + mediaWarnings { + code + message + } + mediaErrors { + code + details + message + } + ... on ExternalVideo { + id + embedUrl + } + ... on MediaImage { + id + updatedAt + createdAt + mimeType + image { + url + width + height + id + } + } + ... on Model3d { + id + filename + sources { + url + format + mimeType + filesize + } + } + ... on Video { + id + updatedAt + createdAt + filename + sources { + url + format + mimeType + fileSize + } + } + } + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + """) + self.instance = TestClass() + + def test_remove_fields_from_query(self): + result = self.instance.remove_fields_from_query(['requiresSellingPlan', 'totalInventory', 'media']) + + expected_query = dedent(""" + query GetProducts($first: Int!, $after: String, $query: String) { + products(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { + edges { + node { + id + title + descriptionHtml + vendor + category { + id + } + tags + handle + publishedAt + createdAt + updatedAt + templateSuffix + status + productType + giftCardTemplateSuffix + hasOnlyDefaultVariant + hasOutOfStockVariants + hasVariantsThatRequiresComponents + isGiftCard + description + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + """).strip() + + # Normalize whitespace for comparison + self.assertEqual( + ''.join(result.split()), + ''.join(expected_query.split()) + ) + + def test_remove_fields_preserves_nested_same_name_fields(self): + """Pruning a top-level field must not remove a same-named field that + appears inside a nested connection (e.g. lineItems.edges.node). + + This is the regression guard for the original customAttributes bug: + the top-level ``customAttributes`` field on the order record and the + ``customAttributes`` field inside ``lineItems.edges.node`` share a + name. If only the order-level one is unselected it must be removed + from ``orders.edges.node`` but left intact inside lineItems. + """ + class OrderLikeStream(Stream): + data_key = "orders" + + def get_query(self): + return dedent(""" + query GetOrders($first: Int!) { + orders(first: $first) { + edges { + node { + id + customAttributes { + key + value + } + lineItems { + edges { + node { + id + customAttributes { + key + value + } + } + } + } + } + } + } + } + """) + + stream = OrderLikeStream() + result = stream.remove_fields_from_query(["customAttributes"]) + + # Top-level customAttributes should be gone; lineItems and its nested + # customAttributes should remain intact. + parsed = parse(result) + + def get_selection_names(selection_set): + return {s.name.value for s in selection_set.selections if isinstance(s, FieldNode)} + + # Navigate: orders → edges → node + orders_node = parsed.definitions[0].selection_set.selections[0] + edges_node = orders_node.selection_set.selections[0] + node_fields = edges_node.selection_set.selections[0] + top_level_names = get_selection_names(node_fields.selection_set) + + self.assertNotIn( + "customAttributes", top_level_names, + "Top-level customAttributes should be pruned from orders.edges.node" + ) + self.assertIn( + "lineItems", top_level_names, + "lineItems should still be present after pruning" + ) + + # Navigate into lineItems → edges → node + line_items_field = next( + s for s in node_fields.selection_set.selections + if isinstance(s, FieldNode) and s.name.value == "lineItems" + ) + line_item_edges = line_items_field.selection_set.selections[0] + line_item_node = line_item_edges.selection_set.selections[0] + line_item_names = get_selection_names(line_item_node.selection_set) + + self.assertIn( + "customAttributes", line_item_names, + "customAttributes inside lineItems.edges.node must NOT be pruned" + ) + + +class TestRemoveFieldsTransactions(unittest.TestCase): + """remove_fields_from_query correctly prunes fields from the + ``transactions { }`` direct-list pattern used by the Transactions stream + (orders.edges.node.transactions { FIELDS }, no nested edges/node wrapper). + """ + + def setUp(self): + self.stream = Transactions() + + def _parse_transaction_fields(self, query_str): + """Return the set of field names directly inside ``transactions { }``.""" + parsed = parse(query_str) + op = parsed.definitions[0] + # orders → edges → node → transactions + orders = op.selection_set.selections[0] # orders + edges = orders.selection_set.selections[0] # edges + node = edges.selection_set.selections[0] # node + transactions = next( + s for s in node.selection_set.selections + if isinstance(s, FieldNode) and s.name.value == "transactions" + ) + return { + s.name.value + for s in transactions.selection_set.selections + if isinstance(s, FieldNode) + } + + def test_unselected_fields_pruned_from_transactions(self): + """Fields that are unselected should be removed from transactions { }.""" + result = self.stream.remove_fields_from_query(["gateway", "errorCode"]) + remaining = self._parse_transaction_fields(result) + + self.assertNotIn("gateway", remaining) + self.assertNotIn("errorCode", remaining) + + def test_selected_fields_kept_in_transactions(self): + """Fields that ARE selected must remain in transactions { }.""" + result = self.stream.remove_fields_from_query(["gateway"]) + remaining = self._parse_transaction_fields(result) + + self.assertIn("id", remaining) + self.assertIn("status", remaining) + self.assertIn("createdAt", remaining) + + def test_empty_fields_to_remove_leaves_query_intact(self): + """Passing an empty list must not alter the transactions fields.""" + original_fields = self._parse_transaction_fields(self.stream.get_query()) + result = self.stream.remove_fields_from_query([]) + pruned_fields = self._parse_transaction_fields(result) + + self.assertEqual(original_fields, pruned_fields) + + +class TestRemoveFieldsInventoryLevels(unittest.TestCase): + """remove_fields_from_query correctly targets the inner + ``inventoryLevels.edges.node { FIELDS }`` selection set and does NOT + prune from the outer ``locations.edges.node`` selection set. + """ + + def setUp(self): + self.stream = InventoryLevels() + + def _parse_outer_location_fields(self, query_str): + """Return field names directly inside ``locations.edges.node { }``.""" + parsed = parse(query_str) + op = parsed.definitions[0] + locations = op.selection_set.selections[0] # locations + edges = locations.selection_set.selections[0] # edges + node = edges.selection_set.selections[0] # outer node + return { + s.name.value + for s in node.selection_set.selections + if isinstance(s, FieldNode) + } + + def _parse_inner_inventory_level_fields(self, query_str): + """Return field names directly inside ``inventoryLevels.edges.node { }``.""" + parsed = parse(query_str) + op = parsed.definitions[0] + locations = op.selection_set.selections[0] + edges = locations.selection_set.selections[0] + outer_node = edges.selection_set.selections[0] + inventory_levels = next( + s for s in outer_node.selection_set.selections + if isinstance(s, FieldNode) and s.name.value == "inventoryLevels" + ) + inner_edges = next( + s for s in inventory_levels.selection_set.selections + if isinstance(s, FieldNode) and s.name.value == "edges" + ) + inner_node = inner_edges.selection_set.selections[0] + return { + s.name.value + for s in inner_node.selection_set.selections + if isinstance(s, FieldNode) + } + + def test_unselected_fields_pruned_from_inner_node(self): + """Unselected inventory-level fields are removed from the inner node.""" + result = self.stream.remove_fields_from_query(["canDeactivate", "deactivationAlert"]) + inner_fields = self._parse_inner_inventory_level_fields(result) + + self.assertNotIn("canDeactivate", inner_fields) + self.assertNotIn("deactivationAlert", inner_fields) + + def test_outer_location_node_is_not_pruned(self): + """The outer locations.edges.node must never be touched by pruning.""" + original_outer = self._parse_outer_location_fields(self.stream.get_query()) + result = self.stream.remove_fields_from_query(["canDeactivate", "id", "updatedAt"]) + pruned_outer = self._parse_outer_location_fields(result) + + # The outer node must be unchanged regardless of what fields_to_remove contains. + self.assertEqual( + original_outer, pruned_outer, + "Pruning inventory-level schema fields must not affect the outer " + "locations.edges.node selection set" + ) + + def test_selected_fields_kept_in_inner_node(self): + """Fields that ARE selected must remain in the inner inventory-level node.""" + result = self.stream.remove_fields_from_query(["canDeactivate"]) + inner_fields = self._parse_inner_inventory_level_fields(result) + + self.assertIn("id", inner_fields) + self.assertIn("updatedAt", inner_fields) + self.assertIn("item", inner_fields) + diff --git a/tests/unittests/test_timeout.py b/tests/unittests/test_timeout.py deleted file mode 100644 index c40c0863..00000000 --- a/tests/unittests/test_timeout.py +++ /dev/null @@ -1,576 +0,0 @@ -import socket -import tap_shopify -from unittest import mock -import pyactiveresource -import shopify -from tap_shopify.context import Context -from tap_shopify.streams.base import get_request_timeout -from tap_shopify.streams.inventory_items import InventoryItems -from tap_shopify.streams.inventory_levels import InventoryLevels -from tap_shopify.streams.locations import Locations -from tap_shopify.streams.order_refunds import OrderRefunds -from tap_shopify.streams.transactions import Transactions -from tap_shopify.streams.abandoned_checkouts import AbandonedCheckouts -from tap_shopify.streams.metafields import Metafields -from tap_shopify.streams.metafields import get_metafields -import unittest - -class TestTimeoutValue(unittest.TestCase): - """ - Verify the timeout value is set as expected by the tap - """ - - def test_timeout_value_not_passed_in_config(self): - """ - Test case to verify that the default value is used when we do not pass request timeout value from config - """ - # initialize config - Context.config = { - "start_date": "2021-01-01", - "api_key": "test_api_key", - "shop": "test_shop", - "results_per_page": 50 - } - - # initialize base class - timeout = get_request_timeout() - # verify the timeout is set as expected - self.assertEquals(timeout, 300) - - def test_timeout_int_value_passed_in_config(self): - """ - Test case to verify that the value we passed on config is set as request timeout value - """ - # initialize config - Context.config = { - "start_date": "2021-01-01", - "api_key": "test_api_key", - "shop": "test_shop", - "results_per_page": 50, - "request_timeout": 100 - } - - # initialize base class - timeout = get_request_timeout() - # verify the timeout is set as expected - self.assertEquals(timeout, 100) - - def test_timeout_string_value_passed_in_config(self): - """ - Test case to verify that the value we passed on config is set as request timeout value - """ - # initialize config - Context.config = { - "start_date": "2021-01-01", - "api_key": "test_api_key", - "shop": "test_shop", - "results_per_page": 50, - "request_timeout": "100" - } - - # initialize base class - timeout = get_request_timeout() - # verify the timeout is set as expected - self.assertEquals(timeout, 100) - - def test_timeout_empty_value_passed_in_config(self): - """ - Test case to verify that the default value is used when pass empty request timeout value from config - """ - # initialize config - Context.config = { - "start_date": "2021-01-01", - "api_key": "test_api_key", - "shop": "test_shop", - "results_per_page": 50, - "request_timeout": "" - } - - # initialize base class - timeout = get_request_timeout() - # verify the timeout is set as expected - self.assertEquals(timeout, 300) - - def test_timeout_0_value_passed_in_config(self): - """ - Test case to verify that the default value is used when pass 0 as request timeout value from config - """ - # initialize config - Context.config = { - "start_date": "2021-01-01", - "api_key": "test_api_key", - "shop": "test_shop", - "results_per_page": 50, - "request_timeout": 0.0 - } - - # initialize base class - timeout = get_request_timeout() - # verify the timeout is set as expected - self.assertEquals(timeout, 300) - - def test_timeout_string_0_value_passed_in_config(self): - """ - Test case to verify that the default value is used when pass string 0 as request timeout value from config - """ - # initialize config - Context.config = { - "start_date": "2021-01-01", - "api_key": "test_api_key", - "shop": "test_shop", - "results_per_page": 50, - "request_timeout": "0.0" - } - - # initialize base class - timeout = get_request_timeout() - # verify the timeout is set as expected - self.assertEquals(timeout, 300) - - @mock.patch("shopify.Shop.set_timeout") - @mock.patch("shopify.Shop.current") - def test_timeout_value_not_passed_in_config__initialize_shopify_client(self, mocked_current, mocked_set_timeout): - """ - Test case to verify that the default value is used when we do not pass request timeout value from config - """ - # initialize config - Context.config = { - "start_date": "2021-01-01", - "api_key": "test_api_key", - "shop": "test_shop", - "results_per_page": 50 - } - - # function call - tap_shopify.initialize_shopify_client() - # verify the timeout is set as expected - mocked_set_timeout.assert_called_with(300) - - @mock.patch("shopify.Shop.set_timeout") - @mock.patch("shopify.Shop.current") - def test_timeout_int_value_passed_in_config__initialize_shopify_client(self, mocked_current, mocked_set_timeout): - """ - Test case to verify that the value we passed on config is set as request timeout value - """ - # initialize config - Context.config = { - "start_date": "2021-01-01", - "api_key": "test_api_key", - "shop": "test_shop", - "results_per_page": 50, - "request_timeout": 100 - } - - # function call - tap_shopify.initialize_shopify_client() - # verify the timeout is set as expected - mocked_set_timeout.assert_called_with(100) - - @mock.patch("shopify.Shop.set_timeout") - @mock.patch("shopify.Shop.current") - def test_timeout_string_value_passed_in_config__initialize_shopify_client(self, mocked_current, mocked_set_timeout): - """ - Test case to verify that the value we passed on config is set as request timeout value - """ - # initialize config - Context.config = { - "start_date": "2021-01-01", - "api_key": "test_api_key", - "shop": "test_shop", - "results_per_page": 50, - "request_timeout": "100" - } - - # function call - tap_shopify.initialize_shopify_client() - # verify the timeout is set as expected - mocked_set_timeout.assert_called_with(100) - - @mock.patch("shopify.Shop.set_timeout") - @mock.patch("shopify.Shop.current") - def test_timeout_empty_value_passed_in_config__initialize_shopify_client(self, mocked_current, mocked_set_timeout): - """ - Test case to verify that the default value is used when pass empty request timeout value from config - """ - # initialize config - Context.config = { - "start_date": "2021-01-01", - "api_key": "test_api_key", - "shop": "test_shop", - "results_per_page": 50, - "request_timeout": "" - } - - # function call - tap_shopify.initialize_shopify_client() - # verify the timeout is set as expected - mocked_set_timeout.assert_called_with(300) - - @mock.patch("shopify.Shop.set_timeout") - @mock.patch("shopify.Shop.current") - def test_timeout_0_value_passed_in_config__initialize_shopify_client(self, mocked_current, mocked_set_timeout): - """ - Test case to verify that the default value is used when pass 0 as request timeout value from config - """ - # initialize config - Context.config = { - "start_date": "2021-01-01", - "api_key": "test_api_key", - "shop": "test_shop", - "results_per_page": 50, - "request_timeout": 0.0 - } - - # function call - tap_shopify.initialize_shopify_client() - # verify the timeout is set as expected - mocked_set_timeout.assert_called_with(300) - - @mock.patch("shopify.Shop.set_timeout") - @mock.patch("shopify.Shop.current") - def test_timeout_string_0_value_passed_in_config__initialize_shopify_client(self, mocked_current, mocked_set_timeout): - """ - Test case to verify that the default value is used when pass string 0 as request timeout value from config - """ - # initialize config - Context.config = { - "start_date": "2021-01-01", - "api_key": "test_api_key", - "shop": "test_shop", - "results_per_page": 50, - "request_timeout": "0.0" - } - - # function call - tap_shopify.initialize_shopify_client() - # verify the timeout is set as expected - mocked_set_timeout.assert_called_with(300) - -class TestTimeoutBackoff(unittest.TestCase): - """ - Verify the tap backoff for 5 times when timeout error occurs - """ - - @mock.patch("time.sleep") - @mock.patch("shopify.Checkout.find") - def test_AbandonedCheckouts_pyactiveresource_error_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'pyactiveresource.connection.Error' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = pyactiveresource.connection.Error('urlopen error _ssl.c:1074: The handshake operation timed out') - - # initialize 'AbandonedCheckouts' as it calls the function 'call_api' from the base class - abandoned_checkouts = AbandonedCheckouts() - try: - # function call - abandoned_checkouts.call_api({}) - except pyactiveresource.connection.Error: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) - - @mock.patch("time.sleep") - @mock.patch("shopify.InventoryItem.find") - def test_InventoryItems_pyactiveresource_error_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'pyactiveresource.connection.Error' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = pyactiveresource.connection.Error('urlopen error _ssl.c:1074: The handshake operation timed out') - - # initialize class - inventory_items = InventoryItems() - try: - # function call - inventory_items.get_inventory_items([1, 2, 3]) - except pyactiveresource.connection.Error: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) - - @mock.patch("time.sleep") - @mock.patch("pyactiveresource.activeresource.ActiveResource.find") - def test_InventoryLevels_pyactiveresource_error_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'pyactiveresource.connection.Error' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = pyactiveresource.connection.Error('urlopen error _ssl.c:1074: The handshake operation timed out') - - # initialize class - inventory_levels = InventoryLevels() - try: - # function call - inventory_levels.api_call_for_inventory_levels(1, 'test') - except pyactiveresource.connection.Error: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) - - @mock.patch("time.sleep") - @mock.patch("pyactiveresource.activeresource.ActiveResource.find") - def test_Locations_pyactiveresource_error_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'pyactiveresource.connection.Error' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = pyactiveresource.connection.Error('urlopen error _ssl.c:1074: The handshake operation timed out') - - # initialize class - locations = Locations() - try: - # function call - locations.replication_object.find() - except pyactiveresource.connection.Error: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) - - @mock.patch("time.sleep") - @mock.patch("shopify.Order.metafields") - def test_Metafields_pyactiveresource_error_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'pyactiveresource.connection.Error' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = pyactiveresource.connection.Error('urlopen error _ssl.c:1074: The handshake operation timed out') - - try: - # function call - get_metafields(shopify.Order, 1, shopify.Order, 100) - except pyactiveresource.connection.Error: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) - - @mock.patch("time.sleep") - @mock.patch("shopify.Refund.find") - def test_OrderRefunds_pyactiveresource_error_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'pyactiveresource.connection.Error' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = pyactiveresource.connection.Error('urlopen error _ssl.c:1074: The handshake operation timed out') - - # initialize class - order_refunds = OrderRefunds() - try: - # function call - order_refunds.get_refunds(shopify.Product, 1) - except pyactiveresource.connection.Error: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) - - @mock.patch("time.sleep") - @mock.patch("pyactiveresource.activeresource.ActiveResource.find") - def test_Transactions_pyactiveresource_error_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'pyactiveresource.connection.Error' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = pyactiveresource.connection.Error('urlopen error _ssl.c:1074: The handshake operation timed out') - - # initialize class - locations = Transactions() - try: - # function call - locations.replication_object.find() - except pyactiveresource.connection.Error: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) - - @mock.patch("time.sleep") - @mock.patch("shopify.Shop.current") - def test_Shop_pyactiveresource_error_timeout_backoff(self, mocked_current, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'pyactiveresource.connection.Error' error occurs - """ - # mock 'Shop' call and raise timeout error - mocked_current.side_effect = pyactiveresource.connection.Error('urlopen error _ssl.c:1074: The handshake operation timed out') - - Context.config = { - "api_key": "test_api_key", - "shop": "test_shop" - } - try: - # function call - tap_shopify.initialize_shopify_client() - except pyactiveresource.connection.Error: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_current.call_count, 5) - - """ - Verify the tap backoff for 5 times when timeout error occurs - """ - - @mock.patch("time.sleep") - @mock.patch("shopify.Checkout.find") - def test_AbandonedCheckouts_socket_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'socket.timeout' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = socket.timeout("The read operation timed out") - - # initialize 'AbandonedCheckouts' as it calls the function 'call_api' from the base class - abandoned_checkouts = AbandonedCheckouts() - try: - # function call - abandoned_checkouts.call_api({}) - except socket.timeout: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) - - @mock.patch("time.sleep") - @mock.patch("shopify.InventoryItem.find") - def test_InventoryItems_socket_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'socket.timeout' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = socket.timeout("The read operation timed out") - - # initialize class - inventory_items = InventoryItems() - try: - # function call - inventory_items.get_inventory_items([1, 2, 3]) - except socket.timeout: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) - - @mock.patch("time.sleep") - @mock.patch("pyactiveresource.activeresource.ActiveResource.find") - def test_InventoryLevels_socket_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'socket.timeout' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = socket.timeout("The read operation timed out") - - # initialize class - inventory_levels = InventoryLevels() - try: - # function call - inventory_levels.api_call_for_inventory_levels(1, 'test') - except socket.timeout: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) - - @mock.patch("time.sleep") - @mock.patch("pyactiveresource.activeresource.ActiveResource.find") - def test_Locations_socket_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'socket.timeout' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = socket.timeout("The read operation timed out") - - # initialize class - locations = Locations() - try: - # function call - locations.replication_object.find() - except socket.timeout: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) - - @mock.patch("time.sleep") - @mock.patch("shopify.Order.metafields") - def test_Metafields_socket_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'socket.timeout' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = socket.timeout("The read operation timed out") - - try: - # function call - get_metafields(shopify.Order, 1, shopify.Order, 100) - except socket.timeout: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) - - @mock.patch("time.sleep") - @mock.patch("shopify.Refund.find") - def test_OrderRefunds_socket_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'socket.timeout' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = socket.timeout("The read operation timed out") - - # initialize class - order_refunds = OrderRefunds() - try: - # function call - order_refunds.get_refunds(shopify.Product, 1) - except socket.timeout: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) - - @mock.patch("time.sleep") - @mock.patch("pyactiveresource.activeresource.ActiveResource.find") - def test_Transactions_socket_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'socket.timeout' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = socket.timeout("The read operation timed out") - - # initialize class - locations = Transactions() - try: - # function call - locations.replication_object.find() - except socket.timeout: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) - - @mock.patch("time.sleep") - @mock.patch("shopify.Shop.current") - def test_Shop_socket_timeout_backoff(self, mocked_current, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'socket.timeout' error occurs - """ - # mock 'Shop' call and raise timeout error - mocked_current.side_effect = socket.timeout("The read operation timed out") - - Context.config = { - "api_key": "test_api_key", - "shop": "test_shop" - } - try: - # function call - tap_shopify.initialize_shopify_client() - except socket.timeout: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_current.call_count, 5) diff --git a/tests/unittests/test_token_check_in_discover.py b/tests/unittests/test_token_check_in_discover.py index 22617ba1..5bbbd1c2 100644 --- a/tests/unittests/test_token_check_in_discover.py +++ b/tests/unittests/test_token_check_in_discover.py @@ -11,6 +11,7 @@ def __init__(self): self.catalog = False self.config = {'api_key': 'test', 'shop': 'shop'} self.state = False + self.dev = False def resource_not_found_raiser(): raise pyactiveresource.connection.ResourceNotFound @@ -24,14 +25,16 @@ def connection_error_raiser(): @mock.patch('tap_shopify.utils.parse_args') @mock.patch('tap_shopify.discover', side_effect=tap_shopify.discover) @mock.patch("builtins.print") +@mock.patch("tap_shopify.has_read_users_access") class TestTokenInDiscoverMode(unittest.TestCase): @mock.patch('tap_shopify.initialize_shopify_client', side_effect=resource_not_found_raiser) - def test_resource_not_found(self, mocked_client, mocked_print, mocked_discover, mocked_args): + def test_resource_not_found(self, mocked_client, mocked_access, mocked_print, mocked_discover, mocked_args): ''' Verify exception is raised for ResourceNotFound with proper error message and test that discover mode is called ''' + mocked_access.return_value = True mocked_args.return_value = Args() try: tap_shopify.main() @@ -42,11 +45,12 @@ def test_resource_not_found(self, mocked_client, mocked_print, mocked_discover, self.assertEqual(mocked_print.call_count, 0) @mock.patch('tap_shopify.initialize_shopify_client', side_effect=unauthorized_access_raiser) - def test_unauthorized_access(self, mocked_client, mocked_print, mocked_discover, mocked_args): + def test_unauthorized_access(self, mocked_client, mocked_access, mocked_print, mocked_discover, mocked_args): ''' Verify exception is raised for UnauthorizedAccess with proper error message and test that discover mode is called ''' + mocked_access.return_value = True mocked_args.return_value = Args() try: tap_shopify.main() @@ -57,11 +61,12 @@ def test_unauthorized_access(self, mocked_client, mocked_print, mocked_discover, self.assertEqual(mocked_print.call_count, 0) @mock.patch('tap_shopify.initialize_shopify_client', side_effect=connection_error_raiser) - def test_connection_error(self, mocked_client, mocked_print, mocked_discover, mocked_args): + def test_connection_error(self, mocked_client, mocked_access, mocked_print, mocked_discover, mocked_args): ''' Verify exception is raised for ConnectionError with proper error message and test that discover mode is called ''' + mocked_access.return_value = True mocked_args.return_value = Args() try: tap_shopify.main() @@ -72,11 +77,12 @@ def test_connection_error(self, mocked_client, mocked_print, mocked_discover, mo self.assertEqual(mocked_print.call_count, 0) @mock.patch('tap_shopify.initialize_shopify_client') - def test_no_error(self, mocked_client, mocked_print, mocked_discover, mocked_args): + def test_no_error(self, mocked_client, mocked_access, mocked_print, mocked_discover, mocked_args): ''' Verify that if no error during discover then print should be called once for writing catalog ''' + mocked_access.return_value = True mocked_args.return_value = Args() tap_shopify.main() self.assertEqual(mocked_discover.call_count, 1) diff --git a/tests/unittests/test_transaction_canonicalize.py b/tests/unittests/test_transaction_canonicalize.py deleted file mode 100644 index fb1ab5d3..00000000 --- a/tests/unittests/test_transaction_canonicalize.py +++ /dev/null @@ -1,40 +0,0 @@ -import unittest -from tap_shopify.streams.transactions import canonicalize - -class TestTransactionCanonicalize(unittest.TestCase): - def test_unmodified_if_not_present(self): - # Note: Canonicalize has a side effect with pop(), must copy test - # record to compare - record = {"receipt": {"foo": "bar"}, "id": 2} - expected_record = {"receipt": {"foo": "bar"}, "id": 2} - canonicalize(record, "token") - self.assertEqual(record, expected_record) - - def test_unmodified_if_only_lower_exists(self): - record = {"receipt": {"foo": "bar"}, "id": 2} - expected_record = {"receipt": {"foo": "bar"}, "id": 2} - canonicalize(record, "foo") - self.assertEqual(record, expected_record) - - def test_lowercases_if_capital_only_exists(self): - record = {"receipt": {"Foo": "bar"}, "id": 2} - expected_record = {"receipt": {"foo": "bar"}, "id": 2} - canonicalize(record, "foo") - self.assertEqual(record, expected_record) - - def test_null_receipt_record(self): - record = {"receipt": None} - expected_record = {"receipt": None} - canonicalize(record, "foo") - self.assertEqual(record, expected_record) - - def test_removes_uppercase_if_both_exist_and_are_equal(self): - record = {"receipt": {"Foo": "bar", "foo": "bar"}, "id": 2} - expected_record = {"receipt": {"foo": "bar"}, "id": 2} - canonicalize(record, "foo") - self.assertEqual(record, expected_record) - - def test_throws_if_both_exist_and_are_not_equal(self): - record = {"receipt": {"Foo": "bark", "foo": "bar"}, "id": 2} - with self.assertRaises(ValueError): - canonicalize(record, "foo")