From 96ff8fbe8f5ed838c844d53d2141285d5482feae Mon Sep 17 00:00:00 2001 From: manand31 <92758907+manand31@users.noreply.github.com> Date: Thu, 3 Feb 2022 08:59:48 -0500 Subject: [PATCH 01/66] handled failure on bookmarks updated test (#136) --- tests/test_bookmarks_updated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_bookmarks_updated.py b/tests/test_bookmarks_updated.py index c39ce1a3..abb9dee1 100644 --- a/tests/test_bookmarks_updated.py +++ b/tests/test_bookmarks_updated.py @@ -72,7 +72,7 @@ def bookmarks_test(self, conn_id, testable_streams): #simulated_states = self.calculated_states_by_stream(first_sync_bookmark) # We are hardcoding the updated state to ensure that we get atleast 1 record in second sync. These values have been provided after reviewing the max bookmark value for each of the streams - simulated_states = {'products': {'updated_at': '2021-12-20T05:10:05.000000Z'}, 'collects': {'updated_at': '2021-09-01T09:08:28.000000Z'}, 'abandoned_checkouts': {'updated_at': '2021-10-28T12:43:14.000000Z'}, 'inventory_levels': {'updated_at': '2021-12-20T05:09:34.000000Z'}, 'locations': {'updated_at': '2021-07-20T09:00:22.000000Z'}, 'events': {'created_at': '2021-12-20T05:09:01.000000Z'}, 'inventory_items': {'updated_at': '2021-09-15T19:44:11.000000Z'}, 'transactions': {'created_at': '2021-12-20T00:08:52-05:00'}, 'metafields': {'updated_at': '2021-09-07T21:18:05.000000Z'}, 'order_refunds': {'created_at': '2021-05-01T17:41:18.000000Z'}, 'customers': {'updated_at': '2021-12-20T05:08:17.000000Z'}, 'orders': {'updated_at': '2021-12-20T05:09:01.000000Z'}, 'custom_collections': {'updated_at': '2021-12-20T17:41:18.000000Z'}} + simulated_states = {'products': {'updated_at': '2021-12-20T05:10:05.000000Z'}, 'collects': {'updated_at': '2021-09-01T09:08:28.000000Z'}, 'abandoned_checkouts': {'updated_at': '2022-02-02T16:00:00.000000Z'}, 'inventory_levels': {'updated_at': '2021-12-20T05:09:34.000000Z'}, 'locations': {'updated_at': '2021-07-20T09:00:22.000000Z'}, 'events': {'created_at': '2021-12-20T05:09:01.000000Z'}, 'inventory_items': {'updated_at': '2021-09-15T19:44:11.000000Z'}, 'transactions': {'created_at': '2021-12-20T00:08:52-05:00'}, 'metafields': {'updated_at': '2021-09-07T21:18:05.000000Z'}, 'order_refunds': {'created_at': '2021-05-01T17:41:18.000000Z'}, 'customers': {'updated_at': '2021-12-20T05:08:17.000000Z'}, 'orders': {'updated_at': '2021-12-20T05:09:01.000000Z'}, 'custom_collections': {'updated_at': '2021-12-20T17:41:18.000000Z'}} for stream, updated_state in simulated_states.items(): new_state['bookmarks'][stream] = updated_state From efe43ae94220efc27f7c5fbe68cb728f4c0e4f22 Mon Sep 17 00:00:00 2001 From: KrisPersonal <66801357+KrisPersonal@users.noreply.github.com> Date: Fri, 15 Apr 2022 06:38:58 -0700 Subject: [PATCH 02/66] API-SDK version change and Fields addition in the schema. (#146) (#147) * TDL-17513: Upgrade API and SDK version (#135) * added missing test cases * updated sdk version and added new fields * removed abandoned_checkouts from all test cases except discovery * updated all fields test to remove value_type from metafields stream * updated api and sdk version * added comment for abandoned checkouts in test cases * added value_type field in metafield containing type value * resolved pylint error * TDL-18045: Fields Addition in the Schema (#140) * added fields in the schema * updated schema of order_refunds * updated arguments field schema * removed unnecessary space in the schema Co-authored-by: Harsh <80324346+harshpatel4crest@users.noreply.github.com> --- setup.py | 2 +- tap_shopify/__init__.py | 5 +- tap_shopify/schemas/abandoned_checkouts.json | 19 + tap_shopify/schemas/definitions.json | 258 +++++++++ tap_shopify/schemas/events.json | 23 + tap_shopify/schemas/metafields.json | 6 + tap_shopify/schemas/order_refunds.json | 524 ++++++++++++++++++- tap_shopify/schemas/orders.json | 169 ++++++ tap_shopify/schemas/products.json | 6 + tap_shopify/schemas/transactions.json | 26 + tap_shopify/streams/metafields.py | 10 +- tests/base.py | 14 +- tests/test_all_fields.py | 100 ++++ tests/test_automatic_fields.py | 17 +- tests/test_bookmarks_updated.py | 20 +- tests/test_discovery.py | 17 +- tests/test_start_date.py | 59 ++- 17 files changed, 1245 insertions(+), 30 deletions(-) create mode 100644 tests/test_all_fields.py diff --git a/setup.py b/setup.py index ac1458ad..69241d2f 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ python_requires='>=3.5.2', py_modules=["tap_shopify"], install_requires=[ - "ShopifyAPI==8.4.1", + "ShopifyAPI==10.0.0", "singer-python==5.12.1", ], extras_require={ diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index e0299bcb..c593e5f7 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -25,7 +25,7 @@ def initialize_shopify_client(): api_key = Context.config['api_key'] shop = Context.config['shop'] - version = '2021-04' + version = '2022-01' session = shopify.Session(shop, version, api_key) shopify.ShopifyResource.activate_session(session) @@ -167,7 +167,8 @@ def sync(): Context.state['bookmarks'] = {} Context.state['bookmarks']['currently_sync_stream'] = stream_id - with Transformer() as transformer: + # some fields have epoch-time as date, hence transform into UTC date + with Transformer(singer.UNIX_SECONDS_INTEGER_DATETIME_PARSING) as transformer: for rec in stream.sync(): extraction_time = singer.utils.now() record_schema = catalog_entry['schema'] diff --git a/tap_shopify/schemas/abandoned_checkouts.json b/tap_shopify/schemas/abandoned_checkouts.json index ee59cfdd..d1d0e52d 100644 --- a/tap_shopify/schemas/abandoned_checkouts.json +++ b/tap_shopify/schemas/abandoned_checkouts.json @@ -591,6 +591,25 @@ "null", "boolean" ] + }, + "buyer_accepts_sms_marketing": { + "type": [ + "null", + "boolean" + ] + }, + "sms_marketing_phone": { + "type": [ + "null", + "string" + ] + }, + "total_duties": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" } } } diff --git a/tap_shopify/schemas/definitions.json b/tap_shopify/schemas/definitions.json index f2c8fe13..e5ed7986 100644 --- a/tap_shopify/schemas/definitions.json +++ b/tap_shopify/schemas/definitions.json @@ -2,6 +2,18 @@ "order_adjustments": { "items": { "properties": { + "amount_set": { + "type": [ + "null", + "object" + ] + }, + "tax_amount_set": { + "type": [ + "null", + "object" + ] + }, "order_id": { "type": [ "null", @@ -424,6 +436,57 @@ "string" ], "format": "date-time" + }, + "tax_exemptions": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "string" + ] + } + }, + "marketing_opt_in_level": { + "type": [ + "null", + "string" + ] + }, + "sms_marketing_consent": { + "type": [ + "null", + "object" + ], + "properties": { + "state": { + "type": [ + "null", + "string" + ] + }, + "opt_in_level": { + "type": [ + "null", + "string" + ] + }, + "consent_updated_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "consent_collected_from": { + "type": [ + "null", + "string" + ] + } + } } } }, @@ -806,6 +869,189 @@ "null", "integer" ] + }, + "presentment_title": { + "type": [ + "null", + "string" + ] + }, + "presentment_variant_title": { + "type": [ + "null", + "string" + ] + }, + "user_id": { + "type": [ + "null", + "integer" + ] + }, + "variant_price": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "duties": {}, + "rank": { + "type": [ + "null", + "string" + ] + }, + "unit_price_measurement": { + "type": [ + "null", + "object" + ], + "properties": { + "measured_type": { + "type": [ + "null", + "string" + ] + }, + "quantity_unit": { + "type": [ + "null", + "string" + ] + }, + "quantity_value": { + "type": [ + "null", + "string" + ] + }, + "reference_unit": { + "type": [ + "null", + "string" + ] + }, + "reference_value": { + "type": [ + "null", + "string" + ] + } + } + }, + "selling_plan_application": { + "type": [ + "null", + "object" + ], + "properties": { + "selling_plan_id": { + "type": [ + "null", + "integer" + ] + }, + "api_client_id": { + "type": [ + "null", + "integer" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "description": { + "type": [ + "null", + "string" + ] + }, + "pricing_policies": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "billing_interval": { + "type": [ + "null", + "string" + ] + }, + "billing_interval_count": { + "type": [ + "null", + "integer" + ] + }, + "billing_min_cycles": { + "type": [ + "null", + "string" + ] + }, + "billing_max_cycles": { + "type": [ + "null", + "integer" + ] + }, + "billing_anchors": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "delivery_interval": { + "type": [ + "null", + "string" + ] + }, + "delivery_interval_count": { + "type": [ + "null", + "integer" + ] + }, + "delivery_anchors": { + "type": [ + "null", + "array" + ], + "items": {} + }, + "delivery_cutoff_period": { + "type": [ + "null", + "string" + ] + }, + "delivery_pre_cutoff_behavior": { + "type": [ + "null", + "string" + ] + }, + "delivery_intent": { + "type": [ + "null", + "string" + ] + } + } + }, + "selling_plan_id": { + "type": [ + "null", + "integer" + ] } }, "type": [ @@ -946,6 +1192,18 @@ "null", "string" ] + }, + "identifier": { + "type": [ + "null", + "string" + ] + }, + "channel_liable": { + "type": [ + "null", + "boolean" + ] } }, "type": [ diff --git a/tap_shopify/schemas/events.json b/tap_shopify/schemas/events.json index fc52b5a2..e829fe3c 100644 --- a/tap_shopify/schemas/events.json +++ b/tap_shopify/schemas/events.json @@ -55,6 +55,29 @@ "null", "string" ] + }, + "description": { + "type": [ + "null", + "string" + ] + }, + "arguments": { + "anyOf": [ + { + "type": [ + "null", + "array" + ], + "items": {} + }, + { + "type": [ + "null", + "string" + ] + } + ] } } } diff --git a/tap_shopify/schemas/metafields.json b/tap_shopify/schemas/metafields.json index ba597731..6a2a73b6 100644 --- a/tap_shopify/schemas/metafields.json +++ b/tap_shopify/schemas/metafields.json @@ -70,6 +70,12 @@ "string" ], "format": "date-time" + }, + "type": { + "type": [ + "null", + "string" + ] } }, "type": "object" diff --git a/tap_shopify/schemas/order_refunds.json b/tap_shopify/schemas/order_refunds.json index d05a153d..6d4486d5 100644 --- a/tap_shopify/schemas/order_refunds.json +++ b/tap_shopify/schemas/order_refunds.json @@ -1,6 +1,12 @@ { "type": "object", "properties": { + "total_duties_set": { + "type": [ + "null", + "object" + ] + }, "order_id": { "type": [ "null", @@ -269,6 +275,12 @@ "null", "number" ] + }, + "channel_liable": { + "type": [ + "null", + "boolean" + ] } }, "type": [ @@ -640,7 +652,120 @@ "null", "string" ] - } + }, + "destination_location": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "integer" + ] + }, + "country_code": { + "type": [ + "null", + "string" + ] + }, + "province_code": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "address1": { + "type": [ + "null", + "string" + ] + }, + "address2": { + "type": [ + "null", + "string" + ] + }, + "city": { + "type": [ + "null", + "string" + ] + }, + "zip": { + "type": [ + "null", + "string" + ] + } + } + }, + "origin_location": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "integer" + ] + }, + "country_code": { + "type": [ + "null", + "string" + ] + }, + "province_code": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "address1": { + "type": [ + "null", + "string" + ] + }, + "address2": { + "type": [ + "null", + "string" + ] + }, + "city": { + "type": [ + "null", + "string" + ] + }, + "zip": { + "type": [ + "null", + "string" + ] + } + } + }, + "duties": {} }, "type": [ "null", @@ -665,6 +790,401 @@ "object" ] } - } + }, + "transactions": { + "type": ["null", "array"], + "items": { + "properties": { + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"], + "format": "singer.decimal" + }, + "authorization": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "error_code": { + "type": ["null", "string"] + }, + "device_id": { + "type": ["null", "integer"] + }, + "gateway": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "kind": { + "type": ["null", "string"] + }, + "location_id": { + "type": ["null", "integer"] + }, + "message": { + "type": ["null", "string"] + }, + "order_id": { + "type": ["null", "integer"] + }, + "parent_id": { + "type": ["null", "integer"] + }, + "payment_details": { + "properties": { + "cvv_result_code": { + "type": ["null", "string"] + }, + "credit_card_bin": { + "type": ["null", "string"] + }, + "credit_card_company": { + "type": ["null", "string"] + }, + "credit_card_number": { + "type": ["null", "string"] + }, + "avs_result_code": { + "type": ["null", "string"] + }, + "payments_refund_attributes": { + "type": ["null", "string"] + }, + "acquirer_reference_number": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "processed_at": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "test": { + "type": ["null", "boolean"] + }, + "source_name": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "receipt": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "balance_transaction": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "exchange_rate": { + "type": ["null", "string"] + } + } + }, + "charge": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "application_fee": { + "type": ["null", "string"] + }, + "balance_transaction": { + "type": ["null", "string"] + }, + "captured": { + "type": ["null", "boolean"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "currency": { + "type": ["null", "string"] + }, + "failure_code": { + "type": ["null", "string"] + }, + "failure_message": { + "type": ["null", "string"] + }, + "fraud_details": { + "type": [ + "null", + "object" + ] + }, + "id": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "mit_params": { + "type": [ + "null", + "object" + ] + }, + "object": { + "type": ["null", "string"] + }, + "outcome": { + "type": ["null", "object"], + "properties": { + "network_status": { + "type": ["null", "string"] + }, + "reason": { + "type": ["null", "string"] + }, + "risk_level": { + "type": ["null", "string"] + }, + "risk_score": { + "type": ["null", "integer"] + }, + "seller_message": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + }, + "metadata": { + "type": ["null", "object"], + "properties": { + "email": { + "type": ["null", "string"] + }, + "manual_entry": { + "type": ["null", "string"] + }, + "order_id": { + "type": ["null", "string"] + }, + "order_transaction_id": { + "type": ["null", "string"] + }, + "payments_charge_id": { + "type": ["null", "string"] + }, + "shop_id": { + "type": ["null", "string"] + }, + "shop_name": { + "type": ["null", "string"] + }, + "transaction_fee_total_amount": { + "type": ["null", "string"] + }, + "transaction_fee_tax_amount": { + "type": ["null", "string"] + } + } + }, + "paid": { + "type": ["null", "boolean"] + }, + "payment_intent": { + "type": ["null", "string"] + }, + "payment_method": { + "type": ["null", "string"] + }, + "payment_method_details": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "card": { + "type": ["null", "object"], + "properties": { + "amount_authorized": { + "type": ["null", "integer"] + }, + "brand": { + "type": ["null", "string"] + }, + "checks": { + "type": ["null", "object"], + "properties": { + "address_line1_check": { + "type": ["null", "string"] + }, + "address_postal_code_check": { + "type": ["null", "string"] + }, + "cvc_check": { + "type": ["null", "string"] + } + } + }, + "country": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "ds_transaction_id": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "iin": { + "type": ["null", "string"] + }, + "installments": { + "type": ["null", "string"] + }, + "issuer": { + "type": ["null", "string"] + }, + "last4": { + "type": ["null", "string"] + }, + "moto": { + "type": ["null", "boolean"] + }, + "network": { + "type": ["null", "string"] + }, + "network_token": { + "type": ["null", "string"] + }, + "network_transaction_id": { + "type": ["null", "string"] + }, + "three_d_secure": { + "type": ["null", "string"] + }, + "wallet": { + "type": ["null", "string"] + } + } + } + } + }, + "refunded": { + "type": ["null", "boolean"] + }, + "source": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + } + } + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "currency": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "object"], + "properties": { + "order_transaction_id": { + "type": ["null", "string"] + }, + "payments_refund_id": { + "type": ["null", "string"] + } + } + }, + "mit_params": { + "type": [ + "null", + "object" + ] + }, + "object": { + "type": ["null", "string"] + }, + "payment_method_details": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "card": { + "type": ["null", "object"], + "properties": { + "acquirer_reference_number": { + "type": ["null", "string"] + }, + "acquirer_reference_number_status": { + "type": ["null", "string"] + } + } + } + } + }, + "reason": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + } + } + }, + "payments_refund_attributes": { + "properties": { + "status": { + "type": ["null", "string"] + }, + "acquirer_reference_number": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "processed_at": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "duties": {} } } diff --git a/tap_shopify/schemas/orders.json b/tap_shopify/schemas/orders.json index dc87ea7e..dca630a6 100644 --- a/tap_shopify/schemas/orders.json +++ b/tap_shopify/schemas/orders.json @@ -12,6 +12,42 @@ "total_price_set": {}, "total_shipping_price_set": {}, "total_tax_set": {}, + "current_subtotal_price_set": { + "type": [ + "null", + "object" + ] + }, + "current_total_duties_set": { + "type": [ + "null", + "object" + ] + }, + "current_total_discounts_set": { + "type": [ + "null", + "object" + ] + }, + "current_total_price_set": { + "type": [ + "null", + "object" + ] + }, + "current_total_tax_set": { + "type": [ + "null", + "object" + ] + }, + "original_total_duties_set": { + "type": [ + "null", + "object" + ] + }, "total_price": { "type": [ "null", @@ -1109,6 +1145,139 @@ "null", "string" ] + }, + "current_subtotal_price": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "current_total_discounts": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "current_total_price": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "current_total_tax": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "total_outstanding": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "payment_terms": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "integer", + "string" + ] + }, + "currency": { + "type": [ + "null", + "string" + ] + }, + "due_in_days": { + "type": [ + "null", + "integer" + ] + }, + "payment_schedules": { + "type": [ + "null", + "array" + ], + "items": { + "properties": { + "amount": { + "type": [ + "null", + "integer", + "string" + ] + }, + "due_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "currency": { + "type": [ + "null", + "string" + ] + }, + "issued_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "completed_at": { + "type": [ + "null", + "string" + ] + }, + "expected_payment_method": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "payment_terms_name": { + "type": [ + "null", + "string" + ] + }, + "payment_terms_type": { + "type": [ + "null", + "string" + ] + } + } + }, + "estimated_taxes": { + "type": [ + "null", + "boolean" + ] } }, "type": "object" diff --git a/tap_shopify/schemas/products.json b/tap_shopify/schemas/products.json index 09f9a4ae..34b8b3b7 100644 --- a/tap_shopify/schemas/products.json +++ b/tap_shopify/schemas/products.json @@ -303,6 +303,12 @@ "null", "string" ] + }, + "product_id": { + "type": [ + "null", + "integer" + ] } }, "type": [ diff --git a/tap_shopify/schemas/transactions.json b/tap_shopify/schemas/transactions.json index bd9d321b..87dcecde 100644 --- a/tap_shopify/schemas/transactions.json +++ b/tap_shopify/schemas/transactions.json @@ -176,6 +176,32 @@ "null", "integer" ] + }, + "processed_at": { + "type": [ + "null", + "string" + ] + }, + "payments_refund_attributes": { + "type": [ + "null", + "object" + ], + "properties": { + "status": { + "type": [ + "null", + "string" + ] + }, + "acquirer_reference_number": { + "type": [ + "null", + "string" + ] + } + } } }, "type": "object" diff --git a/tap_shopify/streams/metafields.py b/tap_shopify/streams/metafields.py index 5db1eecc..2c62d1d0 100644 --- a/tap_shopify/streams/metafields.py +++ b/tap_shopify/streams/metafields.py @@ -62,8 +62,14 @@ def sync(self): # Shop metafields for metafield in self.get_objects(): metafield = metafield.to_dict() - value_type = metafield.get("value_type") - if value_type and value_type == "json_string": + metafield_type = metafield.get("type") + # create "value_type" field in the record + metafield["value_type"] = metafield_type + # the json_string value in "value_type" field will be + # mapped to following "type" value in the new version + # Reference: https://shopify.dev/apps/metafields/types + if metafield_type and metafield_type in ["json", "weight", "volume", \ + "dimension", "rating"]: value = metafield.get("value") try: metafield["value"] = json.loads(value) if value is not None else value diff --git a/tests/base.py b/tests/base.py index 2d45ba7b..8e453eb4 100644 --- a/tests/base.py +++ b/tests/base.py @@ -132,7 +132,12 @@ def expected_metadata(self): def expected_streams(self): """A set of expected stream names""" - return set(self.expected_metadata().keys()) + # removed "abandoned_checkouts", as per the Doc: + # https://help.shopify.com/en/manual/orders/abandoned-checkouts?st_source=admin&st_campaign=abandoned_checkouts_footer&utm_source=admin&utm_campaign=abandoned_checkouts_footer#review-your-abandoned-checkouts + # abandoned checkouts are saved in the Shopify admin for three months. + # Every Monday, abandoned checkouts that are older than three months are removed from your admin. + # Also no POST call is available for this endpoint: https://shopify.dev/api/admin-rest/2022-01/resources/abandoned-checkouts + return set(self.expected_metadata().keys()) - {"abandoned_checkouts"} def child_streams(self): """ @@ -301,7 +306,12 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.start_date = self.get_properties().get("start_date") self.store_1_streams = {'custom_collections', 'orders', 'products', 'customers', 'locations', 'inventory_levels', 'inventory_items', 'events'} - self.store_2_streams = {'abandoned_checkouts', 'collects', 'metafields', 'transactions', 'order_refunds', 'products', 'locations', 'inventory_levels', 'inventory_items', 'events'} + # removed 'abandoned_checkouts' from store 2 streams, as per the Doc: + # https://help.shopify.com/en/manual/orders/abandoned-checkouts?st_source=admin&st_campaign=abandoned_checkouts_footer&utm_source=admin&utm_campaign=abandoned_checkouts_footer#review-your-abandoned-checkouts + # abandoned checkouts are saved in the Shopify admin for three months. + # Every Monday, abandoned checkouts that are older than three months are removed from your admin. + # Also no POST call is available for this endpoint: https://shopify.dev/api/admin-rest/2022-01/resources/abandoned-checkouts + self.store_2_streams = {'collects', 'metafields', 'transactions', 'order_refunds', 'products', 'locations', 'inventory_levels', 'inventory_items', 'events'} #modified this method to accommodate replication key in the current_state def calculated_states_by_stream(self, current_state): diff --git a/tests/test_all_fields.py b/tests/test_all_fields.py new file mode 100644 index 00000000..04d87f5a --- /dev/null +++ b/tests/test_all_fields.py @@ -0,0 +1,100 @@ +import os + +from tap_tester import runner, menagerie +from base import BaseTapTest + +class AllFieldsTest(BaseTapTest): + + @staticmethod + def name(): + return "tap_tester_shopify_all_fields_test" + + def get_properties(self, original: bool = True): + """Configuration properties required for the tap.""" + return_value = { + 'start_date': '2021-04-01T00:00:00Z', + 'shop': 'talenddatawearhouse', + 'date_window_size': 30 + } + + return return_value + + @staticmethod + def get_credentials(original_credentials: bool = True): + """Authentication information for the test account""" + + return { + 'api_key': os.getenv('TAP_SHOPIFY_API_KEY_TALENDDATAWEARHOUSE') + } + + def test_run(self): + """ + Ensure running the tap with all streams and fields selected results in the + replication of all fields. + - Verify no unexpected streams were replicated + - Verify that more than just the automatic fields are replicated for each stream + """ + + expected_streams = self.expected_streams() + + # instantiate connection + conn_id = self.create_connection() + + # run check mode + found_catalogs = menagerie.get_catalogs(conn_id) + + # table and field selection + test_catalogs_all_fields = [catalog for catalog in found_catalogs + if catalog.get('stream_name') in expected_streams] + self.select_all_streams_and_fields(conn_id, test_catalogs_all_fields, select_all_fields=True) + + # grab metadata after performing table-and-field selection to set expectations + stream_to_all_catalog_fields = dict() # used for asserting all fields are replicated + for catalog in test_catalogs_all_fields: + stream_id, stream_name = catalog['stream_id'], catalog['stream_name'] + catalog_entry = menagerie.get_annotated_schema(conn_id, stream_id) + fields_from_field_level_md = [md_entry['breadcrumb'][1] + for md_entry in catalog_entry['metadata'] + if md_entry['breadcrumb'] != []] + stream_to_all_catalog_fields[stream_name] = set(fields_from_field_level_md) + + # run initial sync + record_count_by_stream = self.run_sync(conn_id) + synced_records = runner.get_records_from_target_output() + + # Verify no unexpected streams were replicated + synced_stream_names = set(synced_records.keys()) + self.assertSetEqual(expected_streams, synced_stream_names) + + for stream in expected_streams: + with self.subTest(stream=stream): + + # expected values + expected_automatic_keys = self.expected_primary_keys().get(stream, set()) | self.expected_replication_keys().get(stream, set()) + # get all expected keys + expected_all_keys = stream_to_all_catalog_fields[stream] + + # collect actual values + messages = synced_records.get(stream) + + actual_all_keys = set() + # collect actual values + for message in messages['messages']: + if message['action'] == 'upsert': + actual_all_keys.update(message['data'].keys()) + + # Verify that you get some records for each stream + self.assertGreater(record_count_by_stream.get(stream, -1), 0) + + # verify all fields for a stream were replicated + self.assertGreater(len(expected_all_keys), len(expected_automatic_keys)) + self.assertTrue(expected_automatic_keys.issubset(expected_all_keys), msg=f'{expected_automatic_keys-expected_all_keys} is not in "expected_all_keys"') + + if stream == 'abandoned_checkouts': + expected_all_keys.remove('billing_address') + elif stream == 'orders': + # No field named 'order_adjustments' present in the 'order' object + # Documentation: https://shopify.dev/api/admin-rest/2021-10/resources/order#resource_object + expected_all_keys.remove('order_adjustments') + + self.assertSetEqual(expected_all_keys, actual_all_keys) diff --git a/tests/test_automatic_fields.py b/tests/test_automatic_fields.py index 76158e29..8fa63207 100644 --- a/tests/test_automatic_fields.py +++ b/tests/test_automatic_fields.py @@ -31,6 +31,7 @@ def automatic_test(self, conn_id, testable_streams): """ Verify that for each stream you can get multiple pages of data when no fields are selected and only the automatic fields are replicated. + Verify that all replicated records have unique primary key values. PREREQUISITE For EACH stream add enough data that you surpass the limit of a single @@ -52,6 +53,7 @@ def automatic_test(self, conn_id, testable_streams): record_count_by_stream = self.run_sync(conn_id) actual_fields_by_stream = runner.examine_target_output_for_fields() + synced_records = runner.get_records_from_target_output() for stream in incremental_streams: with self.subTest(stream=stream): @@ -60,6 +62,11 @@ def automatic_test(self, conn_id, testable_streams): # SKIP THIS ASSERTION FOR STREAMS WHERE YOU CANNOT GET # MORE THAN 1 PAGE OF DATA IN THE TEST ACCOUNT stream_metadata = self.expected_metadata().get(stream, {}) + expected_primary_keys = self.expected_primary_keys().get(stream, set()) + + # collect records + messages = synced_records.get(stream) + minimum_record_count = stream_metadata.get( self.API_LIMIT, self.get_properties().get('result_per_page', self.DEFAULT_RESULTS_PER_PAGE) @@ -72,7 +79,15 @@ def automatic_test(self, conn_id, testable_streams): # verify that only the automatic fields are sent to the target self.assertEqual( actual_fields_by_stream.get(stream, set()), - self.expected_primary_keys().get(stream, set()) | + expected_primary_keys | self.expected_replication_keys().get(stream, set()), msg="The fields sent to the target are not the automatic fields" ) + + # Verify that all replicated records have unique primary key values. + records_pks_set = {tuple([message.get('data').get(primary_key) for primary_key in expected_primary_keys]) + for message in messages.get('messages')} + records_pks_list = [tuple([message.get('data').get(primary_key) for primary_key in expected_primary_keys]) + for message in messages.get('messages')] + self.assertCountEqual(records_pks_set, records_pks_list, + msg="We have duplicate records for {}".format(stream)) diff --git a/tests/test_bookmarks_updated.py b/tests/test_bookmarks_updated.py index abb9dee1..2dbfd466 100644 --- a/tests/test_bookmarks_updated.py +++ b/tests/test_bookmarks_updated.py @@ -15,6 +15,17 @@ class BookmarkTest(BaseTapTest): def name(): return "tap_tester_shopify_bookmark_test" + # function for verifying the date format + def is_expected_date_format(self, date): + try: + # parse date + dt.strptime(date, "%Y-%m-%dT%H:%M:%S.%fZ") + except ValueError: + # return False if date is in not expected format + return False + # return True in case of no error + return True + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.start_date = '2021-04-01T00:00:00Z' @@ -26,7 +37,12 @@ def __init__(self, *args, **kwargs): # creating this global variable for store 2 which is required only for this test, all the other test are referencing from base global store_2_streams - store_2_streams = {'abandoned_checkouts', 'collects', 'metafields', 'transactions', 'order_refunds', 'products', 'locations', 'inventory_items', 'events', 'customers', 'custom_collections', 'orders'} + # removed 'abandoned_checkouts' from store 2 streams, as per the Doc: + # https://help.shopify.com/en/manual/orders/abandoned-checkouts?st_source=admin&st_campaign=abandoned_checkouts_footer&utm_source=admin&utm_campaign=abandoned_checkouts_footer#review-your-abandoned-checkouts + # abandoned checkouts are saved in the Shopify admin for three months. + # Every Monday, abandoned checkouts that are older than three months are removed from your admin. + # Also no POST call is available for this endpoint: https://shopify.dev/api/admin-rest/2022-01/resources/abandoned-checkouts + store_2_streams = {'collects', 'metafields', 'transactions', 'order_refunds', 'products', 'locations', 'inventory_items', 'events', 'customers', 'custom_collections', 'orders'} def test_run_store_2(self): with self.subTest(store="store_2"): @@ -112,7 +128,9 @@ def bookmarks_test(self, conn_id, testable_streams): # verify the syncs sets a bookmark of the expected form self.assertIsNotNone(first_bookmark_value) + self.assertTrue(self.is_expected_date_format(first_bookmark_value)) self.assertIsNotNone(second_bookmark_value) + self.assertTrue(self.is_expected_date_format(second_bookmark_value)) # verify the 2nd bookmark is equal to 1st sync bookmark #NOT A BUG (IS the expected behaviour for shopify as they are using date windowing : TDL-17096 : 2nd bookmark value is getting assigned from the execution time rather than the actual bookmark time. This is an invalid assertion for shopify diff --git a/tests/test_discovery.py b/tests/test_discovery.py index 5a2377dd..b35738c0 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -33,23 +33,24 @@ def test_run(self): • verify that all other fields have inclusion of available (metadata and schema) """ conn_id = self.create_connection() + expected_streams = self.expected_streams() | {"abandoned_checkouts"} # Verify number of actual streams discovered match expected found_catalogs = menagerie.get_catalogs(conn_id) self.assertGreater(len(found_catalogs), 0, msg="unable to locate schemas for connection {}".format(conn_id)) self.assertEqual(len(found_catalogs), - len(self.expected_streams()), + len(expected_streams), msg="Expected {} streams, actual was {} for connection {}," " actual {}".format( - len(self.expected_streams()), + len(expected_streams), len(found_catalogs), found_catalogs, conn_id)) # Verify the stream names discovered were what we expect found_catalog_names = {c['tap_stream_id'] for c in found_catalogs} - self.assertEqual(set(self.expected_streams()), + self.assertEqual(set(expected_streams), set(found_catalog_names), msg="Expected streams don't match actual streams") @@ -58,7 +59,7 @@ def test_run(self): self.assertTrue(all([re.fullmatch(r"[a-z_]+", name) for name in found_catalog_names]), msg="One or more streams don't follow standard naming") - for stream in self.expected_streams(): + for stream in expected_streams: with self.subTest(stream=stream): catalog = next(iter([catalog for catalog in found_catalogs if catalog["stream_name"] == stream])) @@ -74,6 +75,14 @@ def test_run(self): self.assertTrue(len(stream_properties) == 1, msg="There is more than one top level breadcrumb") + # collect fields + actual_fields = [] + for md_entry in metadata: + if md_entry['breadcrumb'] != []: + actual_fields.append(md_entry['breadcrumb'][1]) + # Verify there are no duplicate/conflicting metadata entries. + self.assertEqual(len(actual_fields), len(set(actual_fields)), msg="There are duplicate entries in the fields of '{}' stream".format(stream)) + # verify replication key(s) self.assertEqual( set(stream_properties[0].get( diff --git a/tests/test_start_date.py b/tests/test_start_date.py index 52635141..c2b13c22 100644 --- a/tests/test_start_date.py +++ b/tests/test_start_date.py @@ -56,7 +56,14 @@ def test_run(self): # Select all streams and all fields within streams found_catalogs = menagerie.get_catalogs(conn_id) - incremental_streams = {key for key, value in self.expected_replication_method().items() + # removed 'abandoned_checkouts', as per the Doc: + # https://help.shopify.com/en/manual/orders/abandoned-checkouts?st_source=admin&st_campaign=abandoned_checkouts_footer&utm_source=admin&utm_campaign=abandoned_checkouts_footer#review-your-abandoned-checkouts + # abandoned checkouts are saved in the Shopify admin for three months. + # Every Monday, abandoned checkouts that are older than three months are removed from your admin. + # Also no POST call is available for this endpoint: https://shopify.dev/api/admin-rest/2022-01/resources/abandoned-checkouts + expected_replication_method = self.expected_replication_method() + expected_replication_method.pop("abandoned_checkouts") + incremental_streams = {key for key, value in expected_replication_method.items() if value == self.INCREMENTAL} # IF THERE ARE STREAMS THAT SHOULD NOT BE TESTED @@ -72,6 +79,7 @@ def test_run(self): # Count actual rows synced first_sync_records = runner.get_records_from_target_output() + first_min_bookmarks = self.min_bookmarks_by_stream(first_sync_records) # set the start date for a new connection based off bookmarks largest value first_max_bookmarks = self.max_bookmarks_by_stream(first_sync_records) @@ -112,27 +120,48 @@ def test_run(self): for stream in incremental_streams: with self.subTest(stream=stream): + # get primary key values for both sync records + expected_primary_keys = self.expected_primary_keys()[stream] + primary_keys_list_1 = [tuple(message.get('data').get(expected_pk) for expected_pk in expected_primary_keys) + for message in first_sync_records.get(stream).get('messages') + if message.get('action') == 'upsert'] + primary_keys_list_2 = [tuple(message.get('data').get(expected_pk) for expected_pk in expected_primary_keys) + for message in second_sync_records.get(stream).get('messages') + if message.get('action') == 'upsert'] + primary_keys_sync_1 = set(primary_keys_list_1) + primary_keys_sync_2 = set(primary_keys_list_2) + # verify that each stream has less records than the first connection sync self.assertGreaterEqual( first_sync_record_count.get(stream, 0), second_sync_record_count.get(stream, 0), msg="second had more records, start_date usage not verified") - # verify all data from 2nd sync >= start_date - target_mark = second_min_bookmarks.get(stream, {"mark": None}) - target_value = next(iter(target_mark.values())) # there should be only one + # Verify by primary key values, that all records of the 2nd sync are included in the 1st sync since 2nd sync has a later start date. + self.assertTrue(primary_keys_sync_2.issubset(primary_keys_sync_1)) + + # verify all data from both syncs >= start_date + first_sync_target_mark = first_min_bookmarks.get(stream, {"mark": None}) + second_sync_target_mark = second_min_bookmarks.get(stream, {"mark": None}) + + # get start dates for both syncs + first_sync_start_date = self.get_properties()["start_date"] + second_sync_start_date = self.start_date + + for start_date, target_mark in zip((first_sync_start_date, second_sync_start_date), (first_sync_target_mark, second_sync_target_mark)): + target_value = next(iter(target_mark.values())) # there should be only one - if target_value: + if target_value: - # it's okay if there isn't target data for a stream - try: - target_value = self.local_to_utc(parse(target_value)) + # it's okay if there isn't target data for a stream + try: + target_value = self.local_to_utc(parse(target_value)) - # verify that the minimum bookmark sent to the target for the second sync - # is greater than or equal to the start date - self.assertGreaterEqual(target_value, - self.local_to_utc(parse(self.start_date))) + # verify that the minimum bookmark sent to the target for the second sync + # is greater than or equal to the start date + self.assertGreaterEqual(target_value, + self.local_to_utc(parse(start_date))) - except (OverflowError, ValueError, TypeError): - print("bookmarks cannot be converted to dates, " - "can't test start_date for {}".format(stream)) + except (OverflowError, ValueError, TypeError): + print("bookmarks cannot be converted to dates, " + "can't test start_date for {}".format(stream)) From 87615140d89277e4bc155804efe289d1ab76f689 Mon Sep 17 00:00:00 2001 From: KrisPersonal <66801357+KrisPersonal@users.noreply.github.com> Date: Mon, 18 Apr 2022 08:43:04 -0700 Subject: [PATCH 03/66] Bump Version for Shopify for API/SDK Upgrade and New Field Additions (#148) Co-authored-by: KrishnanG --- CHANGELOG.md | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a69eaf56..1918ddd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 1.6.0 + * API/SDK Upgrade to v10.0.0 [#135] (https://github.com/singer-io/tap-shopify/pull/135) + * New Field Additions to Schema [#140] (https://github.com/singer-io/tap-shopify/pull/140) + ## 1.5.1 * Request Timeout Implementation [#129](https://github.com/singer-io/tap-shopify/pull/129) ## 1.5.0 diff --git a/setup.py b/setup.py index 69241d2f..5dd0a3ba 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.5.1", + version="1.6.0", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", From 382747dca267f017f49b4fadccf94c238305daa0 Mon Sep 17 00:00:00 2001 From: Harsh <80324346+harshpatel4crest@users.noreply.github.com> Date: Sat, 30 Apr 2022 05:08:33 +0530 Subject: [PATCH 04/66] TDL-18653: Fix singer.transform.SchemaMismatch: Errors during transform (#149) * updated schema to fix transform errors * added patternProperties for order_refunds stream * updated schema of order_refunds * resolve integration test failure * updated schema for abandoned_checkouts --- tap_shopify/schemas/definitions.json | 192 +----------- tap_shopify/schemas/order_refunds.json | 415 +------------------------ 2 files changed, 7 insertions(+), 600 deletions(-) diff --git a/tap_shopify/schemas/definitions.json b/tap_shopify/schemas/definitions.json index e5ed7986..9c2bdf15 100644 --- a/tap_shopify/schemas/definitions.json +++ b/tap_shopify/schemas/definitions.json @@ -870,190 +870,9 @@ "integer" ] }, - "presentment_title": { - "type": [ - "null", - "string" - ] - }, - "presentment_variant_title": { - "type": [ - "null", - "string" - ] - }, - "user_id": { - "type": [ - "null", - "integer" - ] - }, - "variant_price": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "duties": {}, - "rank": { - "type": [ - "null", - "string" - ] - }, - "unit_price_measurement": { - "type": [ - "null", - "object" - ], - "properties": { - "measured_type": { - "type": [ - "null", - "string" - ] - }, - "quantity_unit": { - "type": [ - "null", - "string" - ] - }, - "quantity_value": { - "type": [ - "null", - "string" - ] - }, - "reference_unit": { - "type": [ - "null", - "string" - ] - }, - "reference_value": { - "type": [ - "null", - "string" - ] - } - } - }, - "selling_plan_application": { - "type": [ - "null", - "object" - ], - "properties": { - "selling_plan_id": { - "type": [ - "null", - "integer" - ] - }, - "api_client_id": { - "type": [ - "null", - "integer" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "description": { - "type": [ - "null", - "string" - ] - }, - "pricing_policies": { - "type": [ - "null", - "array" - ], - "items": {} - }, - "billing_interval": { - "type": [ - "null", - "string" - ] - }, - "billing_interval_count": { - "type": [ - "null", - "integer" - ] - }, - "billing_min_cycles": { - "type": [ - "null", - "string" - ] - }, - "billing_max_cycles": { - "type": [ - "null", - "integer" - ] - }, - "billing_anchors": { - "type": [ - "null", - "array" - ], - "items": {} - }, - "delivery_interval": { - "type": [ - "null", - "string" - ] - }, - "delivery_interval_count": { - "type": [ - "null", - "integer" - ] - }, - "delivery_anchors": { - "type": [ - "null", - "array" - ], - "items": {} - }, - "delivery_cutoff_period": { - "type": [ - "null", - "string" - ] - }, - "delivery_pre_cutoff_behavior": { - "type": [ - "null", - "string" - ] - }, - "delivery_intent": { - "type": [ - "null", - "string" - ] - } - } - }, - "selling_plan_id": { - "type": [ - "null", - "integer" - ] - } + "duties": {} }, + "patternProperties": {".+": {}}, "type": [ "null", "object" @@ -1193,12 +1012,6 @@ "string" ] }, - "identifier": { - "type": [ - "null", - "string" - ] - }, "channel_liable": { "type": [ "null", @@ -1206,6 +1019,7 @@ ] } }, + "patternProperties": {".+": {}}, "type": [ "null", "object" diff --git a/tap_shopify/schemas/order_refunds.json b/tap_shopify/schemas/order_refunds.json index 6d4486d5..d170e05f 100644 --- a/tap_shopify/schemas/order_refunds.json +++ b/tap_shopify/schemas/order_refunds.json @@ -1,12 +1,6 @@ { "type": "object", "properties": { - "total_duties_set": { - "type": [ - "null", - "object" - ] - }, "order_id": { "type": [ "null", @@ -275,14 +269,9 @@ "null", "number" ] - }, - "channel_liable": { - "type": [ - "null", - "boolean" - ] } }, + "patternProperties": {".+": {}}, "type": [ "null", "object" @@ -652,121 +641,9 @@ "null", "string" ] - }, - "destination_location": { - "type": [ - "null", - "object" - ], - "properties": { - "id": { - "type": [ - "null", - "integer" - ] - }, - "country_code": { - "type": [ - "null", - "string" - ] - }, - "province_code": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "address1": { - "type": [ - "null", - "string" - ] - }, - "address2": { - "type": [ - "null", - "string" - ] - }, - "city": { - "type": [ - "null", - "string" - ] - }, - "zip": { - "type": [ - "null", - "string" - ] - } - } - }, - "origin_location": { - "type": [ - "null", - "object" - ], - "properties": { - "id": { - "type": [ - "null", - "integer" - ] - }, - "country_code": { - "type": [ - "null", - "string" - ] - }, - "province_code": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "address1": { - "type": [ - "null", - "string" - ] - }, - "address2": { - "type": [ - "null", - "string" - ] - }, - "city": { - "type": [ - "null", - "string" - ] - }, - "zip": { - "type": [ - "null", - "string" - ] - } - } - }, - "duties": {} + } }, + "patternProperties": {".+": {}}, "type": [ "null", "object" @@ -882,291 +759,7 @@ "status": { "type": ["null", "string"] }, - "receipt": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "integer"] - }, - "balance_transaction": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "object": { - "type": ["null", "string"] - }, - "exchange_rate": { - "type": ["null", "string"] - } - } - }, - "charge": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "integer"] - }, - "application_fee": { - "type": ["null", "string"] - }, - "balance_transaction": { - "type": ["null", "string"] - }, - "captured": { - "type": ["null", "boolean"] - }, - "created": { - "type": ["null", "string"], - "format": "date-time" - }, - "currency": { - "type": ["null", "string"] - }, - "failure_code": { - "type": ["null", "string"] - }, - "failure_message": { - "type": ["null", "string"] - }, - "fraud_details": { - "type": [ - "null", - "object" - ] - }, - "id": { - "type": ["null", "string"] - }, - "livemode": { - "type": ["null", "boolean"] - }, - "mit_params": { - "type": [ - "null", - "object" - ] - }, - "object": { - "type": ["null", "string"] - }, - "outcome": { - "type": ["null", "object"], - "properties": { - "network_status": { - "type": ["null", "string"] - }, - "reason": { - "type": ["null", "string"] - }, - "risk_level": { - "type": ["null", "string"] - }, - "risk_score": { - "type": ["null", "integer"] - }, - "seller_message": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - } - } - }, - "metadata": { - "type": ["null", "object"], - "properties": { - "email": { - "type": ["null", "string"] - }, - "manual_entry": { - "type": ["null", "string"] - }, - "order_id": { - "type": ["null", "string"] - }, - "order_transaction_id": { - "type": ["null", "string"] - }, - "payments_charge_id": { - "type": ["null", "string"] - }, - "shop_id": { - "type": ["null", "string"] - }, - "shop_name": { - "type": ["null", "string"] - }, - "transaction_fee_total_amount": { - "type": ["null", "string"] - }, - "transaction_fee_tax_amount": { - "type": ["null", "string"] - } - } - }, - "paid": { - "type": ["null", "boolean"] - }, - "payment_intent": { - "type": ["null", "string"] - }, - "payment_method": { - "type": ["null", "string"] - }, - "payment_method_details": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "card": { - "type": ["null", "object"], - "properties": { - "amount_authorized": { - "type": ["null", "integer"] - }, - "brand": { - "type": ["null", "string"] - }, - "checks": { - "type": ["null", "object"], - "properties": { - "address_line1_check": { - "type": ["null", "string"] - }, - "address_postal_code_check": { - "type": ["null", "string"] - }, - "cvc_check": { - "type": ["null", "string"] - } - } - }, - "country": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "ds_transaction_id": { - "type": ["null", "string"] - }, - "exp_month": { - "type": ["null", "integer"] - }, - "exp_year": { - "type": ["null", "integer"] - }, - "fingerprint": { - "type": ["null", "string"] - }, - "funding": { - "type": ["null", "string"] - }, - "iin": { - "type": ["null", "string"] - }, - "installments": { - "type": ["null", "string"] - }, - "issuer": { - "type": ["null", "string"] - }, - "last4": { - "type": ["null", "string"] - }, - "moto": { - "type": ["null", "boolean"] - }, - "network": { - "type": ["null", "string"] - }, - "network_token": { - "type": ["null", "string"] - }, - "network_transaction_id": { - "type": ["null", "string"] - }, - "three_d_secure": { - "type": ["null", "string"] - }, - "wallet": { - "type": ["null", "string"] - } - } - } - } - }, - "refunded": { - "type": ["null", "boolean"] - }, - "source": { - "type": ["null", "string"] - }, - "status": { - "type": ["null", "string"] - } - } - }, - "created": { - "type": ["null", "string"], - "format": "date-time" - }, - "currency": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "metadata": { - "type": ["null", "object"], - "properties": { - "order_transaction_id": { - "type": ["null", "string"] - }, - "payments_refund_id": { - "type": ["null", "string"] - } - } - }, - "mit_params": { - "type": [ - "null", - "object" - ] - }, - "object": { - "type": ["null", "string"] - }, - "payment_method_details": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "card": { - "type": ["null", "object"], - "properties": { - "acquirer_reference_number": { - "type": ["null", "string"] - }, - "acquirer_reference_number_status": { - "type": ["null", "string"] - } - } - } - } - }, - "reason": { - "type": ["null", "string"] - }, - "status": { - "type": ["null", "string"] - } - } - }, + "receipt": {}, "payments_refund_attributes": { "properties": { "status": { From 492d9a0ac5a1d56715d851fac9196a8d580f6447 Mon Sep 17 00:00:00 2001 From: KrisPersonal <66801357+KrisPersonal@users.noreply.github.com> Date: Mon, 2 May 2022 14:15:07 -0700 Subject: [PATCH 05/66] Fix transform Issues (#150) Co-authored-by: KrishnanG --- CHANGELOG.md | 2 ++ setup.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1918ddd3..330717ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## 1.6.1 + * Fixing Tranformation Issues [#149] (https://github.com/singer-io/tap-shopify/pull/149) ## 1.6.0 * API/SDK Upgrade to v10.0.0 [#135] (https://github.com/singer-io/tap-shopify/pull/135) * New Field Additions to Schema [#140] (https://github.com/singer-io/tap-shopify/pull/140) diff --git a/setup.py b/setup.py index 5dd0a3ba..db433700 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.6.0", + version="1.6.1", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", From e42f04275026d9b261b3be59ac00d69aa5c1d354 Mon Sep 17 00:00:00 2001 From: Kyle Speer <54034650+kspeer825@users.noreply.github.com> Date: Thu, 18 Aug 2022 11:56:52 -0400 Subject: [PATCH 06/66] save test logs (#154) * save test logs * logger usage * fix indentation error in start date test Co-authored-by: kspeer --- .circleci/config.yml | 5 ++++- tests/base.py | 6 +++--- tests/sync_rows.py | 2 -- tests/test_bookmarks.py | 6 +++--- tests/test_pagination.py | 3 ++- tests/test_start_date.py | 7 ++++--- 6 files changed, 16 insertions(+), 13 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3b6ba989..8ed68124 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,16 +36,19 @@ jobs: path: test_output/report.xml - store_artifacts: path: htmlcov - - add_ssh_keys - run: name: 'Integration Tests' command: | aws s3 cp s3://com-stitchdata-dev-deployment-assets/environments/tap-tester/tap_tester_sandbox dev_env.sh source dev_env.sh + mkdir /tmp/${CIRCLE_PROJECT_REPONAME} + export STITCH_CONFIG_DIR=/tmp/${CIRCLE_PROJECT_REPONAME} source /usr/local/share/virtualenvs/tap-tester/bin/activate run-test --tap=tap-shopify tests - slack/notify-on-failure: only_for_branches: master + - store_artifacts: + path: /tmp/tap-shopify workflows: version: 2 diff --git a/tests/base.py b/tests/base.py index 8e453eb4..fdeb24f3 100644 --- a/tests/base.py +++ b/tests/base.py @@ -2,17 +2,17 @@ Setup expectations for test sub classes Run discovery for as a prerequisite for most tests """ -import unittest import os from datetime import datetime as dt from datetime import timezone as tz import dateutil.parser import pytz from datetime import timedelta -from tap_tester import connections, menagerie, runner +from tap_tester import connections, menagerie, runner +from tap_tester.base_case import BaseCase -class BaseTapTest(unittest.TestCase): +class BaseTapTest(BaseCase): """ Setup expectations for test sub classes Run discovery for as a prerequisite for most tests diff --git a/tests/sync_rows.py b/tests/sync_rows.py index d7d7675d..ad497ffc 100644 --- a/tests/sync_rows.py +++ b/tests/sync_rows.py @@ -11,8 +11,6 @@ from functools import reduce from singer import metadata -LOGGER = singer.get_logger() - # The token used to authenticate our requests was generated on # [2018-09-18](https://github.com/stitchdata/environments/commit/82609cef972fd631c628b8eb733f37eea8f5d4f4). # If it ever expires, you'll need to login to Shopify via the 1Password diff --git a/tests/test_bookmarks.py b/tests/test_bookmarks.py index a6a1f135..18ec767a 100644 --- a/tests/test_bookmarks.py +++ b/tests/test_bookmarks.py @@ -5,7 +5,7 @@ from dateutil.parser import parse -from tap_tester import menagerie, runner +from tap_tester import menagerie, runner, LOGGER from base import BaseTapTest @@ -130,7 +130,7 @@ def bookmarks_test(self, conn_id, testable_streams): dt.utcfromtimestamp(target_min_value)) except (OverflowError, ValueError, TypeError): - print("bookmarks cannot be converted to dates, comparing values directly") + LOGGER.warn("bookmarks cannot be converted to dates, comparing values directly") # verify that there is data with different bookmark values - setup necessary self.assertGreaterEqual(target_value, target_min_value, @@ -157,4 +157,4 @@ def bookmarks_test(self, conn_id, testable_streams): target_value = self.local_to_utc(dt.utcfromtimestamp(target_value)) except (OverflowError, ValueError, TypeError): - print("bookmarks cannot be converted to dates, comparing values directly") + LOGGER.warn("bookmarks cannot be converted to dates, comparing values directly") diff --git a/tests/test_pagination.py b/tests/test_pagination.py index b7b619e7..f41b77d4 100644 --- a/tests/test_pagination.py +++ b/tests/test_pagination.py @@ -13,7 +13,8 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.start_date = '2021-04-01T00:00:00Z' - def name(self): + @staticmethod + def name(): return "tap_tester_shopify_pagination_test" def get_properties(self, *args, **kwargs): diff --git a/tests/test_start_date.py b/tests/test_start_date.py index c2b13c22..dd6b1c82 100644 --- a/tests/test_start_date.py +++ b/tests/test_start_date.py @@ -8,7 +8,7 @@ from dateutil.parser import parse -from tap_tester import menagerie, runner +from tap_tester import menagerie, runner, LOGGER from base import BaseTapTest @@ -163,5 +163,6 @@ def test_run(self): self.local_to_utc(parse(start_date))) except (OverflowError, ValueError, TypeError): - print("bookmarks cannot be converted to dates, " - "can't test start_date for {}".format(stream)) + LOGGER.warn( + "bookmarks cannot be converted to dates, can't test start_date for %s", stream + ) From 60c5935e34d644373543052ad77c125af8b2d771 Mon Sep 17 00:00:00 2001 From: Dylan Sprayberry <28106103+dsprayberry@users.noreply.github.com> Date: Wed, 30 Nov 2022 13:26:08 -0500 Subject: [PATCH 07/66] Canonicalize transactions in OrderRefunds for TDL-21280 (#156) * Canonicalize transactions in OrderRefunds * Move canonicalize to base * Version Bump and Changelog --- CHANGELOG.md | 5 ++++ setup.py | 2 +- tap_shopify/streams/base.py | 41 ++++++++++++++++++++++++++ tap_shopify/streams/order_refunds.py | 6 +++- tap_shopify/streams/transactions.py | 44 ++-------------------------- 5 files changed, 54 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 330717ca..c023e8aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,18 @@ # Changelog +## 1.6.2 + * Add canonicalization of transaction receipts to OrderRefunds [#156] (https://github.com/singer-io/tap-shopify/pull/156) + ## 1.6.1 * Fixing Tranformation Issues [#149] (https://github.com/singer-io/tap-shopify/pull/149) + ## 1.6.0 * API/SDK Upgrade to v10.0.0 [#135] (https://github.com/singer-io/tap-shopify/pull/135) * New Field Additions to Schema [#140] (https://github.com/singer-io/tap-shopify/pull/140) ## 1.5.1 * Request Timeout Implementation [#129](https://github.com/singer-io/tap-shopify/pull/129) + ## 1.5.0 * Adds `events` stream [#127](https://github.com/singer-io/tap-shopify/pull/127) diff --git a/setup.py b/setup.py index db433700..894d720d 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.6.1", + version="1.6.2", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index 724df755..9f7ef759 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -25,6 +25,47 @@ # We will retry a 500 error a maximum of 5 times before giving up MAX_RETRIES = 5 +# We have observed transactions with receipt objects that contain both: +# - `token` and `Token` +# - `version` and `Version` +# - `ack` and `Ack` +# keys on transactions where PayPal is the payment type. We reached out to +# PayPal support and they told us the values should be the same, so one +# can be safely ignored since its a duplicate. Example: The logic is to +# prefer `token` if both are present and equal, convert `Token` -> `token` +# if only `Token` is present, and throw an error if both are present and +# their values are not equal. +def canonicalize(transaction_dict, field_name): + field_name_upper = field_name.capitalize() + # Not all Shopify transactions have receipts. Facebook has been shown + # to push a null receipt through the transaction + receipt = transaction_dict.get('receipt', {}) + if receipt: + value_lower = receipt.get(field_name) + value_upper = receipt.get(field_name_upper) + if value_lower and value_upper: + if value_lower == value_upper: + LOGGER.info(( + "Transaction (id=%d) contains a receipt " + "that has `%s` and `%s` keys with the same " + "value. Removing the `%s` key."), + transaction_dict['id'], + field_name, + field_name_upper, + field_name_upper) + transaction_dict['receipt'].pop(field_name_upper) + else: + raise ValueError(( + "Found Transaction (id={}) with a receipt that has " + "`{}` and `{}` keys with the different " + "values. Contact Shopify/PayPal support.").format( + transaction_dict['id'], + field_name_upper, + field_name)) + elif value_upper: + # pylint: disable=line-too-long + transaction_dict["receipt"][field_name] = transaction_dict['receipt'].pop(field_name_upper) + # function to return request timeout def get_request_timeout(): diff --git a/tap_shopify/streams/order_refunds.py b/tap_shopify/streams/order_refunds.py index 2b168146..23e13c8a 100644 --- a/tap_shopify/streams/order_refunds.py +++ b/tap_shopify/streams/order_refunds.py @@ -3,7 +3,8 @@ from tap_shopify.context import Context from tap_shopify.streams.base import (Stream, shopify_error_handling, - OutOfOrderIdsError) + OutOfOrderIdsError, + canonicalize) class OrderRefunds(Stream): name = 'order_refunds' @@ -48,6 +49,9 @@ def sync(self): refund_dict = refund.to_dict() replication_value = strptime_to_utc(refund_dict[self.replication_key]) if replication_value >= bookmark: + for transaction_dict in refund_dict.get("transactions",[]): + for field_name in ['token', 'version', 'ack', 'timestamp', 'build']: + canonicalize(transaction_dict, field_name) yield refund_dict if replication_value > max_bookmark: diff --git a/tap_shopify/streams/transactions.py b/tap_shopify/streams/transactions.py index fc9b2a55..6615a7cd 100644 --- a/tap_shopify/streams/transactions.py +++ b/tap_shopify/streams/transactions.py @@ -3,7 +3,8 @@ from singer.utils import strftime, strptime_to_utc from tap_shopify.context import Context from tap_shopify.streams.base import (Stream, - shopify_error_handling) + shopify_error_handling, + canonicalize) LOGGER = singer.get_logger() @@ -11,47 +12,6 @@ # order can have no more than 100 transactions associated with it. TRANSACTIONS_RESULTS_PER_PAGE = 100 -# We have observed transactions with receipt objects that contain both: -# - `token` and `Token` -# - `version` and `Version` -# - `ack` and `Ack` -# keys transactions where PayPal is the payment type. We reached out to -# PayPal support and they told us the values should be the same, so one -# can be safely ignored since its a duplicate. Example: The logic is to -# prefer `token` if both are present and equal, convert `Token` -> `token` -# if only `Token` is present, and throw an error if both are present and -# their values are not equal -def canonicalize(transaction_dict, field_name): - field_name_upper = field_name.capitalize() - # Not all Shopify transactions have receipts. Facebook has been shown - # to push a null receipt through the transaction - receipt = transaction_dict.get('receipt', {}) - if receipt: - value_lower = receipt.get(field_name) - value_upper = receipt.get(field_name_upper) - if value_lower and value_upper: - if value_lower == value_upper: - LOGGER.info(( - "Transaction (id=%d) contains a receipt " - "that has `%s` and `%s` keys with the same " - "value. Removing the `%s` key."), - transaction_dict['id'], - field_name, - field_name_upper, - field_name_upper) - transaction_dict['receipt'].pop(field_name_upper) - else: - raise ValueError(( - "Found Transaction (id={}) with a receipt that has " - "`{}` and `{}` keys with the different " - "values. Contact Shopify/PayPal support.").format( - transaction_dict['id'], - field_name_upper, - field_name)) - elif value_upper: - # pylint: disable=line-too-long - transaction_dict["receipt"][field_name] = transaction_dict['receipt'].pop(field_name_upper) - class Transactions(Stream): name = 'transactions' From 31b3a8674916692314270dcdfc0ea968337e26e0 Mon Sep 17 00:00:00 2001 From: rdeshmukh15 <107538720+rdeshmukh15@users.noreply.github.com> Date: Wed, 28 Dec 2022 18:08:11 +0530 Subject: [PATCH 08/66] API version upgrade and schema changes (#157) * Changes : - Updated Shopify API and SDK version - Removed below fields from the customer object of orders streams : last_order_id last_order_name orders_count total_spent - Added email_marketing_consent object in customer resource as well as in customer.order resource * changes in CHANGELOG.md and setup.py (#158) --- CHANGELOG.md | 6 + setup.py | 4 +- tap_shopify/__init__.py | 2 +- tap_shopify/schemas/definitions.json | 453 ++++++++++++++++++++++++++- tap_shopify/schemas/orders.json | 4 +- 5 files changed, 463 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c023e8aa..36fb13a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 1.7.0 + From [#157](https://github.com/singer-io/tap-shopify/pull/157): + * API/SDK Upgrade to v12.0.1 + * New Field Additions to Schema + * Fields removal from the schema + ## 1.6.2 * Add canonicalization of transaction receipts to OrderRefunds [#156] (https://github.com/singer-io/tap-shopify/pull/156) diff --git a/setup.py b/setup.py index 894d720d..efcb2b42 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.6.2", + version="1.7.0", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", @@ -11,7 +11,7 @@ python_requires='>=3.5.2', py_modules=["tap_shopify"], install_requires=[ - "ShopifyAPI==10.0.0", + "ShopifyAPI==12.0.1", "singer-python==5.12.1", ], extras_require={ diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index c593e5f7..40baab79 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -25,7 +25,7 @@ def initialize_shopify_client(): api_key = Context.config['api_key'] shop = Context.config['shop'] - version = '2022-01' + version = '2022-07' session = shopify.Session(shop, version, api_key) shopify.ShopifyResource.activate_session(session) diff --git a/tap_shopify/schemas/definitions.json b/tap_shopify/schemas/definitions.json index 9c2bdf15..a47a632e 100644 --- a/tap_shopify/schemas/definitions.json +++ b/tap_shopify/schemas/definitions.json @@ -455,6 +455,457 @@ "string" ] }, + "email_marketing_consent": { + "type": [ + "null", + "object" + ], + "properties": { + "state": { + "type": [ + "null", + "string" + ] + }, + "opt_in_level": { + "type": [ + "null", + "string" + ] + }, + "consent_updated_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + } + } + }, + "sms_marketing_consent": { + "type": [ + "null", + "object" + ], + "properties": { + "state": { + "type": [ + "null", + "string" + ] + }, + "opt_in_level": { + "type": [ + "null", + "string" + ] + }, + "consent_updated_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "consent_collected_from": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "customer_orders": { + "type": [ + "null", + "object" + ], + "properties": { + "currency": { + "type": [ + "null", + "string" + ] + }, + "email": { + "type": [ + "null", + "string" + ] + }, + "multipass_identifier": { + "type": [ + "null", + "string" + ] + }, + "default_address": { + "type": [ + "null", + "object" + ], + "properties": { + "city": { + "type": [ + "null", + "string" + ] + }, + "address1": { + "type": [ + "null", + "string" + ] + }, + "zip": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "integer" + ] + }, + "country_name": { + "type": [ + "null", + "string" + ] + }, + "province": { + "type": [ + "null", + "string" + ] + }, + "phone": { + "type": [ + "null", + "string" + ] + }, + "country": { + "type": [ + "null", + "string" + ] + }, + "first_name": { + "type": [ + "null", + "string" + ] + }, + "customer_id": { + "type": [ + "null", + "integer" + ] + }, + "default": { + "type": [ + "null", + "boolean" + ] + }, + "last_name": { + "type": [ + "null", + "string" + ] + }, + "country_code": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "province_code": { + "type": [ + "null", + "string" + ] + }, + "address2": { + "type": [ + "null", + "string" + ] + }, + "company": { + "type": [ + "null", + "string" + ] + } + } + }, + "state": { + "type": [ + "null", + "string" + ] + }, + "verified_email": { + "type": [ + "null", + "boolean" + ] + }, + "first_name": { + "type": [ + "null", + "string" + ] + }, + "updated_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "note": { + "type": [ + "null", + "string" + ] + }, + "phone": { + "type": [ + "null", + "string" + ] + }, + "admin_graphql_api_id": { + "type": [ + "null", + "string" + ] + }, + "addresses": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "city": { + "type": [ + "null", + "string" + ] + }, + "address1": { + "type": [ + "null", + "string" + ] + }, + "zip": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "integer" + ] + }, + "country_name": { + "type": [ + "null", + "string" + ] + }, + "province": { + "type": [ + "null", + "string" + ] + }, + "phone": { + "type": [ + "null", + "string" + ] + }, + "country": { + "type": [ + "null", + "string" + ] + }, + "first_name": { + "type": [ + "null", + "string" + ] + }, + "customer_id": { + "type": [ + "null", + "integer" + ] + }, + "default": { + "type": [ + "null", + "boolean" + ] + }, + "last_name": { + "type": [ + "null", + "string" + ] + }, + "country_code": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "province_code": { + "type": [ + "null", + "string" + ] + }, + "address2": { + "type": [ + "null", + "string" + ] + }, + "company": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "last_name": { + "type": [ + "null", + "string" + ] + }, + "tags": { + "type": [ + "null", + "string" + ] + }, + "tax_exempt": { + "type": [ + "null", + "boolean" + ] + }, + "id": { + "type": [ + "null", + "integer" + ] + }, + "accepts_marketing": { + "type": [ + "null", + "boolean" + ] + }, + "accepts_marketing_updated_at": { + "anyOf": [ + { + "type": "string" , + "format": "date-time" + }, + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "created_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "tax_exemptions": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "string" + ] + } + }, + "marketing_opt_in_level": { + "type": [ + "null", + "string" + ] + }, + "email_marketing_consent": { + "type": [ + "null", + "object" + ], + "properties": { + "state": { + "type": [ + "null", + "string" + ] + }, + "opt_in_level": { + "type": [ + "null", + "string" + ] + }, + "consent_updated_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + } + } + }, "sms_marketing_consent": { "type": [ "null", @@ -1030,4 +1481,4 @@ "array" ] } -} +} \ No newline at end of file diff --git a/tap_shopify/schemas/orders.json b/tap_shopify/schemas/orders.json index dca630a6..ed8b533b 100644 --- a/tap_shopify/schemas/orders.json +++ b/tap_shopify/schemas/orders.json @@ -399,7 +399,7 @@ ] }, "customer": { - "$ref": "definitions.json#/customer" + "$ref": "definitions.json#/customer_orders" }, "test": { "type": [ @@ -1281,4 +1281,4 @@ } }, "type": "object" -} +} \ No newline at end of file From ef2a257c47f67bf707b55bb46d22603903f9c307 Mon Sep 17 00:00:00 2001 From: btowles Date: Wed, 1 Feb 2023 19:09:16 +0000 Subject: [PATCH 09/66] Update platform type for tap-tester tests --- tests/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base.py b/tests/base.py index fdeb24f3..d0ebc17b 100644 --- a/tests/base.py +++ b/tests/base.py @@ -37,7 +37,7 @@ def tap_name(): @staticmethod def get_type(): """the expected url route ending""" - return "platform.shopify" + return "platform.shopify-tba" def get_properties(self, original: bool = True): """Configuration properties required for the tap.""" From 12879bb3bf08c46a4f18776c9274ae21d7db1c61 Mon Sep 17 00:00:00 2001 From: bhtowles Date: Wed, 1 Feb 2023 14:17:45 -0500 Subject: [PATCH 10/66] Revert "Update platform type for tap-tester tests" (#160) This reverts commit ef2a257c47f67bf707b55bb46d22603903f9c307. Co-authored-by: btowles --- tests/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base.py b/tests/base.py index d0ebc17b..fdeb24f3 100644 --- a/tests/base.py +++ b/tests/base.py @@ -37,7 +37,7 @@ def tap_name(): @staticmethod def get_type(): """the expected url route ending""" - return "platform.shopify-tba" + return "platform.shopify" def get_properties(self, original: bool = True): """Configuration properties required for the tap.""" From 321d23fdfc90761faa2c4be6243c2750d7c29d0a Mon Sep 17 00:00:00 2001 From: bhtowles Date: Thu, 2 Feb 2023 15:16:15 -0500 Subject: [PATCH 11/66] Update platform type for tap-tester tests (#163) Co-authored-by: btowles --- tests/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base.py b/tests/base.py index fdeb24f3..d0ebc17b 100644 --- a/tests/base.py +++ b/tests/base.py @@ -37,7 +37,7 @@ def tap_name(): @staticmethod def get_type(): """the expected url route ending""" - return "platform.shopify" + return "platform.shopify-tba" def get_properties(self, original: bool = True): """Configuration properties required for the tap.""" From 1661b7d41aac717384eb49cb1a3e39c1e4224832 Mon Sep 17 00:00:00 2001 From: namrata270998 <75604662+namrata270998@users.noreply.github.com> Date: Wed, 22 Mar 2023 22:57:16 +0530 Subject: [PATCH 12/66] TDL-17096 updated the write_bookmark() logic (#143) * updated the write_bookmark() logic * fixed pylint * test commit * test commit * fixed unittests * added intermediate state and updated integration tests * resolved final state written after every page * removed comments * added code comments * resolved comments * version bump and changelog update * fix typo [skip ci] --------- Co-authored-by: Leslie VanDeMark <38043390+leslievandemark@users.noreply.github.com> Co-authored-by: Leslie VanDeMark --- CHANGELOG.md | 3 ++ setup.py | 2 +- tap_shopify/streams/base.py | 9 ++++++ tests/test_bookmarks.py | 37 +++++++++++++++++++++- tests/test_bookmarks_updated.py | 25 ++++++++++++--- tests/unittests/test_updated_bookmark.py | 39 ++++++++++++++++++++++++ 6 files changed, 108 insertions(+), 7 deletions(-) create mode 100644 tests/unittests/test_updated_bookmark.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 36fb13a1..d9a3f05c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 1.7.1 + * Update bookmarking logic [#143](https://github.com/singer-io/tap-shopify/pull/143) + ## 1.7.0 From [#157](https://github.com/singer-io/tap-shopify/pull/157): * API/SDK Upgrade to v12.0.1 diff --git a/setup.py b/setup.py index efcb2b42..64f9d67b 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.7.0", + version="1.7.1", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index 9f7ef759..e57fe8aa 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -9,6 +9,7 @@ import simplejson import singer from singer import metrics, utils +from singer.utils import strptime_to_utc from tap_shopify.context import Context LOGGER = singer.get_logger() @@ -214,6 +215,7 @@ def get_query_params(self, since_id, status_key, updated_at_min, updated_at_max) def get_objects(self): updated_at_min = self.get_bookmark() + max_bookmark = updated_at_min stop_time = singer.utils.now().replace(microsecond=0) date_window_size = float(Context.config.get("date_window_size", DATE_WINDOW_SIZE)) @@ -251,6 +253,9 @@ def get_objects(self): # since_id parameter. raise OutOfOrderIdsError("obj.id < since_id: {} < {}".format( obj.id, since_id)) + replication_value = strptime_to_utc(getattr(obj, self.replication_key)) + if replication_value > max_bookmark: + max_bookmark = replication_value yield obj # You know you're at the end when the current page has @@ -275,6 +280,10 @@ def get_objects(self): self.update_bookmark(since_id, bookmark_key='since_id') updated_at_min = updated_at_max + bookmark = max(min(stop_time, + max_bookmark), + (stop_time - datetime.timedelta(days=date_window_size))) + self.update_bookmark(utils.strftime(bookmark)) def sync(self): """Yield's processed SDK object dicts to the caller. diff --git a/tests/test_bookmarks.py b/tests/test_bookmarks.py index 18ec767a..d0dfd062 100644 --- a/tests/test_bookmarks.py +++ b/tests/test_bookmarks.py @@ -19,6 +19,41 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.start_date = '2021-04-01T00:00:00Z' + def max_bookmarks_by_stream(self, sync_records): + """ + Return the maximum value for the replication key for each stream + which is the bookmark expected value. + + Comparisons are based on the class of the bookmark value. Dates will be + string compared which works for ISO date-time strings + """ + max_bookmarks = {} + for stream, batch in sync_records.items(): + # The metafields fetches the fields from `products`, `customers`, `orders` and `custom_collections` + # if the parent streams are selected along with the `shop` fields. + # These different streams have its own bookmark based on its parent. + # Hence filtered out the main records i.e. the `shop` records from all the records. + if stream != 'metafields': + upsert_messages = [m for m in batch.get('messages') if m['action'] == 'upsert'] + else: + upsert_messages = [m for m in batch.get('messages') if m['action'] == 'upsert' and m.get('data').get('owner_resource') == 'shop'] + stream_bookmark_key = self.expected_replication_keys().get(stream, set()) + assert len(stream_bookmark_key) == 1 # There shouldn't be a compound replication key + stream_bookmark_key = stream_bookmark_key.pop() + + bk_values = [message["data"].get(stream_bookmark_key) for message in upsert_messages] + max_bookmarks[stream] = {stream_bookmark_key: None} + for bk_value in bk_values: + if bk_value is None: + continue + + if max_bookmarks[stream][stream_bookmark_key] is None: + max_bookmarks[stream][stream_bookmark_key] = bk_value + + if bk_value > max_bookmarks[stream][stream_bookmark_key]: + max_bookmarks[stream][stream_bookmark_key] = bk_value + return max_bookmarks + def test_run(self): with self.subTest(store="store_1"): conn_id = self.create_connection(original_credentials=True) @@ -54,7 +89,7 @@ def bookmarks_test(self, conn_id, testable_streams): # Our test data sets for Shopify do not have any abandoned_checkouts our_catalogs = [catalog for catalog in found_catalogs if catalog.get('tap_stream_id') in incremental_streams] - self.select_all_streams_and_fields(conn_id, our_catalogs, select_all_fields=False) + self.select_all_streams_and_fields(conn_id, our_catalogs, select_all_fields=True) # Run a sync job using orchestrator first_sync_record_count = self.run_sync(conn_id) diff --git a/tests/test_bookmarks_updated.py b/tests/test_bookmarks_updated.py index 2dbfd466..458206b1 100644 --- a/tests/test_bookmarks_updated.py +++ b/tests/test_bookmarks_updated.py @@ -59,7 +59,7 @@ def bookmarks_test(self, conn_id, testable_streams): # Our test data sets for Shopify do not have any abandoned_checkouts our_catalogs = [catalog for catalog in found_catalogs if catalog.get('tap_stream_id') in incremental_streams] - self.select_all_streams_and_fields(conn_id, our_catalogs, select_all_fields=False) + self.select_all_streams_and_fields(conn_id, our_catalogs, select_all_fields=True) ################################# # Run first sync @@ -111,10 +111,25 @@ def bookmarks_test(self, conn_id, testable_streams): # information required for assertions from sync 1 and 2 based on expected values first_sync_count = first_sync_record_count.get(stream, 0) second_sync_count = second_sync_record_count.get(stream, 0) - first_sync_messages = [record.get('data') for record in first_sync_records.get(stream, {}).get('messages', []) - if record.get('action') == 'upsert'] - second_sync_messages = [record.get('data') for record in second_sync_records.get(stream, {}).get('messages', []) - if record.get('action') == 'upsert'] + + # The metafields fetches the fields from `products`, `customers`, `orders` and `custom_collections` + # if the parent streams are selected along with the `shop` fields. + # These different streams have its own bookmark based on its parent. + # Hence filtered out the main records i.e. the `shop` records from all the records. + if stream != 'metafields': + first_sync_messages = [record.get('data') for record in first_sync_records.get(stream, {}).get('messages', []) + if record.get('action') == 'upsert'] + else: + first_sync_messages = [record.get('data') for record in first_sync_records.get(stream, {}).get('messages', []) + if record.get('action') == 'upsert' and record.get('data').get('owner_resource') == 'shop'] + + if stream != 'metafields': + second_sync_messages = [record.get('data') for record in second_sync_records.get(stream, {}).get('messages', []) + if record.get('action') == 'upsert'] + else: + second_sync_messages = [record.get('data') for record in second_sync_records.get(stream, {}).get('messages', []) + if record.get('action') == 'upsert' and record.get('data').get('owner_resource') == 'shop'] + first_bookmark_value = first_sync_bookmark.get('bookmarks', {stream: None}).get(stream) first_bookmark_value = list(first_bookmark_value.values())[0] second_bookmark_value = second_sync_bookmark.get('bookmarks', {stream: None}).get(stream) diff --git a/tests/unittests/test_updated_bookmark.py b/tests/unittests/test_updated_bookmark.py new file mode 100644 index 00000000..9bc0b116 --- /dev/null +++ b/tests/unittests/test_updated_bookmark.py @@ -0,0 +1,39 @@ +import unittest +from unittest import mock +from singer.utils import strftime, strptime_to_utc +from tap_shopify.context import Context +import datetime + +from tap_shopify.streams.base import DATE_WINDOW_SIZE + +CUSTOMER_OBJECT = Context.stream_objects['customers']() +Context.state = {} + +class Customer(): + """The customer object to return in th api call""" + def __init__(self, id, updated_at): + self.id = id + self.updated_at = updated_at + + def to_dict(self): + return {"id": self.id, "updated_at": self.updated_at} + + +CUSTOMER_1 = Customer(2, "2021-08-11T01:57:05-04:00") +CUSTOMER_2 = Customer(3, "2021-08-12T01:57:05-04:00") + +NOW_TIME = '2021-08-16T01:56:05-04:00' +class TestUpdateBookmark(unittest.TestCase): + + @mock.patch("tap_shopify.streams.base.Stream.call_api") + @mock.patch("tap_shopify.streams.base.Stream.get_bookmark", return_value=strptime_to_utc('2021-08-11T01:55:05-04:00')) + @mock.patch("tap_shopify.streams.base.Stream.update_bookmark") + @mock.patch("tap_shopify.streams.base.Stream.get_query_params") + @mock.patch("singer.utils.now", return_value=strptime_to_utc(NOW_TIME)) + def test_update_bookmark(self, mock_now, mock_get_query_params, mock_update_bookmark, mock_get_bookmark, mock_call_api): + """Verify that the update_bookmark() is called with correct argument""" + mock_call_api.return_value = [CUSTOMER_1, CUSTOMER_2] + + customers = list(CUSTOMER_OBJECT.get_objects()) + # Verify that the min-max evaluation returns correct results and provided in the update_bookmark() + mock_update_bookmark.assert_called_with(strftime(strptime_to_utc(NOW_TIME) - datetime.timedelta(DATE_WINDOW_SIZE))) From 13dd80630922fad3f43dd091406380c4a51445a4 Mon Sep 17 00:00:00 2001 From: Dylan Sprayberry <28106103+dsprayberry@users.noreply.github.com> Date: Wed, 3 May 2023 11:38:11 -0400 Subject: [PATCH 13/66] TDL-22858 Add URLError to backoff handling (#165) * Add URLError to backoff, write unittest, changelog + version * Whitespace cleanup Co-authored-by: Andy Lu --- CHANGELOG.md | 3 +++ setup.py | 2 +- tap_shopify/streams/base.py | 4 +++- tests/unittests/test_error_handling.py | 23 +++++++++++++++++++++++ 4 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 tests/unittests/test_error_handling.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d9a3f05c..79de8a4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 1.7.2 + * Add URLError (connection reset by peer) to retry logic [#165](https://github.com/singer-io/tap-shopify/pull/165) + ## 1.7.1 * Update bookmarking logic [#143](https://github.com/singer-io/tap-shopify/pull/143) diff --git a/setup.py b/setup.py index 64f9d67b..94d588c5 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.7.1", + version="1.7.2", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index e57fe8aa..a560c6b6 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -3,6 +3,7 @@ import math import sys import socket +from urllib.error import URLError import backoff import pyactiveresource import pyactiveresource.formats @@ -127,7 +128,8 @@ def shopify_error_handling(fnc): @backoff.on_exception(backoff.expo, (pyactiveresource.connection.ServerError, pyactiveresource.formats.Error, - simplejson.scanner.JSONDecodeError), + simplejson.scanner.JSONDecodeError, + URLError), giveup=is_not_status_code_fn(range(500, 599)), on_backoff=retry_handler, max_tries=MAX_RETRIES) diff --git a/tests/unittests/test_error_handling.py b/tests/unittests/test_error_handling.py new file mode 100644 index 00000000..6bed54ab --- /dev/null +++ b/tests/unittests/test_error_handling.py @@ -0,0 +1,23 @@ +import unittest +from unittest.mock import Mock +from urllib.error import URLError +import tap_shopify +from tap_shopify.context import Context + + +class TestShopifyConnectionResetErrorHandling(unittest.TestCase): + + def test_check_access_handle_timeout_error(self): + ''' + Test retry handling of URLError + ''' + + stream = Context.stream_objects['orders']() + stream.replication_object = Mock() + stream.replication_object.find = Mock() + stream.replication_object.find.side_effect = URLError('') + + with self.assertRaises(URLError): + stream.call_api({}) + + self.assertEqual(stream.replication_object.find.call_count, 5) From 6939afb7e521df12ff587bd1ca52b9a748770840 Mon Sep 17 00:00:00 2001 From: Rushikesh Todkar <98420315+RushiT0122@users.noreply.github.com> Date: Wed, 10 May 2023 15:12:18 +0530 Subject: [PATCH 14/66] update bookmarking for interrupted sync (#166) * update bookmarking for interrupted sync * add unit tests * minor fix * fix reviews * fix style suggestion Co-authored-by: Andy Lu * bump version 1.7.3 --------- Co-authored-by: RushiT0122 Co-authored-by: Andy Lu --- CHANGELOG.md | 3 ++ setup.py | 2 +- tap_shopify/streams/base.py | 21 +++++++-- tests/unittests/test_updated_bookmark.py | 56 +++++++++++++++++++++--- 4 files changed, 72 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79de8a4d..5886537e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 1.7.3 + * Update interrupted sync bookmark strategy [#166](https://github.com/singer-io/tap-shopify/pull/166) + ## 1.7.2 * Add URLError (connection reset by peer) to retry logic [#165](https://github.com/singer-io/tap-shopify/pull/165) diff --git a/setup.py b/setup.py index 94d588c5..7d8125a9 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.7.2", + version="1.7.3", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index a560c6b6..6fbc33f8 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -183,6 +183,10 @@ def get_since_id(self): self.name, 'since_id') + def get_updated_at_max(self): + updated_at_max = Context.state.get('bookmarks', {}).get(self.name, {}).get('updated_at_max') + return utils.strptime_with_tz(updated_at_max) if updated_at_max else None + def update_bookmark(self, bookmark_value, bookmark_key=None): # NOTE: Bookmarking can never be updated to not get the most # recent thing it saw the next time you run, because the querying @@ -216,6 +220,7 @@ def get_query_params(self, since_id, status_key, updated_at_min, updated_at_max) } def get_objects(self): + last_sync_interrupted_at = self.get_updated_at_max() updated_at_min = self.get_bookmark() max_bookmark = updated_at_min @@ -235,7 +240,14 @@ def get_objects(self): # think it has something to do with how the API treats # microseconds on its date windows. Maybe it's possible to # drop data due to rounding errors or something like that? - updated_at_max = updated_at_min + datetime.timedelta(days=date_window_size) + # If last sync was interrupted, set updated_at_max to + # updated_at_max bookmarked in the interrupted sync. + # This will make sure that records with lower id than since_id + # which got updated later won't be missed + updated_at_max = (last_sync_interrupted_at + or updated_at_min + datetime.timedelta(days=date_window_size)) + last_sync_interrupted_at = None + if updated_at_max > stop_time: updated_at_max = stop_time while True: @@ -266,7 +278,9 @@ def get_objects(self): # Save the updated_at_max as our bookmark as we've synced all rows up in our # window and can move forward. Also remove the since_id because we want to # restart at 1. - Context.state.get('bookmarks', {}).get(self.name, {}).pop('since_id', None) + stream_bookmarks = Context.state.get('bookmarks', {}).get(self.name, {}) + stream_bookmarks.pop('since_id', None) + stream_bookmarks.pop('updated_at_max', None) self.update_bookmark(utils.strftime(updated_at_max)) break @@ -278,8 +292,9 @@ def get_objects(self): objects[-1].id, max([o.id for o in objects]))) since_id = objects[-1].id - # Put since_id into the state. + # Put since_id and updated_at_max into the state. self.update_bookmark(since_id, bookmark_key='since_id') + self.update_bookmark(utils.strftime(updated_at_max), bookmark_key='updated_at_max') updated_at_min = updated_at_max bookmark = max(min(stop_time, diff --git a/tests/unittests/test_updated_bookmark.py b/tests/unittests/test_updated_bookmark.py index 9bc0b116..342af8ad 100644 --- a/tests/unittests/test_updated_bookmark.py +++ b/tests/unittests/test_updated_bookmark.py @@ -19,21 +19,65 @@ def to_dict(self): return {"id": self.id, "updated_at": self.updated_at} -CUSTOMER_1 = Customer(2, "2021-08-11T01:57:05-04:00") -CUSTOMER_2 = Customer(3, "2021-08-12T01:57:05-04:00") +class DummyShopifyError(Exception): + def __init__(self, error, msg=''): + super().__init__('{}\n{}'.format(error.__class__.__name__, msg)) + + +CUSTOMER_1 = Customer(20, "2021-08-11T01:57:05-04:00") +CUSTOMER_2 = Customer(30, "2021-08-12T01:57:05-04:00") NOW_TIME = '2021-08-16T01:56:05-04:00' class TestUpdateBookmark(unittest.TestCase): - @mock.patch("tap_shopify.streams.base.Stream.call_api") + @mock.patch("tap_shopify.streams.base.Stream.call_api", return_value = [CUSTOMER_1, CUSTOMER_2]) @mock.patch("tap_shopify.streams.base.Stream.get_bookmark", return_value=strptime_to_utc('2021-08-11T01:55:05-04:00')) @mock.patch("tap_shopify.streams.base.Stream.update_bookmark") @mock.patch("tap_shopify.streams.base.Stream.get_query_params") @mock.patch("singer.utils.now", return_value=strptime_to_utc(NOW_TIME)) def test_update_bookmark(self, mock_now, mock_get_query_params, mock_update_bookmark, mock_get_bookmark, mock_call_api): """Verify that the update_bookmark() is called with correct argument""" - mock_call_api.return_value = [CUSTOMER_1, CUSTOMER_2] - - customers = list(CUSTOMER_OBJECT.get_objects()) + list(CUSTOMER_OBJECT.get_objects()) # Verify that the min-max evaluation returns correct results and provided in the update_bookmark() mock_update_bookmark.assert_called_with(strftime(strptime_to_utc(NOW_TIME) - datetime.timedelta(DATE_WINDOW_SIZE))) + + @mock.patch("tap_shopify.streams.base.Stream.call_api", return_value=[CUSTOMER_1, CUSTOMER_2]) + @mock.patch("singer.utils.now", return_value=strptime_to_utc(NOW_TIME)) + def test_since_id_and_updated_at_max_deleted(self, mock_now, mock_call_api): + """Verify after successful sync since_id and updated_at_max keys are deleted from the state""" + Context.state = {"bookmarks": { + "currently_sync_stream": 'customers', + "customers": { + "updated_at": "2021-03-27T00:00:00.000000Z", + "since_id": 15, + "updated_at_max": "2021-04-26T00:00:00.000000Z"}}} + + list(CUSTOMER_OBJECT.get_objects()) + + # Verify keys + self.assertIn("updated_at", Context.state["bookmarks"]["customers"]) + self.assertNotIn("since_id", Context.state["bookmarks"]["customers"]) + self.assertNotIn("updated_at_max", Context.state["bookmarks"]["customers"]) + + @mock.patch("tap_shopify.streams.base.Stream.call_api", side_effect=DummyShopifyError("Dummy Shopify exception...")) + @mock.patch("singer.utils.now", return_value=strptime_to_utc(NOW_TIME)) + def test_interrupted_sync(self, mock_now, mock_call_api): + """Verify if sync is interrrupted twice in a row then since_id and updated_at_max keys are not deleted from the state""" + Context.state = {"bookmarks": { + "currently_sync_stream": 'customers', + "customers": { + "updated_at": "2021-03-27T00:00:00.000000Z", + "since_id": 15, + "updated_at_max": "2021-04-26T00:00:00.000000Z"}}} + + with self.assertRaises(DummyShopifyError): + list(CUSTOMER_OBJECT.get_objects()) + + # Verify keys exist + self.assertIn("since_id", Context.state["bookmarks"]["customers"]) + self.assertIn("updated_at_max", Context.state["bookmarks"]["customers"]) + + # Verify bookmark key values are as expected + self.assertEqual(Context.state["bookmarks"]["customers"]['since_id'], 15) + self.assertEqual(Context.state["bookmarks"]["customers"]['updated_at_max'], "2021-04-26T00:00:00.000000Z") + From 1b26fbc4f8743de71a8f9895c98e1977c5284cc9 Mon Sep 17 00:00:00 2001 From: namrata270998 <75604662+namrata270998@users.noreply.github.com> Date: Tue, 23 May 2023 17:52:36 +0530 Subject: [PATCH 15/66] TDL-6829 added backoff for incompleteread error (#144) * added backoff for incompleteread error * fixed pylint * bump version 1.7.4 --------- Co-authored-by: Rushikesh Todkar <98420315+RushiT0122@users.noreply.github.com> --- CHANGELOG.md | 3 +++ setup.py | 2 +- tap_shopify/streams/base.py | 5 ++++ .../test_backoff_on_incompleteread_error.py | 24 +++++++++++++++++++ 4 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 tests/unittests/test_backoff_on_incompleteread_error.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5886537e..f5bb138c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 1.7.4 + * Add backoff for IncompleteRead [#144](https://github.com/singer-io/tap-shopify/pull/144) + ## 1.7.3 * Update interrupted sync bookmark strategy [#166](https://github.com/singer-io/tap-shopify/pull/166) diff --git a/setup.py b/setup.py index 7d8125a9..82c4cfbc 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.7.3", + version="1.7.4", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index 6fbc33f8..a8b6de43 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -4,6 +4,7 @@ import sys import socket from urllib.error import URLError +import http import backoff import pyactiveresource import pyactiveresource.formats @@ -120,6 +121,10 @@ def is_timeout_error(error_raised): return True def shopify_error_handling(fnc): + @backoff.on_exception(backoff.expo, # IncompleteRead error raised + http.client.IncompleteRead, + max_tries=MAX_RETRIES, + factor=2) @backoff.on_exception(backoff.expo, # timeout error raise by Shopify (pyactiveresource.connection.Error, socket.timeout), giveup=is_timeout_error, diff --git a/tests/unittests/test_backoff_on_incompleteread_error.py b/tests/unittests/test_backoff_on_incompleteread_error.py new file mode 100644 index 00000000..ad3d2f5b --- /dev/null +++ b/tests/unittests/test_backoff_on_incompleteread_error.py @@ -0,0 +1,24 @@ +from unittest import mock +from tap_shopify.streams.base import http +from tap_shopify.streams.transactions import Transactions +import unittest + +class TestIncompleteReadBackoff(unittest.TestCase): + @mock.patch("time.sleep") + @mock.patch("pyactiveresource.activeresource.ActiveResource.find") + def test_Transactions_pyactiveresource_error_incomplete_read_error_backoff(self, mocked_find, mocked_sleep): + """ + Test case to verify that we backoff for 5 times when 'http.client.IncompleteRead' error occurs + """ + # mock 'find' and raise IncompleteRead error + mocked_find.side_effect = http.client.IncompleteRead(b'') + # initialize class + locations = Transactions() + try: + # function call + locations.replication_object.find() + except http.client.IncompleteRead: + pass + + # verify we backoff 5 times + self.assertEquals(mocked_find.call_count, 5) \ No newline at end of file From 1debdba753d0e80617b40c9680ecb07c7048f290 Mon Sep 17 00:00:00 2001 From: Andy Lu Date: Tue, 30 May 2023 20:26:26 +0000 Subject: [PATCH 16/66] Bump version, update changelog --- CHANGELOG.md | 5 ++++- setup.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5bb138c..2e7dbe43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 1.7.5 + * Dependabot update [#167](https://github.com/singer-io/tap-shopify/pull/167) + ## 1.7.4 * Add backoff for IncompleteRead [#144](https://github.com/singer-io/tap-shopify/pull/144) @@ -186,4 +189,4 @@ * Reduce query window to 1 month ## 0.4.0 - * Change bookmarking to use query windows `updated_at_max` as bookmark [#9](https://github.com/singer-io/tap-shopify/pull/9) + * Change bookmarking to use query windows `updated_at_max` as bookmark [#9](https://github.com/singer-io/tap-shopify/pull/9) \ No newline at end of file diff --git a/setup.py b/setup.py index 82c4cfbc..59197e0f 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.7.4", + version="1.7.5", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", @@ -31,4 +31,4 @@ "schemas": ["tap_shopify/schemas/*.json"] }, include_package_data=True, -) +) \ No newline at end of file From d924c8b67d80814ab218d98baafd6e81150124ff Mon Sep 17 00:00:00 2001 From: Andy Lu Date: Wed, 7 Jun 2023 11:04:09 -0400 Subject: [PATCH 17/66] Revert "Bump version, update changelog" (#168) This reverts commit 1debdba753d0e80617b40c9680ecb07c7048f290. --- CHANGELOG.md | 5 +---- setup.py | 4 ++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e7dbe43..f5bb138c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,5 @@ # Changelog -## 1.7.5 - * Dependabot update [#167](https://github.com/singer-io/tap-shopify/pull/167) - ## 1.7.4 * Add backoff for IncompleteRead [#144](https://github.com/singer-io/tap-shopify/pull/144) @@ -189,4 +186,4 @@ * Reduce query window to 1 month ## 0.4.0 - * Change bookmarking to use query windows `updated_at_max` as bookmark [#9](https://github.com/singer-io/tap-shopify/pull/9) \ No newline at end of file + * Change bookmarking to use query windows `updated_at_max` as bookmark [#9](https://github.com/singer-io/tap-shopify/pull/9) diff --git a/setup.py b/setup.py index 59197e0f..82c4cfbc 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.7.5", + version="1.7.4", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", @@ -31,4 +31,4 @@ "schemas": ["tap_shopify/schemas/*.json"] }, include_package_data=True, -) \ No newline at end of file +) From 539564f078d38a84e179c0f7ea282e32f2024f93 Mon Sep 17 00:00:00 2001 From: Rhej <57752047+Rohan397@users.noreply.github.com> Date: Tue, 20 Jun 2023 07:16:37 -0700 Subject: [PATCH 18/66] implemented backoff for ConnectionResetError and added unittest (#169) * implemented backoff for ConnectionResetError and added a corresponding test case * updated commit based on review feedback * Update tap_shopify/streams/base.py * add parameterized dependancy for unittests * Update tests/unittests/test_error_handling.py Co-authored-by: Rushikesh Todkar <98420315+RushiT0122@users.noreply.github.com> * remove trailing whitespace * fixing circleci job * removing makefile since it is redundant * made changes based on review * removed extra code and switched locations to stream * remove extra import --------- Co-authored-by: Rushikesh Todkar <98420315+RushiT0122@users.noreply.github.com> --- .circleci/config.yml | 4 +-- Makefile | 5 ---- tap_shopify/streams/base.py | 4 +-- tests/unittests/test_error_handling.py | 38 ++++++++++++++++++++------ 4 files changed, 34 insertions(+), 17 deletions(-) delete mode 100644 Makefile diff --git a/.circleci/config.yml b/.circleci/config.yml index 8ed68124..69fbc452 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,7 +19,7 @@ jobs: name: 'pylint' command: | source /usr/local/share/virtualenvs/tap-shopify/bin/activate - make test + pylint tap_shopify -d missing-docstring,too-many-branches - run: name: 'JSON Validator' command: | @@ -29,7 +29,7 @@ jobs: name: 'Unit Tests' command: | source /usr/local/share/virtualenvs/tap-shopify/bin/activate - pip install nose coverage + pip install nose coverage parameterized nosetests --with-coverage --cover-erase --cover-package=tap_shopify --cover-html-dir=htmlcov tests/unittests coverage html - store_test_results: diff --git a/Makefile b/Makefile deleted file mode 100644 index 1bd8b48f..00000000 --- a/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -.DEFAULT_GOAL := test - -test: - pylint tap_shopify -d missing-docstring,too-many-branches - nosetests tests/unittests diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index a8b6de43..9c9e14f8 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -121,8 +121,8 @@ def is_timeout_error(error_raised): return True def shopify_error_handling(fnc): - @backoff.on_exception(backoff.expo, # IncompleteRead error raised - http.client.IncompleteRead, + @backoff.on_exception(backoff.expo, + (http.client.IncompleteRead, ConnectionResetError), max_tries=MAX_RETRIES, factor=2) @backoff.on_exception(backoff.expo, # timeout error raise by Shopify diff --git a/tests/unittests/test_error_handling.py b/tests/unittests/test_error_handling.py index 6bed54ab..b202909c 100644 --- a/tests/unittests/test_error_handling.py +++ b/tests/unittests/test_error_handling.py @@ -1,23 +1,45 @@ import unittest -from unittest.mock import Mock +from unittest import mock from urllib.error import URLError import tap_shopify +from tap_shopify.streams.transactions import Transactions +from tap_shopify.streams.orders import Orders from tap_shopify.context import Context - class TestShopifyConnectionResetErrorHandling(unittest.TestCase): - def test_check_access_handle_timeout_error(self): + @mock.patch("time.sleep") + @mock.patch("pyactiveresource.activeresource.ActiveResource.find") + def test_check_access_handle_timeout_error(self, mocked_find, mocked_sleep): ''' Test retry handling of URLError ''' - stream = Context.stream_objects['orders']() - stream.replication_object = Mock() - stream.replication_object.find = Mock() - stream.replication_object.find.side_effect = URLError('') + # mock 'find' and raise URLError + mocked_find.side_effect = URLError('') + + # initialize class + stream = Orders() with self.assertRaises(URLError): stream.call_api({}) - self.assertEqual(stream.replication_object.find.call_count, 5) + self.assertEqual(mocked_find.call_count, 5) + + @mock.patch("time.sleep") + @mock.patch("pyactiveresource.activeresource.ActiveResource.find") + def test_check_access_handle_connection_reset_error(self, mocked_find, mocked_sleep): + ''' + Test retry handling of ConnectionResetError + ''' + + # mock 'find' and raise IncompleteRead error + mocked_find.side_effect = ConnectionResetError + + # initialize class + stream = Transactions() + + with self.assertRaises(ConnectionResetError): + stream.replication_object.find() + + self.assertEqual(mocked_find.call_count, 5) From 59ff4a6feb8124c9ab33d2a94ab129dc7ebba9ee Mon Sep 17 00:00:00 2001 From: Rhej <57752047+Rohan397@users.noreply.github.com> Date: Wed, 21 Jun 2023 09:54:35 -0700 Subject: [PATCH 19/66] bumped version and added to changelog (#170) --- CHANGELOG.md | 2 ++ setup.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5bb138c..658debdd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## 1.7.5 + * Add backoff for ConnectionResetError [#169](https://github.com/singer-io/tap-shopify/pull/169) ## 1.7.4 * Add backoff for IncompleteRead [#144](https://github.com/singer-io/tap-shopify/pull/144) diff --git a/setup.py b/setup.py index 82c4cfbc..a315cdbc 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.7.4", + version="1.7.5", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", From d49b210b4f959233d039499f795d71e737c25f36 Mon Sep 17 00:00:00 2001 From: Robert Hill <30398720+rjh336@users.noreply.github.com> Date: Thu, 20 Jul 2023 10:27:57 -0400 Subject: [PATCH 20/66] add up to max retries for 404 response (#159) --- tap_shopify/streams/base.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index 9c9e14f8..b67c782e 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -138,6 +138,11 @@ def shopify_error_handling(fnc): giveup=is_not_status_code_fn(range(500, 599)), on_backoff=retry_handler, max_tries=MAX_RETRIES) + @backoff.on_exception(backoff.expo, + pyactiveresource.connection.ResourceNotFound, + giveup=is_not_status_code_fn([404]), + on_backoff=retry_handler, + max_tries=MAX_RETRIES) @backoff.on_exception(retry_after_wait_gen, pyactiveresource.connection.ClientError, giveup=is_not_status_code_fn([429]), From edbe7a3b950f063eb7d88cffe48f4247db3e4cae Mon Sep 17 00:00:00 2001 From: rdeshmukh15 <107538720+rdeshmukh15@users.noreply.github.com> Date: Thu, 20 Jul 2023 23:10:55 +0530 Subject: [PATCH 21/66] bump version changes in setup.py and CHANGELOG.md (#173) --- CHANGELOG.md | 3 +++ setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 658debdd..e9274846 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 1.7.6 + * Add backoff for 404 error code [#159](https://github.com/singer-io/tap-shopify/pull/159) + ## 1.7.5 * Add backoff for ConnectionResetError [#169](https://github.com/singer-io/tap-shopify/pull/169) ## 1.7.4 diff --git a/setup.py b/setup.py index a315cdbc..dcfa97d4 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.7.5", + version="1.7.6", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", From a475c70952b8e0e49196a7f2d153f310df9edaa1 Mon Sep 17 00:00:00 2001 From: bhtowles Date: Wed, 16 Aug 2023 13:42:35 -0500 Subject: [PATCH 22/66] TDL-22907-Qa/sync interrupt (#171) * Add interrupted sync test, WIP * Update tap to fix sync order in target file, update test assertions * Clean up for PR review, remove globals, simplify state * PR review comments round 1 * Missed one review comment * Review comments 2 * Review comments, move common function to base, clean up and simplify * Move actual_oldest_resuming_replication_date inside assertion if statement per PR review * Formatting changes from review comments --- tap_shopify/__init__.py | 12 +- tests/base.py | 40 ++++- tests/test_bookmarks_updated.py | 11 -- tests/test_interrupted_sync.py | 265 ++++++++++++++++++++++++++++++++ 4 files changed, 308 insertions(+), 20 deletions(-) create mode 100644 tests/test_interrupted_sync.py diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index 40baab79..97faf59f 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -137,6 +137,12 @@ def sync(): shop_attributes = initialize_shopify_client() sdc_fields = {"_sdc_shop_" + x: shop_attributes[x] for x in SDC_KEYS} + # If there is a currently syncing stream bookmark, shuffle the + # stream order so it gets sync'd first + currently_sync_stream_name = Context.state.get('bookmarks', {}).get('currently_sync_stream') + if currently_sync_stream_name: + shuffle_streams(currently_sync_stream_name) + # Emit all schemas first so we have them for child streams for stream in Context.catalog["streams"]: if Context.is_selected(stream["tap_stream_id"]): @@ -146,12 +152,6 @@ def sync(): bookmark_properties=stream["replication_key"]) Context.counts[stream["tap_stream_id"]] = 0 - # If there is a currently syncing stream bookmark, shuffle the - # stream order so it gets sync'd first - currently_sync_stream_name = Context.state.get('bookmarks', {}).get('currently_sync_stream') - if currently_sync_stream_name: - shuffle_streams(currently_sync_stream_name) - # Loop over streams in catalog for catalog_entry in Context.catalog['streams']: stream_id = catalog_entry['tap_stream_id'] diff --git a/tests/base.py b/tests/base.py index d0ebc17b..85bde65a 100644 --- a/tests/base.py +++ b/tests/base.py @@ -2,12 +2,12 @@ Setup expectations for test sub classes Run discovery for as a prerequisite for most tests """ -import os -from datetime import datetime as dt -from datetime import timezone as tz import dateutil.parser +import os import pytz +from datetime import datetime as dt from datetime import timedelta +from datetime import timezone as tz from tap_tester import connections, menagerie, runner from tap_tester.base_case import BaseCase @@ -363,3 +363,37 @@ def timedelta_formatted(self, dtime, days=0): except ValueError: return Exception("Datetime object is not of the format: {}".format(self.START_DATE_FORMAT)) + + @staticmethod + def parse_date(date_value): + """ + Pass in string-formatted-datetime, parse the value + return it as an un-formatted datetime object. + """ + date_formats = { + "%Y-%m-%dT%H:%M:%S.%fZ", + "%Y-%m-%dT%H:%M:%S%z", + "%Y-%m-%dT%H:%M:%SZ", + "%Y-%m-%dT%H:%M:%S.%f+00:00", + "%Y-%m-%dT%H:%M:%S+00:00", + "%Y-%m-%d" + } + for date_format in date_formats: + try: + date_stripped = dt.strptime(date_value, date_format) + return date_stripped + except ValueError: + pass + + raise NotImplementedError(f"Tests do not account for dates of this format: {date_value}") + + # function for verifying the date format + def is_expected_date_format(self, date): + try: + # parse date + dt.strptime(date, "%Y-%m-%dT%H:%M:%S.%fZ") + except ValueError: + # return False if date is in not expected format + return False + # return True in case of no error + return True diff --git a/tests/test_bookmarks_updated.py b/tests/test_bookmarks_updated.py index 458206b1..d590f239 100644 --- a/tests/test_bookmarks_updated.py +++ b/tests/test_bookmarks_updated.py @@ -15,17 +15,6 @@ class BookmarkTest(BaseTapTest): def name(): return "tap_tester_shopify_bookmark_test" - # function for verifying the date format - def is_expected_date_format(self, date): - try: - # parse date - dt.strptime(date, "%Y-%m-%dT%H:%M:%S.%fZ") - except ValueError: - # return False if date is in not expected format - return False - # return True in case of no error - return True - def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.start_date = '2021-04-01T00:00:00Z' diff --git a/tests/test_interrupted_sync.py b/tests/test_interrupted_sync.py new file mode 100644 index 00000000..aadff660 --- /dev/null +++ b/tests/test_interrupted_sync.py @@ -0,0 +1,265 @@ +""" +Test tap successfully resumes after interrupted sync without missing any records +""" +import random + +from datetime import datetime as dt + +from tap_tester import menagerie, connections, runner, LOGGER +from base import BaseTapTest + + +class InterruptedSyncTest(BaseTapTest): + """Test tap sets a bookmark and respects it for the next sync of a stream""" + @staticmethod + def name(): + return "tap_tester_shopify_int_sync_test" + + def group_streams(self, sync_order, currently_syncing): + self.assertIn(currently_syncing, sync_order, + msg="Currently sycning stream not found in sync order") + index = len(sync_order) + for i, stream in enumerate(sync_order): + if stream == currently_syncing: + index = i + break + return { + "completed": sync_order[:index], + "yet_to_be_synced": sync_order[(index + 1):], + } + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.start_date = '2022-03-01T00:00:00Z' + + def test_run(self): + + conn_id = self.create_connection(original_properties=False, original_credentials=False) + + expected_streams = {'customers', + 'events', + 'metafields', + 'orders', + 'products', + 'transactions'} + + # Select all streams and no fields within streams + found_catalogs = menagerie.get_catalogs(conn_id) + + # Our test data sets for Shopify do not have any abandoned_checkouts + our_catalogs = [catalog for catalog in found_catalogs if + catalog.get('tap_stream_id') in expected_streams] + self.select_all_streams_and_fields(conn_id, our_catalogs, select_all_fields=True) + + ################################# + # Run first sync + ################################# + + first_sync_record_count = self.run_sync(conn_id) + first_sync_state = menagerie.get_state(conn_id) + first_sync_records = runner.get_records_from_target_output() + first_sync_order = runner.get_stream_sync_order_from_target() + + # verify that the sync only sent records to the target for selected streams (catalogs) + self.assertSetEqual(set(first_sync_record_count.keys()), expected_streams) + + # BUG:TDL-17087 : State has additional values which are not streams + # Need to remove additional values from bookmark value + extra_stuff = {'transaction_orders', + 'metafield_products', + 'refund_orders', + 'product_variants'} + + for keys in list(first_sync_state['bookmarks'].keys()): + if keys in extra_stuff: + first_sync_state['bookmarks'].pop(keys) + + ################################ + # Update State between Syncs + ################################ + + # hardcoding the updated state to ensure atleast 1 record in resuming (2nd) sync. + # values have been provided after reviewing the max bookmark value for each of the streams + currently_syncing_stream = random.choice(list(expected_streams)) + LOGGER.info("Randomly selected currently syncing stream: %s", currently_syncing_stream) + + stream_groups = self.group_streams(first_sync_order, currently_syncing_stream) + completed_streams = stream_groups.get('completed') + yet_to_be_synced_streams = stream_groups.get('yet_to_be_synced') + + base_state = {'bookmarks': + {'currently_sync_stream': currently_syncing_stream, + 'customers': {'updated_at': '2023-03-28T18:53:28.000000Z'}, + 'events': {'created_at': '2023-01-22T05:05:53.000000Z'}, + 'metafields': {'updated_at': '2023-01-07T21:18:05.000000Z'}, + 'orders': {'updated_at': '2023-01-22T05:07:44.000000Z'}, + 'products': {'updated_at': '2023-01-22T05:05:56.000000Z'}, + 'transactions': {'created_at': '2022-06-26T00:06:38-04:00'} + }} + + # remove yet to be synced streams from base state and then set new state + new_state = { + 'bookmarks': { + key: val + for key, val in base_state['bookmarks'].items() + if key not in yet_to_be_synced_streams + } + } + + menagerie.set_state(conn_id, new_state) + + ################################ + # Run Resuming (2nd) Sync + ################################ + + resuming_sync_record_count = self.run_sync(conn_id) + resuming_sync_records = runner.get_records_from_target_output() + resuming_sync_state = menagerie.get_state(conn_id) + resuming_sync_order = runner.get_stream_sync_order_from_target() + + LOGGER.info("First sync stream order: %s", first_sync_order) + LOGGER.info("currently syncing stream: %s", currently_syncing_stream) + LOGGER.info("yet to be synced streams: %s", yet_to_be_synced_streams) + LOGGER.info("completed streams: %s", completed_streams) + LOGGER.info("Resuming sync stream order: %s", resuming_sync_order) + + # tap level assertions + self.assertTrue(first_sync_state.get('bookmarks')) + self.assertTrue(resuming_sync_state.get('bookmarks')) + self.assertIsNone(first_sync_state.get('bookmarks', {}).get('currently_sync_stream')) + self.assertIsNone(resuming_sync_state.get('bookmarks', {}).get('currently_sync_stream')) + + # verify streams are shuffled so the resuming sync starts with currently_syncing_stream + self.assertEqual(resuming_sync_order[0], currently_syncing_stream) + + expected_resuming_sync_order = ( + [currently_syncing_stream] + yet_to_be_synced_streams + completed_streams + ) + self.assertListEqual(expected_resuming_sync_order, resuming_sync_order) + + for stream in expected_streams: + with self.subTest(stream=stream): + + # expected values (rep method = incremental for all shopify streams as of Jul-2023) + expected_replication_keys = self.expected_replication_keys() + # information required for assertions from sync 1 and 2 based on expected values + first_sync_count = first_sync_record_count.get(stream, 0) + resuming_sync_count = resuming_sync_record_count.get(stream, 0) + + # The metafields fetches the fields from `products`, `customers`, `orders` and + # `custom_collections` if the parent streams are selected along with the `shop` + # fields. These different streams have their own bookmark based on the parent. + # Hence filtered out the main records i.e. the `shop` records from all the records. + + if stream != 'metafields': + first_sync_messages = [ + record.get('data') for record + in first_sync_records.get(stream, {}).get('messages', []) + if record.get('action') == 'upsert'] + + resuming_sync_messages = [ + record.get('data') for record + in resuming_sync_records.get(stream, {}).get('messages', []) + if record.get('action') == 'upsert'] + + else: + first_sync_messages = [ + record.get('data') for record + in first_sync_records.get(stream, {}).get('messages', []) + if record.get('action') == 'upsert' + and record.get('data', {}).get('owner_resource') == 'shop'] + + resuming_sync_messages = [ + record.get('data') for record + in resuming_sync_records.get(stream, {}).get('messages', []) + if record.get('action') == 'upsert' + and record.get('data', {}).get('owner_resource') == 'shop'] + + replication_key = next(iter(expected_replication_keys[stream])) + first_bookmark_stream = first_sync_state.get('bookmarks', {}).get(stream, {}) + first_bookmark_value = first_bookmark_stream.get(replication_key) + resuming_bookmark_stream = resuming_sync_state.get('bookmarks', {}).get(stream, {}) + resuming_bookmark_value = resuming_bookmark_stream.get(replication_key) + resuming_bookmark_value_utc = self.convert_state_to_utc(resuming_bookmark_value) + + if stream in new_state['bookmarks'].keys(): + simulated_bookmark = new_state['bookmarks'][stream] + simulated_bookmark_value = simulated_bookmark[replication_key] + + youngest_first_sync_date = max( + self.parse_date(record.get(replication_key)) + for record in first_sync_messages) + + # verify the syncs sets a bookmark of the expected form + self.assertIsNotNone(first_bookmark_value) + self.assertTrue(self.is_expected_date_format(first_bookmark_value)) + self.assertIsNotNone(resuming_bookmark_value) + self.assertTrue(self.is_expected_date_format(resuming_bookmark_value)) + + # verify the resuming bookmark is greater than 1st sync bookmark + # This is the expected behaviour for shopify as they are using date windowing + # TDL-17096 : Resuming bookmark value is getting assigned from execution time + # rather than the actual bookmark time for some streams. + # TODO this is not the case for the transactions stream, they are equal. + if stream == 'transactions': + self.assertEqual(resuming_bookmark_value, first_bookmark_value) + else: + self.assertGreater(resuming_bookmark_value, first_bookmark_value) + + # verify oldest record from resuming sync respects bookmark from previous sync + if stream in new_state['bookmarks'].keys() and resuming_sync_messages: + # if metafields owner_resource != 'shop' resuming_sync_messages can be empty + actual_oldest_resuming_replication_date = min( + self.parse_date(record.get(replication_key)) + for record in resuming_sync_messages) + + self.assertEqual(actual_oldest_resuming_replication_date, + self.parse_date(simulated_bookmark_value), + msg="Oldest resuming sync record not respecting bookmark") + + # all interrupted recs are in full recs, interrupted rec counts verified + first_sync_records_after_bookmark = [ + record for record in first_sync_messages + if self.parse_date(record[replication_key]) >= + self.parse_date(simulated_bookmark_value)] + # remove any records that got added after the first sync + filtered_resuming_records = [ + record for record in resuming_sync_messages + if self.parse_date(record[replication_key]) <= + youngest_first_sync_date] + + self.assertEqual(first_sync_records_after_bookmark, filtered_resuming_records, + msg="Incorrect data in the resuming sync") + + for record in resuming_sync_messages: + replication_key_value = record.get(replication_key) + # this assertion is only for completed and interrupted streams + if stream in new_state['bookmarks'].keys(): + # verify 2nd sync rep key value is greater or equal to 1st sync bookmarks + msg = "Resuming sync records do not respect the previous bookmark" + self.assertGreaterEqual(replication_key_value, simulated_bookmark_value, + msg=msg) + # verify the 2nd sync bookmark value is the max rep key value for given stream + msg = ("Resuming sync bookmark was set incorrectly, a record with a greater" + " replication key value was synced") + self.assertLessEqual(replication_key_value, resuming_bookmark_value_utc, + msg=msg) + + # verify less data in 2nd sync for streams that started or completed + if stream in new_state['bookmarks'].keys(): + self.assertLess(resuming_sync_count, first_sync_count, + msg="Resuming sync record count greater than expected") + + # verify yet to be sync'd streams have equal oldest record + if stream in yet_to_be_synced_streams: + oldest_first_sync_replication_date = min( + self.parse_date(record.get(replication_key)) + for record in first_sync_messages) + oldest_resuming_sync_replication_date = min( + self.parse_date(record.get(replication_key)) + for record in resuming_sync_messages) + self.assertEqual(oldest_resuming_sync_replication_date, + oldest_first_sync_replication_date) + + # verify that we get at least 1 record in the resuming sync + self.assertGreater(resuming_sync_count, 0, msg="Resuming sync yielded 0 recs") From 09c0a6716ccb47d441f383d3837517b59dfeadee Mon Sep 17 00:00:00 2001 From: JYOTHINARAYANSETTY <49671483+JYOTHINARAYANSETTY@users.noreply.github.com> Date: Thu, 17 Aug 2023 15:16:03 -0400 Subject: [PATCH 23/66] fix all_fields test (#176) * fix all_fields test * pr review comments * pr review comments --- tests/test_all_fields.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/test_all_fields.py b/tests/test_all_fields.py index 04d87f5a..ba42ccab 100644 --- a/tests/test_all_fields.py +++ b/tests/test_all_fields.py @@ -88,13 +88,14 @@ def test_run(self): # verify all fields for a stream were replicated self.assertGreater(len(expected_all_keys), len(expected_automatic_keys)) - self.assertTrue(expected_automatic_keys.issubset(expected_all_keys), msg=f'{expected_automatic_keys-expected_all_keys} is not in "expected_all_keys"') - - if stream == 'abandoned_checkouts': - expected_all_keys.remove('billing_address') - elif stream == 'orders': - # No field named 'order_adjustments' present in the 'order' object - # Documentation: https://shopify.dev/api/admin-rest/2021-10/resources/order#resource_object - expected_all_keys.remove('order_adjustments') + self.assertTrue(expected_automatic_keys.issubset(expected_all_keys), + msg=f'{expected_automatic_keys-expected_all_keys} is not in "expected_all_keys"') + + if stream == 'orders': + # No field named 'order_adjustments', 'total_price_usd' present in the 'order' object + # Documentation: https://shopify.dev/api/admin-rest/2021-10/resources/order#resource_object + # https://jira.talendforge.org/browse/TDL-15985 + bad_schema_fields = {'order_adjustments', 'total_price_usd'} + expected_all_keys = expected_all_keys - bad_schema_fields self.assertSetEqual(expected_all_keys, actual_all_keys) From ad53f765b39b38ccf84683f4fa9ff2aa51a50cfc Mon Sep 17 00:00:00 2001 From: bhtowles Date: Fri, 15 Sep 2023 08:29:47 -0500 Subject: [PATCH 24/66] Update all fields with old/new field from schema (#177) --- tests/test_all_fields.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_all_fields.py b/tests/test_all_fields.py index ba42ccab..fd2768b5 100644 --- a/tests/test_all_fields.py +++ b/tests/test_all_fields.py @@ -95,7 +95,8 @@ def test_run(self): # No field named 'order_adjustments', 'total_price_usd' present in the 'order' object # Documentation: https://shopify.dev/api/admin-rest/2021-10/resources/order#resource_object # https://jira.talendforge.org/browse/TDL-15985 - bad_schema_fields = {'order_adjustments', 'total_price_usd'} + # total_price_usd showing up in syncd records Sep 2023, still missing from docs + bad_schema_fields = {'order_adjustments'} expected_all_keys = expected_all_keys - bad_schema_fields self.assertSetEqual(expected_all_keys, actual_all_keys) From 43b85b2765e6fca7a3de561b572399dc80f400ed Mon Sep 17 00:00:00 2001 From: Dylan Sprayberry <28106103+dsprayberry@users.noreply.github.com> Date: Thu, 28 Sep 2023 14:05:07 -0400 Subject: [PATCH 25/66] TDL-23255 Upgrade Shopify API to 2023_04 (#178) * upgrade shopify api to 23-04 * try parallel circle tests * Remove fulfillment_service from line_item resource for 2022_10 ----------------------------- Co-authored-by: alu * Updated config.yml * Updated config.yml * Updated config.yml * Updated config.yml * update schema name to avoid test collisions ----------------------------- Co-authored-by: Dylan Sprayberry * decrease parallelism for rate limiting errors ----------------------------- Co-authored-by: Dylan Sprayberry * add fulfillment_service back in since it's nested ----------------------------- Co-authored-by: Dylan Sprayberry * Remove deprecated orders.total_price_usd field for 2022_10 ----------------------------- Co-authored-by: Dylan Sprayberry * Add new orders.merchant_of_record_app_id field ----------------------------- Co-authored-by: Dylan Sprayberry * add new transactions.payment_details fields 2023_01 ----------------------------- Co-authored-by: Dylan Sprayberry * Add new transactions.payment_id field for 2023_01 ----------------------------- Co-authored-by: Dylan Sprayberry * make jsonvalidator happy ----------------------------- Co-authored-by: Dylan Sprayberry * Remove processing_method, payment_details, gateway for 2023_04 ----------------------------- Co-authored-by: Dylan Sprayberry * Add total_unsettled_set object for 2023_04 ----------------------------- Co-authored-by: Dylan Sprayberry * Add current / original fees sets objects for 2023_04 ----------------------------- Co-authored-by: Dylan Sprayberry * Fix json and collapse newly added objects for readability ----------------------------- Co-authored-by: Dylan Sprayberry * version_bump and changelog ----------------------------- Co-authored-by: Dylan Sprayberry * adjust parallelism to 2 so that we don't api calls per second limit ----------------------------- Co-authored-by: Dylan Sprayberry * Fix whitespace --------- Co-authored-by: Leslie VanDeMark Co-authored-by: alu Co-authored-by: Leslie VanDeMark <38043390+leslievandemark@users.noreply.github.com> --- .circleci/config.yml | 84 ++++++++++++++++++- CHANGELOG.md | 5 ++ setup.cfg | 2 - setup.py | 4 +- tap_shopify/__init__.py | 2 +- tap_shopify/schemas/definitions.json | 2 +- tap_shopify/schemas/orders.json | 113 +++++++++++++------------- tap_shopify/schemas/transactions.json | 56 +++++++++++++ tests/test_bookmarks_updated.py | 2 +- 9 files changed, 202 insertions(+), 68 deletions(-) delete mode 100644 setup.cfg diff --git a/.circleci/config.yml b/.circleci/config.yml index 69fbc452..625c2196 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,10 +2,19 @@ version: 2.1 orbs: slack: circleci/slack@3.4.2 -jobs: - build: +executors: + docker-executor: docker: - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/circle-ci:stitch-tap-tester + +jobs: + build: + executor: docker-executor + steps: + - run: echo "CI Done" + + ensure_env: + executor: docker-executor steps: - checkout - run: @@ -15,16 +24,39 @@ jobs: source /usr/local/share/virtualenvs/tap-shopify/bin/activate pip install -U 'pip<19.2' 'setuptools<51.0.0' pip install .[dev] + - persist_to_workspace: + root: /usr/local/share/virtualenvs + paths: + - tap-shopify + - dev_env.sh + run_pylint: + executor: docker-executor + steps: + - checkout + - attach_workspace: + at: /usr/local/share/virtualenvs - run: name: 'pylint' command: | source /usr/local/share/virtualenvs/tap-shopify/bin/activate pylint tap_shopify -d missing-docstring,too-many-branches + json_validator: + executor: docker-executor + steps: + - checkout + - attach_workspace: + at: /usr/local/share/virtualenvs - run: name: 'JSON Validator' command: | source /usr/local/share/virtualenvs/tap-tester/bin/activate stitch-validate-json tap_shopify/schemas/*.json + run_unit_tests: + executor: docker-executor + steps: + - checkout + - attach_workspace: + at: /usr/local/share/virtualenvs - run: name: 'Unit Tests' command: | @@ -36,6 +68,13 @@ jobs: path: test_output/report.xml - store_artifacts: path: htmlcov + run_integration_tests: + executor: docker-executor + parallelism: 2 + steps: + - checkout + - attach_workspace: + at: /usr/local/share/virtualenvs - run: name: 'Integration Tests' command: | @@ -44,7 +83,14 @@ jobs: mkdir /tmp/${CIRCLE_PROJECT_REPONAME} export STITCH_CONFIG_DIR=/tmp/${CIRCLE_PROJECT_REPONAME} source /usr/local/share/virtualenvs/tap-tester/bin/activate - run-test --tap=tap-shopify tests + circleci tests glob "tests/test_*.py" | circleci tests split > ./tests-to-run + if [ -s ./tests-to-run ]; then + for test_file in $(cat ./tests-to-run) + do + echo $test_file > $STITCH_CONFIG_DIR/tap_test.txt + run-test --tap=${CIRCLE_PROJECT_REPONAME} $test_file + done + fi - slack/notify-on-failure: only_for_branches: master - store_artifacts: @@ -54,10 +100,42 @@ workflows: version: 2 commit: &commit_jobs jobs: + - ensure_env: + context: + - circleci-user + - tier-1-tap-user + - run_pylint: + context: + - circleci-user + - tier-1-tap-user + requires: + - ensure_env + - json_validator: + context: + - circleci-user + - tier-1-tap-user + requires: + - ensure_env + - run_unit_tests: + context: + - circleci-user + - tier-1-tap-user + requires: + - ensure_env + - run_integration_tests: + context: + - circleci-user + - tier-1-tap-user + requires: + - ensure_env - build: context: - circleci-user - tier-1-tap-user + requires: + - run_pylint + - run_unit_tests + - run_integration_tests build_daily: <<: *commit_jobs triggers: diff --git a/CHANGELOG.md b/CHANGELOG.md index e9274846..08abf955 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 1.8.0 + * Updates the Shopify SDK to 12.3.0 + * Updates API version used to 2023_04 + * Adds and removes fields per Shopify API changelog for versions 2022_10, 2023_01, 2023_04 [#178](https://github.com/singer-io/tap-shopify/pull/178) + ## 1.7.6 * Add backoff for 404 error code [#159](https://github.com/singer-io/tap-shopify/pull/159) diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index b88034e4..00000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -description-file = README.md diff --git a/setup.py b/setup.py index dcfa97d4..b85afa27 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.7.6", + version="1.8.0", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", @@ -11,7 +11,7 @@ python_requires='>=3.5.2', py_modules=["tap_shopify"], install_requires=[ - "ShopifyAPI==12.0.1", + "ShopifyAPI==12.3.0", "singer-python==5.12.1", ], extras_require={ diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index 97faf59f..64803e03 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -25,7 +25,7 @@ def initialize_shopify_client(): api_key = Context.config['api_key'] shop = Context.config['shop'] - version = '2022-07' + version = '2023-04' session = shopify.Session(shop, version, api_key) shopify.ShopifyResource.activate_session(session) diff --git a/tap_shopify/schemas/definitions.json b/tap_shopify/schemas/definitions.json index a47a632e..f8e080ff 100644 --- a/tap_shopify/schemas/definitions.json +++ b/tap_shopify/schemas/definitions.json @@ -1481,4 +1481,4 @@ "array" ] } -} \ No newline at end of file +} diff --git a/tap_shopify/schemas/orders.json b/tap_shopify/schemas/orders.json index ed8b533b..26b386ac 100644 --- a/tap_shopify/schemas/orders.json +++ b/tap_shopify/schemas/orders.json @@ -58,12 +58,6 @@ "line_items": { "$ref": "definitions.json#/line_items" }, - "processing_method": { - "type": [ - "null", - "string" - ] - }, "order_number": { "type": [ "null", @@ -414,44 +408,6 @@ ], "format": "singer.decimal" }, - "payment_details": { - "properties": { - "avs_result_code": { - "type": [ - "null", - "string" - ] - }, - "credit_card_company": { - "type": [ - "null", - "string" - ] - }, - "cvv_result_code": { - "type": [ - "null", - "string" - ] - }, - "credit_card_bin": { - "type": [ - "null", - "string" - ] - }, - "credit_card_number": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, "number": { "type": [ "null", @@ -574,13 +530,6 @@ "object" ] }, - "total_price_usd": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, "closed_at": { "type": [ "null", @@ -987,12 +936,6 @@ "integer" ] }, - "gateway": { - "type": [ - "null", - "string" - ] - }, "cart_token": { "type": [ "null", @@ -1278,7 +1221,61 @@ "null", "boolean" ] + }, + "merchant_of_record_app_id": { + "type": ["null", "integer"] + }, + "current_total_additional_fees_set": { + "type": ["null", "object"], + "properties": { + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"], + "format": "singer.decimal"}, + "currency": { + "type": ["null", "string"]} + } + }, + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"], + "format": "singer.decimal"}, + "currency": { + "type": ["null", "string"]} + } + } + } + }, + "original_total_additional_fees_set": { + "type": ["null", "object"], + "properties": { + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"], + "format": "singer.decimal"}, + "currency": { + "type": ["null", "string"] + } + } + }, + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"], + "format": "singer.decimal"}, + "currency": { + "type": ["null", "string"]} + } + } + } } }, "type": "object" -} \ No newline at end of file +} diff --git a/tap_shopify/schemas/transactions.json b/tap_shopify/schemas/transactions.json index 87dcecde..24bfc2fd 100644 --- a/tap_shopify/schemas/transactions.json +++ b/tap_shopify/schemas/transactions.json @@ -111,12 +111,36 @@ "string" ] }, + "credit_card_expiration_month": { + "type": [ + "null", + "integer" + ] + }, + "credit_card_expiration_year": { + "type": [ + "null", + "integer" + ] + }, + "credit_card_name": { + "type": [ + "null", + "string" + ] + }, "credit_card_number": { "type": [ "null", "string" ] }, + "credit_card_wallet": { + "type": [ + "null", + "string" + ] + }, "avs_result_code": { "type": [ "null", @@ -202,6 +226,38 @@ ] } } + }, + "payment_id": { + "type": [ + "null", + "string" + ] + }, + "total_unsettled_set": { + "type": ["null", "object"], + "properties": { + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"], + "format": "singer.decimal"}, + "currency": { + "type": ["null", "string"] + } + } + }, + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"], + "format": "singer.decimal"}, + "currency": { + "type": ["null", "string"]} + } + } + } } }, "type": "object" diff --git a/tests/test_bookmarks_updated.py b/tests/test_bookmarks_updated.py index d590f239..c6d29b3c 100644 --- a/tests/test_bookmarks_updated.py +++ b/tests/test_bookmarks_updated.py @@ -13,7 +13,7 @@ class BookmarkTest(BaseTapTest): """Test tap sets a bookmark and respects it for the next sync of a stream""" @staticmethod def name(): - return "tap_tester_shopify_bookmark_test" + return "tap_tester_shopify_bookmark_update_test" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) From ebda0708011e2917e8a98754c0d009ab429184b7 Mon Sep 17 00:00:00 2001 From: bhtowles Date: Mon, 2 Oct 2023 14:44:33 -0500 Subject: [PATCH 26/66] Update platform for tests (#180) * Update platform for tests * fix date --- tests/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/base.py b/tests/base.py index 85bde65a..8c469635 100644 --- a/tests/base.py +++ b/tests/base.py @@ -37,7 +37,8 @@ def tap_name(): @staticmethod def get_type(): """the expected url route ending""" - return "platform.shopify-tba" + # return "platform.shopify-tba" # moved to alpha Sep 28, 2023 + return "platform.shopify" # new connections after Sep 28, 2023 def get_properties(self, original: bool = True): """Configuration properties required for the tap.""" From e5bfd423a0e73073d5369735a4c054e0bb4576c6 Mon Sep 17 00:00:00 2001 From: bhtowles Date: Thu, 5 Oct 2023 13:56:56 -0500 Subject: [PATCH 27/66] Revert "Update platform for tests (#180)" (#182) This reverts commit ebda0708011e2917e8a98754c0d009ab429184b7. --- tests/base.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/base.py b/tests/base.py index 8c469635..85bde65a 100644 --- a/tests/base.py +++ b/tests/base.py @@ -37,8 +37,7 @@ def tap_name(): @staticmethod def get_type(): """the expected url route ending""" - # return "platform.shopify-tba" # moved to alpha Sep 28, 2023 - return "platform.shopify" # new connections after Sep 28, 2023 + return "platform.shopify-tba" def get_properties(self, original: bool = True): """Configuration properties required for the tap.""" From 4867d3c6be0f0e2f6003925cc32b740a0735d663 Mon Sep 17 00:00:00 2001 From: bhtowles Date: Tue, 17 Oct 2023 12:58:32 -0500 Subject: [PATCH 28/66] Transition to updated platform type (#183) --- tests/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/base.py b/tests/base.py index 85bde65a..919bba9d 100644 --- a/tests/base.py +++ b/tests/base.py @@ -37,7 +37,8 @@ def tap_name(): @staticmethod def get_type(): """the expected url route ending""" - return "platform.shopify-tba" + # return "platform.shopify-tba" # moved to alpha Oct 13, 2023 + return "platform.shopify" # new connections after Oct 13th def get_properties(self, original: bool = True): """Configuration properties required for the tap.""" From fea2b88bfc0d9dcc00acd4d8978232119ff35add Mon Sep 17 00:00:00 2001 From: Leslie VanDeMark <38043390+leslievandemark@users.noreply.github.com> Date: Wed, 24 Jan 2024 11:53:49 -0500 Subject: [PATCH 29/66] Python 3.11 updates (#186) * test tap-shopify on python 3.11 * remove outdated backoff wait generator that was causing errors * pylint updates and version bump * keep on standard base image [skip ci] ----------------------------- --- .circleci/config.yml | 9 ++++----- CHANGELOG.md | 3 +++ setup.py | 6 +++--- tap_shopify/__init__.py | 4 ++-- tap_shopify/streams/base.py | 19 ++----------------- 5 files changed, 14 insertions(+), 27 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 625c2196..4c430e07 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,7 +22,7 @@ jobs: command: | python3 -mvenv /usr/local/share/virtualenvs/tap-shopify source /usr/local/share/virtualenvs/tap-shopify/bin/activate - pip install -U 'pip<19.2' 'setuptools<51.0.0' + pip install -U 'pip==23.3.2' 'setuptools<56.0.0' pip install .[dev] - persist_to_workspace: root: /usr/local/share/virtualenvs @@ -39,7 +39,7 @@ jobs: name: 'pylint' command: | source /usr/local/share/virtualenvs/tap-shopify/bin/activate - pylint tap_shopify -d missing-docstring,too-many-branches + pylint tap_shopify -d missing-docstring,too-many-branches,consider-using-f-string,consider-using-generator,consider-using-dict-items,unnecessary-dunder-call,duplicate-code json_validator: executor: docker-executor steps: @@ -61,9 +61,8 @@ jobs: name: 'Unit Tests' command: | source /usr/local/share/virtualenvs/tap-shopify/bin/activate - pip install nose coverage parameterized - nosetests --with-coverage --cover-erase --cover-package=tap_shopify --cover-html-dir=htmlcov tests/unittests - coverage html + pip install nose2 parameterized nose2[coverage_plugin]>=0.6.5 + nose2 --with-coverage -v -s tests/unittests - store_test_results: path: test_output/report.xml - store_artifacts: diff --git a/CHANGELOG.md b/CHANGELOG.md index 08abf955..62a12422 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 1.9.0 + * Updates to run on python 3.11 [#186](https://github.com/singer-io/tap-shopify/pull/186) + ## 1.8.0 * Updates the Shopify SDK to 12.3.0 * Updates API version used to 2023_04 diff --git a/setup.py b/setup.py index b85afa27..660035a3 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.8.0", + version="1.9.0", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", @@ -12,11 +12,11 @@ py_modules=["tap_shopify"], install_requires=[ "ShopifyAPI==12.3.0", - "singer-python==5.12.1", + "singer-python==6.0.0", ], extras_require={ 'dev': [ - 'pylint==2.7.4', + 'pylint==3.0.3', 'ipdb', 'requests==2.20.0', 'nose', diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index 64803e03..57fddbef 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -47,7 +47,7 @@ def load_schemas(): for filename in os.listdir(get_abs_path('schemas')): path = get_abs_path('schemas') + '/' + filename schema_name = filename.replace('.json', '') - with open(path) as file: + with open(path, encoding='UTF-8') as file: schemas[schema_name] = json.load(file) return schemas @@ -74,7 +74,7 @@ def load_schema_references(): shared_schema_path = get_abs_path('schemas/') refs = {} - with open(os.path.join(shared_schema_path, shared_schema_file)) as data_file: + with open(os.path.join(shared_schema_path, shared_schema_file), encoding='UTF-8') as data_file: refs[shared_schema_file] = json.load(data_file) return refs diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index b67c782e..54482923 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -1,7 +1,5 @@ import datetime import functools -import math -import sys import socket from urllib.error import URLError import http @@ -97,18 +95,6 @@ def retry_handler(details): LOGGER.info("Received 500 or retryable error -- Retry %s/%s", details['tries'], MAX_RETRIES) -#pylint: disable=unused-argument -def retry_after_wait_gen(**kwargs): - # This is called in an except block so we can retrieve the exception - # and check it. - exc_info = sys.exc_info() - resp = exc_info[1].response - # Retry-After is an undocumented header. But honoring - # it was proven to work in our spikes. - # It's been observed to come through as lowercase, so fallback if not present - sleep_time_str = resp.headers.get('Retry-After', resp.headers.get('retry-after')) - yield math.floor(float(sleep_time_str)) - # boolean function to check if the error is 'timeout' error or not def is_timeout_error(error_raised): """ @@ -143,7 +129,7 @@ def shopify_error_handling(fnc): giveup=is_not_status_code_fn([404]), on_backoff=retry_handler, max_tries=MAX_RETRIES) - @backoff.on_exception(retry_after_wait_gen, + @backoff.on_exception(backoff.expo, pyactiveresource.connection.ClientError, giveup=is_not_status_code_fn([429]), on_backoff=leaky_bucket_handler, @@ -258,8 +244,7 @@ def get_objects(self): or updated_at_min + datetime.timedelta(days=date_window_size)) last_sync_interrupted_at = None - if updated_at_max > stop_time: - updated_at_max = stop_time + updated_at_max = min(updated_at_max, stop_time) while True: status_key = self.status_key or "status" query_params = self.get_query_params(since_id, From df35a37cd1289e304abf18809458f664429efc42 Mon Sep 17 00:00:00 2001 From: rdeshmukh15 <107538720+rdeshmukh15@users.noreply.github.com> Date: Thu, 29 Feb 2024 17:44:21 +0530 Subject: [PATCH 30/66] Upgrade Shopify API to 2024-01 (#187) * upgrade shopify api to 2024-01 * 2023-07 schema changes * Version bump & changelog * removed deprecated fields from customer schema * removed few keys from actual keys to check as data is not available for them * refactoring * updated the comment --- CHANGELOG.md | 5 +++++ setup.py | 4 ++-- tap_shopify/__init__.py | 2 +- tap_shopify/schemas/definitions.json | 32 ---------------------------- tap_shopify/schemas/orders.json | 12 +++++++++++ tests/test_all_fields.py | 5 ++++- 6 files changed, 24 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62a12422..f980a9fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 1.10.0 + * Updates the Shopify SDK to 12.3.0 + * Updates API version used to 2024-01 + * Incarporates schema changes [#187](https://github.com/singer-io/tap-shopify/pull/187) + ## 1.9.0 * Updates to run on python 3.11 [#186](https://github.com/singer-io/tap-shopify/pull/186) diff --git a/setup.py b/setup.py index 660035a3..30f2aec2 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.9.0", + version="1.10.0", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", @@ -11,7 +11,7 @@ python_requires='>=3.5.2', py_modules=["tap_shopify"], install_requires=[ - "ShopifyAPI==12.3.0", + "ShopifyAPI==12.4.0", "singer-python==6.0.0", ], extras_require={ diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index 57fddbef..c6323c90 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -25,7 +25,7 @@ def initialize_shopify_client(): api_key = Context.config['api_key'] shop = Context.config['shop'] - version = '2023-04' + version = '2024-01' session = shopify.Session(shop, version, api_key) shopify.ShopifyResource.activate_session(session) diff --git a/tap_shopify/schemas/definitions.json b/tap_shopify/schemas/definitions.json index f8e080ff..95156d5b 100644 --- a/tap_shopify/schemas/definitions.json +++ b/tap_shopify/schemas/definitions.json @@ -81,12 +81,6 @@ "string" ] }, - "currency": { - "type": [ - "null", - "string" - ] - }, "email": { "type": [ "null", @@ -410,26 +404,6 @@ "integer" ] }, - "accepts_marketing": { - "type": [ - "null", - "boolean" - ] - }, - "accepts_marketing_updated_at": { - "anyOf": [ - { - "type": "string" , - "format": "date-time" - }, - { - "type": "string" - }, - { - "type": "null" - } - ] - }, "created_at": { "type": [ "null", @@ -449,12 +423,6 @@ ] } }, - "marketing_opt_in_level": { - "type": [ - "null", - "string" - ] - }, "email_marketing_consent": { "type": [ "null", diff --git a/tap_shopify/schemas/orders.json b/tap_shopify/schemas/orders.json index 26b386ac..df5411ad 100644 --- a/tap_shopify/schemas/orders.json +++ b/tap_shopify/schemas/orders.json @@ -70,6 +70,18 @@ "boolean" ] }, + "taxExempt": { + "type": [ + "null", + "boolean" + ] + }, + "poNumber": { + "type": [ + "null", + "string" + ] + }, "total_discounts": { "type": [ "null", diff --git a/tests/test_all_fields.py b/tests/test_all_fields.py index fd2768b5..e3c8814d 100644 --- a/tests/test_all_fields.py +++ b/tests/test_all_fields.py @@ -97,6 +97,9 @@ def test_run(self): # https://jira.talendforge.org/browse/TDL-15985 # total_price_usd showing up in syncd records Sep 2023, still missing from docs bad_schema_fields = {'order_adjustments'} - expected_all_keys = expected_all_keys - bad_schema_fields + # missing data for 'taxExempt' and 'poNumber' in 'orders' stream + # https://jira.talendforge.org/browse/TDL-25173 + missing_fields = {'taxExempt', 'poNumber'} + expected_all_keys = expected_all_keys - bad_schema_fields - missing_fields self.assertSetEqual(expected_all_keys, actual_all_keys) From 6e3fb7ff6b43dcbc886518a1d4641c4b4c6b3df8 Mon Sep 17 00:00:00 2001 From: bhtowles Date: Wed, 20 Mar 2024 15:20:44 -0500 Subject: [PATCH 31/66] Update interrupted sync orders stream Mar-2024 (#188) --- tests/test_interrupted_sync.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_interrupted_sync.py b/tests/test_interrupted_sync.py index aadff660..508c7cca 100644 --- a/tests/test_interrupted_sync.py +++ b/tests/test_interrupted_sync.py @@ -200,8 +200,8 @@ def test_run(self): # This is the expected behaviour for shopify as they are using date windowing # TDL-17096 : Resuming bookmark value is getting assigned from execution time # rather than the actual bookmark time for some streams. - # TODO this is not the case for the transactions stream, they are equal. - if stream == 'transactions': + # TODO orders and transactions streams are equal, confirm this behavior is correct + if stream == 'transactions' or stream == 'orders': self.assertEqual(resuming_bookmark_value, first_bookmark_value) else: self.assertGreater(resuming_bookmark_value, first_bookmark_value) From c260e11cd0a85d260ffa8802b97c689dca547179 Mon Sep 17 00:00:00 2001 From: bhtowles Date: Wed, 1 May 2024 15:52:23 -0500 Subject: [PATCH 32/66] Revert accidental commit to main (#189) * Update orders stream for interrupted sync again * Revert "Update orders stream for interrupted sync again" This reverts commit 6921131db1fe6e1a5073a392d98559f5eecb39f0. * Push fix the correct way this time, update int sync orders stream * Fix if elif else bug --- tests/test_interrupted_sync.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_interrupted_sync.py b/tests/test_interrupted_sync.py index 508c7cca..d34aff54 100644 --- a/tests/test_interrupted_sync.py +++ b/tests/test_interrupted_sync.py @@ -200,9 +200,12 @@ def test_run(self): # This is the expected behaviour for shopify as they are using date windowing # TDL-17096 : Resuming bookmark value is getting assigned from execution time # rather than the actual bookmark time for some streams. - # TODO orders and transactions streams are equal, confirm this behavior is correct - if stream == 'transactions' or stream == 'orders': + # TODO transactions stream has equal bookmarks, orders stream has shown both equal + # and greater than bookmark behavior, confirm if this is correct + if stream == 'transactions': self.assertEqual(resuming_bookmark_value, first_bookmark_value) + elif stream == 'orders': + self.assertGreaterEqual(resuming_bookmark_value, first_bookmark_value) else: self.assertGreater(resuming_bookmark_value, first_bookmark_value) From 86d6d99dfabb2d047c13bd90e8de362d55b2b90b Mon Sep 17 00:00:00 2001 From: Eivin Giske Skaaren Date: Tue, 3 Sep 2024 12:40:51 +0200 Subject: [PATCH 33/66] Enable copilot usage in PR template according to Qlik policy --- .github/pull_request_template.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index c71d3b31..3ac0252c 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -9,3 +9,7 @@ # Rollback steps - revert this branch + +#### AI generated code +https://internal.qlik.dev/general/ways-of-working/code-reviews/#guidelines-for-ai-generated-code +- [ ] this PR has been written with the help of GitHub Copilot or another generative AI tool From 6e478958a4546df15d8d2a0088b0631b7673992d Mon Sep 17 00:00:00 2001 From: Vishal Pachpinde Date: Fri, 24 Jan 2025 18:27:55 +0530 Subject: [PATCH 34/66] Migrate to GraphQL Streams (#193) GraphQL Migration for deprecated streams --------- Co-authored-by: Sourabh Gandhi Co-authored-by: prijendev --- CHANGELOG.md | 6 + setup.py | 7 +- tap_shopify/__init__.py | 2 +- tap_shopify/schemas/inventory_items.json | 44 ++- tap_shopify/schemas/metafields.json | 28 +- tap_shopify/schemas/product_variants.json | 164 ++++++++ tap_shopify/schemas/products.json | 424 +++++++++++++-------- tap_shopify/schemas/transactions.json | 3 +- tap_shopify/streams/__init__.py | 1 + tap_shopify/streams/base.py | 9 + tap_shopify/streams/graphql/__init__.py | 16 + tap_shopify/streams/graphql/gql_base.py | 127 ++++++ tap_shopify/streams/graphql/gql_queries.py | 412 ++++++++++++++++++++ tap_shopify/streams/inventory_items.py | 82 ++-- tap_shopify/streams/metafields.py | 242 ++++++++---- tap_shopify/streams/product_variants.py | 34 ++ tap_shopify/streams/products.py | 40 +- tests/base.py | 16 +- tests/test_bookmarks.py | 1 + tests/test_bookmarks_metafields.py | 211 ++++++++++ tests/test_bookmarks_updated.py | 4 +- tests/test_interrupted_sync.py | 26 +- tests/unittests/test_graphql_base.py | 106 ++++++ tests/unittests/test_inventory_items.py | 92 ----- tests/unittests/test_timeout.py | 73 ---- 25 files changed, 1661 insertions(+), 509 deletions(-) create mode 100644 tap_shopify/schemas/product_variants.json create mode 100644 tap_shopify/streams/graphql/__init__.py create mode 100644 tap_shopify/streams/graphql/gql_base.py create mode 100644 tap_shopify/streams/graphql/gql_queries.py create mode 100644 tap_shopify/streams/product_variants.py create mode 100644 tests/test_bookmarks_metafields.py create mode 100644 tests/unittests/test_graphql_base.py delete mode 100644 tests/unittests/test_inventory_items.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f980a9fd..be3465cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 2.0.0 + * Deprecated REST Admin API for products + * GraphQL Support added for deprecated streams + * New Stream `Product Variants` added + * More Details here: [#193](https://github.com/singer-io/tap-shopify/pull/193) + ## 1.10.0 * Updates the Shopify SDK to 12.3.0 * Updates API version used to 2024-01 diff --git a/setup.py b/setup.py index 30f2aec2..45748823 100755 --- a/setup.py +++ b/setup.py @@ -1,9 +1,10 @@ #!/usr/bin/env python from setuptools import setup +from setuptools import find_packages setup( name="tap-shopify", - version="1.10.0", + version="2.0.0", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", @@ -11,7 +12,7 @@ python_requires='>=3.5.2', py_modules=["tap_shopify"], install_requires=[ - "ShopifyAPI==12.4.0", + "ShopifyAPI==12.6.0", "singer-python==6.0.0", ], extras_require={ @@ -26,7 +27,7 @@ [console_scripts] tap-shopify=tap_shopify:main """, - packages=["tap_shopify"], + packages=find_packages(), package_data = { "schemas": ["tap_shopify/schemas/*.json"] }, diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index c6323c90..91f25689 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -25,7 +25,7 @@ def initialize_shopify_client(): api_key = Context.config['api_key'] shop = Context.config['shop'] - version = '2024-01' + version = '2024-07' session = shopify.Session(shop, version, api_key) shopify.ShopifyResource.activate_session(session) diff --git a/tap_shopify/schemas/inventory_items.json b/tap_shopify/schemas/inventory_items.json index f6ce0838..47828eaa 100644 --- a/tap_shopify/schemas/inventory_items.json +++ b/tap_shopify/schemas/inventory_items.json @@ -4,7 +4,7 @@ "id": { "type": [ "null", - "integer" + "string" ] }, "sku": { @@ -13,49 +13,57 @@ "string" ] }, - "created_at": { + "createdAt": { "type": [ "null", "string" ], "format": "date-time" }, - "updated_at": { + "updatedAt": { "type": [ "null", "string" ], "format": "date-time" }, - "requires_shipping": { + "requiresShipping": { "type": [ "null", "boolean" ] }, - "cost": { + "unitCost": { "type": [ "null", - "string" + "object" ], - "format": "singer.decimal" + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + } + } }, - "country_code_of_origin": { + "countryCodeOfOrigin": { "type": [ "null", "string" ] }, - "province_code_of_origin": { + "provinceCodeOfOrigin": { "type": [ "null", "string" ] }, - "harmonized_system_code": { + "harmonizedSystemCode": { "type": [ "null", - "integer" + "string" ] }, "tracked": { @@ -64,7 +72,7 @@ "boolean" ] }, - "country_harmonized_system_codes": { + "countryHarmonizedSystemCodes": { "type": [ "null", "array" @@ -75,13 +83,13 @@ "object" ], "properties": { - "harmonized_system_code": { + "countryCode": { "type": [ "null", "string" ] }, - "country_code": { + "harmonizedSystemCode": { "type": [ "null", "string" @@ -89,12 +97,6 @@ } } } - }, - "admin_graphql_api_id": { - "type": [ - "null", - "string" - ] } } -} \ No newline at end of file +} diff --git a/tap_shopify/schemas/metafields.json b/tap_shopify/schemas/metafields.json index 6a2a73b6..839ac7ce 100644 --- a/tap_shopify/schemas/metafields.json +++ b/tap_shopify/schemas/metafields.json @@ -1,18 +1,20 @@ { "properties": { - "owner_id": { + "owner": { "type": [ "null", - "integer" - ] - }, - "admin_graphql_api_id": { - "type": [ - "null", - "string" - ] + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } }, - "owner_resource": { + "ownerType": { "type": [ "null", "string" @@ -30,7 +32,7 @@ "string" ] }, - "created_at": { + "createdAt": { "type": [ "null", "string" @@ -40,7 +42,7 @@ "id": { "type": [ "null", - "integer" + "string" ] }, "namespace": { @@ -64,7 +66,7 @@ ], "properties": {} }, - "updated_at": { + "updatedAt": { "type": [ "null", "string" diff --git a/tap_shopify/schemas/product_variants.json b/tap_shopify/schemas/product_variants.json new file mode 100644 index 00000000..385c6569 --- /dev/null +++ b/tap_shopify/schemas/product_variants.json @@ -0,0 +1,164 @@ +{ + "type": "object", + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "createdAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "barcode": { + "type": [ + "null", + "string" + ] + }, + "availableForSale": { + "type": [ + "null", + "boolean" + ] + }, + "compareAtPrice": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "displayName": { + "type": [ + "null", + "string" + ] + }, + "image": { + "type": [ + "null", + "object" + ], + "properties": { + "altText": { + "type": [ + "null", + "string" + ] + }, + "height": { + "type": [ + "null", + "integer" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "url": { + "type": [ + "null", + "string" + ], + "format": "uri" + }, + "width": { + "type": [ + "null", + "integer" + ] + } + } + }, + "inventoryPolicy": { + "type": [ + "null", + "string" + ] + }, + "inventoryQuantity": { + "type": [ + "null", + "integer" + ] + }, + "position": { + "type": [ + "null", + "integer" + ] + }, + "price": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "requiresComponents": { + "type": [ + "null", + "boolean" + ] + }, + "sellableOnlineQuantity": { + "type": [ + "null", + "integer" + ] + }, + "sku": { + "type": [ + "null", + "string" + ] + }, + "taxCode": { + "type": [ + "null", + "string" + ] + }, + "taxable": { + "type": [ + "null", + "boolean" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "updatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "product": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + } + } +} diff --git a/tap_shopify/schemas/products.json b/tap_shopify/schemas/products.json index 34b8b3b7..4e599b59 100644 --- a/tap_shopify/schemas/products.json +++ b/tap_shopify/schemas/products.json @@ -6,46 +6,34 @@ "string" ] }, - "published_at": { + "publishedAt": { "type": [ "null", "string" ], "format": "date-time" }, - "created_at": { + "createdAt": { "type": [ "null", "string" ], "format": "date-time" }, - "published_scope": { - "type": [ - "null", - "string" - ] - }, "vendor": { "type": [ "null", "string" ] }, - "updated_at": { + "updatedAt": { "type": [ "null", "string" ], "format": "date-time" }, - "body_html": { - "type": [ - "null", - "string" - ] - }, - "product_type": { + "productType": { "type": [ "null", "string" @@ -70,12 +58,6 @@ "string" ] }, - "product_id": { - "type": [ - "null", - "integer" - ] - }, "values": { "type": [ "null", @@ -91,7 +73,7 @@ "id": { "type": [ "null", - "integer" + "string" ] }, "position": { @@ -107,227 +89,357 @@ ] } }, - "image": { - "$ref": "definitions.json#/image" - }, "handle": { "type": [ "null", "string" ] }, - "images": { + "templateSuffix": { "type": [ "null", - "array" - ], - "items": { - "$ref": "definitions.json#/image" - } + "string" + ] }, - "template_suffix": { + "title": { "type": [ "null", "string" ] }, - "title": { + "id": { "type": [ "null", "string" ] }, - "variants": { + "giftCardTemplateSuffix": { "type": [ "null", - "array" - ], - "items": { + "string" + ] + }, + "hasOnlyDefaultVariant": { + "type": [ + "null", + "string" + ] + }, + "hasOutOfStockVariants": { + "type": [ + "null", + "boolean" + ] + }, + "hasVariantsThatRequiresComponents": { + "type": [ + "null", + "boolean" + ] + }, + "isGiftCard": { + "type": [ + "null", + "boolean" + ] + }, + "description": { + "type": [ + "null", + "string" + ] + }, + "descriptionHtml": { + "type": [ + "null", + "string" + ] + }, + "compareAtPriceRange": { + "maxVariantCompareAtPrice": { + "type": [ + "null", + "object" + ], "properties": { - "barcode": { - "type": [ - "null", - "string" - ] - }, - "tax_code": { + "amount": { "type": [ "null", - "string" - ] - }, - "created_at": { - "type": [ - "null", - "string" + "string" ], - "format": "date-time" + "format": "singer.decimal" }, - "weight_unit": { + "currencyCode": { "type": [ "null", "string" ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "position": { - "type": [ - "null", - "integer" - ] - }, - "price": { + } + } + }, + "minVariantCompareAtPrice": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { "type": [ "null", - "string" + "string" ], "format": "singer.decimal" }, - "image_id": { - "type": [ - "null", - "integer" - ] - }, - "inventory_policy": { + "currencyCode": { "type": [ "null", "string" ] - }, - "sku": { + } + } + } + }, + "category": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + }, + "featuredMedia": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "mediaContentType": { + "type": [ + "null", + "string" + ] + }, + "status": { + "type": [ + "null", + "string" + ] + } + } + }, + "requiresSellingPlan": { + "type": [ + "null", + "boolean" + ] + }, + "totalInventory": { + "type": [ + "null", + "integer" + ] + }, + "tracksInventory": { + "type": [ + "null", + "boolean" + ] + }, + "media": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { "type": [ "null", "string" ] }, - "inventory_item_id": { - "type": [ - "null", - "integer" - ] - }, - "fulfillment_service": { + "alt": { "type": [ "null", "string" ] }, - "title": { + "status": { "type": [ "null", "string" ] }, - "weight": { - "type": [ - "null", - "number" - ] - }, - "inventory_management": { + "mediaContentType": { "type": [ "null", "string" ] }, - "taxable": { + "embedUrl": { "type": [ "null", - "boolean" + "string" ] }, - "admin_graphql_api_id": { + "mimeType": { "type": [ "null", "string" ] }, - "option1": { + "filename": { "type": [ "null", "string" ] }, - "compare_at_price": { + "image": { "type": [ "null", - "string" + "object" ], - "format": "singer.decimal" + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "url": { + "type": [ + "null", + "string" + ] + }, + "width": { + "type": [ + "null", + "string" + ] + }, + "height": { + "type": [ + "null", + "string" + ] + } + } }, - "updated_at": { + "sources": { "type": [ "null", - "string" + "array" ], - "format": "date-time" - }, - "option2": { - "type": [ - "null", - "string" - ] - }, - "old_inventory_quantity": { - "type": [ - "null", - "integer" - ] - }, - "requires_shipping": { - "type": [ - "null", - "boolean" - ] - }, - "inventory_quantity": { - "type": [ - "null", - "integer" - ] - }, - "grams": { - "type": [ - "null", - "integer" - ] + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "url": { + "type": [ + "null", + "string" + ] + }, + "format": { + "type": [ + "null", + "string" + ] + }, + "mimeType": { + "type": [ + "null", + "string" + ] + }, + "fileSize": { + "type": [ + "null", + "string" + ] + } + } + } }, - "option3": { + "mediaWarnings": { "type": [ "null", - "string" - ] + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "code": { + "type": [ + "null", + "string" + ] + }, + "message": { + "type": [ + "null", + "string" + ] + } + } + } }, - "product_id": { + "mediaErrors": { "type": [ "null", - "integer" - ] + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "code": { + "type": [ + "null", + "string" + ] + }, + "details": { + "type": [ + "null", + "string" + ] + }, + "message": { + "type": [ + "null", + "string" + ] + } + } + } } - }, - "type": [ - "null", - "object" - ] + } } - }, - "admin_graphql_api_id": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] } }, "type": "object" diff --git a/tap_shopify/schemas/transactions.json b/tap_shopify/schemas/transactions.json index 24bfc2fd..fd5d97d8 100644 --- a/tap_shopify/schemas/transactions.json +++ b/tap_shopify/schemas/transactions.json @@ -83,7 +83,8 @@ "type": [ "null", "string" - ] + ], + "format": "date-time" }, "status": { "type": [ diff --git a/tap_shopify/streams/__init__.py b/tap_shopify/streams/__init__.py index 27a03e90..362f1280 100644 --- a/tap_shopify/streams/__init__.py +++ b/tap_shopify/streams/__init__.py @@ -10,4 +10,5 @@ import tap_shopify.streams.locations import tap_shopify.streams.inventory_levels import tap_shopify.streams.inventory_items +import tap_shopify.streams.product_variants import tap_shopify.streams.events diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index 54482923..3ab2d44e 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -173,6 +173,15 @@ def get_bookmark(self): or Context.config["start_date"]) return utils.strptime_with_tz(bookmark) + def get_bookmark_by_name(self, bookmark_key): + # name is overridden by some substreams + bookmark = (singer.get_bookmark(Context.state, + self.name, + bookmark_key) + or Context.config["start_date"]) + return utils.strptime_with_tz(bookmark) + + def get_since_id(self): return singer.get_bookmark(Context.state, # name is overridden by some substreams diff --git a/tap_shopify/streams/graphql/__init__.py b/tap_shopify/streams/graphql/__init__.py new file mode 100644 index 00000000..d4eec738 --- /dev/null +++ b/tap_shopify/streams/graphql/__init__.py @@ -0,0 +1,16 @@ +""" +tap shopify graphql module +""" + +from .gql_base import ShopifyGqlStream, ShopifyGraphQLError + +from .gql_queries import ( + get_inventory_items_query, + get_metafield_query_collection, + get_metafield_query_order, + get_metafield_query_customers, + get_parent_ids_query, + get_product_variant_query, + get_metafield_query_product, + get_metafield_query_shop, + get_products_query) diff --git a/tap_shopify/streams/graphql/gql_base.py b/tap_shopify/streams/graphql/gql_base.py new file mode 100644 index 00000000..b6ef5b5a --- /dev/null +++ b/tap_shopify/streams/graphql/gql_base.py @@ -0,0 +1,127 @@ +from datetime import timedelta +import json +import shopify + +from singer import( + metrics, + get_logger, + utils +) +from tap_shopify.context import Context +from tap_shopify.streams.base import ( + Stream, + shopify_error_handling, + DATE_WINDOW_SIZE, + ) + + +LOGGER = get_logger() + +class ShopifyGraphQLError(Exception): + """Custom exception for GraphQL errors""" + +class ShopifyGqlStream(Stream): + + data_key = None + + def get_query(self): + """ + Provides GraphQL query + """ + raise NotImplementedError("Function Not Implemented") + + def transform_object(self, obj): + """ + Modify this to perform custom transformation on each object + """ + return obj + + # pylint: disable=W0221 + def get_query_params(self, updated_at_min, updated_at_max, cursor=None): + """ + Construct query parameters for GraphQL requests. + + Args: + updated_at_min (str): Minimum updated_at timestamp. + updated_at_max (str): Maximum updated_at timestamp. + cursor (str): Pagination cursor, if any. + + Returns: + dict: Dictionary of query parameters. + """ + rkey = self.replication_key + params = { + "query": f"{rkey}:>='{updated_at_min}' AND {rkey}:<'{updated_at_max}'", + "first": self.results_per_page, + } + if cursor: + params["after"] = cursor + return params + + @shopify_error_handling + def call_api(self, query_params): + """ + - Modifies the default call api implementation to support GraphQL + - Returns response Object dict + """ + + try: + query = self.get_query() + LOGGER.info("Fetching %s %s", self.name, query_params) + response = shopify.GraphQL().execute(query=query, variables=query_params) + response = json.loads(response) + if "errors" in response.keys(): + raise ShopifyGraphQLError(response['errors']) + data = response.get("data", {}).get(self.data_key, {}) + return data + except ShopifyGraphQLError as gql_error: + LOGGER.error("GraphQL Error %s", gql_error) + raise ShopifyGraphQLError("An error occurred with the GraphQL API.") from gql_error + except Exception as e: + LOGGER.error("Unexpected error occurred.",) + raise e + + def get_objects(self): + """ + Returns: + - Yields list of objects for the stream + Performs + - Pagination & Filtering of stream + - Transformation and bookmarking + """ + + last_updated_at = self.get_bookmark() + current_bookmark = last_updated_at + sync_start = utils.now().replace(microsecond=0) + date_window_size = float(Context.config.get("date_window_size", DATE_WINDOW_SIZE)) + + while last_updated_at < sync_start: + date_window_end = last_updated_at + timedelta(days=date_window_size) + query_end = min(sync_start, date_window_end) + has_next_page, cursor = True, None + + while has_next_page: + query_params = self.get_query_params(last_updated_at, query_end, cursor) + + with metrics.http_request_timer(self.name): + data = self.call_api(query_params) + + for edge in data.get("edges"): + obj = self.transform_object(edge.get("node")) + replication_value = utils.strptime_to_utc(obj[self.replication_key]) + if replication_value > current_bookmark: + current_bookmark = replication_value + yield obj + + page_info = data.get("pageInfo") + cursor , has_next_page = page_info.get("endCursor"), page_info.get("hasNextPage") + + last_updated_at = query_end + self.update_bookmark(utils.strftime(current_bookmark)) + + def sync(self): + """ + Default implementation for sync method + """ + for obj in self.get_objects(): + yield obj diff --git a/tap_shopify/streams/graphql/gql_queries.py b/tap_shopify/streams/graphql/gql_queries.py new file mode 100644 index 00000000..fda6e5cf --- /dev/null +++ b/tap_shopify/streams/graphql/gql_queries.py @@ -0,0 +1,412 @@ +""" +Stores all the GraphQl Queries for shopify api +""" + +def get_products_query(): + """ + Returns GraphQL query to get all products + """ + return """ + query GetProducts($first: Int!, $after: String, $query: String) { + products(first: $first, after: $after, query: $query) { + edges { + node { + id + title + descriptionHtml + vendor + category { + id + } + tags + handle + publishedAt + createdAt + updatedAt + templateSuffix + status + productType + options { + id + name + position + values + } + giftCardTemplateSuffix + hasOnlyDefaultVariant + hasOutOfStockVariants + hasVariantsThatRequiresComponents + isGiftCard + description + compareAtPriceRange { + maxVariantCompareAtPrice { + amount + currencyCode + } + minVariantCompareAtPrice { + amount + currencyCode + } + } + featuredMedia { + id + mediaContentType + status + } + requiresSellingPlan + totalInventory + tracksInventory + media(first: 250) { + edges { + node { + id + alt + status + mediaContentType + mediaWarnings { + code + message + } + mediaErrors { + code + details + message + } + ... on ExternalVideo { + id + embedUrl + } + ... on MediaImage { + id + updatedAt + createdAt + mimeType + image { + url + width + height + id + } + } + ... on Model3d { + id + filename + sources { + url + format + mimeType + filesize + } + } + ... on Video { + id + updatedAt + createdAt + filename + sources { + url + format + mimeType + fileSize + } + + } + } + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + """ + +def get_product_variant_query(): + """ + Returns GraphQL query to get all product variants + """ + return """ + query GetProductVariants($first: Int!, $after: String, $query: String) { + productVariants(first: $first, after: $after, query: $query) { + edges { + node { + id + createdAt + barcode + availableForSale + compareAtPrice + displayName + image { + altText + height + id + url + width + } + inventoryPolicy + inventoryQuantity + position + price + requiresComponents + sellableOnlineQuantity + sku + taxCode + taxable + title + updatedAt + product { id } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + """ + +def get_inventory_items_query(): + """ + Returns GraphQL query to get all inventory items + """ + return """ + query GetinventoryItems($first: Int!, $after: String, $query: String) { + inventoryItems(first: $first, after: $after, query: $query) { + edges { + node { + id + createdAt + sku + updatedAt + requiresShipping + countryCodeOfOrigin + provinceCodeOfOrigin + harmonizedSystemCode + tracked + unitCost { + amount + } + countryHarmonizedSystemCodes(first: 175) { + edges { + node { + countryCode + harmonizedSystemCode + } + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + """ + +def get_parent_ids_query(resource): + """ + Returns GraphQL query to get id for all metaftield + supported parent streams. + """ + + qry = """ + query getParentEntities( $first: Int!, $after: String $query: String) { + RESOURCE(first: $first after: $after query: $query) { + edges { + node { + id + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + qry = qry.replace("RESOURCE", resource) + return qry + +def get_metafield_query_customers(): + """ + Returns GraphQL query to get customer metafields + """ + qry = """ + query GetMetafields($pk_id: ID! $first: Int!, $after: String) { + customer(id: $pk_id) { + metafields(first: $first after: $after ){ + edges { + node { + id + ownerType + value + type + key + createdAt + namespace + description + updatedAt + owner { + ... on Customer { + id + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + } + """ + return qry + +def get_metafield_query_product(): + """ + Returns GraphQL query to get product metafields + """ + qry = """ + query GetMetafields($pk_id: ID! $first: Int!, $after: String) { + product(id: $pk_id) { + metafields(first: $first after: $after ){ + edges { + node { + id + ownerType + value + type + key + createdAt + namespace + description + updatedAt + owner { + ... on Product { + id + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + } + """ + return qry + +def get_metafield_query_collection(): + """ + Returns GraphQL query to get collection metafields + """ + qry = """ + query GetMetafields($pk_id: ID! $first: Int!, $after: String) { + collection(id: $pk_id) { + metafields(first: $first after: $after ){ + edges { + node { + id + ownerType + value + type + key + createdAt + namespace + description + updatedAt + owner { + ... on Collection { + id + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + } + """ + return qry + +def get_metafield_query_order(): + """ + Returns GraphQL query to get order metafields + """ + qry = """ + query GetMetafields($pk_id: ID! $first: Int!, $after: String) { + order(id: $pk_id) { + metafields(first: $first after: $after ){ + edges { + node { + id + ownerType + value + type + key + createdAt + namespace + description + updatedAt + owner { + ... on Order { + id + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + } + """ + return qry + +def get_metafield_query_shop(): + """ + Returns GraphQL query to get shop metafields + """ + qry = """ + query GetMetafields($first: Int!, $after: String) { + shop{ + metafields(first: $first after: $after ){ + edges { + node { + id + ownerType + value + type + key + createdAt + namespace + description + updatedAt + owner { + ... on Shop { + id + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + } + """ + return qry diff --git a/tap_shopify/streams/inventory_items.py b/tap_shopify/streams/inventory_items.py index 8b2dc676..641ff969 100644 --- a/tap_shopify/streams/inventory_items.py +++ b/tap_shopify/streams/inventory_items.py @@ -1,56 +1,40 @@ -import singer -import shopify -from singer.utils import strftime,strptime_to_utc -from tap_shopify.streams.base import (Stream, shopify_error_handling) from tap_shopify.context import Context +from tap_shopify.streams.graphql import get_inventory_items_query +from tap_shopify.streams.graphql import ShopifyGqlStream -LOGGER = singer.get_logger() -RESULTS_PER_PAGE = 250 -class InventoryItems(Stream): +class InventoryItems(ShopifyGqlStream): name = 'inventory_items' - replication_object = shopify.InventoryItem - - @shopify_error_handling - def get_inventory_items(self, inventory_items_ids): - # set timeout - self.replication_object.set_timeout(self.request_timeout) - return self.replication_object.find( - ids=inventory_items_ids, - limit=RESULTS_PER_PAGE) - - def get_objects(self): - - selected_parent = Context.stream_objects['products']() - selected_parent.name = "product_variants" - - # Page through all `products`, bookmarking at `product_variants` - for parent_object in selected_parent.get_objects(): - - product_variants = parent_object.variants - inventory_items_ids = ",".join( - [str(product_variant.inventory_item_id) for product_variant in product_variants]) - - # Max limit of IDs is 100 and Max limit of product_variants in one product is also 100 - # hence we can directly pass all inventory_items_ids - inventory_items = self.get_inventory_items(inventory_items_ids) - - for inventory_item in inventory_items: - yield inventory_item - - def sync(self): - bookmark = self.get_bookmark() - max_bookmark = bookmark - for inventory_item in self.get_objects(): - inventory_item_dict = inventory_item.to_dict() - replication_value = strptime_to_utc(inventory_item_dict[self.replication_key]) - if replication_value >= bookmark: - yield inventory_item_dict - - if replication_value > max_bookmark: - max_bookmark = replication_value - - self.update_bookmark(strftime(max_bookmark)) + data_key = "inventoryItems" + replication_key = "updatedAt" + + def get_query(self): + return get_inventory_items_query() + + # pylint: disable=W0221 + def get_query_params(self, updated_at_min, updated_at_max, cursor=None): + """ + Returns query and params for filtering, pagination + """ + filter_key = "updated_at" + params = { + "query": f"{filter_key}:>='{updated_at_min}' AND {filter_key}:<'{updated_at_max}'", + "first": self.results_per_page, + } + if cursor: + params["after"] = cursor + return params + + def transform_object(self, obj): + hsc = obj.get("countryHarmonizedSystemCodes") + hsc_list = [] + if hsc and "edges" in hsc: + for edge in hsc.get("edges"): + node = edge.get("node") + if node: + hsc_list.append(node) + obj["countryHarmonizedSystemCodes"] = hsc_list + return obj Context.stream_objects['inventory_items'] = InventoryItems diff --git a/tap_shopify/streams/metafields.py b/tap_shopify/streams/metafields.py index 2c62d1d0..6264358c 100644 --- a/tap_shopify/streams/metafields.py +++ b/tap_shopify/streams/metafields.py @@ -1,82 +1,182 @@ +from datetime import timedelta import json import shopify -import singer + +from singer import utils, get_logger from tap_shopify.context import Context -from tap_shopify.streams.base import (Stream, - shopify_error_handling, - RESULTS_PER_PAGE, - OutOfOrderIdsError) - -LOGGER = singer.get_logger() - -def get_selected_parents(): - for parent_stream in ['orders', 'customers', 'products', 'custom_collections']: - if Context.is_selected(parent_stream): - yield Context.stream_objects[parent_stream]() - -@shopify_error_handling -def get_metafields(parent_object, since_id, parent_replication_object, timeout): - # set timeout - parent_replication_object.set_timeout(timeout) - # This call results in an HTTP request - the parent object never has a - # cache of this data so we have to issue that request. - return parent_object.metafields( - limit=Context.get_results_per_page(RESULTS_PER_PAGE), - since_id=since_id) - -class Metafields(Stream): +from tap_shopify.streams.graphql import ( + get_parent_ids_query, + get_metafield_query_customers, + get_metafield_query_product, + get_metafield_query_collection, + get_metafield_query_order, + get_metafield_query_shop, +) +from tap_shopify.streams.graphql.gql_base import ( + ShopifyGqlStream, + ShopifyGraphQLError, + DATE_WINDOW_SIZE, + shopify_error_handling, + ) + + +LOGGER = get_logger() + + + +class Metafields(ShopifyGqlStream): name = 'metafields' - replication_object = shopify.Metafield + data_key = "metafields" + replication_key = "updatedAt" + + selected_parent = None + + parent_alias = { + "custom_collections":"collections" + } + + + # maps object list identifier to single object access identifier + # eg customers -> customer + resource_alias = { + "customers":"customer", + "products":"product", + "collections": "collection", + "orders": "order" + } + + def get_query(self): + return None + + # pylint: disable=W0221 + def get_query_params(self, updated_at_min, updated_at_max, cursor=None): + """ + Returns query and params for filtering, pagination + """ + rkey = "updated_at" + params = { + "query": f"{rkey}:>='{updated_at_min}' AND {rkey}:<'{updated_at_max}'", + "first": self.results_per_page, + } + if cursor: + params["after"] = cursor + return params + + def get_resource_type_query(self, resource): + return { + "customer": get_metafield_query_customers, + "product": get_metafield_query_product, + "collection": get_metafield_query_collection, + "order": get_metafield_query_order, + "shop":get_metafield_query_shop + }.get(resource) + + @shopify_error_handling + def call_api(self, query_params, query, data_key): + LOGGER.info("Fetching %s %s", self.name, query_params) + response = shopify.GraphQL().execute(query=query, variables=query_params) + response = json.loads(response) + if "errors" in response.keys(): + raise ShopifyGraphQLError(response['errors']) + data = response.get("data", {}).get(data_key, {}) + return data + + def get_parents(self): + sync_start = utils.now().replace(microsecond=0) + for parent in ['shop','orders', 'customers', 'products', 'custom_collections']: + parent = self.parent_alias.get(parent, parent) + resource_alias = self.resource_alias.get(parent, parent) + LOGGER.info("Fetching id's for %s %s", parent, resource_alias) + + if parent == "shop": + yield None, resource_alias + continue + + last_updated_at = self.get_bookmark_by_name(f'{self.name}_{resource_alias}') + date_window_size = float(Context.config.get("date_window_size", DATE_WINDOW_SIZE)) + + while last_updated_at < sync_start: + + date_window_end = last_updated_at + timedelta(days=date_window_size) + query_end = min(sync_start, date_window_end) + has_next_page, cursor = True, None + + while has_next_page: + query_params = self.get_query_params(last_updated_at, query_end, cursor) + query = get_parent_ids_query(parent) + data = self.call_api(query_params, query, parent) + LOGGER.info("%s : %s", parent, len(data.get("edges"))) + + for edge in data.get("edges"): + yield (edge.get("node"), resource_alias) + + page_info = data.get("pageInfo") + cursor = page_info.get("endCursor") + has_next_page = page_info.get("hasNextPage") + + last_updated_at = query_end def get_objects(self): - # Get top-level shop metafields - yield from super().get_objects() - # Get parent objects, bookmarking at `metafield_` - for selected_parent in get_selected_parents(): - # The name member controls many things, but most importantly - # the bookmark key. This switches us over to the - # `metafield_` bookmark. We track that separately - # to make resetting individual streams easier. - selected_parent.name = "metafield_{}".format(selected_parent.name) - for parent_object in selected_parent.get_objects(): - since_id = 1 - while True: - metafields = get_metafields(parent_object, - since_id, - selected_parent.replication_object, - self.request_timeout) - for metafield in metafields: - if metafield.id < since_id: - raise OutOfOrderIdsError("metafield.id < since_id: {} < {}".format( - metafield.id, since_id)) - yield metafield - if len(metafields) < self.results_per_page: - break - if metafields[-1].id != max([o.id for o in metafields]): - raise OutOfOrderIdsError("{} is not the max id in metafields ({})".format( - metafields[-1].id, max([o.id for o in metafields]))) - since_id += metafields[-1].id + + for parent_obj, resource_type in self.get_parents(): + qury_fnc = self.get_resource_type_query(resource_type) + + if qury_fnc: + query = qury_fnc() + else: + raise ShopifyGraphQLError("Invalid Resource Type") + + has_next_page, cursor = True, None + query_params = {"first": self.results_per_page} + if resource_type != "shop": + query_params["pk_id"] = parent_obj["id"] + + while has_next_page: + if cursor: + query_params["after"] = cursor + response = self.call_api(query_params, query, resource_type) + data = (response.get("metafields") or {}) + + for edge in data.get("edges"): + obj = self.transform_object(edge.get("node")) + yield (obj, resource_type) + + page_info = data.get("pageInfo") + cursor, has_next_page = page_info.get("endCursor"), page_info.get("hasNextPage") + + def transform_object(self, obj): + obj["value_type"] = obj["type"] or None + obj["updated_at"] = obj["updatedAt"] + if obj["value_type"] in ["json", "weight", "volume", "dimension", "rating"]: + value = obj.get("value") + try: + obj["value"] = json.loads(value) if value is not None else value + except json.decoder.JSONDecodeError: + LOGGER.info("Failed to decode JSON value for obj %s", obj.get('id')) + obj["value"] = value + return obj def sync(self): - # Shop metafields - for metafield in self.get_objects(): - metafield = metafield.to_dict() - metafield_type = metafield.get("type") - # create "value_type" field in the record - metafield["value_type"] = metafield_type - # the json_string value in "value_type" field will be - # mapped to following "type" value in the new version - # Reference: https://shopify.dev/apps/metafields/types - if metafield_type and metafield_type in ["json", "weight", "volume", \ - "dimension", "rating"]: - value = metafield.get("value") - try: - metafield["value"] = json.loads(value) if value is not None else value - except json.decoder.JSONDecodeError: - LOGGER.info("Failed to decode JSON value for metafield %s", metafield.get('id')) - metafield["value"] = value - - yield metafield + start_time = utils.now().replace(microsecond=0) + current_bookmarks = {} + last_bookmarks = {} + + for obj, resource_type in self.get_objects(): + replication_value = utils.strptime_to_utc(obj[self.replication_key]) + + if resource_type not in current_bookmarks or resource_type not in last_bookmarks: + bookmark = self.get_bookmark_by_name(f"{self.name}_{resource_type}") + last_bookmarks[resource_type] = bookmark + current_bookmarks[resource_type] = bookmark + + if replication_value >= last_bookmarks[resource_type]: + current_bookmarks[resource_type] = max(replication_value, \ + current_bookmarks[resource_type]) + yield obj + + for res, bookmark_val in current_bookmarks.items(): + bookmark_val = min(start_time, bookmark_val) + self.update_bookmark(utils.strftime(bookmark_val), f"{self.name}_{res}") Context.stream_objects['metafields'] = Metafields diff --git a/tap_shopify/streams/product_variants.py b/tap_shopify/streams/product_variants.py new file mode 100644 index 00000000..080bbc40 --- /dev/null +++ b/tap_shopify/streams/product_variants.py @@ -0,0 +1,34 @@ + +from tap_shopify.context import Context +from tap_shopify.streams.graphql import get_product_variant_query +from tap_shopify.streams.graphql import ShopifyGqlStream + + + +class ProductVariants(ShopifyGqlStream): + name = 'product_variants' + data_key = "productVariants" + replication_key = "updatedAt" + + def get_query(self): + return get_product_variant_query() + + # pylint: disable=W0221 + def get_query_params(self, updated_at_min, updated_at_max, cursor=None): + """ + Returns query and params for filtering, pagination + """ + filter_key = "updated_at" + params = { + "query": f"{filter_key}:>='{updated_at_min}' AND {filter_key}:<'{updated_at_max}'", + "first": self.results_per_page, + } + if cursor: + params["after"] = cursor + return params + + + def transform_object(self, obj): + return obj + +Context.stream_objects['product_variants'] = ProductVariants diff --git a/tap_shopify/streams/products.py b/tap_shopify/streams/products.py index fbfcb6a0..07da3d26 100644 --- a/tap_shopify/streams/products.py +++ b/tap_shopify/streams/products.py @@ -1,12 +1,42 @@ -import shopify -from tap_shopify.streams.base import Stream from tap_shopify.context import Context +from tap_shopify.streams.graphql import get_products_query +from tap_shopify.streams.graphql import ShopifyGqlStream -class Products(Stream): + +class Products(ShopifyGqlStream): name = 'products' - replication_object = shopify.Product - status_key = "published_status" + data_key = "products" + replication_key = "updatedAt" + + def get_query(self): + return get_products_query() + + # pylint: disable=W0221 + def get_query_params(self, updated_at_min, updated_at_max, cursor=None): + """ + Returns query and params for filtering, pagination + """ + filter_key = "updated_at" + params = { + "query": f"{filter_key}:>='{updated_at_min}' AND {filter_key}:<'{updated_at_max}'", + "first": self.results_per_page, + } + if cursor: + params["after"] = cursor + return params + + + def transform_object(self, obj): + media = obj.get("media") + media_list = [] + if media and "edges" in media: + for edge in media.get("edges"): + node = edge.get("node") + if node: + media_list.append(node) + obj["media"] = media_list + return obj Context.stream_objects['products'] = Products diff --git a/tests/base.py b/tests/base.py index 919bba9d..730e8538 100644 --- a/tests/base.py +++ b/tests/base.py @@ -83,7 +83,8 @@ def expected_metadata(self): self.API_LIMIT: self.DEFAULT_RESULTS_PER_PAGE} meta = default.copy() - meta.update({self.FOREIGN_KEYS: {"owner_id", "owner_resource"}}) + meta[self.REPLICATION_KEYS] = {"updatedAt"} + meta.update({self.FOREIGN_KEYS: {"owner", "ownerType"}}) return { "abandoned_checkouts": { @@ -101,8 +102,17 @@ def expected_metadata(self): self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.API_LIMIT: self.DEFAULT_RESULTS_PER_PAGE}, - "products": default, - "inventory_items": {self.REPLICATION_KEYS: {"updated_at"}, + "products": { + self.REPLICATION_KEYS: {"updatedAt"}, + self.PRIMARY_KEYS: {"id"}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.API_LIMIT: self.DEFAULT_RESULTS_PER_PAGE}, + "product_variants": { + self.REPLICATION_KEYS: {"updatedAt"}, + self.PRIMARY_KEYS: {"id"}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.API_LIMIT: self.DEFAULT_RESULTS_PER_PAGE}, + "inventory_items": {self.REPLICATION_KEYS: {"updatedAt"}, self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.API_LIMIT: 250}, diff --git a/tests/test_bookmarks.py b/tests/test_bookmarks.py index d0dfd062..d7a1f3a7 100644 --- a/tests/test_bookmarks.py +++ b/tests/test_bookmarks.py @@ -85,6 +85,7 @@ def bookmarks_test(self, conn_id, testable_streams): found_catalogs = menagerie.get_catalogs(conn_id) incremental_streams = {key for key, value in self.expected_replication_method().items() if value == self.INCREMENTAL and key in testable_streams} + incremental_streams = incremental_streams - {'metafields'} # Created a separate test for metafields # Our test data sets for Shopify do not have any abandoned_checkouts our_catalogs = [catalog for catalog in found_catalogs if diff --git a/tests/test_bookmarks_metafields.py b/tests/test_bookmarks_metafields.py new file mode 100644 index 00000000..2d33a6df --- /dev/null +++ b/tests/test_bookmarks_metafields.py @@ -0,0 +1,211 @@ +""" +Test tap sets a bookmark and respects it for the next sync of a stream +""" +from datetime import datetime as dt, timezone + +from dateutil.parser import parse + +from tap_tester import menagerie, runner, LOGGER +from base import BaseTapTest + +class BookmarkMetafieldsTest(BaseTapTest): + """Test tap sets a bookmark and respects it for the next sync of a stream""" + @staticmethod + def name(): + return "tap_tester_shopify_bookmark_metafields_test" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.start_date = '2025-01-01T00:00:00Z' + self.metafields_dict = {} + + def min_max_bookmarks_by_stream(self, sync_records): + """ + Return the maximum value for the replication key for each stream + which is the bookmark expected value. + + Comparisons are based on the class of the bookmark value. Dates will be + string compared which works for ISO date-time strings + """ + max_bookmarks = {} + min_bookmarks = {} + for stream, batch in sync_records.items(): + for message in batch.get('messages'): + if message['action'] == 'upsert': + data = message.get('data') + if data.get('ownerType') == "SHOP": + self.metafields_dict.setdefault("metafields_shop", []).append(data) + elif data.get('ownerType') == "ORDER": + self.metafields_dict.setdefault("metafields_order", []).append(data) + elif data.get('ownerType') == "PRODUCT": + self.metafields_dict.setdefault("metafields_product", []).append(data) + elif data.get('ownerType') == "CUSTOMER": + self.metafields_dict.setdefault("metafields_customer", []).append(data) + elif data.get('ownerType') == "COLLECTION": + self.metafields_dict.setdefault("metafields_collection", []).append(data) + + stream_bookmark_key = self.expected_replication_keys().get(stream, set()) + assert len(stream_bookmark_key) == 1 # There shouldn't be a compound replication key + stream_bookmark_key = stream_bookmark_key.pop() + + for metafields_owner, messages in self.metafields_dict.items(): + max_bookmarks, min_bookmarks = self.fetch_max_min(metafields_owner, stream_bookmark_key, messages, max_bookmarks, min_bookmarks) + + max_bookmarks["metafields"] = max_bookmarks + min_bookmarks["metafields"] = min_bookmarks + + return max_bookmarks, min_bookmarks + + + def fetch_max_min(self, owner_type, stream_bookmark_key, messages, max_bookmarks, min_bookmarks): + """ + Updates the maximum and minimum bookmark values for a given owner type based on a list of messages. + + Returns: + tuple: Updated dictionaries for maximum and minimum bookmark values. + """ + bk_values = [message.get(stream_bookmark_key) for message in messages] + max_bookmarks[owner_type] = None + min_bookmarks[owner_type] = None + for bk_value in bk_values: + if bk_value is None: + continue + + if max_bookmarks[owner_type] is None: + max_bookmarks[owner_type] = bk_value + + if bk_value > max_bookmarks[owner_type]: + max_bookmarks[owner_type] = bk_value + + if min_bookmarks[owner_type] is None: + min_bookmarks[owner_type] = bk_value + + if bk_value < min_bookmarks[owner_type]: + min_bookmarks[owner_type] = bk_value + return max_bookmarks, min_bookmarks + + def test_run(self): + """ + Verify that metafields stream you can do a sync which records bookmarks. + That the bookmark is the maximum value sent to the target for the replication key. + That a second sync respects the bookmark + All data of the second sync is > the bookmark from the first sync + The number of records in the 2nd sync is less then the first (This assumes that + new data added to the stream is done at a rate slow enough that you haven't + doubled the amount of data from the start date to the first sync between + the first sync and second sync run in this test) + + Verify that only data for incremental streams is sent to the target + + PREREQUISITE + For metafields stream that is incrementally replicated there are multiple rows of data with + different values for the replication key + """ + conn_id = self.create_connection(original_credentials=True) + + # Select all streams and no fields within streams + found_catalogs = menagerie.get_catalogs(conn_id) + incremental_stream = {'metafields'} + + # Our test data sets for Shopify do not have any abandoned_checkouts + our_catalogs = [catalog for catalog in found_catalogs if + catalog.get('tap_stream_id') in incremental_stream] + self.select_all_streams_and_fields(conn_id, our_catalogs, select_all_fields=True) + + # Run a sync job using orchestrator + first_sync_record_count = self.run_sync(conn_id) + + # verify that the sync only sent records to the target for selected streams (catalogs) + self.assertEqual(set(first_sync_record_count.keys()), + incremental_stream) + + first_sync_state = menagerie.get_state(conn_id) + + # Get data about actual rows synced + first_sync_records = runner.get_records_from_target_output() + + # Run a second sync job using orchestrator + second_sync_record_count = self.run_sync(conn_id) + + # Get data about rows synced + second_sync_records = runner.get_records_from_target_output() + + first_max_bookmarks, first_min_bookmarks = self.min_max_bookmarks_by_stream(first_sync_records) + + _ , second_min_bookmarks = self.min_max_bookmarks_by_stream(second_sync_records) + + for stream in incremental_stream: + with self.subTest(stream=stream): + + # Verify that you get less data the 2nd time around + self.assertGreater( + first_sync_record_count.get(stream, 0), + second_sync_record_count.get(stream, 0), + msg="second sync didn't have less records, bookmark usage not verified") + + # Get bookmark values from state and target data + stream_bookmark_key = self.expected_replication_keys().get(stream, set()) + assert len( + stream_bookmark_key) == 1 # There shouldn't be a compound replication key + stream_bookmark_key = stream_bookmark_key.pop() + + for metafield_key in self.metafields_dict.keys(): + + state_value = first_sync_state.get("bookmarks", {}).get( + stream, {None: None}).get(metafield_key) + target_value = first_max_bookmarks.get( + stream, {None: None}).get(metafield_key) + target_min_value = first_min_bookmarks.get( + stream, {None: None}).get(metafield_key) + + try: + # Attempt to parse the bookmark as a date + if state_value: + if isinstance(state_value, str): + state_value = self.local_to_utc(parse(state_value)) + if isinstance(state_value, int): + state_value = self.local_to_utc(dt.fromtimestamp(state_value, tz=timezone.utc)) + + if target_value: + if isinstance(target_value, str): + target_value = self.local_to_utc(parse(target_value)) + if isinstance(target_value, int): + target_value = self.local_to_utc(dt.fromtimestamp(target_value, tz=timezone.utc)) + + if target_min_value: + if isinstance(target_min_value, str): + target_min_value = self.local_to_utc(parse(target_min_value)) + if isinstance(target_min_value, int): + target_min_value = self.local_to_utc( + dt.fromtimestamp(target_min_value, tz=timezone.utc)) + + except (OverflowError, ValueError, TypeError): + LOGGER.warn("Bookmarks cannot be converted to dates, comparing values directly") + + # Verify that there is data with different bookmark values - setup necessary + self.assertGreaterEqual(target_value, target_min_value, + msg="Data isn't set up to be able to test bookmarks") + + # Verify state agrees with target data after 1st sync + self.assertGreaterEqual(state_value, target_value, + msg="The bookmark value isn't correct based on target data") + + # Verify all data from 2nd sync >= 1st bookmark + target_value = second_min_bookmarks.get( + stream, {None: None}).get(metafield_key) + try: + if target_value: + if isinstance(target_value, str): + target_value = self.local_to_utc(parse(target_value)) + if isinstance(target_value, int): + target_value = self.local_to_utc(dt.fromtimestamp(target_value, tz=timezone.utc)) + + except (OverflowError, ValueError, TypeError): + LOGGER.warn("bookmarks cannot be converted to dates, comparing values directly") + + for message in second_sync_records.get(stream).get('messages'): + if message['action'] == 'upsert': + if message.get('data').get('ownerType') == metafield_key: + self.assertGreaterEqual( + message.get('data').get(stream_bookmark_key), target_value, + msg="Data from 2nd sync is not all >= bookmark from 1st sync") diff --git a/tests/test_bookmarks_updated.py b/tests/test_bookmarks_updated.py index c6d29b3c..f7d8c93f 100644 --- a/tests/test_bookmarks_updated.py +++ b/tests/test_bookmarks_updated.py @@ -77,7 +77,7 @@ def bookmarks_test(self, conn_id, testable_streams): #simulated_states = self.calculated_states_by_stream(first_sync_bookmark) # We are hardcoding the updated state to ensure that we get atleast 1 record in second sync. These values have been provided after reviewing the max bookmark value for each of the streams - simulated_states = {'products': {'updated_at': '2021-12-20T05:10:05.000000Z'}, 'collects': {'updated_at': '2021-09-01T09:08:28.000000Z'}, 'abandoned_checkouts': {'updated_at': '2022-02-02T16:00:00.000000Z'}, 'inventory_levels': {'updated_at': '2021-12-20T05:09:34.000000Z'}, 'locations': {'updated_at': '2021-07-20T09:00:22.000000Z'}, 'events': {'created_at': '2021-12-20T05:09:01.000000Z'}, 'inventory_items': {'updated_at': '2021-09-15T19:44:11.000000Z'}, 'transactions': {'created_at': '2021-12-20T00:08:52-05:00'}, 'metafields': {'updated_at': '2021-09-07T21:18:05.000000Z'}, 'order_refunds': {'created_at': '2021-05-01T17:41:18.000000Z'}, 'customers': {'updated_at': '2021-12-20T05:08:17.000000Z'}, 'orders': {'updated_at': '2021-12-20T05:09:01.000000Z'}, 'custom_collections': {'updated_at': '2021-12-20T17:41:18.000000Z'}} + simulated_states = {'products': {'updatedAt': '2024-09-14T03:01:11.000000Z'}, 'collects': {'updated_at': '2021-09-01T09:08:28.000000Z'}, 'abandoned_checkouts': {'updated_at': '2022-02-02T16:00:00.000000Z'}, 'inventory_levels': {'updated_at': '2021-12-20T05:09:34.000000Z'}, 'locations': {'updated_at': '2021-07-20T09:00:22.000000Z'}, 'events': {'created_at': '2021-12-20T05:09:01.000000Z'}, 'inventory_items': {'updatedAt': '2021-09-15T19:44:11.000000Z'}, 'transactions': {'created_at': '2021-12-20T00:08:52-05:00'}, 'metafields': {'metafields_customer': '2025-01-21T13:28:24.000000Z'}, 'order_refunds': {'created_at': '2021-05-01T17:41:18.000000Z'}, 'customers': {'updated_at': '2021-12-20T05:08:17.000000Z'}, 'orders': {'updated_at': '2021-12-20T05:09:01.000000Z'}, 'custom_collections': {'updated_at': '2024-12-13T08:42:56.000000Z'}} for stream, updated_state in simulated_states.items(): new_state['bookmarks'][stream] = updated_state @@ -148,7 +148,7 @@ def bookmarks_test(self, conn_id, testable_streams): for record in second_sync_messages: replication_key_value = record.get(replication_key) # verify the 2nd sync replication key value is greater or equal to the 1st sync bookmarks - self.assertGreaterEqual(replication_key_value, simulated_bookmark_value, msg="Second sync records do not respect the previous bookmark") + self.assertGreaterEqual(self.convert_state_to_utc(replication_key_value), simulated_bookmark_value, msg="Second sync records do not respect the previous bookmark") # verify the 2nd sync bookmark value is the max replication key value for a given stream self.assertLessEqual(replication_key_value, second_bookmark_value_utc, msg="Second sync bookmark was set incorrectly, a record with a greater replication key value was synced") diff --git a/tests/test_interrupted_sync.py b/tests/test_interrupted_sync.py index d34aff54..70a21a07 100644 --- a/tests/test_interrupted_sync.py +++ b/tests/test_interrupted_sync.py @@ -38,7 +38,6 @@ def test_run(self): expected_streams = {'customers', 'events', - 'metafields', 'orders', 'products', 'transactions'} @@ -89,12 +88,11 @@ def test_run(self): base_state = {'bookmarks': {'currently_sync_stream': currently_syncing_stream, - 'customers': {'updated_at': '2023-03-28T18:53:28.000000Z'}, - 'events': {'created_at': '2023-01-22T05:05:53.000000Z'}, - 'metafields': {'updated_at': '2023-01-07T21:18:05.000000Z'}, - 'orders': {'updated_at': '2023-01-22T05:07:44.000000Z'}, - 'products': {'updated_at': '2023-01-22T05:05:56.000000Z'}, - 'transactions': {'created_at': '2022-06-26T00:06:38-04:00'} + 'customers': first_sync_state.get('bookmarks').get('customers'), + 'events': first_sync_state.get('bookmarks').get('events'), + 'orders': first_sync_state.get('bookmarks').get('orders'), + 'products': first_sync_state.get('bookmarks').get('products'), + 'transactions': first_sync_state.get('bookmarks').get('transactions') }} # remove yet to be synced streams from base state and then set new state @@ -196,18 +194,8 @@ def test_run(self): self.assertIsNotNone(resuming_bookmark_value) self.assertTrue(self.is_expected_date_format(resuming_bookmark_value)) - # verify the resuming bookmark is greater than 1st sync bookmark - # This is the expected behaviour for shopify as they are using date windowing - # TDL-17096 : Resuming bookmark value is getting assigned from execution time - # rather than the actual bookmark time for some streams. - # TODO transactions stream has equal bookmarks, orders stream has shown both equal - # and greater than bookmark behavior, confirm if this is correct - if stream == 'transactions': - self.assertEqual(resuming_bookmark_value, first_bookmark_value) - elif stream == 'orders': - self.assertGreaterEqual(resuming_bookmark_value, first_bookmark_value) - else: - self.assertGreater(resuming_bookmark_value, first_bookmark_value) + # verify the resuming bookmark is greater or equal than 1st sync bookmark + self.assertGreaterEqual(resuming_bookmark_value, first_bookmark_value) # verify oldest record from resuming sync respects bookmark from previous sync if stream in new_state['bookmarks'].keys() and resuming_sync_messages: diff --git a/tests/unittests/test_graphql_base.py b/tests/unittests/test_graphql_base.py new file mode 100644 index 00000000..bfd10c2d --- /dev/null +++ b/tests/unittests/test_graphql_base.py @@ -0,0 +1,106 @@ +import json +import unittest +from unittest.mock import patch, MagicMock +from datetime import datetime +from dateutil.tz import tzlocal +from itertools import cycle +from tap_shopify.streams.graphql.gql_base import ShopifyGqlStream, ShopifyGraphQLError +from tap_shopify.context import Context + +class TestShopifyGqlStream(unittest.TestCase): + + def setUp(self): + self.stream = ShopifyGqlStream() + self.stream.data_key = "products" + # Mock the Context.config to include start_date + self.original_config = Context.config + Context.config = { + "start_date": "2025-01-01T00:00:00Z", + "date_window_size": 30 + } + + def tearDown(self): + # Reset Context.config to its original state + Context.config = self.original_config + + @patch('shopify.GraphQL') + @patch.object(ShopifyGqlStream, 'get_query', return_value='mocked_query') + def test_call_api_success(self, mock_get_query, mock_graphql): + """Test successful GraphQL query execution.""" + # Mock the response from Shopify GraphQL API + mock_response = { + "data": { + "products": { + "edges": [{"node": {"id": "mocked_id", "updated_at": "2025-01-01T00:00:00Z"}}], + "pageInfo": {"endCursor": "cursor_123", "hasNextPage": False}, + } + } + } + mock_graphql.return_value.execute.return_value = json.dumps(mock_response) + + query_params = self.stream.get_query_params("2025-01-01T00:00:00Z", "2025-01-02T00:00:00Z") + result = self.stream.call_api(query_params) + + self.assertEqual(result, mock_response["data"]["products"]) + + @patch('shopify.GraphQL') + @patch.object(ShopifyGqlStream, 'get_query', return_value='mocked_query') + def test_call_api_error(self, mock_get_query, mock_graphql): + """Test GraphQL query execution with an error.""" + # Mock an error response from Shopify GraphQL API + mock_graphql.return_value.execute.side_effect = ShopifyGraphQLError("GraphQL error") + + query_params = self.stream.get_query_params("2025-01-01T00:00:00Z", "2025-01-02T00:00:00Z") + + with self.assertRaises(ShopifyGraphQLError): + self.stream.call_api(query_params) + + @patch('shopify.GraphQL') + @patch.object(ShopifyGqlStream, 'get_query', return_value='mocked_query') + def test_call_api_empty_response(self, mock_get_query, mock_graphql): + """Test GraphQL query execution with an empty response.""" + # Mock an empty response from Shopify GraphQL API + mock_response = {} + mock_graphql.return_value.execute.return_value = json.dumps(mock_response) + + query_params = self.stream.get_query_params("2025-01-01T00:00:00Z", "2025-01-02T00:00:00Z") + result = self.stream.call_api(query_params) + + self.assertEqual(result, {}) + + @patch('shopify.GraphQL') + @patch.object(ShopifyGqlStream, 'get_query', return_value='mocked_query') + @patch.object(ShopifyGqlStream, 'transform_object', side_effect=lambda x: x) + @patch('tap_shopify.streams.graphql.gql_base.utils.now', return_value=datetime(2025, 2, 1, 0, 0, tzinfo=tzlocal())) + def test_get_objects(self, mock_now, mock_transform_object, mock_get_query, mock_graphql): + """Test get_objects with pagination and bookmarking.""" + # Mock the response from Shopify GraphQL API + mock_response_page_1 = { + "data": { + "products": { + "edges": [{"node": {"id": "mocked_id_1", "updated_at": "2025-01-01T00:00:00Z"}}], + "pageInfo": {"endCursor": "cursor_123", "hasNextPage": True}, + } + } + } + mock_response_page_2 = { + "data": { + "products": { + "edges": [{"node": {"id": "mocked_id_2", "updated_at": "2025-01-01T00:00:00Z"}}], + "pageInfo": {"endCursor": "cursor_456", "hasNextPage": False}, + } + } + } + alternating_responses = cycle([ + json.dumps(mock_response_page_1), + json.dumps(mock_response_page_2) + ]) + + # Set side_effect to use the infinite alternating cycle + mock_graphql.return_value.execute.side_effect = lambda *args, **kwargs: next(alternating_responses) + + objects = list(self.stream.get_objects()) + + self.assertEqual(len(objects), 4) + self.assertEqual(objects[0], {"id": "mocked_id_1", "updated_at": "2025-01-01T00:00:00Z"}) + self.assertEqual(objects[1], {"id": "mocked_id_2", "updated_at": "2025-01-01T00:00:00Z"}) diff --git a/tests/unittests/test_inventory_items.py b/tests/unittests/test_inventory_items.py deleted file mode 100644 index 76721c44..00000000 --- a/tests/unittests/test_inventory_items.py +++ /dev/null @@ -1,92 +0,0 @@ -import unittest -from unittest import mock -from singer.utils import strptime_to_utc -from tap_shopify.context import Context - -INVENTORY_ITEM_OBJECT = Context.stream_objects['inventory_items']() - -class Product(): - def __init__(self, id, variants): - self.id = id - self.variants = variants - -class ProductVariant(): - def __init__(self, id, inventory_item_id): - self.id = id - self.inventory_item_id = inventory_item_id - -class InventoryItems(): - def __init__(self, id, updated_at): - self.id = id - self.updated_at = updated_at - - def to_dict(self): - return {"id": self.id, "updated_at": self.updated_at} - -ITEM_1 = InventoryItems("i11", "2021-08-11T01:57:05-04:00") -ITEM_2 = InventoryItems("i12", "2021-08-12T01:57:05-04:00") -ITEM_3 = InventoryItems("i21", "2021-08-13T01:57:05-04:00") -ITEM_4 = InventoryItems("i22", "2021-08-14T01:57:05-04:00") - -class TestInventoryItems(unittest.TestCase): - - @mock.patch("tap_shopify.streams.base.Stream.get_objects") - @mock.patch("tap_shopify.streams.inventory_items.InventoryItems.get_inventory_items") - def test_get_objects_with_product_variant(self, mock_get_inventory_items, mock_parent_object): - - expected_inventory_items = [ITEM_1, ITEM_2, ITEM_3, ITEM_4] - product1 = Product("p1", [ProductVariant("v11", "i11"), ProductVariant("v21", "i21")]) - product2 = Product("p2", [ProductVariant("v12", "i12"), ProductVariant("v22", "i22")]) - - mock_get_inventory_items.side_effect = [[ITEM_1, ITEM_2], [ITEM_3, ITEM_4]] - mock_parent_object.return_value = [product1, product2] - - actual_inventory_items = list(INVENTORY_ITEM_OBJECT.get_objects()) - - #Verify that it returns inventory_item of all product variant - self.assertEqual(actual_inventory_items, expected_inventory_items) - - - @mock.patch("tap_shopify.streams.base.Stream.get_objects") - @mock.patch("tap_shopify.streams.inventory_items.InventoryItems.get_inventory_items") - def test_get_objects_with_product_but_no_variant(self, mock_get_inventory_items, mock_parent_object): - - expected_inventory_items = [ITEM_3, ITEM_4] - - #Product1 contain no variant - product1 = Product("p1", []) - - product2 = Product("p2", [ProductVariant("v12", "i12"), ProductVariant("v22", "i22")]) - mock_parent_object.return_value = [product1, product2] - - mock_get_inventory_items.side_effect = [[], [ITEM_3, ITEM_4]] - - actual_inventory_items = list(INVENTORY_ITEM_OBJECT.get_objects()) - #Verify that it returns inventory_item of existing product variant - self.assertEqual(actual_inventory_items, expected_inventory_items) - - - @mock.patch("tap_shopify.streams.base.Stream.get_objects") - @mock.patch("tap_shopify.streams.inventory_items.InventoryItems.get_inventory_items") - def test_get_objects_with_no_product(self, mock_get_inventory_items, mock_parent_object): - - #No product exist - mock_parent_object.return_value = [] - expected_inventory_items = [] - - actual_inventory_items = list(INVENTORY_ITEM_OBJECT.get_objects()) - self.assertEqual(actual_inventory_items, expected_inventory_items) - - @mock.patch("tap_shopify.streams.base.Stream.get_bookmark") - @mock.patch("tap_shopify.streams.inventory_items.InventoryItems.get_objects") - def test_sync(self, mock_get_objects, mock_get_bookmark): - - expected_sync = [ITEM_3.to_dict(), ITEM_4.to_dict()] - mock_get_objects.return_value = [ITEM_1, ITEM_2, ITEM_3, ITEM_4] - - mock_get_bookmark.return_value = strptime_to_utc("2021-08-13T01:05:05-04:00") - - actual_sync = list(INVENTORY_ITEM_OBJECT.sync()) - - #Verify that only 2 record syncs - self.assertEqual(actual_sync, expected_sync) \ No newline at end of file diff --git a/tests/unittests/test_timeout.py b/tests/unittests/test_timeout.py index c40c0863..dfe09ee9 100644 --- a/tests/unittests/test_timeout.py +++ b/tests/unittests/test_timeout.py @@ -12,7 +12,6 @@ from tap_shopify.streams.transactions import Transactions from tap_shopify.streams.abandoned_checkouts import AbandonedCheckouts from tap_shopify.streams.metafields import Metafields -from tap_shopify.streams.metafields import get_metafields import unittest class TestTimeoutValue(unittest.TestCase): @@ -271,23 +270,6 @@ def test_AbandonedCheckouts_pyactiveresource_error_timeout_backoff(self, mocked_ # verify we backoff 5 times self.assertEquals(mocked_find.call_count, 5) - @mock.patch("time.sleep") - @mock.patch("shopify.InventoryItem.find") - def test_InventoryItems_pyactiveresource_error_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'pyactiveresource.connection.Error' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = pyactiveresource.connection.Error('urlopen error _ssl.c:1074: The handshake operation timed out') - - # initialize class - inventory_items = InventoryItems() - try: - # function call - inventory_items.get_inventory_items([1, 2, 3]) - except pyactiveresource.connection.Error: - pass - # verify we backoff 5 times self.assertEquals(mocked_find.call_count, 5) @@ -331,24 +313,6 @@ def test_Locations_pyactiveresource_error_timeout_backoff(self, mocked_find, moc # verify we backoff 5 times self.assertEquals(mocked_find.call_count, 5) - @mock.patch("time.sleep") - @mock.patch("shopify.Order.metafields") - def test_Metafields_pyactiveresource_error_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'pyactiveresource.connection.Error' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = pyactiveresource.connection.Error('urlopen error _ssl.c:1074: The handshake operation timed out') - - try: - # function call - get_metafields(shopify.Order, 1, shopify.Order, 100) - except pyactiveresource.connection.Error: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) - @mock.patch("time.sleep") @mock.patch("shopify.Refund.find") def test_OrderRefunds_pyactiveresource_error_timeout_backoff(self, mocked_find, mocked_sleep): @@ -435,25 +399,6 @@ def test_AbandonedCheckouts_socket_timeout_backoff(self, mocked_find, mocked_sle # verify we backoff 5 times self.assertEquals(mocked_find.call_count, 5) - @mock.patch("time.sleep") - @mock.patch("shopify.InventoryItem.find") - def test_InventoryItems_socket_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'socket.timeout' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = socket.timeout("The read operation timed out") - - # initialize class - inventory_items = InventoryItems() - try: - # function call - inventory_items.get_inventory_items([1, 2, 3]) - except socket.timeout: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) @mock.patch("time.sleep") @mock.patch("pyactiveresource.activeresource.ActiveResource.find") @@ -495,24 +440,6 @@ def test_Locations_socket_timeout_backoff(self, mocked_find, mocked_sleep): # verify we backoff 5 times self.assertEquals(mocked_find.call_count, 5) - @mock.patch("time.sleep") - @mock.patch("shopify.Order.metafields") - def test_Metafields_socket_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'socket.timeout' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = socket.timeout("The read operation timed out") - - try: - # function call - get_metafields(shopify.Order, 1, shopify.Order, 100) - except socket.timeout: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) - @mock.patch("time.sleep") @mock.patch("shopify.Refund.find") def test_OrderRefunds_socket_timeout_backoff(self, mocked_find, mocked_sleep): From 73dec1e77d9b9b36f82b2c663aa0a08826a3817e Mon Sep 17 00:00:00 2001 From: Vishal Pachpinde Date: Mon, 27 Jan 2025 20:11:11 +0530 Subject: [PATCH 35/66] Fix GraphQL Client Exception handling and logging (#195) remove print statements --- CHANGELOG.md | 9 ++++++--- setup.py | 3 ++- tap_shopify/streams/graphql/gql_base.py | 22 ++++++++++++++++++++++ tap_shopify/streams/metafields.py | 3 --- 4 files changed, 30 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be3465cc..542b447d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,13 @@ # Changelog +## 2.0.1 + * Fixed error handling for GraphQL client [#195](https://github.com/singer-io/tap-shopify/pull/195) + ## 2.0.0 * Deprecated REST Admin API for products - * GraphQL Support added for deprecated streams - * New Stream `Product Variants` added - * More Details here: [#193](https://github.com/singer-io/tap-shopify/pull/193) + * GraphQL support added for deprecated streams + * New stream `Product Variants` added + * More details here: [#193](https://github.com/singer-io/tap-shopify/pull/193) ## 1.10.0 * Updates the Shopify SDK to 12.3.0 diff --git a/setup.py b/setup.py index 45748823..9768fe1b 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="2.0.0", + version="2.0.1", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", @@ -12,6 +12,7 @@ python_requires='>=3.5.2', py_modules=["tap_shopify"], install_requires=[ + # Important: review the monkey-patched method in the GraphQL client when upgrading this dependency. "ShopifyAPI==12.6.0", "singer-python==6.0.0", ], diff --git a/tap_shopify/streams/graphql/gql_base.py b/tap_shopify/streams/graphql/gql_base.py index b6ef5b5a..6c16f9d5 100644 --- a/tap_shopify/streams/graphql/gql_base.py +++ b/tap_shopify/streams/graphql/gql_base.py @@ -1,5 +1,6 @@ from datetime import timedelta import json +import urllib import shopify from singer import( @@ -20,6 +21,27 @@ class ShopifyGraphQLError(Exception): """Custom exception for GraphQL errors""" + +def execute_gql(self, query, variables=None, operation_name=None): + """ + This overrides the `execute` method from ShopifyAPI(v12.6.0) to remove the print statement. + Ensure to check the original impl before making any changes or upgrading the SDK version, + as this modification may affect future updates + """ + default_headers = {"Accept": "application/json", "Content-Type": "application/json"} + headers = self.merge_headers(default_headers, self.headers) + data = {"query": query, "variables": variables, "operationName": operation_name} + + req = urllib.request.Request(self.endpoint, json.dumps(data).encode("utf-8"), headers) + + try: + with urllib.request.urlopen(req) as response: + return response.read().decode("utf-8") + except urllib.error.HTTPError as e: + raise e + +shopify.GraphQL.execute = execute_gql + class ShopifyGqlStream(Stream): data_key = None diff --git a/tap_shopify/streams/metafields.py b/tap_shopify/streams/metafields.py index 6264358c..e17d4f86 100644 --- a/tap_shopify/streams/metafields.py +++ b/tap_shopify/streams/metafields.py @@ -74,7 +74,6 @@ def get_resource_type_query(self, resource): @shopify_error_handling def call_api(self, query_params, query, data_key): - LOGGER.info("Fetching %s %s", self.name, query_params) response = shopify.GraphQL().execute(query=query, variables=query_params) response = json.loads(response) if "errors" in response.keys(): @@ -106,8 +105,6 @@ def get_parents(self): query_params = self.get_query_params(last_updated_at, query_end, cursor) query = get_parent_ids_query(parent) data = self.call_api(query_params, query, parent) - LOGGER.info("%s : %s", parent, len(data.get("edges"))) - for edge in data.get("edges"): yield (edge.get("node"), resource_alias) From e3d7819885f43f2f9b1fed31c24ef1b91082d985 Mon Sep 17 00:00:00 2001 From: Prijen Khokhani <88327452+prijendev@users.noreply.github.com> Date: Wed, 5 Feb 2025 18:00:55 +0530 Subject: [PATCH 36/66] Update bookmark logic for transactions and order_refunds (#197) * Update bookmark logic * Update changelog * Resolve comments --- CHANGELOG.md | 3 +++ setup.py | 2 +- tap_shopify/streams/order_refunds.py | 13 +++++++------ tap_shopify/streams/transactions.py | 12 ++++++------ 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 542b447d..cd1cbe4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 2.0.2 + * Update bookmark logic for transactions and order_refunds stream [#197](https://github.com/singer-io/tap-shopify/pull/197) + ## 2.0.1 * Fixed error handling for GraphQL client [#195](https://github.com/singer-io/tap-shopify/pull/195) diff --git a/setup.py b/setup.py index 9768fe1b..001878b2 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="2.0.1", + version="2.0.2", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/streams/order_refunds.py b/tap_shopify/streams/order_refunds.py index 23e13c8a..c6d35718 100644 --- a/tap_shopify/streams/order_refunds.py +++ b/tap_shopify/streams/order_refunds.py @@ -1,5 +1,6 @@ import shopify -from singer.utils import strftime, strptime_to_utc +import singer +from singer.utils import strptime_to_utc from tap_shopify.context import Context from tap_shopify.streams.base import (Stream, shopify_error_handling, @@ -10,6 +11,7 @@ class OrderRefunds(Stream): name = 'order_refunds' replication_object = shopify.Refund replication_key = 'created_at' + parent_stream = None @shopify_error_handling def get_refunds(self, parent_object, since_id): @@ -24,6 +26,7 @@ def get_refunds(self, parent_object, since_id): def get_objects(self): selected_parent = Context.stream_objects['orders']() selected_parent.name = "refund_orders" + self.parent_stream = selected_parent # Page through all `orders`, bookmarking at `refund_orders` for parent_object in selected_parent.get_objects(): @@ -44,7 +47,6 @@ def get_objects(self): def sync(self): bookmark = self.get_bookmark() - max_bookmark = bookmark for refund in self.get_objects(): refund_dict = refund.to_dict() replication_value = strptime_to_utc(refund_dict[self.replication_key]) @@ -54,10 +56,9 @@ def sync(self): canonicalize(transaction_dict, field_name) yield refund_dict - if replication_value > max_bookmark: - max_bookmark = replication_value - - self.update_bookmark(strftime(max_bookmark)) + max_bookmark = singer.get_bookmark( + Context.state, self.parent_stream.name, self.parent_stream.replication_key) + self.update_bookmark(max_bookmark) Context.stream_objects['order_refunds'] = OrderRefunds diff --git a/tap_shopify/streams/transactions.py b/tap_shopify/streams/transactions.py index 6615a7cd..652efaa4 100644 --- a/tap_shopify/streams/transactions.py +++ b/tap_shopify/streams/transactions.py @@ -1,6 +1,6 @@ import shopify import singer -from singer.utils import strftime, strptime_to_utc +from singer.utils import strptime_to_utc from tap_shopify.context import Context from tap_shopify.streams.base import (Stream, shopify_error_handling, @@ -17,6 +17,7 @@ class Transactions(Stream): name = 'transactions' replication_key = 'created_at' replication_object = shopify.Transaction + parent_stream = None # Added decorator over functions of shopify SDK replication_object.find = shopify_error_handling(replication_object.find) # Transactions have no updated_at property. Therefore we have @@ -59,6 +60,7 @@ def get_objects(self): # Get transactions, bookmarking at `transaction_orders` selected_parent = Context.stream_objects['orders']() selected_parent.name = "transaction_orders" + self.parent_stream = selected_parent # Page through all `orders`, bookmarking at `transaction_orders` for parent_object in selected_parent.get_objects(): @@ -68,7 +70,6 @@ def get_objects(self): def sync(self): bookmark = self.get_bookmark() - max_bookmark = bookmark for transaction in self.get_objects(): transaction_dict = transaction.to_dict() replication_value = strptime_to_utc(transaction_dict[self.replication_key]) @@ -77,9 +78,8 @@ def sync(self): canonicalize(transaction_dict, field_name) yield transaction_dict - if replication_value > max_bookmark: - max_bookmark = replication_value - - self.update_bookmark(strftime(max_bookmark)) + max_bookmark = singer.get_bookmark( + Context.state, self.parent_stream.name, self.parent_stream.replication_key) + self.update_bookmark(max_bookmark) Context.stream_objects['transactions'] = Transactions From 189fd6f8ce4aef82dd1de9922d2c153286b04903 Mon Sep 17 00:00:00 2001 From: Sourabh Gandhi <105213416+sgandhi1311@users.noreply.github.com> Date: Wed, 19 Feb 2025 22:43:36 +0530 Subject: [PATCH 37/66] Optimize Shopify Metafields Sync Performance (#200) * added backoff and fix interuptible sync * add the metafields query * update query and update the pagination logic * Fix the bookmarking logic * replicate the records exact matching to the bookmark value * fix the pylint issues * fix the failing unit test cases * remove shop owner from the other resources query * remove redundant query * update according to the review comments * remove redundant code * Add new field - inventoryItem * update setup and changelog * remove redundant function call * optimize the bookmarking logic * add space above class initiation --------- Co-authored-by: Vi6hal --- CHANGELOG.md | 5 + setup.py | 2 +- tap_shopify/__init__.py | 1 + tap_shopify/schemas/product_variants.json | 14 ++ tap_shopify/streams/base.py | 5 +- tap_shopify/streams/graphql/__init__.py | 1 + tap_shopify/streams/graphql/gql_base.py | 17 +- tap_shopify/streams/graphql/gql_queries.py | 57 ++++++ tap_shopify/streams/metafields.py | 200 +++++++++++---------- tests/unittests/test_graphql_base.py | 6 +- 10 files changed, 205 insertions(+), 103 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd1cbe4f..0cff3b44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 2.1.0 + * Optimize Shopify Metafields Sync Performance [#200](https://github.com/singer-io/tap-shopify/pull/200) + * Include the new field (= inventoryItem) in the product variants schema. + * Add retry for the interruptible sync error from the server side. + ## 2.0.2 * Update bookmark logic for transactions and order_refunds stream [#197](https://github.com/singer-io/tap-shopify/pull/197) diff --git a/setup.py b/setup.py index 001878b2..16152346 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="2.0.2", + version="2.1.0", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index 91f25689..a8dd82c1 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -166,6 +166,7 @@ def sync(): if not Context.state.get('bookmarks'): Context.state['bookmarks'] = {} Context.state['bookmarks']['currently_sync_stream'] = stream_id + singer.write_state(Context.state) # some fields have epoch-time as date, hence transform into UTC date with Transformer(singer.UNIX_SECONDS_INTEGER_DATETIME_PARSING) as transformer: diff --git a/tap_shopify/schemas/product_variants.json b/tap_shopify/schemas/product_variants.json index 385c6569..81b67ef1 100644 --- a/tap_shopify/schemas/product_variants.json +++ b/tap_shopify/schemas/product_variants.json @@ -159,6 +159,20 @@ ] } } + }, + "inventoryItem": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } } } } diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index 3ab2d44e..44c414d6 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -108,7 +108,7 @@ def is_timeout_error(error_raised): def shopify_error_handling(fnc): @backoff.on_exception(backoff.expo, - (http.client.IncompleteRead, ConnectionResetError), + (http.client.IncompleteRead, ConnectionResetError, ShopifyAPIError), max_tries=MAX_RETRIES, factor=2) @backoff.on_exception(backoff.expo, # timeout error raise by Shopify @@ -146,6 +146,9 @@ class Error(Exception): class OutOfOrderIdsError(Error): """Raised if our expectation of ordering by ID is violated""" +class ShopifyAPIError(Error): + """Raised for any unexpected api error without a valid status code""" + class Stream(): # Used for bookmarking and stream identification. Is overridden by # subclasses to change the bookmark key. diff --git a/tap_shopify/streams/graphql/__init__.py b/tap_shopify/streams/graphql/__init__.py index d4eec738..4a9b2523 100644 --- a/tap_shopify/streams/graphql/__init__.py +++ b/tap_shopify/streams/graphql/__init__.py @@ -5,6 +5,7 @@ from .gql_base import ShopifyGqlStream, ShopifyGraphQLError from .gql_queries import ( + get_metafields_query, get_inventory_items_query, get_metafield_query_collection, get_metafield_query_order, diff --git a/tap_shopify/streams/graphql/gql_base.py b/tap_shopify/streams/graphql/gql_base.py index 6c16f9d5..657f817b 100644 --- a/tap_shopify/streams/graphql/gql_base.py +++ b/tap_shopify/streams/graphql/gql_base.py @@ -12,13 +12,14 @@ from tap_shopify.streams.base import ( Stream, shopify_error_handling, + ShopifyAPIError, DATE_WINDOW_SIZE, ) LOGGER = get_logger() -class ShopifyGraphQLError(Exception): +class ShopifyGraphQLError(ShopifyAPIError): """Custom exception for GraphQL errors""" @@ -37,8 +38,8 @@ def execute_gql(self, query, variables=None, operation_name=None): try: with urllib.request.urlopen(req) as response: return response.read().decode("utf-8") - except urllib.error.HTTPError as e: - raise e + except urllib.error.HTTPError as http_error: + raise http_error shopify.GraphQL.execute = execute_gql @@ -93,15 +94,15 @@ def call_api(self, query_params): response = shopify.GraphQL().execute(query=query, variables=query_params) response = json.loads(response) if "errors" in response.keys(): - raise ShopifyGraphQLError(response['errors']) + raise ShopifyAPIError(response['errors']) data = response.get("data", {}).get(self.data_key, {}) return data - except ShopifyGraphQLError as gql_error: + except ShopifyAPIError as gql_error: LOGGER.error("GraphQL Error %s", gql_error) - raise ShopifyGraphQLError("An error occurred with the GraphQL API.") from gql_error - except Exception as e: + raise ShopifyAPIError("An error occurred with the GraphQL API.") from gql_error + except Exception as exc: LOGGER.error("Unexpected error occurred.",) - raise e + raise exc def get_objects(self): """ diff --git a/tap_shopify/streams/graphql/gql_queries.py b/tap_shopify/streams/graphql/gql_queries.py index fda6e5cf..7145ebe8 100644 --- a/tap_shopify/streams/graphql/gql_queries.py +++ b/tap_shopify/streams/graphql/gql_queries.py @@ -158,6 +158,7 @@ def get_product_variant_query(): title updatedAt product { id } + inventoryItem { id } } } pageInfo { @@ -231,6 +232,62 @@ def get_parent_ids_query(resource): qry = qry.replace("RESOURCE", resource) return qry +def get_metafields_query(resource): + """Returns the GraphQL query for fetching metafields""" + if resource == 'shop': + return get_metafield_query_shop() + + qry = """ + query getMetafields( $first: Int!, $after: String $query: String) { + RESOURCE(first: $first after: $after query: $query) { + edges { + node { + metafields(first: $first) { + edges { + node { + id + ownerType + value + type + key + createdAt + namespace + description + updatedAt + owner { + ... on Customer { + id + } + ... on Product { + id + } + ... on Order { + id + } + ... on Collection { + id + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + + qry = qry.replace("RESOURCE", resource) + return qry + def get_metafield_query_customers(): """ Returns GraphQL query to get customer metafields diff --git a/tap_shopify/streams/metafields.py b/tap_shopify/streams/metafields.py index e17d4f86..afc61ecf 100644 --- a/tap_shopify/streams/metafields.py +++ b/tap_shopify/streams/metafields.py @@ -6,7 +6,7 @@ from tap_shopify.context import Context from tap_shopify.streams.graphql import ( - get_parent_ids_query, + get_metafields_query, get_metafield_query_customers, get_metafield_query_product, get_metafield_query_collection, @@ -15,16 +15,14 @@ ) from tap_shopify.streams.graphql.gql_base import ( ShopifyGqlStream, - ShopifyGraphQLError, + ShopifyAPIError, DATE_WINDOW_SIZE, shopify_error_handling, - ) - +) LOGGER = get_logger() - class Metafields(ShopifyGqlStream): name = 'metafields' data_key = "metafields" @@ -32,16 +30,12 @@ class Metafields(ShopifyGqlStream): selected_parent = None - parent_alias = { - "custom_collections":"collections" - } - - # maps object list identifier to single object access identifier # eg customers -> customer resource_alias = { - "customers":"customer", - "products":"product", + "shop": "shop", + "customers": "customer", + "products": "product", "collections": "collection", "orders": "order" } @@ -49,7 +43,7 @@ class Metafields(ShopifyGqlStream): def get_query(self): return None - # pylint: disable=W0221 + # pylint: disable=arguments-differ def get_query_params(self, updated_at_min, updated_at_max, cursor=None): """ Returns query and params for filtering, pagination @@ -69,109 +63,135 @@ def get_resource_type_query(self, resource): "product": get_metafield_query_product, "collection": get_metafield_query_collection, "order": get_metafield_query_order, - "shop":get_metafield_query_shop - }.get(resource) + "shop": get_metafield_query_shop + }.get(resource) @shopify_error_handling - def call_api(self, query_params, query, data_key): - response = shopify.GraphQL().execute(query=query, variables=query_params) - response = json.loads(response) - if "errors" in response.keys(): - raise ShopifyGraphQLError(response['errors']) - data = response.get("data", {}).get(data_key, {}) - return data - - def get_parents(self): + def call_api(self, query_params, query): + try: + response = shopify.GraphQL().execute(query=query, variables=query_params) + response = json.loads(response) + if "errors" in response.keys(): + raise ShopifyAPIError(response['errors']) + data = response.get("data", {}) + return data + except ShopifyAPIError as gql_error: + LOGGER.error("GraphQL Error %s", gql_error) + raise ShopifyAPIError("An error occurred with the GraphQL API.") from gql_error + except Exception as exception: + LOGGER.error("Unexpected error occurred.",) + raise exception + + def transform_object(self, obj): + obj["value_type"] = obj["type"] or None + obj["updated_at"] = obj["updatedAt"] + if obj["value_type"] in ["json", "weight", "volume", "dimension", "rating"]: + value = obj.get("value") + try: + obj["value"] = json.loads(value) if value is not None else value + except json.decoder.JSONDecodeError: + LOGGER.info("Failed to decode JSON value for obj %s", obj.get('id')) + obj["value"] = value + return obj + + def get_next_page_metafields(self, metafields, query_params, gql_resource): + owner_id = metafields.get("edges", [])[0]["node"]["owner"]["id"] + query = self.get_resource_type_query(gql_resource)() + has_next_page = True + cursor = metafields.get("pageInfo", {}).get("endCursor") + while has_next_page: + query_params["after"] = cursor + query_params["pk_id"] = owner_id + data = self.call_api(query_params, query).get(gql_resource, {}) + metafields = data.get("metafields", {}) + for edge in metafields.get("edges", []): + yield edge + page_info = metafields.get("pageInfo", {}) + cursor = page_info.get("endCursor") + has_next_page = page_info.get("hasNextPage", False) + + # pylint: disable=too-many-locals, too-many-nested-blocks + def get_objects(self): + """Main iterator to yield metafield objects""" sync_start = utils.now().replace(microsecond=0) - for parent in ['shop','orders', 'customers', 'products', 'custom_collections']: - parent = self.parent_alias.get(parent, parent) - resource_alias = self.resource_alias.get(parent, parent) - LOGGER.info("Fetching id's for %s %s", parent, resource_alias) - if parent == "shop": - yield None, resource_alias + for resource_type, gql_resource in self.resource_alias.items(): + LOGGER.info("Syncing metafields for %s", resource_type) + + # Handle shop metafields separately + if resource_type == 'shop': + query = get_metafields_query(gql_resource) + has_next_page = True + cursor = None + + while has_next_page: + params = {"first": self.results_per_page} + if cursor: + params["after"] = cursor + + data = self.call_api(params, query) + metafields = data.get("shop", {}).get("metafields", {}) + + for edge in metafields.get("edges", []): + yield self.transform_object(edge["node"]) + + page_info = metafields.get("pageInfo", {}) + cursor = page_info.get("endCursor") + has_next_page = page_info.get("hasNextPage", False) continue - last_updated_at = self.get_bookmark_by_name(f'{self.name}_{resource_alias}') + # Handle other resources + last_updated_at = self.get_bookmark_by_name(f'{self.name}_{resource_type}') date_window_size = float(Context.config.get("date_window_size", DATE_WINDOW_SIZE)) while last_updated_at < sync_start: - date_window_end = last_updated_at + timedelta(days=date_window_size) query_end = min(sync_start, date_window_end) - has_next_page, cursor = True, None + + has_next_page = True + cursor = None while has_next_page: query_params = self.get_query_params(last_updated_at, query_end, cursor) - query = get_parent_ids_query(parent) - data = self.call_api(query_params, query, parent) - for edge in data.get("edges"): - yield (edge.get("node"), resource_alias) - - page_info = data.get("pageInfo") + query = get_metafields_query(resource_type) + data = self.call_api(query_params, query) + + resource_data = data.get(resource_type, {}) + for data_edge in resource_data.get("edges", []): + metafields = data_edge["node"].get("metafields", {}) + for edge in metafields.get("edges", []): + yield self.transform_object(edge["node"]) + meta_page_info = metafields.get("pageInfo", {}) + meta_has_next_page = meta_page_info.get("hasNextPage", False) + if meta_has_next_page: + for edge in self.get_next_page_metafields( + metafields, query_params, gql_resource + ): + yield self.transform_object(edge["node"]) + + page_info = resource_data.get("pageInfo", {}) cursor = page_info.get("endCursor") - has_next_page = page_info.get("hasNextPage") + has_next_page = page_info.get("hasNextPage", False) last_updated_at = query_end - def get_objects(self): - - for parent_obj, resource_type in self.get_parents(): - qury_fnc = self.get_resource_type_query(resource_type) - - if qury_fnc: - query = qury_fnc() - else: - raise ShopifyGraphQLError("Invalid Resource Type") - - has_next_page, cursor = True, None - query_params = {"first": self.results_per_page} - if resource_type != "shop": - query_params["pk_id"] = parent_obj["id"] - - while has_next_page: - if cursor: - query_params["after"] = cursor - response = self.call_api(query_params, query, resource_type) - data = (response.get("metafields") or {}) - - for edge in data.get("edges"): - obj = self.transform_object(edge.get("node")) - yield (obj, resource_type) - - page_info = data.get("pageInfo") - cursor, has_next_page = page_info.get("endCursor"), page_info.get("hasNextPage") - - def transform_object(self, obj): - obj["value_type"] = obj["type"] or None - obj["updated_at"] = obj["updatedAt"] - if obj["value_type"] in ["json", "weight", "volume", "dimension", "rating"]: - value = obj.get("value") - try: - obj["value"] = json.loads(value) if value is not None else value - except json.decoder.JSONDecodeError: - LOGGER.info("Failed to decode JSON value for obj %s", obj.get('id')) - obj["value"] = value - return obj - def sync(self): + """Sync metafields and update bookmarks""" start_time = utils.now().replace(microsecond=0) current_bookmarks = {} - last_bookmarks = {} - for obj, resource_type in self.get_objects(): - replication_value = utils.strptime_to_utc(obj[self.replication_key]) + for obj in self.get_objects(): + resource_type = obj["ownerType"].lower() + replication_value = utils.strptime_to_utc(obj["updated_at"]) + current_bookmark_value = self.get_bookmark_by_name(f"{self.name}_{resource_type}") - if resource_type not in current_bookmarks or resource_type not in last_bookmarks: - bookmark = self.get_bookmark_by_name(f"{self.name}_{resource_type}") - last_bookmarks[resource_type] = bookmark - current_bookmarks[resource_type] = bookmark + if replication_value >= current_bookmarks.get(resource_type, current_bookmark_value): + current_bookmarks[resource_type] = replication_value - if replication_value >= last_bookmarks[resource_type]: - current_bookmarks[resource_type] = max(replication_value, \ - current_bookmarks[resource_type]) + if replication_value >= current_bookmark_value: yield obj + # Update bookmarks for res, bookmark_val in current_bookmarks.items(): bookmark_val = min(start_time, bookmark_val) self.update_bookmark(utils.strftime(bookmark_val), f"{self.name}_{res}") diff --git a/tests/unittests/test_graphql_base.py b/tests/unittests/test_graphql_base.py index bfd10c2d..54de74cf 100644 --- a/tests/unittests/test_graphql_base.py +++ b/tests/unittests/test_graphql_base.py @@ -4,7 +4,7 @@ from datetime import datetime from dateutil.tz import tzlocal from itertools import cycle -from tap_shopify.streams.graphql.gql_base import ShopifyGqlStream, ShopifyGraphQLError +from tap_shopify.streams.graphql.gql_base import ShopifyGqlStream, ShopifyAPIError from tap_shopify.context import Context class TestShopifyGqlStream(unittest.TestCase): @@ -48,11 +48,11 @@ def test_call_api_success(self, mock_get_query, mock_graphql): def test_call_api_error(self, mock_get_query, mock_graphql): """Test GraphQL query execution with an error.""" # Mock an error response from Shopify GraphQL API - mock_graphql.return_value.execute.side_effect = ShopifyGraphQLError("GraphQL error") + mock_graphql.return_value.execute.side_effect = ShopifyAPIError("GraphQL error") query_params = self.stream.get_query_params("2025-01-01T00:00:00Z", "2025-01-02T00:00:00Z") - with self.assertRaises(ShopifyGraphQLError): + with self.assertRaises(ShopifyAPIError): self.stream.call_api(query_params) @patch('shopify.GraphQL') From abf5ddceaeeb850bf1e63cbd6a4435d2be8e554b Mon Sep 17 00:00:00 2001 From: Sourabh Gandhi <105213416+sgandhi1311@users.noreply.github.com> Date: Sat, 29 Mar 2025 02:40:08 +0530 Subject: [PATCH 38/66] Migrate Shopify Streams from REST API to GraphQL API (#201) * add new queries * update schemas * schemas for abandoned checkouts, collections and customers * uncomment transactions query * revert the changes for the collects stream * schema changes for customers stream and update the queries * update inventory levels schema * update transactions and order_refunds schema * update schemas for the collections stream * add new schema for the metafield_products, metafields_collections, metafields_customers and metafields_orders * remove redundant queries * update the order_refunds and transactions schema * rename filename * update checkout schema * fix the schemas * update orders schema * fix collections schema * Migrate Shopify Streams from REST API to GraphQL API (#203) * graphql implementataion for custoemrs, events and locations stream * Use graphql endpoint for orders * add the queries in the respective stream class * Update the logic for inventory levels stream * Optimize inventory levels logic * update transactions and order_refunds logic * Initial commit for abandoned checkouts and collections streams * Remove reference for the collects and custom collections * optimize the order_refunds and transactions stream logic * partial logic for collections * update locations query * add collections streams changes and separate out metafields as individual streams * remove gql query file * remove gql query file reference and include the query in the code * fix abandoned checkouts logic * format the code * remove metafields reference from init file * update checkout schema * fix typo * set the default page size to 250 * make pylint happy and format the code * upgrade python packages * fix circleci pylint issues * modify variable name in collections stream * optimize get query params method * move the graphql base code to the base file * removed reference to the gql_base file * fix the pylint * optimize the source code * update unit test cases * update exception in unit test * Refactor the code - 1. Use the single query for inventiry levels to fetch the child object as well. 2. Add Sortkey in the products query 3. Update query names for the respective metafields * move date_window_size to the parent class * make pylint happy * make pylint happy * Make metafields related stream, order_refunds and transactions stream as pseudo incremental * make pylint happy * the similar scencario is tested in the integration test, so we can skip this test. * Update Integration Tests for GraphQL Migration (#205) * make initial base file changes * add individual meta streams * update all fields integration test * update locations replication key in test suite * remove old references of the stream * avoid testing on the events stream * reduce the default page size for metafields * update state file in the bookmarks * include events in the expected streams * fix interrupt sync * update foreign key for transactions table * update the custom field name in the collections stream * print the catalog in the sorted order of streams * update README and changelog --- CHANGELOG.md | 5 + README.md | 53 +- setup.py | 10 +- tap_shopify/__init__.py | 5 +- tap_shopify/schemas/abandoned_checkouts.json | 895 +++----- tap_shopify/schemas/collections.json | 154 ++ tap_shopify/schemas/collects.json | 49 - tap_shopify/schemas/customers.json | 480 +++- tap_shopify/schemas/events.json | 138 +- tap_shopify/schemas/inventory_levels.json | 66 +- tap_shopify/schemas/locations.json | 43 +- ...ields.json => metafields_collections.json} | 4 +- ...ections.json => metafields_customers.json} | 85 +- tap_shopify/schemas/metafields_orders.json | 84 + tap_shopify/schemas/metafields_products.json | 84 + tap_shopify/schemas/order_refunds.json | 776 +------ tap_shopify/schemas/orders.json | 1944 +++++++---------- tap_shopify/schemas/products.json | 4 +- tap_shopify/schemas/transactions.json | 392 +++- tap_shopify/streams/__init__.py | 8 +- tap_shopify/streams/abandoned_checkouts.py | 276 ++- tap_shopify/streams/base.py | 296 +-- tap_shopify/streams/collections.py | 122 ++ tap_shopify/streams/collects.py | 50 - tap_shopify/streams/custom_collections.py | 11 - tap_shopify/streams/customers.py | 114 +- tap_shopify/streams/events.py | 93 +- tap_shopify/streams/graphql/__init__.py | 17 - tap_shopify/streams/graphql/gql_base.py | 150 -- tap_shopify/streams/graphql/gql_queries.py | 469 ---- tap_shopify/streams/inventory_items.py | 79 +- tap_shopify/streams/inventory_levels.py | 187 +- tap_shopify/streams/locations.py | 93 +- tap_shopify/streams/metafields.py | 257 +-- tap_shopify/streams/metafields_collections.py | 58 + tap_shopify/streams/metafields_customers.py | 57 + tap_shopify/streams/metafields_orders.py | 57 + tap_shopify/streams/metafields_products.py | 57 + tap_shopify/streams/order_refunds.py | 206 +- tap_shopify/streams/orders.py | 535 ++++- tap_shopify/streams/product_variants.py | 82 +- tap_shopify/streams/products.py | 161 +- tap_shopify/streams/transactions.py | 250 ++- tests/base.py | 65 +- tests/test_all_fields.py | 24 +- tests/test_bookmarks.py | 11 +- tests/test_bookmarks_metafields.py | 211 -- tests/test_bookmarks_updated.py | 48 +- tests/test_interrupted_sync.py | 40 +- .../test_backoff_on_incompleteread_error.py | 24 - .../{test_graphql_base.py => test_base.py} | 26 +- tests/unittests/test_error_handling.py | 90 +- tests/unittests/test_inventory_levels.py | 76 - tests/unittests/test_locations.py | 43 - tests/unittests/test_timeout.py | 503 ----- .../test_transaction_canonicalize.py | 40 - tests/unittests/test_updated_bookmark.py | 83 - 57 files changed, 5073 insertions(+), 5167 deletions(-) create mode 100644 tap_shopify/schemas/collections.json delete mode 100644 tap_shopify/schemas/collects.json rename tap_shopify/schemas/{metafields.json => metafields_collections.json} (97%) rename tap_shopify/schemas/{custom_collections.json => metafields_customers.json} (52%) create mode 100644 tap_shopify/schemas/metafields_orders.json create mode 100644 tap_shopify/schemas/metafields_products.json create mode 100644 tap_shopify/streams/collections.py delete mode 100644 tap_shopify/streams/collects.py delete mode 100644 tap_shopify/streams/custom_collections.py delete mode 100644 tap_shopify/streams/graphql/__init__.py delete mode 100644 tap_shopify/streams/graphql/gql_base.py delete mode 100644 tap_shopify/streams/graphql/gql_queries.py create mode 100644 tap_shopify/streams/metafields_collections.py create mode 100644 tap_shopify/streams/metafields_customers.py create mode 100644 tap_shopify/streams/metafields_orders.py create mode 100644 tap_shopify/streams/metafields_products.py delete mode 100644 tests/test_bookmarks_metafields.py delete mode 100644 tests/unittests/test_backoff_on_incompleteread_error.py rename tests/unittests/{test_graphql_base.py => test_base.py} (82%) delete mode 100644 tests/unittests/test_inventory_levels.py delete mode 100644 tests/unittests/test_locations.py delete mode 100644 tests/unittests/test_timeout.py delete mode 100644 tests/unittests/test_transaction_canonicalize.py delete mode 100644 tests/unittests/test_updated_bookmark.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0cff3b44..a11b7375 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 3.0.0 + * Migrate Remaining Shopify Streams from REST API to GraphQL API [#201](https://github.com/singer-io/tap-shopify/pull/201) + * Introduce new streams - `collections`, `metafields_collections`, `metafields_customers`, `metafields_orders`, `metafields_products` + * Delete streams - `collects`, `custom_collections` and `metafields` + ## 2.1.0 * Optimize Shopify Metafields Sync Performance [#200](https://github.com/singer-io/tap-shopify/pull/200) * Include the new field (= inventoryItem) in the product variants schema. diff --git a/README.md b/README.md index 15d57ca9..d6a53059 100644 --- a/README.md +++ b/README.md @@ -6,22 +6,47 @@ spec](https://github.com/singer-io/getting-started/blob/master/SPEC.md). This tap: -- Pulls raw data from [Shopify](https://help.shopify.com/en/api/reference) +- Pulls raw data from [Shopify Graphql Admin API](https://shopify.dev/docs/api/admin-graphql/latest) - Extracts the following resources: - - [Abandoned Checkouts](https://help.shopify.com/en/api/reference/orders/abandoned_checkouts) - - [Collects](https://help.shopify.com/en/api/reference/products/collect) - - [Custom Collections](https://help.shopify.com/en/api/reference/products/customcollection) - - [Customers](https://help.shopify.com/en/api/reference/customers) - - [Metafields](https://help.shopify.com/en/api/reference/metafield) - - [Orders](https://help.shopify.com/en/api/reference/orders) - - [Products](https://help.shopify.com/en/api/reference/products) - - [Transactions](https://help.shopify.com/en/api/reference/orders/transaction) - - [Locations](https://help.shopify.com/en/api/reference/inventory/location) - - [Inventory Levels](https://help.shopify.com/en/api/reference/inventory/inventorylevel) - - [Inventory Item](https://help.shopify.com/en/api/reference/inventory/inventoryitem) + - [Abandoned Checkouts](https://shopify.dev/docs/api/admin-graphql/latest/queries/abandonedcheckouts) + - [Collections](https://shopify.dev/docs/api/admin-graphql/latest/queries/collections) + - [Customers](https://shopify.dev/docs/api/admin-graphql/latest/queries/customers) + - [Metafields Collections](https://shopify.dev/docs/api/admin-graphql/latest/queries/collections) + - [Metafields Customers](https://shopify.dev/docs/api/admin-graphql/latest/queries/customers) + - [Metafields Orders](https://shopify.dev/docs/api/admin-graphql/latest/queries/orders) + - [Metafields Products](https://shopify.dev/docs/api/admin-graphql/latest/queries/products) + - [Orders](https://shopify.dev/docs/api/admin-graphql/latest/queries/orders) + - [Products](https://shopify.dev/docs/api/admin-graphql/latest/queries/products) + - [Product Variants](https://shopify.dev/docs/api/admin-graphql/latest/queries/productVariants) + - [Transactions](https://shopify.dev/docs/api/admin-graphql/latest/queries/orders) + - [Locations](https://shopify.dev/docs/api/admin-graphql/latest/queries/locations) + - [Inventory Levels](https://shopify.dev/docs/api/admin-graphql/latest/queries/inventorylevel) + - [Inventory Item](https://shopify.dev/docs/api/admin-graphql/latest/queries/inventoryitems) - Outputs the schema for each resource - Incrementally pulls data based on the input state -- When Metafields are selected, this tap will sync the Shopify store's top-level Metafields and any additional Metafields for selected tables that also have them (ie: Orders, Products, Customers) + +## Stream Details + +| Stream Name | Replication Key | Key Properties | +|------------------------|----------------|---------------| +| abandoned_checkouts | updatedAt | id | +| collections | updatedAt | id | +| customers | updatedAt | id | +| events | createdAt | id | +| inventory_items | updatedAt | id | +| inventory_levels | updatedAt | id | +| locations | createdAt | id | +| metafields_collections | updatedAt | id | +| metafields_customers | updatedAt | id | +| metafields_orders | updatedAt | id | +| metafields_products | updatedAt | id | +| order_refunds | updatedAt | id | +| orders | updatedAt | id | +| product_variants | updatedAt | id | +| products | updatedAt | id | +| transactions | createdAt | id | + +Currently, `locations` graphql endpoint doesn't support querying on the `updatedAt`, therefore, `createdAt` is made the replication key. ## Quick Start @@ -64,4 +89,4 @@ This tap: --- -Copyright © 2019 Stitch +Copyright © 2025 Stitch diff --git a/setup.py b/setup.py index 16152346..1365a671 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="2.1.0", + version="3.0.0", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", @@ -13,14 +13,14 @@ py_modules=["tap_shopify"], install_requires=[ # Important: review the monkey-patched method in the GraphQL client when upgrading this dependency. - "ShopifyAPI==12.6.0", - "singer-python==6.0.0", + "ShopifyAPI==12.7.0", + "singer-python==6.1.1", ], extras_require={ 'dev': [ - 'pylint==3.0.3', + 'pylint==3.3.6', 'ipdb', - 'requests==2.20.0', + 'requests==2.32.3', 'nose', ] }, diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index a8dd82c1..fc2f0946 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -15,7 +15,6 @@ from tap_shopify.context import Context from tap_shopify.exceptions import ShopifyError from tap_shopify.streams.base import shopify_error_handling, get_request_timeout -import tap_shopify.streams # Load stream objects into Context REQUIRED_CONFIG_KEYS = ["shop", "api_key"] LOGGER = singer.get_logger() @@ -25,7 +24,7 @@ def initialize_shopify_client(): api_key = Context.config['api_key'] shop = Context.config['shop'] - version = '2024-07' + version = '2025-01' session = shopify.Session(shop, version, api_key) shopify.ShopifyResource.activate_session(session) @@ -44,7 +43,7 @@ def load_schemas(): # This schema represents many of the currency values as JSON schema # 'number's, which may result in lost precision. - for filename in os.listdir(get_abs_path('schemas')): + for filename in sorted(os.listdir(get_abs_path('schemas'))): path = get_abs_path('schemas') + '/' + filename schema_name = filename.replace('.json', '') with open(path, encoding='UTF-8') as file: diff --git a/tap_shopify/schemas/abandoned_checkouts.json b/tap_shopify/schemas/abandoned_checkouts.json index d1d0e52d..bb003f1f 100644 --- a/tap_shopify/schemas/abandoned_checkouts.json +++ b/tap_shopify/schemas/abandoned_checkouts.json @@ -1,615 +1,350 @@ { "type": "object", "properties": { - "note_attributes": { - "type": [ - "null", - "array" - ], - "items": { - "type": [ - "null", - "object" - ], + "note": {"type": ["null", "string"]}, + "completedAt": {"type": ["null", "string"], "format": "date-time"}, + "billingAddress": {"type": ["null", "object"], "properties": { + "phone": { "type": ["null", "string"] }, + "country": { "type": ["null", "string"] }, + "firstName": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "latitude": { "type": ["null", "number"] }, + "zip": { "type": ["null", "string"] }, + "lastName": { "type": ["null", "string"] }, + "province": { "type": ["null", "string"] }, + "address2": { "type": ["null", "string"] }, + "address1": { "type": ["null", "string"] }, + "countryCodeV2": { "type": ["null", "string"] }, + "city": { "type": ["null", "string"] }, + "company": { "type": ["null", "string"] }, + "provinceCode": { "type": ["null", "string"] }, + "longitude": { "type": ["null", "number"] }, + "coordinatesValidated": { "type": ["null", "boolean"] }, + "formattedArea": { "type": ["null", "string"] }, + "id": { "type": ["null", "string"] }, + "timeZone": { "type": ["null", "string"] }, + "validationResultSummary": { "type": ["null", "string"] } + }}, + "discountCodes": {"type": "array", "items": {"type": ["null", "string"]}}, + "createdAt": {"type": ["null", "string"], "format": "date-time"}, + "updatedAt": {"type": ["null", "string"], "format": "date-time"}, + "taxLines": {"type": "array", "items": {"type": ["null", "object"], "properties": { + "priceSet": { + "type": ["null", "object"], "properties": { - "name": { - "type": [ - "null", - "string" - ] + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } }, - "value": { - "type": [ - "null", - "string" - ] + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "title": { "type": ["null", "string"] }, + "rate": { "type": ["null", "number"] }, + "source": { "type": ["null", "string"] }, + "channelLiable": { "type": ["null", "boolean"] }, + "ratePercentage": { "type": ["null", "number"] } + }}}, + "totalLineItemsPriceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": {"type": ["null", "string"]} + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": {"type": ["null", "string"]} } } } }, - "location_id": { - "type": [ - "null", - "integer" - ] - }, - "buyer_accepts_marketing": { - "type": [ - "null", - "boolean" - ] - }, - "currency": { - "type": [ - "null", - "string" - ] - }, - "completed_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "token": { - "type": [ - "null", - "string" - ] - }, - "billing_address": { - "type": [ - "null", - "object" - ], + "id": {"type": ["null", "string"]}, + "name": {"type": ["null", "string"]}, + "totalTaxSet": { + "type": ["null", "object"], "properties": { - "phone": { - "type": [ - "null", - "string" - ] - }, - "country": { - "type": [ - "null", - "string" - ] - }, - "first_name": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "latitude": { - "type": [ - "null", - "number" - ] - }, - "zip": { - "type": [ - "null", - "string" - ] - }, - "last_name": { - "type": [ - "null", - "string" - ] - }, - "province": { - "type": [ - "null", - "string" - ] - }, - "address2": { - "type": [ - "null", - "string" - ] - }, - "address1": { - "type": [ - "null", - "string" - ] - }, - "country_code": { - "type": [ - "null", - "string" - ] - }, - "city": { - "type": [ - "null", - "string" - ] - }, - "company": { - "type": [ - "null", - "string" - ] - }, - "province_code": { - "type": [ - "null", - "string" - ] + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": {"type": ["null", "string"]} + } }, - "longitude": { - "type": [ - "null", - "number" - ] + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": {"type": ["null", "string"]} + } } } }, - "email": { - "type": [ - "null", - "string" - ] + "shippingAddress": {"type": ["null", "object"], "properties": { + "phone": { "type": ["null", "string"] }, + "country": { "type": ["null", "string"] }, + "firstName": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "latitude": { "type": ["null", "string"], "format": "singer.decimal" }, + "zip": { "type": ["null", "string"] }, + "lastName": { "type": ["null", "string"] }, + "province": { "type": ["null", "string"] }, + "address2": { "type": ["null", "string"] }, + "address1": { "type": ["null", "string"] }, + "countryCodeV2": { "type": ["null", "string"] }, + "city": { "type": ["null", "string"] }, + "company": { "type": ["null", "string"] }, + "provinceCode": { "type": ["null", "string"] }, + "longitude": { "type": ["null", "string"], "format": "singer.decimal" }, + "coordinatesValidated": { "type": ["null", "boolean"] }, + "formattedArea": { "type": ["null", "string"] }, + "id": { "type": ["null", "string"] }, + "timeZone": { "type": ["null", "string"] }, + "validationResultSummary": { "type": ["null", "string"] } + }}, + "abandonedCheckoutUrl": {"type": ["null", "string"]}, + "totalDiscountSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": {"type": ["null", "string"]} + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": {"type": ["null", "string"]} + } + } + } }, - "discount_codes": { - "type": [ - "null", - "array" - ], - "items": { - "type": [ - "null", - "object" - ], + "taxesIncluded": {"type": "boolean"}, + "totalDutiesSet": {"type": ["null", "object"], "properties": { + "presentmentMoney": { + "type": ["null", "object"], "properties": { - "type": { - "type": [ - "null", - "string" - ] - }, - "amount": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "code": { - "type": [ - "null", - "string" - ] + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": {"type": ["null", "string"]} + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": {"type": ["null", "string"]} + } + }}}, + "totalPriceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": {"type": ["null", "string"]} + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": {"type": ["null", "string"]} } } } }, - "customer_locale": { - "type": [ - "null", - "string" - ] - }, - "created_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "updated_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "gateway": { - "type": [ - "null", - "string" - ] - }, - "referring_site": { - "type": [ - "null", - "string" - ] - }, - "source_identifier": { - "type": [ - "null", - "string" - ] - }, - "total_weight": { - "type": [ - "null", - "integer" - ] - }, - "tax_lines": { - "$ref": "definitions.json#/tax_lines" - }, - "total_line_items_price": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "closed_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "device_id": { - "type": [ - "null", - "integer" - ] - }, - "phone": { - "type": [ - "null", - "string" - ] - }, - "source_name": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "total_tax": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "subtotal_price": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "line_items": { - "$ref": "definitions.json#/line_items" - }, - "source_url": { - "type": [ - "null", - "string" - ] - }, - "total_discounts": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "note": { - "type": [ - "null", - "string" - ] - }, - "presentment_currency": { - "type": [ - "null", - "string" - ] - }, - "shipping_lines": { - "type": [ - "null", - "array" - ], + "lineItems": { + "type": "array", "items": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { - "applied_discounts": { - "type": [ - "null", - "array" - ], + "edges": { + "type": "array", "items": { - "type": [ - "null", - "object" - ] + "type": ["null", "object"], + "properties": { + "node": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "string"] }, + "quantity": { "type": ["null", "integer"] }, + "sku": { "type": ["null", "string"] }, + "title": { "type": ["null", "string"] }, + "variantTitle": { "type": ["null", "string"] }, + "variant": { + "type": ["null", "object"], + "properties": { + "title": { "type": ["null", "string"] }, + "id": { "type": ["null", "string"] } + } + }, + "discountedTotalPriceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "components": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "string"] }, + "quantity": { "type": ["null", "integer"] }, + "title": { "type": ["null", "string"] }, + "variantTitle": { "type": ["null", "string"] } + } + } + }, + "customAttributes": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "key": { "type": ["null", "string"] }, + "value": { "type": ["null", "string"] } + } + } + }, + "product": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "string"] } + } + }, + "discountedUnitPriceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "discountedUnitPriceWithCodeDiscount": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "originalTotalPriceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "originalUnitPriceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + } + } + } + } } - }, - "custom_tax_lines": { - "type": [ - "null", - "array" - ], - "items": { - "type": [ - "null", - "object" - ] - } - }, - "phone": { - "type": [ - "null", - "string" - ] - }, - "validation_context": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "string" - ] - }, - "carrier_identifier": { - "type": [ - "null", - "string" - ] - }, - "api_client_id": { - "type": [ - "null", - "integer" - ] - }, - "price": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "requested_fulfillment_service_id": { - "type": [ - "null", - "string" - ] - }, - "title": { - "type": [ - "null", - "string" - ] - }, - "code": { - "type": [ - "null", - "string" - ] - }, - "tax_lines": { - "$ref": "definitions.json#/tax_lines" - }, - "carrier_service_id": { - "type": [ - "null", - "integer" - ] - }, - "delivery_category": { - "type": [ - "null", - "string" - ] - }, - "markup": { - "type": [ - "null", - "string" - ] - }, - "source": { - "type": [ - "null", - "string" - ] } } } }, - "user_id": { - "type": [ - "null", - "integer" - ] - }, - "source": { - "type": [ - "null", - "string" - ] - }, - "shipping_address": { - "type": [ - "null", - "object" - ], + "subtotalPriceSet": { + "type": ["null", "object"], "properties": { - "phone": { - "type": [ - "null", - "string" - ] - }, - "country": { - "type": [ - "null", - "string" - ] - }, - "first_name": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "latitude": { - "type": [ - "null", - "number" - ] - }, - "zip": { - "type": [ - "null", - "string" - ] - }, - "last_name": { - "type": [ - "null", - "string" - ] - }, - "province": { - "type": [ - "null", - "string" - ] - }, - "address2": { - "type": [ - "null", - "string" - ] - }, - "address1": { - "type": [ - "null", - "string" - ] - }, - "country_code": { - "type": [ - "null", - "string" - ] - }, - "city": { - "type": [ - "null", - "string" - ] - }, - "company": { - "type": [ - "null", - "string" - ] - }, - "province_code": { - "type": [ - "null", - "string" - ] + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": {"type": ["null", "string"]} + } }, - "longitude": { - "type": [ - "null", - "number" - ] + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": {"type": ["null", "string"]} + } } } - }, - "abandoned_checkout_url": { - "type": [ - "null", - "string" - ] - }, - "landing_site": { - "type": [ - "null", - "string" - ] - }, - "customer": { - "$ref": "definitions.json#/customer" - }, - "total_price": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "cart_token": { - "type": [ - "null", - "string" - ] - }, - "taxes_included": { - "type": [ - "null", - "boolean" - ] - }, - "buyer_accepts_sms_marketing": { - "type": [ - "null", - "boolean" - ] - }, - "sms_marketing_phone": { - "type": [ - "null", - "string" - ] - }, - "total_duties": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" } } } diff --git a/tap_shopify/schemas/collections.json b/tap_shopify/schemas/collections.json new file mode 100644 index 00000000..d7adf254 --- /dev/null +++ b/tap_shopify/schemas/collections.json @@ -0,0 +1,154 @@ +{ + "type": "object", + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "handle": { + "type": [ + "null", + "string" + ] + }, + "collectionType": { + "type": [ + "null", + "string" + ] + }, + "updatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "productsCount": { + "type": [ + "null", + "object" + ], + "properties": { + "count": { + "type": [ + "null", + "integer" + ] + }, + "precision": { + "type": [ + "null", + "string" + ] + } + } + }, + "sortOrder": { + "type": [ + "null", + "string" + ] + }, + "ruleSet": { + "type": [ + "null", + "object" + ], + "properties": { + "appliedDisjunctively": { + "type": [ + "null", + "boolean" + ] + }, + "rules": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "column": { + "type": [ + "null", + "string" + ] + }, + "condition": { + "type": [ + "null", + "string" + ] + }, + "relation": { + "type": [ + "null", + "string" + ] + } + } + } + } + } + }, + "seo": { + "type": [ + "null", + "object" + ], + "properties": { + "description": { + "type": [ + "null", + "string" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + } + } + }, + "feedback": { + "type": [ + "null", + "object" + ], + "properties": { + "summary": { + "type": [ + "null", + "string" + ] + } + } + }, + "products": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "string" + ] + } + } + } +} diff --git a/tap_shopify/schemas/collects.json b/tap_shopify/schemas/collects.json deleted file mode 100644 index acaab19c..00000000 --- a/tap_shopify/schemas/collects.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": [ - "null", - "integer" - ] - }, - "collection_id": { - "type": [ - "null", - "integer" - ] - }, - "created_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "position": { - "type": [ - "null", - "integer" - ] - }, - "product_id": { - "type": [ - "null", - "integer" - ] - }, - "sort_value": { - "type": [ - "null", - "string" - ] - }, - "updated_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - } - } -} diff --git a/tap_shopify/schemas/customers.json b/tap_shopify/schemas/customers.json index 00fb3d0b..60280cb4 100644 --- a/tap_shopify/schemas/customers.json +++ b/tap_shopify/schemas/customers.json @@ -1,3 +1,481 @@ { - "$ref": "definitions.json#/customer" + "type": "object", + "properties": { + "email": { + "type": [ + "null", + "string" + ] + }, + "multipassIdentifier": { + "type": [ + "null", + "string" + ] + }, + "defaultAddress": { + "type": [ + "null", + "object" + ], + "properties": { + "city": { + "type": [ + "null", + "string" + ] + }, + "address1": { + "type": [ + "null", + "string" + ] + }, + "zip": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "province": { + "type": [ + "null", + "string" + ] + }, + "phone": { + "type": [ + "null", + "string" + ] + }, + "country": { + "type": [ + "null", + "string" + ] + }, + "firstName": { + "type": [ + "null", + "string" + ] + }, + "lastName": { + "type": [ + "null", + "string" + ] + }, + "countryCodeV2": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "provinceCode": { + "type": [ + "null", + "string" + ] + }, + "address2": { + "type": [ + "null", + "string" + ] + }, + "company": { + "type": [ + "null", + "string" + ] + }, + "timeZone": { + "type": [ + "null", + "string" + ] + }, + "validationResultSummary": { + "type": [ + "null", + "string" + ] + }, + "latitude": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "longitude": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "coordinatesValidated": { + "type": [ + "null", + "boolean" + ] + }, + "formattedArea": { + "type": [ + "null", + "string" + ] + } + } + }, + "numberOfOrders": { + "type": [ + "null", + "string" + ] + }, + "state": { + "type": [ + "null", + "string" + ] + }, + "verifiedEmail": { + "type": [ + "null", + "boolean" + ] + }, + "firstName": { + "type": [ + "null", + "string" + ] + }, + "updatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "note": { + "type": [ + "null", + "string" + ] + }, + "phone": { + "type": [ + "null", + "string" + ] + }, + "addresses": { + "type": "array", + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "city": { + "type": [ + "null", + "string" + ] + }, + "address1": { + "type": [ + "null", + "string" + ] + }, + "zip": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "province": { + "type": [ + "null", + "string" + ] + }, + "phone": { + "type": [ + "null", + "string" + ] + }, + "country": { + "type": [ + "null", + "string" + ] + }, + "firstName": { + "type": [ + "null", + "string" + ] + }, + "lastName": { + "type": [ + "null", + "string" + ] + }, + "countryCodeV2": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "provinceCode": { + "type": [ + "null", + "string" + ] + }, + "address2": { + "type": [ + "null", + "string" + ] + }, + "company": { + "type": [ + "null", + "string" + ] + }, + "timeZone": { + "type": [ + "null", + "string" + ] + }, + "validationResultSummary": { + "type": [ + "null", + "string" + ] + }, + "latitude": { + "type": [ + "number", + "null" + ] + }, + "longitude": { + "type": [ + "number", + "null" + ] + }, + "coordinatesValidated": { + "type": [ + "null", + "boolean" + ] + }, + "formattedArea": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "lastName": { + "type": [ + "null", + "string" + ] + }, + "tags": { + "type": "array", + "items": { + "type": [ + "null", + "string" + ] + } + }, + "taxExempt": { + "type": [ + "null", + "boolean" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "createdAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "taxExemptions": { + "type": "array", + "items": { + "type": [ + "null", + "string" + ] + } + }, + "emailMarketingConsent": { + "type": [ + "null", + "object" + ], + "properties": { + "consentUpdatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "marketingOptInLevel": { + "type": [ + "null", + "string" + ] + }, + "marketingState": { + "type": [ + "null", + "string" + ] + } + } + }, + "smsMarketingConsent": { + "type": [ + "object", + "null" + ], + "properties": { + "consentCollectedFrom": { + "type": [ + "null", + "string" + ] + }, + "consentUpdatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "marketingOptInLevel": { + "type": [ + "null", + "string" + ] + }, + "marketingState": { + "type": [ + "null", + "string" + ] + } + } + }, + "validEmailAddress": { + "type": [ + "null", + "boolean" + ] + }, + "productSubscriberStatus": { + "type": [ + "null", + "string" + ] + }, + "amountSpent": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "dataSaleOptOut": { + "type": [ + "null", + "boolean" + ] + }, + "displayName": { + "type": [ + "null", + "string" + ] + }, + "locale": { + "type": [ + "null", + "string" + ] + }, + "lifetimeDuration": { + "type": [ + "null", + "string" + ] + } + } } diff --git a/tap_shopify/schemas/events.json b/tap_shopify/schemas/events.json index e829fe3c..6dfdc6f1 100644 --- a/tap_shopify/schemas/events.json +++ b/tap_shopify/schemas/events.json @@ -4,79 +4,177 @@ "id": { "type": [ "null", - "integer" + "string" ] }, - "created_at": { + "createdAt": { "type": [ "null", "string" ], "format": "date-time" }, - "body": { + "action": { "type": [ "null", "string" ] }, - "path": { + "appTitle": { "type": [ "null", "string" ] }, + "attributeToApp": { + "type": [ + "null", + "boolean" + ] + }, + "attributeToUser": { + "type": [ + "null", + "boolean" + ] + }, + "criticalAlert": { + "type": [ + "null", + "boolean" + ] + }, "message": { "type": [ "null", "string" ] }, - "subject_id": { + "subjectId": { "type": [ "null", - "integer" + "string" ] }, - "subject_type": { + "subjectType": { "type": [ "null", "string" ] }, - "verb": { + "additionalContent": { "type": [ "null", "string" ] }, - "author": { + "additionalData": { "type": [ "null", "string" ] }, - "description": { + "arguments": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "string" + ] + } + }, + "hasAdditionalContent": { + "type": [ + "null", + "boolean" + ] + }, + "secondaryMessage": { "type": [ "null", "string" ] }, - "arguments": { - "anyOf": [ - { - "type": [ - "null", - "array" - ], - "items": {} - }, - { + "attachments": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "fileExtension": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "size": { + "type": [ + "null", + "integer" + ] + }, + "url": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "author": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { "type": [ "null", "string" ] } + } + }, + "canDelete": { + "type": [ + "null", + "boolean" + ] + }, + "canEdit": { + "type": [ + "null", + "boolean" + ] + }, + "edited": { + "type": [ + "null", + "boolean" + ] + }, + "rawMessage": { + "type": [ + "null", + "string" ] } } diff --git a/tap_shopify/schemas/inventory_levels.json b/tap_shopify/schemas/inventory_levels.json index fb9e6f10..c365da36 100644 --- a/tap_shopify/schemas/inventory_levels.json +++ b/tap_shopify/schemas/inventory_levels.json @@ -1,21 +1,67 @@ { + "type": "object", "properties": { - "available": { - "type": ["null", "integer"] + "id": { + "type": ["null", "string"] }, - "inventory_item_id": { - "type": ["null", "integer"] + "canDeactivate": { + "type": ["null", "boolean"] }, - "updated_at": { + "createdAt": { "type": ["null", "string"], "format": "date-time" }, - "location_id": { - "type": ["null", "integer"] + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" }, - "admin_graphql_api_id": { + "deactivationAlert": { "type": ["null", "string"] + }, + "item": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "variant": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + } + } + } + } + }, + "location": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + } + } + }, + "quantities": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + } + } + } } - }, - "type": "object" + } } diff --git a/tap_shopify/schemas/locations.json b/tap_shopify/schemas/locations.json index 577e6917..b8d55375 100644 --- a/tap_shopify/schemas/locations.json +++ b/tap_shopify/schemas/locations.json @@ -1,3 +1,44 @@ { - "$ref": "definitions.json#/location" + "type": "object", + "properties": { + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "updatedAt": { "type": ["null", "string"], "format": "date-time" }, + "createdAt": { "type": ["null", "string"], "format": "date-time" }, + "isActive": { "type": ["null", "boolean"] }, + "addressVerified": { "type": ["null", "boolean"] }, + "deactivatable": { "type": ["null", "boolean"] }, + "deactivatedAt": { "type": ["null", "string"], "format": "date-time" }, + "deletable": { "type": ["null", "boolean"] }, + "fulfillsOnlineOrders": { "type": ["null", "boolean"] }, + "hasActiveInventory": { "type": ["null", "boolean"] }, + "hasUnfulfilledOrders": { "type": ["null", "boolean"] }, + "isFulfillmentService": { "type": ["null", "boolean"] }, + "legacyResourceId": { "type": ["null", "string"] }, + "shipsInventory": { "type": ["null", "boolean"] }, + "address": { + "type": ["null", "object"], + "properties": { + "countryCode": { "type": ["null", "string"] }, + "address1": { "type": ["null", "string"] }, + "city": { "type": ["null", "string"] }, + "address2": { "type": ["null", "string"] }, + "provinceCode": { "type": ["null", "string"] }, + "zip": { "type": ["null", "string"] }, + "province": { "type": ["null", "string"] }, + "phone": { "type": ["null", "string"] }, + "country": { "type": ["null", "string"] }, + "formatted": { "type": ["null", "array"], "items": { "type": ["null", "string"] } }, + "latitude": { "type": ["null", "string"], "format": "singer.decimal" }, + "longitude": { "type": ["null", "string"], "format": "singer.decimal" } + } + }, + "localPickupSettingsV2": { + "type": ["null", "object"], + "properties": { + "instructions": { "type": ["null", "string"] }, + "pickupTime": { "type": ["null", "string"] } + } + } + } } diff --git a/tap_shopify/schemas/metafields.json b/tap_shopify/schemas/metafields_collections.json similarity index 97% rename from tap_shopify/schemas/metafields.json rename to tap_shopify/schemas/metafields_collections.json index 839ac7ce..5226a246 100644 --- a/tap_shopify/schemas/metafields.json +++ b/tap_shopify/schemas/metafields_collections.json @@ -1,4 +1,5 @@ { + "type": "object", "properties": { "owner": { "type": [ @@ -79,6 +80,5 @@ "string" ] } - }, - "type": "object" + } } diff --git a/tap_shopify/schemas/custom_collections.json b/tap_shopify/schemas/metafields_customers.json similarity index 52% rename from tap_shopify/schemas/custom_collections.json rename to tap_shopify/schemas/metafields_customers.json index 4698ef45..5226a246 100644 --- a/tap_shopify/schemas/custom_collections.json +++ b/tap_shopify/schemas/metafields_customers.json @@ -1,103 +1,84 @@ { + "type": "object", "properties": { - "handle": { + "owner": { "type": [ "null", - "string" - ] + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } }, - "sort_order": { + "ownerType": { "type": [ "null", "string" ] }, - "body_html": { + "value_type": { "type": [ "null", "string" ] }, - "title": { + "key": { "type": [ "null", "string" ] }, - "id": { + "createdAt": { "type": [ "null", - "integer" - ] + "string" + ], + "format": "date-time" }, - "published_scope": { + "id": { "type": [ "null", "string" ] }, - "admin_graphql_api_id": { + "namespace": { "type": [ "null", "string" ] }, - "updated_at": { + "description": { "type": [ "null", "string" ] }, - "image": { - "properties": { - "alt": { - "type": [ - "null", - "string" - ] - }, - "src": { - "type": [ - "null", - "string" - ] - }, - "width": { - "type": [ - "null", - "integer" - ] - }, - "created_at": { - "type": [ - "null", - "string" - ] - }, - "height": { - "type": [ - "null", - "integer" - ] - } - }, + "value": { "type": [ "null", - "object" - ] + "integer", + "object", + "string" + ], + "properties": {} }, - "published_at": { + "updatedAt": { "type": [ "null", "string" - ] + ], + "format": "date-time" }, - "template_suffix": { + "type": { "type": [ "null", "string" ] } - }, - "type": "object" + } } diff --git a/tap_shopify/schemas/metafields_orders.json b/tap_shopify/schemas/metafields_orders.json new file mode 100644 index 00000000..5226a246 --- /dev/null +++ b/tap_shopify/schemas/metafields_orders.json @@ -0,0 +1,84 @@ +{ + "type": "object", + "properties": { + "owner": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + }, + "ownerType": { + "type": [ + "null", + "string" + ] + }, + "value_type": { + "type": [ + "null", + "string" + ] + }, + "key": { + "type": [ + "null", + "string" + ] + }, + "createdAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "namespace": { + "type": [ + "null", + "string" + ] + }, + "description": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "integer", + "object", + "string" + ], + "properties": {} + }, + "updatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "type": { + "type": [ + "null", + "string" + ] + } + } +} diff --git a/tap_shopify/schemas/metafields_products.json b/tap_shopify/schemas/metafields_products.json new file mode 100644 index 00000000..5226a246 --- /dev/null +++ b/tap_shopify/schemas/metafields_products.json @@ -0,0 +1,84 @@ +{ + "type": "object", + "properties": { + "owner": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + }, + "ownerType": { + "type": [ + "null", + "string" + ] + }, + "value_type": { + "type": [ + "null", + "string" + ] + }, + "key": { + "type": [ + "null", + "string" + ] + }, + "createdAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "namespace": { + "type": [ + "null", + "string" + ] + }, + "description": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "integer", + "object", + "string" + ], + "properties": {} + }, + "updatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "type": { + "type": [ + "null", + "string" + ] + } + } +} diff --git a/tap_shopify/schemas/order_refunds.json b/tap_shopify/schemas/order_refunds.json index d170e05f..583917e0 100644 --- a/tap_shopify/schemas/order_refunds.json +++ b/tap_shopify/schemas/order_refunds.json @@ -1,31 +1,23 @@ { "type": "object", "properties": { - "order_id": { - "type": [ - "null", - "integer" - ] - }, - "restock": { + "id": { "type": [ "null", - "boolean" + "string" ] }, - "order_adjustments": { - "$ref": "definitions.json#/order_adjustments" - }, - "processed_at": { + "createdAt": { "type": [ "null", "string" - ] + ], + "format": "date-time" }, - "user_id": { + "legacyResourceId": { "type": [ "null", - "integer" + "string" ] }, "note": { @@ -34,750 +26,214 @@ "string" ] }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "created_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "admin_graphql_api_id": { + "order": { "type": [ "null", - "string" - ] + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } }, - "refund_line_items": { + "refundLineItems": { "type": [ "null", "array" ], "items": { + "type": [ + "null", + "object" + ], "properties": { - "location_id": { + "id": { + "type": [ + "null", + "string" + ] + }, + "quantity": { "type": [ "null", "integer" ] }, - "subtotal_set": { + "priceSet": { + "type": [ + "null", + "object" + ], "properties": { - "shop_money": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], "properties": { - "currency_code": { + "amount": { "type": [ "null", "string" - ] + ], + "format": "singer.decimal" }, - "amount": { + "currencyCode": { "type": [ "null", "string" ] } - }, + } + }, + "shopMoney": { "type": [ "null", "object" - ] - }, - "presentment_money": { + ], "properties": { - "currency_code": { + "amount": { "type": [ "null", "string" - ] + ], + "format": "singer.decimal" }, - "amount": { + "currencyCode": { "type": [ "null", "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "restockType": { "type": [ "null", - "object" + "string" + ] + }, + "restocked": { + "type": [ + "null", + "boolean" ] }, - "total_tax_set": { + "subtotalSet": { + "type": [ + "null", + "object" + ], "properties": { - "shop_money": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], "properties": { - "currency_code": { + "amount": { "type": [ "null", "string" - ] + ], + "format": "singer.decimal" }, - "amount": { + "currencyCode": { "type": [ "null", "string" ] } - }, + } + }, + "shopMoney": { "type": [ "null", "object" - ] - }, - "presentment_money": { + ], "properties": { - "currency_code": { + "amount": { "type": [ "null", "string" - ] + ], + "format": "singer.decimal" }, - "amount": { + "currencyCode": { "type": [ "null", "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, - "type": [ - "null", - "object" - ] + } }, - "line_item_id": { + "totalTaxSet": { "type": [ "null", - "integer" - ] - }, - "total_tax": { - "type": [ - "null", - "number" - ] - }, - "quantity": { - "type": [ - "null", - "integer" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "line_item": { + "object" + ], "properties": { - "gift_card": { - "type": [ - "null", - "boolean" - ] - }, - "price": { - "type": [ - "null", - "string" - ] - }, - "tax_lines": { - "type": [ - "null", - "array" - ], - "items": { - "properties": { - "price_set": { - "properties": { - "shop_money": { - "properties": { - "currency_code": { - "type": [ - "null", - "string" - ] - }, - "amount": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "presentment_money": { - "properties": { - "currency_code": { - "type": [ - "null", - "string" - ] - }, - "amount": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "price": { - "type": [ - "null", - "string" - ] - }, - "title": { - "type": [ - "null", - "string" - ] - }, - "rate": { - "type": [ - "null", - "number" - ] - } - }, - "patternProperties": {".+": {}}, - "type": [ - "null", - "object" - ] - } - }, - "fulfillment_service": { - "type": [ - "null", - "string" - ] - }, - "sku": { + "presentmentMoney": { "type": [ "null", - "string" - ] - }, - "fulfillment_status": { - "type": [ - "null", - "string" - ] - }, - "properties": { - "type": [ - "null", - "array" + "object" ], - "items": { - "properties": { - "name": { - "type": [ - "null", - "string" - ] - }, - "value": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - } - }, - "quantity": { - "type": [ - "null", - "integer" - ] - }, - "variant_id": { - "type": [ - "null", - "integer" - ] - }, - "grams": { - "type": [ - "null", - "integer" - ] - }, - "requires_shipping": { - "type": [ - "null", - "boolean" - ] - }, - "vendor": { - "type": [ - "null", - "string" - ] - }, - "price_set": { "properties": { - "shop_money": { - "properties": { - "currency_code": { - "type": [ - "null", - "string" - ] - }, - "amount": { - "type": [ - "null", - "string" - ] - } - }, + "amount": { "type": [ "null", - "object" - ] + "string" + ], + "format": "singer.decimal" }, - "presentment_money": { - "properties": { - "currency_code": { - "type": [ - "null", - "string" - ] - }, - "amount": { - "type": [ - "null", - "string" - ] - } - }, + "currencyCode": { "type": [ "null", - "object" + "string" ] } - }, - "type": [ - "null", - "object" - ] - }, - "variant_inventory_management": { - "type": [ - "null", - "string" - ] - }, - "pre_tax_price": { - "type": [ - "null", - "string" - ] - }, - "variant_title": { - "type": [ - "null", - "string" - ] + } }, - "total_discount_set": { - "properties": { - "shop_money": { - "properties": { - "currency_code": { - "type": [ - "null", - "string" - ] - }, - "amount": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "presentment_money": { - "properties": { - "currency_code": { - "type": [ - "null", - "string" - ] - }, - "amount": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - } - }, + "shopMoney": { "type": [ "null", "object" - ] - }, - "discount_allocations": { - "type": [ - "null", - "array" ], - "items": { - "properties": { - "amount": { - "type": [ - "null", - "string" - ] - }, - "amount_set": { - "properties": { - "shop_money": { - "properties": { - "currency_code": { - "type": [ - "null", - "string" - ] - }, - "amount": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "presentment_money": { - "properties": { - "currency_code": { - "type": [ - "null", - "string" - ] - }, - "amount": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "discount_application_index": { - "type": [ - "null", - "integer" - ] - } - }, - "type": [ - "null", - "object" - ] - } - }, - "pre_tax_price_set": { "properties": { - "shop_money": { - "properties": { - "currency_code": { - "type": [ - "null", - "string" - ] - }, - "amount": { - "type": [ - "null", - "string" - ] - } - }, + "amount": { "type": [ "null", - "object" - ] + "string" + ], + "format": "singer.decimal" }, - "presentment_money": { - "properties": { - "currency_code": { - "type": [ - "null", - "string" - ] - }, - "amount": { - "type": [ - "null", - "string" - ] - } - }, + "currencyCode": { "type": [ "null", - "object" + "string" ] } - }, - "type": [ - "null", - "object" - ] - }, - "fulfillable_quantity": { - "type": [ - "null", - "integer" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "admin_graphql_api_id": { - "type": [ - "null", - "string" - ] - }, - "total_discount": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "product_exists": { - "type": [ - "null", - "boolean" - ] - }, - "taxable": { - "type": [ - "null", - "boolean" - ] - }, - "product_id": { - "type": [ - "null", - "integer" - ] - }, - "title": { - "type": [ - "null", - "string" - ] - } - }, - "patternProperties": {".+": {}}, - "type": [ - "null", - "object" - ] - }, - "subtotal": { - "type": [ - "null", - "number" - ] - }, - "restock_type": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - } - }, - "transactions": { - "type": ["null", "array"], - "items": { - "properties": { - "admin_graphql_api_id": { - "type": ["null", "string"] - }, - "amount": { - "type": ["null", "string"], - "format": "singer.decimal" - }, - "authorization": { - "type": ["null", "string"] - }, - "created_at": { - "type": ["null", "string"] - }, - "currency": { - "type": ["null", "string"] - }, - "error_code": { - "type": ["null", "string"] - }, - "device_id": { - "type": ["null", "integer"] - }, - "gateway": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "kind": { - "type": ["null", "string"] - }, - "location_id": { - "type": ["null", "integer"] - }, - "message": { - "type": ["null", "string"] - }, - "order_id": { - "type": ["null", "integer"] - }, - "parent_id": { - "type": ["null", "integer"] - }, - "payment_details": { - "properties": { - "cvv_result_code": { - "type": ["null", "string"] - }, - "credit_card_bin": { - "type": ["null", "string"] - }, - "credit_card_company": { - "type": ["null", "string"] - }, - "credit_card_number": { - "type": ["null", "string"] - }, - "avs_result_code": { - "type": ["null", "string"] - }, - "payments_refund_attributes": { - "type": ["null", "string"] - }, - "acquirer_reference_number": { - "type": ["null", "string"] - }, - "status": { - "type": ["null", "string"] - }, - "processed_at": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "user_id": { - "type": ["null", "integer"] - }, - "test": { - "type": ["null", "boolean"] - }, - "source_name": { - "type": ["null", "string"] - }, - "status": { - "type": ["null", "string"] - }, - "receipt": {}, - "payments_refund_attributes": { - "properties": { - "status": { - "type": ["null", "string"] - }, - "acquirer_reference_number": { - "type": ["null", "string"] + } } - }, - "type": ["null", "object"] - }, - "processed_at": { - "type": ["null", "string"] + } } - }, - "type": ["null", "object"] + } } }, - "duties": {} + "updatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + } } } diff --git a/tap_shopify/schemas/orders.json b/tap_shopify/schemas/orders.json index df5411ad..87d5afda 100644 --- a/tap_shopify/schemas/orders.json +++ b/tap_shopify/schemas/orders.json @@ -1,1293 +1,865 @@ { + "type": "object", "properties": { - "presentment_currency": { - "type": [ - "null", - "string" - ] - }, - "subtotal_price_set": {}, - "total_discounts_set": {}, - "total_line_items_price_set": {}, - "total_price_set": {}, - "total_shipping_price_set": {}, - "total_tax_set": {}, - "current_subtotal_price_set": { - "type": [ - "null", - "object" - ] - }, - "current_total_duties_set": { - "type": [ - "null", - "object" - ] - }, - "current_total_discounts_set": { - "type": [ - "null", - "object" - ] - }, - "current_total_price_set": { - "type": [ - "null", - "object" - ] - }, - "current_total_tax_set": { - "type": [ - "null", - "object" - ] - }, - "original_total_duties_set": { - "type": [ - "null", - "object" - ] - }, - "total_price": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "line_items": { - "$ref": "definitions.json#/line_items" - }, - "order_number": { - "type": [ - "null", - "integer" - ] - }, - "confirmed": { - "type": [ - "null", - "boolean" - ] - }, - "taxExempt": { - "type": [ - "null", - "boolean" - ] - }, - "poNumber": { - "type": [ - "null", - "string" - ] - }, - "total_discounts": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "total_line_items_price": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "order_adjustments": { - "$ref": "definitions.json#/order_adjustments" - }, - "shipping_lines": { + "additionalFees": { + "type": "array", "items": { + "type": "object", "properties": { - "tax_lines": { - "$ref": "definitions.json#/tax_lines" - }, - "phone": { - "type": [ - "null", - "string" - ] - }, - "discounted_price_set": {}, - "price_set": {}, + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, "price": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "title": { - "type": [ - "null", - "string" - ] + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } }, - "discount_allocations": { + "taxLines": { + "type": ["null", "array"], "items": { + "type": "object", "properties": { - "discount_application_index": { - "type": [ - "null", - "integer" - ] + "channelLiable": { "type": ["null", "boolean"] }, + "priceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } }, - "amount": { - "type": [ - "null", - "number" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "delivery_category": { - "type": [ - "null", - "string" - ] - }, - "discounted_price": { - "type": [ - "null", - "number" - ] - }, - "code": { - "type": [ - "null", - "string" - ] - }, - "requested_fulfillment_service_id": { - "type": [ - "null", - "string" - ] - }, - "carrier_identifier": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "source": { - "type": [ - "null", - "string" - ] + "rate": { "type": ["null", "number"] }, + "ratePercentage": { "type": ["null", "number"] }, + "source": { "type": ["null", "string"] }, + "title": { "type": ["null", "string"] } + } + } } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "admin_graphql_api_id": { - "type": [ - "null", - "string" - ] - }, - "device_id": { - "type": [ - "null", - "integer" - ] + } + } }, - "cancel_reason": { - "type": [ - "null", - "string" - ] + "alerts": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "content": { "type": ["null", "string"] }, + "dismissibleHandle": { "type": ["null", "string"] }, + "icon": { "type": ["null", "string"] }, + "severity": { "type": ["null", "string"] }, + "title": { "type": ["null", "string"] }, + "actions": { + "type": ["null", "object"], + "properties": { + "primary": { "type": ["null", "boolean"] }, + "show": { "type": ["null", "boolean"] }, + "title": { "type": ["null", "string"] }, + "url": { "type": ["null", "string"] } + } + } + } + } }, - "currency": { - "type": [ - "null", - "string" - ] + "app": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "icon": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "string"] } + } + } + } }, - "payment_gateway_names": { - "items": { - "type": [ - "null", - "string" - ] - }, - "type": [ - "null", - "array" - ] + "billingAddress": { + "type": ["null", "object"], + "properties": { + "address1": { "type": ["null", "string"] }, + "address2": { "type": ["null", "string"] }, + "city": { "type": ["null", "string"] }, + "company": { "type": ["null", "string"] }, + "coordinatesValidated": { "type": ["null", "boolean"] }, + "country": { "type": ["null", "string"] }, + "countryCodeV2": { "type": ["null", "string"] }, + "firstName": { "type": ["null", "string"] }, + "formattedArea": { "type": ["null", "string"] }, + "id": { "type": ["null", "string"] }, + "lastName": { "type": ["null", "string"] }, + "latitude": { "type": ["null", "number"] }, + "longitude": { "type": ["null", "number"] }, + "name": { "type": ["null", "string"] }, + "phone": { "type": ["null", "string"] }, + "province": { "type": ["null", "string"] }, + "provinceCode": { "type": ["null", "string"] }, + "timeZone": { "type": ["null", "string"] }, + "validationResultSummary": { "type": ["null", "string"] }, + "zip": { "type": ["null", "string"] } + } }, - "source_identifier": { - "type": [ - "null", - "string" - ] + "billingAddressMatchesShippingAddress": { "type": ["null", "boolean"] }, + "canMarkAsPaid": { "type": ["null", "boolean"] }, + "canNotifyCustomer": { "type": ["null", "boolean"] }, + "cancelReason": { "type": ["null", "string"] }, + "cancellation": { + "type": ["null", "object"], + "properties": { + "staffNote": { "type": ["null", "string"] } + } }, - "id": { - "type": [ - "null", - "integer" - ] + "cancelledAt": { "type": ["null", "string"], "format": "date-time" }, + "capturable": { "type": ["null", "boolean"] }, + "cartDiscountAmountSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } }, - "processed_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" + "channelInformation": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "string"] }, + "channelId": { "type": ["null", "string"] } + } }, - "referring_site": { - "type": [ - "null", - "string" - ] + "clientIp": { "type": ["null", "string"] }, + "closed": { "type": ["null", "boolean"] }, + "closedAt": { "type": ["null", "string"], "format": "date-time" }, + "confirmationNumber": { "type": ["null", "string"] }, + "confirmed": { "type": ["null", "boolean"] }, + "createdAt": { "type": ["null", "string"], "format": "date-time" }, + "currencyCode": { "type": ["null", "string"] }, + "currentCartDiscountAmountSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } }, - "contact_email": { - "type": [ - "null", - "string" - ] + "currentShippingPriceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } }, - "location_id": { - "type": [ - "null", - "integer" - ] + "currentSubtotalLineItemsQuantity": { "type": ["null", "integer"] }, + "currentSubtotalPriceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } }, - "fulfillments": { + "currentTaxLines": { + "type": ["null", "array"], "items": { + "type": "object", "properties": { - "location_id": { - "type": [ - "null", - "integer" - ] - }, - "receipt": { + "channelLiable": { "type": ["null", "boolean"] }, + "priceSet": { "type": ["null", "object"], "properties": { - "testcase": { - "type": ["null", "boolean"] + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } }, - "authorization": { - "type": ["null", "string"] + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } } } }, - "tracking_number": { - "type": [ - "null", - "string" - ] - }, - "created_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "shipment_status": { - "type": [ - "null", - "string" - ] - }, - "line_items": { - "$ref": "definitions.json#/line_items" - }, - "tracking_url": { - "type": [ - "null", - "string" - ] - }, - "service": { - "type": [ - "null", - "string" - ] - }, - "status": { - "type": [ - "null", - "string" - ] - }, - "admin_graphql_api_id": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "tracking_urls": { - "items": { - "type": [ - "null", - "string" - ] - }, - "type": [ - "null", - "array" - ] - }, - "tracking_numbers": { - "items": { - "type": [ - "null", - "string" - ] - }, - "type": [ - "null", - "array" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "tracking_company": { - "type": [ - "null", - "string" - ] - }, - "updated_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" + "rate": { "type": ["null", "number"] }, + "ratePercentage": { "type": ["null", "number"] }, + "source": { "type": ["null", "string"] }, + "title": { "type": ["null", "string"] } + } + } + }, + "currentTotalAdditionalFeesSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } } }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "customer": { - "$ref": "definitions.json#/customer_orders" - }, - "test": { - "type": [ - "null", - "boolean" - ] - }, - "total_tax": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "number": { - "type": [ - "null", - "integer" - ] - }, - "email": { - "type": [ - "null", - "string" - ] - }, - "source_name": { - "type": [ - "null", - "string" - ] - }, - "landing_site_ref": { - "type": [ - "null", - "string" - ] + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } }, - "shipping_address": { + "currentTotalDiscountsSet": { + "type": ["null", "object"], "properties": { - "phone": { - "type": [ - "null", - "string" - ] - }, - "country": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "address1": { - "type": [ - "null", - "string" - ] - }, - "longitude": { - "type": [ - "null", - "number" - ] - }, - "address2": { - "type": [ - "null", - "string" - ] - }, - "last_name": { - "type": [ - "null", - "string" - ] - }, - "first_name": { - "type": [ - "null", - "string" - ] - }, - "province": { - "type": [ - "null", - "string" - ] - }, - "city": { - "type": [ - "null", - "string" - ] - }, - "company": { - "type": [ - "null", - "string" - ] - }, - "latitude": { - "type": [ - "null", - "number" - ] - }, - "country_code": { - "type": [ - "null", - "string" - ] - }, - "province_code": { - "type": [ - "null", - "string" - ] + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } }, - "zip": { - "type": [ - "null", - "string" - ] + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } } - }, - "type": [ - "null", - "object" - ] - }, - "closed_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" + } }, - "discount_applications": { - "items": { - "properties": { - "target_type": { - "type": [ - "null", - "string" - ] - }, - "code": { - "type": [ - "null", - "string" - ] - }, - "description": { - "type": [ - "null", - "string" - ] - }, - "type": { - "type": [ - "null", - "string" - ] - }, - "target_selection": { - "type": [ - "null", - "string" - ] - }, - "allocation_method": { - "type": [ - "null", - "string" - ] - }, - "title": { - "type": [ - "null", - "string" - ] - }, - "value_type": { - "type": [ - "null", - "string" - ] - }, - "value": { - "type": [ - "null", - "number" - ] + "currentTotalDutiesSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } } }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "note": { - "type": [ - "null", - "string" - ] - }, - "user_id": { - "type": [ - "null", - "integer" - ] - }, - "source_url": { - "type": [ - "null", - "string" - ] - }, - "subtotal_price": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } }, - "billing_address": { + "currentTotalPriceSet": { + "type": ["null", "object"], "properties": { - "phone": { - "type": [ - "null", - "string" - ] - }, - "country": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "address1": { - "type": [ - "null", - "string" - ] - }, - "longitude": { - "type": [ - "null", - "number" - ] - }, - "address2": { - "type": [ - "null", - "string" - ] - }, - "last_name": { - "type": [ - "null", - "string" - ] - }, - "first_name": { - "type": [ - "null", - "string" - ] - }, - "province": { - "type": [ - "null", - "string" - ] - }, - "city": { - "type": [ - "null", - "string" - ] - }, - "company": { - "type": [ - "null", - "string" - ] - }, - "latitude": { - "type": [ - "null", - "number" - ] - }, - "country_code": { - "type": [ - "null", - "string" - ] - }, - "province_code": { - "type": [ - "null", - "string" - ] + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } }, - "zip": { - "type": [ - "null", - "string" - ] + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } } - }, - "type": [ - "null", - "object" - ] - }, - "landing_site": { - "type": [ - "null", - "string" - ] - }, - "taxes_included": { - "type": [ - "null", - "boolean" - ] - }, - "token": { - "type": [ - "null", - "string" - ] - }, - "app_id": { - "type": [ - "null", - "integer" - ] - }, - "total_tip_received": { - "type": [ - "null", - "string" - ] - }, - "browser_ip": { - "type": [ - "null", - "string" - ] + } }, - "discount_codes": { - "items": { - "properties": { - "code": { - "type": [ - "null", - "string" - ] - }, - "amount": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "type": { - "type": [ - "null", - "string" - ] + "currentTotalTaxSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } } }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "tax_lines": { - "$ref": "definitions.json#/tax_lines" + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } }, - "phone": { - "type": [ - "null", - "string" - ] + "currentTotalWeight": { "type": ["null", "number"] }, + "customer": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "string"] }, + "email": { "type": ["null", "string"] }, + "firstName": { "type": ["null", "string"] }, + "lastName": { "type": ["null", "string"] } + } }, - "note_attributes": { + "customerAcceptsMarketing": { "type": ["null", "boolean"] }, + "customerLocale": { "type": ["null", "string"] }, + "discountCodes": {"type": ["null", "array"], "items": { "type": "string" } }, + "discountCode": { "type": ["null", "string"] }, + "displayFinancialStatus": { "type": ["null", "string"] }, + "displayFulfillmentStatus": { "type": ["null", "string"] }, + "disputes": { + "type": ["null", "array"], "items": { + "type": "object", "properties": { - "name": { - "type": [ - "null", - "string" - ] - }, - "value": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "fulfillment_status": { - "type": [ - "null", - "string" - ] + "id": { "type": ["null", "string"] }, + "initiatedAs": { "type": ["null", "string"] }, + "status": { "type": ["null", "string"] } + } + } }, - "order_status_url": { - "type": [ - "null", - "string" - ] + "dutiesIncluded": { "type": ["null", "boolean"] }, + "email": { "type": ["null", "string"] }, + "edited": { "type": ["null", "boolean"] }, + "estimatedTaxes": { "type": ["null", "boolean"] }, + "fulfillable": { "type": ["null", "boolean"] }, + "fullyPaid": { "type": ["null", "boolean"] }, + "hasTimelineComment": { "type": ["null", "boolean"] }, + "fulfillmentsCount": { + "type": ["null", "object"], + "properties": { + "count": { "type": ["null", "integer"] }, + "precision": { "type": ["null", "string"] } + } }, - "client_details": { + "id": { "type": "string" }, + "legacyResourceId": { "type": ["null", "string"] }, + "merchantBusinessEntity": { + "type": ["null", "object"], "properties": { - "session_hash": { - "type": [ - "null", - "string" - ] - }, - "accept_language": { - "type": [ - "null", - "string" - ] - }, - "browser_width": { - "type": [ - "null", - "integer" - ] - }, - "user_agent": { - "type": [ - "null", - "string" - ] + "address": { + "type": ["null", "object"], + "properties": { + "address1": { "type": ["null", "string"] }, + "address2": { "type": ["null", "string"] }, + "city": { "type": ["null", "string"] }, + "countryCode": { "type": ["null", "string"] }, + "province": { "type": ["null", "string"] }, + "zip": { "type": ["null", "string"] } + } }, - "browser_ip": { - "type": [ - "null", - "string" - ] + "companyName": { "type": ["null", "string"] }, + "displayName": { "type": ["null", "string"] }, + "id": { "type": ["null", "string"] }, + "primary": { "type": ["null", "boolean"] } + } + }, + "name": { "type": ["null", "string"] }, + "note": { "type": ["null", "string"] }, + "netPaymentSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } }, - "browser_height": { - "type": [ - "null", - "integer" - ] + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } } - }, - "type": [ - "null", - "object" - ] - }, - "buyer_accepts_marketing": { - "type": [ - "null", - "boolean" - ] - }, - "checkout_token": { - "type": [ - "null", - "string" - ] + } }, - "tags": { - "type": [ - "null", - "string" - ] + "originalTotalAdditionalFeesSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } }, - "financial_status": { - "type": [ - "null", - "string" - ] + "originalTotalDutiesSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } }, - "customer_locale": { - "type": [ - "null", - "string" - ] + "originalTotalPriceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } }, - "checkout_id": { - "type": [ - "null", - "integer" - ] + "paymentGatewayNames": {"type": ["null", "array"], "items": { "type": "string" } }, + "phone": { "type": ["null", "string"] }, + "poNumber": { "type": ["null", "string"] }, + "presentmentCurrencyCode": { "type": ["null", "string"] }, + "processedAt": { "type": ["null", "string"], "format": "date-time" }, + "refundable": { "type": ["null", "boolean"] }, + "refundDiscrepancySet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } }, - "total_weight": { - "type": [ - "null", - "integer" - ] + "registeredSourceUrl": { "type": ["null", "string"] }, + "requiresShipping": { "type": ["null", "boolean"] }, + "restockable": { "type": ["null", "boolean"] }, + "returnStatus": { "type": ["null", "string"] }, + "shippingAddress": { + "type": ["null", "object"], + "properties": { + "address1": { "type": ["null", "string"] }, + "address2": { "type": ["null", "string"] }, + "city": { "type": ["null", "string"] }, + "company": { "type": ["null", "string"] }, + "coordinatesValidated": { "type": ["null", "boolean"] }, + "country": { "type": ["null", "string"] }, + "countryCodeV2": { "type": ["null", "string"] }, + "firstName": { "type": ["null", "string"] }, + "formattedArea": { "type": ["null", "string"] }, + "id": { "type": ["null", "string"] }, + "lastName": { "type": ["null", "string"] }, + "latitude": { "type": ["null", "number"] }, + "longitude": { "type": ["null", "number"] }, + "name": { "type": ["null", "string"] }, + "phone": { "type": ["null", "string"] }, + "province": { "type": ["null", "string"] }, + "provinceCode": { "type": ["null", "string"] }, + "timeZone": { "type": ["null", "string"] }, + "validationResultSummary": { "type": ["null", "string"] }, + "zip": { "type": ["null", "string"] } + } }, - "cart_token": { - "type": [ - "null", - "string" - ] + "shopifyProtect": { + "type": ["null", "object"], + "properties": { + "eligibility": { + "type": ["null", "object"], + "properties": { + "status": { "type": ["null", "string"] } + } + }, + "status": { "type": ["null", "string"] } + } }, - "cancelled_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" + "sourceIdentifier": { "type": ["null", "string"] }, + "sourceName": { "type": ["null", "string"] }, + "statusPageUrl": { "type": ["null", "string"] }, + "subtotalLineItemsQuantity": { "type": ["null", "integer"] }, + "subtotalPriceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } }, - "refunds": { + "tags": {"type": ["null", "array"], "items": { "type": "string" } }, + "taxExempt": { "type": ["null", "boolean"] }, + "taxLines": { + "type": ["null", "array"], "items": { + "type": "object", "properties": { - "admin_graphql_api_id": { - "type": [ - "null", - "string" - ] - }, - "refund_line_items": { - "items": { - "properties": { - "line_item": { - "$ref": "definitions.json#/line_item" - }, - "location_id": { - "type": [ - "null", - "integer" - ] - }, - "line_item_id": { - "type": [ - "null", - "integer" - ] - }, - "quantity": { - "type": [ - "null", - "integer" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "total_tax": { - "type": [ - "null", - "number" - ] - }, - "restock_type": { - "type": [ - "null", - "string" - ] - }, - "subtotal": { - "type": [ - "null", - "number" - ] + "channelLiable": { "type": ["null", "boolean"] }, + "priceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } } }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "restock": { - "type": [ - "null", - "boolean" - ] - }, - "note": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "user_id": { - "type": [ - "null", - "integer" - ] - }, - "created_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "processed_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } }, - "order_adjustments": { - "$ref": "definitions.json#/order_adjustments" + "rate": { "type": ["null", "number"] }, + "ratePercentage": { "type": ["null", "number"] }, + "source": { "type": ["null", "string"] }, + "title": { "type": ["null", "string"] } + } + } + }, + "taxesIncluded": { "type": ["null", "boolean"] }, + "test": { "type": ["null", "boolean"] }, + "totalCapturableSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } } }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "created_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "updated_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "reference": { - "type": [ - "null", - "string" - ] - }, - "current_subtotal_price": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "current_total_discounts": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "current_total_price": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "current_total_tax": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } }, - "total_outstanding": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" + "totalCashRoundingAdjustment": { + "type": ["null", "object"], + "properties": { + "paymentSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "refundSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + } + } }, - "payment_terms": { - "type": [ - "null", - "object" - ], + "totalDiscountsSet": { + "type": ["null", "object"], "properties": { - "amount": { - "type": [ - "null", - "integer", - "string" - ] + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } }, - "currency": { - "type": [ - "null", - "string" - ] + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "totalOutstandingSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } }, - "due_in_days": { - "type": [ - "null", - "integer" - ] + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "totalPriceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } }, - "payment_schedules": { - "type": [ - "null", - "array" - ], - "items": { - "properties": { - "amount": { - "type": [ - "null", - "integer", - "string" - ] - }, - "due_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "currency": { - "type": [ - "null", - "string" - ] - }, - "issued_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "completed_at": { - "type": [ - "null", - "string" - ] - }, - "expected_payment_method": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "totalReceivedSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } } }, - "payment_terms_name": { - "type": [ - "null", - "string" - ] + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "totalRefundedSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } }, - "payment_terms_type": { - "type": [ - "null", - "string" - ] + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } } } }, - "estimated_taxes": { - "type": [ - "null", - "boolean" - ] + "totalRefundedShippingSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } }, - "merchant_of_record_app_id": { - "type": ["null", "integer"] + "totalShippingPriceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } + } + } + } }, - "current_total_additional_fees_set": { + "totalTaxSet": { "type": ["null", "object"], "properties": { - "presentment_money": { + "presentmentMoney": { "type": ["null", "object"], "properties": { - "amount": { - "type": ["null", "string"], - "format": "singer.decimal"}, - "currency": { - "type": ["null", "string"]} + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } } }, - "shop_money": { + "shopMoney": { "type": ["null", "object"], "properties": { - "amount": { - "type": ["null", "string"], - "format": "singer.decimal"}, - "currency": { - "type": ["null", "string"]} + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } } } } }, - "original_total_additional_fees_set": { + "totalTipReceivedSet": { "type": ["null", "object"], "properties": { - "presentment_money": { + "presentmentMoney": { "type": ["null", "object"], "properties": { - "amount": { - "type": ["null", "string"], - "format": "singer.decimal"}, - "currency": { - "type": ["null", "string"] - } + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } } }, - "shop_money": { + "shopMoney": { "type": ["null", "object"], "properties": { - "amount": { - "type": ["null", "string"], - "format": "singer.decimal"}, - "currency": { - "type": ["null", "string"]} + "amount": { "type": ["null", "number"] }, + "currencyCode": { "type": ["null", "string"] } } } } - } - }, - "type": "object" + }, + "totalWeight": { "type": ["null", "number"] }, + "transactionsCount": { + "type": ["null", "object"], + "properties": { + "count": { "type": ["null", "integer"] }, + "precision": { "type": ["null", "string"] } + } + }, + "unpaid": { "type": ["null", "boolean"] }, + "updatedAt": { "type": ["null", "string"], "format": "date-time" } + } } diff --git a/tap_shopify/schemas/products.json b/tap_shopify/schemas/products.json index 4e599b59..04d87ce0 100644 --- a/tap_shopify/schemas/products.json +++ b/tap_shopify/schemas/products.json @@ -1,4 +1,5 @@ { + "type": "object", "properties": { "status": { "type": [ @@ -441,6 +442,5 @@ } } } - }, - "type": "object" + } } diff --git a/tap_shopify/schemas/transactions.json b/tap_shopify/schemas/transactions.json index fd5d97d8..23afde96 100644 --- a/tap_shopify/schemas/transactions.json +++ b/tap_shopify/schemas/transactions.json @@ -1,265 +1,427 @@ { + "type": "object", "properties": { - "error_code": { + "accountNumber": { "type": [ "null", "string" ] }, - "device_id": { + "amountRoundingSet": { "type": [ "null", - "integer" - ] + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } }, - "user_id": { + "amountSet": { "type": [ "null", - "integer" - ] + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } }, - "parent_id": { + "authorizationCode": { "type": [ "null", - "integer" + "string" ] }, - "test": { + "authorizationExpiresAt": { "type": [ "null", - "boolean" - ] + "string" + ], + "format": "date-time" }, - "kind": { + "createdAt": { "type": [ "null", "string" - ] + ], + "format": "date-time" }, - "order_id": { + "errorCode": { "type": [ "null", - "integer" + "string" ] }, - "amount": { + "formattedGateway": { "type": [ "null", "string" - ], - "format": "singer.decimal" + ] }, - "authorization": { + "gateway": { "type": [ "null", "string" ] }, - "currency": { + "id": { "type": [ "null", "string" ] }, - "source_name": { + "kind": { "type": [ "null", "string" ] }, - "message": { + "manualPaymentGateway": { "type": [ "null", - "string" + "boolean" ] }, - "id": { + "maximumRefundableV2": { "type": [ "null", - "integer" + "object" ] }, - "created_at": { + "multiCapturable": { "type": [ "null", - "string" - ], - "format": "date-time" + "boolean" + ] }, - "status": { + "order": { "type": [ "null", - "string" - ] - }, - "payment_details": { + "object" + ], "properties": { - "cvv_result_code": { + "id": { "type": [ "null", "string" ] - }, - "credit_card_bin": { + } + } + }, + "parentTransaction": { + "type": [ + "null", + "object" + ], + "properties": { + "accountNumber": { "type": [ "null", "string" ] }, - "credit_card_company": { + "createdAt": { "type": [ "null", "string" - ] - }, - "credit_card_expiration_month": { - "type": [ - "null", - "integer" - ] + ], + "format": "date-time" }, - "credit_card_expiration_year": { + "id": { "type": [ "null", - "integer" + "string" ] }, - "credit_card_name": { + "status": { "type": [ "null", "string" ] }, - "credit_card_number": { + "paymentId": { "type": [ "null", "string" ] }, - "credit_card_wallet": { + "processedAt": { "type": [ "null", "string" - ] + ], + "format": "date-time" }, - "avs_result_code": { + "amountSet": { "type": [ "null", - "string" - ] + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } } - }, + } + }, + "paymentId": { "type": [ "null", - "object" + "string" ] }, - "gateway": { + "processedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "receiptJson": { "type": [ "null", "string" ] }, - "admin_graphql_api_id": { + "settlementCurrency": { "type": [ "null", "string" ] }, - "receipt": { + "settlementCurrencyRate": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "shopifyPaymentsSet": { "type": [ "null", "object" ], "properties": { - "fee_amount": { + "extendedAuthorizationSet": { "type": [ "null", - "string" + "object" ], - "format": "singer.decimal" - }, - "gross_amount": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" + "properties": { + "extendedAuthorizationExpiresAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "standardAuthorizationExpiresAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + } + } }, - "tax_amount": { + "refundSet": { "type": [ "null", - "string" + "object" ], - "format": "singer.decimal" + "properties": { + "acquirerReferenceNumber": { + "type": [ + "null", + "string" + ] + } + } } - }, - "patternProperties": {".+": {}} + } }, - "location_id": { + "status": { "type": [ "null", - "integer" + "string" ] }, - "processed_at": { + "test": { "type": [ "null", - "string" + "boolean" ] }, - "payments_refund_attributes": { + "totalUnsettledSet": { "type": [ "null", "object" ], "properties": { - "status": { + "presentmentMoney": { "type": [ "null", - "string" - ] - }, - "acquirer_reference_number": { - "type": [ - "null", - "string" - ] - } - } - }, - "payment_id": { - "type": [ - "null", - "string" - ] - }, - "total_unsettled_set": { - "type": ["null", "object"], - "properties": { - "presentment_money": { - "type": ["null", "object"], + "object" + ], "properties": { "amount": { - "type": ["null", "string"], - "format": "singer.decimal"}, - "currency": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] } } }, - "shop_money": { - "type": ["null", "object"], + "shopMoney": { + "type": [ + "null", + "object" + ], "properties": { "amount": { - "type": ["null", "string"], - "format": "singer.decimal"}, - "currency": { - "type": ["null", "string"]} + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } } } } } - }, - "type": "object" + } } diff --git a/tap_shopify/streams/__init__.py b/tap_shopify/streams/__init__.py index 362f1280..78242bf5 100644 --- a/tap_shopify/streams/__init__.py +++ b/tap_shopify/streams/__init__.py @@ -2,13 +2,15 @@ import tap_shopify.streams.customers import tap_shopify.streams.orders import tap_shopify.streams.order_refunds -import tap_shopify.streams.metafields +import tap_shopify.streams.metafields_products +import tap_shopify.streams.metafields_collections +import tap_shopify.streams.metafields_customers +import tap_shopify.streams.metafields_orders import tap_shopify.streams.transactions import tap_shopify.streams.products -import tap_shopify.streams.collects -import tap_shopify.streams.custom_collections import tap_shopify.streams.locations import tap_shopify.streams.inventory_levels import tap_shopify.streams.inventory_items import tap_shopify.streams.product_variants import tap_shopify.streams.events +import tap_shopify.streams.collections diff --git a/tap_shopify/streams/abandoned_checkouts.py b/tap_shopify/streams/abandoned_checkouts.py index e3887f37..f0867b66 100644 --- a/tap_shopify/streams/abandoned_checkouts.py +++ b/tap_shopify/streams/abandoned_checkouts.py @@ -1,9 +1,277 @@ -import shopify from tap_shopify.context import Context from tap_shopify.streams.base import Stream + class AbandonedCheckouts(Stream): - name = 'abandoned_checkouts' - replication_object = shopify.Checkout + """Stream class for Abandoned Checkouts in Shopify.""" + name = "abandoned_checkouts" + data_key = "abandonedCheckouts" + replication_key = "updatedAt" + + @classmethod + def process_sub_entities(cls, data, entity_name): + """ + Processes sub-entities from the response data. + + Args: + data (dict): Response data. + entity_name (str): Name of the entity to process. + + Returns: + list: List of processed sub-entities. + """ + sub_entities = [] + for item in data[entity_name]["edges"]: + if node := item.get("node"): + sub_entities.append(node) + return sub_entities + + def transform_object(self, obj): + """ + Transforms the object by processing its sub-entities. + + Args: + obj (dict): Object to transform. + + Returns: + dict: Transformed object. + """ + obj["lineItems"] = self.process_sub_entities(obj, entity_name="lineItems") + return obj + + def get_query(self): + """ + Returns the GraphQL query for fetching abandoned checkouts. + + Returns: + str: GraphQL query string. + """ + return """ + query abandonedcheckouts($first: Int!, $after: String, $query: String) { + abandonedCheckouts(first: $first, after: $after, query: $query) { + edges { + node { + note + completedAt + billingAddress { + phone + country + firstName + name + latitude + zip + lastName + province + address2 + address1 + countryCodeV2 + city + company + provinceCode + longitude + coordinatesValidated + formattedArea + id + timeZone + validationResultSummary + } + discountCodes + createdAt + updatedAt + taxLines { + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + title + rate + source + channelLiable + ratePercentage + } + totalLineItemsPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + id + name + totalTaxSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + shippingAddress { + phone + country + firstName + name + latitude + zip + lastName + province + address2 + address1 + countryCodeV2 + city + company + provinceCode + longitude + coordinatesValidated + formattedArea + id + timeZone + validationResultSummary + } + abandonedCheckoutUrl + totalDiscountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + taxesIncluded + totalDutiesSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + lineItems(first: 250) { + edges { + node { + id + quantity + sku + title + variantTitle + variant { + title + id + } + discountedTotalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + components { + id + quantity + title + variantTitle + } + customAttributes { + key + value + } + product { + id + } + discountedUnitPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + discountedUnitPriceWithCodeDiscount { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + originalTotalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + originalUnitPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + subtotalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + """ + -Context.stream_objects['abandoned_checkouts'] = AbandonedCheckouts +Context.stream_objects["abandoned_checkouts"] = AbandonedCheckouts diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index 44c414d6..8737b4b5 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -1,71 +1,32 @@ -import datetime +from datetime import timedelta import functools +import json +import re import socket +import urllib from urllib.error import URLError import http import backoff import pyactiveresource import pyactiveresource.formats +import shopify import simplejson import singer from singer import metrics, utils -from singer.utils import strptime_to_utc from tap_shopify.context import Context LOGGER = singer.get_logger() -RESULTS_PER_PAGE = 175 +RESULTS_PER_PAGE = 250 # set default timeout of 300 seconds REQUEST_TIMEOUT = 300 -# We've observed 500 errors returned if this is too large (30 days was too -# large for a customer) -DATE_WINDOW_SIZE = 1 +DATE_WINDOW_SIZE = 30 # We will retry a 500 error a maximum of 5 times before giving up MAX_RETRIES = 5 -# We have observed transactions with receipt objects that contain both: -# - `token` and `Token` -# - `version` and `Version` -# - `ack` and `Ack` -# keys on transactions where PayPal is the payment type. We reached out to -# PayPal support and they told us the values should be the same, so one -# can be safely ignored since its a duplicate. Example: The logic is to -# prefer `token` if both are present and equal, convert `Token` -> `token` -# if only `Token` is present, and throw an error if both are present and -# their values are not equal. -def canonicalize(transaction_dict, field_name): - field_name_upper = field_name.capitalize() - # Not all Shopify transactions have receipts. Facebook has been shown - # to push a null receipt through the transaction - receipt = transaction_dict.get('receipt', {}) - if receipt: - value_lower = receipt.get(field_name) - value_upper = receipt.get(field_name_upper) - if value_lower and value_upper: - if value_lower == value_upper: - LOGGER.info(( - "Transaction (id=%d) contains a receipt " - "that has `%s` and `%s` keys with the same " - "value. Removing the `%s` key."), - transaction_dict['id'], - field_name, - field_name_upper, - field_name_upper) - transaction_dict['receipt'].pop(field_name_upper) - else: - raise ValueError(( - "Found Transaction (id={}) with a receipt that has " - "`{}` and `{}` keys with the different " - "values. Contact Shopify/PayPal support.").format( - transaction_dict['id'], - field_name_upper, - field_name)) - elif value_upper: - # pylint: disable=line-too-long - transaction_dict["receipt"][field_name] = transaction_dict['receipt'].pop(field_name_upper) # function to return request timeout def get_request_timeout(): @@ -79,6 +40,27 @@ def get_request_timeout(): return request_timeout +def execute_gql(self, query, variables=None, operation_name=None, timeout=None): + """ + This overrides the `execute` method from ShopifyAPI(v12.6.0) to remove the print statement + and also to explicitly pass the timeout value to the urlopen method. + Ensure to check the original impl before making any changes or upgrading the SDK version, + as this modification may affect future updates + """ + default_headers = {"Accept": "application/json", "Content-Type": "application/json"} + headers = self.merge_headers(default_headers, self.headers) + data = {"query": query, "variables": variables, "operationName": operation_name} + + req = urllib.request.Request(self.endpoint, json.dumps(data).encode("utf-8"), headers) + + try: + with urllib.request.urlopen(req, timeout=timeout) as response: + return response.read().decode("utf-8") + except urllib.error.HTTPError as http_error: + raise http_error + +shopify.GraphQL.execute = execute_gql + def is_not_status_code_fn(status_code): def gen_fn(exc): if getattr(exc, 'code', None) and exc.code not in status_code: @@ -154,12 +136,10 @@ class Stream(): # subclasses to change the bookmark key. name = None replication_method = 'INCREMENTAL' - replication_key = 'updated_at' + replication_key = 'updatedAt' key_properties = ['id'] - # Controls which SDK object we use to call the API by default. - replication_object = None - # Status parameter override option - status_key = None + date_window_size = float(Context.config.get("date_window_size", DATE_WINDOW_SIZE)) + data_key = None results_per_page = None def __init__(self): @@ -168,6 +148,37 @@ def __init__(self): # set request timeout self.request_timeout = get_request_timeout() + def get_query(self): + """ + Provides GraphQL query + """ + raise NotImplementedError("Function Not Implemented") + + def transform_object(self, obj): + """ + Modify this to perform custom transformation on each object + """ + return obj + + @classmethod + def camel_to_snake(cls, name): + """ + Convert camelCase to snake_case + + Args: + name (str): Input string in camelCase + + Returns: + str: Converted string in snake_case + """ + # Handle special cases + if not name: + return name + + # Use regex to insert underscore before capital letters + pattern = re.compile(r'(?='{updated_at_min}' AND {rkey}:<'{updated_at_max}'", + "first": self.results_per_page, } + if cursor: + params["after"] = cursor + return params def get_objects(self): - last_sync_interrupted_at = self.get_updated_at_max() - updated_at_min = self.get_bookmark() - max_bookmark = updated_at_min - - stop_time = singer.utils.now().replace(microsecond=0) - date_window_size = float(Context.config.get("date_window_size", DATE_WINDOW_SIZE)) - - # Page through till the end of the resultset - while updated_at_min < stop_time: - # Bookmarking can also occur on the since_id - since_id = self.get_since_id() or 1 - - if since_id != 1: - LOGGER.info("Resuming sync from since_id %d", since_id) - - # It's important that `updated_at_min` has microseconds - # truncated. Why has been lost to the mists of time but we - # think it has something to do with how the API treats - # microseconds on its date windows. Maybe it's possible to - # drop data due to rounding errors or something like that? - # If last sync was interrupted, set updated_at_max to - # updated_at_max bookmarked in the interrupted sync. - # This will make sure that records with lower id than since_id - # which got updated later won't be missed - updated_at_max = (last_sync_interrupted_at - or updated_at_min + datetime.timedelta(days=date_window_size)) - last_sync_interrupted_at = None - - updated_at_max = min(updated_at_max, stop_time) - while True: - status_key = self.status_key or "status" - query_params = self.get_query_params(since_id, - status_key, - updated_at_min, - updated_at_max) + """ + Returns: + - Yields list of objects for the stream + Performs + - Pagination & Filtering of stream + - Transformation and bookmarking + """ + + last_updated_at = self.get_bookmark() + sync_start = utils.now().replace(microsecond=0) + + while last_updated_at < sync_start: + date_window_end = last_updated_at + timedelta(days=self.date_window_size) + query_end = min(sync_start, date_window_end) + has_next_page, cursor = True, None + + while has_next_page: + query_params = self.get_query_params(last_updated_at, query_end, cursor) with metrics.http_request_timer(self.name): - objects = self.call_api(query_params) - - for obj in objects: - if obj.id < since_id: - # This verifies the api behavior expectation we - # have that all results actually honor the - # since_id parameter. - raise OutOfOrderIdsError("obj.id < since_id: {} < {}".format( - obj.id, since_id)) - replication_value = strptime_to_utc(getattr(obj, self.replication_key)) - if replication_value > max_bookmark: - max_bookmark = replication_value + data = self.call_api(query_params) + + for edge in data.get("edges"): + obj = self.transform_object(edge.get("node")) yield obj - # You know you're at the end when the current page has - # less than the request size limits you set. - if len(objects) < self.results_per_page: - # Save the updated_at_max as our bookmark as we've synced all rows up in our - # window and can move forward. Also remove the since_id because we want to - # restart at 1. - stream_bookmarks = Context.state.get('bookmarks', {}).get(self.name, {}) - stream_bookmarks.pop('since_id', None) - stream_bookmarks.pop('updated_at_max', None) - self.update_bookmark(utils.strftime(updated_at_max)) - break - - if objects[-1].id != max([o.id for o in objects]): - # This verifies the api behavior expectation we have - # that all pages are internally ordered by the - # `since_id`. - raise OutOfOrderIdsError("{} is not the max id in objects ({})".format( - objects[-1].id, max([o.id for o in objects]))) - since_id = objects[-1].id - - # Put since_id and updated_at_max into the state. - self.update_bookmark(since_id, bookmark_key='since_id') - self.update_bookmark(utils.strftime(updated_at_max), bookmark_key='updated_at_max') - - updated_at_min = updated_at_max - bookmark = max(min(stop_time, - max_bookmark), - (stop_time - datetime.timedelta(days=date_window_size))) - self.update_bookmark(utils.strftime(bookmark)) + page_info = data.get("pageInfo") + cursor , has_next_page = page_info.get("endCursor"), page_info.get("hasNextPage") - def sync(self): - """Yield's processed SDK object dicts to the caller. + last_updated_at = query_end - This is the default implementation. Get's all of self's objects - and calls to_dict on them with no further processing. + def sync(self): """ + Default implementation for sync method + """ + start_time = utils.now().replace(microsecond=0) + max_bookmark_value = self.get_bookmark() + for obj in self.get_objects(): - yield obj.to_dict() + max_bookmark_value = max( + max_bookmark_value, + utils.strptime_to_utc(obj[self.replication_key]) + ) + yield obj + + # Update bookmark to the latest value, but not beyond sync start time + max_bookmark_value = min(start_time, max_bookmark_value) + self.update_bookmark(utils.strftime(max_bookmark_value)) diff --git a/tap_shopify/streams/collections.py b/tap_shopify/streams/collections.py new file mode 100644 index 00000000..9b607f01 --- /dev/null +++ b/tap_shopify/streams/collections.py @@ -0,0 +1,122 @@ +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream + + +class Collections(Stream): + """Stream class for Shopify collections.""" + name = "collections" + data_key = "collections" + replication_key = "updatedAt" + + def transform_products(self, data): + """ + Transforms the products data by extracting product IDs and handling pagination. + + Args: + data (dict): Product data. + + Returns: + list: List of product IDs. + """ + # Extract product IDs from the first page + product_ids = [ + node["id"] + for item in data["products"]["edges"] + if (node := item.get("node")) and "id" in node + ] + + # Handle pagination + page_info = data["products"].get("pageInfo", {}) + while page_info.get("hasNextPage"): + params = { + "first": self.results_per_page, + "query": f"id:{data['id'].split('/')[-1]}", + "childafter": page_info.get("endCursor"), + } + + # Fetch the next page of data + response = self.call_api(params) + products_data = response.get("node", {}).get("products", {}) + product_ids.extend( + node["id"] + for item in products_data.get("edges", []) + if (node := item.get("node")) and "id" in node + ) + page_info = products_data.get("pageInfo", {}) + + return product_ids + + def transform_object(self, obj): + """ + Transforms a collection object. + + Args: + obj (dict): Collection object. + + Returns: + dict: Transformed collection object. + """ + obj["collectionType"] = "SMART" if obj.get("ruleSet") else "MANUAL" + if obj.get("products"): + obj["products"] = self.transform_products(obj) + return obj + + def get_query(self): + """ + Returns the GraphQL query for fetching collections. + + Returns: + str: GraphQL query string. + """ + return """ + query Collections($first: Int!, $after: String, $query: String, $childafter: String) { + collections(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { + edges { + node { + id + title + handle + updatedAt + productsCount { + count + precision + } + sortOrder + ruleSet { + appliedDisjunctively + rules { + column + condition + relation + } + } + seo { + description + title + } + feedback { + summary + } + products(first: 250, sortKey: ID, after: $childafter) { + edges { + node { + id + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + + +Context.stream_objects["collections"] = Collections diff --git a/tap_shopify/streams/collects.py b/tap_shopify/streams/collects.py deleted file mode 100644 index 83342ee5..00000000 --- a/tap_shopify/streams/collects.py +++ /dev/null @@ -1,50 +0,0 @@ -import shopify -import singer -from singer import utils -from tap_shopify.streams.base import (Stream, - OutOfOrderIdsError) -from tap_shopify.context import Context - -LOGGER = singer.get_logger() - -class Collects(Stream): - name = 'collects' - replication_object = shopify.Collect - replication_key = 'updated_at' - - def get_objects(self): - since_id = 1 - bookmark = self.get_bookmark() - max_bookmark = utils.strftime(utils.now()) - while True: - query_params = { - "since_id": since_id, - "limit": self.results_per_page, - } - - objects = self.call_api(query_params) - - for obj in objects: - # Syncing Collects is a full sync every time but emitting - # records that have an updated_date greater than the - # bookmark - if not obj.updated_at and obj.id: - LOGGER.info('Collect with id: %d does not have an updated_at, syncing it!', - obj.id) - if not obj.updated_at or utils.strptime_with_tz(obj.updated_at) > bookmark: - if obj.id < since_id: - raise OutOfOrderIdsError("obj.id < since_id: {} < {}".format( - obj.id, since_id)) - yield obj - - if len(objects) < self.results_per_page: - # Update the bookmark at the end of the last page - self.update_bookmark(max_bookmark) - break - if objects[-1].id != max([o.id for o in objects]): - raise OutOfOrderIdsError("{} is not the max id in objects ({})".format( - objects[-1].id, max([o.id for o in objects]))) - since_id = objects[-1].id - - -Context.stream_objects['collects'] = Collects diff --git a/tap_shopify/streams/custom_collections.py b/tap_shopify/streams/custom_collections.py deleted file mode 100644 index 5e5d0f49..00000000 --- a/tap_shopify/streams/custom_collections.py +++ /dev/null @@ -1,11 +0,0 @@ -import shopify - -from tap_shopify.streams.base import Stream -from tap_shopify.context import Context - - -class CustomCollections(Stream): - name = 'custom_collections' - replication_object = shopify.CustomCollection - -Context.stream_objects['custom_collections'] = CustomCollections diff --git a/tap_shopify/streams/customers.py b/tap_shopify/streams/customers.py index 8afab28c..2ff65502 100644 --- a/tap_shopify/streams/customers.py +++ b/tap_shopify/streams/customers.py @@ -1,11 +1,113 @@ -import shopify - -from tap_shopify.streams.base import Stream from tap_shopify.context import Context +from tap_shopify.streams.base import Stream class Customers(Stream): - name = 'customers' - replication_object = shopify.Customer + """Stream class for Shopify Customers.""" + name = "customers" + data_key = "customers" + replication_key = "updatedAt" + + def get_query(self): + """ + Returns the GraphQL query for fetching customers. + + Returns: + str: GraphQL query string. + """ + return """ + query Customers($first: Int!, $after: String, $query: String) { + customers(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { + edges { + node { + email + multipassIdentifier + defaultAddress { + city + address1 + zip + id + province + phone + country + firstName + lastName + countryCodeV2 + name + provinceCode + address2 + company + timeZone + validationResultSummary + latitude + longitude + coordinatesValidated + formattedArea + } + numberOfOrders + state + verifiedEmail + firstName + updatedAt + note + phone + addresses(first: 250) { + city + address1 + zip + id + province + phone + country + firstName + lastName + countryCodeV2 + name + provinceCode + address2 + company + timeZone + validationResultSummary + latitude + longitude + coordinatesValidated + formattedArea + } + lastName + tags + taxExempt + id + createdAt + taxExemptions + emailMarketingConsent { + consentUpdatedAt + marketingOptInLevel + marketingState + } + smsMarketingConsent { + consentCollectedFrom + consentUpdatedAt + marketingOptInLevel + marketingState + } + validEmailAddress + productSubscriberStatus + amountSpent { + amount + currencyCode + } + dataSaleOptOut + displayName + locale + lifetimeDuration + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ -Context.stream_objects['customers'] = Customers +Context.stream_objects["customers"] = Customers diff --git a/tap_shopify/streams/events.py b/tap_shopify/streams/events.py index bd59aff2..16f6b70c 100644 --- a/tap_shopify/streams/events.py +++ b/tap_shopify/streams/events.py @@ -1,21 +1,84 @@ -import shopify - -from tap_shopify.streams.base import Stream from tap_shopify.context import Context +from tap_shopify.streams.base import Stream class Events(Stream): - name = 'events' - replication_object = shopify.Event - replication_key = "created_at" + """Stream class for Shopify Events.""" + + name = "events" + data_key = "events" + replication_key = "createdAt" + + def get_query(self): + """ + Returns the GraphQL query for fetching events. + + Returns: + str: GraphQL query string. + """ + return """ + query GetEvents($first: Int!, $after: String, $query: String) { + events(first: $first, after: $after, query: $query, sortKey: CREATED_AT) { + edges { + node { + id + createdAt + action + appTitle + attributeToApp + attributeToUser + criticalAlert + message + ... on BasicEvent { + id + subjectId + subjectType + action + additionalContent + additionalData + appTitle + arguments + attributeToApp + attributeToUser + createdAt + criticalAlert + hasAdditionalContent + message + secondaryMessage + } + ... on CommentEvent { + id + action + appTitle + attachments { + fileExtension + id + name + size + url + } + attributeToApp + attributeToUser + author { + id + } + canDelete + canEdit + createdAt + criticalAlert + edited + message + rawMessage + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ - def get_query_params(self, since_id, status_key, updated_at_min, updated_at_max): - return { - "since_id": since_id, - "created_at_min": updated_at_min, - "created_at_max": updated_at_max, - "limit": self.results_per_page, - status_key: "any" - } -Context.stream_objects['events'] = Events +Context.stream_objects["events"] = Events diff --git a/tap_shopify/streams/graphql/__init__.py b/tap_shopify/streams/graphql/__init__.py deleted file mode 100644 index 4a9b2523..00000000 --- a/tap_shopify/streams/graphql/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -""" -tap shopify graphql module -""" - -from .gql_base import ShopifyGqlStream, ShopifyGraphQLError - -from .gql_queries import ( - get_metafields_query, - get_inventory_items_query, - get_metafield_query_collection, - get_metafield_query_order, - get_metafield_query_customers, - get_parent_ids_query, - get_product_variant_query, - get_metafield_query_product, - get_metafield_query_shop, - get_products_query) diff --git a/tap_shopify/streams/graphql/gql_base.py b/tap_shopify/streams/graphql/gql_base.py deleted file mode 100644 index 657f817b..00000000 --- a/tap_shopify/streams/graphql/gql_base.py +++ /dev/null @@ -1,150 +0,0 @@ -from datetime import timedelta -import json -import urllib -import shopify - -from singer import( - metrics, - get_logger, - utils -) -from tap_shopify.context import Context -from tap_shopify.streams.base import ( - Stream, - shopify_error_handling, - ShopifyAPIError, - DATE_WINDOW_SIZE, - ) - - -LOGGER = get_logger() - -class ShopifyGraphQLError(ShopifyAPIError): - """Custom exception for GraphQL errors""" - - -def execute_gql(self, query, variables=None, operation_name=None): - """ - This overrides the `execute` method from ShopifyAPI(v12.6.0) to remove the print statement. - Ensure to check the original impl before making any changes or upgrading the SDK version, - as this modification may affect future updates - """ - default_headers = {"Accept": "application/json", "Content-Type": "application/json"} - headers = self.merge_headers(default_headers, self.headers) - data = {"query": query, "variables": variables, "operationName": operation_name} - - req = urllib.request.Request(self.endpoint, json.dumps(data).encode("utf-8"), headers) - - try: - with urllib.request.urlopen(req) as response: - return response.read().decode("utf-8") - except urllib.error.HTTPError as http_error: - raise http_error - -shopify.GraphQL.execute = execute_gql - -class ShopifyGqlStream(Stream): - - data_key = None - - def get_query(self): - """ - Provides GraphQL query - """ - raise NotImplementedError("Function Not Implemented") - - def transform_object(self, obj): - """ - Modify this to perform custom transformation on each object - """ - return obj - - # pylint: disable=W0221 - def get_query_params(self, updated_at_min, updated_at_max, cursor=None): - """ - Construct query parameters for GraphQL requests. - - Args: - updated_at_min (str): Minimum updated_at timestamp. - updated_at_max (str): Maximum updated_at timestamp. - cursor (str): Pagination cursor, if any. - - Returns: - dict: Dictionary of query parameters. - """ - rkey = self.replication_key - params = { - "query": f"{rkey}:>='{updated_at_min}' AND {rkey}:<'{updated_at_max}'", - "first": self.results_per_page, - } - if cursor: - params["after"] = cursor - return params - - @shopify_error_handling - def call_api(self, query_params): - """ - - Modifies the default call api implementation to support GraphQL - - Returns response Object dict - """ - - try: - query = self.get_query() - LOGGER.info("Fetching %s %s", self.name, query_params) - response = shopify.GraphQL().execute(query=query, variables=query_params) - response = json.loads(response) - if "errors" in response.keys(): - raise ShopifyAPIError(response['errors']) - data = response.get("data", {}).get(self.data_key, {}) - return data - except ShopifyAPIError as gql_error: - LOGGER.error("GraphQL Error %s", gql_error) - raise ShopifyAPIError("An error occurred with the GraphQL API.") from gql_error - except Exception as exc: - LOGGER.error("Unexpected error occurred.",) - raise exc - - def get_objects(self): - """ - Returns: - - Yields list of objects for the stream - Performs - - Pagination & Filtering of stream - - Transformation and bookmarking - """ - - last_updated_at = self.get_bookmark() - current_bookmark = last_updated_at - sync_start = utils.now().replace(microsecond=0) - date_window_size = float(Context.config.get("date_window_size", DATE_WINDOW_SIZE)) - - while last_updated_at < sync_start: - date_window_end = last_updated_at + timedelta(days=date_window_size) - query_end = min(sync_start, date_window_end) - has_next_page, cursor = True, None - - while has_next_page: - query_params = self.get_query_params(last_updated_at, query_end, cursor) - - with metrics.http_request_timer(self.name): - data = self.call_api(query_params) - - for edge in data.get("edges"): - obj = self.transform_object(edge.get("node")) - replication_value = utils.strptime_to_utc(obj[self.replication_key]) - if replication_value > current_bookmark: - current_bookmark = replication_value - yield obj - - page_info = data.get("pageInfo") - cursor , has_next_page = page_info.get("endCursor"), page_info.get("hasNextPage") - - last_updated_at = query_end - self.update_bookmark(utils.strftime(current_bookmark)) - - def sync(self): - """ - Default implementation for sync method - """ - for obj in self.get_objects(): - yield obj diff --git a/tap_shopify/streams/graphql/gql_queries.py b/tap_shopify/streams/graphql/gql_queries.py deleted file mode 100644 index 7145ebe8..00000000 --- a/tap_shopify/streams/graphql/gql_queries.py +++ /dev/null @@ -1,469 +0,0 @@ -""" -Stores all the GraphQl Queries for shopify api -""" - -def get_products_query(): - """ - Returns GraphQL query to get all products - """ - return """ - query GetProducts($first: Int!, $after: String, $query: String) { - products(first: $first, after: $after, query: $query) { - edges { - node { - id - title - descriptionHtml - vendor - category { - id - } - tags - handle - publishedAt - createdAt - updatedAt - templateSuffix - status - productType - options { - id - name - position - values - } - giftCardTemplateSuffix - hasOnlyDefaultVariant - hasOutOfStockVariants - hasVariantsThatRequiresComponents - isGiftCard - description - compareAtPriceRange { - maxVariantCompareAtPrice { - amount - currencyCode - } - minVariantCompareAtPrice { - amount - currencyCode - } - } - featuredMedia { - id - mediaContentType - status - } - requiresSellingPlan - totalInventory - tracksInventory - media(first: 250) { - edges { - node { - id - alt - status - mediaContentType - mediaWarnings { - code - message - } - mediaErrors { - code - details - message - } - ... on ExternalVideo { - id - embedUrl - } - ... on MediaImage { - id - updatedAt - createdAt - mimeType - image { - url - width - height - id - } - } - ... on Model3d { - id - filename - sources { - url - format - mimeType - filesize - } - } - ... on Video { - id - updatedAt - createdAt - filename - sources { - url - format - mimeType - fileSize - } - - } - } - } - } - } - } - pageInfo { - hasNextPage - endCursor - } - } - } - """ - -def get_product_variant_query(): - """ - Returns GraphQL query to get all product variants - """ - return """ - query GetProductVariants($first: Int!, $after: String, $query: String) { - productVariants(first: $first, after: $after, query: $query) { - edges { - node { - id - createdAt - barcode - availableForSale - compareAtPrice - displayName - image { - altText - height - id - url - width - } - inventoryPolicy - inventoryQuantity - position - price - requiresComponents - sellableOnlineQuantity - sku - taxCode - taxable - title - updatedAt - product { id } - inventoryItem { id } - } - } - pageInfo { - hasNextPage - endCursor - } - } - } - """ - -def get_inventory_items_query(): - """ - Returns GraphQL query to get all inventory items - """ - return """ - query GetinventoryItems($first: Int!, $after: String, $query: String) { - inventoryItems(first: $first, after: $after, query: $query) { - edges { - node { - id - createdAt - sku - updatedAt - requiresShipping - countryCodeOfOrigin - provinceCodeOfOrigin - harmonizedSystemCode - tracked - unitCost { - amount - } - countryHarmonizedSystemCodes(first: 175) { - edges { - node { - countryCode - harmonizedSystemCode - } - } - } - } - } - pageInfo { - hasNextPage - endCursor - } - } - } - """ - -def get_parent_ids_query(resource): - """ - Returns GraphQL query to get id for all metaftield - supported parent streams. - """ - - qry = """ - query getParentEntities( $first: Int!, $after: String $query: String) { - RESOURCE(first: $first after: $after query: $query) { - edges { - node { - id - } - } - pageInfo { - endCursor - hasNextPage - } - } - } - """ - qry = qry.replace("RESOURCE", resource) - return qry - -def get_metafields_query(resource): - """Returns the GraphQL query for fetching metafields""" - if resource == 'shop': - return get_metafield_query_shop() - - qry = """ - query getMetafields( $first: Int!, $after: String $query: String) { - RESOURCE(first: $first after: $after query: $query) { - edges { - node { - metafields(first: $first) { - edges { - node { - id - ownerType - value - type - key - createdAt - namespace - description - updatedAt - owner { - ... on Customer { - id - } - ... on Product { - id - } - ... on Order { - id - } - ... on Collection { - id - } - } - } - } - pageInfo { - hasNextPage - endCursor - } - } - } - } - pageInfo { - endCursor - hasNextPage - } - } - } - """ - - qry = qry.replace("RESOURCE", resource) - return qry - -def get_metafield_query_customers(): - """ - Returns GraphQL query to get customer metafields - """ - qry = """ - query GetMetafields($pk_id: ID! $first: Int!, $after: String) { - customer(id: $pk_id) { - metafields(first: $first after: $after ){ - edges { - node { - id - ownerType - value - type - key - createdAt - namespace - description - updatedAt - owner { - ... on Customer { - id - } - } - } - } - pageInfo { - hasNextPage - endCursor - } - } - } - } - """ - return qry - -def get_metafield_query_product(): - """ - Returns GraphQL query to get product metafields - """ - qry = """ - query GetMetafields($pk_id: ID! $first: Int!, $after: String) { - product(id: $pk_id) { - metafields(first: $first after: $after ){ - edges { - node { - id - ownerType - value - type - key - createdAt - namespace - description - updatedAt - owner { - ... on Product { - id - } - } - } - } - pageInfo { - hasNextPage - endCursor - } - } - } - } - """ - return qry - -def get_metafield_query_collection(): - """ - Returns GraphQL query to get collection metafields - """ - qry = """ - query GetMetafields($pk_id: ID! $first: Int!, $after: String) { - collection(id: $pk_id) { - metafields(first: $first after: $after ){ - edges { - node { - id - ownerType - value - type - key - createdAt - namespace - description - updatedAt - owner { - ... on Collection { - id - } - } - } - } - pageInfo { - hasNextPage - endCursor - } - } - } - } - """ - return qry - -def get_metafield_query_order(): - """ - Returns GraphQL query to get order metafields - """ - qry = """ - query GetMetafields($pk_id: ID! $first: Int!, $after: String) { - order(id: $pk_id) { - metafields(first: $first after: $after ){ - edges { - node { - id - ownerType - value - type - key - createdAt - namespace - description - updatedAt - owner { - ... on Order { - id - } - } - } - } - pageInfo { - hasNextPage - endCursor - } - } - } - } - """ - return qry - -def get_metafield_query_shop(): - """ - Returns GraphQL query to get shop metafields - """ - qry = """ - query GetMetafields($first: Int!, $after: String) { - shop{ - metafields(first: $first after: $after ){ - edges { - node { - id - ownerType - value - type - key - createdAt - namespace - description - updatedAt - owner { - ... on Shop { - id - } - } - } - } - pageInfo { - hasNextPage - endCursor - } - } - } - } - """ - return qry diff --git a/tap_shopify/streams/inventory_items.py b/tap_shopify/streams/inventory_items.py index 641ff969..dde493b4 100644 --- a/tap_shopify/streams/inventory_items.py +++ b/tap_shopify/streams/inventory_items.py @@ -1,34 +1,26 @@ from tap_shopify.context import Context -from tap_shopify.streams.graphql import get_inventory_items_query -from tap_shopify.streams.graphql import ShopifyGqlStream +from tap_shopify.streams.base import Stream +class InventoryItems(Stream): + """Stream class for inventory items.""" -class InventoryItems(ShopifyGqlStream): - name = 'inventory_items' + name = "inventory_items" data_key = "inventoryItems" replication_key = "updatedAt" - def get_query(self): - return get_inventory_items_query() - - # pylint: disable=W0221 - def get_query_params(self, updated_at_min, updated_at_max, cursor=None): - """ - Returns query and params for filtering, pagination + def transform_object(self, obj): """ - filter_key = "updated_at" - params = { - "query": f"{filter_key}:>='{updated_at_min}' AND {filter_key}:<'{updated_at_max}'", - "first": self.results_per_page, - } - if cursor: - params["after"] = cursor - return params + Transforms the object by extracting country harmonized system codes. - def transform_object(self, obj): + Args: + obj (dict): The object to transform. + + Returns: + dict: Transformed object. + """ hsc = obj.get("countryHarmonizedSystemCodes") - hsc_list = [] + hsc_list = [] if hsc and "edges" in hsc: for edge in hsc.get("edges"): node = edge.get("node") @@ -37,4 +29,47 @@ def transform_object(self, obj): obj["countryHarmonizedSystemCodes"] = hsc_list return obj -Context.stream_objects['inventory_items'] = InventoryItems + def get_query(self): + """ + Returns GraphQL query to get all inventory items. + + Returns: + str: GraphQL query string. + """ + return """ + query GetinventoryItems($first: Int!, $after: String, $query: String) { + inventoryItems(first: $first, after: $after, query: $query) { + edges { + node { + id + createdAt + sku + updatedAt + requiresShipping + countryCodeOfOrigin + provinceCodeOfOrigin + harmonizedSystemCode + tracked + unitCost { + amount + } + countryHarmonizedSystemCodes(first: 175) { + edges { + node { + countryCode + harmonizedSystemCode + } + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + """ + + +Context.stream_objects["inventory_items"] = InventoryItems diff --git a/tap_shopify/streams/inventory_levels.py b/tap_shopify/streams/inventory_levels.py index 8cc6efda..e202ab81 100644 --- a/tap_shopify/streams/inventory_levels.py +++ b/tap_shopify/streams/inventory_levels.py @@ -1,60 +1,149 @@ -import shopify -from singer.utils import strftime, strptime_to_utc -from tap_shopify.streams.base import (Stream, - RESULTS_PER_PAGE, - shopify_error_handling) +from datetime import timedelta +from singer import metrics, utils from tap_shopify.context import Context +from tap_shopify.streams.base import Stream + class InventoryLevels(Stream): - name = 'inventory_levels' - replication_key = 'updated_at' - key_properties = ['location_id', 'inventory_item_id'] - replication_object = shopify.InventoryLevel - # Added decorator over functions of shopify SDK - replication_object.find = shopify_error_handling(replication_object.find) - - def api_call_for_inventory_levels(self, parent_object_id, bookmark): - # set timeout - self.replication_object.set_timeout(self.request_timeout) - return self.replication_object.find( - updated_at_min = bookmark, - limit = RESULTS_PER_PAGE, - location_ids=parent_object_id - ) - - def get_inventory_levels(self, parent_object, bookmark): - inventory_page = self.api_call_for_inventory_levels(parent_object, bookmark) - yield from inventory_page - - while inventory_page.has_next_page(): - inventory_page = inventory_page.next_page() - yield from inventory_page + """Stream class for inventory levels.""" + + name = "inventory_levels" + data_key = "locations" + child_data_key = "inventoryLevels" + replication_key = "updatedAt" + + def get_next_page_child(self, parent_id, cursor, child_query): + """ + Gets all child objects efficiently with pagination. + + Args: + parent_id (str): The ID of the parent object. + cursor (str): The cursor for pagination. + child_query (str): The query for child objects. + Yields: + dict: The child object. + """ + has_next_page = True + while has_next_page: + child_query_params = { + "first": self.results_per_page, + "parentquery": f"id:{parent_id.split('/')[-1]}", + "query": child_query, + "childafter": cursor, + } + data = self.call_api(child_query_params) + for edge in data.get("edges", []): + node = edge.get("node", {}) + child_data = node.get(self.child_data_key, {}) + child_edges = child_data.get("edges", []) + yield from child_edges + + page_info = child_data.get("pageInfo", {}) + cursor = page_info.get("endCursor") + has_next_page = page_info.get("hasNextPage", False) + + # pylint: disable=too-many-locals def get_objects(self): - bookmark = self.get_bookmark() + """ + Retrieves objects in paginated batches. + + Yields: + dict: The transformed object. + """ + last_updated_at = self.get_bookmark() + sync_start = utils.now().replace(microsecond=0) + + while last_updated_at < sync_start: + date_window_end = last_updated_at + timedelta(days=self.date_window_size) + query_end = min(sync_start, date_window_end) + has_next_page, cursor = True, None + + while has_next_page: + query_params = self.get_query_params(last_updated_at, query_end, cursor) + with metrics.http_request_timer(self.name): + data = self.call_api(query_params) + + # Process parent objects + for edge in data.get("edges", []): + node = edge.get("node", {}) + + # Handle already fetched child objects + child_edges = node.get(self.child_data_key, {}).get("edges", []) + for child_obj in child_edges: + obj = self.transform_object(child_obj.get("node")) + yield obj + + # Check if more child pages are needed + child_page_info = node.get(self.child_data_key, {}).get("pageInfo", {}) + if child_page_info.get("hasNextPage", False): + parent_id = node.get("id") + child_cursor = child_page_info.get("endCursor") + + # Get remaining child pages + for child_obj in self.get_next_page_child( + parent_id, child_cursor, query_params["query"] + ): + transformed_obj = self.transform_object(child_obj.get("node")) + yield transformed_obj - selected_parent = Context.stream_objects['locations']() - selected_parent.name = "inventory_level_locations" + page_info = data.get("pageInfo", {}) + cursor = page_info.get("endCursor") + has_next_page = page_info.get("hasNextPage") - # Get all locations data as location id is used for Inventory Level - # If we get locations updated after a bookmark - # then there is possibility of data loss for Inventory Level - # because location is not updated when any Inventory Level is updated inside it. - for parent_object in selected_parent.get_locations_data(): - yield from self.get_inventory_levels(parent_object.id, bookmark) + last_updated_at = query_end - def sync(self): - bookmark = self.get_bookmark() - max_bookmark = bookmark - for inventory_level in self.get_objects(): - inventory_level_dict = inventory_level.to_dict() - replication_value = strptime_to_utc(inventory_level_dict[self.replication_key]) - if replication_value >= bookmark: - yield inventory_level_dict + # pylint: disable=C0301 + def get_query(self): + """ + Returns the GraphQL query for inventory levels. - if replication_value > max_bookmark: - max_bookmark = replication_value + Returns: + str: The GraphQL query string. + """ + return """query GetInventoryLevels($first: Int!, $after: String, $query: String, $childafter: String, $parentquery: String) { + locations(first: $first, after: $after, query: $parentquery, sortKey: ID, includeInactive: true, includeLegacy: true) { + edges { + node { + inventoryLevels(first: $first, query: $query, after: $childafter) { + edges { + node { + canDeactivate + createdAt + deactivationAlert + id + location { + id + } + updatedAt + item { + id + variant { + id + } + } + quantities(names: ["available", "committed", "damaged", "incoming", "on_hand", "quality_control", "reserved", "safety_stock"]) { + id + name + quantity + updatedAt + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + id + } + } + pageInfo { + endCursor + hasNextPage + } + } + }""" - self.update_bookmark(strftime(max_bookmark)) -Context.stream_objects['inventory_levels'] = InventoryLevels +Context.stream_objects["inventory_levels"] = InventoryLevels diff --git a/tap_shopify/streams/locations.py b/tap_shopify/streams/locations.py index c3bd3c6c..67b60ec1 100644 --- a/tap_shopify/streams/locations.py +++ b/tap_shopify/streams/locations.py @@ -1,40 +1,69 @@ -import shopify -from singer import utils -from tap_shopify.streams.base import (Stream, shopify_error_handling) from tap_shopify.context import Context +from tap_shopify.streams.base import Stream class Locations(Stream): - name = 'locations' - replication_object = shopify.Location - # Added decorator over functions of shopify SDK - replication_object.find = shopify_error_handling(replication_object.find) + """Stream class for Shopify Locations""" - def get_locations_data(self): - # set timeout - self.replication_object.set_timeout(self.request_timeout) - location_page = self.replication_object.find() - yield from location_page + name = "locations" + data_key = "locations" + # Currently, the replication key is set to 'createdAt' because the Shopify + # locations graphql endpoint doesn't allow the filter on 'updatedAt' field. + replication_key = "createdAt" - while location_page.has_next_page(): - location_page = location_page.next_page() - yield from location_page + def get_query(self): + """ + Returns the GraphQL query for fetching locations. - def sync(self): - bookmark = self.get_bookmark() - max_bookmark = bookmark + Returns: + str: GraphQL query string. + """ + return """ + query GetLocations($first: Int!, $after: String, $query: String) { + locations(first: $first, after: $after, query: $query, sortKey: ID) { + edges { + node { + address { + countryCode + address1 + city + address2 + provinceCode + zip + province + phone + country + formatted + latitude + longitude + } + name + id + updatedAt + createdAt + isActive + addressVerified + deactivatable + deactivatedAt + deletable + fulfillsOnlineOrders + hasActiveInventory + hasUnfulfilledOrders + isFulfillmentService + legacyResourceId + localPickupSettingsV2 { + instructions + pickupTime + } + shipsInventory + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ - for location in self.get_locations_data(): - location_dict = location.to_dict() - replication_value = utils.strptime_to_utc(location_dict[self.replication_key]) - - if replication_value >= bookmark: - yield location_dict - - # update max bookmark if "replication_value" of current location is greater - if replication_value > max_bookmark: - max_bookmark = replication_value - - self.update_bookmark(utils.strftime(max_bookmark)) - -Context.stream_objects['locations'] = Locations +Context.stream_objects["locations"] = Locations diff --git a/tap_shopify/streams/metafields.py b/tap_shopify/streams/metafields.py index afc61ecf..d5c32d2f 100644 --- a/tap_shopify/streams/metafields.py +++ b/tap_shopify/streams/metafields.py @@ -1,199 +1,130 @@ +from abc import ABC, abstractmethod from datetime import timedelta import json -import shopify - -from singer import utils, get_logger +from singer import utils, get_logger, metrics +from tap_shopify.streams.base import Stream from tap_shopify.context import Context -from tap_shopify.streams.graphql import ( - get_metafields_query, - get_metafield_query_customers, - get_metafield_query_product, - get_metafield_query_collection, - get_metafield_query_order, - get_metafield_query_shop, -) -from tap_shopify.streams.graphql.gql_base import ( - ShopifyGqlStream, - ShopifyAPIError, - DATE_WINDOW_SIZE, - shopify_error_handling, -) LOGGER = get_logger() -class Metafields(ShopifyGqlStream): - name = 'metafields' - data_key = "metafields" - replication_key = "updatedAt" - - selected_parent = None +class Metafields(Stream, ABC): + """Stream class for Shopify Metafields""" - # maps object list identifier to single object access identifier - # eg customers -> customer - resource_alias = { - "shop": "shop", - "customers": "customer", - "products": "product", - "collections": "collection", - "orders": "order" - } + name = None + data_key = None + child_data_key = "metafields" + replication_key = "updatedAt" + @abstractmethod def get_query(self): - return None + """Placeholder for get_query method.""" - # pylint: disable=arguments-differ - def get_query_params(self, updated_at_min, updated_at_max, cursor=None): + def transform_object(self, obj): """ - Returns query and params for filtering, pagination + Transforms a metafield object for output. """ - rkey = "updated_at" - params = { - "query": f"{rkey}:>='{updated_at_min}' AND {rkey}:<'{updated_at_max}'", - "first": self.results_per_page, - } - if cursor: - params["after"] = cursor - return params - - def get_resource_type_query(self, resource): - return { - "customer": get_metafield_query_customers, - "product": get_metafield_query_product, - "collection": get_metafield_query_collection, - "order": get_metafield_query_order, - "shop": get_metafield_query_shop - }.get(resource) - - @shopify_error_handling - def call_api(self, query_params, query): - try: - response = shopify.GraphQL().execute(query=query, variables=query_params) - response = json.loads(response) - if "errors" in response.keys(): - raise ShopifyAPIError(response['errors']) - data = response.get("data", {}) - return data - except ShopifyAPIError as gql_error: - LOGGER.error("GraphQL Error %s", gql_error) - raise ShopifyAPIError("An error occurred with the GraphQL API.") from gql_error - except Exception as exception: - LOGGER.error("Unexpected error occurred.",) - raise exception - - def transform_object(self, obj): - obj["value_type"] = obj["type"] or None - obj["updated_at"] = obj["updatedAt"] + obj["value_type"] = obj.get("type") or None + obj["updated_at"] = obj.get("updatedAt") if obj["value_type"] in ["json", "weight", "volume", "dimension", "rating"]: value = obj.get("value") try: obj["value"] = json.loads(value) if value is not None else value except json.decoder.JSONDecodeError: - LOGGER.info("Failed to decode JSON value for obj %s", obj.get('id')) + LOGGER.info("Failed to decode JSON value for obj %s", obj.get("id")) obj["value"] = value return obj - def get_next_page_metafields(self, metafields, query_params, gql_resource): - owner_id = metafields.get("edges", [])[0]["node"]["owner"]["id"] - query = self.get_resource_type_query(gql_resource)() - has_next_page = True - cursor = metafields.get("pageInfo", {}).get("endCursor") - while has_next_page: - query_params["after"] = cursor - query_params["pk_id"] = owner_id - data = self.call_api(query_params, query).get(gql_resource, {}) - metafields = data.get("metafields", {}) - for edge in metafields.get("edges", []): - yield edge - page_info = metafields.get("pageInfo", {}) - cursor = page_info.get("endCursor") - has_next_page = page_info.get("hasNextPage", False) + def fetch_paginated_child_data(self, initial_child_data, parent_id): + """ + Fetches all pages of child data by handling pagination. + """ + # Extract the numeric ID from the full path + numeric_id = parent_id.split('/')[-1] + page_info = initial_child_data.get("pageInfo", {}) + + while page_info.get("hasNextPage"): + query_params = { + "first": self.results_per_page, + "query": f"id:{numeric_id}", + "childafter": page_info.get("endCursor"), + } + + response = self.call_api(query_params) + response_edges = response.get("edges", []) + if not response_edges: + break + + first_edge = response_edges[0] + child_data = first_edge.get("node", {}).get(self.child_data_key, {}) + + yield from child_data.get("edges", []) + + page_info = child_data.get("pageInfo", {}) # pylint: disable=too-many-locals, too-many-nested-blocks def get_objects(self): - """Main iterator to yield metafield objects""" + """ + Main iterator to yield metafield objects. + """ sync_start = utils.now().replace(microsecond=0) - - for resource_type, gql_resource in self.resource_alias.items(): - LOGGER.info("Syncing metafields for %s", resource_type) - - # Handle shop metafields separately - if resource_type == 'shop': - query = get_metafields_query(gql_resource) - has_next_page = True - cursor = None - - while has_next_page: - params = {"first": self.results_per_page} - if cursor: - params["after"] = cursor - - data = self.call_api(params, query) - metafields = data.get("shop", {}).get("metafields", {}) - - for edge in metafields.get("edges", []): - yield self.transform_object(edge["node"]) - - page_info = metafields.get("pageInfo", {}) - cursor = page_info.get("endCursor") - has_next_page = page_info.get("hasNextPage", False) - continue - - # Handle other resources - last_updated_at = self.get_bookmark_by_name(f'{self.name}_{resource_type}') - date_window_size = float(Context.config.get("date_window_size", DATE_WINDOW_SIZE)) - - while last_updated_at < sync_start: - date_window_end = last_updated_at + timedelta(days=date_window_size) - query_end = min(sync_start, date_window_end) - - has_next_page = True - cursor = None - - while has_next_page: - query_params = self.get_query_params(last_updated_at, query_end, cursor) - query = get_metafields_query(resource_type) - data = self.call_api(query_params, query) - - resource_data = data.get(resource_type, {}) - for data_edge in resource_data.get("edges", []): - metafields = data_edge["node"].get("metafields", {}) - for edge in metafields.get("edges", []): - yield self.transform_object(edge["node"]) - meta_page_info = metafields.get("pageInfo", {}) - meta_has_next_page = meta_page_info.get("hasNextPage", False) - if meta_has_next_page: - for edge in self.get_next_page_metafields( - metafields, query_params, gql_resource - ): - yield self.transform_object(edge["node"]) - - page_info = resource_data.get("pageInfo", {}) - cursor = page_info.get("endCursor") - has_next_page = page_info.get("hasNextPage", False) - - last_updated_at = query_end + # Will always fetch the data from the start date and filter it in the code + # Shopify doesn't support filtering by updated_at for metafields + last_updated_at = utils.strptime_with_tz(Context.config["start_date"]) + + while last_updated_at < sync_start: + date_window_end = last_updated_at + timedelta(days=self.date_window_size) + query_end = min(sync_start, date_window_end) + + has_next_page = True + cursor = None + + while has_next_page: + query_params = self.get_query_params(last_updated_at, query_end, cursor) + with metrics.http_request_timer(self.name): + data = self.call_api(query_params) + + # Process parent objects + for edge in data.get("edges", []): + node = edge.get("node", {}) + + # First handle the already fetched child objects + child_edges = node.get(self.child_data_key).get("edges", []) + for child_obj in child_edges: + obj = self.transform_object(child_obj.get("node")) + yield obj + + # Check if we need to get more child pages + child_page_info = node.get(self.child_data_key, {}).get("pageInfo", {}) + if child_page_info.get("hasNextPage", False): + parent_id = node.get("id") + for child_obj in self.fetch_paginated_child_data( + node.get(self.child_data_key), parent_id + ): + transformed_obj = self.transform_object(child_obj.get("node")) + yield transformed_obj + + page_info = data.get("pageInfo", {}) + cursor, has_next_page = page_info.get("endCursor"), page_info.get("hasNextPage") + + last_updated_at = query_end def sync(self): - """Sync metafields and update bookmarks""" + """ + Performs pseudo incremental sync. + """ start_time = utils.now().replace(microsecond=0) - current_bookmarks = {} + max_bookmark_value = current_bookmark_value = self.get_bookmark() for obj in self.get_objects(): - resource_type = obj["ownerType"].lower() - replication_value = utils.strptime_to_utc(obj["updated_at"]) - current_bookmark_value = self.get_bookmark_by_name(f"{self.name}_{resource_type}") + replication_value = utils.strptime_to_utc(obj[self.replication_key]) - if replication_value >= current_bookmarks.get(resource_type, current_bookmark_value): - current_bookmarks[resource_type] = replication_value + max_bookmark_value = max(max_bookmark_value, replication_value) if replication_value >= current_bookmark_value: yield obj - # Update bookmarks - for res, bookmark_val in current_bookmarks.items(): - bookmark_val = min(start_time, bookmark_val) - self.update_bookmark(utils.strftime(bookmark_val), f"{self.name}_{res}") - -Context.stream_objects['metafields'] = Metafields + # Update bookmark to the latest value, but not beyond sync start time + max_bookmark_value = min(start_time, max_bookmark_value) + self.update_bookmark(utils.strftime(max_bookmark_value)) diff --git a/tap_shopify/streams/metafields_collections.py b/tap_shopify/streams/metafields_collections.py new file mode 100644 index 00000000..1b94cd4e --- /dev/null +++ b/tap_shopify/streams/metafields_collections.py @@ -0,0 +1,58 @@ +"""MetafieldsCollections stream for Shopify tap.""" + +from tap_shopify.context import Context +from tap_shopify.streams.metafields import Metafields + + +class MetafieldsCollections(Metafields): + """Stream class for metafields associated with collections.""" + + name = "metafields_collections" + data_key = "collections" + + def get_query(self): + """Return the GraphQL query for fetching collection metafields.""" + return """ + query getCollectionsMetafields( + $first: Int!, $after: String, $query: String, $childafter: String + ) { + collections(first: $first, after: $after, query: $query) { + edges { + node { + metafields(first: $first, after: $childafter) { + edges { + node { + id + ownerType + value + type + key + createdAt + namespace + description + updatedAt + owner { + ... on Collection { + id + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + id + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + + +Context.stream_objects["metafields_collections"] = MetafieldsCollections diff --git a/tap_shopify/streams/metafields_customers.py b/tap_shopify/streams/metafields_customers.py new file mode 100644 index 00000000..e3290c78 --- /dev/null +++ b/tap_shopify/streams/metafields_customers.py @@ -0,0 +1,57 @@ +"""MetafieldsCustomers stream for Shopify tap.""" + +from tap_shopify.context import Context +from tap_shopify.streams.metafields import Metafields + + +class MetafieldsCustomers(Metafields): + """Stream class for metafields associated with customers.""" + + name = "metafields_customers" + data_key = "customers" + + def get_query(self): + """Return the GraphQL query for fetching customer metafields.""" + return """ + query getCustomerMetafields( + $first: Int!, $after: String, $query: String, $childafter: String + ) { + customers(first: $first, after: $after, query: $query) { + edges { + node { + metafields(first: $first, after: $childafter) { + edges { + node { + id + ownerType + value + type + key + createdAt + namespace + description + updatedAt + owner { + ... on Customer { + id + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + id + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + +Context.stream_objects["metafields_customers"] = MetafieldsCustomers diff --git a/tap_shopify/streams/metafields_orders.py b/tap_shopify/streams/metafields_orders.py new file mode 100644 index 00000000..2c7f0d36 --- /dev/null +++ b/tap_shopify/streams/metafields_orders.py @@ -0,0 +1,57 @@ +"""MetafieldsOrders stream for Shopify tap.""" + +from tap_shopify.context import Context +from tap_shopify.streams.metafields import Metafields + + +class MetafieldsOrders(Metafields): + """Stream class for metafields associated with orders.""" + name = "metafields_orders" + data_key = "orders" + + def get_query(self): + """Returns the GraphQL query for fetching order metafields.""" + return """ + query getOrderMetafields( + $first: Int!, $after: String, $query: String, $childafter: String + ) { + orders(first: $first, after: $after, query: $query) { + edges { + node { + metafields(first: $first, after: $childafter) { + edges { + node { + id + ownerType + value + type + key + createdAt + namespace + description + updatedAt + owner { + ... on Order { + id + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + id + } + } + pageInfo { + endCursor + hasNextPage + } + } + }""" + + +# Register the stream object in the context +Context.stream_objects["metafields_orders"] = MetafieldsOrders diff --git a/tap_shopify/streams/metafields_products.py b/tap_shopify/streams/metafields_products.py new file mode 100644 index 00000000..c4c5e6e3 --- /dev/null +++ b/tap_shopify/streams/metafields_products.py @@ -0,0 +1,57 @@ +"""MetafieldsProducts stream for Shopify tap.""" + +from tap_shopify.context import Context +from tap_shopify.streams.metafields import Metafields + + +class MetafieldsProducts(Metafields): + """Stream class for product metafields.""" + name = "metafields_products" + data_key = "products" + + def get_query(self): + """Return the GraphQL query for product metafields.""" + query = """ + query getProductMetafields( + $first: Int!, $after: String, $query: String, $childafter: String + ) { + products(first: $first, after: $after, query: $query) { + edges { + node { + metafields(first: $first, after: $childafter) { + edges { + node { + id + ownerType + value + type + key + createdAt + namespace + description + updatedAt + owner { + ... on Product { + id + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + id + } + } + pageInfo { + endCursor + hasNextPage + } + } + }""" + return query + + +Context.stream_objects["metafields_products"] = MetafieldsProducts diff --git a/tap_shopify/streams/order_refunds.py b/tap_shopify/streams/order_refunds.py index c6d35718..178e79f5 100644 --- a/tap_shopify/streams/order_refunds.py +++ b/tap_shopify/streams/order_refunds.py @@ -1,64 +1,162 @@ -import shopify -import singer -from singer.utils import strptime_to_utc +from datetime import timedelta +from singer import metrics, utils from tap_shopify.context import Context -from tap_shopify.streams.base import (Stream, - shopify_error_handling, - OutOfOrderIdsError, - canonicalize) +from tap_shopify.streams.base import Stream + class OrderRefunds(Stream): - name = 'order_refunds' - replication_object = shopify.Refund - replication_key = 'created_at' - parent_stream = None - - @shopify_error_handling - def get_refunds(self, parent_object, since_id): - # set timeout - self.replication_object.set_timeout(self.request_timeout) - return self.replication_object.find( - order_id=parent_object.id, - limit=self.results_per_page, - since_id=since_id, - order='id asc') + """Stream class for fetching order refunds from Shopify""" + + name = "order_refunds" + data_key = "orders" + child_data_key = "refunds" + replication_key = "updatedAt" def get_objects(self): - selected_parent = Context.stream_objects['orders']() - selected_parent.name = "refund_orders" - self.parent_stream = selected_parent + """ + Fetch order refund objects within date windows, yielding each refund individually. + + Yields: + dict: Transformed refund object. + """ + # Will always fetch the data from the start date and filter it in the code + # Shopify doesn't support filtering by updated_at for metafields + last_updated_at = utils.strptime_with_tz(Context.config["start_date"]) + sync_start = utils.now().replace(microsecond=0) + + # Process each date window + while last_updated_at < sync_start: + date_window_end = last_updated_at + timedelta(days=self.date_window_size) + query_end = min(sync_start, date_window_end) + cursor = None - # Page through all `orders`, bookmarking at `refund_orders` - for parent_object in selected_parent.get_objects(): - since_id = 1 while True: - refunds = self.get_refunds(parent_object, since_id) - for refund in refunds: - if refund.id < since_id: - raise OutOfOrderIdsError("refund.id < since_id: {} < {}".format( - refund.id, since_id)) - yield refund - if len(refunds) < self.results_per_page: + query_params = self.get_query_params(last_updated_at, query_end, cursor) + + with metrics.http_request_timer(self.name): + data = self.call_api(query_params) + + # Process parent objects and their refunds + edges = data.get("edges", []) + for edge in edges: + node = edge.get("node", {}) + child_edges = node.get(self.child_data_key, []) + + # Yield each transformed refund + for child_obj in child_edges: + yield self.transform_object(child_obj) + + # Handle pagination + page_info = data.get("pageInfo", {}) + cursor = page_info.get("endCursor") + if not page_info.get("hasNextPage", False): break - if refunds[-1].id != max([o.id for o in refunds]): - raise OutOfOrderIdsError("{} is not the max id in refunds ({})".format( - refunds[-1].id, max([o.id for o in refunds]))) - since_id = refunds[-1].id + + last_updated_at = query_end + + def transform_object(self, obj): + """ + Transform refund objects by extracting refund line items from edges. + + Args: + obj (dict): Refund object. + + Returns: + dict: Transformed refund object. + """ + obj["refundLineItems"] = [ + edge.get("node") for edge in obj.get("refundLineItems", {}).get("edges", []) + ] + return obj def sync(self): - bookmark = self.get_bookmark() - for refund in self.get_objects(): - refund_dict = refund.to_dict() - replication_value = strptime_to_utc(refund_dict[self.replication_key]) - if replication_value >= bookmark: - for transaction_dict in refund_dict.get("transactions",[]): - for field_name in ['token', 'version', 'ack', 'timestamp', 'build']: - canonicalize(transaction_dict, field_name) - yield refund_dict - - max_bookmark = singer.get_bookmark( - Context.state, self.parent_stream.name, self.parent_stream.replication_key) - self.update_bookmark(max_bookmark) - - -Context.stream_objects['order_refunds'] = OrderRefunds + """ + Performs pseudo incremental sync. + """ + start_time = utils.now().replace(microsecond=0) + max_bookmark_value = current_bookmark_value = self.get_bookmark() + + for obj in self.get_objects(): + replication_value = utils.strptime_to_utc(obj[self.replication_key]) + + max_bookmark_value = max(max_bookmark_value, replication_value) + + if replication_value >= current_bookmark_value: + yield obj + + # Update bookmark to the latest value, but not beyond sync start time + max_bookmark_value = min(start_time, max_bookmark_value) + self.update_bookmark(utils.strftime(max_bookmark_value)) + + def get_query(self): + """ + Returns the GraphQL query for fetching order refunds. + + Returns: + str: GraphQL query string. + """ + return """query GetOrderRefunds($first: Int!, $after: String, $query: String) { + orders(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { + edges { + node { + refunds(first: 250) { + id + createdAt + legacyResourceId + note + order { + id + } + refundLineItems(first: 250) { + edges { + node { + id + quantity + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + restockType + restocked + subtotalSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalTaxSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + } + } + } + updatedAt + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + }""" + + +Context.stream_objects["order_refunds"] = OrderRefunds diff --git a/tap_shopify/streams/orders.py b/tap_shopify/streams/orders.py index bb360204..7b948cd5 100644 --- a/tap_shopify/streams/orders.py +++ b/tap_shopify/streams/orders.py @@ -1,10 +1,535 @@ -import shopify - from tap_shopify.context import Context from tap_shopify.streams.base import Stream + class Orders(Stream): - name = 'orders' - replication_object = shopify.Order + """Stream class for Shopify Orders.""" + + name = "orders" + data_key = "orders" + replication_key = "updatedAt" + + def get_query(self): + """ + Returns the GraphQL query for fetching orders. + + Returns: + str: GraphQL query string. + """ + return """ + query orders($first: Int!, $after: String, $query: String) { + orders(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { + edges { + node { + additionalFees { + id + name + price { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + taxLines { + channelLiable + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + rate + ratePercentage + source + title + } + } + alerts { + content + dismissibleHandle + icon + severity + title + actions { + primary + show + title + url + } + } + app { + id + name + icon { + id + } + } + billingAddress { + address1 + address2 + city + company + coordinatesValidated + country + countryCodeV2 + firstName + formattedArea + id + lastName + latitude + longitude + name + phone + province + provinceCode + timeZone + validationResultSummary + zip + } + billingAddressMatchesShippingAddress + canMarkAsPaid + canNotifyCustomer + cancelReason + cancellation { + staffNote + } + cancelledAt + capturable + cartDiscountAmountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + channelInformation { + id + channelId + } + clientIp + closed + closedAt + confirmationNumber + confirmed + createdAt + currencyCode + currentCartDiscountAmountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentShippingPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentSubtotalLineItemsQuantity + currentSubtotalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentTaxLines { + channelLiable + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + rate + ratePercentage + source + title + } + currentTotalAdditionalFeesSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentTotalDiscountsSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentTotalDutiesSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentTotalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentTotalTaxSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentTotalWeight + customer { + id + email + firstName + lastName + } + customerAcceptsMarketing + customerLocale + discountCodes + discountCode + displayFinancialStatus + displayFulfillmentStatus + disputes { + id + initiatedAs + status + } + dutiesIncluded + email + edited + estimatedTaxes + fulfillable + fullyPaid + hasTimelineComment + fulfillmentsCount { + count + precision + } + id + legacyResourceId + merchantBusinessEntity { + address { + address1 + address2 + city + countryCode + province + zip + } + companyName + displayName + id + primary + } + name + note + netPaymentSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + originalTotalAdditionalFeesSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + originalTotalDutiesSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + originalTotalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + paymentGatewayNames + phone + poNumber + presentmentCurrencyCode + processedAt + refundable + refundDiscrepancySet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + registeredSourceUrl + requiresShipping + restockable + returnStatus + shippingAddress { + address1 + address2 + city + company + coordinatesValidated + country + countryCodeV2 + firstName + formattedArea + id + lastName + latitude + longitude + name + phone + province + provinceCode + timeZone + validationResultSummary + zip + } + shopifyProtect { + eligibility { + status + } + status + } + sourceIdentifier + sourceName + statusPageUrl + subtotalLineItemsQuantity + subtotalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + tags + taxExempt + taxLines { + channelLiable + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + rate + ratePercentage + source + title + } + taxesIncluded + test + totalCapturableSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalCashRoundingAdjustment { + paymentSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + refundSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + } + totalDiscountsSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalOutstandingSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalReceivedSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalRefundedSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalRefundedShippingSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalShippingPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalTaxSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalTipReceivedSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalWeight + transactionsCount { + count + precision + } + unpaid + updatedAt + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + -Context.stream_objects['orders'] = Orders +Context.stream_objects["orders"] = Orders diff --git a/tap_shopify/streams/product_variants.py b/tap_shopify/streams/product_variants.py index 080bbc40..8760355f 100644 --- a/tap_shopify/streams/product_variants.py +++ b/tap_shopify/streams/product_variants.py @@ -1,34 +1,72 @@ - from tap_shopify.context import Context -from tap_shopify.streams.graphql import get_product_variant_query -from tap_shopify.streams.graphql import ShopifyGqlStream - +from tap_shopify.streams.base import Stream -class ProductVariants(ShopifyGqlStream): - name = 'product_variants' +class ProductVariants(Stream): + """Stream class for Product Variants in Shopify.""" + name = "product_variants" data_key = "productVariants" replication_key = "updatedAt" - def get_query(self): - return get_product_variant_query() + def transform_object(self, obj): + """ + Transforms the object if needed. + + Args: + obj (dict): The object to transform. - # pylint: disable=W0221 - def get_query_params(self, updated_at_min, updated_at_max, cursor=None): + Returns: + dict: The transformed object. """ - Returns query and params for filtering, pagination + return obj + + def get_query(self): """ - filter_key = "updated_at" - params = { - "query": f"{filter_key}:>='{updated_at_min}' AND {filter_key}:<'{updated_at_max}'", - "first": self.results_per_page, - } - if cursor: - params["after"] = cursor - return params + Returns the GraphQL query to get all product variants. + Returns: + str: The GraphQL query string. + """ + return """ + query GetProductVariants($first: Int!, $after: String, $query: String) { + productVariants(first: $first, after: $after, query: $query) { + edges { + node { + id + createdAt + barcode + availableForSale + compareAtPrice + displayName + image { + altText + height + id + url + width + } + inventoryPolicy + inventoryQuantity + position + price + requiresComponents + sellableOnlineQuantity + sku + taxCode + taxable + title + updatedAt + product { id } + inventoryItem { id } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + """ - def transform_object(self, obj): - return obj -Context.stream_objects['product_variants'] = ProductVariants +Context.stream_objects["product_variants"] = ProductVariants diff --git a/tap_shopify/streams/products.py b/tap_shopify/streams/products.py index 07da3d26..e8c564c2 100644 --- a/tap_shopify/streams/products.py +++ b/tap_shopify/streams/products.py @@ -1,36 +1,26 @@ - from tap_shopify.context import Context -from tap_shopify.streams.graphql import get_products_query -from tap_shopify.streams.graphql import ShopifyGqlStream +from tap_shopify.streams.base import Stream +class Products(Stream): + """Stream class for Shopify Products""" -class Products(ShopifyGqlStream): - name = 'products' + name = "products" data_key = "products" replication_key = "updatedAt" - def get_query(self): - return get_products_query() - - # pylint: disable=W0221 - def get_query_params(self, updated_at_min, updated_at_max, cursor=None): - """ - Returns query and params for filtering, pagination + def transform_object(self, obj): """ - filter_key = "updated_at" - params = { - "query": f"{filter_key}:>='{updated_at_min}' AND {filter_key}:<'{updated_at_max}'", - "first": self.results_per_page, - } - if cursor: - params["after"] = cursor - return params + Transforms the product object by extracting media information. + Args: + obj (dict): Product object. - def transform_object(self, obj): + Returns: + dict: Transformed product object. + """ media = obj.get("media") - media_list = [] + media_list = [] if media and "edges" in media: for edge in media.get("edges"): node = edge.get("node") @@ -39,4 +29,129 @@ def transform_object(self, obj): obj["media"] = media_list return obj -Context.stream_objects['products'] = Products + def get_query(self): + """ + Returns the GraphQL query to fetch all products. + + Returns: + str: GraphQL query string. + """ + return """ + query GetProducts($first: Int!, $after: String, $query: String) { + products(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { + edges { + node { + id + title + descriptionHtml + vendor + category { + id + } + tags + handle + publishedAt + createdAt + updatedAt + templateSuffix + status + productType + options { + id + name + position + values + } + giftCardTemplateSuffix + hasOnlyDefaultVariant + hasOutOfStockVariants + hasVariantsThatRequiresComponents + isGiftCard + description + compareAtPriceRange { + maxVariantCompareAtPrice { + amount + currencyCode + } + minVariantCompareAtPrice { + amount + currencyCode + } + } + featuredMedia { + id + mediaContentType + status + } + requiresSellingPlan + totalInventory + tracksInventory + media(first: 250) { + edges { + node { + id + alt + status + mediaContentType + mediaWarnings { + code + message + } + mediaErrors { + code + details + message + } + ... on ExternalVideo { + id + embedUrl + } + ... on MediaImage { + id + updatedAt + createdAt + mimeType + image { + url + width + height + id + } + } + ... on Model3d { + id + filename + sources { + url + format + mimeType + filesize + } + } + ... on Video { + id + updatedAt + createdAt + filename + sources { + url + format + mimeType + fileSize + } + } + } + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + """ + + +Context.stream_objects["products"] = Products diff --git a/tap_shopify/streams/transactions.py b/tap_shopify/streams/transactions.py index 652efaa4..96e11117 100644 --- a/tap_shopify/streams/transactions.py +++ b/tap_shopify/streams/transactions.py @@ -1,85 +1,181 @@ -import shopify -import singer -from singer.utils import strptime_to_utc +from datetime import timedelta +from singer import metrics, utils from tap_shopify.context import Context -from tap_shopify.streams.base import (Stream, - shopify_error_handling, - canonicalize) - -LOGGER = singer.get_logger() - -# https://help.shopify.com/en/api/reference/orders/transaction An -# order can have no more than 100 transactions associated with it. -TRANSACTIONS_RESULTS_PER_PAGE = 100 +from tap_shopify.streams.base import Stream class Transactions(Stream): - name = 'transactions' - replication_key = 'created_at' - replication_object = shopify.Transaction - parent_stream = None - # Added decorator over functions of shopify SDK - replication_object.find = shopify_error_handling(replication_object.find) - # Transactions have no updated_at property. Therefore we have - # nothing to set the `replication_method` member to. - # https://help.shopify.com/en/api/reference/orders/transaction#properties - - def call_api_for_transactions(self, parent_object): - # set timeout - self.replication_object.set_timeout(self.request_timeout) - return self.replication_object.find( - limit=TRANSACTIONS_RESULTS_PER_PAGE, - order_id=parent_object.id, - ) - - def get_transactions(self, parent_object): - # We do not need to support paging on this substream. If that - # were to become untrue, reference Metafields. - # - # We do not user the `transactions` method of the order object - # like in metafield because they overrode it here to not - # support limit overrides. - # - # https://github.com/Shopify/shopify_python_api/blob/e8c475ccc84b1516912b37f691d00ecd24921e9b/shopify/resources/order.py#L17-L18 - - page = self.call_api_for_transactions(parent_object) - yield from page - - while page.has_next_page(): - page = page.next_page() - yield from page + """Stream class for Shopify transactions.""" + + name = "transactions" + data_key = "orders" + child_data_key = "transactions" + replication_key = "createdAt" def get_objects(self): - # Right now, it's ok for the user to select 'transactions' but not - # 'orders'. This data may not be all that useful but we're taking - # the less opinionated approach to begin with to favor simplicity. - # This is where you would need to add the behavior for enforcing - # that 'orders' is selected if we want to go that route in the - # future. - - # Get transactions, bookmarking at `transaction_orders` - selected_parent = Context.stream_objects['orders']() - selected_parent.name = "transaction_orders" - self.parent_stream = selected_parent - - # Page through all `orders`, bookmarking at `transaction_orders` - for parent_object in selected_parent.get_objects(): - transactions = self.get_transactions(parent_object) - for transaction in transactions: - yield transaction + """ + Fetch transaction objects within date windows, yielding each transaction individually. + + Yields: + dict: Transformed transaction object. + """ + # Will always fetch the data from the start date and filter it in the code + # Shopify doesn't support filtering by updated_at for metafields + last_updated_at = utils.strptime_with_tz(Context.config["start_date"]) + sync_start = utils.now().replace(microsecond=0) + + while last_updated_at < sync_start: + date_window_end = last_updated_at + timedelta(days=self.date_window_size) + query_end = min(sync_start, date_window_end) + cursor = None + + while True: + query_params = self.get_query_params(last_updated_at, query_end, cursor) + + with metrics.http_request_timer(self.name): + data = self.call_api(query_params) + + edges = data.get("edges", []) + for edge in edges: + node = edge.get("node", {}) + child_edges = node.get(self.child_data_key, []) + + yield from (self.transform_object(child_obj) for child_obj in child_edges) + + page_info = data.get("pageInfo", {}) + cursor = page_info.get("endCursor") + if not page_info.get("hasNextPage", False): + break + + last_updated_at = query_end def sync(self): - bookmark = self.get_bookmark() - for transaction in self.get_objects(): - transaction_dict = transaction.to_dict() - replication_value = strptime_to_utc(transaction_dict[self.replication_key]) - if replication_value >= bookmark: - for field_name in ['token', 'version', 'ack', 'timestamp', 'build']: - canonicalize(transaction_dict, field_name) - yield transaction_dict - - max_bookmark = singer.get_bookmark( - Context.state, self.parent_stream.name, self.parent_stream.replication_key) - self.update_bookmark(max_bookmark) - -Context.stream_objects['transactions'] = Transactions + """ + Performs pseudo incremental sync. + """ + start_time = utils.now().replace(microsecond=0) + max_bookmark_value = current_bookmark_value = self.get_bookmark() + + for obj in self.get_objects(): + replication_value = utils.strptime_to_utc(obj[self.replication_key]) + + max_bookmark_value = max(max_bookmark_value, replication_value) + + if replication_value >= current_bookmark_value: + yield obj + + # Update bookmark to the latest value, but not beyond sync start time + max_bookmark_value = min(start_time, max_bookmark_value) + self.update_bookmark(utils.strftime(max_bookmark_value)) + + def get_query(self): + """ + Returns query for fetching transactions. + + Note: + Shopify has a limit of 100 transactions per order. + + Returns: + str: GraphQL query string. + """ + return """ + query GetTransactions($first: Int!, $after: String, $query: String) { + orders(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { + edges { + node { + transactions(first: 100) { + accountNumber + amountRoundingSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + amountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + authorizationCode + authorizationExpiresAt + createdAt + errorCode + formattedGateway + gateway + id + kind + manualPaymentGateway + maximumRefundableV2 { + amount + currencyCode + } + multiCapturable + order { + id + } + parentTransaction { + accountNumber + createdAt + id + status + paymentId + processedAt + amountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + } + paymentId + processedAt + receiptJson + settlementCurrency + settlementCurrencyRate + shopifyPaymentsSet { + extendedAuthorizationSet { + extendedAuthorizationExpiresAt + standardAuthorizationExpiresAt + } + refundSet { + acquirerReferenceNumber + } + } + status + test + totalUnsettledSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + + +Context.stream_objects["transactions"] = Transactions diff --git a/tests/base.py b/tests/base.py index 730e8538..43a2db41 100644 --- a/tests/base.py +++ b/tests/base.py @@ -27,7 +27,7 @@ class BaseTapTest(BaseCase): FULL = "FULL_TABLE" START_DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" BOOKMARK_COMPARISON_FORMAT = "%Y-%m-%dT00:00:00+00:00" - DEFAULT_RESULTS_PER_PAGE = 175 + DEFAULT_RESULTS_PER_PAGE = 250 @staticmethod def tap_name(): @@ -45,7 +45,7 @@ def get_properties(self, original: bool = True): return_value = { 'start_date': '2017-07-01T00:00:00Z', 'shop': 'stitchdatawearhouse', - 'date_window_size': 30, + 'date_window_size': 180, # BUG: https://jira.talendforge.org/browse/TDL-13180 # 'results_per_page': '50' } @@ -77,64 +77,49 @@ def expected_metadata(self): """The expected streams and metadata about the streams""" default = { - self.REPLICATION_KEYS: {"updated_at"}, + self.REPLICATION_KEYS: {"updatedAt"}, self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.API_LIMIT: self.DEFAULT_RESULTS_PER_PAGE} meta = default.copy() meta[self.REPLICATION_KEYS] = {"updatedAt"} + meta[self.API_LIMIT] = 30 meta.update({self.FOREIGN_KEYS: {"owner", "ownerType"}}) return { "abandoned_checkouts": { - self.REPLICATION_KEYS: {"updated_at"}, + self.REPLICATION_KEYS: {"updatedAt"}, self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.INCREMENTAL, - # BUG: https://jira.talendforge.org/browse/TDL-13180 + # BUG: https://qlik-dev.atlassian.net/browse/TDL-13180 self.API_LIMIT: 50}, - "collects": default, - "custom_collections": default, + "collections": default, "customers": default, "orders": default, - "order_refunds": { - self.REPLICATION_KEYS: {"created_at"}, - self.PRIMARY_KEYS: {"id"}, - self.REPLICATION_METHOD: self.INCREMENTAL, - self.API_LIMIT: self.DEFAULT_RESULTS_PER_PAGE}, - "products": { - self.REPLICATION_KEYS: {"updatedAt"}, - self.PRIMARY_KEYS: {"id"}, - self.REPLICATION_METHOD: self.INCREMENTAL, - self.API_LIMIT: self.DEFAULT_RESULTS_PER_PAGE}, - "product_variants": { - self.REPLICATION_KEYS: {"updatedAt"}, - self.PRIMARY_KEYS: {"id"}, - self.REPLICATION_METHOD: self.INCREMENTAL, - self.API_LIMIT: self.DEFAULT_RESULTS_PER_PAGE}, - "inventory_items": {self.REPLICATION_KEYS: {"updatedAt"}, - self.PRIMARY_KEYS: {"id"}, - self.REPLICATION_METHOD: self.INCREMENTAL, - self.API_LIMIT: 250}, - "metafields": meta, + "order_refunds": default, + "products": default, + "product_variants": default, + "inventory_items": default, + "metafields_collections": meta, + "metafields_customers": meta, + "metafields_orders": meta, + "metafields_products": meta, "transactions": { - self.REPLICATION_KEYS: {"created_at"}, + self.REPLICATION_KEYS: {"createdAt"}, self.PRIMARY_KEYS: {"id"}, - self.FOREIGN_KEYS: {"order_id"}, + self.FOREIGN_KEYS: {"order"}, self.REPLICATION_METHOD: self.INCREMENTAL, - self.API_LIMIT: self.DEFAULT_RESULTS_PER_PAGE}, + self.API_LIMIT: 150}, "locations": { - self.REPLICATION_KEYS: {"updated_at"}, + self.REPLICATION_KEYS: {"createdAt"}, self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.INCREMENTAL, - self.API_LIMIT: 0}, - "inventory_levels": { - self.REPLICATION_KEYS: {"updated_at"}, - self.PRIMARY_KEYS: {"location_id", "inventory_item_id"}, - self.REPLICATION_METHOD: self.INCREMENTAL, - self.API_LIMIT: self.DEFAULT_RESULTS_PER_PAGE}, + self.API_LIMIT: 2 + }, + "inventory_levels": default, "events": { - self.REPLICATION_KEYS: {"created_at"}, + self.REPLICATION_KEYS: {"createdAt"}, self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.API_LIMIT: 50 @@ -316,13 +301,13 @@ def select_all_streams_and_fields(conn_id, catalogs, select_all_fields: bool = T def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.start_date = self.get_properties().get("start_date") - self.store_1_streams = {'custom_collections', 'orders', 'products', 'customers', 'locations', 'inventory_levels', 'inventory_items', 'events'} + self.store_1_streams = {'collections', 'orders', 'products', 'customers', 'locations', 'inventory_levels', 'inventory_items'} # removed 'abandoned_checkouts' from store 2 streams, as per the Doc: # https://help.shopify.com/en/manual/orders/abandoned-checkouts?st_source=admin&st_campaign=abandoned_checkouts_footer&utm_source=admin&utm_campaign=abandoned_checkouts_footer#review-your-abandoned-checkouts # abandoned checkouts are saved in the Shopify admin for three months. # Every Monday, abandoned checkouts that are older than three months are removed from your admin. # Also no POST call is available for this endpoint: https://shopify.dev/api/admin-rest/2022-01/resources/abandoned-checkouts - self.store_2_streams = {'collects', 'metafields', 'transactions', 'order_refunds', 'products', 'locations', 'inventory_levels', 'inventory_items', 'events'} + self.store_2_streams = {'metafields_products', 'transactions', 'order_refunds', 'products', 'locations', 'inventory_levels', 'inventory_items'} #modified this method to accommodate replication key in the current_state def calculated_states_by_stream(self, current_state): diff --git a/tests/test_all_fields.py b/tests/test_all_fields.py index e3c8814d..4c5b42a7 100644 --- a/tests/test_all_fields.py +++ b/tests/test_all_fields.py @@ -3,6 +3,18 @@ from tap_tester import runner, menagerie from base import BaseTapTest + +KNOWN_MISSING_FIELDS = { + 'events': { + 'attachments', + 'edited', + 'author', + 'canDelete', + 'rawMessage', + 'canEdit', + } +} + class AllFieldsTest(BaseTapTest): @staticmethod @@ -91,15 +103,5 @@ def test_run(self): self.assertTrue(expected_automatic_keys.issubset(expected_all_keys), msg=f'{expected_automatic_keys-expected_all_keys} is not in "expected_all_keys"') - if stream == 'orders': - # No field named 'order_adjustments', 'total_price_usd' present in the 'order' object - # Documentation: https://shopify.dev/api/admin-rest/2021-10/resources/order#resource_object - # https://jira.talendforge.org/browse/TDL-15985 - # total_price_usd showing up in syncd records Sep 2023, still missing from docs - bad_schema_fields = {'order_adjustments'} - # missing data for 'taxExempt' and 'poNumber' in 'orders' stream - # https://jira.talendforge.org/browse/TDL-25173 - missing_fields = {'taxExempt', 'poNumber'} - expected_all_keys = expected_all_keys - bad_schema_fields - missing_fields - + expected_all_keys = expected_all_keys - KNOWN_MISSING_FIELDS.get(stream, set()) self.assertSetEqual(expected_all_keys, actual_all_keys) diff --git a/tests/test_bookmarks.py b/tests/test_bookmarks.py index d7a1f3a7..332a681f 100644 --- a/tests/test_bookmarks.py +++ b/tests/test_bookmarks.py @@ -29,14 +29,7 @@ def max_bookmarks_by_stream(self, sync_records): """ max_bookmarks = {} for stream, batch in sync_records.items(): - # The metafields fetches the fields from `products`, `customers`, `orders` and `custom_collections` - # if the parent streams are selected along with the `shop` fields. - # These different streams have its own bookmark based on its parent. - # Hence filtered out the main records i.e. the `shop` records from all the records. - if stream != 'metafields': - upsert_messages = [m for m in batch.get('messages') if m['action'] == 'upsert'] - else: - upsert_messages = [m for m in batch.get('messages') if m['action'] == 'upsert' and m.get('data').get('owner_resource') == 'shop'] + upsert_messages = [m for m in batch.get('messages') if m['action'] == 'upsert'] stream_bookmark_key = self.expected_replication_keys().get(stream, set()) assert len(stream_bookmark_key) == 1 # There shouldn't be a compound replication key stream_bookmark_key = stream_bookmark_key.pop() @@ -85,7 +78,7 @@ def bookmarks_test(self, conn_id, testable_streams): found_catalogs = menagerie.get_catalogs(conn_id) incremental_streams = {key for key, value in self.expected_replication_method().items() if value == self.INCREMENTAL and key in testable_streams} - incremental_streams = incremental_streams - {'metafields'} # Created a separate test for metafields + incremental_streams = incremental_streams # Our test data sets for Shopify do not have any abandoned_checkouts our_catalogs = [catalog for catalog in found_catalogs if diff --git a/tests/test_bookmarks_metafields.py b/tests/test_bookmarks_metafields.py deleted file mode 100644 index 2d33a6df..00000000 --- a/tests/test_bookmarks_metafields.py +++ /dev/null @@ -1,211 +0,0 @@ -""" -Test tap sets a bookmark and respects it for the next sync of a stream -""" -from datetime import datetime as dt, timezone - -from dateutil.parser import parse - -from tap_tester import menagerie, runner, LOGGER -from base import BaseTapTest - -class BookmarkMetafieldsTest(BaseTapTest): - """Test tap sets a bookmark and respects it for the next sync of a stream""" - @staticmethod - def name(): - return "tap_tester_shopify_bookmark_metafields_test" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.start_date = '2025-01-01T00:00:00Z' - self.metafields_dict = {} - - def min_max_bookmarks_by_stream(self, sync_records): - """ - Return the maximum value for the replication key for each stream - which is the bookmark expected value. - - Comparisons are based on the class of the bookmark value. Dates will be - string compared which works for ISO date-time strings - """ - max_bookmarks = {} - min_bookmarks = {} - for stream, batch in sync_records.items(): - for message in batch.get('messages'): - if message['action'] == 'upsert': - data = message.get('data') - if data.get('ownerType') == "SHOP": - self.metafields_dict.setdefault("metafields_shop", []).append(data) - elif data.get('ownerType') == "ORDER": - self.metafields_dict.setdefault("metafields_order", []).append(data) - elif data.get('ownerType') == "PRODUCT": - self.metafields_dict.setdefault("metafields_product", []).append(data) - elif data.get('ownerType') == "CUSTOMER": - self.metafields_dict.setdefault("metafields_customer", []).append(data) - elif data.get('ownerType') == "COLLECTION": - self.metafields_dict.setdefault("metafields_collection", []).append(data) - - stream_bookmark_key = self.expected_replication_keys().get(stream, set()) - assert len(stream_bookmark_key) == 1 # There shouldn't be a compound replication key - stream_bookmark_key = stream_bookmark_key.pop() - - for metafields_owner, messages in self.metafields_dict.items(): - max_bookmarks, min_bookmarks = self.fetch_max_min(metafields_owner, stream_bookmark_key, messages, max_bookmarks, min_bookmarks) - - max_bookmarks["metafields"] = max_bookmarks - min_bookmarks["metafields"] = min_bookmarks - - return max_bookmarks, min_bookmarks - - - def fetch_max_min(self, owner_type, stream_bookmark_key, messages, max_bookmarks, min_bookmarks): - """ - Updates the maximum and minimum bookmark values for a given owner type based on a list of messages. - - Returns: - tuple: Updated dictionaries for maximum and minimum bookmark values. - """ - bk_values = [message.get(stream_bookmark_key) for message in messages] - max_bookmarks[owner_type] = None - min_bookmarks[owner_type] = None - for bk_value in bk_values: - if bk_value is None: - continue - - if max_bookmarks[owner_type] is None: - max_bookmarks[owner_type] = bk_value - - if bk_value > max_bookmarks[owner_type]: - max_bookmarks[owner_type] = bk_value - - if min_bookmarks[owner_type] is None: - min_bookmarks[owner_type] = bk_value - - if bk_value < min_bookmarks[owner_type]: - min_bookmarks[owner_type] = bk_value - return max_bookmarks, min_bookmarks - - def test_run(self): - """ - Verify that metafields stream you can do a sync which records bookmarks. - That the bookmark is the maximum value sent to the target for the replication key. - That a second sync respects the bookmark - All data of the second sync is > the bookmark from the first sync - The number of records in the 2nd sync is less then the first (This assumes that - new data added to the stream is done at a rate slow enough that you haven't - doubled the amount of data from the start date to the first sync between - the first sync and second sync run in this test) - - Verify that only data for incremental streams is sent to the target - - PREREQUISITE - For metafields stream that is incrementally replicated there are multiple rows of data with - different values for the replication key - """ - conn_id = self.create_connection(original_credentials=True) - - # Select all streams and no fields within streams - found_catalogs = menagerie.get_catalogs(conn_id) - incremental_stream = {'metafields'} - - # Our test data sets for Shopify do not have any abandoned_checkouts - our_catalogs = [catalog for catalog in found_catalogs if - catalog.get('tap_stream_id') in incremental_stream] - self.select_all_streams_and_fields(conn_id, our_catalogs, select_all_fields=True) - - # Run a sync job using orchestrator - first_sync_record_count = self.run_sync(conn_id) - - # verify that the sync only sent records to the target for selected streams (catalogs) - self.assertEqual(set(first_sync_record_count.keys()), - incremental_stream) - - first_sync_state = menagerie.get_state(conn_id) - - # Get data about actual rows synced - first_sync_records = runner.get_records_from_target_output() - - # Run a second sync job using orchestrator - second_sync_record_count = self.run_sync(conn_id) - - # Get data about rows synced - second_sync_records = runner.get_records_from_target_output() - - first_max_bookmarks, first_min_bookmarks = self.min_max_bookmarks_by_stream(first_sync_records) - - _ , second_min_bookmarks = self.min_max_bookmarks_by_stream(second_sync_records) - - for stream in incremental_stream: - with self.subTest(stream=stream): - - # Verify that you get less data the 2nd time around - self.assertGreater( - first_sync_record_count.get(stream, 0), - second_sync_record_count.get(stream, 0), - msg="second sync didn't have less records, bookmark usage not verified") - - # Get bookmark values from state and target data - stream_bookmark_key = self.expected_replication_keys().get(stream, set()) - assert len( - stream_bookmark_key) == 1 # There shouldn't be a compound replication key - stream_bookmark_key = stream_bookmark_key.pop() - - for metafield_key in self.metafields_dict.keys(): - - state_value = first_sync_state.get("bookmarks", {}).get( - stream, {None: None}).get(metafield_key) - target_value = first_max_bookmarks.get( - stream, {None: None}).get(metafield_key) - target_min_value = first_min_bookmarks.get( - stream, {None: None}).get(metafield_key) - - try: - # Attempt to parse the bookmark as a date - if state_value: - if isinstance(state_value, str): - state_value = self.local_to_utc(parse(state_value)) - if isinstance(state_value, int): - state_value = self.local_to_utc(dt.fromtimestamp(state_value, tz=timezone.utc)) - - if target_value: - if isinstance(target_value, str): - target_value = self.local_to_utc(parse(target_value)) - if isinstance(target_value, int): - target_value = self.local_to_utc(dt.fromtimestamp(target_value, tz=timezone.utc)) - - if target_min_value: - if isinstance(target_min_value, str): - target_min_value = self.local_to_utc(parse(target_min_value)) - if isinstance(target_min_value, int): - target_min_value = self.local_to_utc( - dt.fromtimestamp(target_min_value, tz=timezone.utc)) - - except (OverflowError, ValueError, TypeError): - LOGGER.warn("Bookmarks cannot be converted to dates, comparing values directly") - - # Verify that there is data with different bookmark values - setup necessary - self.assertGreaterEqual(target_value, target_min_value, - msg="Data isn't set up to be able to test bookmarks") - - # Verify state agrees with target data after 1st sync - self.assertGreaterEqual(state_value, target_value, - msg="The bookmark value isn't correct based on target data") - - # Verify all data from 2nd sync >= 1st bookmark - target_value = second_min_bookmarks.get( - stream, {None: None}).get(metafield_key) - try: - if target_value: - if isinstance(target_value, str): - target_value = self.local_to_utc(parse(target_value)) - if isinstance(target_value, int): - target_value = self.local_to_utc(dt.fromtimestamp(target_value, tz=timezone.utc)) - - except (OverflowError, ValueError, TypeError): - LOGGER.warn("bookmarks cannot be converted to dates, comparing values directly") - - for message in second_sync_records.get(stream).get('messages'): - if message['action'] == 'upsert': - if message.get('data').get('ownerType') == metafield_key: - self.assertGreaterEqual( - message.get('data').get(stream_bookmark_key), target_value, - msg="Data from 2nd sync is not all >= bookmark from 1st sync") diff --git a/tests/test_bookmarks_updated.py b/tests/test_bookmarks_updated.py index f7d8c93f..fde27ec1 100644 --- a/tests/test_bookmarks_updated.py +++ b/tests/test_bookmarks_updated.py @@ -31,7 +31,7 @@ def __init__(self, *args, **kwargs): # abandoned checkouts are saved in the Shopify admin for three months. # Every Monday, abandoned checkouts that are older than three months are removed from your admin. # Also no POST call is available for this endpoint: https://shopify.dev/api/admin-rest/2022-01/resources/abandoned-checkouts - store_2_streams = {'collects', 'metafields', 'transactions', 'order_refunds', 'products', 'locations', 'inventory_items', 'events', 'customers', 'custom_collections', 'orders'} + store_2_streams = {'metafields_products', 'transactions', 'order_refunds', 'products', 'locations', 'inventory_items', 'customers', 'collections', 'orders'} def test_run_store_2(self): with self.subTest(store="store_2"): @@ -77,7 +77,20 @@ def bookmarks_test(self, conn_id, testable_streams): #simulated_states = self.calculated_states_by_stream(first_sync_bookmark) # We are hardcoding the updated state to ensure that we get atleast 1 record in second sync. These values have been provided after reviewing the max bookmark value for each of the streams - simulated_states = {'products': {'updatedAt': '2024-09-14T03:01:11.000000Z'}, 'collects': {'updated_at': '2021-09-01T09:08:28.000000Z'}, 'abandoned_checkouts': {'updated_at': '2022-02-02T16:00:00.000000Z'}, 'inventory_levels': {'updated_at': '2021-12-20T05:09:34.000000Z'}, 'locations': {'updated_at': '2021-07-20T09:00:22.000000Z'}, 'events': {'created_at': '2021-12-20T05:09:01.000000Z'}, 'inventory_items': {'updatedAt': '2021-09-15T19:44:11.000000Z'}, 'transactions': {'created_at': '2021-12-20T00:08:52-05:00'}, 'metafields': {'metafields_customer': '2025-01-21T13:28:24.000000Z'}, 'order_refunds': {'created_at': '2021-05-01T17:41:18.000000Z'}, 'customers': {'updated_at': '2021-12-20T05:08:17.000000Z'}, 'orders': {'updated_at': '2021-12-20T05:09:01.000000Z'}, 'custom_collections': {'updated_at': '2024-12-13T08:42:56.000000Z'}} + simulated_states = { + 'products': {'updatedAt': '2025-01-23T14:08:21.000000Z'}, + 'abandoned_checkouts': {'updatedAt': '2025-01-20T06:56:01.000000Z'}, + 'inventory_levels': {'updatedAt': '2024-12-05T09:26:47.000000Z'}, + 'locations': {'createdAt': '2021-07-29T08:38:44.000000Z'}, + 'inventory_items': {'updatedAt': '2021-09-15T19:44:11.000000Z'}, + 'transactions': {'createdAt': '2024-12-05T09:58:40.000000Z'}, + 'metafields_customer': '2025-01-21T13:28:24.000000Z', + 'order_refunds': {'updatedAt': '2024-12-05T09:58:40.000000Z'}, + 'customers': {'updatedAt': '2025-01-19T20:55:07.000000Z'}, + 'orders': {'updatedAt': '2024-12-05T09:59:35.000000Z'}, + 'collections': {'updatedAt': '2025-01-21T13:29:06.000000Z'}, + 'metafields_products': {'updatedAt': '2025-01-21T03:11:54.000000Z'} + } for stream, updated_state in simulated_states.items(): new_state['bookmarks'][stream] = updated_state @@ -101,23 +114,11 @@ def bookmarks_test(self, conn_id, testable_streams): first_sync_count = first_sync_record_count.get(stream, 0) second_sync_count = second_sync_record_count.get(stream, 0) - # The metafields fetches the fields from `products`, `customers`, `orders` and `custom_collections` - # if the parent streams are selected along with the `shop` fields. - # These different streams have its own bookmark based on its parent. - # Hence filtered out the main records i.e. the `shop` records from all the records. - if stream != 'metafields': - first_sync_messages = [record.get('data') for record in first_sync_records.get(stream, {}).get('messages', []) - if record.get('action') == 'upsert'] - else: - first_sync_messages = [record.get('data') for record in first_sync_records.get(stream, {}).get('messages', []) - if record.get('action') == 'upsert' and record.get('data').get('owner_resource') == 'shop'] - - if stream != 'metafields': - second_sync_messages = [record.get('data') for record in second_sync_records.get(stream, {}).get('messages', []) - if record.get('action') == 'upsert'] - else: - second_sync_messages = [record.get('data') for record in second_sync_records.get(stream, {}).get('messages', []) - if record.get('action') == 'upsert' and record.get('data').get('owner_resource') == 'shop'] + first_sync_messages = [record.get('data') for record in first_sync_records.get(stream, {}).get('messages', []) + if record.get('action') == 'upsert'] + + second_sync_messages = [record.get('data') for record in second_sync_records.get(stream, {}).get('messages', []) + if record.get('action') == 'upsert'] first_bookmark_value = first_sync_bookmark.get('bookmarks', {stream: None}).get(stream) first_bookmark_value = list(first_bookmark_value.values())[0] @@ -153,11 +154,8 @@ def bookmarks_test(self, conn_id, testable_streams): self.assertLessEqual(replication_key_value, second_bookmark_value_utc, msg="Second sync bookmark was set incorrectly, a record with a greater replication key value was synced") # verify that we get less data in the 2nd sync - # collects has all the records with the same value of replication key, so we are removing from this assertion - if stream not in ('collects'): - self.assertLess(second_sync_count, first_sync_count, - msg="Second sync does not have less records, bookmark usage not verified") + self.assertLess(second_sync_count, first_sync_count, + msg="Second sync does not have less records, bookmark usage not verified") # verify that we get atleast 1 record in the second sync - if stream not in ('collects'): - self.assertGreater(second_sync_count, 0, msg="Second sync did not yield any records") + self.assertGreater(second_sync_count, 0, msg="Second sync did not yield any records") diff --git a/tests/test_interrupted_sync.py b/tests/test_interrupted_sync.py index 70a21a07..aa1ec44f 100644 --- a/tests/test_interrupted_sync.py +++ b/tests/test_interrupted_sync.py @@ -37,7 +37,7 @@ def test_run(self): conn_id = self.create_connection(original_properties=False, original_credentials=False) expected_streams = {'customers', - 'events', + 'collections', 'orders', 'products', 'transactions'} @@ -89,7 +89,6 @@ def test_run(self): base_state = {'bookmarks': {'currently_sync_stream': currently_syncing_stream, 'customers': first_sync_state.get('bookmarks').get('customers'), - 'events': first_sync_state.get('bookmarks').get('events'), 'orders': first_sync_state.get('bookmarks').get('orders'), 'products': first_sync_state.get('bookmarks').get('products'), 'transactions': first_sync_state.get('bookmarks').get('transactions') @@ -144,34 +143,15 @@ def test_run(self): first_sync_count = first_sync_record_count.get(stream, 0) resuming_sync_count = resuming_sync_record_count.get(stream, 0) - # The metafields fetches the fields from `products`, `customers`, `orders` and - # `custom_collections` if the parent streams are selected along with the `shop` - # fields. These different streams have their own bookmark based on the parent. - # Hence filtered out the main records i.e. the `shop` records from all the records. - - if stream != 'metafields': - first_sync_messages = [ - record.get('data') for record - in first_sync_records.get(stream, {}).get('messages', []) - if record.get('action') == 'upsert'] - - resuming_sync_messages = [ - record.get('data') for record - in resuming_sync_records.get(stream, {}).get('messages', []) - if record.get('action') == 'upsert'] - - else: - first_sync_messages = [ - record.get('data') for record - in first_sync_records.get(stream, {}).get('messages', []) - if record.get('action') == 'upsert' - and record.get('data', {}).get('owner_resource') == 'shop'] - - resuming_sync_messages = [ - record.get('data') for record - in resuming_sync_records.get(stream, {}).get('messages', []) - if record.get('action') == 'upsert' - and record.get('data', {}).get('owner_resource') == 'shop'] + first_sync_messages = [ + record.get('data') for record + in first_sync_records.get(stream, {}).get('messages', []) + if record.get('action') == 'upsert'] + + resuming_sync_messages = [ + record.get('data') for record + in resuming_sync_records.get(stream, {}).get('messages', []) + if record.get('action') == 'upsert'] replication_key = next(iter(expected_replication_keys[stream])) first_bookmark_stream = first_sync_state.get('bookmarks', {}).get(stream, {}) diff --git a/tests/unittests/test_backoff_on_incompleteread_error.py b/tests/unittests/test_backoff_on_incompleteread_error.py deleted file mode 100644 index ad3d2f5b..00000000 --- a/tests/unittests/test_backoff_on_incompleteread_error.py +++ /dev/null @@ -1,24 +0,0 @@ -from unittest import mock -from tap_shopify.streams.base import http -from tap_shopify.streams.transactions import Transactions -import unittest - -class TestIncompleteReadBackoff(unittest.TestCase): - @mock.patch("time.sleep") - @mock.patch("pyactiveresource.activeresource.ActiveResource.find") - def test_Transactions_pyactiveresource_error_incomplete_read_error_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'http.client.IncompleteRead' error occurs - """ - # mock 'find' and raise IncompleteRead error - mocked_find.side_effect = http.client.IncompleteRead(b'') - # initialize class - locations = Transactions() - try: - # function call - locations.replication_object.find() - except http.client.IncompleteRead: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) \ No newline at end of file diff --git a/tests/unittests/test_graphql_base.py b/tests/unittests/test_base.py similarity index 82% rename from tests/unittests/test_graphql_base.py rename to tests/unittests/test_base.py index 54de74cf..12fe4842 100644 --- a/tests/unittests/test_graphql_base.py +++ b/tests/unittests/test_base.py @@ -4,13 +4,13 @@ from datetime import datetime from dateutil.tz import tzlocal from itertools import cycle -from tap_shopify.streams.graphql.gql_base import ShopifyGqlStream, ShopifyAPIError +from tap_shopify.streams.base import Stream, ShopifyAPIError from tap_shopify.context import Context -class TestShopifyGqlStream(unittest.TestCase): +class TestStream(unittest.TestCase): def setUp(self): - self.stream = ShopifyGqlStream() + self.stream = Stream() self.stream.data_key = "products" # Mock the Context.config to include start_date self.original_config = Context.config @@ -24,7 +24,7 @@ def tearDown(self): Context.config = self.original_config @patch('shopify.GraphQL') - @patch.object(ShopifyGqlStream, 'get_query', return_value='mocked_query') + @patch.object(Stream, 'get_query', return_value='mocked_query') def test_call_api_success(self, mock_get_query, mock_graphql): """Test successful GraphQL query execution.""" # Mock the response from Shopify GraphQL API @@ -44,7 +44,7 @@ def test_call_api_success(self, mock_get_query, mock_graphql): self.assertEqual(result, mock_response["data"]["products"]) @patch('shopify.GraphQL') - @patch.object(ShopifyGqlStream, 'get_query', return_value='mocked_query') + @patch.object(Stream, 'get_query', return_value='mocked_query') def test_call_api_error(self, mock_get_query, mock_graphql): """Test GraphQL query execution with an error.""" # Mock an error response from Shopify GraphQL API @@ -56,7 +56,7 @@ def test_call_api_error(self, mock_get_query, mock_graphql): self.stream.call_api(query_params) @patch('shopify.GraphQL') - @patch.object(ShopifyGqlStream, 'get_query', return_value='mocked_query') + @patch.object(Stream, 'get_query', return_value='mocked_query') def test_call_api_empty_response(self, mock_get_query, mock_graphql): """Test GraphQL query execution with an empty response.""" # Mock an empty response from Shopify GraphQL API @@ -69,16 +69,16 @@ def test_call_api_empty_response(self, mock_get_query, mock_graphql): self.assertEqual(result, {}) @patch('shopify.GraphQL') - @patch.object(ShopifyGqlStream, 'get_query', return_value='mocked_query') - @patch.object(ShopifyGqlStream, 'transform_object', side_effect=lambda x: x) - @patch('tap_shopify.streams.graphql.gql_base.utils.now', return_value=datetime(2025, 2, 1, 0, 0, tzinfo=tzlocal())) + @patch.object(Stream, 'get_query', return_value='mocked_query') + @patch.object(Stream, 'transform_object', side_effect=lambda x: x) + @patch('tap_shopify.streams.base.utils.now', return_value=datetime(2025, 2, 1, 0, 0, tzinfo=tzlocal())) def test_get_objects(self, mock_now, mock_transform_object, mock_get_query, mock_graphql): """Test get_objects with pagination and bookmarking.""" # Mock the response from Shopify GraphQL API mock_response_page_1 = { "data": { "products": { - "edges": [{"node": {"id": "mocked_id_1", "updated_at": "2025-01-01T00:00:00Z"}}], + "edges": [{"node": {"id": "mocked_id_1", "updatedAt": "2025-01-01T00:00:00Z"}}], "pageInfo": {"endCursor": "cursor_123", "hasNextPage": True}, } } @@ -86,7 +86,7 @@ def test_get_objects(self, mock_now, mock_transform_object, mock_get_query, mock mock_response_page_2 = { "data": { "products": { - "edges": [{"node": {"id": "mocked_id_2", "updated_at": "2025-01-01T00:00:00Z"}}], + "edges": [{"node": {"id": "mocked_id_2", "updatedAt": "2025-01-01T00:00:00Z"}}], "pageInfo": {"endCursor": "cursor_456", "hasNextPage": False}, } } @@ -102,5 +102,5 @@ def test_get_objects(self, mock_now, mock_transform_object, mock_get_query, mock objects = list(self.stream.get_objects()) self.assertEqual(len(objects), 4) - self.assertEqual(objects[0], {"id": "mocked_id_1", "updated_at": "2025-01-01T00:00:00Z"}) - self.assertEqual(objects[1], {"id": "mocked_id_2", "updated_at": "2025-01-01T00:00:00Z"}) + self.assertEqual(objects[0], {"id": "mocked_id_1", "updatedAt": "2025-01-01T00:00:00Z"}) + self.assertEqual(objects[1], {"id": "mocked_id_2", "updatedAt": "2025-01-01T00:00:00Z"}) diff --git a/tests/unittests/test_error_handling.py b/tests/unittests/test_error_handling.py index b202909c..57f901bd 100644 --- a/tests/unittests/test_error_handling.py +++ b/tests/unittests/test_error_handling.py @@ -1,45 +1,49 @@ import unittest -from unittest import mock +import http.client +import socket +from parameterized import parameterized +import pyactiveresource +import simplejson from urllib.error import URLError -import tap_shopify -from tap_shopify.streams.transactions import Transactions -from tap_shopify.streams.orders import Orders -from tap_shopify.context import Context - -class TestShopifyConnectionResetErrorHandling(unittest.TestCase): - - @mock.patch("time.sleep") - @mock.patch("pyactiveresource.activeresource.ActiveResource.find") - def test_check_access_handle_timeout_error(self, mocked_find, mocked_sleep): - ''' - Test retry handling of URLError - ''' - - # mock 'find' and raise URLError - mocked_find.side_effect = URLError('') - - # initialize class - stream = Orders() - - with self.assertRaises(URLError): - stream.call_api({}) - - self.assertEqual(mocked_find.call_count, 5) - - @mock.patch("time.sleep") - @mock.patch("pyactiveresource.activeresource.ActiveResource.find") - def test_check_access_handle_connection_reset_error(self, mocked_find, mocked_sleep): - ''' - Test retry handling of ConnectionResetError - ''' - - # mock 'find' and raise IncompleteRead error - mocked_find.side_effect = ConnectionResetError - - # initialize class - stream = Transactions() - - with self.assertRaises(ConnectionResetError): - stream.replication_object.find() - - self.assertEqual(mocked_find.call_count, 5) +from unittest.mock import patch, MagicMock +import itertools +from tap_shopify.streams.products import Products +from tap_shopify.streams.base import ShopifyAPIError + +class MockResponse: + def __init__(self, msg, url, code): + self.msg = msg + self.url = url + self.code = code + +class TestShopifyErrorHandling(unittest.TestCase): + + @parameterized.expand([ + ["http_incompleteread_error", lambda: http.client.IncompleteRead(10), http.client.IncompleteRead], + ["connection_reset_error", lambda: ConnectionResetError("Connection reset by peer"), ConnectionResetError], + ["shopify_api_error", lambda: ShopifyAPIError("Shopify API error"), ShopifyAPIError], + ["pyactiveresource_connection_error", lambda: pyactiveresource.connection.Error("Resource connection error with timed out"), pyactiveresource.connection.Error], + ["socket_timeout_error", lambda: socket.timeout("The read operation timed out"), socket.timeout], + ["server_error", lambda: pyactiveresource.connection.ServerError(MockResponse("Server error", "https://shopify.com", 500)), pyactiveresource.connection.ServerError], + ["formats_error", lambda: pyactiveresource.formats.Error("Format error"), pyactiveresource.formats.Error], + ["json_decode_error", lambda: simplejson.scanner.JSONDecodeError("JSON decode error", "doc", 0), simplejson.scanner.JSONDecodeError], + ["url_error", lambda: URLError("URL error"), URLError] + ]) + @patch("shopify.GraphQL") + def test_api_errors(self, name, error_fn, expected_exception, mock_graphql): + """Test handling of different API errors""" + + side_effect = error_fn() # Call the lambda to create the exception instance + + with self.subTest(name=name, side_effect=side_effect, expected_exception=expected_exception): + # Mock GraphQL execute method to simulate errors + mock_graphql_instance = mock_graphql.return_value + mock_graphql_instance.execute = MagicMock(side_effect=itertools.repeat(side_effect, 5)) + + obj = Products() + obj.data_key = "products" + + with self.assertRaises(expected_exception): + obj.call_api(query_params={}) + + self.assertEqual(mock_graphql_instance.execute.call_count, 5) diff --git a/tests/unittests/test_inventory_levels.py b/tests/unittests/test_inventory_levels.py deleted file mode 100644 index cf2992a8..00000000 --- a/tests/unittests/test_inventory_levels.py +++ /dev/null @@ -1,76 +0,0 @@ -import unittest -from unittest import mock -from singer.utils import strptime_to_utc -from tap_shopify.context import Context - -INVENTORY_LEVEL_OBJECT = Context.stream_objects['inventory_levels']() - -class Location(): - def __init__(self, id): - self.id = id - -class InventoryLevels(): - def __init__(self, id, updated_at): - self.id = id - self.updated_at = updated_at - - def to_dict(self): - return {"id": self.id, "updated_at": self.updated_at} - -LEVEL_1 = InventoryLevels("inv_level1", "2021-08-11T01:57:05-04:00") -LEVEL_2 = InventoryLevels("inv_level2", "2021-08-12T01:57:05-04:00") -LEVEL_3 = InventoryLevels("inv_level3", "2021-08-13T01:57:05-04:00") -LEVEL_4 = InventoryLevels("inv_level4", "2021-08-14T01:57:05-04:00") - -@mock.patch("tap_shopify.streams.base.Stream.get_bookmark") -class TestInventoryItems(unittest.TestCase): - - @mock.patch("tap_shopify.streams.locations.Locations.get_locations_data") - @mock.patch("tap_shopify.streams.inventory_levels.InventoryLevels.get_inventory_levels") - def test_get_objects_with_locations(self, mock_get_inventory_levels, mock_parent_object, mock_get_bookmark): - ''' - Verify that expected data should be emitted for inventory_levels if locations found. - ''' - expected_inventory_levels = [LEVEL_1, LEVEL_2, LEVEL_3, LEVEL_4] - location1 = Location("location1") - location2 = Location("location2") - - mock_get_inventory_levels.side_effect = [[LEVEL_1, LEVEL_2], [LEVEL_3, LEVEL_4]] - mock_parent_object.return_value = [location1, location2] - - actual_inventory_levels = list(INVENTORY_LEVEL_OBJECT.get_objects()) - - #Verify that it returns inventory_levels for all locations - self.assertEqual(actual_inventory_levels, expected_inventory_levels) - - @mock.patch("tap_shopify.streams.locations.Locations.get_locations_data") - @mock.patch("tap_shopify.streams.inventory_levels.InventoryLevels.get_inventory_levels") - def test_get_objects_with_no_locations(self, mock_get_inventory_levels, mock_parent_object, mock_get_bookmark): - ''' - Verify that no data should be emitted for inventory_levels if no locations found. - ''' - # No data for parent stream location - mock_parent_object.return_value = [] - expected_inventory_levels = [] - - actual_inventory_levels = list(INVENTORY_LEVEL_OBJECT.get_objects()) - - # No get_inventory_levels should be called and no data should be returned - self.assertEqual(actual_inventory_levels, expected_inventory_levels) - self.assertEqual(mock_get_inventory_levels.call_count, 0) - - @mock.patch("tap_shopify.streams.inventory_levels.InventoryLevels.get_objects") - def test_sync(self, mock_get_objects, mock_get_bookmark): - ''' - Verify that only data updated after specific bookmark are yielded from sync. - ''' - - expected_sync = [LEVEL_3.to_dict(), LEVEL_4.to_dict()] - mock_get_objects.return_value = [LEVEL_1, LEVEL_2, LEVEL_3, LEVEL_4] - - mock_get_bookmark.return_value = strptime_to_utc("2021-08-13T01:05:05-04:00") - - actual_sync = list(INVENTORY_LEVEL_OBJECT.sync()) - - #Verify that only 2 record syncs - self.assertEqual(actual_sync, expected_sync) diff --git a/tests/unittests/test_locations.py b/tests/unittests/test_locations.py deleted file mode 100644 index 6f398a4f..00000000 --- a/tests/unittests/test_locations.py +++ /dev/null @@ -1,43 +0,0 @@ -import unittest -from unittest import mock -from singer import utils -from singer.utils import strptime_to_utc, strftime -from tap_shopify.context import Context - -LOCATIONS_OBJECT = Context.stream_objects['locations']() - - -class Locations(): - def __init__(self, id, updated_at): - self.id = id - self.updated_at = updated_at - - def to_dict(self): - return {"id": self.id, "updated_at": self.updated_at} - - -LOCATION_1 = Locations("i11", "2021-08-11T01:57:05-04:00") -LOCATION_2 = Locations("i12", "2021-08-12T01:57:05-04:00") -LOCATION_3 = Locations("i21", "2021-08-13T01:57:05-04:00") -LOCATION_4 = Locations("i22", "2021-08-14T01:57:05-04:00") - - -class TestLocations(unittest.TestCase): - @mock.patch("tap_shopify.streams.base.Stream.update_bookmark") - @mock.patch("tap_shopify.streams.base.Stream.get_bookmark") - @mock.patch("tap_shopify.streams.locations.Locations.get_locations_data") - def test_sync(self, mock_get_locations_data, mock_get_bookmark, mock_update_bookmark): - - expected_sync = [LOCATION_3.to_dict(), LOCATION_4.to_dict()] - mock_get_locations_data.return_value = [LOCATION_1, LOCATION_2, LOCATION_3, LOCATION_4] - - mock_get_bookmark.return_value = strptime_to_utc("2021-08-13T01:05:05-04:00") - - actual_sync = list(LOCATIONS_OBJECT.sync()) - - # Verify that only 2 record syncs - self.assertEqual(actual_sync, expected_sync) - max_bookmark = strptime_to_utc("2021-08-14T01:57:05-04:00") - - # Verify that maximum replication key of all keys is updated as bookmark - mock_update_bookmark.assert_called_with(utils.strftime(max_bookmark)) diff --git a/tests/unittests/test_timeout.py b/tests/unittests/test_timeout.py deleted file mode 100644 index dfe09ee9..00000000 --- a/tests/unittests/test_timeout.py +++ /dev/null @@ -1,503 +0,0 @@ -import socket -import tap_shopify -from unittest import mock -import pyactiveresource -import shopify -from tap_shopify.context import Context -from tap_shopify.streams.base import get_request_timeout -from tap_shopify.streams.inventory_items import InventoryItems -from tap_shopify.streams.inventory_levels import InventoryLevels -from tap_shopify.streams.locations import Locations -from tap_shopify.streams.order_refunds import OrderRefunds -from tap_shopify.streams.transactions import Transactions -from tap_shopify.streams.abandoned_checkouts import AbandonedCheckouts -from tap_shopify.streams.metafields import Metafields -import unittest - -class TestTimeoutValue(unittest.TestCase): - """ - Verify the timeout value is set as expected by the tap - """ - - def test_timeout_value_not_passed_in_config(self): - """ - Test case to verify that the default value is used when we do not pass request timeout value from config - """ - # initialize config - Context.config = { - "start_date": "2021-01-01", - "api_key": "test_api_key", - "shop": "test_shop", - "results_per_page": 50 - } - - # initialize base class - timeout = get_request_timeout() - # verify the timeout is set as expected - self.assertEquals(timeout, 300) - - def test_timeout_int_value_passed_in_config(self): - """ - Test case to verify that the value we passed on config is set as request timeout value - """ - # initialize config - Context.config = { - "start_date": "2021-01-01", - "api_key": "test_api_key", - "shop": "test_shop", - "results_per_page": 50, - "request_timeout": 100 - } - - # initialize base class - timeout = get_request_timeout() - # verify the timeout is set as expected - self.assertEquals(timeout, 100) - - def test_timeout_string_value_passed_in_config(self): - """ - Test case to verify that the value we passed on config is set as request timeout value - """ - # initialize config - Context.config = { - "start_date": "2021-01-01", - "api_key": "test_api_key", - "shop": "test_shop", - "results_per_page": 50, - "request_timeout": "100" - } - - # initialize base class - timeout = get_request_timeout() - # verify the timeout is set as expected - self.assertEquals(timeout, 100) - - def test_timeout_empty_value_passed_in_config(self): - """ - Test case to verify that the default value is used when pass empty request timeout value from config - """ - # initialize config - Context.config = { - "start_date": "2021-01-01", - "api_key": "test_api_key", - "shop": "test_shop", - "results_per_page": 50, - "request_timeout": "" - } - - # initialize base class - timeout = get_request_timeout() - # verify the timeout is set as expected - self.assertEquals(timeout, 300) - - def test_timeout_0_value_passed_in_config(self): - """ - Test case to verify that the default value is used when pass 0 as request timeout value from config - """ - # initialize config - Context.config = { - "start_date": "2021-01-01", - "api_key": "test_api_key", - "shop": "test_shop", - "results_per_page": 50, - "request_timeout": 0.0 - } - - # initialize base class - timeout = get_request_timeout() - # verify the timeout is set as expected - self.assertEquals(timeout, 300) - - def test_timeout_string_0_value_passed_in_config(self): - """ - Test case to verify that the default value is used when pass string 0 as request timeout value from config - """ - # initialize config - Context.config = { - "start_date": "2021-01-01", - "api_key": "test_api_key", - "shop": "test_shop", - "results_per_page": 50, - "request_timeout": "0.0" - } - - # initialize base class - timeout = get_request_timeout() - # verify the timeout is set as expected - self.assertEquals(timeout, 300) - - @mock.patch("shopify.Shop.set_timeout") - @mock.patch("shopify.Shop.current") - def test_timeout_value_not_passed_in_config__initialize_shopify_client(self, mocked_current, mocked_set_timeout): - """ - Test case to verify that the default value is used when we do not pass request timeout value from config - """ - # initialize config - Context.config = { - "start_date": "2021-01-01", - "api_key": "test_api_key", - "shop": "test_shop", - "results_per_page": 50 - } - - # function call - tap_shopify.initialize_shopify_client() - # verify the timeout is set as expected - mocked_set_timeout.assert_called_with(300) - - @mock.patch("shopify.Shop.set_timeout") - @mock.patch("shopify.Shop.current") - def test_timeout_int_value_passed_in_config__initialize_shopify_client(self, mocked_current, mocked_set_timeout): - """ - Test case to verify that the value we passed on config is set as request timeout value - """ - # initialize config - Context.config = { - "start_date": "2021-01-01", - "api_key": "test_api_key", - "shop": "test_shop", - "results_per_page": 50, - "request_timeout": 100 - } - - # function call - tap_shopify.initialize_shopify_client() - # verify the timeout is set as expected - mocked_set_timeout.assert_called_with(100) - - @mock.patch("shopify.Shop.set_timeout") - @mock.patch("shopify.Shop.current") - def test_timeout_string_value_passed_in_config__initialize_shopify_client(self, mocked_current, mocked_set_timeout): - """ - Test case to verify that the value we passed on config is set as request timeout value - """ - # initialize config - Context.config = { - "start_date": "2021-01-01", - "api_key": "test_api_key", - "shop": "test_shop", - "results_per_page": 50, - "request_timeout": "100" - } - - # function call - tap_shopify.initialize_shopify_client() - # verify the timeout is set as expected - mocked_set_timeout.assert_called_with(100) - - @mock.patch("shopify.Shop.set_timeout") - @mock.patch("shopify.Shop.current") - def test_timeout_empty_value_passed_in_config__initialize_shopify_client(self, mocked_current, mocked_set_timeout): - """ - Test case to verify that the default value is used when pass empty request timeout value from config - """ - # initialize config - Context.config = { - "start_date": "2021-01-01", - "api_key": "test_api_key", - "shop": "test_shop", - "results_per_page": 50, - "request_timeout": "" - } - - # function call - tap_shopify.initialize_shopify_client() - # verify the timeout is set as expected - mocked_set_timeout.assert_called_with(300) - - @mock.patch("shopify.Shop.set_timeout") - @mock.patch("shopify.Shop.current") - def test_timeout_0_value_passed_in_config__initialize_shopify_client(self, mocked_current, mocked_set_timeout): - """ - Test case to verify that the default value is used when pass 0 as request timeout value from config - """ - # initialize config - Context.config = { - "start_date": "2021-01-01", - "api_key": "test_api_key", - "shop": "test_shop", - "results_per_page": 50, - "request_timeout": 0.0 - } - - # function call - tap_shopify.initialize_shopify_client() - # verify the timeout is set as expected - mocked_set_timeout.assert_called_with(300) - - @mock.patch("shopify.Shop.set_timeout") - @mock.patch("shopify.Shop.current") - def test_timeout_string_0_value_passed_in_config__initialize_shopify_client(self, mocked_current, mocked_set_timeout): - """ - Test case to verify that the default value is used when pass string 0 as request timeout value from config - """ - # initialize config - Context.config = { - "start_date": "2021-01-01", - "api_key": "test_api_key", - "shop": "test_shop", - "results_per_page": 50, - "request_timeout": "0.0" - } - - # function call - tap_shopify.initialize_shopify_client() - # verify the timeout is set as expected - mocked_set_timeout.assert_called_with(300) - -class TestTimeoutBackoff(unittest.TestCase): - """ - Verify the tap backoff for 5 times when timeout error occurs - """ - - @mock.patch("time.sleep") - @mock.patch("shopify.Checkout.find") - def test_AbandonedCheckouts_pyactiveresource_error_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'pyactiveresource.connection.Error' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = pyactiveresource.connection.Error('urlopen error _ssl.c:1074: The handshake operation timed out') - - # initialize 'AbandonedCheckouts' as it calls the function 'call_api' from the base class - abandoned_checkouts = AbandonedCheckouts() - try: - # function call - abandoned_checkouts.call_api({}) - except pyactiveresource.connection.Error: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) - - @mock.patch("time.sleep") - @mock.patch("pyactiveresource.activeresource.ActiveResource.find") - def test_InventoryLevels_pyactiveresource_error_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'pyactiveresource.connection.Error' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = pyactiveresource.connection.Error('urlopen error _ssl.c:1074: The handshake operation timed out') - - # initialize class - inventory_levels = InventoryLevels() - try: - # function call - inventory_levels.api_call_for_inventory_levels(1, 'test') - except pyactiveresource.connection.Error: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) - - @mock.patch("time.sleep") - @mock.patch("pyactiveresource.activeresource.ActiveResource.find") - def test_Locations_pyactiveresource_error_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'pyactiveresource.connection.Error' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = pyactiveresource.connection.Error('urlopen error _ssl.c:1074: The handshake operation timed out') - - # initialize class - locations = Locations() - try: - # function call - locations.replication_object.find() - except pyactiveresource.connection.Error: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) - - @mock.patch("time.sleep") - @mock.patch("shopify.Refund.find") - def test_OrderRefunds_pyactiveresource_error_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'pyactiveresource.connection.Error' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = pyactiveresource.connection.Error('urlopen error _ssl.c:1074: The handshake operation timed out') - - # initialize class - order_refunds = OrderRefunds() - try: - # function call - order_refunds.get_refunds(shopify.Product, 1) - except pyactiveresource.connection.Error: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) - - @mock.patch("time.sleep") - @mock.patch("pyactiveresource.activeresource.ActiveResource.find") - def test_Transactions_pyactiveresource_error_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'pyactiveresource.connection.Error' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = pyactiveresource.connection.Error('urlopen error _ssl.c:1074: The handshake operation timed out') - - # initialize class - locations = Transactions() - try: - # function call - locations.replication_object.find() - except pyactiveresource.connection.Error: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) - - @mock.patch("time.sleep") - @mock.patch("shopify.Shop.current") - def test_Shop_pyactiveresource_error_timeout_backoff(self, mocked_current, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'pyactiveresource.connection.Error' error occurs - """ - # mock 'Shop' call and raise timeout error - mocked_current.side_effect = pyactiveresource.connection.Error('urlopen error _ssl.c:1074: The handshake operation timed out') - - Context.config = { - "api_key": "test_api_key", - "shop": "test_shop" - } - try: - # function call - tap_shopify.initialize_shopify_client() - except pyactiveresource.connection.Error: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_current.call_count, 5) - - """ - Verify the tap backoff for 5 times when timeout error occurs - """ - - @mock.patch("time.sleep") - @mock.patch("shopify.Checkout.find") - def test_AbandonedCheckouts_socket_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'socket.timeout' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = socket.timeout("The read operation timed out") - - # initialize 'AbandonedCheckouts' as it calls the function 'call_api' from the base class - abandoned_checkouts = AbandonedCheckouts() - try: - # function call - abandoned_checkouts.call_api({}) - except socket.timeout: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) - - - @mock.patch("time.sleep") - @mock.patch("pyactiveresource.activeresource.ActiveResource.find") - def test_InventoryLevels_socket_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'socket.timeout' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = socket.timeout("The read operation timed out") - - # initialize class - inventory_levels = InventoryLevels() - try: - # function call - inventory_levels.api_call_for_inventory_levels(1, 'test') - except socket.timeout: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) - - @mock.patch("time.sleep") - @mock.patch("pyactiveresource.activeresource.ActiveResource.find") - def test_Locations_socket_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'socket.timeout' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = socket.timeout("The read operation timed out") - - # initialize class - locations = Locations() - try: - # function call - locations.replication_object.find() - except socket.timeout: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) - - @mock.patch("time.sleep") - @mock.patch("shopify.Refund.find") - def test_OrderRefunds_socket_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'socket.timeout' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = socket.timeout("The read operation timed out") - - # initialize class - order_refunds = OrderRefunds() - try: - # function call - order_refunds.get_refunds(shopify.Product, 1) - except socket.timeout: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) - - @mock.patch("time.sleep") - @mock.patch("pyactiveresource.activeresource.ActiveResource.find") - def test_Transactions_socket_timeout_backoff(self, mocked_find, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'socket.timeout' error occurs - """ - # mock 'find' and raise timeout error - mocked_find.side_effect = socket.timeout("The read operation timed out") - - # initialize class - locations = Transactions() - try: - # function call - locations.replication_object.find() - except socket.timeout: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_find.call_count, 5) - - @mock.patch("time.sleep") - @mock.patch("shopify.Shop.current") - def test_Shop_socket_timeout_backoff(self, mocked_current, mocked_sleep): - """ - Test case to verify that we backoff for 5 times when 'socket.timeout' error occurs - """ - # mock 'Shop' call and raise timeout error - mocked_current.side_effect = socket.timeout("The read operation timed out") - - Context.config = { - "api_key": "test_api_key", - "shop": "test_shop" - } - try: - # function call - tap_shopify.initialize_shopify_client() - except socket.timeout: - pass - - # verify we backoff 5 times - self.assertEquals(mocked_current.call_count, 5) diff --git a/tests/unittests/test_transaction_canonicalize.py b/tests/unittests/test_transaction_canonicalize.py deleted file mode 100644 index fb1ab5d3..00000000 --- a/tests/unittests/test_transaction_canonicalize.py +++ /dev/null @@ -1,40 +0,0 @@ -import unittest -from tap_shopify.streams.transactions import canonicalize - -class TestTransactionCanonicalize(unittest.TestCase): - def test_unmodified_if_not_present(self): - # Note: Canonicalize has a side effect with pop(), must copy test - # record to compare - record = {"receipt": {"foo": "bar"}, "id": 2} - expected_record = {"receipt": {"foo": "bar"}, "id": 2} - canonicalize(record, "token") - self.assertEqual(record, expected_record) - - def test_unmodified_if_only_lower_exists(self): - record = {"receipt": {"foo": "bar"}, "id": 2} - expected_record = {"receipt": {"foo": "bar"}, "id": 2} - canonicalize(record, "foo") - self.assertEqual(record, expected_record) - - def test_lowercases_if_capital_only_exists(self): - record = {"receipt": {"Foo": "bar"}, "id": 2} - expected_record = {"receipt": {"foo": "bar"}, "id": 2} - canonicalize(record, "foo") - self.assertEqual(record, expected_record) - - def test_null_receipt_record(self): - record = {"receipt": None} - expected_record = {"receipt": None} - canonicalize(record, "foo") - self.assertEqual(record, expected_record) - - def test_removes_uppercase_if_both_exist_and_are_equal(self): - record = {"receipt": {"Foo": "bar", "foo": "bar"}, "id": 2} - expected_record = {"receipt": {"foo": "bar"}, "id": 2} - canonicalize(record, "foo") - self.assertEqual(record, expected_record) - - def test_throws_if_both_exist_and_are_not_equal(self): - record = {"receipt": {"Foo": "bark", "foo": "bar"}, "id": 2} - with self.assertRaises(ValueError): - canonicalize(record, "foo") diff --git a/tests/unittests/test_updated_bookmark.py b/tests/unittests/test_updated_bookmark.py deleted file mode 100644 index 342af8ad..00000000 --- a/tests/unittests/test_updated_bookmark.py +++ /dev/null @@ -1,83 +0,0 @@ -import unittest -from unittest import mock -from singer.utils import strftime, strptime_to_utc -from tap_shopify.context import Context -import datetime - -from tap_shopify.streams.base import DATE_WINDOW_SIZE - -CUSTOMER_OBJECT = Context.stream_objects['customers']() -Context.state = {} - -class Customer(): - """The customer object to return in th api call""" - def __init__(self, id, updated_at): - self.id = id - self.updated_at = updated_at - - def to_dict(self): - return {"id": self.id, "updated_at": self.updated_at} - - -class DummyShopifyError(Exception): - def __init__(self, error, msg=''): - super().__init__('{}\n{}'.format(error.__class__.__name__, msg)) - - -CUSTOMER_1 = Customer(20, "2021-08-11T01:57:05-04:00") -CUSTOMER_2 = Customer(30, "2021-08-12T01:57:05-04:00") - -NOW_TIME = '2021-08-16T01:56:05-04:00' -class TestUpdateBookmark(unittest.TestCase): - - @mock.patch("tap_shopify.streams.base.Stream.call_api", return_value = [CUSTOMER_1, CUSTOMER_2]) - @mock.patch("tap_shopify.streams.base.Stream.get_bookmark", return_value=strptime_to_utc('2021-08-11T01:55:05-04:00')) - @mock.patch("tap_shopify.streams.base.Stream.update_bookmark") - @mock.patch("tap_shopify.streams.base.Stream.get_query_params") - @mock.patch("singer.utils.now", return_value=strptime_to_utc(NOW_TIME)) - def test_update_bookmark(self, mock_now, mock_get_query_params, mock_update_bookmark, mock_get_bookmark, mock_call_api): - """Verify that the update_bookmark() is called with correct argument""" - list(CUSTOMER_OBJECT.get_objects()) - # Verify that the min-max evaluation returns correct results and provided in the update_bookmark() - mock_update_bookmark.assert_called_with(strftime(strptime_to_utc(NOW_TIME) - datetime.timedelta(DATE_WINDOW_SIZE))) - - @mock.patch("tap_shopify.streams.base.Stream.call_api", return_value=[CUSTOMER_1, CUSTOMER_2]) - @mock.patch("singer.utils.now", return_value=strptime_to_utc(NOW_TIME)) - def test_since_id_and_updated_at_max_deleted(self, mock_now, mock_call_api): - """Verify after successful sync since_id and updated_at_max keys are deleted from the state""" - Context.state = {"bookmarks": { - "currently_sync_stream": 'customers', - "customers": { - "updated_at": "2021-03-27T00:00:00.000000Z", - "since_id": 15, - "updated_at_max": "2021-04-26T00:00:00.000000Z"}}} - - list(CUSTOMER_OBJECT.get_objects()) - - # Verify keys - self.assertIn("updated_at", Context.state["bookmarks"]["customers"]) - self.assertNotIn("since_id", Context.state["bookmarks"]["customers"]) - self.assertNotIn("updated_at_max", Context.state["bookmarks"]["customers"]) - - @mock.patch("tap_shopify.streams.base.Stream.call_api", side_effect=DummyShopifyError("Dummy Shopify exception...")) - @mock.patch("singer.utils.now", return_value=strptime_to_utc(NOW_TIME)) - def test_interrupted_sync(self, mock_now, mock_call_api): - """Verify if sync is interrrupted twice in a row then since_id and updated_at_max keys are not deleted from the state""" - Context.state = {"bookmarks": { - "currently_sync_stream": 'customers', - "customers": { - "updated_at": "2021-03-27T00:00:00.000000Z", - "since_id": 15, - "updated_at_max": "2021-04-26T00:00:00.000000Z"}}} - - with self.assertRaises(DummyShopifyError): - list(CUSTOMER_OBJECT.get_objects()) - - # Verify keys exist - self.assertIn("since_id", Context.state["bookmarks"]["customers"]) - self.assertIn("updated_at_max", Context.state["bookmarks"]["customers"]) - - # Verify bookmark key values are as expected - self.assertEqual(Context.state["bookmarks"]["customers"]['since_id'], 15) - self.assertEqual(Context.state["bookmarks"]["customers"]['updated_at_max'], "2021-04-26T00:00:00.000000Z") - From 2f8355b56d8c96401bfe00a96fbf56b091739e3a Mon Sep 17 00:00:00 2001 From: Prijen Khokhani <88327452+prijendev@users.noreply.github.com> Date: Fri, 4 Apr 2025 20:33:26 +0530 Subject: [PATCH 39/66] Fix schema orders (#208) * updated orders schema * update orders schema * reduce query cost by limiting less line items * Add refunds into order schema * updated page limits * Fix pylint issue * removed order_refund from query and schema * updated pagination logic * fix refund schema * update orders pagination * Add missing fields in schema * DIsable pylint for order_refunds query * Fix json decode error * Update version in changelog * fixed review comments * update schema * Update datatype for addresses --------- Co-authored-by: Dylan Sprayberry Co-authored-by: vishalpachpinde --- CHANGELOG.md | 3 + setup.py | 2 +- tap_shopify/schemas/order_refunds.json | 638 ++++++++++++++ tap_shopify/schemas/orders.json | 1056 ++++++++++++++++++++++-- tap_shopify/streams/order_refunds.py | 180 +++- tap_shopify/streams/orders.py | 289 ++++++- 6 files changed, 2071 insertions(+), 97 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a11b7375..0c539a4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 3.1.0 + * Add missing fields into the schema of orders and order_refunds stream [#208](https://github.com/singer-io/tap-shopify/pull/208) + ## 3.0.0 * Migrate Remaining Shopify Streams from REST API to GraphQL API [#201](https://github.com/singer-io/tap-shopify/pull/201) * Introduce new streams - `collections`, `metafields_collections`, `metafields_customers`, `metafields_orders`, `metafields_products` diff --git a/setup.py b/setup.py index 1365a671..b12cb174 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="3.0.0", + version="3.1.0", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/schemas/order_refunds.json b/tap_shopify/schemas/order_refunds.json index 583917e0..02565300 100644 --- a/tap_shopify/schemas/order_refunds.json +++ b/tap_shopify/schemas/order_refunds.json @@ -125,6 +125,20 @@ "boolean" ] }, + "location": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + }, "subtotalSet": { "type": [ "null", @@ -224,6 +238,630 @@ } } } + }, + "lineItem": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "vendor": { + "type": [ + "null", + "string" + ] + }, + "quantity": { + "type": [ + "null", + "integer" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "requiresShipping": { + "type": [ + "null", + "boolean" + ] + }, + "originalTotalSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + } + } + }, + "taxLines": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "priceSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "number" + ] + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "number" + ] + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "rate": { + "type": [ + "null", + "number" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "source": { + "type": [ + "null", + "string" + ] + }, + "channelLiable": { + "type": [ + "null", + "boolean" + ] + } + } + } + }, + "taxable": { + "type": [ + "null", + "boolean" + ] + }, + "isGiftCard": { + "type": [ + "null", + "boolean" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "discountedTotalSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + } + } + }, + "sku": { + "type": [ + "null", + "string" + ] + }, + "product": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + }, + "discountAllocations": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "allocatedAmountSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "number" + ] + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "number" + ] + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "discountApplication": { + "type": [ + "null", + "object" + ], + "properties": { + "index": { + "type": [ + "null", + "integer" + ] + }, + "targetType": { + "type": [ + "null", + "string" + ] + }, + "targetSelection": { + "type": [ + "null", + "string" + ] + }, + "allocationMethod": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "object" + ], + "properties": { + "__typename": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "percentage": { + "type": [ + "null", + "number" + ] + } + } + } + } + } + } + } + }, + "customAttributes": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "key": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "totalDiscountSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + } + } + }, + "duties": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "harmonizedSystemCode": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "taxLines": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "rate": { + "type": [ + "null", + "number" + ] + }, + "source": { + "type": [ + "null", + "string" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "channelLiable": { + "type": [ + "null", + "boolean" + ] + }, + "priceSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "number" + ] + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "number" + ] + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + } + } + } + }, + "countryCodeOfOrigin": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "discountedUnitPriceSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + } + } + } + } } } } diff --git a/tap_shopify/schemas/orders.json b/tap_shopify/schemas/orders.json index 87d5afda..46790777 100644 --- a/tap_shopify/schemas/orders.json +++ b/tap_shopify/schemas/orders.json @@ -62,28 +62,6 @@ } } }, - "alerts": { - "type": ["null", "array"], - "items": { - "type": "object", - "properties": { - "content": { "type": ["null", "string"] }, - "dismissibleHandle": { "type": ["null", "string"] }, - "icon": { "type": ["null", "string"] }, - "severity": { "type": ["null", "string"] }, - "title": { "type": ["null", "string"] }, - "actions": { - "type": ["null", "object"], - "properties": { - "primary": { "type": ["null", "boolean"] }, - "show": { "type": ["null", "boolean"] }, - "title": { "type": ["null", "string"] }, - "url": { "type": ["null", "string"] } - } - } - } - } - }, "app": { "type": ["null", "object"], "properties": { @@ -225,38 +203,6 @@ } } }, - "currentTaxLines": { - "type": ["null", "array"], - "items": { - "type": "object", - "properties": { - "channelLiable": { "type": ["null", "boolean"] }, - "priceSet": { - "type": ["null", "object"], - "properties": { - "presentmentMoney": { - "type": ["null", "object"], - "properties": { - "amount": { "type": ["null", "number"] }, - "currencyCode": { "type": ["null", "string"] } - } - }, - "shopMoney": { - "type": ["null", "object"], - "properties": { - "amount": { "type": ["null", "number"] }, - "currencyCode": { "type": ["null", "string"] } - } - } - } - }, - "rate": { "type": ["null", "number"] }, - "ratePercentage": { "type": ["null", "number"] }, - "source": { "type": ["null", "string"] }, - "title": { "type": ["null", "string"] } - } - } - }, "currentTotalAdditionalFeesSet": { "type": ["null", "object"], "properties": { @@ -354,12 +300,332 @@ }, "currentTotalWeight": { "type": ["null", "number"] }, "customer": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { - "id": { "type": ["null", "string"] }, - "email": { "type": ["null", "string"] }, - "firstName": { "type": ["null", "string"] }, - "lastName": { "type": ["null", "string"] } + "id": { + "type": [ + "null", + "string" + ] + }, + "email": { + "type": [ + "null", + "string" + ] + }, + "firstName": { + "type": [ + "null", + "string" + ] + }, + "lastName": { + "type": [ + "null", + "string" + ] + }, + "state": { + "type": [ + "null", + "string" + ] + }, + "verifiedEmail": { + "type": [ + "null", + "boolean" + ] + }, + "lastOrder": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + }, + "updatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "createdAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "note": { + "type": [ + "null", + "string" + ] + }, + "multipassIdentifier": { + "type": [ + "null", + "string" + ] + }, + "tags": { + "type": [ + "null", + "string" + ] + }, + "taxExempt": { + "type": [ + "null", + "string" + ] + }, + "taxExemptions": { + "type": [ + "null", + "string" + ] + }, + "defaultAddress": { + "type": [ + "null", + "object" + ], + "properties": { + "address1": { + "type": [ + "null", + "string" + ] + }, + "address2": { + "type": [ + "null", + "string" + ] + }, + "city": { + "type": [ + "null", + "string" + ] + }, + "countryCodeV2": { + "type": [ + "null", + "string" + ] + }, + "province": { + "type": [ + "null", + "string" + ] + }, + "zip": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "country": { + "type": [ + "null", + "string" + ] + }, + "lastName": { + "type": [ + "null", + "string" + ] + }, + "phone": { + "type": [ + "null", + "string" + ] + }, + "firstName": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "provinceCode": { + "type": [ + "null", + "string" + ] + }, + "company": { + "type": [ + "null", + "string" + ] + } + } + }, + "addresses": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "address1": { + "type": [ + "null", + "string" + ] + }, + "address2": { + "type": [ + "null", + "string" + ] + }, + "city": { + "type": [ + "null", + "string" + ] + }, + "countryCodeV2": { + "type": [ + "null", + "string" + ] + }, + "province": { + "type": [ + "null", + "string" + ] + }, + "zip": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "country": { + "type": [ + "null", + "string" + ] + }, + "lastName": { + "type": [ + "null", + "string" + ] + }, + "phone": { + "type": [ + "null", + "string" + ] + }, + "firstName": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "provinceCode": { + "type": [ + "null", + "string" + ] + }, + "company": { + "type": [ + "null", + "string" + ] + } + } + } + } + } + }, + "customerJourneySummary": { + "type": [ + "null", + "object" + ], + "lastVisit": { + "type": [ + "null", + "object" + ], + "properties": { + "landingPage": { + "type": [ + "null", + "string" + ] + }, + "referrerUrl": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "merchantOfRecordApp": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } } }, "customerAcceptsMarketing": { "type": ["null", "boolean"] }, @@ -860,6 +1126,676 @@ } }, "unpaid": { "type": ["null", "boolean"] }, - "updatedAt": { "type": ["null", "string"], "format": "date-time" } + "updatedAt": { "type": ["null", "string"], "format": "date-time" }, + "fulfillments": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "id": { "type": "string" }, + "name": { "type": ["null", "string"] }, + "status": { "type": ["null", "string"] }, + "totalQuantity": { "type": ["null", "number"] }, + "updatedAt": { "type": ["null", "string"], "format": "date-time" }, + "deliveredAt": { "type": ["null", "string"], "format": "date-time" }, + "estimatedDeliveryAt": { "type": ["null", "string"], "format": "date-time" }, + "requiresShipping": { "type": ["null", "boolean"] }, + "inTransitAt" : { "type": ["null", "string"], "format": "date-time" }, + "trackingInfo": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "number": { "type": ["null", "string"] }, + "company": { "type": ["null", "string"] }, + "url": { "type": ["null", "string"] } + } + } + }, + "service": { + "type": ["null", "object"], + "properties": { + "serviceName": { "type": ["null", "string"] }, + "id": { "type": ["null", "string"] }, + "handle": { "type": ["null", "string"] }, + "trackingSupport": { "type": ["null", "boolean"] }, + "type": { "type": ["null", "string"] }, + "permitsSkuSharing": { "type": ["null", "boolean"] }, + "inventoryManagement": { "type": ["null", "boolean"] } + } + } + } + } + }, + "lineItems":{ + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "vendor": { + "type": [ + "null", + "string" + ] + }, + "quantity": { + "type": [ + "null", + "number" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "requiresShipping": { + "type": [ + "null", + "boolean" + ] + }, + "originalTotalSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + } + } + }, + "taxLines": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "priceSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "number" + ] + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "number" + ] + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "rate": { + "type": [ + "null", + "number" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "source": { + "type": [ + "null", + "string" + ] + }, + "channelLiable": { + "type": [ + "null", + "boolean" + ] + } + } + } + }, + "taxable": { + "type": [ + "null", + "boolean" + ] + }, + "isGiftCard": { + "type": [ + "null", + "boolean" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "discountedTotalSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + } + } + }, + "sku": { + "type": [ + "null", + "string" + ] + }, + "product": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + }, + "discountAllocations": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "allocatedAmountSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "number" + ] + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "number" + ] + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "discountApplication": { + "type": [ + "null", + "object" + ], + "properties": { + "index": { + "type": [ + "null", + "integer" + ] + }, + "targetType": { + "type": [ + "null", + "string" + ] + }, + "targetSelection": { + "type": [ + "null", + "string" + ] + }, + "allocationMethod": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "object" + ], + "properties": { + "__typename": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "percentage": { + "type": [ + "null", + "number" + ] + } + } + } + } + } + } + } + }, + "customAttributes": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "key": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "totalDiscountSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + } + } + }, + "duties": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "harmonizedSystemCode": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "taxLines": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "rate": { + "type": [ + "null", + "number" + ] + }, + "source": { + "type": [ + "null", + "string" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "channelLiable": { + "type": [ + "null", + "boolean" + ] + }, + "priceSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "number" + ] + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "number" + ] + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + } + } + } + }, + "countryCodeOfOrigin": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "discountedUnitPriceSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + } + } + } + } + } + } } } diff --git a/tap_shopify/streams/order_refunds.py b/tap_shopify/streams/order_refunds.py index 178e79f5..933489b0 100644 --- a/tap_shopify/streams/order_refunds.py +++ b/tap_shopify/streams/order_refunds.py @@ -54,6 +54,44 @@ def get_objects(self): last_updated_at = query_end + def transform_lineitems(self, data): + """ + Transforms the order lineitems data by extracting order IDs and handling pagination. + + Args: + data (dict): Order data. + + Returns: + list: List of refunds with lineitems. + """ + + lineitems = [ + node for item in data["refundLineItems"]["edges"] + if (node := item.get("node")) + ] + + # Handle pagination + page_info = data["refundLineItems"].get("pageInfo", {}) + order_parent = data.get("order").get("id") + while page_info.get("hasNextPage"): + params = { + "first": self.results_per_page, + "query": f"id:{order_parent.split('/')[-1]}", + "childafter": page_info.get("endCursor"), + } + + # Fetch the next page of data + response = self.call_api(params) + nodes = response.get("edges", [])[0].get("node", {}) + lineitems_data = nodes.get("refunds")[0] + lineitems.extend( + node for item in lineitems_data["refundLineItems"]["edges"] + if (node := item.get("node")) + ) + page_info = lineitems_data.get("pageInfo", {}) + + return lineitems + def transform_object(self, obj): """ Transform refund objects by extracting refund line items from edges. @@ -64,9 +102,9 @@ def transform_object(self, obj): Returns: dict: Transformed refund object. """ - obj["refundLineItems"] = [ - edge.get("node") for edge in obj.get("refundLineItems", {}).get("edges", []) - ] + + if obj.get("refundLineItems"): + obj["refundLineItems"] = self.transform_lineitems(obj) return obj def sync(self): @@ -95,7 +133,8 @@ def get_query(self): Returns: str: GraphQL query string. """ - return """query GetOrderRefunds($first: Int!, $after: String, $query: String) { + # pylint: disable=line-too-long + return """query GetOrderRefunds($first: Int!, $after: String, $query: String, $childafter: String) { orders(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { edges { node { @@ -107,7 +146,7 @@ def get_query(self): order { id } - refundLineItems(first: 250) { + refundLineItems(first: 50, after: $childafter) { edges { node { id @@ -124,6 +163,9 @@ def get_query(self): } restockType restocked + location { + id + } subtotalSet { presentmentMoney { amount @@ -144,8 +186,136 @@ def get_query(self): currencyCode } } + lineItem { + id + vendor + quantity + title + requiresShipping + originalTotalSet { + presentmentMoney { + currencyCode + amount + } + shopMoney { + amount + currencyCode + } + } + taxLines(first: 250) { + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + rate + title + source + channelLiable + } + taxable + isGiftCard + name + discountedTotalSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + sku + product { + id + } + discountAllocations { + allocatedAmountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + discountApplication { + index + targetType + targetSelection + allocationMethod + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + } + customAttributes { + key + value + } + totalDiscountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + duties { + harmonizedSystemCode + id + taxLines { + rate + source + title + channelLiable + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + } + countryCodeOfOrigin + } + discountedUnitPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + } } } + pageInfo { + hasNextPage + endCursor + } } updatedAt } diff --git a/tap_shopify/streams/orders.py b/tap_shopify/streams/orders.py index 7b948cd5..669779a3 100644 --- a/tap_shopify/streams/orders.py +++ b/tap_shopify/streams/orders.py @@ -9,6 +9,55 @@ class Orders(Stream): data_key = "orders" replication_key = "updatedAt" + def transform_lineitems(self, data): + """ + Transforms the order lineitems data by extracting order IDs and handling pagination. + + Args: + data (dict): Order data. + + Returns: + list: List of lineItems. + """ + lineitems = [ + node for item in data["lineItems"]["edges"] + if (node := item.get("node")) + ] + + # Handle pagination + page_info = data["lineItems"].get("pageInfo", {}) + while page_info.get("hasNextPage"): + params = { + "first": self.results_per_page, + "query": f"id:{data['id'].split('/')[-1]}", + "childafter": page_info.get("endCursor"), + } + + # Fetch the next page of data + response = self.call_api(params) + lineitems_data = response.get("edges", [])[0].get("node", {}) + lineitems.extend( + node for item in lineitems_data["lineItems"]["edges"] + if (node := item.get("node")) + ) + page_info = lineitems_data.get("pageInfo", {}) + + return lineitems + + def transform_object(self, obj): + """ + Transforms a collection object. + + Args: + obj (dict): Collection object. + + Returns: + dict: Transformed collection object. + """ + if obj.get("lineItems"): + obj["lineItems"] = self.transform_lineitems(obj) + return obj + def get_query(self): """ Returns the GraphQL query for fetching orders. @@ -17,7 +66,7 @@ def get_query(self): str: GraphQL query string. """ return """ - query orders($first: Int!, $after: String, $query: String) { + query orders($first: Int!, $after: String, $query: String, $childafter: String) { orders(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { edges { node { @@ -52,19 +101,6 @@ def get_query(self): title } } - alerts { - content - dismissibleHandle - icon - severity - title - actions { - primary - show - title - url - } - } app { id name @@ -155,23 +191,6 @@ def get_query(self): currencyCode } } - currentTaxLines { - channelLiable - priceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - rate - ratePercentage - source - title - } currentTotalAdditionalFeesSet { presentmentMoney { amount @@ -228,6 +247,56 @@ def get_query(self): email firstName lastName + addresses { + address1 + address2 + city + countryCodeV2 + country + company + firstName + lastName + id + name + phone + province + provinceCode + zip + } + state + verifiedEmail + updatedAt + taxExempt + tags + taxExemptions + note + multipassIdentifier + createdAt + defaultAddress { + address1 + address2 + city + company + country + countryCodeV2 + firstName + id + lastName + name + province + phone + provinceCode + zip + } + } + customerJourneySummary { + lastVisit { + landingPage + referrerUrl + } + } + merchantOfRecordApp { + id } customerAcceptsMarketing customerLocale @@ -521,6 +590,164 @@ def get_query(self): } unpaid updatedAt + fulfillments(first: 250) { + id + name + status + totalQuantity + updatedAt + createdAt + deliveredAt + estimatedDeliveryAt + requiresShipping + inTransitAt + trackingInfo(first: 250) { + number + company + url + } + service { + serviceName + id + handle + trackingSupport + type + permitsSkuSharing + inventoryManagement + } + } + lineItems(first: 25, after: $childafter) { + edges { + node { + id + vendor + quantity + title + requiresShipping + originalTotalSet { + presentmentMoney { + currencyCode + amount + } + shopMoney { + amount + currencyCode + } + } + taxLines(first: 250) { + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + rate + title + source + channelLiable + } + taxable + isGiftCard + name + discountedTotalSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + sku + product { + id + } + discountAllocations { + allocatedAmountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + discountApplication { + index + targetType + targetSelection + allocationMethod + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + } + customAttributes { + key + value + } + totalDiscountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + duties { + harmonizedSystemCode + id + taxLines { + rate + source + title + channelLiable + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + } + countryCodeOfOrigin + } + discountedUnitPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } } } pageInfo { From beb71fca403871cd96d4ea392cd083913b1a0008 Mon Sep 17 00:00:00 2001 From: Sourabh Gandhi <105213416+sgandhi1311@users.noreply.github.com> Date: Wed, 16 Apr 2025 21:42:32 +0530 Subject: [PATCH 40/66] Enhance tap with additional fields across streams (#209) * Add missing fields * --------- Co-authored-by: Vishal Pachpinde --- CHANGELOG.md | 4 + setup.py | 2 +- tap_shopify/__init__.py | 20 +- tap_shopify/schemas/abandoned_checkouts.json | 1311 ++++++++++++---- tap_shopify/schemas/customers.json | 14 + tap_shopify/schemas/definitions.json | 1452 ------------------ tap_shopify/schemas/order_refunds.json | 126 ++ tap_shopify/schemas/orders.json | 317 ++++ tap_shopify/schemas/product_variants.json | 41 +- tap_shopify/schemas/transactions.json | 187 +++ tap_shopify/streams/abandoned_checkouts.py | 6 + tap_shopify/streams/base.py | 21 +- tap_shopify/streams/customers.py | 177 +-- tap_shopify/streams/inventory_levels.py | 10 + tap_shopify/streams/metafields.py | 8 +- tap_shopify/streams/order_refunds.py | 75 +- tap_shopify/streams/orders.py | 1390 ++++++++++------- tap_shopify/streams/product_variants.py | 14 +- tap_shopify/streams/transactions.py | 221 +-- 19 files changed, 2879 insertions(+), 2517 deletions(-) delete mode 100644 tap_shopify/schemas/definitions.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c539a4a..7875dad5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 3.2.0 + * Enhance tap with additional fields across streams [#209](https://github.com/singer-io/tap-shopify/pull/209) + * Bookmarks are now updated after fetching records within the specified date range. + ## 3.1.0 * Add missing fields into the schema of orders and order_refunds stream [#208](https://github.com/singer-io/tap-shopify/pull/208) diff --git a/setup.py b/setup.py index b12cb174..6a6b0f6a 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="3.1.0", + version="3.2.0", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index fc2f0946..d0c5d001 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -68,16 +68,6 @@ def get_discovery_metadata(stream, schema): return metadata.to_list(mdata) -def load_schema_references(): - shared_schema_file = "definitions.json" - shared_schema_path = get_abs_path('schemas/') - - refs = {} - with open(os.path.join(shared_schema_path, shared_schema_file), encoding='UTF-8') as data_file: - refs[shared_schema_file] = json.load(data_file) - - return refs - def add_synthetic_key_to_schema(schema): for k in SDC_KEYS: schema['properties']['_sdc_shop_' + k] = {'type': ["null", SDC_KEYS[k]]} @@ -89,20 +79,12 @@ def discover(): raw_schemas = load_schemas() streams = [] - refs = load_schema_references() for schema_name, schema in raw_schemas.items(): if schema_name not in Context.stream_objects: continue stream = Context.stream_objects[schema_name]() - - # resolve_schema_references() is changing value of passed refs. - # Customer is a stream and it's a nested field of orders and abandoned_checkouts streams - # and those 3 _sdc fields are also added inside nested field customer for above 2 stream - # so create a copy of refs before passing it to resolve_schema_references(). - refs_copy = copy.deepcopy(refs) - catalog_schema = add_synthetic_key_to_schema( - singer.resolve_schema_references(schema, refs_copy)) + catalog_schema = add_synthetic_key_to_schema(schema) # create and add catalog entry catalog_entry = { diff --git a/tap_shopify/schemas/abandoned_checkouts.json b/tap_shopify/schemas/abandoned_checkouts.json index bb003f1f..040b555c 100644 --- a/tap_shopify/schemas/abandoned_checkouts.json +++ b/tap_shopify/schemas/abandoned_checkouts.json @@ -1,324 +1,1046 @@ { "type": "object", "properties": { - "note": {"type": ["null", "string"]}, - "completedAt": {"type": ["null", "string"], "format": "date-time"}, - "billingAddress": {"type": ["null", "object"], "properties": { - "phone": { "type": ["null", "string"] }, - "country": { "type": ["null", "string"] }, - "firstName": { "type": ["null", "string"] }, - "name": { "type": ["null", "string"] }, - "latitude": { "type": ["null", "number"] }, - "zip": { "type": ["null", "string"] }, - "lastName": { "type": ["null", "string"] }, - "province": { "type": ["null", "string"] }, - "address2": { "type": ["null", "string"] }, - "address1": { "type": ["null", "string"] }, - "countryCodeV2": { "type": ["null", "string"] }, - "city": { "type": ["null", "string"] }, - "company": { "type": ["null", "string"] }, - "provinceCode": { "type": ["null", "string"] }, - "longitude": { "type": ["null", "number"] }, - "coordinatesValidated": { "type": ["null", "boolean"] }, - "formattedArea": { "type": ["null", "string"] }, - "id": { "type": ["null", "string"] }, - "timeZone": { "type": ["null", "string"] }, - "validationResultSummary": { "type": ["null", "string"] } - }}, - "discountCodes": {"type": "array", "items": {"type": ["null", "string"]}}, - "createdAt": {"type": ["null", "string"], "format": "date-time"}, - "updatedAt": {"type": ["null", "string"], "format": "date-time"}, - "taxLines": {"type": "array", "items": {"type": ["null", "object"], "properties": { - "priceSet": { - "type": ["null", "object"], + "note": { + "type": [ + "null", + "string" + ] + }, + "completedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "billingAddress": { + "type": [ + "null", + "object" + ], + "properties": { + "phone": { + "type": [ + "null", + "string" + ] + }, + "country": { + "type": [ + "null", + "string" + ] + }, + "firstName": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "latitude": { + "type": [ + "null", + "number" + ] + }, + "zip": { + "type": [ + "null", + "string" + ] + }, + "lastName": { + "type": [ + "null", + "string" + ] + }, + "province": { + "type": [ + "null", + "string" + ] + }, + "address2": { + "type": [ + "null", + "string" + ] + }, + "address1": { + "type": [ + "null", + "string" + ] + }, + "countryCodeV2": { + "type": [ + "null", + "string" + ] + }, + "city": { + "type": [ + "null", + "string" + ] + }, + "company": { + "type": [ + "null", + "string" + ] + }, + "provinceCode": { + "type": [ + "null", + "string" + ] + }, + "longitude": { + "type": [ + "null", + "number" + ] + }, + "coordinatesValidated": { + "type": [ + "null", + "boolean" + ] + }, + "formattedArea": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "timeZone": { + "type": [ + "null", + "string" + ] + }, + "validationResultSummary": { + "type": [ + "null", + "string" + ] + } + } + }, + "discountCodes": { + "type": "array", + "items": { + "type": [ + "null", + "string" + ] + } + }, + "createdAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "updatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "taxLines": { + "type": "array", + "items": { + "type": [ + "null", + "object" + ], "properties": { - "presentmentMoney": { - "type": ["null", "object"], + "priceSet": { + "type": [ + "null", + "object" + ], "properties": { - "amount": { "type": ["null", "string"], "format": "singer.decimal" }, - "currencyCode": { "type": ["null", "string"] } + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } } }, - "shopMoney": { - "type": ["null", "object"], - "properties": { - "amount": { "type": ["null", "string"], "format": "singer.decimal" }, - "currencyCode": { "type": ["null", "string"] } - } + "title": { + "type": [ + "null", + "string" + ] + }, + "rate": { + "type": [ + "null", + "number" + ] + }, + "source": { + "type": [ + "null", + "string" + ] + }, + "channelLiable": { + "type": [ + "null", + "boolean" + ] + }, + "ratePercentage": { + "type": [ + "null", + "number" + ] } } - }, - "title": { "type": ["null", "string"] }, - "rate": { "type": ["null", "number"] }, - "source": { "type": ["null", "string"] }, - "channelLiable": { "type": ["null", "boolean"] }, - "ratePercentage": { "type": ["null", "number"] } - }}}, + } + }, "totalLineItemsPriceSet": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "presentmentMoney": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { - "amount": { "type": ["null", "string"], "format": "singer.decimal" }, - "currencyCode": {"type": ["null", "string"]} + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } } }, "shopMoney": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { - "amount": { "type": ["null", "string"], "format": "singer.decimal" }, - "currencyCode": {"type": ["null", "string"]} + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } } } } }, - "id": {"type": ["null", "string"]}, - "name": {"type": ["null", "string"]}, + "id": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, "totalTaxSet": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "presentmentMoney": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { - "amount": { "type": ["null", "string"], "format": "singer.decimal" }, - "currencyCode": {"type": ["null", "string"]} + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } } }, "shopMoney": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { - "amount": { "type": ["null", "string"], "format": "singer.decimal" }, - "currencyCode": {"type": ["null", "string"]} + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } } } } }, - "shippingAddress": {"type": ["null", "object"], "properties": { - "phone": { "type": ["null", "string"] }, - "country": { "type": ["null", "string"] }, - "firstName": { "type": ["null", "string"] }, - "name": { "type": ["null", "string"] }, - "latitude": { "type": ["null", "string"], "format": "singer.decimal" }, - "zip": { "type": ["null", "string"] }, - "lastName": { "type": ["null", "string"] }, - "province": { "type": ["null", "string"] }, - "address2": { "type": ["null", "string"] }, - "address1": { "type": ["null", "string"] }, - "countryCodeV2": { "type": ["null", "string"] }, - "city": { "type": ["null", "string"] }, - "company": { "type": ["null", "string"] }, - "provinceCode": { "type": ["null", "string"] }, - "longitude": { "type": ["null", "string"], "format": "singer.decimal" }, - "coordinatesValidated": { "type": ["null", "boolean"] }, - "formattedArea": { "type": ["null", "string"] }, - "id": { "type": ["null", "string"] }, - "timeZone": { "type": ["null", "string"] }, - "validationResultSummary": { "type": ["null", "string"] } - }}, - "abandonedCheckoutUrl": {"type": ["null", "string"]}, + "shippingAddress": { + "type": [ + "null", + "object" + ], + "properties": { + "phone": { + "type": [ + "null", + "string" + ] + }, + "country": { + "type": [ + "null", + "string" + ] + }, + "firstName": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "latitude": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "zip": { + "type": [ + "null", + "string" + ] + }, + "lastName": { + "type": [ + "null", + "string" + ] + }, + "province": { + "type": [ + "null", + "string" + ] + }, + "address2": { + "type": [ + "null", + "string" + ] + }, + "address1": { + "type": [ + "null", + "string" + ] + }, + "countryCodeV2": { + "type": [ + "null", + "string" + ] + }, + "city": { + "type": [ + "null", + "string" + ] + }, + "company": { + "type": [ + "null", + "string" + ] + }, + "provinceCode": { + "type": [ + "null", + "string" + ] + }, + "longitude": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "coordinatesValidated": { + "type": [ + "null", + "boolean" + ] + }, + "formattedArea": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "timeZone": { + "type": [ + "null", + "string" + ] + }, + "validationResultSummary": { + "type": [ + "null", + "string" + ] + } + } + }, + "abandonedCheckoutUrl": { + "type": [ + "null", + "string" + ] + }, "totalDiscountSet": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "presentmentMoney": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { - "amount": { "type": ["null", "string"], "format": "singer.decimal" }, - "currencyCode": {"type": ["null", "string"]} + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } } }, "shopMoney": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { - "amount": { "type": ["null", "string"], "format": "singer.decimal" }, - "currencyCode": {"type": ["null", "string"]} + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } } } } }, - "taxesIncluded": {"type": "boolean"}, - "totalDutiesSet": {"type": ["null", "object"], "properties": { - "presentmentMoney": { - "type": ["null", "object"], - "properties": { - "amount": { "type": ["null", "string"], "format": "singer.decimal" }, - "currencyCode": {"type": ["null", "string"]} - } - }, - "shopMoney": { - "type": ["null", "object"], - "properties": { - "amount": { "type": ["null", "string"], "format": "singer.decimal" }, - "currencyCode": {"type": ["null", "string"]} + "taxesIncluded": { + "type": "boolean" + }, + "totalDutiesSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } } - }}}, + } + }, "totalPriceSet": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "presentmentMoney": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { - "amount": { "type": ["null", "string"], "format": "singer.decimal" }, - "currencyCode": {"type": ["null", "string"]} + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } } }, "shopMoney": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { - "amount": { "type": ["null", "string"], "format": "singer.decimal" }, - "currencyCode": {"type": ["null", "string"]} + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } } } } }, "lineItems": { - "type": "array", + "type": [ + "null", + "array" + ], "items": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { - "edges": { - "type": "array", + "id": { + "type": [ + "null", + "string" + ] + }, + "quantity": { + "type": [ + "null", + "integer" + ] + }, + "sku": { + "type": [ + "null", + "string" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "variantTitle": { + "type": [ + "null", + "string" + ] + }, + "variant": { + "type": [ + "null", + "object" + ], + "properties": { + "title": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + } + } + }, + "discountedTotalPriceSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "components": { + "type": [ + "null", + "array" + ], "items": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { - "node": { - "type": ["null", "object"], - "properties": { - "id": { "type": ["null", "string"] }, - "quantity": { "type": ["null", "integer"] }, - "sku": { "type": ["null", "string"] }, - "title": { "type": ["null", "string"] }, - "variantTitle": { "type": ["null", "string"] }, - "variant": { - "type": ["null", "object"], - "properties": { - "title": { "type": ["null", "string"] }, - "id": { "type": ["null", "string"] } - } - }, - "discountedTotalPriceSet": { - "type": ["null", "object"], - "properties": { - "presentmentMoney": { - "type": ["null", "object"], - "properties": { - "amount": { "type": ["null", "string"], "format": "singer.decimal" }, - "currencyCode": { "type": ["null", "string"] } - } - }, - "shopMoney": { - "type": ["null", "object"], - "properties": { - "amount": { "type": ["null", "string"], "format": "singer.decimal" }, - "currencyCode": { "type": ["null", "string"] } - } - } - } - }, - "components": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { "type": ["null", "string"] }, - "quantity": { "type": ["null", "integer"] }, - "title": { "type": ["null", "string"] }, - "variantTitle": { "type": ["null", "string"] } - } - } - }, - "customAttributes": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "key": { "type": ["null", "string"] }, - "value": { "type": ["null", "string"] } - } - } - }, - "product": { - "type": ["null", "object"], - "properties": { - "id": { "type": ["null", "string"] } - } - }, - "discountedUnitPriceSet": { - "type": ["null", "object"], - "properties": { - "presentmentMoney": { - "type": ["null", "object"], - "properties": { - "amount": { "type": ["null", "string"], "format": "singer.decimal" }, - "currencyCode": { "type": ["null", "string"] } - } - }, - "shopMoney": { - "type": ["null", "object"], - "properties": { - "amount": { "type": ["null", "string"], "format": "singer.decimal" }, - "currencyCode": { "type": ["null", "string"] } - } - } - } - }, - "discountedUnitPriceWithCodeDiscount": { - "type": ["null", "object"], - "properties": { - "presentmentMoney": { - "type": ["null", "object"], - "properties": { - "amount": { "type": ["null", "string"], "format": "singer.decimal" }, - "currencyCode": { "type": ["null", "string"] } - } - }, - "shopMoney": { - "type": ["null", "object"], - "properties": { - "amount": { "type": ["null", "string"], "format": "singer.decimal" }, - "currencyCode": { "type": ["null", "string"] } - } - } - } - }, - "originalTotalPriceSet": { - "type": ["null", "object"], - "properties": { - "presentmentMoney": { - "type": ["null", "object"], - "properties": { - "amount": { "type": ["null", "string"], "format": "singer.decimal" }, - "currencyCode": { "type": ["null", "string"] } - } - }, - "shopMoney": { - "type": ["null", "object"], - "properties": { - "amount": { "type": ["null", "string"], "format": "singer.decimal" }, - "currencyCode": { "type": ["null", "string"] } - } - } - } - }, - "originalUnitPriceSet": { - "type": ["null", "object"], - "properties": { - "presentmentMoney": { - "type": ["null", "object"], - "properties": { - "amount": { "type": ["null", "string"], "format": "singer.decimal" }, - "currencyCode": { "type": ["null", "string"] } - } - }, - "shopMoney": { - "type": ["null", "object"], - "properties": { - "amount": { "type": ["null", "string"], "format": "singer.decimal" }, - "currencyCode": { "type": ["null", "string"] } - } - } - } - } + "id": { + "type": [ + "null", + "string" + ] + }, + "quantity": { + "type": [ + "null", + "integer" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "variantTitle": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "customAttributes": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "key": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "product": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + }, + "discountedUnitPriceSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "discountedUnitPriceWithCodeDiscount": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "originalTotalPriceSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "originalUnitPriceSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] } } } @@ -328,20 +1050,79 @@ } }, "subtotalPriceSet": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "presentmentMoney": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { - "amount": { "type": ["null", "string"], "format": "singer.decimal" }, - "currencyCode": {"type": ["null", "string"]} + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } } }, "shopMoney": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "customer": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "lastOrder": { + "type": [ + "null", + "object" + ], "properties": { - "amount": { "type": ["null", "string"], "format": "singer.decimal" }, - "currencyCode": {"type": ["null", "string"]} + "id": { + "type": [ + "null", + "string" + ] + } } } } diff --git a/tap_shopify/schemas/customers.json b/tap_shopify/schemas/customers.json index 60280cb4..8cc77457 100644 --- a/tap_shopify/schemas/customers.json +++ b/tap_shopify/schemas/customers.json @@ -476,6 +476,20 @@ "null", "string" ] + }, + "lastOrder": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } } } } diff --git a/tap_shopify/schemas/definitions.json b/tap_shopify/schemas/definitions.json deleted file mode 100644 index 95156d5b..00000000 --- a/tap_shopify/schemas/definitions.json +++ /dev/null @@ -1,1452 +0,0 @@ -{ - "order_adjustments": { - "items": { - "properties": { - "amount_set": { - "type": [ - "null", - "object" - ] - }, - "tax_amount_set": { - "type": [ - "null", - "object" - ] - }, - "order_id": { - "type": [ - "null", - "integer" - ] - }, - "tax_amount": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "refund_id": { - "type": [ - "null", - "integer" - ] - }, - "amount": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "kind": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "reason": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "customer": { - "type": [ - "null", - "object" - ], - "properties": { - "last_order_name": { - "type": [ - "null", - "string" - ] - }, - "email": { - "type": [ - "null", - "string" - ] - }, - "multipass_identifier": { - "type": [ - "null", - "string" - ] - }, - "default_address": { - "type": [ - "null", - "object" - ], - "properties": { - "city": { - "type": [ - "null", - "string" - ] - }, - "address1": { - "type": [ - "null", - "string" - ] - }, - "zip": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "country_name": { - "type": [ - "null", - "string" - ] - }, - "province": { - "type": [ - "null", - "string" - ] - }, - "phone": { - "type": [ - "null", - "string" - ] - }, - "country": { - "type": [ - "null", - "string" - ] - }, - "first_name": { - "type": [ - "null", - "string" - ] - }, - "customer_id": { - "type": [ - "null", - "integer" - ] - }, - "default": { - "type": [ - "null", - "boolean" - ] - }, - "last_name": { - "type": [ - "null", - "string" - ] - }, - "country_code": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "province_code": { - "type": [ - "null", - "string" - ] - }, - "address2": { - "type": [ - "null", - "string" - ] - }, - "company": { - "type": [ - "null", - "string" - ] - } - } - }, - "orders_count": { - "type": [ - "null", - "integer" - ] - }, - "state": { - "type": [ - "null", - "string" - ] - }, - "verified_email": { - "type": [ - "null", - "boolean" - ] - }, - "total_spent": { - "type": [ - "null", - "string" - ] - }, - "last_order_id": { - "type": [ - "null", - "integer" - ] - }, - "first_name": { - "type": [ - "null", - "string" - ] - }, - "updated_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "note": { - "type": [ - "null", - "string" - ] - }, - "phone": { - "type": [ - "null", - "string" - ] - }, - "admin_graphql_api_id": { - "type": [ - "null", - "string" - ] - }, - "addresses": { - "type": [ - "null", - "array" - ], - "items": { - "type": [ - "null", - "object" - ], - "properties": { - "city": { - "type": [ - "null", - "string" - ] - }, - "address1": { - "type": [ - "null", - "string" - ] - }, - "zip": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "country_name": { - "type": [ - "null", - "string" - ] - }, - "province": { - "type": [ - "null", - "string" - ] - }, - "phone": { - "type": [ - "null", - "string" - ] - }, - "country": { - "type": [ - "null", - "string" - ] - }, - "first_name": { - "type": [ - "null", - "string" - ] - }, - "customer_id": { - "type": [ - "null", - "integer" - ] - }, - "default": { - "type": [ - "null", - "boolean" - ] - }, - "last_name": { - "type": [ - "null", - "string" - ] - }, - "country_code": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "province_code": { - "type": [ - "null", - "string" - ] - }, - "address2": { - "type": [ - "null", - "string" - ] - }, - "company": { - "type": [ - "null", - "string" - ] - } - } - } - }, - "last_name": { - "type": [ - "null", - "string" - ] - }, - "tags": { - "type": [ - "null", - "string" - ] - }, - "tax_exempt": { - "type": [ - "null", - "boolean" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "created_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "tax_exemptions": { - "type": [ - "null", - "array" - ], - "items": { - "type": [ - "null", - "string" - ] - } - }, - "email_marketing_consent": { - "type": [ - "null", - "object" - ], - "properties": { - "state": { - "type": [ - "null", - "string" - ] - }, - "opt_in_level": { - "type": [ - "null", - "string" - ] - }, - "consent_updated_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - } - } - }, - "sms_marketing_consent": { - "type": [ - "null", - "object" - ], - "properties": { - "state": { - "type": [ - "null", - "string" - ] - }, - "opt_in_level": { - "type": [ - "null", - "string" - ] - }, - "consent_updated_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "consent_collected_from": { - "type": [ - "null", - "string" - ] - } - } - } - } - }, - "customer_orders": { - "type": [ - "null", - "object" - ], - "properties": { - "currency": { - "type": [ - "null", - "string" - ] - }, - "email": { - "type": [ - "null", - "string" - ] - }, - "multipass_identifier": { - "type": [ - "null", - "string" - ] - }, - "default_address": { - "type": [ - "null", - "object" - ], - "properties": { - "city": { - "type": [ - "null", - "string" - ] - }, - "address1": { - "type": [ - "null", - "string" - ] - }, - "zip": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "country_name": { - "type": [ - "null", - "string" - ] - }, - "province": { - "type": [ - "null", - "string" - ] - }, - "phone": { - "type": [ - "null", - "string" - ] - }, - "country": { - "type": [ - "null", - "string" - ] - }, - "first_name": { - "type": [ - "null", - "string" - ] - }, - "customer_id": { - "type": [ - "null", - "integer" - ] - }, - "default": { - "type": [ - "null", - "boolean" - ] - }, - "last_name": { - "type": [ - "null", - "string" - ] - }, - "country_code": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "province_code": { - "type": [ - "null", - "string" - ] - }, - "address2": { - "type": [ - "null", - "string" - ] - }, - "company": { - "type": [ - "null", - "string" - ] - } - } - }, - "state": { - "type": [ - "null", - "string" - ] - }, - "verified_email": { - "type": [ - "null", - "boolean" - ] - }, - "first_name": { - "type": [ - "null", - "string" - ] - }, - "updated_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "note": { - "type": [ - "null", - "string" - ] - }, - "phone": { - "type": [ - "null", - "string" - ] - }, - "admin_graphql_api_id": { - "type": [ - "null", - "string" - ] - }, - "addresses": { - "type": [ - "null", - "array" - ], - "items": { - "type": [ - "null", - "object" - ], - "properties": { - "city": { - "type": [ - "null", - "string" - ] - }, - "address1": { - "type": [ - "null", - "string" - ] - }, - "zip": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "country_name": { - "type": [ - "null", - "string" - ] - }, - "province": { - "type": [ - "null", - "string" - ] - }, - "phone": { - "type": [ - "null", - "string" - ] - }, - "country": { - "type": [ - "null", - "string" - ] - }, - "first_name": { - "type": [ - "null", - "string" - ] - }, - "customer_id": { - "type": [ - "null", - "integer" - ] - }, - "default": { - "type": [ - "null", - "boolean" - ] - }, - "last_name": { - "type": [ - "null", - "string" - ] - }, - "country_code": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "province_code": { - "type": [ - "null", - "string" - ] - }, - "address2": { - "type": [ - "null", - "string" - ] - }, - "company": { - "type": [ - "null", - "string" - ] - } - } - } - }, - "last_name": { - "type": [ - "null", - "string" - ] - }, - "tags": { - "type": [ - "null", - "string" - ] - }, - "tax_exempt": { - "type": [ - "null", - "boolean" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "accepts_marketing": { - "type": [ - "null", - "boolean" - ] - }, - "accepts_marketing_updated_at": { - "anyOf": [ - { - "type": "string" , - "format": "date-time" - }, - { - "type": "string" - }, - { - "type": "null" - } - ] - }, - "created_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "tax_exemptions": { - "type": [ - "null", - "array" - ], - "items": { - "type": [ - "null", - "string" - ] - } - }, - "marketing_opt_in_level": { - "type": [ - "null", - "string" - ] - }, - "email_marketing_consent": { - "type": [ - "null", - "object" - ], - "properties": { - "state": { - "type": [ - "null", - "string" - ] - }, - "opt_in_level": { - "type": [ - "null", - "string" - ] - }, - "consent_updated_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - } - } - }, - "sms_marketing_consent": { - "type": [ - "null", - "object" - ], - "properties": { - "state": { - "type": [ - "null", - "string" - ] - }, - "opt_in_level": { - "type": [ - "null", - "string" - ] - }, - "consent_updated_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "consent_collected_from": { - "type": [ - "null", - "string" - ] - } - } - } - } - }, - "location": { - "properties": { - "country_code": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "address1": { - "type": [ - "null", - "string" - ] - }, - "city": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "address2": { - "type": [ - "null", - "string" - ] - }, - "province_code": { - "type": [ - "null", - "string" - ] - }, - "zip": { - "type": [ - "null", - "string" - ] - }, - "localized_province_name": { - "type": [ - "null", - "string" - ] - }, - "localized_country_name": { - "type": [ - "null", - "string" - ] - }, - "updated_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "province": { - "type": [ - "null", - "string" - ] - }, - "phone": { - "type": [ - "null", - "string" - ] - }, - "legacy": { - "type": [ - "null", - "boolean"] - }, - "created_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "country": { - "type": [ - "null", - "string" - ] - }, - "active": { - "type": [ - "null", - "boolean"] - }, - "admin_graphql_api_id": { - "type": [ - "null", - "string"] - }, - "country_name": { - "type": [ - "null", - "string"] - } - }, - "type": [ - "null", - "object" - ] - }, - "line_item": { - "properties": { - "applied_discounts": { - "items": { - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "total_discount_set": {}, - "pre_tax_price_set": {}, - "price_set": {}, - "grams": { - "type": [ - "null", - "integer" - ] - }, - "compare_at_price": { - "type": [ - "null", - "string" - ] - }, - "destination_location_id": { - "type": [ - "null", - "integer" - ] - }, - "key": { - "type": [ - "null", - "string" - ] - }, - "line_price": { - "type": [ - "null", - "string" - ] - }, - "origin_location_id": { - "type": [ - "null", - "integer" - ] - }, - "applied_discount": { - "type": [ - "null", - "integer" - ] - }, - "fulfillable_quantity": { - "type": [ - "null", - "integer" - ] - }, - "variant_title": { - "type": [ - "null", - "string" - ] - }, - "properties": { - "anyOf": [ - { - "items": { - "properties": { - "name": { - "type": [ - "null", - "string" - ] - }, - "value": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - { - "properties": {}, - "type": ["null", "object"] - } - ] - }, - "tax_code": { - "type": [ - "null", - "string" - ] - }, - "discount_allocations": { - "items": { - "properties": { - "discount_application_index": { - "type": [ - "null", - "integer" - ] - }, - "amount_set": {}, - "amount": { - "type": [ - "null", - "number" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "admin_graphql_api_id": { - "type": [ - "null", - "string" - ] - }, - "pre_tax_price": { - "type": [ - "null", - "number" - ] - }, - "sku": { - "type": [ - "null", - "string" - ] - }, - "product_exists": { - "type": [ - "null", - "boolean" - ] - }, - "total_discount": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "fulfillment_status": { - "type": [ - "null", - "string" - ] - }, - "gift_card": { - "type": [ - "null", - "boolean" - ] - }, - "id": { - "type": ["null", "integer", "string"] - }, - "taxable": { - "type": [ - "null", - "boolean" - ] - }, - "vendor": { - "type": [ - "null", - "string" - ] - }, - "tax_lines": { - "$ref": "definitions.json#/tax_lines" - }, - "origin_location": { - "$ref": "definitions.json#/location" - }, - "price": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "requires_shipping": { - "type": [ - "null", - "boolean" - ] - }, - "fulfillment_service": { - "type": [ - "null", - "string" - ] - }, - "variant_inventory_management": { - "type": [ - "null", - "string" - ] - }, - "title": { - "type": [ - "null", - "string" - ] - }, - "destination_location": { - "$ref": "definitions.json#/location" - }, - "quantity": { - "type": [ - "null", - "integer" - ] - }, - "product_id": { - "type": [ - "null", - "integer" - ] - }, - "variant_id": { - "type": [ - "null", - "integer" - ] - }, - "duties": {} - }, - "patternProperties": {".+": {}}, - "type": [ - "null", - "object" - ] - }, - "line_items": { - "items": { - "$ref": "definitions.json#/line_item" - }, - "type": [ - "null", - "array" - ] - }, - "image": { - "properties": { - "updated_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "created_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "variant_ids": { - "type": [ - "null", - "array" - ], - "items": { - "type": [ - "null", - "integer" - ] - } - }, - "height": { - "type": [ - "null", - "integer" - ] - }, - "alt": { - "type": [ - "null", - "string" - ] - }, - "src": { - "type": [ - "null", - "string" - ] - }, - "position": { - "type": [ - "null", - "integer" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "admin_graphql_api_id": { - "type": [ - "null", - "string" - ] - }, - "width": { - "type": [ - "null", - "integer" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "tax_lines": { - "items": { - "properties": { - "price_set": { - }, - "price": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "title": { - "type": [ - "null", - "string" - ] - }, - "rate": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "compare_at": { - "type": [ - "null", - "string" - ] - }, - "position": { - "type": [ - "null", - "integer" - ] - }, - "source": { - "type": [ - "null", - "string" - ] - }, - "zone": { - "type": [ - "null", - "string" - ] - }, - "channel_liable": { - "type": [ - "null", - "boolean" - ] - } - }, - "patternProperties": {".+": {}}, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - } -} diff --git a/tap_shopify/schemas/order_refunds.json b/tap_shopify/schemas/order_refunds.json index 02565300..a08d0b63 100644 --- a/tap_shopify/schemas/order_refunds.json +++ b/tap_shopify/schemas/order_refunds.json @@ -866,6 +866,132 @@ } } }, + "orderAdjustments": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "amountSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "reason": { + "type": [ + "null", + "string" + ] + }, + "taxAmountSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + } + } + } + }, "updatedAt": { "type": [ "null", diff --git a/tap_shopify/schemas/orders.json b/tap_shopify/schemas/orders.json index 46790777..9a2d1803 100644 --- a/tap_shopify/schemas/orders.json +++ b/tap_shopify/schemas/orders.json @@ -1499,6 +1499,12 @@ "integer" ] }, + "title": { + "type": [ + "null", + "string" + ] + }, "targetType": { "type": [ "null", @@ -1793,9 +1799,320 @@ } } } + }, + "originalUnitPriceSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + } + } + }, + "unfulfilledDiscountedTotalSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + } + } + }, + "unfulfilledOriginalTotalSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + } + } + }, + "variant": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } } } } + }, + "shippingLine": { + "type": ["null", "object"], + "properties": { + "carrierIdentifier": { "type": ["null", "string"] }, + "code": { "type": ["null", "string"] }, + "currentDiscountedPriceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "custom": { "type": ["null", "boolean"] }, + "deliveryCategory": { "type": ["null", "string"] }, + "discountAllocations": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "allocatedAmountSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "discountApplication": { + "type": ["null", "object"], + "properties": { + "__typename": { "type": ["null", "string"] }, + "allocationMethod": { "type": ["null", "string"] }, + "index": { "type": ["null", "integer"] }, + "targetSelection": { "type": ["null", "string"] }, + "targetType": { "type": ["null", "string"] }, + "title": { "type": ["null", "string"] }, + "code": { "type": ["null", "string"] }, + "description": { "type": ["null", "string"] }, + "value": { + "type": ["null", "object"], + "properties": { + "__typename": { "type": ["null", "string"] }, + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] }, + "percentage": { "type": ["null", "number"] } + } + } + } + } + } + } + }, + "discountedPriceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "id": { "type": ["null", "string"] }, + "isRemoved": { "type": ["null", "boolean"] }, + "originalPriceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "phone": { "type": ["null", "string"] }, + "shippingRateHandle": { "type": ["null", "string"] }, + "source": { "type": ["null", "string"] }, + "taxLines": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "channelLiable": { "type": ["null", "boolean"] }, + "priceSet": { + "type": ["null", "object"], + "properties": { + "presentmentMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + }, + "shopMoney": { + "type": ["null", "object"], + "properties": { + "amount": { "type": ["null", "string"], "format": "singer.decimal" }, + "currencyCode": { "type": ["null", "string"] } + } + } + } + }, + "rate": { "type": ["null", "number"] }, + "ratePercentage": { "type": ["null", "string"] }, + "source": { "type": ["null", "string"] }, + "title": { "type": ["null", "string"] } + } + } + }, + "title": { "type": ["null", "string"] } + } } } } diff --git a/tap_shopify/schemas/product_variants.json b/tap_shopify/schemas/product_variants.json index 81b67ef1..e9518bf8 100644 --- a/tap_shopify/schemas/product_variants.json +++ b/tap_shopify/schemas/product_variants.json @@ -28,11 +28,11 @@ }, "compareAtPrice": { "type": [ - "null", - "string" + "null", + "string" ], "format": "singer.decimal" - }, + }, "displayName": { "type": [ "null", @@ -98,11 +98,11 @@ }, "price": { "type": [ - "null", - "string" + "null", + "string" ], "format": "singer.decimal" - }, + }, "requiresComponents": { "type": [ "null", @@ -171,6 +171,35 @@ "null", "string" ] + }, + "measurement": { + "type": [ + "null", + "object" + ], + "properties": { + "weight": { + "type": [ + "null", + "object" + ], + "properties": { + "unit": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + } + } + } + } } } } diff --git a/tap_shopify/schemas/transactions.json b/tap_shopify/schemas/transactions.json index 23afde96..53ebd02d 100644 --- a/tap_shopify/schemas/transactions.json +++ b/tap_shopify/schemas/transactions.json @@ -422,6 +422,193 @@ } } } + }, + "fees": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "rate": { + "type": [ + "null", + "number" + ] + }, + "rateName": { + "type": [ + "null", + "string" + ] + }, + "taxAmount": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "type": { + "type": [ + "null", + "string" + ] + }, + "flatFeeName": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "flatFee": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + } + }, + "manuallyCapturable": { + "type": [ + "null", + "boolean" + ] + }, + "paymentDetails": { + "type": [ + "null", + "object" + ], + "properties": { + "avsResultCode": { + "type": [ + "null", + "string" + ] + }, + "bin": { + "type": [ + "null", + "string" + ] + }, + "company": { + "type": [ + "null", + "string" + ] + }, + "cvvResultCode": { + "type": [ + "null", + "string" + ] + }, + "expirationMonth": { + "type": [ + "null", + "integer" + ] + }, + "expirationYear": { + "type": [ + "null", + "integer" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "number": { + "type": [ + "null", + "string" + ] + }, + "paymentMethodName": { + "type": [ + "null", + "string" + ] + }, + "wallet": { + "type": [ + "null", + "string" + ] + }, + "paymentDescriptor": { + "type": [ + "null", + "string" + ] + } + } } } } diff --git a/tap_shopify/streams/abandoned_checkouts.py b/tap_shopify/streams/abandoned_checkouts.py index f0867b66..2b356cb2 100644 --- a/tap_shopify/streams/abandoned_checkouts.py +++ b/tap_shopify/streams/abandoned_checkouts.py @@ -263,6 +263,12 @@ def get_query(self): currencyCode } } + customer { + id + lastOrder { + id + } + } } } pageInfo { diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index 8737b4b5..597057db 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -272,6 +272,7 @@ def get_query_params(self, updated_at_min, updated_at_max, cursor=None): "query": f"{rkey}:>='{updated_at_min}' AND {rkey}:<'{updated_at_max}'", "first": self.results_per_page, } + if cursor: params["after"] = cursor return params @@ -286,6 +287,7 @@ def get_objects(self): """ last_updated_at = self.get_bookmark() + current_bookmark = last_updated_at sync_start = utils.now().replace(microsecond=0) while last_updated_at < sync_start: @@ -301,27 +303,20 @@ def get_objects(self): for edge in data.get("edges"): obj = self.transform_object(edge.get("node")) + replication_value = utils.strptime_to_utc(obj[self.replication_key]) + current_bookmark = max(current_bookmark, replication_value) yield obj page_info = data.get("pageInfo") cursor , has_next_page = page_info.get("endCursor"), page_info.get("hasNextPage") last_updated_at = query_end + # Update bookmark to the latest value, but not beyond sync start time + max_bookmark_value = min(sync_start, current_bookmark) + self.update_bookmark(utils.strftime(max_bookmark_value)) def sync(self): """ Default implementation for sync method """ - start_time = utils.now().replace(microsecond=0) - max_bookmark_value = self.get_bookmark() - - for obj in self.get_objects(): - max_bookmark_value = max( - max_bookmark_value, - utils.strptime_to_utc(obj[self.replication_key]) - ) - yield obj - - # Update bookmark to the latest value, but not beyond sync start time - max_bookmark_value = min(start_time, max_bookmark_value) - self.update_bookmark(utils.strftime(max_bookmark_value)) + yield from self.get_objects() diff --git a/tap_shopify/streams/customers.py b/tap_shopify/streams/customers.py index 2ff65502..f589abd6 100644 --- a/tap_shopify/streams/customers.py +++ b/tap_shopify/streams/customers.py @@ -17,96 +17,99 @@ def get_query(self): """ return """ query Customers($first: Int!, $after: String, $query: String) { - customers(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { - edges { - node { - email - multipassIdentifier - defaultAddress { - city - address1 - zip - id - province - phone - country - firstName - lastName - countryCodeV2 - name - provinceCode - address2 - company - timeZone - validationResultSummary - latitude - longitude - coordinatesValidated - formattedArea - } - numberOfOrders - state - verifiedEmail - firstName - updatedAt - note - phone - addresses(first: 250) { - city - address1 - zip - id - province - phone - country - firstName - lastName - countryCodeV2 - name - provinceCode - address2 - company - timeZone - validationResultSummary - latitude - longitude - coordinatesValidated - formattedArea - } - lastName - tags - taxExempt - id - createdAt - taxExemptions - emailMarketingConsent { - consentUpdatedAt - marketingOptInLevel - marketingState - } - smsMarketingConsent { - consentCollectedFrom - consentUpdatedAt - marketingOptInLevel - marketingState - } - validEmailAddress - productSubscriberStatus - amountSpent { - amount - currencyCode - } - dataSaleOptOut - displayName - locale - lifetimeDuration - } + customers(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { + edges { + node { + email + multipassIdentifier + defaultAddress { + city + address1 + zip + id + province + phone + country + firstName + lastName + countryCodeV2 + name + provinceCode + address2 + company + timeZone + validationResultSummary + latitude + longitude + coordinatesValidated + formattedArea } - pageInfo { - endCursor - hasNextPage + numberOfOrders + state + verifiedEmail + firstName + updatedAt + note + phone + addresses(first: 250) { + city + address1 + zip + id + province + phone + country + firstName + lastName + countryCodeV2 + name + provinceCode + address2 + company + timeZone + validationResultSummary + latitude + longitude + coordinatesValidated + formattedArea } + lastName + tags + taxExempt + id + createdAt + taxExemptions + emailMarketingConsent { + consentUpdatedAt + marketingOptInLevel + marketingState + } + smsMarketingConsent { + consentCollectedFrom + consentUpdatedAt + marketingOptInLevel + marketingState + } + validEmailAddress + productSubscriberStatus + amountSpent { + amount + currencyCode + } + dataSaleOptOut + displayName + locale + lifetimeDuration + lastOrder { + id + } + } + } + pageInfo { + endCursor + hasNextPage } + } } """ diff --git a/tap_shopify/streams/inventory_levels.py b/tap_shopify/streams/inventory_levels.py index e202ab81..a778d140 100644 --- a/tap_shopify/streams/inventory_levels.py +++ b/tap_shopify/streams/inventory_levels.py @@ -52,6 +52,7 @@ def get_objects(self): dict: The transformed object. """ last_updated_at = self.get_bookmark() + current_bookmark = last_updated_at sync_start = utils.now().replace(microsecond=0) while last_updated_at < sync_start: @@ -72,6 +73,8 @@ def get_objects(self): child_edges = node.get(self.child_data_key, {}).get("edges", []) for child_obj in child_edges: obj = self.transform_object(child_obj.get("node")) + replication_value = utils.strptime_to_utc(obj[self.replication_key]) + current_bookmark = max(current_bookmark, replication_value) yield obj # Check if more child pages are needed @@ -85,6 +88,10 @@ def get_objects(self): parent_id, child_cursor, query_params["query"] ): transformed_obj = self.transform_object(child_obj.get("node")) + replication_value = utils.strptime_to_utc( + transformed_obj[self.replication_key] + ) + current_bookmark = max(current_bookmark, replication_value) yield transformed_obj page_info = data.get("pageInfo", {}) @@ -92,6 +99,9 @@ def get_objects(self): has_next_page = page_info.get("hasNextPage") last_updated_at = query_end + # Update bookmark to the latest value, but not beyond sync start time + max_bookmark_value = min(sync_start, current_bookmark) + self.update_bookmark(utils.strftime(max_bookmark_value)) # pylint: disable=C0301 def get_query(self): diff --git a/tap_shopify/streams/metafields.py b/tap_shopify/streams/metafields.py index d5c32d2f..261e6299 100644 --- a/tap_shopify/streams/metafields.py +++ b/tap_shopify/streams/metafields.py @@ -4,7 +4,6 @@ from singer import utils, get_logger, metrics from tap_shopify.streams.base import Stream -from tap_shopify.context import Context LOGGER = get_logger() @@ -69,9 +68,10 @@ def get_objects(self): Main iterator to yield metafield objects. """ sync_start = utils.now().replace(microsecond=0) - # Will always fetch the data from the start date and filter it in the code - # Shopify doesn't support filtering by updated_at for metafields - last_updated_at = utils.strptime_with_tz(Context.config["start_date"]) + + # Use the lastest bookmark value as the starting point, contrary to the earlier + # implementation where the start date was used. + last_updated_at = self.get_bookmark() while last_updated_at < sync_start: date_window_end = last_updated_at + timedelta(days=self.date_window_size) diff --git a/tap_shopify/streams/order_refunds.py b/tap_shopify/streams/order_refunds.py index 933489b0..905809bc 100644 --- a/tap_shopify/streams/order_refunds.py +++ b/tap_shopify/streams/order_refunds.py @@ -92,6 +92,44 @@ def transform_lineitems(self, data): return lineitems + def transform_orderadjustments(self, data): + """ + Transforms the order adjustments data by extracting order IDs and handling pagination. + + Args: + data (dict): Order data. + + Returns: + list: List of adjustments. + """ + + orderadjustments = [ + node for item in data["orderAdjustments"]["edges"] + if (node := item.get("node")) + ] + + # Handle pagination + page_info = data["orderAdjustments"].get("pageInfo", {}) + order_parent = data.get("order").get("id") + while page_info.get("hasNextPage"): + params = { + "first": self.results_per_page, + "query": f"id:{order_parent.split('/')[-1]}", + "orderadjustments_after": page_info.get("endCursor"), + } + + # Fetch the next page of data + response = self.call_api(params) + nodes = response.get("edges", [])[0].get("node", {}) + refunds_data = nodes.get("refunds")[0] + orderadjustments.extend( + node for item in refunds_data["orderAdjustments"]["edges"] + if (node := item.get("node")) + ) + page_info = refunds_data.get("pageInfo", {}) + + return orderadjustments + def transform_object(self, obj): """ Transform refund objects by extracting refund line items from edges. @@ -105,6 +143,9 @@ def transform_object(self, obj): if obj.get("refundLineItems"): obj["refundLineItems"] = self.transform_lineitems(obj) + + if obj.get("orderAdjustments"): + obj["orderAdjustments"] = self.transform_orderadjustments(obj) return obj def sync(self): @@ -134,7 +175,7 @@ def get_query(self): str: GraphQL query string. """ # pylint: disable=line-too-long - return """query GetOrderRefunds($first: Int!, $after: String, $query: String, $childafter: String) { + return """query GetOrderRefunds($first: Int!, $after: String, $query: String, $childafter: String, $orderadjustments_after: String) { orders(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { edges { node { @@ -146,6 +187,38 @@ def get_query(self): order { id } + orderAdjustments(first: 100, after: $orderadjustments_after) { + edges { + node { + amountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + id + reason + taxAmountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + } + } + pageInfo { + endCursor + hasNextPage + } + } refundLineItems(first: 50, after: $childafter) { edges { node { diff --git a/tap_shopify/streams/orders.py b/tap_shopify/streams/orders.py index 669779a3..3d02ec56 100644 --- a/tap_shopify/streams/orders.py +++ b/tap_shopify/streams/orders.py @@ -9,6 +9,33 @@ class Orders(Stream): data_key = "orders" replication_key = "updatedAt" + # pylint: disable=W0221,fixme + def get_query_params(self, updated_at_min, updated_at_max, cursor=None): + """ + Construct query parameters for GraphQL requests. + + Args: + updated_at_min (str): Minimum updated_at timestamp. + updated_at_max (str): Maximum updated_at timestamp. + cursor (str): Pagination cursor, if any. + + Returns: + dict: Dictionary of query parameters. + """ + rkey = self.camel_to_snake(self.replication_key) + + # TODO: In future once the dynamic query generation logic is setup remove the below + # condition for the orders stream. As by default we will ask the customers to select + # few fields or reduce the page size + params = { + "query": f"{rkey}:>='{updated_at_min}' AND {rkey}:<'{updated_at_max}'", + "first": self.results_per_page if self.results_per_page <= 150 else 150, + } + + if cursor: + params["after"] = cursor + return params + def transform_lineitems(self, data): """ Transforms the order lineitems data by extracting order IDs and handling pagination. @@ -24,11 +51,14 @@ def transform_lineitems(self, data): if (node := item.get("node")) ] + # TODO: In future once the dynamic query generation logic is setup remove the below + # condition for the orders stream. Moving forward we should ask the customers to select + # few fields or reduce the page size # Handle pagination page_info = data["lineItems"].get("pageInfo", {}) while page_info.get("hasNextPage"): params = { - "first": self.results_per_page, + "first": self.results_per_page if self.results_per_page <= 150 else 150, "query": f"id:{data['id'].split('/')[-1]}", "childafter": page_info.get("endCursor"), } @@ -66,162 +96,576 @@ def get_query(self): str: GraphQL query string. """ return """ - query orders($first: Int!, $after: String, $query: String, $childafter: String) { + query orders($first: Int!, $after: String, $query: String, $childafter: String) { orders(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { edges { - node { - additionalFees { - id - name - price { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - taxLines { - channelLiable - priceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - rate - ratePercentage - source - title - } + node { + additionalFees { + id + name + price { + presentmentMoney { + amount + currencyCode } - app { - id - name - icon { - id - } - } - billingAddress { - address1 - address2 - city - company - coordinatesValidated - country - countryCodeV2 - firstName - formattedArea - id - lastName - latitude - longitude - name - phone - province - provinceCode - timeZone - validationResultSummary - zip - } - billingAddressMatchesShippingAddress - canMarkAsPaid - canNotifyCustomer - cancelReason - cancellation { - staffNote - } - cancelledAt - capturable - cartDiscountAmountSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } + shopMoney { + amount + currencyCode } - channelInformation { - id - channelId + } + taxLines { + channelLiable + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + rate + ratePercentage + source + title + } + } + app { + id + name + icon { + id + } + } + billingAddress { + address1 + address2 + city + company + coordinatesValidated + country + countryCodeV2 + firstName + formattedArea + id + lastName + latitude + longitude + name + phone + province + provinceCode + timeZone + validationResultSummary + zip + } + billingAddressMatchesShippingAddress + canMarkAsPaid + canNotifyCustomer + cancelReason + cancellation { + staffNote + } + cancelledAt + capturable + cartDiscountAmountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + channelInformation { + id + channelId + } + clientIp + closed + closedAt + confirmationNumber + confirmed + createdAt + currencyCode + currentCartDiscountAmountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentShippingPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentSubtotalLineItemsQuantity + currentSubtotalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentTotalAdditionalFeesSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentTotalDiscountsSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentTotalDutiesSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentTotalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentTotalTaxSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentTotalWeight + customer { + id + email + firstName + lastName + addresses { + address1 + address2 + city + countryCodeV2 + country + company + firstName + lastName + id + name + phone + province + provinceCode + zip + } + state + verifiedEmail + updatedAt + taxExempt + tags + taxExemptions + note + multipassIdentifier + createdAt + defaultAddress { + address1 + address2 + city + company + country + countryCodeV2 + firstName + id + lastName + name + province + phone + provinceCode + zip + } + } + customerJourneySummary { + lastVisit { + landingPage + referrerUrl + } + } + merchantOfRecordApp { + id + } + customerAcceptsMarketing + customerLocale + discountCodes + discountCode + displayFinancialStatus + displayFulfillmentStatus + disputes { + id + initiatedAs + status + } + dutiesIncluded + email + edited + estimatedTaxes + fulfillable + fullyPaid + hasTimelineComment + fulfillmentsCount { + count + precision + } + id + legacyResourceId + merchantBusinessEntity { + address { + address1 + address2 + city + countryCode + province + zip + } + companyName + displayName + id + primary + } + name + note + netPaymentSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + originalTotalAdditionalFeesSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + originalTotalDutiesSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + originalTotalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + paymentGatewayNames + phone + poNumber + presentmentCurrencyCode + processedAt + refundable + refundDiscrepancySet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + registeredSourceUrl + requiresShipping + restockable + returnStatus + shippingAddress { + address1 + address2 + city + company + coordinatesValidated + country + countryCodeV2 + firstName + formattedArea + id + lastName + latitude + longitude + name + phone + province + provinceCode + timeZone + validationResultSummary + zip + } + shopifyProtect { + eligibility { + status + } + status + } + sourceIdentifier + sourceName + statusPageUrl + subtotalLineItemsQuantity + subtotalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + tags + taxExempt + taxLines { + channelLiable + priceSet { + presentmentMoney { + amount + currencyCode } - clientIp - closed - closedAt - confirmationNumber - confirmed - createdAt + shopMoney { + amount currencyCode - currentCartDiscountAmountSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } } - currentShippingPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } + } + rate + ratePercentage + source + title + } + taxesIncluded + test + totalCapturableSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalCashRoundingAdjustment { + paymentSet { + presentmentMoney { + amount + currencyCode } - currentSubtotalLineItemsQuantity - currentSubtotalPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } + shopMoney { + amount + currencyCode } - currentTotalAdditionalFeesSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } + } + refundSet { + presentmentMoney { + amount + currencyCode } - currentTotalDiscountsSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } + shopMoney { + amount + currencyCode } - currentTotalDutiesSet { + } + } + totalDiscountsSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalOutstandingSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalReceivedSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalRefundedSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalRefundedShippingSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalShippingPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalTaxSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalTipReceivedSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalWeight + transactionsCount { + count + precision + } + unpaid + updatedAt + fulfillments(first: 250) { + id + name + status + totalQuantity + updatedAt + createdAt + deliveredAt + estimatedDeliveryAt + requiresShipping + inTransitAt + trackingInfo(first: 250) { + number + company + url + } + service { + serviceName + id + handle + trackingSupport + type + permitsSkuSharing + inventoryManagement + } + } + lineItems(first: 25, after: $childafter) { + edges { + node { + id + vendor + quantity + title + requiresShipping + originalTotalSet { presentmentMoney { - amount - currencyCode + currencyCode + amount } shopMoney { - amount - currencyCode + amount + currencyCode } } - currentTotalPriceSet { + taxLines(first: 250) { + priceSet { presentmentMoney { amount currencyCode @@ -230,115 +674,31 @@ def get_query(self): amount currencyCode } + } + rate + title + source + channelLiable } - currentTotalTaxSet { + taxable + isGiftCard + name + discountedTotalSet { presentmentMoney { - amount - currencyCode + amount + currencyCode } shopMoney { - amount - currencyCode + amount + currencyCode } } - currentTotalWeight - customer { - id - email - firstName - lastName - addresses { - address1 - address2 - city - countryCodeV2 - country - company - firstName - lastName - id - name - phone - province - provinceCode - zip - } - state - verifiedEmail - updatedAt - taxExempt - tags - taxExemptions - note - multipassIdentifier - createdAt - defaultAddress { - address1 - address2 - city - company - country - countryCodeV2 - firstName - id - lastName - name - province - phone - provinceCode - zip - } - } - customerJourneySummary { - lastVisit { - landingPage - referrerUrl - } - } - merchantOfRecordApp { - id - } - customerAcceptsMarketing - customerLocale - discountCodes - discountCode - displayFinancialStatus - displayFulfillmentStatus - disputes { + sku + product { id - initiatedAs - status - } - dutiesIncluded - email - edited - estimatedTaxes - fulfillable - fullyPaid - hasTimelineComment - fulfillmentsCount { - count - precision } - id - legacyResourceId - merchantBusinessEntity { - address { - address1 - address2 - city - countryCode - province - zip - } - companyName - displayName - id - primary - } - name - note - netPaymentSet { + discountAllocations { + allocatedAmountSet { presentmentMoney { amount currencyCode @@ -347,415 +707,291 @@ def get_query(self): amount currencyCode } - } - originalTotalAdditionalFeesSet { - presentmentMoney { - amount - currencyCode } - shopMoney { - amount - currencyCode + discountApplication { + index + targetType + targetSelection + allocationMethod + ... on AutomaticDiscountApplication { + title } - } - originalTotalDutiesSet { - presentmentMoney { - amount - currencyCode + ... on ManualDiscountApplication { + title } - shopMoney { - amount - currencyCode + ... on ScriptDiscountApplication { + title } - } - originalTotalPriceSet { - presentmentMoney { + value { + ... on MoneyV2 { + __typename amount currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } } - shopMoney { - amount - currencyCode } } - paymentGatewayNames - phone - poNumber - presentmentCurrencyCode - processedAt - refundable - refundDiscrepancySet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } + customAttributes { + key + value } - registeredSourceUrl - requiresShipping - restockable - returnStatus - shippingAddress { - address1 - address2 - city - company - coordinatesValidated - country - countryCodeV2 - firstName - formattedArea - id - lastName - latitude - longitude - name - phone - province - provinceCode - timeZone - validationResultSummary - zip - } - shopifyProtect { - eligibility { - status - } - status - } - sourceIdentifier - sourceName - statusPageUrl - subtotalLineItemsQuantity - subtotalPriceSet { + totalDiscountSet { presentmentMoney { - amount - currencyCode + amount + currencyCode } shopMoney { - amount - currencyCode + amount + currencyCode } } - tags - taxExempt - taxLines { - channelLiable - priceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } + duties { + harmonizedSystemCode + id + taxLines { rate - ratePercentage source title - } - taxesIncluded - test - totalCapturableSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { + channelLiable + priceSet { + presentmentMoney { amount currencyCode - } - } - totalCashRoundingAdjustment { - paymentSet { - presentmentMoney { - amount - currencyCode } shopMoney { - amount - currencyCode - } - } - refundSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - } - totalDiscountsSet { - presentmentMoney { amount currencyCode + } } - shopMoney { - amount - currencyCode } + countryCodeOfOrigin } - totalOutstandingSet { + discountedUnitPriceSet { presentmentMoney { - amount - currencyCode + amount + currencyCode } shopMoney { - amount - currencyCode + amount + currencyCode } } - totalPriceSet { + originalUnitPriceSet { presentmentMoney { - amount - currencyCode + amount + currencyCode } shopMoney { - amount - currencyCode + amount + currencyCode } } - totalReceivedSet { + unfulfilledDiscountedTotalSet { presentmentMoney { - amount - currencyCode + amount + currencyCode } shopMoney { - amount - currencyCode + amount + currencyCode } } - totalRefundedSet { + unfulfilledOriginalTotalSet { presentmentMoney { - amount - currencyCode + amount + currencyCode } shopMoney { - amount - currencyCode + amount + currencyCode } } - totalRefundedShippingSet { - presentmentMoney { + variant { + id + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + shippingLine { + carrierIdentifier + code + currentDiscountedPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + custom + deliveryCategory + discountAllocations { + allocatedAmountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + discountApplication { + allocationMethod + index + targetSelection + targetType + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + ... on AutomaticDiscountApplication { + __typename + allocationMethod + index + targetSelection + targetType + title + value { + ... on MoneyV2 { + __typename amount currencyCode } - shopMoney { - amount - currencyCode + ... on PricingPercentageValue { + __typename + percentage + } } } - totalShippingPriceSet { - presentmentMoney { + ... on DiscountCodeApplication { + __typename + allocationMethod + code + index + targetSelection + targetType + value { + ... on MoneyV2 { + __typename amount currencyCode } - shopMoney { - amount - currencyCode + ... on PricingPercentageValue { + __typename + percentage + } } } - totalTaxSet { - presentmentMoney { + ... on ManualDiscountApplication { + description + allocationMethod + index + targetSelection + targetType + title + value { + ... on MoneyV2 { + __typename amount currencyCode } - shopMoney { - amount - currencyCode + ... on PricingPercentageValue { + __typename + percentage + } } } - totalTipReceivedSet { - presentmentMoney { + ... on ScriptDiscountApplication { + __typename + allocationMethod + index + targetSelection + targetType + title + value { + ... on MoneyV2 { + __typename amount currencyCode } - shopMoney { - amount - currencyCode + ... on PricingPercentageValue { + __typename + percentage + } } } - totalWeight - transactionsCount { - count - precision } - unpaid - updatedAt - fulfillments(first: 250) { - id - name - status - totalQuantity - updatedAt - createdAt - deliveredAt - estimatedDeliveryAt - requiresShipping - inTransitAt - trackingInfo(first: 250) { - number - company - url - } - service { - serviceName - id - handle - trackingSupport - type - permitsSkuSharing - inventoryManagement - } - } - lineItems(first: 25, after: $childafter) { - edges { - node { - id - vendor - quantity - title - requiresShipping - originalTotalSet { - presentmentMoney { - currencyCode - amount - } - shopMoney { - amount - currencyCode - } - } - taxLines(first: 250) { - priceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - rate - title - source - channelLiable - } - taxable - isGiftCard - name - discountedTotalSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - sku - product { - id - } - discountAllocations { - allocatedAmountSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - discountApplication { - index - targetType - targetSelection - allocationMethod - value { - ... on MoneyV2 { - __typename - amount - currencyCode - } - ... on PricingPercentageValue { - __typename - percentage - } - } - } - } - customAttributes { - key - value - } - totalDiscountSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - duties { - harmonizedSystemCode - id - taxLines { - rate - source - title - channelLiable - priceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - } - countryCodeOfOrigin - } - discountedUnitPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - } - } - pageInfo { - hasNextPage - endCursor - } + } + discountedPriceSet { + presentmentMoney { + amount + currencyCode } + shopMoney { + amount + currencyCode + } + } + id + isRemoved + originalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + phone + shippingRateHandle + source + taxLines { + channelLiable + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + rate + ratePercentage + source + title + } + title } } + } pageInfo { - endCursor - hasNextPage + endCursor + hasNextPage } } - } + } """ diff --git a/tap_shopify/streams/product_variants.py b/tap_shopify/streams/product_variants.py index 8760355f..4f18cbe8 100644 --- a/tap_shopify/streams/product_variants.py +++ b/tap_shopify/streams/product_variants.py @@ -56,8 +56,18 @@ def get_query(self): taxable title updatedAt - product { id } - inventoryItem { id } + product { + id + } + inventoryItem { + id + measurement { + weight { + unit + value + } + } + } } } pageInfo { diff --git a/tap_shopify/streams/transactions.py b/tap_shopify/streams/transactions.py index 96e11117..a4f9f8a4 100644 --- a/tap_shopify/streams/transactions.py +++ b/tap_shopify/streams/transactions.py @@ -80,100 +80,141 @@ def get_query(self): """ return """ query GetTransactions($first: Int!, $after: String, $query: String) { - orders(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { - edges { - node { - transactions(first: 100) { - accountNumber - amountRoundingSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - amountSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - authorizationCode - authorizationExpiresAt - createdAt - errorCode - formattedGateway - gateway - id - kind - manualPaymentGateway - maximumRefundableV2 { - amount - currencyCode - } - multiCapturable - order { - id - } - parentTransaction { - accountNumber - createdAt - id - status - paymentId - processedAt - amountSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - } - paymentId - processedAt - receiptJson - settlementCurrency - settlementCurrencyRate - shopifyPaymentsSet { - extendedAuthorizationSet { - extendedAuthorizationExpiresAt - standardAuthorizationExpiresAt - } - refundSet { - acquirerReferenceNumber - } - } - status - test - totalUnsettledSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - } + orders(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { + edges { + node { + transactions(first: 100) { + accountNumber + amountRoundingSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + amountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + authorizationCode + authorizationExpiresAt + createdAt + errorCode + formattedGateway + gateway + id + kind + manualPaymentGateway + maximumRefundableV2 { + amount + currencyCode + } + multiCapturable + order { + id + } + parentTransaction { + accountNumber + createdAt + id + status + paymentId + processedAt + amountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + } + paymentId + processedAt + receiptJson + settlementCurrency + settlementCurrencyRate + shopifyPaymentsSet { + extendedAuthorizationSet { + extendedAuthorizationExpiresAt + standardAuthorizationExpiresAt + } + refundSet { + acquirerReferenceNumber + } + } + status + test + totalUnsettledSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + fees { + id + rate + rateName + taxAmount { + amount + currencyCode + } + type + flatFeeName + amount { + amount + currencyCode + } + flatFee { + amount + currencyCode + } + } + manuallyCapturable + paymentDetails { + ... on CardPaymentDetails { + avsResultCode + bin + company + cvvResultCode + expirationMonth + expirationYear + name + number + paymentMethodName + wallet + } + ... on LocalPaymentMethodsPaymentDetails { + paymentDescriptor + paymentMethodName + } + ... on ShopPayInstallmentsPaymentDetails { + paymentMethodName } } - pageInfo { - endCursor - hasNextPage } } + } + pageInfo { + endCursor + hasNextPage + } + } } """ From 4fbc565bef0b23bb018e338f2f6470094d4060b7 Mon Sep 17 00:00:00 2001 From: Sourabh Gandhi <105213416+sgandhi1311@users.noreply.github.com> Date: Thu, 24 Apr 2025 14:01:35 +0530 Subject: [PATCH 41/66] Optimize Refund and Transactions sync by fetching only updated parent objects (#212) * remove the pseudo incremental sync logic from the order_refunds stream * setup and changelog update * exiting bookmark value should not decrease * perform pseudo sync on the order_refunds stream * replicate record that exactly matches the bookmark value * update transactions stream bookmarking logic * add pylint fix * add custom get query function * update changelog * make pylint happy --- CHANGELOG.md | 3 ++ setup.py | 2 +- tap_shopify/streams/metafields.py | 7 +-- tap_shopify/streams/order_refunds.py | 37 ++++++--------- tap_shopify/streams/transactions.py | 71 ++++++++++++++++++---------- 5 files changed, 69 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7875dad5..0d21e64a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 3.2.1 + * Refactor refund and transactions sync logic to rely on updated parent objects [#212](https://github.com/singer-io/tap-shopify/pull/212) + ## 3.2.0 * Enhance tap with additional fields across streams [#209](https://github.com/singer-io/tap-shopify/pull/209) * Bookmarks are now updated after fetching records within the specified date range. diff --git a/setup.py b/setup.py index 6a6b0f6a..c8e66746 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="3.2.0", + version="3.2.1", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/streams/metafields.py b/tap_shopify/streams/metafields.py index 261e6299..2d2b79e7 100644 --- a/tap_shopify/streams/metafields.py +++ b/tap_shopify/streams/metafields.py @@ -69,9 +69,10 @@ def get_objects(self): """ sync_start = utils.now().replace(microsecond=0) - # Use the lastest bookmark value as the starting point, contrary to the earlier - # implementation where the start date was used. - last_updated_at = self.get_bookmark() + # Set the initial last updated time to the bookmark minus one minute + # to ensure we don't miss any updates as it was observed shopify + # updates the parent object initially and then the child objects + last_updated_at = self.get_bookmark() - timedelta(minutes=1) while last_updated_at < sync_start: date_window_end = last_updated_at + timedelta(days=self.date_window_size) diff --git a/tap_shopify/streams/order_refunds.py b/tap_shopify/streams/order_refunds.py index 905809bc..e9033cd2 100644 --- a/tap_shopify/streams/order_refunds.py +++ b/tap_shopify/streams/order_refunds.py @@ -12,6 +12,7 @@ class OrderRefunds(Stream): child_data_key = "refunds" replication_key = "updatedAt" + # pylint: disable=too-many-locals def get_objects(self): """ Fetch order refund objects within date windows, yielding each refund individually. @@ -19,9 +20,12 @@ def get_objects(self): Yields: dict: Transformed refund object. """ - # Will always fetch the data from the start date and filter it in the code - # Shopify doesn't support filtering by updated_at for metafields - last_updated_at = utils.strptime_with_tz(Context.config["start_date"]) + + # Set the initial last updated time to the bookmark minus one minute + # to ensure we don't miss any updates as its observed shopify updates + # the parent object initially and then the child objects + last_updated_at = self.get_bookmark() - timedelta(minutes=1) + initial_bookmark_time = current_bookmark = self.get_bookmark() sync_start = utils.now().replace(microsecond=0) # Process each date window @@ -44,7 +48,11 @@ def get_objects(self): # Yield each transformed refund for child_obj in child_edges: - yield self.transform_object(child_obj) + replication_value = utils.strptime_with_tz(child_obj[self.replication_key]) + current_bookmark = max(current_bookmark, replication_value) + # Perform the pseudo sync for the child objects + if replication_value >= initial_bookmark_time: + yield self.transform_object(child_obj) # Handle pagination page_info = data.get("pageInfo", {}) @@ -53,6 +61,9 @@ def get_objects(self): break last_updated_at = query_end + # Update bookmark to the latest value, but not beyond sync start time + max_bookmark_value = min(sync_start, current_bookmark) + self.update_bookmark(utils.strftime(max_bookmark_value)) def transform_lineitems(self, data): """ @@ -148,24 +159,6 @@ def transform_object(self, obj): obj["orderAdjustments"] = self.transform_orderadjustments(obj) return obj - def sync(self): - """ - Performs pseudo incremental sync. - """ - start_time = utils.now().replace(microsecond=0) - max_bookmark_value = current_bookmark_value = self.get_bookmark() - - for obj in self.get_objects(): - replication_value = utils.strptime_to_utc(obj[self.replication_key]) - - max_bookmark_value = max(max_bookmark_value, replication_value) - - if replication_value >= current_bookmark_value: - yield obj - - # Update bookmark to the latest value, but not beyond sync start time - max_bookmark_value = min(start_time, max_bookmark_value) - self.update_bookmark(utils.strftime(max_bookmark_value)) def get_query(self): """ diff --git a/tap_shopify/streams/transactions.py b/tap_shopify/streams/transactions.py index a4f9f8a4..6ab767bf 100644 --- a/tap_shopify/streams/transactions.py +++ b/tap_shopify/streams/transactions.py @@ -12,6 +12,34 @@ class Transactions(Stream): child_data_key = "transactions" replication_key = "createdAt" + # pylint: disable=W0221 + def get_query_params(self, updated_at_min, updated_at_max, cursor=None): + """ + Construct query parameters for GraphQL requests. + + Args: + updated_at_min (str): Minimum updated_at timestamp. + updated_at_max (str): Maximum updated_at timestamp. + cursor (str): Pagination cursor, if any. + + Returns: + dict: Dictionary of query parameters. + """ + parent_filter_key = "updated_at" + query = ( + f"{parent_filter_key}:>='{updated_at_min}' " + f"AND {parent_filter_key}:<'{updated_at_max}'" + ) + params = { + "query": query, + "first": self.results_per_page, + } + + if cursor: + params["after"] = cursor + return params + + # pylint: disable=too-many-locals def get_objects(self): """ Fetch transaction objects within date windows, yielding each transaction individually. @@ -19,9 +47,11 @@ def get_objects(self): Yields: dict: Transformed transaction object. """ - # Will always fetch the data from the start date and filter it in the code - # Shopify doesn't support filtering by updated_at for metafields - last_updated_at = utils.strptime_with_tz(Context.config["start_date"]) + # Set the initial last updated time to the bookmark minus one minute + # to ensure we don't miss any updates as its observed shopify updates + # the parent object initially and then the child objects + last_updated_at = self.get_bookmark() - timedelta(minutes=1) + initial_bookmark_time = current_bookmark = self.get_bookmark() sync_start = utils.now().replace(microsecond=0) while last_updated_at < sync_start: @@ -40,7 +70,13 @@ def get_objects(self): node = edge.get("node", {}) child_edges = node.get(self.child_data_key, []) - yield from (self.transform_object(child_obj) for child_obj in child_edges) + # Yield each transformed transaction object + for child_obj in child_edges: + replication_value = utils.strptime_with_tz(child_obj[self.replication_key]) + current_bookmark = max(current_bookmark, replication_value) + # Perform the pseudo sync for the child objects + if replication_value >= initial_bookmark_time: + yield self.transform_object(child_obj) page_info = data.get("pageInfo", {}) cursor = page_info.get("endCursor") @@ -48,32 +84,17 @@ def get_objects(self): break last_updated_at = query_end - - def sync(self): - """ - Performs pseudo incremental sync. - """ - start_time = utils.now().replace(microsecond=0) - max_bookmark_value = current_bookmark_value = self.get_bookmark() - - for obj in self.get_objects(): - replication_value = utils.strptime_to_utc(obj[self.replication_key]) - - max_bookmark_value = max(max_bookmark_value, replication_value) - - if replication_value >= current_bookmark_value: - yield obj - - # Update bookmark to the latest value, but not beyond sync start time - max_bookmark_value = min(start_time, max_bookmark_value) - self.update_bookmark(utils.strftime(max_bookmark_value)) + # Update bookmark to the latest value, but not beyond sync start time + max_bookmark_value = min(sync_start, current_bookmark) + self.update_bookmark(utils.strftime(max_bookmark_value)) def get_query(self): """ Returns query for fetching transactions. Note: - Shopify has a limit of 100 transactions per order. + Shopify has a limit of 100 transactions per order as per shopify support. + To be on the safer side, we are limiting the transactions to 250. Returns: str: GraphQL query string. @@ -83,7 +104,7 @@ def get_query(self): orders(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { edges { node { - transactions(first: 100) { + transactions(first: 250) { accountNumber amountRoundingSet { presentmentMoney { From a14bae1ca76679820e1c285c0206f9c52421acc4 Mon Sep 17 00:00:00 2001 From: Sourabh Gandhi <105213416+sgandhi1311@users.noreply.github.com> Date: Mon, 26 May 2025 19:05:41 +0530 Subject: [PATCH 42/66] Catalog-driven dynamic query pruning and Error handling (#214) * Implement dynamic query builder * fix pylint and unit test * Include graphql query and the request-id in the logs * add unit test for schema validation * fix pylint * add retry mechanism for the ShopifyError * make the condition in positive tone --- CHANGELOG.md | 3 + setup.py | 3 +- tap_shopify/__init__.py | 2 + tap_shopify/context.py | 16 ++ tap_shopify/streams/base.py | 65 +++++- tap_shopify/streams/collections.py | 3 +- tap_shopify/streams/inventory_levels.py | 6 +- tap_shopify/streams/metafields.py | 7 +- tap_shopify/streams/order_refunds.py | 9 +- tap_shopify/streams/orders.py | 3 +- tap_shopify/streams/transactions.py | 3 +- tests/unittests/test_base.py | 34 ++- tests/unittests/test_schema_validation.py | 244 ++++++++++++++++++++++ 13 files changed, 380 insertions(+), 18 deletions(-) create mode 100644 tests/unittests/test_schema_validation.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d21e64a..8c29c927 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 3.3.0 + * Dynamically generate graphql query [#214](https://github.com/singer-io/tap-shopify/pull/214) + ## 3.2.1 * Refactor refund and transactions sync logic to rely on updated parent objects [#212](https://github.com/singer-io/tap-shopify/pull/212) diff --git a/setup.py b/setup.py index c8e66746..29546c74 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="3.2.1", + version="3.3.0", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", @@ -15,6 +15,7 @@ # Important: review the monkey-patched method in the GraphQL client when upgrading this dependency. "ShopifyAPI==12.7.0", "singer-python==6.1.1", + "graphql-core==3.2.6" ], extras_require={ 'dev': [ diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index d0c5d001..9e5bc4e4 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -206,6 +206,8 @@ def main(): msg = body.get('errors') finally: raise ShopifyError(exc, msg) from exc + except ShopifyError as error: + raise error except Exception as exc: raise ShopifyError(exc) from exc diff --git a/tap_shopify/context.py b/tap_shopify/context.py index 3408bb8f..ef29a69f 100644 --- a/tap_shopify/context.py +++ b/tap_shopify/context.py @@ -23,6 +23,22 @@ def is_selected(cls, stream_name): stream_metadata = metadata.to_map(stream['metadata']) return metadata.get(stream_metadata, (), 'selected') + @classmethod + def get_unselected_fields(cls, stream_name): + stream = cls.get_catalog_entry(stream_name) + stream_metadata = metadata.to_map(stream['metadata']) + # All fields defined in the schema + all_fields = set(stream["schema"]["properties"].keys()) + + # Selected fields from metadata + selected_fields = set() + for breadcrumb, data in stream_metadata.items(): + if len(breadcrumb) == 2: + if data.get('selected') or data.get('inclusion') == 'automatic': + selected_fields.add(breadcrumb[1]) + + return list(all_fields - selected_fields) + @classmethod def get_results_per_page(cls, default_results_per_page): results_per_page = default_results_per_page diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index 597057db..1ca2c906 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -13,7 +13,10 @@ import simplejson import singer from singer import metrics, utils +from graphql import parse, print_ast, visit +from graphql.language import Visitor, FieldNode, SelectionSetNode, OperationDefinitionNode, NameNode from tap_shopify.context import Context +from tap_shopify.exceptions import ShopifyError LOGGER = singer.get_logger() @@ -90,7 +93,8 @@ def is_timeout_error(error_raised): def shopify_error_handling(fnc): @backoff.on_exception(backoff.expo, - (http.client.IncompleteRead, ConnectionResetError, ShopifyAPIError), + (http.client.IncompleteRead, ConnectionResetError, + ShopifyAPIError, ShopifyError), max_tries=MAX_RETRIES, factor=2) @backoff.on_exception(backoff.expo, # timeout error raise by Shopify @@ -125,9 +129,6 @@ def wrapper(*args, **kwargs): class Error(Exception): """Base exception for the API interaction module""" -class OutOfOrderIdsError(Error): - """Raised if our expectation of ordering by ID is violated""" - class ShopifyAPIError(Error): """Raised for any unexpected api error without a valid status code""" @@ -219,6 +220,41 @@ def update_bookmark(self, bookmark_value, bookmark_key=None): ) singer.write_state(Context.state) + def remove_fields_from_query(self, fields_to_remove: list) -> str: + ast = parse(self.get_query()) + used_variable_names = set() + + class FieldRemover(Visitor): + def enter_selection_set(self, node, _key, _parent, _path, _ancestors): + new_selections = [] + for selection in node.selections: + if isinstance(selection, FieldNode): + if selection.name.value in fields_to_remove: + continue + # Check field arguments for variable usage + for arg in selection.arguments or []: + if hasattr(arg.value, "name") and isinstance(arg.value.name, NameNode): + used_variable_names.add(arg.value.name.value) + new_selections.append(selection) + return SelectionSetNode(selections=new_selections) + + def leave_operation_definition(self, node, *_): + # Keep only variable definitions that are used + new_var_defs = [ + var_def for var_def in node.variable_definitions or [] + if var_def.variable.name.value in used_variable_names + ] + return OperationDefinitionNode( + operation=node.operation, + name=node.name, + variable_definitions=new_var_defs, + directives=node.directives, + selection_set=node.selection_set + ) + + # Start visiting the AST and dynamically gather used variable names + modified_ast = visit(ast, FieldRemover()) + return print_ast(modified_ast) # This function can be overridden by subclasses for specialized API # interactions. If you override it you need to remember to decorate it @@ -250,6 +286,20 @@ def call_api(self, query_params, query=None, data_key=None): LOGGER.error("GraphQL Error: %s", gql_error) raise ShopifyAPIError("An error occurred with the GraphQL API.") from gql_error + except urllib.error.HTTPError as http_error: + # Extract X-Request-ID from the error response headers + request_id = http_error.headers.get("X-Request-ID") + error_body = http_error.read().decode("utf-8") if http_error.fp else None + error_message = ( + f"{http_error.reason} - {error_body}" + if error_body + else http_error.reason + ) + raise ShopifyError(http_error, + f"GraphQL request failed for stream '{self.name}' with status {http_error.code} " + f"and X-Request-ID '{request_id or 'N/A'}', Reason: {error_message}." + ) from http_error + except Exception as exc: LOGGER.error("Unexpected error occurred.") raise exc @@ -277,11 +327,12 @@ def get_query_params(self, updated_at_min, updated_at_max, cursor=None): params["after"] = cursor return params + # pylint: disable=too-many-locals def get_objects(self): """ Returns: - Yields list of objects for the stream - Performs + Performs: - Pagination & Filtering of stream - Transformation and bookmarking """ @@ -289,6 +340,8 @@ def get_objects(self): last_updated_at = self.get_bookmark() current_bookmark = last_updated_at sync_start = utils.now().replace(microsecond=0) + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) + LOGGER.info("GraphQL query for stream '%s': %s", self.name, ' '.join(query.split())) while last_updated_at < sync_start: date_window_end = last_updated_at + timedelta(days=self.date_window_size) @@ -299,7 +352,7 @@ def get_objects(self): query_params = self.get_query_params(last_updated_at, query_end, cursor) with metrics.http_request_timer(self.name): - data = self.call_api(query_params) + data = self.call_api(query_params, query=query) for edge in data.get("edges"): obj = self.transform_object(edge.get("node")) diff --git a/tap_shopify/streams/collections.py b/tap_shopify/streams/collections.py index 9b607f01..3b9cedab 100644 --- a/tap_shopify/streams/collections.py +++ b/tap_shopify/streams/collections.py @@ -24,6 +24,7 @@ def transform_products(self, data): for item in data["products"]["edges"] if (node := item.get("node")) and "id" in node ] + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) # Handle pagination page_info = data["products"].get("pageInfo", {}) @@ -35,7 +36,7 @@ def transform_products(self, data): } # Fetch the next page of data - response = self.call_api(params) + response = self.call_api(params, query=query) products_data = response.get("node", {}).get("products", {}) product_ids.extend( node["id"] diff --git a/tap_shopify/streams/inventory_levels.py b/tap_shopify/streams/inventory_levels.py index a778d140..eb12bc6d 100644 --- a/tap_shopify/streams/inventory_levels.py +++ b/tap_shopify/streams/inventory_levels.py @@ -25,6 +25,7 @@ def get_next_page_child(self, parent_id, cursor, child_query): dict: The child object. """ has_next_page = True + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) while has_next_page: child_query_params = { "first": self.results_per_page, @@ -32,7 +33,7 @@ def get_next_page_child(self, parent_id, cursor, child_query): "query": child_query, "childafter": cursor, } - data = self.call_api(child_query_params) + data = self.call_api(child_query_params, query=query) for edge in data.get("edges", []): node = edge.get("node", {}) child_data = node.get(self.child_data_key, {}) @@ -54,6 +55,7 @@ def get_objects(self): last_updated_at = self.get_bookmark() current_bookmark = last_updated_at sync_start = utils.now().replace(microsecond=0) + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) while last_updated_at < sync_start: date_window_end = last_updated_at + timedelta(days=self.date_window_size) @@ -63,7 +65,7 @@ def get_objects(self): while has_next_page: query_params = self.get_query_params(last_updated_at, query_end, cursor) with metrics.http_request_timer(self.name): - data = self.call_api(query_params) + data = self.call_api(query_params, query=query) # Process parent objects for edge in data.get("edges", []): diff --git a/tap_shopify/streams/metafields.py b/tap_shopify/streams/metafields.py index 2d2b79e7..417682e9 100644 --- a/tap_shopify/streams/metafields.py +++ b/tap_shopify/streams/metafields.py @@ -3,6 +3,7 @@ import json from singer import utils, get_logger, metrics +from tap_shopify.context import Context from tap_shopify.streams.base import Stream LOGGER = get_logger() @@ -42,6 +43,7 @@ def fetch_paginated_child_data(self, initial_child_data, parent_id): # Extract the numeric ID from the full path numeric_id = parent_id.split('/')[-1] page_info = initial_child_data.get("pageInfo", {}) + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) while page_info.get("hasNextPage"): query_params = { @@ -50,7 +52,7 @@ def fetch_paginated_child_data(self, initial_child_data, parent_id): "childafter": page_info.get("endCursor"), } - response = self.call_api(query_params) + response = self.call_api(query_params, query=query) response_edges = response.get("edges", []) if not response_edges: break @@ -73,6 +75,7 @@ def get_objects(self): # to ensure we don't miss any updates as it was observed shopify # updates the parent object initially and then the child objects last_updated_at = self.get_bookmark() - timedelta(minutes=1) + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) while last_updated_at < sync_start: date_window_end = last_updated_at + timedelta(days=self.date_window_size) @@ -84,7 +87,7 @@ def get_objects(self): while has_next_page: query_params = self.get_query_params(last_updated_at, query_end, cursor) with metrics.http_request_timer(self.name): - data = self.call_api(query_params) + data = self.call_api(query_params, query=query) # Process parent objects for edge in data.get("edges", []): diff --git a/tap_shopify/streams/order_refunds.py b/tap_shopify/streams/order_refunds.py index e9033cd2..175580e3 100644 --- a/tap_shopify/streams/order_refunds.py +++ b/tap_shopify/streams/order_refunds.py @@ -27,6 +27,7 @@ def get_objects(self): last_updated_at = self.get_bookmark() - timedelta(minutes=1) initial_bookmark_time = current_bookmark = self.get_bookmark() sync_start = utils.now().replace(microsecond=0) + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) # Process each date window while last_updated_at < sync_start: @@ -38,7 +39,7 @@ def get_objects(self): query_params = self.get_query_params(last_updated_at, query_end, cursor) with metrics.http_request_timer(self.name): - data = self.call_api(query_params) + data = self.call_api(query_params, query=query) # Process parent objects and their refunds edges = data.get("edges", []) @@ -80,6 +81,7 @@ def transform_lineitems(self, data): node for item in data["refundLineItems"]["edges"] if (node := item.get("node")) ] + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) # Handle pagination page_info = data["refundLineItems"].get("pageInfo", {}) @@ -92,7 +94,7 @@ def transform_lineitems(self, data): } # Fetch the next page of data - response = self.call_api(params) + response = self.call_api(params, query=query) nodes = response.get("edges", [])[0].get("node", {}) lineitems_data = nodes.get("refunds")[0] lineitems.extend( @@ -118,6 +120,7 @@ def transform_orderadjustments(self, data): node for item in data["orderAdjustments"]["edges"] if (node := item.get("node")) ] + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) # Handle pagination page_info = data["orderAdjustments"].get("pageInfo", {}) @@ -130,7 +133,7 @@ def transform_orderadjustments(self, data): } # Fetch the next page of data - response = self.call_api(params) + response = self.call_api(params, query=query) nodes = response.get("edges", [])[0].get("node", {}) refunds_data = nodes.get("refunds")[0] orderadjustments.extend( diff --git a/tap_shopify/streams/orders.py b/tap_shopify/streams/orders.py index 3d02ec56..71e6b1ab 100644 --- a/tap_shopify/streams/orders.py +++ b/tap_shopify/streams/orders.py @@ -56,6 +56,7 @@ def transform_lineitems(self, data): # few fields or reduce the page size # Handle pagination page_info = data["lineItems"].get("pageInfo", {}) + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) while page_info.get("hasNextPage"): params = { "first": self.results_per_page if self.results_per_page <= 150 else 150, @@ -64,7 +65,7 @@ def transform_lineitems(self, data): } # Fetch the next page of data - response = self.call_api(params) + response = self.call_api(params, query=query) lineitems_data = response.get("edges", [])[0].get("node", {}) lineitems.extend( node for item in lineitems_data["lineItems"]["edges"] diff --git a/tap_shopify/streams/transactions.py b/tap_shopify/streams/transactions.py index 6ab767bf..fb3de78a 100644 --- a/tap_shopify/streams/transactions.py +++ b/tap_shopify/streams/transactions.py @@ -53,6 +53,7 @@ def get_objects(self): last_updated_at = self.get_bookmark() - timedelta(minutes=1) initial_bookmark_time = current_bookmark = self.get_bookmark() sync_start = utils.now().replace(microsecond=0) + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) while last_updated_at < sync_start: date_window_end = last_updated_at + timedelta(days=self.date_window_size) @@ -63,7 +64,7 @@ def get_objects(self): query_params = self.get_query_params(last_updated_at, query_end, cursor) with metrics.http_request_timer(self.name): - data = self.call_api(query_params) + data = self.call_api(query_params, query=query) edges = data.get("edges", []) for edge in edges: diff --git a/tests/unittests/test_base.py b/tests/unittests/test_base.py index 12fe4842..203a762f 100644 --- a/tests/unittests/test_base.py +++ b/tests/unittests/test_base.py @@ -9,15 +9,47 @@ class TestStream(unittest.TestCase): + def mock_query(): + return """ + { + products(first: 10, after: "cursor") { + edges { + node { + id + updatedAt + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ def setUp(self): self.stream = Stream() self.stream.data_key = "products" + self.stream.name = "products" # Mock the Context.config to include start_date self.original_config = Context.config Context.config = { "start_date": "2025-01-01T00:00:00Z", "date_window_size": 30 } + Context.catalog = { + "streams": [ + { + "tap_stream_id": "products", + "schema": { + "properties": { + "id": {"type": "string"}, + "updatedAt": {"type": "string"} + } + }, + "metadata": [] + } + ] + } def tearDown(self): # Reset Context.config to its original state @@ -69,7 +101,7 @@ def test_call_api_empty_response(self, mock_get_query, mock_graphql): self.assertEqual(result, {}) @patch('shopify.GraphQL') - @patch.object(Stream, 'get_query', return_value='mocked_query') + @patch.object(Stream, 'get_query', return_value=mock_query()) @patch.object(Stream, 'transform_object', side_effect=lambda x: x) @patch('tap_shopify.streams.base.utils.now', return_value=datetime(2025, 2, 1, 0, 0, tzinfo=tzlocal())) def test_get_objects(self, mock_now, mock_transform_object, mock_get_query, mock_graphql): diff --git a/tests/unittests/test_schema_validation.py b/tests/unittests/test_schema_validation.py new file mode 100644 index 00000000..a0b9df6b --- /dev/null +++ b/tests/unittests/test_schema_validation.py @@ -0,0 +1,244 @@ +import unittest +from unittest.mock import patch +import json +from graphql import parse, FieldNode, SelectionSetNode +from pathlib import Path +from tap_shopify.streams.orders import Orders +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream +from textwrap import dedent + +SCHEMA_PATH = Path("tap_shopify/schemas/orders.json") + +def load_schema_fields(schema, prefix=""): + """Recursively extract schema field paths.""" + fields = set() + for key, value in schema.get("properties", {}).items(): + full_key = f"{prefix}.{key}" if prefix else key + fields.add(full_key) + return fields + + +def extract_top_level_fields_from_node(selection_set: SelectionSetNode, path=["orders", "edges", "node"]): + """Extract top-level field names from a selection set under the given field path.""" + + def find_selection(selections, name): + for sel in selections: + if isinstance(sel, FieldNode) and sel.name.value == name: + return sel + return None + + current_set = selection_set + for key in path: + node = find_selection(current_set.selections, key) + if node is None or node.selection_set is None: + return set() + current_set = node.selection_set + + return { + field.name.value + for field in current_set.selections + if isinstance(field, FieldNode) + } + +class TestGraphQLSchemaMatch(unittest.TestCase): + def test_schema_matches_graphql_query(self): + # Load and parse schema + schema = json.loads(SCHEMA_PATH.read_text()) + schema_fields = load_schema_fields(schema) + + # Load and parse GraphQL query + query_str = Orders().get_query() + ast = parse(query_str) + query_fields = set() + + for definition in ast.definitions: + if hasattr(definition, "selection_set") and isinstance(definition.selection_set, SelectionSetNode): + query_fields.update(extract_top_level_fields_from_node(definition.selection_set)) + + # Compare fields + missing_in_query = schema_fields - query_fields + missing_in_schema = query_fields - schema_fields + + self.assertFalse(missing_in_query, f"Schema fields missing in query: {missing_in_query}") + self.assertFalse(missing_in_schema, f"Query fields missing in schema: {missing_in_schema}") + + @patch('singer.metadata.to_map') + @patch.object(Context, 'get_catalog_entry') + def test_get_unselected_fields(self, mock_get_catalog_entry, mock_to_map): + # Mock schema + mock_schema = { + "properties": { + "id": {"type": "string"}, + "name": {"type": "string"}, + "email": {"type": "string"}, + "created_at": {"type": "string"}, + } + } + + # Mock metadata (breadcrumb: metadata dict) + mock_metadata_map = { + (): {"inclusion": "available"}, # Root - should be skipped + ("properties", "id"): {"selected": True}, + ("properties", "name"): {"inclusion": "automatic"}, + ("properties", "email"): {"selected": False}, # Not selected + ("properties", "created_at"): {"selected": False}, # Not selected + } + + # Mock return values + mock_get_catalog_entry.return_value = { + "schema": mock_schema, + "metadata": "dummy_metadata" + } + mock_to_map.return_value = mock_metadata_map + + # Call the method + result = Context.get_unselected_fields("test_stream") + + # Expecting unselected fields + expected = ["email", "created_at"] + self.assertCountEqual(result, expected) + + def setUp(self): + class TestClass(Stream): + def get_query(self): + return dedent(""" + query GetProducts($first: Int!, $after: String, $query: String) { + products(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { + edges { + node { + id + title + descriptionHtml + vendor + category { + id + } + tags + handle + publishedAt + createdAt + updatedAt + templateSuffix + status + productType + giftCardTemplateSuffix + hasOnlyDefaultVariant + hasOutOfStockVariants + hasVariantsThatRequiresComponents + isGiftCard + description + requiresSellingPlan + totalInventory + media(first: 250) { + edges { + node { + id + alt + status + mediaContentType + mediaWarnings { + code + message + } + mediaErrors { + code + details + message + } + ... on ExternalVideo { + id + embedUrl + } + ... on MediaImage { + id + updatedAt + createdAt + mimeType + image { + url + width + height + id + } + } + ... on Model3d { + id + filename + sources { + url + format + mimeType + filesize + } + } + ... on Video { + id + updatedAt + createdAt + filename + sources { + url + format + mimeType + fileSize + } + } + } + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + """) + self.instance = TestClass() + + def test_remove_fields_from_query(self): + result = self.instance.remove_fields_from_query(['requiresSellingPlan', 'totalInventory', 'media']) + + expected_query = dedent(""" + query GetProducts($first: Int!, $after: String, $query: String) { + products(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { + edges { + node { + id + title + descriptionHtml + vendor + category { + id + } + tags + handle + publishedAt + createdAt + updatedAt + templateSuffix + status + productType + giftCardTemplateSuffix + hasOnlyDefaultVariant + hasOutOfStockVariants + hasVariantsThatRequiresComponents + isGiftCard + description + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + """).strip() + + # Normalize whitespace for comparison + self.assertEqual( + ''.join(result.split()), + ''.join(expected_query.split()) + ) From a36de47a4414aef3c3d6c905044843e75b6f7ba7 Mon Sep 17 00:00:00 2001 From: Vishal Pachpinde Date: Thu, 29 May 2025 18:14:43 +0530 Subject: [PATCH 43/66] fixes KeyError for abandoned checkouts (#218) * bugfix & updated changelog --- CHANGELOG.md | 3 +++ setup.py | 2 +- tap_shopify/streams/abandoned_checkouts.py | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c29c927..536a86e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 3.3.1 + * Fix transformation bug for abandoned checkouts [#218](https://github.com/singer-io/tap-shopify/pull/218) + ## 3.3.0 * Dynamically generate graphql query [#214](https://github.com/singer-io/tap-shopify/pull/214) diff --git a/setup.py b/setup.py index 29546c74..ce212d52 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="3.3.0", + version="3.3.1", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/streams/abandoned_checkouts.py b/tap_shopify/streams/abandoned_checkouts.py index 2b356cb2..a1712ff8 100644 --- a/tap_shopify/streams/abandoned_checkouts.py +++ b/tap_shopify/streams/abandoned_checkouts.py @@ -36,7 +36,8 @@ def transform_object(self, obj): Returns: dict: Transformed object. """ - obj["lineItems"] = self.process_sub_entities(obj, entity_name="lineItems") + if obj.get("lineItems"): + obj["lineItems"] = self.process_sub_entities(obj, entity_name="lineItems") return obj def get_query(self): From aa686ced79911b7bb84837326072e3b83e4d8d6d Mon Sep 17 00:00:00 2001 From: Prijen Khokhani <88327452+prijendev@users.noreply.github.com> Date: Thu, 5 Jun 2025 16:06:47 +0530 Subject: [PATCH 44/66] Assign config provided value to date_window (#221) * Fix date window bug * Enhance date_window extraction logic * Update changelog * Add unit tests * Update class name in unit tests * Fix pylint issue --- CHANGELOG.md | 3 ++ setup.py | 2 +- tap_shopify/streams/base.py | 6 ++- tests/unittests/test_date_window.py | 58 +++++++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 tests/unittests/test_date_window.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 536a86e7..f67872e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 3.3.2 + * Fix date_window bug[#221](https://github.com/singer-io/tap-shopify/pull/221) + ## 3.3.1 * Fix transformation bug for abandoned checkouts [#218](https://github.com/singer-io/tap-shopify/pull/218) diff --git a/setup.py b/setup.py index ce212d52..4eae2d27 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="3.3.1", + version="3.3.2", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index 1ca2c906..2532d934 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -25,7 +25,7 @@ # set default timeout of 300 seconds REQUEST_TIMEOUT = 300 -DATE_WINDOW_SIZE = 30 +DEFAULT_DATE_WINDOW = 30 # We will retry a 500 error a maximum of 5 times before giving up MAX_RETRIES = 5 @@ -139,12 +139,14 @@ class Stream(): replication_method = 'INCREMENTAL' replication_key = 'updatedAt' key_properties = ['id'] - date_window_size = float(Context.config.get("date_window_size", DATE_WINDOW_SIZE)) + date_window_size = None data_key = None results_per_page = None def __init__(self): self.results_per_page = Context.get_results_per_page(RESULTS_PER_PAGE) + self.date_window_size = float(Context.config.get("date_window_size") or + DEFAULT_DATE_WINDOW) or DEFAULT_DATE_WINDOW # set request timeout self.request_timeout = get_request_timeout() diff --git a/tests/unittests/test_date_window.py b/tests/unittests/test_date_window.py new file mode 100644 index 00000000..a0796812 --- /dev/null +++ b/tests/unittests/test_date_window.py @@ -0,0 +1,58 @@ +import unittest +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream + +class TestShopifyDateWindowHandling(unittest.TestCase): + + def test_no_date_window_value(self): + """Test that no value for date_window is handled correctly.""" + Context.config = { + "start_date": "2025-01-01T00:00:00Z", + } + streams = Stream() + self.assertEquals(streams.date_window_size, 30) + + def test_valid_int_date_window_value(self): + """Test that valid integer value for date_window is handled correctly.""" + Context.config = { + "start_date": "2025-01-01T00:00:00Z", + "date_window_size": 10 + } + streams = Stream() + self.assertEquals(streams.date_window_size, 10) + + def test_valid_str_date_window_value(self): + """Test that valid str value for date_window is handled correctly..""" + Context.config = { + "start_date": "2025-01-01T00:00:00Z", + "date_window_size": "10" + } + streams = Stream() + self.assertEquals(streams.date_window_size, 10) + + def test_valid_float_date_window_value(self): + """Test that valid float value for date_window is handled correctly.""" + Context.config = { + "start_date": "2025-01-01T00:00:00Z", + "date_window_size": "11.00" + } + streams = Stream() + self.assertEquals(streams.date_window_size, 11) + + def test_zero_str_date_window_value(self): + """Test that valid zero string value for date_window is handled correctly.""" + Context.config = { + "start_date": "2025-01-01T00:00:00Z", + "date_window_size": "0" + } + streams = Stream() + self.assertEquals(streams.date_window_size, 30) + + def test_zero_int_date_window_value(self): + """Test that valid zero integer value for date_window is handled correctly.""" + Context.config = { + "start_date": "2025-01-01T00:00:00Z", + "date_window_size": 0 + } + streams = Stream() + self.assertEquals(streams.date_window_size, 30) From 5fc87c695f7a5c19f3f91e43467b7a72a0d276fc Mon Sep 17 00:00:00 2001 From: Prijen Khokhani <88327452+prijendev@users.noreply.github.com> Date: Thu, 19 Jun 2025 09:54:11 +0530 Subject: [PATCH 45/66] Introduce new stream order_shipping_lines (#213) * Initial commit for the shipping lines stream * update query * Add order_shipping_lines stream * Update changelog and bump version * Implement dynamic query builder * fix pylint and unit test * Update changelog --------- Co-authored-by: Sourabh Gandhi --- CHANGELOG.md | 4 + setup.py | 2 +- tap_shopify/schemas/order_shipping_lines.json | 475 ++++++++++++++++++ tap_shopify/streams/__init__.py | 1 + tap_shopify/streams/order_shipping_lines.py | 303 +++++++++++ tests/base.py | 8 +- tests/test_bookmarks_updated.py | 3 +- tests/test_start_date.py | 2 +- 8 files changed, 794 insertions(+), 4 deletions(-) create mode 100644 tap_shopify/schemas/order_shipping_lines.json create mode 100644 tap_shopify/streams/order_shipping_lines.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f67872e2..29513c51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 3.4.0 + * Introduce new stream order_shipping_lines. [#213](https://github.com/singer-io/tap-shopify/pull/213) + * shippingLine is a nested object within the orders stream and it represents individual shipping methods. + ## 3.3.2 * Fix date_window bug[#221](https://github.com/singer-io/tap-shopify/pull/221) diff --git a/setup.py b/setup.py index 4eae2d27..f336fb9b 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="3.3.2", + version="3.4.0", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/schemas/order_shipping_lines.json b/tap_shopify/schemas/order_shipping_lines.json new file mode 100644 index 00000000..43425e5d --- /dev/null +++ b/tap_shopify/schemas/order_shipping_lines.json @@ -0,0 +1,475 @@ +{ + "type": "object", + "properties": { + "carrierIdentifier": { + "type": [ + "null", + "string" + ] + }, + "code": { + "type": [ + "null", + "string" + ] + }, + "orderId": { + "type": [ + "null", + "string" + ] + }, + "currentDiscountedPriceSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "custom": { + "type": [ + "null", + "boolean" + ] + }, + "deliveryCategory": { + "type": [ + "null", + "string" + ] + }, + "discountAllocations": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "allocatedAmountSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "discountApplication": { + "type": [ + "null", + "object" + ], + "properties": { + "__typename": { + "type": [ + "null", + "string" + ] + }, + "allocationMethod": { + "type": [ + "null", + "string" + ] + }, + "index": { + "type": [ + "null", + "integer" + ] + }, + "targetSelection": { + "type": [ + "null", + "string" + ] + }, + "targetType": { + "type": [ + "null", + "string" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "code": { + "type": [ + "null", + "string" + ] + }, + "description": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "object" + ], + "properties": { + "__typename": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "percentage": { + "type": [ + "null", + "number" + ] + } + } + } + } + } + } + } + }, + "discountedPriceSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "isRemoved": { + "type": [ + "null", + "boolean" + ] + }, + "originalPriceSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "phone": { + "type": [ + "null", + "string" + ] + }, + "shippingRateHandle": { + "type": [ + "null", + "string" + ] + }, + "source": { + "type": [ + "null", + "string" + ] + }, + "taxLines": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "channelLiable": { + "type": [ + "null", + "boolean" + ] + }, + "priceSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "rate": { + "type": [ + "null", + "number" + ] + }, + "ratePercentage": { + "type": [ + "null", + "string" + ] + }, + "source": { + "type": [ + "null", + "string" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "updatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + } + } +} \ No newline at end of file diff --git a/tap_shopify/streams/__init__.py b/tap_shopify/streams/__init__.py index 78242bf5..3271a925 100644 --- a/tap_shopify/streams/__init__.py +++ b/tap_shopify/streams/__init__.py @@ -14,3 +14,4 @@ import tap_shopify.streams.product_variants import tap_shopify.streams.events import tap_shopify.streams.collections +import tap_shopify.streams.order_shipping_lines diff --git a/tap_shopify/streams/order_shipping_lines.py b/tap_shopify/streams/order_shipping_lines.py new file mode 100644 index 00000000..6786b35e --- /dev/null +++ b/tap_shopify/streams/order_shipping_lines.py @@ -0,0 +1,303 @@ +from datetime import timedelta +from singer import metrics, utils +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream + + +class OrderShippingLines(Stream): + """Stream class for fetching order shippingLines from Shopify""" + + name = "order_shipping_lines" + data_key = "orders" + child_data_key = "shippingLines" + replication_key = "updatedAt" + + # pylint: disable=too-many-locals + def get_objects(self): + """ + Fetch order shipping lines objects within date windows, yielding each shipping line. + + Yields: + dict: Transformed shipping line object. + """ + + # Set the initial last updated time to the bookmark minus one minute + # to ensure we don't miss any updates as its observed shopify updates + # the parent object initially and then the child objects + last_updated_at = self.get_bookmark() - timedelta(minutes=1) + current_bookmark = self.get_bookmark() + sync_start = utils.now().replace(microsecond=0) + + # Process each date window + while last_updated_at < sync_start: + date_window_end = last_updated_at + timedelta(days=self.date_window_size) + query_end = min(sync_start, date_window_end) + cursor = None + + while True: + query_params = self.get_query_params(last_updated_at, query_end, cursor) + + with metrics.http_request_timer(self.name): + data = self.call_api(query_params) + + # Process parent objects and their shippinglines + edges = data.get("edges", []) + for edge in edges: + node = edge.get("node", {}) + + for shipping_line in self.paginate_shipping_lines(node): + shipping_line["orderId"] = node["id"].split("/")[-1] + shipping_line["updatedAt"] = node["updatedAt"] + + replication_value = utils.strptime_with_tz( + shipping_line[self.replication_key] + ) + current_bookmark = max(current_bookmark, replication_value) + yield self.transform_object(shipping_line) + + # Handle pagination + page_info = data.get("pageInfo", {}) + cursor = page_info.get("endCursor") + if not page_info.get("hasNextPage", False): + break + + last_updated_at = query_end + # Update bookmark to the latest value, but not beyond sync start time + max_bookmark_value = min(sync_start, current_bookmark) + self.update_bookmark(utils.strftime(max_bookmark_value)) + + def paginate_shipping_lines(self, data): + """ + Transforms the shippingLines data by handling pagination. + + Args: + data (dict): Order data. + + Returns: + list: List of shippingLines. + """ + + for item in data["shippingLines"]["edges"]: + node = item.get("node") + if node: + yield node + + # Handle pagination + page_info = data.get("pageInfo", {}) + order_parent = data["id"] + while page_info.get("hasNextPage"): + params = { + "first": self.results_per_page, + "query": f"id:{order_parent.split('/')[-1]}", + "childafter": page_info.get("endCursor"), + } + + # Fetch the next page of data + response = self.call_api(params) + node = response.get("edges", [])[0].get("node", {}) + shipping_lines_data = node.get("shippingLines") + for item in shipping_lines_data["edges"]: + node = item.get("node") + if node: + yield node + + page_info = shipping_lines_data.get("pageInfo", {}) + + def get_query(self): + """ + Returns the GraphQL query for fetching order shipping lines. + + Returns: + str: GraphQL query string. + """ + return """ + query GetShippingLines($first: Int!, $after: String, $query: String, $childafter: String) { + orders(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { + edges { + node { + id + updatedAt + shippingLines(first: $first, after: $childafter) { + edges { + node { + carrierIdentifier + code + currentDiscountedPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + custom + deliveryCategory + discountAllocations { + allocatedAmountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + discountApplication { + allocationMethod + index + targetSelection + targetType + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + ... on AutomaticDiscountApplication { + __typename + allocationMethod + index + targetSelection + targetType + title + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + ... on DiscountCodeApplication { + __typename + allocationMethod + code + index + targetSelection + targetType + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + ... on ManualDiscountApplication { + description + allocationMethod + index + targetSelection + targetType + title + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + ... on ScriptDiscountApplication { + __typename + allocationMethod + index + targetSelection + targetType + title + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + } + } + discountedPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + id + isRemoved + originalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + phone + shippingRateHandle + source + taxLines { + channelLiable + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + rate + ratePercentage + source + title + } + title + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + + +Context.stream_objects["order_shipping_lines"] = OrderShippingLines diff --git a/tests/base.py b/tests/base.py index 43a2db41..d57d8ff9 100644 --- a/tests/base.py +++ b/tests/base.py @@ -97,6 +97,12 @@ def expected_metadata(self): "collections": default, "customers": default, "orders": default, + "order_shipping_lines": { + self.REPLICATION_KEYS: {"updatedAt"}, + self.PRIMARY_KEYS: {"id"}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.API_LIMIT: 15 + }, "order_refunds": default, "products": default, "product_variants": default, @@ -307,7 +313,7 @@ def __init__(self, *args, **kwargs): # abandoned checkouts are saved in the Shopify admin for three months. # Every Monday, abandoned checkouts that are older than three months are removed from your admin. # Also no POST call is available for this endpoint: https://shopify.dev/api/admin-rest/2022-01/resources/abandoned-checkouts - self.store_2_streams = {'metafields_products', 'transactions', 'order_refunds', 'products', 'locations', 'inventory_levels', 'inventory_items'} + self.store_2_streams = {'metafields_products', 'transactions', 'order_refunds', 'products', 'locations', 'inventory_levels', 'inventory_items', 'order_shipping_lines'} #modified this method to accommodate replication key in the current_state def calculated_states_by_stream(self, current_state): diff --git a/tests/test_bookmarks_updated.py b/tests/test_bookmarks_updated.py index fde27ec1..e87eea71 100644 --- a/tests/test_bookmarks_updated.py +++ b/tests/test_bookmarks_updated.py @@ -31,7 +31,7 @@ def __init__(self, *args, **kwargs): # abandoned checkouts are saved in the Shopify admin for three months. # Every Monday, abandoned checkouts that are older than three months are removed from your admin. # Also no POST call is available for this endpoint: https://shopify.dev/api/admin-rest/2022-01/resources/abandoned-checkouts - store_2_streams = {'metafields_products', 'transactions', 'order_refunds', 'products', 'locations', 'inventory_items', 'customers', 'collections', 'orders'} + store_2_streams = {'metafields_products', 'transactions', 'order_refunds', 'products', 'locations', 'inventory_items', 'customers', 'collections', 'orders', 'order_shipping_lines'} def test_run_store_2(self): with self.subTest(store="store_2"): @@ -89,6 +89,7 @@ def bookmarks_test(self, conn_id, testable_streams): 'customers': {'updatedAt': '2025-01-19T20:55:07.000000Z'}, 'orders': {'updatedAt': '2024-12-05T09:59:35.000000Z'}, 'collections': {'updatedAt': '2025-01-21T13:29:06.000000Z'}, + 'order_shipping_lines': {'updatedAt': '2021-09-30T01:02:21.000000Z'}, 'metafields_products': {'updatedAt': '2025-01-21T03:11:54.000000Z'} } diff --git a/tests/test_start_date.py b/tests/test_start_date.py index dd6b1c82..7536b83d 100644 --- a/tests/test_start_date.py +++ b/tests/test_start_date.py @@ -37,7 +37,7 @@ def get_properties(self, original: bool = True): if original: return return_value - return_value["start_date"] = '2021-04-21T00:00:00Z' + return_value["start_date"] = '2021-09-15T00:00:00Z' return return_value @staticmethod From 40452f07a1c85f1cc1c376980805cbc8ae44b441 Mon Sep 17 00:00:00 2001 From: Sourabh Gandhi <105213416+sgandhi1311@users.noreply.github.com> Date: Wed, 9 Jul 2025 10:25:11 +0530 Subject: [PATCH 46/66] Add fields in orders stream - retailLocation and location ID (within fulfillments) (#219) * Initial commit for the shipping lines stream * update query * Add order_shipping_lines stream * Update changelog and bump version * Implement dynamic query builder * fix pylint and unit test * add retailLocation and DiscountCodeApplication fields to the orders schema * Update changelog * add location id in the orders.fulfillments * update pylint command * Skip the author field on missing `read_users` scope (#223) * Skip author field * mark the author field as unsupported for the public app * fix the unit test and pylint * fix the integration test * include scenario for unsupported field in the discovery metadata check * update review comments * changelog and setup update --------- Co-authored-by: prijendev --- .circleci/config.yml | 2 +- CHANGELOG.md | 4 + setup.py | 2 +- tap_shopify/__init__.py | 27 + tap_shopify/context.py | 2 + tap_shopify/schemas/orders.json | 74 + tap_shopify/streams/metafields.py | 1 + tap_shopify/streams/orders.py | 1810 +++++++++-------- tests/test_discovery.py | 22 +- .../unittests/test_token_check_in_discover.py | 13 +- 10 files changed, 1096 insertions(+), 861 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4c430e07..ff2ada48 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,7 +39,7 @@ jobs: name: 'pylint' command: | source /usr/local/share/virtualenvs/tap-shopify/bin/activate - pylint tap_shopify -d missing-docstring,too-many-branches,consider-using-f-string,consider-using-generator,consider-using-dict-items,unnecessary-dunder-call,duplicate-code + pylint tap_shopify -d missing-docstring,too-many-branches,consider-using-f-string,consider-using-generator,consider-using-dict-items,unnecessary-dunder-call,duplicate-code,too-many-lines json_validator: executor: docker-executor steps: diff --git a/CHANGELOG.md b/CHANGELOG.md index 29513c51..4f3bbeea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 3.5.0 + * Add fields in orders stream - retailLocation and location ID (within fulfillments) [#219](https://github.com/singer-io/tap-shopify/pull/219) + * Skip the author field on missing read_users scope [#223](https://github.com/singer-io/tap-shopify/pull/223) + ## 3.4.0 * Introduce new stream order_shipping_lines. [#213](https://github.com/singer-io/tap-shopify/pull/213) * shippingLine is a nested object within the orders stream and it represents individual shipping methods. diff --git a/setup.py b/setup.py index f336fb9b..09dcbac6 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="3.4.0", + version="3.5.0", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index 9e5bc4e4..807b5202 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -19,6 +19,7 @@ REQUIRED_CONFIG_KEYS = ["shop", "api_key"] LOGGER = singer.get_logger() SDC_KEYS = {'id': 'integer', 'name': 'string', 'myshopify_domain': 'string'} +UNSUPPORTED_FIELDS = {"author"} @shopify_error_handling def initialize_shopify_client(): @@ -34,6 +35,30 @@ def initialize_shopify_client(): # Shop.current() makes a call for shop details with provided shop and api_key return shopify.Shop.current().attributes +# Add helper +def fetch_app_scopes(): + query = """ + query { + currentAppInstallation { + accessScopes { + handle + } + } + } + """ + data = json.loads(shopify.GraphQL().execute(query)) + return {s["handle"] for s in data["data"]["currentAppInstallation"]["accessScopes"]} + +def has_read_users_access(): + # If the app does not have the 'read_users' scope, return False + if 'read_users' not in fetch_app_scopes(): + LOGGER.warning( + "Skipping '%s' field: 'read_users' scope is not granted for public apps.", + ", ".join(UNSUPPORTED_FIELDS) + ) + return False + return True + def get_abs_path(path): return os.path.join(os.path.dirname(os.path.realpath(__file__)), path) @@ -63,6 +88,8 @@ def get_discovery_metadata(stream, schema): for field_name in schema['properties'].keys(): if field_name in stream.key_properties or field_name == stream.replication_key: mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'automatic') + elif field_name in UNSUPPORTED_FIELDS and not has_read_users_access(): + mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'unsupported') else: mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'available') diff --git a/tap_shopify/context.py b/tap_shopify/context.py index ef29a69f..fdd0a6c7 100644 --- a/tap_shopify/context.py +++ b/tap_shopify/context.py @@ -34,6 +34,8 @@ def get_unselected_fields(cls, stream_name): selected_fields = set() for breadcrumb, data in stream_metadata.items(): if len(breadcrumb) == 2: + if data.get('inclusion') == 'unsupported': + continue if data.get('selected') or data.get('inclusion') == 'automatic': selected_fields.add(breadcrumb[1]) diff --git a/tap_shopify/schemas/orders.json b/tap_shopify/schemas/orders.json index 9a2d1803..000f6398 100644 --- a/tap_shopify/schemas/orders.json +++ b/tap_shopify/schemas/orders.json @@ -1163,6 +1163,12 @@ "permitsSkuSharing": { "type": ["null", "boolean"] }, "inventoryManagement": { "type": ["null", "boolean"] } } + }, + "location": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "string"] } + } } } } @@ -2113,6 +2119,74 @@ }, "title": { "type": ["null", "string"] } } + }, + "retailLocation": { + "type": ["null", "object"], + "properties": { + "activatable": { "type": ["null", "boolean"] }, + "address": { + "type": ["null", "object"], + "properties": { + "address1": { "type": ["null", "string"] }, + "address2": { "type": ["null", "string"] }, + "city": { "type": ["null", "string"] }, + "country": { "type": ["null", "string"] }, + "countryCode": { "type": ["null", "string"] }, + "formatted": { "type": ["null", "string"] }, + "latitude": { "type": ["null", "number"] }, + "longitude": { "type": ["null", "number"] }, + "phone": { "type": ["null", "string"] }, + "province": { "type": ["null", "string"] }, + "provinceCode": { "type": ["null", "string"] }, + "zip": { "type": ["null", "string"] } + } + }, + "addressVerified": { "type": ["null", "boolean"] }, + "createdAt": { "type": ["null", "string"], "format": "date-time" }, + "deactivatable": { "type": ["null", "boolean"] }, + "deactivatedAt": { "type": ["null", "string"], "format": "date-time" }, + "deletable": { "type": ["null", "boolean"] }, + "fulfillmentService": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "string"] } + } + }, + "fulfillsOnlineOrders": { "type": ["null", "boolean"] }, + "hasActiveInventory": { "type": ["null", "boolean"] }, + "hasUnfulfilledOrders": { "type": ["null", "boolean"] }, + "id": { "type": ["null", "string"] }, + "isActive": { "type": ["null", "boolean"] }, + "isFulfillmentService": { "type": ["null", "boolean"] }, + "legacyResourceId": { "type": ["null", "string"] }, + "localPickupSettingsV2": { + "type": ["null", "object"], + "properties": { + "instructions": { "type": ["null", "string"] }, + "pickupTime": { "type": ["null", "string"] } + } + }, + "name": { "type": ["null", "string"] }, + "shipsInventory": { "type": ["null", "boolean"] }, + "updatedAt": { "type": ["null", "string"], "format": "date-time" }, + "suggestedAddresses": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "address1": { "type": ["null", "string"] }, + "address2": { "type": ["null", "string"] }, + "city": { "type": ["null", "string"] }, + "country": { "type": ["null", "string"] }, + "countryCode": { "type": ["null", "string"] }, + "formatted": { "type": ["null", "string"] }, + "province": { "type": ["null", "string"] }, + "provinceCode": { "type": ["null", "string"] }, + "zip": { "type": ["null", "string"] } + } + } + } + } } } } diff --git a/tap_shopify/streams/metafields.py b/tap_shopify/streams/metafields.py index 417682e9..cc830fdf 100644 --- a/tap_shopify/streams/metafields.py +++ b/tap_shopify/streams/metafields.py @@ -76,6 +76,7 @@ def get_objects(self): # updates the parent object initially and then the child objects last_updated_at = self.get_bookmark() - timedelta(minutes=1) query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) + LOGGER.info("GraphQL query for stream '%s': %s", self.name, ' '.join(query.split())) while last_updated_at < sync_start: date_window_end = last_updated_at + timedelta(days=self.date_window_size) diff --git a/tap_shopify/streams/orders.py b/tap_shopify/streams/orders.py index 71e6b1ab..b4ed6c98 100644 --- a/tap_shopify/streams/orders.py +++ b/tap_shopify/streams/orders.py @@ -98,900 +98,1018 @@ def get_query(self): """ return """ query orders($first: Int!, $after: String, $query: String, $childafter: String) { - orders(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { - edges { - node { - additionalFees { - id - name - price { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - taxLines { - channelLiable - priceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - rate - ratePercentage - source - title - } - } - app { - id - name - icon { - id - } - } - billingAddress { - address1 - address2 - city - company - coordinatesValidated - country - countryCodeV2 - firstName - formattedArea - id - lastName - latitude - longitude - name - phone - province - provinceCode - timeZone - validationResultSummary - zip - } - billingAddressMatchesShippingAddress - canMarkAsPaid - canNotifyCustomer - cancelReason - cancellation { - staffNote - } - cancelledAt - capturable - cartDiscountAmountSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - channelInformation { - id - channelId - } - clientIp - closed - closedAt - confirmationNumber - confirmed - createdAt - currencyCode - currentCartDiscountAmountSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - currentShippingPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - currentSubtotalLineItemsQuantity - currentSubtotalPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - currentTotalAdditionalFeesSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - currentTotalDiscountsSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - currentTotalDutiesSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - currentTotalPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - currentTotalTaxSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - currentTotalWeight - customer { - id - email - firstName - lastName - addresses { - address1 - address2 - city - countryCodeV2 - country - company - firstName - lastName - id - name - phone - province - provinceCode - zip - } - state - verifiedEmail - updatedAt - taxExempt - tags - taxExemptions - note - multipassIdentifier - createdAt - defaultAddress { - address1 - address2 - city - company - country - countryCodeV2 - firstName - id - lastName - name - province - phone - provinceCode - zip - } - } - customerJourneySummary { - lastVisit { - landingPage - referrerUrl - } - } - merchantOfRecordApp { - id - } - customerAcceptsMarketing - customerLocale - discountCodes - discountCode - displayFinancialStatus - displayFulfillmentStatus - disputes { - id - initiatedAs - status - } - dutiesIncluded - email - edited - estimatedTaxes - fulfillable - fullyPaid - hasTimelineComment - fulfillmentsCount { - count - precision - } - id - legacyResourceId - merchantBusinessEntity { - address { - address1 - address2 - city - countryCode - province - zip - } - companyName - displayName - id - primary - } - name - note - netPaymentSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - originalTotalAdditionalFeesSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - originalTotalDutiesSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - originalTotalPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - paymentGatewayNames - phone - poNumber - presentmentCurrencyCode - processedAt - refundable - refundDiscrepancySet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - registeredSourceUrl - requiresShipping - restockable - returnStatus - shippingAddress { - address1 - address2 - city - company - coordinatesValidated - country - countryCodeV2 - firstName - formattedArea - id - lastName - latitude - longitude - name - phone - province - provinceCode - timeZone - validationResultSummary - zip - } - shopifyProtect { - eligibility { - status - } - status - } - sourceIdentifier - sourceName - statusPageUrl - subtotalLineItemsQuantity - subtotalPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - tags - taxExempt - taxLines { - channelLiable - priceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - rate - ratePercentage - source - title - } - taxesIncluded - test - totalCapturableSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - totalCashRoundingAdjustment { - paymentSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - refundSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - } - totalDiscountsSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - totalOutstandingSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - totalPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - totalReceivedSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - totalRefundedSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - totalRefundedShippingSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - totalShippingPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - totalTaxSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - totalTipReceivedSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - totalWeight - transactionsCount { - count - precision - } - unpaid - updatedAt - fulfillments(first: 250) { - id - name - status - totalQuantity - updatedAt - createdAt - deliveredAt - estimatedDeliveryAt - requiresShipping - inTransitAt - trackingInfo(first: 250) { - number - company - url - } - service { - serviceName - id - handle - trackingSupport - type - permitsSkuSharing - inventoryManagement - } - } - lineItems(first: 25, after: $childafter) { + orders(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { edges { node { - id - vendor - quantity - title - requiresShipping - originalTotalSet { - presentmentMoney { - currencyCode - amount + additionalFees { + id + name + price { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + taxLines { + channelLiable + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + rate + ratePercentage + source + title + } } - shopMoney { - amount - currencyCode + app { + id + name + icon { + id + } } - } - taxLines(first: 250) { - priceSet { - presentmentMoney { - amount - currencyCode + billingAddress { + address1 + address2 + city + company + coordinatesValidated + country + countryCodeV2 + firstName + formattedArea + id + lastName + latitude + longitude + name + phone + province + provinceCode + timeZone + validationResultSummary + zip } - shopMoney { - amount - currencyCode + billingAddressMatchesShippingAddress + canMarkAsPaid + canNotifyCustomer + cancelReason + cancellation { + staffNote } + cancelledAt + capturable + cartDiscountAmountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } } - rate - title - source - channelLiable - } - taxable - isGiftCard - name - discountedTotalSet { - presentmentMoney { - amount - currencyCode + channelInformation { + id + channelId } - shopMoney { - amount + clientIp + closed + closedAt + confirmationNumber + confirmed + createdAt currencyCode + currentCartDiscountAmountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } } - } - sku - product { - id - } - discountAllocations { - allocatedAmountSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - discountApplication { - index - targetType - targetSelection - allocationMethod - ... on AutomaticDiscountApplication { - title + currentShippingPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } } - ... on ManualDiscountApplication { - title + currentSubtotalLineItemsQuantity + currentSubtotalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } } - ... on ScriptDiscountApplication { - title + currentTotalAdditionalFeesSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } } - value { - ... on MoneyV2 { - __typename - amount - currencyCode + currentTotalDiscountsSet { + presentmentMoney { + amount + currencyCode } - ... on PricingPercentageValue { - __typename - percentage + shopMoney { + amount + currencyCode } } + currentTotalDutiesSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } } - } - customAttributes { - key - value - } - totalDiscountSet { - presentmentMoney { - amount - currencyCode + currentTotalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } } - shopMoney { - amount - currencyCode + currentTotalTaxSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentTotalWeight + customer { + id + email + firstName + lastName + addresses { + address1 + address2 + city + countryCodeV2 + country + company + firstName + lastName + id + name + phone + province + provinceCode + zip + } + state + verifiedEmail + updatedAt + taxExempt + tags + taxExemptions + note + multipassIdentifier + createdAt + defaultAddress { + address1 + address2 + city + company + country + countryCodeV2 + firstName + id + lastName + name + province + phone + provinceCode + zip + } + } + customerJourneySummary { + lastVisit { + landingPage + referrerUrl + } + } + merchantOfRecordApp { + id + } + customerAcceptsMarketing + customerLocale + discountCodes + discountCode + displayFinancialStatus + displayFulfillmentStatus + disputes { + id + initiatedAs + status + } + dutiesIncluded + email + edited + estimatedTaxes + fulfillable + fullyPaid + hasTimelineComment + fulfillmentsCount { + count + precision } - } - duties { - harmonizedSystemCode id - taxLines { - rate - source - title - channelLiable - priceSet { + legacyResourceId + merchantBusinessEntity { + address { + address1 + address2 + city + countryCode + province + zip + } + companyName + displayName + id + primary + } + name + note + netPaymentSet { presentmentMoney { - amount - currencyCode + amount + currencyCode } shopMoney { - amount - currencyCode + amount + currencyCode } } + originalTotalAdditionalFeesSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } } - countryCodeOfOrigin - } - discountedUnitPriceSet { - presentmentMoney { - amount - currencyCode + originalTotalDutiesSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } } - shopMoney { - amount - currencyCode + originalTotalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } } - } - originalUnitPriceSet { - presentmentMoney { - amount - currencyCode + paymentGatewayNames + phone + poNumber + presentmentCurrencyCode + processedAt + refundable + refundDiscrepancySet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } } - shopMoney { - amount - currencyCode + registeredSourceUrl + requiresShipping + restockable + returnStatus + shippingAddress { + address1 + address2 + city + company + coordinatesValidated + country + countryCodeV2 + firstName + formattedArea + id + lastName + latitude + longitude + name + phone + province + provinceCode + timeZone + validationResultSummary + zip } - } - unfulfilledDiscountedTotalSet { - presentmentMoney { - amount - currencyCode + shopifyProtect { + eligibility { + status + } + status } - shopMoney { - amount - currencyCode + sourceIdentifier + sourceName + statusPageUrl + subtotalLineItemsQuantity + subtotalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } } - } - unfulfilledOriginalTotalSet { - presentmentMoney { - amount - currencyCode + tags + taxExempt + taxLines { + channelLiable + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + rate + ratePercentage + source + title } - shopMoney { - amount - currencyCode + taxesIncluded + test + totalCapturableSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } } - } - variant { - id - } - } - } - pageInfo { - hasNextPage - endCursor - } - } - shippingLine { - carrierIdentifier - code - currentDiscountedPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - custom - deliveryCategory - discountAllocations { - allocatedAmountSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - discountApplication { - allocationMethod - index - targetSelection - targetType - value { - ... on MoneyV2 { - __typename - amount - currencyCode + totalCashRoundingAdjustment { + paymentSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + refundSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } } - ... on PricingPercentageValue { - __typename - percentage + totalDiscountsSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } } - } - ... on AutomaticDiscountApplication { - __typename - allocationMethod - index - targetSelection - targetType - title - value { - ... on MoneyV2 { - __typename - amount - currencyCode - } - ... on PricingPercentageValue { - __typename - percentage + totalOutstandingSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } } + totalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } } - } - ... on DiscountCodeApplication { - __typename - allocationMethod - code - index - targetSelection - targetType - value { - ... on MoneyV2 { - __typename - amount - currencyCode - } - ... on PricingPercentageValue { - __typename - percentage + totalReceivedSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalRefundedSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } } + totalRefundedShippingSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } } - } - ... on ManualDiscountApplication { - description - allocationMethod - index - targetSelection - targetType - title - value { - ... on MoneyV2 { - __typename - amount - currencyCode - } - ... on PricingPercentageValue { - __typename - percentage + totalShippingPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } } + totalTaxSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } } - } - ... on ScriptDiscountApplication { - __typename - allocationMethod - index - targetSelection - targetType - title - value { - ... on MoneyV2 { - __typename - amount - currencyCode - } - ... on PricingPercentageValue { - __typename - percentage + totalTipReceivedSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } } + totalWeight + transactionsCount { + count + precision + } + unpaid + updatedAt + fulfillments(first: 250) { + id + name + status + totalQuantity + updatedAt + createdAt + deliveredAt + estimatedDeliveryAt + requiresShipping + inTransitAt + trackingInfo(first: 250) { + number + company + url + } + service { + serviceName + id + handle + trackingSupport + type + permitsSkuSharing + inventoryManagement + } + location { + id + } + } + lineItems(first: 25, after: $childafter) { + edges { + node { + id + vendor + quantity + title + requiresShipping + originalTotalSet { + presentmentMoney { + currencyCode + amount + } + shopMoney { + amount + currencyCode + } + } + taxLines(first: 250) { + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + rate + title + source + channelLiable + } + taxable + isGiftCard + name + discountedTotalSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + sku + product { + id + } + discountAllocations { + allocatedAmountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + discountApplication { + index + targetType + targetSelection + allocationMethod + ... on AutomaticDiscountApplication { + title + allocationMethod + index + targetSelection + targetType + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + ... on ManualDiscountApplication { + title + allocationMethod + description + index + targetSelection + targetType + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + ... on ScriptDiscountApplication { + title + allocationMethod + index + targetSelection + targetType + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + ... on DiscountCodeApplication { + __typename + allocationMethod + code + index + targetSelection + targetType + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + } + } + customAttributes { + key + value + } + totalDiscountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + duties { + harmonizedSystemCode + id + taxLines { + rate + source + title + channelLiable + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + } + countryCodeOfOrigin + } + discountedUnitPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + originalUnitPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + unfulfilledDiscountedTotalSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + unfulfilledOriginalTotalSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + variant { + id + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + shippingLine { + carrierIdentifier + code + currentDiscountedPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + custom + deliveryCategory + discountAllocations { + allocatedAmountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + discountApplication { + allocationMethod + index + targetSelection + targetType + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + ... on AutomaticDiscountApplication { + __typename + allocationMethod + index + targetSelection + targetType + title + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + ... on DiscountCodeApplication { + __typename + allocationMethod + code + index + targetSelection + targetType + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + ... on ManualDiscountApplication { + description + allocationMethod + index + targetSelection + targetType + title + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + ... on ScriptDiscountApplication { + __typename + allocationMethod + index + targetSelection + targetType + title + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + } + } + discountedPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + id + isRemoved + originalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + phone + shippingRateHandle + source + taxLines { + channelLiable + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + rate + ratePercentage + source + title + } + title + } + retailLocation { + activatable + address { + address1 + address2 + city + country + countryCode + formatted + latitude + longitude + phone + province + provinceCode + zip + } + addressVerified + createdAt + deactivatable + deactivatedAt + deletable + fulfillmentService { + id + } + fulfillsOnlineOrders + hasActiveInventory + hasUnfulfilledOrders + id + isActive + isFulfillmentService + legacyResourceId + localPickupSettingsV2 { + instructions + pickupTime + } + name + shipsInventory + updatedAt + suggestedAddresses { + address1 + address2 + city + country + countryCode + formatted + province + provinceCode + zip + } } } - } - } - discountedPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - id - isRemoved - originalPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - phone - shippingRateHandle - source - taxLines { - channelLiable - priceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - rate - ratePercentage - source - title } - title + pageInfo { + endCursor + hasNextPage } } - } - pageInfo { - endCursor - hasNextPage - } - } } """ diff --git a/tests/test_discovery.py b/tests/test_discovery.py index b35738c0..5e4df1b3 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -140,13 +140,18 @@ def test_run(self): expected_automatic_fields, actual_automatic_fields)) - # verify that all other fields have inclusion of available - # This assumes there are no unsupported fields for SaaS sources + # verify that all other fields have inclusion of available or unsupported self.assertTrue( - all({value.get("inclusion") == "available" for key, value - in schema["properties"].items() - if key not in actual_automatic_fields}), - msg="Not all non key properties are set to available in annotated schema") + all( + ( + value.get("inclusion") == "available" + or value.get("inclusion") == "unsupported" + ) + for key, value in schema["properties"].items() + if key not in actual_automatic_fields + ), + msg="Not all non key properties are set to available in annotated schema" + ) # verify that primary, replication and foreign keys # are given the inclusion of automatic in metadata. @@ -160,10 +165,9 @@ def test_run(self): expected_automatic_fields, actual_automatic_fields)) - # verify that all other fields have inclusion of available - # This assumes there are no unsupported fields for SaaS sources + # verify that all other fields have inclusion of available or unsupported self.assertTrue( - all({item.get("metadata").get("inclusion") == "available" + all({(item.get("metadata").get("inclusion") == "available" or item.get("metadata").get("inclusion") == "unsupported") for item in metadata if item.get("breadcrumb", []) != [] and item.get("breadcrumb", ["properties", None])[1] diff --git a/tests/unittests/test_token_check_in_discover.py b/tests/unittests/test_token_check_in_discover.py index 22617ba1..0fe2d981 100644 --- a/tests/unittests/test_token_check_in_discover.py +++ b/tests/unittests/test_token_check_in_discover.py @@ -24,14 +24,16 @@ def connection_error_raiser(): @mock.patch('tap_shopify.utils.parse_args') @mock.patch('tap_shopify.discover', side_effect=tap_shopify.discover) @mock.patch("builtins.print") +@mock.patch("tap_shopify.has_read_users_access") class TestTokenInDiscoverMode(unittest.TestCase): @mock.patch('tap_shopify.initialize_shopify_client', side_effect=resource_not_found_raiser) - def test_resource_not_found(self, mocked_client, mocked_print, mocked_discover, mocked_args): + def test_resource_not_found(self, mocked_client, mocked_access, mocked_print, mocked_discover, mocked_args): ''' Verify exception is raised for ResourceNotFound with proper error message and test that discover mode is called ''' + mocked_access.return_value = True mocked_args.return_value = Args() try: tap_shopify.main() @@ -42,11 +44,12 @@ def test_resource_not_found(self, mocked_client, mocked_print, mocked_discover, self.assertEqual(mocked_print.call_count, 0) @mock.patch('tap_shopify.initialize_shopify_client', side_effect=unauthorized_access_raiser) - def test_unauthorized_access(self, mocked_client, mocked_print, mocked_discover, mocked_args): + def test_unauthorized_access(self, mocked_client, mocked_access, mocked_print, mocked_discover, mocked_args): ''' Verify exception is raised for UnauthorizedAccess with proper error message and test that discover mode is called ''' + mocked_access.return_value = True mocked_args.return_value = Args() try: tap_shopify.main() @@ -57,11 +60,12 @@ def test_unauthorized_access(self, mocked_client, mocked_print, mocked_discover, self.assertEqual(mocked_print.call_count, 0) @mock.patch('tap_shopify.initialize_shopify_client', side_effect=connection_error_raiser) - def test_connection_error(self, mocked_client, mocked_print, mocked_discover, mocked_args): + def test_connection_error(self, mocked_client, mocked_access, mocked_print, mocked_discover, mocked_args): ''' Verify exception is raised for ConnectionError with proper error message and test that discover mode is called ''' + mocked_access.return_value = True mocked_args.return_value = Args() try: tap_shopify.main() @@ -72,11 +76,12 @@ def test_connection_error(self, mocked_client, mocked_print, mocked_discover, mo self.assertEqual(mocked_print.call_count, 0) @mock.patch('tap_shopify.initialize_shopify_client') - def test_no_error(self, mocked_client, mocked_print, mocked_discover, mocked_args): + def test_no_error(self, mocked_client, mocked_access, mocked_print, mocked_discover, mocked_args): ''' Verify that if no error during discover then print should be called once for writing catalog ''' + mocked_access.return_value = True mocked_args.return_value = Args() tap_shopify.main() self.assertEqual(mocked_discover.call_count, 1) From ef98374adc9ef1ad97ee8d4ba26f58dd45378cfc Mon Sep 17 00:00:00 2001 From: Prijen Khokhani <88327452+prijendev@users.noreply.github.com> Date: Wed, 9 Jul 2025 17:00:53 +0530 Subject: [PATCH 47/66] SAC-27727: Add new stream fulfillment orders (#222) * Add new stream fulfillment * Remove merchant requests * Update pagination logic * Remove heldBy as it require app_reads scope * Fix type error for events field * Fetch closed fulfillment_orders as well * Raise exception on missing scopes * Fix results_per_page type in tests * Add new stream fulfillment * Remove merchant requests * Update pagination logic * Remove heldBy as it require app_reads scope * Fix type error for events field * Fetch closed fulfillment_orders as well * Raise exception on missing scopes * Fix results_per_page type in tests * Update __init__ * Fix base tests * Update bookmark tests --- CHANGELOG.md | 3 + README.md | 4 + setup.py | 2 +- tap_shopify/__init__.py | 39 +- tap_shopify/schemas/fulfillment_orders.json | 979 ++++++++++++++++++++ tap_shopify/streams/__init__.py | 1 + tap_shopify/streams/fulfillment_orders.py | 321 +++++++ tests/base.py | 8 +- tests/test_all_fields.py | 5 +- tests/test_bookmarks.py | 2 +- tests/test_bookmarks_updated.py | 2 +- tests/test_pagination.py | 2 +- tests/test_shop_info_fields.py | 7 +- tests/test_start_date.py | 2 +- 14 files changed, 1354 insertions(+), 23 deletions(-) create mode 100644 tap_shopify/schemas/fulfillment_orders.json create mode 100644 tap_shopify/streams/fulfillment_orders.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f3bbeea..2c43823f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 3.6.0 + * Introduce new stream fulfillment_orders. [#222](https://github.com/singer-io/tap-shopify/pull/222) + ## 3.5.0 * Add fields in orders stream - retailLocation and location ID (within fulfillments) [#219](https://github.com/singer-io/tap-shopify/pull/219) * Skip the author field on missing read_users scope [#223](https://github.com/singer-io/tap-shopify/pull/223) diff --git a/README.md b/README.md index d6a53059..f44dc09a 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,8 @@ This tap: - [Metafields Orders](https://shopify.dev/docs/api/admin-graphql/latest/queries/orders) - [Metafields Products](https://shopify.dev/docs/api/admin-graphql/latest/queries/products) - [Orders](https://shopify.dev/docs/api/admin-graphql/latest/queries/orders) + - [OrderShippingLines](https://shopify.dev/docs/api/admin-graphql/latest/objects/ShippingLine) + - [FulfillmentOrders](https://shopify.dev/docs/api/admin-graphql/latest/queries/fulfillmentorders) - [Products](https://shopify.dev/docs/api/admin-graphql/latest/queries/products) - [Product Variants](https://shopify.dev/docs/api/admin-graphql/latest/queries/productVariants) - [Transactions](https://shopify.dev/docs/api/admin-graphql/latest/queries/orders) @@ -42,6 +44,8 @@ This tap: | metafields_products | updatedAt | id | | order_refunds | updatedAt | id | | orders | updatedAt | id | +| order_shipping_lines | updatedAt | id | +| fulfillment_orders | updatedAt | id | | product_variants | updatedAt | id | | products | updatedAt | id | | transactions | createdAt | id | diff --git a/setup.py b/setup.py index 09dcbac6..5616ec26 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="3.5.0", + version="3.6.0", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index 807b5202..e880c5cb 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -14,7 +14,7 @@ from singer import Transformer from tap_shopify.context import Context from tap_shopify.exceptions import ShopifyError -from tap_shopify.streams.base import shopify_error_handling, get_request_timeout +from tap_shopify.streams.base import shopify_error_handling, get_request_timeout, ShopifyAPIError REQUIRED_CONFIG_KEYS = ["shop", "api_key"] LOGGER = singer.get_logger() @@ -144,6 +144,7 @@ def shuffle_streams(stream_name): def sync(): shop_attributes = initialize_shopify_client() sdc_fields = {"_sdc_shop_" + x: shop_attributes[x] for x in SDC_KEYS} + require_reauth = False # If there is a currently syncing stream bookmark, shuffle the # stream order so it gets sync'd first @@ -176,19 +177,25 @@ def sync(): Context.state['bookmarks']['currently_sync_stream'] = stream_id singer.write_state(Context.state) - # some fields have epoch-time as date, hence transform into UTC date - with Transformer(singer.UNIX_SECONDS_INTEGER_DATETIME_PARSING) as transformer: - for rec in stream.sync(): - extraction_time = singer.utils.now() - record_schema = catalog_entry['schema'] - record_metadata = metadata.to_map(catalog_entry['metadata']) - rec = transformer.transform({**rec, **sdc_fields}, - record_schema, - record_metadata) - singer.write_record(stream_id, - rec, - time_extracted=extraction_time) - Context.counts[stream_id] += 1 + try: + # some fields have epoch-time as date, hence transform into UTC date + with Transformer(singer.UNIX_SECONDS_INTEGER_DATETIME_PARSING) as transformer: + for rec in stream.sync(): + extraction_time = singer.utils.now() + record_schema = catalog_entry['schema'] + record_metadata = metadata.to_map(catalog_entry['metadata']) + rec = transformer.transform({**rec, **sdc_fields}, + record_schema, + record_metadata) + singer.write_record(stream_id, + rec, + time_extracted=extraction_time) + Context.counts[stream_id] += 1 + except ShopifyAPIError as e: + if stream_id == 'fulfillment_orders' and 'Access denied' in str(e.__cause__): + require_reauth = True + continue + raise e Context.state['bookmarks'].pop('currently_sync_stream') singer.write_state(Context.state) @@ -198,6 +205,10 @@ def sync(): LOGGER.info('%s: %d', stream_id, stream_count) LOGGER.info('----------------------') + if require_reauth: + raise ShopifyAPIError("Required scopes are missing for the `fulfillment_orders` stream. " \ + "Please re-authorize the connection to sync this stream.") + @utils.handle_top_exception(LOGGER) def main(): try: diff --git a/tap_shopify/schemas/fulfillment_orders.json b/tap_shopify/schemas/fulfillment_orders.json new file mode 100644 index 00000000..59fda378 --- /dev/null +++ b/tap_shopify/schemas/fulfillment_orders.json @@ -0,0 +1,979 @@ +{ + "type": "object", + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "orderId": { + "type": [ + "null", + "string" + ] + }, + "updatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "supportedActions": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "action": { + "type": [ + "null", + "string" + ] + }, + "externalUrl": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "status": { + "type": [ + "null", + "string" + ] + }, + "requestStatus": { + "type": [ + "null", + "string" + ] + }, + "orderProcessedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "orderName": { + "type": [ + "null", + "string" + ] + }, + "channelId": { + "type": [ + "null", + "string" + ] + }, + "fulfillAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "fulfillBy": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "createdAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "destination": { + "type": [ + "null", + "object" + ], + "properties": { + "address1": { + "type": [ + "null", + "string" + ] + }, + "address2": { + "type": [ + "null", + "string" + ] + }, + "city": { + "type": [ + "null", + "string" + ] + }, + "province": { + "type": [ + "null", + "string" + ] + }, + "zip": { + "type": [ + "null", + "string" + ] + }, + "countryCode": { + "type": [ + "null", + "string" + ] + }, + "company": { + "type": [ + "null", + "string" + ] + }, + "email": { + "type": [ + "null", + "string" + ] + }, + "firstName": { + "type": [ + "null", + "string" + ] + }, + "lastName": { + "type": [ + "null", + "string" + ] + }, + "phone": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "location": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "fulfillmentHolds": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "displayReason": { + "type": [ + "null", + "string" + ] + }, + "handle": { + "type": [ + "null", + "string" + ] + }, + "reason": { + "type": [ + "null", + "string" + ] + }, + "reasonNotes": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "heldByRequestingApp": { + "type": [ + "null", + "boolean" + ] + } + } + } + }, + "internationalDuties": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "incoterm": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "deliveryMethod": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "methodType": { + "type": [ + "null", + "string" + ] + }, + "presentedName": { + "type": [ + "null", + "string" + ] + }, + "serviceCode": { + "type": [ + "null", + "string" + ] + }, + "sourceReference": { + "type": [ + "null", + "string" + ] + }, + "minDeliveryDateTime": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "maxDeliveryDateTime": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "brandedPromise": { + "type": [ + "null", + "object" + ], + "properties": { + "handle": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + } + } + }, + "additionalInformation": { + "type": [ + "null", + "object" + ], + "properties": { + "instructions": { + "type": [ + "null", + "string" + ] + }, + "phone": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "assignedLocation": { + "type": [ + "null", + "object" + ], + "properties": { + "address1": { + "type": [ + "null", + "string" + ] + }, + "address2": { + "type": [ + "null", + "string" + ] + }, + "city": { + "type": [ + "null", + "string" + ] + }, + "province": { + "type": [ + "null", + "string" + ] + }, + "zip": { + "type": [ + "null", + "string" + ] + }, + "countryCode": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "phone": { + "type": [ + "null", + "string" + ] + }, + "location": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "merchantRequests": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "kind": { + "type": [ + "null", + "string" + ] + }, + "message": { + "type": [ + "null", + "string" + ] + }, + "requestOptions": { + "type": [ + "null", + "object" + ] + }, + "responseData": { + "type": [ + "null", + "object" + ] + }, + "sentAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "fulfillmentOrder": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + } + } + } + }, + "locationsForMove": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "message": { + "type": [ + "null", + "string" + ] + }, + "movable": { + "type": [ + "null", + "boolean" + ] + }, + "unavailableLineItemsCount": { + "type": [ + "null", + "object" + ], + "properties": { + "count": { + "type": [ + "null", + "integer" + ] + }, + "precision": { + "type": [ + "null", + "string" + ] + } + } + }, + "availableLineItemsCount": { + "type": [ + "null", + "object" + ], + "properties": { + "count": { + "type": [ + "null", + "integer" + ] + }, + "precision": { + "type": [ + "null", + "string" + ] + } + } + }, + "location": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + }, + "availableLineItems": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "action": { + "id": [ + "null", + "string" + ] + } + } + } + }, + "unavailableLineItems": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + } + } + } + } + }, + "fulfillmentOrdersForMerge": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "fulfillments": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "status": { + "type": [ + "null", + "string" + ] + }, + "displayStatus": { + "type": [ + "null", + "string" + ] + }, + "requiresShipping": { + "type": [ + "null", + "boolean" + ] + }, + "totalQuantity": { + "type": [ + "null", + "integer" + ] + }, + "createdAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "updatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "inTransitAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "deliveredAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "estimatedDeliveryAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "originAddress": { + "type": [ + "null", + "object" + ], + "properties": { + "address1": { + "type": [ + "null", + "string" + ] + }, + "address2": { + "type": [ + "null", + "string" + ] + }, + "city": { + "type": [ + "null", + "string" + ] + }, + "provinceCode": { + "type": [ + "null", + "string" + ] + }, + "zip": { + "type": [ + "null", + "string" + ] + }, + "countryCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "trackingInfo": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "company": { + "type": [ + "null", + "string" + ] + }, + "number": { + "type": [ + "null", + "string" + ] + }, + "url": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "service": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + }, + "location": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + }, + "fulfillmentOrders": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "fulfillmentLineItems": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "quantity": { + "type": [ + "null", + "number" + ] + }, + "originalTotalSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + } + } + }, + "discountedTotalSet": { + "type": [ + "null", + "object" + ], + "properties": { + "presentmentMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + }, + "shopMoney": { + "type": [ + "null", + "object" + ], + "properties": { + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + } + } + } + } + }, + "lineItem": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + } + } + } + }, + "legacyResourceId": { + "type": [ + "null", + "string" + ] + }, + "order": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + }, + "events": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "id": { + "type": [ + "null", + "string" + ] + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/tap_shopify/streams/__init__.py b/tap_shopify/streams/__init__.py index 3271a925..b647db5c 100644 --- a/tap_shopify/streams/__init__.py +++ b/tap_shopify/streams/__init__.py @@ -14,4 +14,5 @@ import tap_shopify.streams.product_variants import tap_shopify.streams.events import tap_shopify.streams.collections +import tap_shopify.streams.fulfillment_orders import tap_shopify.streams.order_shipping_lines diff --git a/tap_shopify/streams/fulfillment_orders.py b/tap_shopify/streams/fulfillment_orders.py new file mode 100644 index 00000000..ac8d599c --- /dev/null +++ b/tap_shopify/streams/fulfillment_orders.py @@ -0,0 +1,321 @@ +import singer +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream + +LOGGER = singer.get_logger() + + +class FulfillmentOrders(Stream): + """Stream class for Shopify fulfillment_orders.""" + + name = "fulfillment_orders" + data_key = "fulfillmentOrders" + replication_key = "updatedAt" + + def transform_childitems(self, data, parent_id, key, next_page_key): + """ + Paginate child items. + """ + child_records = [ + node for item in data["edges"] + if (node := item.get("node")) + ] + # Handle pagination + page_info = data.get("pageInfo", {}) + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) + while page_info.get("hasNextPage"): + params = { + "first": self.results_per_page, + "query": f"id:{parent_id.split('/')[-1]}", + next_page_key: page_info.get("endCursor"), + } + + # Fetch the next page of data + response = self.call_api(params, query=query) + node = response.get("edges", [])[0].get("node", {}) + child_records.extend( + node for item in node[key]["edges"] + if (node := item.get("node")) + ) + page_info = node.get("pageInfo", {}) + + return child_records + + def transform_object(self, obj): + """ + Transforms a collection object. + Args: + obj (dict): Collection object. + Returns: + dict: Transformed collection object. + """ + if obj.get("merchantRequests"): + obj["merchantRequests"] = self.transform_childitems( + obj.get("merchantRequests"), + obj["id"], "merchantRequests", + "merchant_request_after" + ) + + if obj.get("locationsForMove"): + obj["locationsForMove"] = self.transform_childitems( + obj.get("locationsForMove"), obj["id"], + "locationsForMove", + "locations_move_after" + ) + for item in obj["locationsForMove"]: + item["availableLineItems"] = item["availableLineItems"]["nodes"] + item["unavailableLineItems"] = item["unavailableLineItems"]["nodes"] + + if obj.get("fulfillments"): + obj["fulfillments"] = self.transform_childitems( + obj.get("fulfillments"), obj["id"], + "fulfillments", + "fulfillments_after" + ) + for item in obj["fulfillments"]: + item["fulfillmentOrders"] = item["fulfillmentOrders"]["nodes"] + item["fulfillmentLineItems"] = item["fulfillmentLineItems"]["nodes"] + item["events"] = item["events"]["nodes"] + + if obj.get("fulfillmentOrdersForMerge"): + obj["fulfillmentOrdersForMerge"] = obj["fulfillmentOrdersForMerge"]["nodes"] + + return obj + + def get_query(self): + """ + Returns the GraphQL query for fetching fulfillmentOrders. + Returns: + str: GraphQL query string. + """ + return """ + query fulfillmentOrders($first: Int!, $after: String, $query: String, $merchant_request_after: String, $locations_move_after: String, $fulfillments_after: String) { + fulfillmentOrders(first: $first, after: $after, query: $query, includeClosed: true, sortKey: UPDATED_AT) { + edges { + node { + id + orderId + updatedAt + supportedActions { + action + externalUrl + } + status + requestStatus + orderProcessedAt + orderName + channelId + fulfillAt + fulfillBy + createdAt + destination { + address1 + address2 + city + countryCode + company + email + firstName + id + lastName + province + phone + zip + location { + id + } + } + fulfillmentHolds { + displayReason + handle + heldByRequestingApp + id + reason + reasonNotes + } + internationalDuties { + incoterm + } + deliveryMethod { + id + maxDeliveryDateTime + methodType + minDeliveryDateTime + presentedName + serviceCode + sourceReference + brandedPromise { + handle + name + } + additionalInformation { + instructions + phone + } + } + assignedLocation { + address1 + address2 + city + countryCode + name + phone + province + zip + location { + id + } + } + merchantRequests(first: 3, after: $merchant_request_after) { + edges { + node { + id + kind + message + requestOptions + responseData + sentAt + fulfillmentOrder { + id + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + locationsForMove(first: 3, after: $locations_move_after) { + edges { + node { + message + movable + location { + id + } + unavailableLineItemsCount { + count + precision + } + availableLineItemsCount { + count + precision + } + availableLineItems(first: 250) { + nodes { + id + } + } + unavailableLineItems(first:250) { + nodes { + id + } + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + fulfillmentOrdersForMerge(first: 250) { + nodes { + id + } + } + fulfillments(first: 3, after: $fulfillments_after) { + edges { + node { + createdAt + deliveredAt + displayStatus + estimatedDeliveryAt + id + inTransitAt + name + requiresShipping + status + totalQuantity + updatedAt + originAddress { + address1 + address2 + city + countryCode + provinceCode + zip + } + trackingInfo { + company + number + url + } + service { + id + } + location { + id + } + fulfillmentOrders(first: 250) { + nodes { + id + } + } + fulfillmentLineItems(first: 3) { + nodes { + id + quantity + originalTotalSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + discountedTotalSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + currencyCode + amount + } + } + lineItem { + id + } + } + + } + legacyResourceId + order { + id + } + events(first: 250) { + nodes { + id + } + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + + +Context.stream_objects["fulfillment_orders"] = FulfillmentOrders diff --git a/tests/base.py b/tests/base.py index d57d8ff9..ef414d2e 100644 --- a/tests/base.py +++ b/tests/base.py @@ -97,6 +97,12 @@ def expected_metadata(self): "collections": default, "customers": default, "orders": default, + "fulfillment_orders": { + self.REPLICATION_KEYS: {"updatedAt"}, + self.PRIMARY_KEYS: {"id"}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.API_LIMIT: 30 + }, "order_shipping_lines": { self.REPLICATION_KEYS: {"updatedAt"}, self.PRIMARY_KEYS: {"id"}, @@ -307,7 +313,7 @@ def select_all_streams_and_fields(conn_id, catalogs, select_all_fields: bool = T def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.start_date = self.get_properties().get("start_date") - self.store_1_streams = {'collections', 'orders', 'products', 'customers', 'locations', 'inventory_levels', 'inventory_items'} + self.store_1_streams = {'collections', 'orders', 'products', 'customers', 'locations', 'inventory_levels', 'inventory_items', 'fulfillment_orders'} # removed 'abandoned_checkouts' from store 2 streams, as per the Doc: # https://help.shopify.com/en/manual/orders/abandoned-checkouts?st_source=admin&st_campaign=abandoned_checkouts_footer&utm_source=admin&utm_campaign=abandoned_checkouts_footer#review-your-abandoned-checkouts # abandoned checkouts are saved in the Shopify admin for three months. diff --git a/tests/test_all_fields.py b/tests/test_all_fields.py index 4c5b42a7..ff1e8179 100644 --- a/tests/test_all_fields.py +++ b/tests/test_all_fields.py @@ -24,9 +24,10 @@ def name(): def get_properties(self, original: bool = True): """Configuration properties required for the tap.""" return_value = { - 'start_date': '2021-04-01T00:00:00Z', + 'start_date': '2025-01-01T00:00:00Z', 'shop': 'talenddatawearhouse', - 'date_window_size': 30 + 'date_window_size': 30, + 'results_per_page': '30' } return return_value diff --git a/tests/test_bookmarks.py b/tests/test_bookmarks.py index 332a681f..a28c3dd7 100644 --- a/tests/test_bookmarks.py +++ b/tests/test_bookmarks.py @@ -83,7 +83,7 @@ def bookmarks_test(self, conn_id, testable_streams): # Our test data sets for Shopify do not have any abandoned_checkouts our_catalogs = [catalog for catalog in found_catalogs if catalog.get('tap_stream_id') in incremental_streams] - self.select_all_streams_and_fields(conn_id, our_catalogs, select_all_fields=True) + self.select_all_streams_and_fields(conn_id, our_catalogs, select_all_fields=False) # Run a sync job using orchestrator first_sync_record_count = self.run_sync(conn_id) diff --git a/tests/test_bookmarks_updated.py b/tests/test_bookmarks_updated.py index e87eea71..fad0067a 100644 --- a/tests/test_bookmarks_updated.py +++ b/tests/test_bookmarks_updated.py @@ -48,7 +48,7 @@ def bookmarks_test(self, conn_id, testable_streams): # Our test data sets for Shopify do not have any abandoned_checkouts our_catalogs = [catalog for catalog in found_catalogs if catalog.get('tap_stream_id') in incremental_streams] - self.select_all_streams_and_fields(conn_id, our_catalogs, select_all_fields=True) + self.select_all_streams_and_fields(conn_id, our_catalogs, select_all_fields=False) ################################# # Run first sync diff --git a/tests/test_pagination.py b/tests/test_pagination.py index f41b77d4..65773ae0 100644 --- a/tests/test_pagination.py +++ b/tests/test_pagination.py @@ -19,7 +19,7 @@ def name(): def get_properties(self, *args, **kwargs): props = super().get_properties(*args, **kwargs) - props['results_per_page'] = '50' + props['results_per_page'] = '30' return props def test_run(self): diff --git a/tests/test_shop_info_fields.py b/tests/test_shop_info_fields.py index ee188569..1aeb7102 100644 --- a/tests/test_shop_info_fields.py +++ b/tests/test_shop_info_fields.py @@ -13,12 +13,17 @@ class ShopInfoFieldsTest(BaseTapTest): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.start_date = '2021-04-01T00:00:00Z' + self.start_date = '2024-12-01T00:00:00Z' @staticmethod def name(): return "tap_tester_shopify_shop_info_fields_test" + def get_properties(self, *args, **kwargs): + props = super().get_properties(*args, **kwargs) + props['results_per_page'] = '30' + return props + def test_run(self): """ Verify shop information fields are present in catalog for every streams. diff --git a/tests/test_start_date.py b/tests/test_start_date.py index 7536b83d..54d793ea 100644 --- a/tests/test_start_date.py +++ b/tests/test_start_date.py @@ -31,7 +31,7 @@ def get_properties(self, original: bool = True): 'shop': 'talenddatawearhouse', 'date_window_size': 30, # BUG: https://jira.talendforge.org/browse/TDL-13180 - 'results_per_page': '50' + 'results_per_page': '30' } if original: From 832bfe8199598ca63bc8a909ca17750e59327f9a Mon Sep 17 00:00:00 2001 From: Andy Lu Date: Thu, 17 Jul 2025 15:06:02 -0400 Subject: [PATCH 48/66] Bump versions (#228) * Bump versions * Bump setup.py version, update changelog --- CHANGELOG.md | 3 +++ setup.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c43823f..b28a412c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 3.6.1 + * Dependency upgrades [#228](https://github.com/singer-io/tap-shopify/pull/228) + ## 3.6.0 * Introduce new stream fulfillment_orders. [#222](https://github.com/singer-io/tap-shopify/pull/222) diff --git a/setup.py b/setup.py index 5616ec26..e76a0932 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="3.6.0", + version="3.6.1", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", @@ -21,7 +21,7 @@ 'dev': [ 'pylint==3.3.6', 'ipdb', - 'requests==2.32.3', + 'requests==2.32.4', 'nose', ] }, From 2a16deda1ed0162f73e7b57e284711bea2677ccc Mon Sep 17 00:00:00 2001 From: Prijen Khokhani <88327452+prijendev@users.noreply.github.com> Date: Tue, 22 Jul 2025 09:40:56 +0530 Subject: [PATCH 49/66] Set max size of pagination to 30 for fullfilment_orders (#230) * Set max size for pagination to 30 * Bump tap version * Update change log --- CHANGELOG.md | 3 +++ setup.py | 2 +- tap_shopify/streams/fulfillment_orders.py | 24 +++++++++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b28a412c..43fd0d8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +### 3.6.2 + * Set max_size of pagination to 30 for fullfilment_orders [#230](https://github.com/singer-io/tap-shopify/pull/230) + ## 3.6.1 * Dependency upgrades [#228](https://github.com/singer-io/tap-shopify/pull/228) diff --git a/setup.py b/setup.py index e76a0932..44e4f5e2 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="3.6.1", + version="3.6.2", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/streams/fulfillment_orders.py b/tap_shopify/streams/fulfillment_orders.py index ac8d599c..e42344ad 100644 --- a/tap_shopify/streams/fulfillment_orders.py +++ b/tap_shopify/streams/fulfillment_orders.py @@ -12,6 +12,30 @@ class FulfillmentOrders(Stream): data_key = "fulfillmentOrders" replication_key = "updatedAt" + # pylint: disable=W0221,fixme + def get_query_params(self, updated_at_min, updated_at_max, cursor=None): + """ + Construct query parameters for GraphQL requests. + + Args: + updated_at_min (str): Minimum updated_at timestamp. + updated_at_max (str): Maximum updated_at timestamp. + cursor (str): Pagination cursor, if any. + + Returns: + dict: Dictionary of query parameters. + """ + rkey = self.camel_to_snake(self.replication_key) + + params = { + "query": f"{rkey}:>='{updated_at_min}' AND {rkey}:<'{updated_at_max}'", + "first": self.results_per_page if self.results_per_page <= 30 else 30, + } + + if cursor: + params["after"] = cursor + return params + def transform_childitems(self, data, parent_id, key, next_page_key): """ Paginate child items. From 64e1c7e6f51ae9a34b991ad8a5b61ee2fd779619 Mon Sep 17 00:00:00 2001 From: Sourabh Gandhi <105213416+sgandhi1311@users.noreply.github.com> Date: Tue, 5 Aug 2025 15:17:11 +0530 Subject: [PATCH 50/66] =?UTF-8?q?Refactor=20Orders=20Stream:=20standard=20?= =?UTF-8?q?GraphQL=20=E2=86=92=20Bulk=20API=20Migration=20(#227)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Initial commit for the shipping lines stream * update query * Add order_shipping_lines stream * Update changelog and bump version * Implement dynamic query builder * fix pylint and unit test * add retailLocation and DiscountCodeApplication fields to the orders schema * Update changelog * add location id in the orders.fulfillments * update pylint command * implement bulk query operation * working code * working code * Add requests module in the required * increase the poll time to 1 hour * Update logger statement and add date_window_size in the __init__ * set the max time limit for polling to 2 hours * Revert "Update logger statement and add date_window_size in the __init__" This reverts commit 4214f6feb24d1f185a4d328f3f466d649067bc73. * Add bulk operation ID in the logger statement * add wait time to 3 hours from 2 hours * fetch the last bulk operation id if it wasn't processed * fix pylint * add location and retail location back to the query * update wait time * Adjust the exceptions and few corner case scenarios * include unsupported fields as well * clear previous bulk id * customer specific, remove older bulk operation ID * update the order ID * Optimise query * include x-request-id in the log * Add new fields in the orders stream * update bulp op id * I have updated the poll wait time to 23 hours * add createdAt field in the fulfillments stream * display JSONL file size * Shopify claims that parent and child records will always be clubbed together * remove requests dependency as it's added in dev * update known missing field in the all fields test * add requests package as required * add customAttributes and number fields (#232) * setup and changelog changes * reduce the date window size for the bulk query operation to complete * clear previous bulk operation details if the date window size is modified * add custom message for the already bulk op running * remove orders from the few integration tests to save the time * make pylint happy * trying to reduce the circleci execution time * increase date window size for tests * remove orders from automatic test to save time --------- Co-authored-by: prijendev Co-authored-by: Robert Hill <30398720+rjh336@users.noreply.github.com> --- CHANGELOG.md | 3 + setup.py | 5 +- tap_shopify/__init__.py | 8 +- tap_shopify/context.py | 7 + tap_shopify/exceptions.py | 3 + tap_shopify/schemas/orders.json | 261 +++- tap_shopify/streams/base.py | 8 +- tap_shopify/streams/orders.py | 2350 +++++++++++++++++-------------- tests/test_all_fields.py | 3 +- tests/test_automatic_fields.py | 2 +- tests/test_bookmarks_updated.py | 3 +- tests/test_pagination.py | 7 +- tests/test_start_date.py | 1 + 13 files changed, 1540 insertions(+), 1121 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43fd0d8f..d0e008d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +### 3.7.0 + * Refactor Orders Stream: standard GraphQL → Bulk API Migration [#227](https://github.com/singer-io/tap-shopify/pull/227) + ### 3.6.2 * Set max_size of pagination to 30 for fullfilment_orders [#230](https://github.com/singer-io/tap-shopify/pull/230) diff --git a/setup.py b/setup.py index 44e4f5e2..d02c439d 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="3.6.2", + version="3.7.0", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", @@ -15,7 +15,8 @@ # Important: review the monkey-patched method in the GraphQL client when upgrading this dependency. "ShopifyAPI==12.7.0", "singer-python==6.1.1", - "graphql-core==3.2.6" + "graphql-core==3.2.6", + 'requests==2.32.4', ], extras_require={ 'dev': [ diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index e880c5cb..7add471a 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -13,8 +13,8 @@ from singer import metadata from singer import Transformer from tap_shopify.context import Context -from tap_shopify.exceptions import ShopifyError -from tap_shopify.streams.base import shopify_error_handling, get_request_timeout, ShopifyAPIError +from tap_shopify.exceptions import ShopifyError, ShopifyAPIError +from tap_shopify.streams.base import shopify_error_handling, get_request_timeout REQUIRED_CONFIG_KEYS = ["shop", "api_key"] LOGGER = singer.get_logger() @@ -25,7 +25,7 @@ def initialize_shopify_client(): api_key = Context.config['api_key'] shop = Context.config['shop'] - version = '2025-01' + version = '2025-07' session = shopify.Session(shop, version, api_key) shopify.ShopifyResource.activate_session(session) @@ -246,6 +246,8 @@ def main(): raise ShopifyError(exc, msg) from exc except ShopifyError as error: raise error + except ShopifyAPIError as error: + raise error except Exception as exc: raise ShopifyError(exc) from exc diff --git a/tap_shopify/context.py b/tap_shopify/context.py index fdd0a6c7..dca79f79 100644 --- a/tap_shopify/context.py +++ b/tap_shopify/context.py @@ -41,6 +41,13 @@ def get_unselected_fields(cls, stream_name): return list(all_fields - selected_fields) + @classmethod + def get_all_fields(cls, stream_name): + stream = cls.get_catalog_entry(stream_name) + # All fields defined in the schema + all_fields = set(stream["schema"]["properties"].keys()) + return all_fields + @classmethod def get_results_per_page(cls, default_results_per_page): results_per_page = default_results_per_page diff --git a/tap_shopify/exceptions.py b/tap_shopify/exceptions.py index 94fa62dd..417a01c6 100644 --- a/tap_shopify/exceptions.py +++ b/tap_shopify/exceptions.py @@ -1,3 +1,6 @@ class ShopifyError(Exception): def __init__(self, error, msg=''): super().__init__('{}\n{}'.format(error.__class__.__name__, msg)) + +class ShopifyAPIError(Exception): + """Raised for any unexpected api error without a valid status code""" diff --git a/tap_shopify/schemas/orders.json b/tap_shopify/schemas/orders.json index 000f6398..eea4bb33 100644 --- a/tap_shopify/schemas/orders.json +++ b/tap_shopify/schemas/orders.json @@ -299,6 +299,32 @@ } }, "currentTotalWeight": { "type": ["null", "number"] }, + "customAttributes": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "key": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "string" + ] + } + } + } + }, "customer": { "type": [ "null", @@ -593,23 +619,63 @@ "null", "object" ], - "lastVisit": { - "type": [ - "null", - "object" - ], - "properties": { - "landingPage": { - "type": [ - "null", - "string" - ] - }, - "referrerUrl": { + "properties": { + "lastVisit": { "type": [ + "null", + "object" + ], + "properties": { + "landingPage": { + "type": [ + "null", + "string" + ] + }, + "referrerUrl": { + "type": [ + "null", + "string" + ] + } + } + }, + "utmParameters": { + "type": [ "null", - "string" - ] + "object" + ], + "properties": { + "campaign": { + "type": [ + "null", + "string" + ] + }, + "content": { + "type": [ + "null", + "string" + ] + }, + "medium": { + "type": [ + "null", + "string" + ] + }, + "source": { + "type": [ + "null", + "string" + ] + }, + "term": { + "type": [ + "null", + "string" + ] + } } } } @@ -683,6 +749,7 @@ }, "name": { "type": ["null", "string"] }, "note": { "type": ["null", "string"] }, + "number": { "type": ["null", "number"] }, "netPaymentSet": { "type": ["null", "object"], "properties": { @@ -1137,6 +1204,7 @@ "status": { "type": ["null", "string"] }, "totalQuantity": { "type": ["null", "number"] }, "updatedAt": { "type": ["null", "string"], "format": "date-time" }, + "createdAt": { "type": ["null", "string"], "format": "date-time" }, "deliveredAt": { "type": ["null", "string"], "format": "date-time" }, "estimatedDeliveryAt": { "type": ["null", "string"], "format": "date-time" }, "requiresShipping": { "type": ["null", "boolean"] }, @@ -1166,6 +1234,12 @@ }, "location": { "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "string"] } + } + }, + "location": { + "type": ["null", "object"], "properties": { "id": { "type": ["null", "string"] } } @@ -1963,6 +2037,67 @@ ] } } + }, + "lineItemGroup": { + "type": [ + "null", + "object" + ], + "properties": { + "customAttributes": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "key": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "quantity": { + "type": [ + "null", + "integer" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "variantId": { + "type": [ + "null", + "string" + ] + }, + "variantSku": { + "type": [ + "null", + "string" + ] + } + } } } } @@ -2187,6 +2322,100 @@ } } } - } + }, + "discountApplications": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "allocationMethod": { + "type": [ + "null", + "string" + ] + }, + "index": { + "type": [ + "null", + "integer" + ] + }, + "targetSelection": { + "type": [ + "null", + "string" + ] + }, + "targetType": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "object" + ], + "properties": { + "__typename": { + "type": [ + "null", + "string" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + }, + "currencyCode": { + "type": [ + "null", + "string" + ] + }, + "percentage": { + "type": [ + "null", + "number" + ] + } + } + }, + "__typename": { + "type": [ + "null", + "string" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "description": { + "type": [ + "null", + "string" + ] + }, + "code": { + "type": [ + "null", + "string" + ] + } + } + } + } } } diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index 2532d934..e7caabed 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -16,7 +16,7 @@ from graphql import parse, print_ast, visit from graphql.language import Visitor, FieldNode, SelectionSetNode, OperationDefinitionNode, NameNode from tap_shopify.context import Context -from tap_shopify.exceptions import ShopifyError +from tap_shopify.exceptions import ShopifyError, ShopifyAPIError LOGGER = singer.get_logger() @@ -126,12 +126,6 @@ def wrapper(*args, **kwargs): return fnc(*args, **kwargs) return wrapper -class Error(Exception): - """Base exception for the API interaction module""" - -class ShopifyAPIError(Error): - """Raised for any unexpected api error without a valid status code""" - class Stream(): # Used for bookmarking and stream identification. Is overridden by # subclasses to change the bookmark key. diff --git a/tap_shopify/streams/orders.py b/tap_shopify/streams/orders.py index b4ed6c98..35d0f98e 100644 --- a/tap_shopify/streams/orders.py +++ b/tap_shopify/streams/orders.py @@ -1,1117 +1,1295 @@ +from datetime import timedelta +import json +import time +import re +import requests +import shopify +import singer +from singer import metrics, utils from tap_shopify.context import Context from tap_shopify.streams.base import Stream +from tap_shopify.exceptions import ShopifyAPIError +LOGGER = singer.get_logger() class Orders(Stream): - """Stream class for Shopify Orders.""" - name = "orders" data_key = "orders" replication_key = "updatedAt" - # pylint: disable=W0221,fixme - def get_query_params(self, updated_at_min, updated_at_max, cursor=None): + def get_query(self): + """ + Returns the GraphQL query string for the bulk operation. + The date filters will be injected via the bulk operation variables. + """ + return """ + { + orders(query: "%s") { + edges { + node { + additionalFees { + id + name + price { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + taxLines { + channelLiable + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + rate + ratePercentage + source + title + } + } + app { + id + name + icon { + id + } + } + billingAddress { + address1 + address2 + city + company + coordinatesValidated + country + countryCodeV2 + firstName + formattedArea + id + lastName + latitude + longitude + name + phone + province + provinceCode + timeZone + validationResultSummary + zip + } + billingAddressMatchesShippingAddress + canMarkAsPaid + canNotifyCustomer + cancelReason + cancellation { + staffNote + } + cancelledAt + capturable + cartDiscountAmountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + channelInformation { + id + channelId + } + clientIp + closed + closedAt + confirmationNumber + confirmed + createdAt + currencyCode + currentCartDiscountAmountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentShippingPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentSubtotalLineItemsQuantity + currentSubtotalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentTotalAdditionalFeesSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentTotalDiscountsSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentTotalDutiesSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentTotalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentTotalTaxSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + currentTotalWeight + customAttributes { + key + value + } + customer { + id + email + firstName + lastName + addresses { + address1 + address2 + city + countryCodeV2 + country + company + firstName + lastName + id + name + phone + province + provinceCode + zip + } + state + verifiedEmail + updatedAt + taxExempt + tags + taxExemptions + note + multipassIdentifier + createdAt + defaultAddress { + address1 + address2 + city + company + country + countryCodeV2 + firstName + id + lastName + name + province + phone + provinceCode + zip + } + } + customerJourneySummary { + lastVisit { + landingPage + referrerUrl + } + } + merchantOfRecordApp { + id + } + customerAcceptsMarketing + customerLocale + discountCodes + discountCode + displayFinancialStatus + displayFulfillmentStatus + disputes { + id + initiatedAs + status + } + dutiesIncluded + email + edited + estimatedTaxes + fulfillable + fullyPaid + hasTimelineComment + fulfillmentsCount { + count + precision + } + id + legacyResourceId + merchantBusinessEntity { + address { + address1 + address2 + city + countryCode + province + zip + } + companyName + displayName + id + primary + } + name + note + netPaymentSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + number + originalTotalAdditionalFeesSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + originalTotalDutiesSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + originalTotalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + paymentGatewayNames + phone + poNumber + presentmentCurrencyCode + processedAt + refundable + refundDiscrepancySet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + registeredSourceUrl + requiresShipping + restockable + returnStatus + shippingAddress { + address1 + address2 + city + company + coordinatesValidated + country + countryCodeV2 + firstName + formattedArea + id + lastName + latitude + longitude + name + phone + province + provinceCode + timeZone + validationResultSummary + zip + } + shopifyProtect { + eligibility { + status + } + status + } + sourceIdentifier + sourceName + statusPageUrl + subtotalLineItemsQuantity + subtotalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + tags + taxExempt + taxLines { + channelLiable + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + rate + ratePercentage + source + title + } + taxesIncluded + test + totalCapturableSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalCashRoundingAdjustment { + paymentSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + refundSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + } + totalDiscountsSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalOutstandingSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalReceivedSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalRefundedSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalRefundedShippingSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalShippingPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalTaxSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalTipReceivedSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalWeight + transactionsCount { + count + precision + } + unpaid + updatedAt + fulfillments { + id + name + status + totalQuantity + updatedAt + createdAt + deliveredAt + estimatedDeliveryAt + requiresShipping + inTransitAt + trackingInfo { + number + company + url + } + service { + serviceName + id + handle + trackingSupport + type + permitsSkuSharing + inventoryManagement + } + location { + id + } + } + lineItems { + edges { + node { + id + vendor + quantity + title + requiresShipping + originalTotalSet { + presentmentMoney { + currencyCode + amount + } + shopMoney { + amount + currencyCode + } + } + taxLines { + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + rate + title + source + channelLiable + } + taxable + isGiftCard + name + discountedTotalSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + sku + product { + id + } + discountAllocations { + allocatedAmountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + discountApplication { + index + targetType + targetSelection + allocationMethod + ... on AutomaticDiscountApplication { + title + } + ... on ManualDiscountApplication { + title + } + ... on ScriptDiscountApplication { + title + } + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + } + customAttributes { + key + value + } + totalDiscountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + duties { + harmonizedSystemCode + id + taxLines { + rate + source + title + channelLiable + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + } + countryCodeOfOrigin + } + discountedUnitPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + originalUnitPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + unfulfilledDiscountedTotalSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + unfulfilledOriginalTotalSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + variant { + id + } + lineItemGroup { + customAttributes { + key + value + } + id + quantity + title + variantId + variantSku + } + } + } + } + shippingLine { + carrierIdentifier + code + currentDiscountedPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + custom + deliveryCategory + discountAllocations { + allocatedAmountSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + discountApplication { + allocationMethod + index + targetSelection + targetType + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + ... on AutomaticDiscountApplication { + __typename + allocationMethod + index + targetSelection + targetType + title + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + ... on DiscountCodeApplication { + __typename + allocationMethod + code + index + targetSelection + targetType + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + ... on ManualDiscountApplication { + description + allocationMethod + index + targetSelection + targetType + title + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + ... on ScriptDiscountApplication { + __typename + allocationMethod + index + targetSelection + targetType + title + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + } + } + } + discountedPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + id + isRemoved + originalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + phone + shippingRateHandle + source + taxLines { + channelLiable + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + rate + ratePercentage + source + title + } + title + } + retailLocation { + activatable + address { + address1 + address2 + city + country + countryCode + formatted + latitude + longitude + phone + province + provinceCode + zip + } + addressVerified + createdAt + deactivatable + deactivatedAt + deletable + fulfillmentService { + id + } + fulfillsOnlineOrders + hasActiveInventory + hasUnfulfilledOrders + id + isActive + isFulfillmentService + legacyResourceId + localPickupSettingsV2 { + instructions + pickupTime + } + name + shipsInventory + updatedAt + suggestedAddresses { + address1 + address2 + city + country + countryCode + formatted + province + provinceCode + zip + } + } + discountApplications { + edges { + node { + allocationMethod + index + targetSelection + targetType + value { + ... on MoneyV2 { + __typename + amount + currencyCode + } + ... on PricingPercentageValue { + __typename + percentage + } + } + ... on AutomaticDiscountApplication { + __typename + title + } + ... on DiscountCodeApplication { + __typename + code + } + ... on ManualDiscountApplication { + title + description + } + ... on ScriptDiscountApplication { + __typename + title + } + } + } + } + } + } + } + } """ - Construct query parameters for GraphQL requests. - Args: - updated_at_min (str): Minimum updated_at timestamp. - updated_at_max (str): Maximum updated_at timestamp. - cursor (str): Pagination cursor, if any. + def update_bookmark(self, bookmark_value, bookmark_key=None, bulk_op_metadata=None): + # Standard Singer bookmark + singer.write_bookmark( + Context.state, + self.name, + bookmark_key or self.replication_key, + bookmark_value + ) - Returns: - dict: Dictionary of query parameters. - """ - rkey = self.camel_to_snake(self.replication_key) + # Store under orders -> bulk_operation + if bulk_op_metadata: + orders_bookmark = Context.state.setdefault("bookmarks", {}).setdefault("orders", {}) + orders_bookmark["bulk_operation"] = bulk_op_metadata + + singer.write_state(Context.state) - # TODO: In future once the dynamic query generation logic is setup remove the below - # condition for the orders stream. As by default we will ask the customers to select - # few fields or reduce the page size - params = { - "query": f"{rkey}:>='{updated_at_min}' AND {rkey}:<'{updated_at_max}'", - "first": self.results_per_page if self.results_per_page <= 150 else 150, + def build_query_filter(self, updated_at_min, updated_at_max): + return f"updated_at:>='{updated_at_min}' AND updated_at:<'{updated_at_max}'" + + def submit_bulk_query(self, query_string): + url = f"https://{Context.config.get('shop')}.myshopify.com/admin/api/2025-07/graphql.json" + headers = { + "Content-Type": "application/json", + "X-Shopify-Access-Token": Context.config.get("api_key"), + } + operation = { + "query": """ + mutation bulkOperationRunQuery($query: String!) { + bulkOperationRunQuery(query: $query) { + bulkOperation { + id + status + createdAt + } + userErrors { + field + message + } + } + } + """, + "variables": { + "query": query_string + } } + response = requests.post(url, headers=headers, json=operation, timeout=300) + LOGGER.info("X-request-ID for the bulk operation: %s", response.headers.get("X-Request-ID")) - if cursor: - params["after"] = cursor - return params + return response.json() - def transform_lineitems(self, data): - """ - Transforms the order lineitems data by extracting order IDs and handling pagination. + def poll_bulk_completion(self, current_bookmark, bulk_op_id, timeout=82800): + def fetch_bulk_operation(op_id): + query = f""" + {{ + node(id: "{op_id}") {{ + ... on BulkOperation {{ + id + status + errorCode + createdAt + completedAt + objectCount + fileSize + url + }} + }} + }} + """ + response = json.loads(shopify.GraphQL().execute(query=query)) + if not isinstance(response, dict): + raise ShopifyAPIError(f"Unexpected GraphQL response: {response}") + return response.get("data", {}).get("node") - Args: - data (dict): Order data. + start = time.time() + last_status = None - Returns: - list: List of lineItems. - """ - lineitems = [ - node for item in data["lineItems"]["edges"] - if (node := item.get("node")) - ] + while time.time() - start < timeout: + op = fetch_bulk_operation(bulk_op_id) - # TODO: In future once the dynamic query generation logic is setup remove the below - # condition for the orders stream. Moving forward we should ask the customers to select - # few fields or reduce the page size - # Handle pagination - page_info = data["lineItems"].get("pageInfo", {}) - query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) - while page_info.get("hasNextPage"): - params = { - "first": self.results_per_page if self.results_per_page <= 150 else 150, - "query": f"id:{data['id'].split('/')[-1]}", - "childafter": page_info.get("endCursor"), - } + if not op: + LOGGER.warning("Bulk operation not found: %s", bulk_op_id) + return None + if not isinstance(op, dict): + raise ShopifyAPIError(f"Unexpected bulk operation format: {op}") - # Fetch the next page of data - response = self.call_api(params, query=query) - lineitems_data = response.get("edges", [])[0].get("node", {}) - lineitems.extend( - node for item in lineitems_data["lineItems"]["edges"] - if (node := item.get("node")) - ) - page_info = lineitems_data.get("pageInfo", {}) + current_status = op.get("status") - return lineitems + if current_status != last_status: + LOGGER.info( + "Bulk operation - %s, status: %s, created at - %s, completed at - %s", + op.get("id"), + current_status, + op.get("createdAt"), + op.get("completedAt") or "N/A" + ) + last_status = current_status - def transform_object(self, obj): - """ - Transforms a collection object. + if current_status == "COMPLETED": + LOGGER.info("Bulk operation completed. File size: %s bytes", op.get("fileSize")) + return op.get("url") - Args: - obj (dict): Collection object. + if current_status in ["FAILED", "CANCELED"]: + raise ShopifyAPIError(f"Bulk operation failed: {op.get('errorCode')}") - Returns: - dict: Transformed collection object. - """ - if obj.get("lineItems"): - obj["lineItems"] = self.transform_lineitems(obj) - return obj + time.sleep(60) - def get_query(self): - """ - Returns the GraphQL query for fetching orders. + # Save bookmark if timeout occurs + self.update_bookmark( + bookmark_value=utils.strftime(current_bookmark), + bulk_op_metadata={ + "bulk_operation_id": op.get("id"), + "status": op.get("status"), + "created_at": op.get("createdAt"), + "last_date_window": self.date_window_size, + } + ) + + elapsed = int(time.time() - start) + raise ShopifyAPIError( + f"Bulk operation id - {op.get('id') or 'UNKNOWN'} did not complete " + f"within {elapsed} seconds. " + "Please contact Shopify support with the operation ID for assistance." + ) - Returns: - str: GraphQL query string. + # pylint: disable=unsupported-assignment-operation + def parse_bulk_jsonl(self, url): """ - return """ - query orders($first: Int!, $after: String, $query: String, $childafter: String) { - orders(first: $first, after: $after, query: $query, sortKey: UPDATED_AT) { - edges { - node { - additionalFees { - id - name - price { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - taxLines { - channelLiable - priceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - rate - ratePercentage - source - title - } - } - app { - id - name - icon { - id - } - } - billingAddress { - address1 - address2 - city - company - coordinatesValidated - country - countryCodeV2 - firstName - formattedArea - id - lastName - latitude - longitude - name - phone - province - provinceCode - timeZone - validationResultSummary - zip - } - billingAddressMatchesShippingAddress - canMarkAsPaid - canNotifyCustomer - cancelReason - cancellation { - staffNote - } - cancelledAt - capturable - cartDiscountAmountSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - channelInformation { - id - channelId - } - clientIp - closed - closedAt - confirmationNumber - confirmed - createdAt - currencyCode - currentCartDiscountAmountSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - currentShippingPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - currentSubtotalLineItemsQuantity - currentSubtotalPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - currentTotalAdditionalFeesSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - currentTotalDiscountsSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - currentTotalDutiesSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - currentTotalPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - currentTotalTaxSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - currentTotalWeight - customer { - id - email - firstName - lastName - addresses { - address1 - address2 - city - countryCodeV2 - country - company - firstName - lastName - id - name - phone - province - provinceCode - zip - } - state - verifiedEmail - updatedAt - taxExempt - tags - taxExemptions - note - multipassIdentifier - createdAt - defaultAddress { - address1 - address2 - city - company - country - countryCodeV2 - firstName - id - lastName - name - province - phone - provinceCode - zip - } - } - customerJourneySummary { - lastVisit { - landingPage - referrerUrl - } - } - merchantOfRecordApp { - id - } - customerAcceptsMarketing - customerLocale - discountCodes - discountCode - displayFinancialStatus - displayFulfillmentStatus - disputes { - id - initiatedAs - status - } - dutiesIncluded - email - edited - estimatedTaxes - fulfillable - fullyPaid - hasTimelineComment - fulfillmentsCount { - count - precision - } - id - legacyResourceId - merchantBusinessEntity { - address { - address1 - address2 - city - countryCode - province - zip - } - companyName - displayName - id - primary - } - name - note - netPaymentSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - originalTotalAdditionalFeesSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - originalTotalDutiesSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - originalTotalPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - paymentGatewayNames - phone - poNumber - presentmentCurrencyCode - processedAt - refundable - refundDiscrepancySet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - registeredSourceUrl - requiresShipping - restockable - returnStatus - shippingAddress { - address1 - address2 - city - company - coordinatesValidated - country - countryCodeV2 - firstName - formattedArea - id - lastName - latitude - longitude - name - phone - province - provinceCode - timeZone - validationResultSummary - zip - } - shopifyProtect { - eligibility { - status - } - status - } - sourceIdentifier - sourceName - statusPageUrl - subtotalLineItemsQuantity - subtotalPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - tags - taxExempt - taxLines { - channelLiable - priceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - rate - ratePercentage - source - title - } - taxesIncluded - test - totalCapturableSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - totalCashRoundingAdjustment { - paymentSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - refundSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - } - totalDiscountsSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - totalOutstandingSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - totalPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - totalReceivedSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - totalRefundedSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - totalRefundedShippingSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - totalShippingPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - totalTaxSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - totalTipReceivedSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - totalWeight - transactionsCount { - count - precision - } - unpaid - updatedAt - fulfillments(first: 250) { - id - name - status - totalQuantity - updatedAt - createdAt - deliveredAt - estimatedDeliveryAt - requiresShipping - inTransitAt - trackingInfo(first: 250) { - number - company - url - } - service { - serviceName - id - handle - trackingSupport - type - permitsSkuSharing - inventoryManagement - } - location { - id - } - } - lineItems(first: 25, after: $childafter) { - edges { - node { - id - vendor - quantity - title - requiresShipping - originalTotalSet { - presentmentMoney { - currencyCode - amount - } - shopMoney { - amount - currencyCode - } - } - taxLines(first: 250) { - priceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - rate - title - source - channelLiable - } - taxable - isGiftCard - name - discountedTotalSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - sku - product { - id - } - discountAllocations { - allocatedAmountSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - discountApplication { - index - targetType - targetSelection - allocationMethod - ... on AutomaticDiscountApplication { - title - allocationMethod - index - targetSelection - targetType - value { - ... on MoneyV2 { - __typename - amount - currencyCode - } - ... on PricingPercentageValue { - __typename - percentage - } - } - } - ... on ManualDiscountApplication { - title - allocationMethod - description - index - targetSelection - targetType - value { - ... on MoneyV2 { - __typename - amount - currencyCode - } - ... on PricingPercentageValue { - __typename - percentage - } - } - } - ... on ScriptDiscountApplication { - title - allocationMethod - index - targetSelection - targetType - value { - ... on MoneyV2 { - __typename - amount - currencyCode - } - ... on PricingPercentageValue { - __typename - percentage - } - } - } - value { - ... on MoneyV2 { - __typename - amount - currencyCode - } - ... on PricingPercentageValue { - __typename - percentage - } - } - ... on DiscountCodeApplication { - __typename - allocationMethod - code - index - targetSelection - targetType - value { - ... on MoneyV2 { - __typename - amount - currencyCode - } - ... on PricingPercentageValue { - __typename - percentage - } - } - } - } - } - customAttributes { - key - value - } - totalDiscountSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - duties { - harmonizedSystemCode - id - taxLines { - rate - source - title - channelLiable - priceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - } - countryCodeOfOrigin - } - discountedUnitPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - originalUnitPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - unfulfilledDiscountedTotalSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - unfulfilledOriginalTotalSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - variant { - id - } - } - } - pageInfo { - hasNextPage - endCursor - } - } - shippingLine { - carrierIdentifier - code - currentDiscountedPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - custom - deliveryCategory - discountAllocations { - allocatedAmountSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - discountApplication { - allocationMethod - index - targetSelection - targetType - value { - ... on MoneyV2 { - __typename - amount - currencyCode - } - ... on PricingPercentageValue { - __typename - percentage - } - } - ... on AutomaticDiscountApplication { - __typename - allocationMethod - index - targetSelection - targetType - title - value { - ... on MoneyV2 { - __typename - amount - currencyCode - } - ... on PricingPercentageValue { - __typename - percentage - } - } - } - ... on DiscountCodeApplication { - __typename - allocationMethod - code - index - targetSelection - targetType - value { - ... on MoneyV2 { - __typename - amount - currencyCode - } - ... on PricingPercentageValue { - __typename - percentage - } - } - } - ... on ManualDiscountApplication { - description - allocationMethod - index - targetSelection - targetType - title - value { - ... on MoneyV2 { - __typename - amount - currencyCode - } - ... on PricingPercentageValue { - __typename - percentage - } - } - } - ... on ScriptDiscountApplication { - __typename - allocationMethod - index - targetSelection - targetType - title - value { - ... on MoneyV2 { - __typename - amount - currencyCode - } - ... on PricingPercentageValue { - __typename - percentage - } - } - } - } - } - discountedPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - id - isRemoved - originalPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - phone - shippingRateHandle - source - taxLines { - channelLiable - priceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - rate - ratePercentage - source - title - } - title - } - retailLocation { - activatable - address { - address1 - address2 - city - country - countryCode - formatted - latitude - longitude - phone - province - provinceCode - zip - } - addressVerified - createdAt - deactivatable - deactivatedAt - deletable - fulfillmentService { - id - } - fulfillsOnlineOrders - hasActiveInventory - hasUnfulfilledOrders - id - isActive - isFulfillmentService - legacyResourceId - localPickupSettingsV2 { - instructions - pickupTime - } - name - shipsInventory - updatedAt - suggestedAddresses { - address1 - address2 - city - country - countryCode - formatted - province - provinceCode - zip - } - } - } - } - pageInfo { - endCursor - hasNextPage - } - } - } + Streams and yields one order at a time, with its associated line items, + without holding all orders/line_items in memory. """ + resp = requests.get(url, stream=True, timeout=60) + current_order = None + current_line_items = [] + + for line in resp.iter_lines(): + if not line: + continue + rec = json.loads(line) + if not isinstance(rec, dict): + LOGGER.warning("Skipping unexpected JSONL line (not a dict): %s", rec) + continue + # Detect line item (child) or order (parent) + if '__parentId' in rec: + # It's a line item belonging to current_order + current_line_items.append(rec) + else: + if current_order: + current_order["lineItems"] = current_line_items + yield current_order + # Start tracking new parent group + current_order = rec + current_line_items = [] + # Yield the last parent group (if exists) + if current_order: + current_order["lineItems"] = current_line_items + yield current_order + + def transform_object(self, obj): + if obj.get("lineItems", {}).get("edges"): + obj["lineItems"] = [item["node"] for item in obj["lineItems"]["edges"]] + return obj + + def clear_bulk_operation_state(self): + orders_bookmark = Context.state.get("bookmarks", {}).get("orders", {}) + if "bulk_operation" in orders_bookmark: + del orders_bookmark["bulk_operation"] + singer.write_state(Context.state) + + # pylint: disable=too-many-locals,too-many-statements + def get_objects(self): + last_updated_at = self.get_bookmark() + current_bookmark = last_updated_at + sync_start = utils.now().replace(microsecond=0) + query_template = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) + LOGGER.info( + "GraphQL query for stream '%s': %s", + self.name, + ' '.join(query_template.split()) + ) + + bulk_op = Context.state.get("bookmarks", {}).get("orders", {}).get("bulk_operation") + op_id = None + existing_url = None + + if bulk_op: + if bulk_op.get("last_date_window") != self.date_window_size: + LOGGER.info( + "Clearing existing bulk operation state due to date " + "window size change from %s to %s", + bulk_op.get("last_date_window"), + self.date_window_size + ) + self.clear_bulk_operation_state() + + else: + op_id = bulk_op.get("bulk_operation_id") + op_status = bulk_op.get("status") + + if op_status in ["RUNNING", "COMPLETED"]: + LOGGER.info("Resuming polling for existing bulk operation ID: %s", op_id) + existing_url = self.poll_bulk_completion(current_bookmark, op_id) + else: + self.clear_bulk_operation_state() + + while last_updated_at < sync_start: + date_window_end = last_updated_at + timedelta(days=self.date_window_size) + query_end = min(sync_start, date_window_end) + + if not existing_url: + with metrics.http_request_timer(self.name): + query_filter = self.build_query_filter( + utils.strftime(last_updated_at), + utils.strftime(query_end) + ) + query = query_template % query_filter + LOGGER.info("Fetching records in date range: %s", query_filter) + + bulk_op_data = self.submit_bulk_query(query) + + user_errors = ( + bulk_op_data.get("data", {}) + .get("bulkOperationRunQuery", {}) + .get("userErrors") + ) + if user_errors: + for error in user_errors: + message = error.get("message", "") + if ( + "bulk query operation for this app and shop is already in progress" + in message + ): + # Try to extract BulkOperation ID using regex + match = re.search( + r"gid://shopify/BulkOperation/\d+", message + ) + bulk_op_id = match.group(0) if match else "UNKNOWN" + raise ShopifyAPIError( + "Shopify limitation: Only one bulk operation of this type " + "is allowed at a time. A bulk operation is already in progress " + f"(ID: {bulk_op_id}). To proceed, please adjust the anchor time" + " to avoid overlapping operations." + ) + raise ShopifyAPIError("Bulk query error: {}".format(user_errors)) + + bulk_operation = ( + bulk_op_data.get("data", {}) + .get("bulkOperationRunQuery", {}) + .get("bulkOperation") + ) + bulk_op_id = bulk_operation.get("id") if bulk_operation else None + if not bulk_op_id: + raise ShopifyAPIError("Invalid bulk operation response: {}". + format(bulk_op_data)) + + existing_url = self.poll_bulk_completion(current_bookmark, bulk_op_id) + + if existing_url: + for obj in self.parse_bulk_jsonl(existing_url): + replication_value = utils.strptime_to_utc(obj[self.replication_key]) + current_bookmark = max(current_bookmark, replication_value) + + yield obj + + self.clear_bulk_operation_state() + else: + LOGGER.info("No data returned for the date range: %s to %s", + last_updated_at, query_end) + last_updated_at = query_end + max_bookmark_value = min(sync_start, current_bookmark) + self.update_bookmark(utils.strftime(max_bookmark_value)) + existing_url = None Context.stream_objects["orders"] = Orders diff --git a/tests/test_all_fields.py b/tests/test_all_fields.py index ff1e8179..e066a1c5 100644 --- a/tests/test_all_fields.py +++ b/tests/test_all_fields.py @@ -12,7 +12,8 @@ 'canDelete', 'rawMessage', 'canEdit', - } + }, + 'orders': {'discountApplications'} } class AllFieldsTest(BaseTapTest): diff --git a/tests/test_automatic_fields.py b/tests/test_automatic_fields.py index 8fa63207..5f64d873 100644 --- a/tests/test_automatic_fields.py +++ b/tests/test_automatic_fields.py @@ -21,7 +21,7 @@ def __init__(self, *args, **kwargs): def test_run(self): with self.subTest(store="store_1"): conn_id = self.create_connection(original_credentials=True) - self.automatic_test(conn_id, self.store_1_streams) + self.automatic_test(conn_id, self.store_1_streams - {"orders"}) with self.subTest(store="store_2"): conn_id = self.create_connection(original_properties=False, original_credentials=False) diff --git a/tests/test_bookmarks_updated.py b/tests/test_bookmarks_updated.py index fad0067a..b61dbe2a 100644 --- a/tests/test_bookmarks_updated.py +++ b/tests/test_bookmarks_updated.py @@ -31,7 +31,7 @@ def __init__(self, *args, **kwargs): # abandoned checkouts are saved in the Shopify admin for three months. # Every Monday, abandoned checkouts that are older than three months are removed from your admin. # Also no POST call is available for this endpoint: https://shopify.dev/api/admin-rest/2022-01/resources/abandoned-checkouts - store_2_streams = {'metafields_products', 'transactions', 'order_refunds', 'products', 'locations', 'inventory_items', 'customers', 'collections', 'orders', 'order_shipping_lines'} + store_2_streams = {'metafields_products', 'transactions', 'order_refunds', 'products', 'locations', 'inventory_items', 'customers', 'collections', 'order_shipping_lines'} def test_run_store_2(self): with self.subTest(store="store_2"): @@ -87,7 +87,6 @@ def bookmarks_test(self, conn_id, testable_streams): 'metafields_customer': '2025-01-21T13:28:24.000000Z', 'order_refunds': {'updatedAt': '2024-12-05T09:58:40.000000Z'}, 'customers': {'updatedAt': '2025-01-19T20:55:07.000000Z'}, - 'orders': {'updatedAt': '2024-12-05T09:59:35.000000Z'}, 'collections': {'updatedAt': '2025-01-21T13:29:06.000000Z'}, 'order_shipping_lines': {'updatedAt': '2021-09-30T01:02:21.000000Z'}, 'metafields_products': {'updatedAt': '2025-01-21T03:11:54.000000Z'} diff --git a/tests/test_pagination.py b/tests/test_pagination.py index 65773ae0..f4b8ab2e 100644 --- a/tests/test_pagination.py +++ b/tests/test_pagination.py @@ -28,9 +28,11 @@ def test_run(self): # limit of records returned in 1 page # Documentation: https://help.shopify.com/en/manual/locations/setting-up-your-locations # 'inventory_items': - # As it can call for max 100 product_variants and + # As it can call for max 100 product_variants and # we can generate only one inventory_item for one product_variants - excepted_streams = {'locations', 'inventory_items'} + # 'orders': + # Pagination is not supported for orders BULK API + excepted_streams = {'locations', 'inventory_items', 'orders'} with self.subTest(store="store_1"): conn_id = self.create_connection(original_credentials=True) @@ -40,7 +42,6 @@ def test_run(self): conn_id = self.create_connection(original_properties=False, original_credentials=False) self.pagination_test(conn_id, self.store_2_streams - excepted_streams) - def pagination_test(self, conn_id, testable_streams): """ Verify that for each stream you can get multiple pages of data diff --git a/tests/test_start_date.py b/tests/test_start_date.py index 54d793ea..d2284913 100644 --- a/tests/test_start_date.py +++ b/tests/test_start_date.py @@ -63,6 +63,7 @@ def test_run(self): # Also no POST call is available for this endpoint: https://shopify.dev/api/admin-rest/2022-01/resources/abandoned-checkouts expected_replication_method = self.expected_replication_method() expected_replication_method.pop("abandoned_checkouts") + expected_replication_method.pop("orders") incremental_streams = {key for key, value in expected_replication_method.items() if value == self.INCREMENTAL} From 04b7709b1dfd97038452317adf7a59d19afe3254 Mon Sep 17 00:00:00 2001 From: Jonathan Sphar <80364060+JJSphar@users.noreply.github.com> Date: Thu, 7 Aug 2025 03:28:20 -0700 Subject: [PATCH 51/66] Load all collection.products results, not just the first page. Closes #224 (#231) --- tap_shopify/streams/collections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_shopify/streams/collections.py b/tap_shopify/streams/collections.py index 3b9cedab..dc584207 100644 --- a/tap_shopify/streams/collections.py +++ b/tap_shopify/streams/collections.py @@ -37,7 +37,7 @@ def transform_products(self, data): # Fetch the next page of data response = self.call_api(params, query=query) - products_data = response.get("node", {}).get("products", {}) + products_data = response.get("edges", [{}])[0].get("node", {}).get("products", {}) product_ids.extend( node["id"] for item in products_data.get("edges", []) From 39300ea3df4c0279307befe035f981bb0f6de318 Mon Sep 17 00:00:00 2001 From: Sourabh Gandhi <105213416+sgandhi1311@users.noreply.github.com> Date: Thu, 7 Aug 2025 19:36:48 +0530 Subject: [PATCH 52/66] update setup and changelog (#233) --- CHANGELOG.md | 3 +++ setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0e008d9..dca6f526 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +### 3.7.1 + * Query and extract multiple pages of products for each collection [#227](https://github.com/singer-io/tap-shopify/pull/231) + ### 3.7.0 * Refactor Orders Stream: standard GraphQL → Bulk API Migration [#227](https://github.com/singer-io/tap-shopify/pull/227) diff --git a/setup.py b/setup.py index d02c439d..2c8ac3ef 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="3.7.0", + version="3.7.1", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", From 4ea44ed775c1a79af8b231ba2e270f01626a628d Mon Sep 17 00:00:00 2001 From: Ben Allred Date: Thu, 14 Aug 2025 12:39:17 -0600 Subject: [PATCH 53/66] SAC-28318: update circle config to use uv (#234) * update circle config to use uv * increase parallelism * revert parallelism increase --- .circleci/config.yml | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ff2ada48..421c33fb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,7 @@ orbs: executors: docker-executor: docker: - - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/circle-ci:stitch-tap-tester + - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/circle-ci:stitch-tap-tester-uv jobs: build: @@ -20,21 +20,23 @@ jobs: - run: name: 'Setup virtual env' command: | - python3 -mvenv /usr/local/share/virtualenvs/tap-shopify + uv venv --python 3.11 /usr/local/share/virtualenvs/tap-shopify source /usr/local/share/virtualenvs/tap-shopify/bin/activate - pip install -U 'pip==23.3.2' 'setuptools<56.0.0' - pip install .[dev] + uv pip install -U 'pip==23.3.2' 'setuptools<56.0.0' + uv pip install .[dev] + uv pip install --upgrade awscli - persist_to_workspace: - root: /usr/local/share/virtualenvs + root: / paths: - - tap-shopify - - dev_env.sh + - root/.local/share/uv/python + - usr/local/share/virtualenvs/tap-shopify + run_pylint: executor: docker-executor steps: - checkout - attach_workspace: - at: /usr/local/share/virtualenvs + at: / - run: name: 'pylint' command: | @@ -45,7 +47,7 @@ jobs: steps: - checkout - attach_workspace: - at: /usr/local/share/virtualenvs + at: / - run: name: 'JSON Validator' command: | @@ -56,13 +58,14 @@ jobs: steps: - checkout - attach_workspace: - at: /usr/local/share/virtualenvs + at: / - run: name: 'Unit Tests' command: | source /usr/local/share/virtualenvs/tap-shopify/bin/activate - pip install nose2 parameterized nose2[coverage_plugin]>=0.6.5 - nose2 --with-coverage -v -s tests/unittests + pip install pytest coverage parameterized nose2[coverage_plugin]>=0.6.5 + coverage run -m pytest tests/unittests + coverage html - store_test_results: path: test_output/report.xml - store_artifacts: @@ -73,10 +76,11 @@ jobs: steps: - checkout - attach_workspace: - at: /usr/local/share/virtualenvs + at: / - run: name: 'Integration Tests' command: | + source /usr/local/share/virtualenvs/tap-shopify/bin/activate aws s3 cp s3://com-stitchdata-dev-deployment-assets/environments/tap-tester/tap_tester_sandbox dev_env.sh source dev_env.sh mkdir /tmp/${CIRCLE_PROJECT_REPONAME} From 0ca13f8d2d3a759721b38b3ad46c9235365a9fdd Mon Sep 17 00:00:00 2001 From: Bryant Gray Date: Thu, 4 Sep 2025 10:01:13 -0400 Subject: [PATCH 54/66] upgrade circleci build to use python 3.12 (#236) * upgrade circleci build to use python 3.12 * assertEquals -> assertEqual --- .circleci/config.yml | 4 ++-- tests/unittests/test_date_window.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 421c33fb..1aa31125 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,9 +20,9 @@ jobs: - run: name: 'Setup virtual env' command: | - uv venv --python 3.11 /usr/local/share/virtualenvs/tap-shopify + uv venv --python 3.12 /usr/local/share/virtualenvs/tap-shopify source /usr/local/share/virtualenvs/tap-shopify/bin/activate - uv pip install -U 'pip==23.3.2' 'setuptools<56.0.0' + uv pip install -U pip setuptools uv pip install .[dev] uv pip install --upgrade awscli - persist_to_workspace: diff --git a/tests/unittests/test_date_window.py b/tests/unittests/test_date_window.py index a0796812..402dc8d7 100644 --- a/tests/unittests/test_date_window.py +++ b/tests/unittests/test_date_window.py @@ -10,7 +10,7 @@ def test_no_date_window_value(self): "start_date": "2025-01-01T00:00:00Z", } streams = Stream() - self.assertEquals(streams.date_window_size, 30) + self.assertEqual(streams.date_window_size, 30) def test_valid_int_date_window_value(self): """Test that valid integer value for date_window is handled correctly.""" @@ -19,7 +19,7 @@ def test_valid_int_date_window_value(self): "date_window_size": 10 } streams = Stream() - self.assertEquals(streams.date_window_size, 10) + self.assertEqual(streams.date_window_size, 10) def test_valid_str_date_window_value(self): """Test that valid str value for date_window is handled correctly..""" @@ -28,7 +28,7 @@ def test_valid_str_date_window_value(self): "date_window_size": "10" } streams = Stream() - self.assertEquals(streams.date_window_size, 10) + self.assertEqual(streams.date_window_size, 10) def test_valid_float_date_window_value(self): """Test that valid float value for date_window is handled correctly.""" @@ -37,7 +37,7 @@ def test_valid_float_date_window_value(self): "date_window_size": "11.00" } streams = Stream() - self.assertEquals(streams.date_window_size, 11) + self.assertEqual(streams.date_window_size, 11) def test_zero_str_date_window_value(self): """Test that valid zero string value for date_window is handled correctly.""" @@ -46,7 +46,7 @@ def test_zero_str_date_window_value(self): "date_window_size": "0" } streams = Stream() - self.assertEquals(streams.date_window_size, 30) + self.assertEqual(streams.date_window_size, 30) def test_zero_int_date_window_value(self): """Test that valid zero integer value for date_window is handled correctly.""" @@ -55,4 +55,4 @@ def test_zero_int_date_window_value(self): "date_window_size": 0 } streams = Stream() - self.assertEquals(streams.date_window_size, 30) + self.assertEqual(streams.date_window_size, 30) From 9436167b2eeadebedff7c1b0ab4238123f19708c Mon Sep 17 00:00:00 2001 From: Sourabh Gandhi <105213416+sgandhi1311@users.noreply.github.com> Date: Wed, 17 Sep 2025 09:41:07 +0530 Subject: [PATCH 55/66] New automatic_keys Support and Bulk State Persistence Logic (#237) * Store the bulk operation details in the state once it is complete for the extraction if the existing fails due to the time limit of 23 hours * add automatic keys properties * update the automatic key name * Increase the chunk size to 1MB * remove chunk size modifications * update setup and changelog * update discovery test * update automatic test for the order_refunds stream * include extra automatic keys logic in the automatic test --- CHANGELOG.md | 3 +++ setup.py | 2 +- tap_shopify/__init__.py | 3 ++- tap_shopify/streams/base.py | 1 + tap_shopify/streams/order_refunds.py | 1 + tap_shopify/streams/orders.py | 12 ++++++++++-- tests/test_automatic_fields.py | 4 +++- tests/test_discovery.py | 4 ++++ 8 files changed, 25 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dca6f526..a0a117ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +### 3.7.2 + * New automatic_keys Support and Bulk State Persistence Logic [#237](https://github.com/singer-io/tap-shopify/pull/237) + ### 3.7.1 * Query and extract multiple pages of products for each collection [#227](https://github.com/singer-io/tap-shopify/pull/231) diff --git a/setup.py b/setup.py index 2c8ac3ef..19fa631f 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="3.7.1", + version="3.7.2", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index 7add471a..0544dad6 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -86,7 +86,8 @@ def get_discovery_metadata(stream, schema): mdata = metadata.write(mdata, (), 'valid-replication-keys', [stream.replication_key]) for field_name in schema['properties'].keys(): - if field_name in stream.key_properties or field_name == stream.replication_key: + if field_name in stream.key_properties or field_name == stream.replication_key \ + or field_name in stream.automatic_keys: mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'automatic') elif field_name in UNSUPPORTED_FIELDS and not has_read_users_access(): mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'unsupported') diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index e7caabed..ada0aa81 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -132,6 +132,7 @@ class Stream(): name = None replication_method = 'INCREMENTAL' replication_key = 'updatedAt' + automatic_keys = [] key_properties = ['id'] date_window_size = None data_key = None diff --git a/tap_shopify/streams/order_refunds.py b/tap_shopify/streams/order_refunds.py index 175580e3..1b11cd16 100644 --- a/tap_shopify/streams/order_refunds.py +++ b/tap_shopify/streams/order_refunds.py @@ -11,6 +11,7 @@ class OrderRefunds(Stream): data_key = "orders" child_data_key = "refunds" replication_key = "updatedAt" + automatic_keys = ["order"] # pylint: disable=too-many-locals def get_objects(self): diff --git a/tap_shopify/streams/orders.py b/tap_shopify/streams/orders.py index 35d0f98e..d6137c0e 100644 --- a/tap_shopify/streams/orders.py +++ b/tap_shopify/streams/orders.py @@ -1118,6 +1118,15 @@ def fetch_bulk_operation(op_id): if current_status == "COMPLETED": LOGGER.info("Bulk operation completed. File size: %s bytes", op.get("fileSize")) + self.update_bookmark( + bookmark_value=utils.strftime(current_bookmark), + bulk_op_metadata={ + "bulk_operation_id": op.get("id"), + "status": current_status, + "created_at": op.get("createdAt"), + "last_date_window": self.date_window_size, + } + ) return op.get("url") if current_status in ["FAILED", "CANCELED"]: @@ -1281,12 +1290,11 @@ def get_objects(self): current_bookmark = max(current_bookmark, replication_value) yield obj - - self.clear_bulk_operation_state() else: LOGGER.info("No data returned for the date range: %s to %s", last_updated_at, query_end) + self.clear_bulk_operation_state() last_updated_at = query_end max_bookmark_value = min(sync_start, current_bookmark) self.update_bookmark(utils.strftime(max_bookmark_value)) diff --git a/tests/test_automatic_fields.py b/tests/test_automatic_fields.py index 5f64d873..ad8756cb 100644 --- a/tests/test_automatic_fields.py +++ b/tests/test_automatic_fields.py @@ -64,6 +64,8 @@ def automatic_test(self, conn_id, testable_streams): stream_metadata = self.expected_metadata().get(stream, {}) expected_primary_keys = self.expected_primary_keys().get(stream, set()) + extra_automatic_keys = {"order"} if stream == "order_refunds" else set() + # collect records messages = synced_records.get(stream) @@ -79,7 +81,7 @@ def automatic_test(self, conn_id, testable_streams): # verify that only the automatic fields are sent to the target self.assertEqual( actual_fields_by_stream.get(stream, set()), - expected_primary_keys | + expected_primary_keys | extra_automatic_keys | self.expected_replication_keys().get(stream, set()), msg="The fields sent to the target are not the automatic fields" ) diff --git a/tests/test_discovery.py b/tests/test_discovery.py index 5e4df1b3..89aa4a49 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -129,6 +129,10 @@ def test_run(self): expected_primary_keys = self.expected_primary_keys()[stream] expected_replication_keys = self.expected_replication_keys()[stream] expected_automatic_fields = expected_primary_keys | expected_replication_keys + + # In the order_refunds stream, order field is explicitly marked as automatic + if stream == "order_refunds": + expected_automatic_fields = expected_automatic_fields | {"order"} # verify that primary, replication and foreign keys # are given the inclusion of automatic in annotated schema. From 313e366e0e835a3406888ae31270b687bab6c33c Mon Sep 17 00:00:00 2001 From: Atul Tiwari Date: Fri, 17 Oct 2025 16:41:09 +0530 Subject: [PATCH 56/66] GL-Work (#242) * Added Stream Schema files (#238) * SAC-28916: Added schema.json files * SAC-28916: Updated schema files * SAC-28916: Update in schams file * SAC-28916: Updated replication key for resource_feedback * Implement data sync with pagination, bookmark handling (#239) * SAC-28916: Added streams.py files with sync logic * SAC-28916: Updated sync implementation to adderess Copilot and new implementation * SAC-28916: updated unittests * SAC-28916: Addressed copilot issues * SAC-28916: Update for integration tests * SAC-28916: Removed parent field from metadata * parent-tap-stream-id added * SAC-28916: Updated readme.md file * SAC-28916: Removed replication keys for full table streams * SAC-28916: Removed replication keys from schema catalog entry * SAC-28916: Remove article author from integration due access issue * SAC-28916: commit for test how many streams have access issue * SAC-28916: commit for test how many streams have access issue * SAC-28916: Added access scope permission check at discovery * SAC-28916: Updated integration tests * SAC-28916: Updated integration tests test_all_fields * SAC-28916: Updated integration tests test_full_replication * SAC-28916: updated circle.ci for all_field test * SAC-28916: updated circle.ci to run all tests * SAC-28916: updated test_full_replication test * SAC-28916: updated test_bookmark test * SAC-28916: updated base.py for api limit * SAC-28916: updated full_replication.py file * SAC-28916: updated full_pagination and bookmark.py file * SAC-28916: updated test_discovery.py * SAC-28916: updated marketing event replication key * SAC-28916: updated interupted sync.py * SAC-28916: updated test_start_date.py * SAC-28916: updated for running selected test cases * SAC-28916: update test_start_date * SAC-28916: enable all intergration tests and updated readme file with access scope info --------- Co-authored-by: Tushar Mittal * SAC-28916:Addressed review comments * SAC-28916:Addressed review comments --------- Co-authored-by: Tushar Mittal * SAC-28933: Updated setup.py and changelog.md * SAC-28933: Updated changelog.md * SAC-28885:removed parent stream metafield --------- Co-authored-by: Tushar Mittal --- CHANGELOG.md | 3 + README.md | 105 +- setup.py | 2 +- tap_shopify/__init__.py | 39 +- tap_shopify/schemas/application_charges.json | 56 + tap_shopify/schemas/application_credits.json | 50 + tap_shopify/schemas/article_authors.json | 11 + tap_shopify/schemas/article_tags.json | 11 + tap_shopify/schemas/blogs.json | 165 +++ tap_shopify/schemas/carrier_services.json | 41 + tap_shopify/schemas/comments.json | 146 +++ tap_shopify/schemas/currencies.json | 36 + tap_shopify/schemas/custom_collections.json | 92 ++ tap_shopify/schemas/draft_orders.json | 1021 +++++++++++++++++ tap_shopify/schemas/fulfillment_services.json | 59 + tap_shopify/schemas/marketing_events.json | 110 ++ tap_shopify/schemas/pages.json | 68 ++ tap_shopify/schemas/policies.json | 43 + tap_shopify/schemas/price_rules.json | 103 ++ .../recurring_application_charges.json | 186 +++ tap_shopify/schemas/redirects.json | 30 + tap_shopify/schemas/resource_feedback.json | 88 ++ tap_shopify/schemas/script_tags.json | 49 + tap_shopify/schemas/shipping_zones.json | 243 ++++ tap_shopify/schemas/shop.json | 415 +++++++ tap_shopify/schemas/smart_collections.json | 117 ++ tap_shopify/schemas/tender_transactions.json | 90 ++ tap_shopify/schemas/themes.json | 61 + tap_shopify/schemas/webhooks.json | 143 +++ tap_shopify/streams/__init__.py | 25 + tap_shopify/streams/abandoned_checkouts.py | 5 +- tap_shopify/streams/application_charges.py | 48 + tap_shopify/streams/application_credits.py | 47 + tap_shopify/streams/article_authors.py | 29 + tap_shopify/streams/article_tags.py | 60 + tap_shopify/streams/base.py | 60 +- tap_shopify/streams/blogs.py | 54 + tap_shopify/streams/carrier_services.py | 33 + tap_shopify/streams/collections.py | 5 +- tap_shopify/streams/comments.py | 51 + tap_shopify/streams/currencies.py | 45 + tap_shopify/streams/custom_collections.py | 64 ++ tap_shopify/streams/customers.py | 1 + tap_shopify/streams/draft_orders.py | 218 ++++ tap_shopify/streams/fulfillment_orders.py | 9 +- tap_shopify/streams/fulfillment_services.py | 65 ++ tap_shopify/streams/inventory_items.py | 5 +- tap_shopify/streams/inventory_levels.py | 1 + tap_shopify/streams/locations.py | 1 + tap_shopify/streams/marketing_events.py | 43 + tap_shopify/streams/metafields.py | 2 +- tap_shopify/streams/metafields_collections.py | 1 + tap_shopify/streams/metafields_customers.py | 1 + tap_shopify/streams/metafields_orders.py | 1 + tap_shopify/streams/metafields_products.py | 1 + tap_shopify/streams/order_refunds.py | 5 +- tap_shopify/streams/order_shipping_lines.py | 1 + tap_shopify/streams/orders.py | 3 +- tap_shopify/streams/pages.py | 37 + tap_shopify/streams/policies.py | 88 ++ tap_shopify/streams/price_rules.py | 148 +++ tap_shopify/streams/product_variants.py | 5 +- tap_shopify/streams/products.py | 5 +- .../streams/recurring_application_charges.py | 97 ++ tap_shopify/streams/redirects.py | 52 + tap_shopify/streams/resource_feedback.py | 89 ++ tap_shopify/streams/script_tags.py | 35 + tap_shopify/streams/shipping_zones.py | 82 ++ tap_shopify/streams/shop.py | 116 ++ tap_shopify/streams/smart_collections.py | 70 ++ tap_shopify/streams/tender_transactions.py | 46 + tap_shopify/streams/themes.py | 92 ++ tap_shopify/streams/transactions.py | 1 + tap_shopify/streams/webhooks.py | 56 + tests/base.py | 136 ++- tests/test_all_fields.py | 27 +- tests/test_bookmarks.py | 39 +- tests/test_discovery.py | 26 +- tests/test_full_replication.py | 15 +- tests/test_interrupted_sync.py | 3 +- tests/test_pagination.py | 4 +- tests/test_shop_info_fields.py | 2 +- tests/test_start_date.py | 18 +- tests/unittests/test_base.py | 2 +- tests/unittests/test_date_window.py | 2 +- .../unittests/test_token_check_in_discover.py | 51 +- 86 files changed, 5732 insertions(+), 79 deletions(-) create mode 100644 tap_shopify/schemas/application_charges.json create mode 100644 tap_shopify/schemas/application_credits.json create mode 100644 tap_shopify/schemas/article_authors.json create mode 100644 tap_shopify/schemas/article_tags.json create mode 100644 tap_shopify/schemas/blogs.json create mode 100644 tap_shopify/schemas/carrier_services.json create mode 100644 tap_shopify/schemas/comments.json create mode 100644 tap_shopify/schemas/currencies.json create mode 100644 tap_shopify/schemas/custom_collections.json create mode 100644 tap_shopify/schemas/draft_orders.json create mode 100644 tap_shopify/schemas/fulfillment_services.json create mode 100644 tap_shopify/schemas/marketing_events.json create mode 100644 tap_shopify/schemas/pages.json create mode 100644 tap_shopify/schemas/policies.json create mode 100644 tap_shopify/schemas/price_rules.json create mode 100644 tap_shopify/schemas/recurring_application_charges.json create mode 100644 tap_shopify/schemas/redirects.json create mode 100644 tap_shopify/schemas/resource_feedback.json create mode 100644 tap_shopify/schemas/script_tags.json create mode 100644 tap_shopify/schemas/shipping_zones.json create mode 100644 tap_shopify/schemas/shop.json create mode 100644 tap_shopify/schemas/smart_collections.json create mode 100644 tap_shopify/schemas/tender_transactions.json create mode 100644 tap_shopify/schemas/themes.json create mode 100644 tap_shopify/schemas/webhooks.json create mode 100644 tap_shopify/streams/application_charges.py create mode 100644 tap_shopify/streams/application_credits.py create mode 100644 tap_shopify/streams/article_authors.py create mode 100644 tap_shopify/streams/article_tags.py create mode 100644 tap_shopify/streams/blogs.py create mode 100644 tap_shopify/streams/carrier_services.py create mode 100644 tap_shopify/streams/comments.py create mode 100644 tap_shopify/streams/currencies.py create mode 100644 tap_shopify/streams/custom_collections.py create mode 100644 tap_shopify/streams/draft_orders.py create mode 100644 tap_shopify/streams/fulfillment_services.py create mode 100644 tap_shopify/streams/marketing_events.py create mode 100644 tap_shopify/streams/pages.py create mode 100644 tap_shopify/streams/policies.py create mode 100644 tap_shopify/streams/price_rules.py create mode 100644 tap_shopify/streams/recurring_application_charges.py create mode 100644 tap_shopify/streams/redirects.py create mode 100644 tap_shopify/streams/resource_feedback.py create mode 100644 tap_shopify/streams/script_tags.py create mode 100644 tap_shopify/streams/shipping_zones.py create mode 100644 tap_shopify/streams/shop.py create mode 100644 tap_shopify/streams/smart_collections.py create mode 100644 tap_shopify/streams/tender_transactions.py create mode 100644 tap_shopify/streams/themes.py create mode 100644 tap_shopify/streams/webhooks.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a0a117ad..937ea295 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +### 3.8.0 + * Introduced new streams [#242](https://github.com/singer-io/tap-shopify/pull/242) + ### 3.7.2 * New automatic_keys Support and Bulk State Persistence Logic [#237](https://github.com/singer-io/tap-shopify/pull/237) diff --git a/README.md b/README.md index f44dc09a..122f666e 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,7 @@ # tap-shopify This is a [Singer](https://singer.io) tap that produces JSON-formatted data -following the [Singer -spec](https://github.com/singer-io/getting-started/blob/master/SPEC.md). +following the [Singer spec](https://github.com/singer-io/getting-started/blob/master/SPEC.md). This tap: @@ -24,34 +23,96 @@ This tap: - [Locations](https://shopify.dev/docs/api/admin-graphql/latest/queries/locations) - [Inventory Levels](https://shopify.dev/docs/api/admin-graphql/latest/queries/inventorylevel) - [Inventory Item](https://shopify.dev/docs/api/admin-graphql/latest/queries/inventoryitems) + - [Application Charges](https://shopify.dev/docs/api/admin-graphql/latest/queries/currentAppInstallation?example=retrieves-a-list-of-application-charges) + - [Application Credits](https://shopify.dev/docs/api/admin-graphql/latest/queries/appInstallation?example=retrieves-all-application-credits) + - [Article Authors](https://shopify.dev/docs/api/admin-rest/latest/resources/article#get-articles-authors) + - [Article Tags](https://shopify.dev/docs/api/admin-rest/latest/resources/article#get-articles-tags) + - [Blogs](https://shopify.dev/docs/api/admin-graphql/latest/queries/blogs?example=retrieve-a-list-of-all-blogs) + - [Carrier Services](https://shopify.dev/docs/api/admin-rest/latest/resources/carrierservice#get-carrier-services) + - [Comments](https://shopify.dev/docs/api/admin-graphql/latest/queries/comments?example=retrieves-a-list-of-comments) + - [Currencies](https://shopify.dev/docs/api/admin-graphql/latest/queries/shop?example=retrieves-a-list-of-currencies-enabled-on-a-shop) + - [Custom Collections](https://shopify.dev/docs/api/admin-graphql/latest/queries/collections?example=retrieve-a-list-of-custom-collections) + - [Draft Orders](https://shopify.dev/docs/api/admin-graphql/latest/queries/draftOrders?example=retrieves-a-list-of-draft-orders) + - [Fulfillment Services](https://shopify.dev/docs/api/admin-graphql/latest/queries/shop?example=receive-a-list-of-all-fulfillmentservices) + - [Marketing Events](https://shopify.dev/docs/api/admin-graphql/latest/queries/marketingEvents?example=retrieves-a-list-of-all-marketing-events) + - [Pages](https://shopify.dev/docs/api/admin-graphql/latest/queries/pages?example=retrieves-a-list-of-pages) + - [Policies](https://shopify.dev/docs/api/admin-graphql/latest/queries/shop?example=retrieves-a-list-of-the-shops-policies) + - [Price Rules](https://shopify.dev/docs/api/admin-graphql/latest/queries/discountNodes) + - [Recurring Application Charges](https://shopify.dev/docs/api/admin-graphql/latest/queries/currentAppInstallationexample=retrieves-a-list-of-recurring-application-charges) + - [Redirects](https://shopify.dev/docs/api/admin-graphql/latest/queries/urlRedirects?example=retrieves-a-list-of-url-redirects) + - [Resource Feedback](https://shopify.dev/docs/api/admin-graphql/latest/queries/app?example=get-the-feedback-field) + - [Script Tags](https://shopify.dev/docs/api/admin-graphql/latest/queries/scriptTags?example=retrieves-a-list-of-all-script-tags) + - [Shipping Zones](https://shopify.dev/docs/api/admin-graphql/latest/queries/deliveryProfiles?example=receive-a-list-of-all-shippingzones) + - [Shop](https://shopify.dev/docs/api/admin-rest/latest/resources/shop#get-shop) + - [Smart Collections](https://shopify.dev/docs/api/admin-graphql/latest/queries/collections?example=retrieve-a-list-of-smart-collections) + - [Tender Transactions](https://shopify.dev/docs/api/admin-graphql/latest/queries/tenderTransactions?example=retrieves-a-list-of-tender-transactions) + - [Themes](https://shopify.dev/docs/api/admin-graphql/latest/queries/themes?example=retrieves-a-list-of-themes) + - [Webhooks](https://shopify.dev/docs/api/admin-graphql/latest/queries/webhookSubscriptions) - Outputs the schema for each resource - Incrementally pulls data based on the input state ## Stream Details -| Stream Name | Replication Key | Key Properties | -|------------------------|----------------|---------------| -| abandoned_checkouts | updatedAt | id | -| collections | updatedAt | id | -| customers | updatedAt | id | -| events | createdAt | id | -| inventory_items | updatedAt | id | -| inventory_levels | updatedAt | id | -| locations | createdAt | id | -| metafields_collections | updatedAt | id | -| metafields_customers | updatedAt | id | -| metafields_orders | updatedAt | id | -| metafields_products | updatedAt | id | -| order_refunds | updatedAt | id | -| orders | updatedAt | id | -| order_shipping_lines | updatedAt | id | -| fulfillment_orders | updatedAt | id | -| product_variants | updatedAt | id | -| products | updatedAt | id | -| transactions | createdAt | id | +| Stream Name | Replication Key | Key Properties| +|------------------------|--------------------------|---------------| +| abandoned_checkouts | updatedAt | id | +| collections | updatedAt | id | +| customers | updatedAt | id | +| events | createdAt | id | +| inventory_items | updatedAt | id | +| inventory_levels | updatedAt | id | +| locations | createdAt | id | +| metafields_collections | updatedAt | id | +| metafields_customers | updatedAt | id | +| metafields_orders | updatedAt | id | +| metafields_products | updatedAt | id | +| order_refunds | updatedAt | id | +| orders | updatedAt | id | +| order_shipping_lines | updatedAt | id | +| fulfillment_orders | updatedAt | id | +| product_variants | updatedAt | id | +| products | updatedAt | id | +| transactions | createdAt | id | +| blogs | updatedAt | id | +| comments | updatedAt | id | +| custom_collections | updatedAt | id | +| draft_orders | updatedAt | id | +| marketing_events | started_at | id | +| pages | updatedAt | id | +| policies | updatedAt | id | +| redirects | createdAt | id | +| resource_feedback | feedback_generated_at | app_id | +| script_tags | updatedAt | id | +| shop | updatedAt | id | +| smart_collections | updatedAt | id | +| tender_transactions | processedAt | id | +| themes | updatedAt | id | +| webhooks | updatedAt | id | Currently, `locations` graphql endpoint doesn't support querying on the `updatedAt`, therefore, `createdAt` is made the replication key. +## Required Account Access Scopes + +1. CarrierServices, ShippingZones - shipping +2. Comments, Pages, Redirect, Article, Blog - content +3. Collections, CustomCollection , SmartCollection, ProductVariants, Products, ResourceFeedback - products +5. DraftOrders - draft_orders +6. TenderTransaction, Transactions, AbandonedCheckouts, OrderRefund, OrderShippingLines - orders +7. FulfillmentService - fulfillments +8. Marketing Events - marketing_events +9. ResourceFeedback - resource_feedbacks +10. ScriptTags - script_tags +11. Themes - themes +12. PriceRules - price_rules +13. Customers - customers +14. FulfillmentOrders - fulfillments_orders +15. InventoryItems, InventoryLevels - inventory +16. Locations - location +17. MarketingEvents - marketing_events +18. Policies - legal_policies +19. PriceRules - discount +20. Redirects - online_store_navigation + ## Quick Start 1. Install diff --git a/setup.py b/setup.py index 19fa631f..808eebae 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="3.7.2", + version="3.8.0", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index 0544dad6..be1855ec 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -59,6 +59,19 @@ def has_read_users_access(): return False return True +def has_access_scope(stream, scopes): + """If the app does not have the required scope, return False""" + app_access_scopes = fetch_app_scopes() + + if not any(access_scope in scopes for access_scope in app_access_scopes): + message = "The account credentials supplied do not have '{0}' access scope " \ + "to the following stream: {1}. The data for these streams would not be " \ + "collected due to lack of required permission.".format(", ".join(scopes), stream) + LOGGER.warning(message) + return False + + return True + def get_abs_path(path): return os.path.join(os.path.dirname(os.path.realpath(__file__)), path) @@ -82,7 +95,7 @@ def get_discovery_metadata(stream, schema): mdata = metadata.write(mdata, (), 'table-key-properties', stream.key_properties) mdata = metadata.write(mdata, (), 'forced-replication-method', stream.replication_method) - if stream.replication_key: + if stream.replication_method=="INCREMENTAL" and stream.replication_key: mdata = metadata.write(mdata, (), 'valid-replication-keys', [stream.replication_key]) for field_name in schema['properties'].keys(): @@ -106,12 +119,15 @@ def discover(): raw_schemas = load_schemas() streams = [] + error_list = [] for schema_name, schema in raw_schemas.items(): if schema_name not in Context.stream_objects: continue stream = Context.stream_objects[schema_name]() + if stream.access_scope and not has_access_scope(stream.name, stream.access_scope): + error_list.append({stream.name: stream.access_scope}) catalog_schema = add_synthetic_key_to_schema(schema) # create and add catalog entry @@ -121,11 +137,25 @@ def discover(): 'schema': catalog_schema, 'metadata': get_discovery_metadata(stream, schema), 'key_properties': stream.key_properties, - 'replication_key': stream.replication_key, - 'replication_method': stream.replication_method } streams.append(catalog_entry) + if error_list: + total_stream = len(raw_schemas.keys()) + missing_scopes_msg = [ + f"{stream}: {', '.join(scopes)}" + for item in error_list + for stream, scopes in item.items() + ] + message = "The account credentials supplied do not have access to the following " \ + "stream(s): {}. The data for these streams would not be collected due to lack of " \ + "required permission.".format(", ".join(missing_scopes_msg)) + + if len(error_list) != total_stream: + LOGGER.warning(message) + else: + raise ShopifyAPIError(message) + return {'streams': streams} def shuffle_streams(stream_name): @@ -158,8 +188,7 @@ def sync(): if Context.is_selected(stream["tap_stream_id"]): singer.write_schema(stream["tap_stream_id"], stream["schema"], - stream["key_properties"], - bookmark_properties=stream["replication_key"]) + stream["key_properties"]) Context.counts[stream["tap_stream_id"]] = 0 # Loop over streams in catalog diff --git a/tap_shopify/schemas/application_charges.json b/tap_shopify/schemas/application_charges.json new file mode 100644 index 00000000..1f735f0c --- /dev/null +++ b/tap_shopify/schemas/application_charges.json @@ -0,0 +1,56 @@ +{ + "type": "object", + "properties": { + "createdAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "id": { + "type": [ + "string", + "null" + ] + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "price": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "number", + "null" + ] + }, + "currencyCode": { + "type": [ + "string", + "null" + ] + } + } + }, + "status": { + "type": [ + "string", + "null" + ] + }, + "test": { + "type": [ + "boolean", + "null" + ] + } + } +} diff --git a/tap_shopify/schemas/application_credits.json b/tap_shopify/schemas/application_credits.json new file mode 100644 index 00000000..ab45aaf7 --- /dev/null +++ b/tap_shopify/schemas/application_credits.json @@ -0,0 +1,50 @@ +{ + "type": "object", + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "createdAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "description": { + "type": [ + "string", + "null" + ] + }, + "test": { + "type": [ + "boolean", + "null" + ] + }, + "amount": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "number", + "null" + ] + }, + "currencyCode": { + "type": [ + "string", + "null" + ] + } + } + } + } +} diff --git a/tap_shopify/schemas/article_authors.json b/tap_shopify/schemas/article_authors.json new file mode 100644 index 00000000..f11bd16a --- /dev/null +++ b/tap_shopify/schemas/article_authors.json @@ -0,0 +1,11 @@ +{ + "type": "object", + "properties": { + "name": { + "type": [ + "string", + "null" + ] + } + } +} diff --git a/tap_shopify/schemas/article_tags.json b/tap_shopify/schemas/article_tags.json new file mode 100644 index 00000000..4327dd00 --- /dev/null +++ b/tap_shopify/schemas/article_tags.json @@ -0,0 +1,11 @@ +{ + "type": "object", + "properties": { + "article_tag": { + "type": [ + "string", + "null" + ] + } + } +} diff --git a/tap_shopify/schemas/blogs.json b/tap_shopify/schemas/blogs.json new file mode 100644 index 00000000..8e9067bf --- /dev/null +++ b/tap_shopify/schemas/blogs.json @@ -0,0 +1,165 @@ +{ + "type": "object", + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "handle": { + "type": [ + "string", + "null" + ] + }, + "commentPolicy": { + "type": [ + "string", + "null" + ] + }, + "createdAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "updatedAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "tags": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "string", + "null" + ] + } + }, + "title": { + "type": [ + "string", + "null" + ] + }, + "templateSuffix": { + "type": [ + "string", + "null" + ] + }, + "articles": { + "type": [ + "object", + "null" + ], + "properties": { + "edges": { + "type": [ + "array", + "null" + ], + "items": { + "type": "object", + "properties": { + "node": { + "type": [ + "object", + "null" + ], + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "title": { + "type": [ + "string", + "null" + ] + }, + "body": { + "type": [ + "string", + "null" + ] + }, + "createdAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "updatedAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "publishedAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "isPublished": { + "type": [ + "boolean", + "null" + ] + }, + "tags": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "string", + "null" + ] + } + } + } + } + } + } + } + } + }, + "feed": { + "type": [ + "object", + "null" + ], + "properties": { + "location": { + "type": [ + "string", + "null" + ] + }, + "path": { + "type": [ + "string", + "null" + ] + } + } + } + } +} diff --git a/tap_shopify/schemas/carrier_services.json b/tap_shopify/schemas/carrier_services.json new file mode 100644 index 00000000..6bcf5434 --- /dev/null +++ b/tap_shopify/schemas/carrier_services.json @@ -0,0 +1,41 @@ +{ + "type": "object", + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "active": { + "type": [ + "boolean", + "null" + ] + }, + "callbackUrl": { + "type": [ + "string", + "null" + ] + }, + "formattedName": { + "type": [ + "string", + "null" + ] + }, + "supportsServiceDiscovery": { + "type": [ + "boolean", + "null" + ] + } + } +} diff --git a/tap_shopify/schemas/comments.json b/tap_shopify/schemas/comments.json new file mode 100644 index 00000000..01ffd525 --- /dev/null +++ b/tap_shopify/schemas/comments.json @@ -0,0 +1,146 @@ +{ + "type": "object", + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "body": { + "type": [ + "string", + "null" + ] + }, + "bodyHtml": { + "type": [ + "string", + "null" + ] + }, + "author": { + "type": [ + "object", + "null" + ], + "properties": { + "email": { + "type": [ + "string", + "null" + ] + }, + "name": { + "type": [ + "string", + "null" + ] + } + } + }, + "createdAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "updatedAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "userAgent": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": [ + "string", + "null" + ] + }, + "publishedAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "isPublished": { + "type": [ + "boolean", + "null" + ] + }, + "ip": { + "type": [ + "string", + "null" + ] + }, + "article": { + "type": [ + "object", + "null" + ], + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "title": { + "type": [ + "string", + "null" + ] + }, + "tags": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "string", + "null" + ] + } + }, + "publishedAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "isPublished": { + "type": [ + "boolean", + "null" + ] + }, + "createdAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "body": { + "type": [ + "string", + "null" + ] + } + } + } + } +} diff --git a/tap_shopify/schemas/currencies.json b/tap_shopify/schemas/currencies.json new file mode 100644 index 00000000..8d81499d --- /dev/null +++ b/tap_shopify/schemas/currencies.json @@ -0,0 +1,36 @@ +{ + "type": "object", + "properties": { + "currencyCode": { + "type": [ + "string", + "null" + ] + }, + "currencyName": { + "type": [ + "string", + "null" + ] + }, + "enabled": { + "type": [ + "boolean", + "null" + ] + }, + "manualRate": { + "type": [ + "number", + "null" + ] + }, + "rateUpdatedAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + } + } +} diff --git a/tap_shopify/schemas/custom_collections.json b/tap_shopify/schemas/custom_collections.json new file mode 100644 index 00000000..e842c5f9 --- /dev/null +++ b/tap_shopify/schemas/custom_collections.json @@ -0,0 +1,92 @@ +{ + "type": "object", + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "title": { + "type": [ + "string", + "null" + ] + }, + "description": { + "type": [ + "string", + "null" + ] + }, + "descriptionHtml": { + "type": [ + "string", + "null" + ] + }, + "updatedAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "handle": { + "type": [ + "string", + "null" + ] + }, + "publishedOnCurrentPublication": { + "type": [ + "boolean", + "null" + ] + }, + "sortOrder": { + "type": [ + "string", + "null" + ] + }, + "templateSuffix": { + "type": [ + "string", + "null" + ] + }, + "image": { + "type": [ + "object", + "null" + ], + "properties": { + "url": { + "type": [ + "string", + "null" + ] + }, + "altText": { + "type": [ + "string", + "null" + ] + }, + "height": { + "type": [ + "integer", + "null" + ] + }, + "width": { + "type": [ + "integer", + "null" + ] + } + } + } + } +} diff --git a/tap_shopify/schemas/draft_orders.json b/tap_shopify/schemas/draft_orders.json new file mode 100644 index 00000000..75bc91b6 --- /dev/null +++ b/tap_shopify/schemas/draft_orders.json @@ -0,0 +1,1021 @@ +{ + "type": "object", + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "note2": { + "type": [ + "string", + "null" + ] + }, + "email": { + "type": [ + "string", + "null" + ] + }, + "taxesIncluded": { + "type": [ + "boolean", + "null" + ] + }, + "currencyCode": { + "type": [ + "string", + "null" + ] + }, + "invoiceSentAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "createdAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "updatedAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "taxExempt": { + "type": [ + "boolean", + "null" + ] + }, + "completedAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": [ + "string", + "null" + ] + }, + "invoiceUrl": { + "type": [ + "string", + "null" + ] + }, + "tags": { + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + } + }, + "phone": { + "type": [ + "string", + "null" + ] + }, + "totalShippingPriceSet": { + "type": [ + "object", + "null" + ], + "properties": { + "presentmentMoney": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currencyCode": { + "type": [ + "string", + "null" + ] + } + } + }, + "shopMoney": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currencyCode": { + "type": [ + "string", + "null" + ] + } + } + } + } + }, + "totalPriceSet": { + "type": [ + "object", + "null" + ], + "properties": { + "presentmentMoney": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currencyCode": { + "type": [ + "string", + "null" + ] + } + } + }, + "shopMoney": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currencyCode": { + "type": [ + "string", + "null" + ] + } + } + } + } + }, + "subtotalPriceSet": { + "type": [ + "object", + "null" + ], + "properties": { + "presentmentMoney": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currencyCode": { + "type": [ + "string", + "null" + ] + } + } + }, + "shopMoney": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currencyCode": { + "type": [ + "string", + "null" + ] + } + } + } + } + }, + "totalTaxSet": { + "type": [ + "object", + "null" + ], + "properties": { + "presentmentMoney": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currencyCode": { + "type": [ + "string", + "null" + ] + } + } + }, + "shopMoney": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currencyCode": { + "type": [ + "string", + "null" + ] + } + } + } + } + }, + "lineItems": { + "type": [ + "object", + "null" + ], + "properties": { + "edges": { + "type": [ + "array", + "null" + ], + "items": { + "type": "object", + "properties": { + "node": { + "type": [ + "object", + "null" + ], + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "quantity": { + "type": [ + "integer", + "null" + ] + }, + "sku": { + "type": [ + "string", + "null" + ] + }, + "title": { + "type": [ + "string", + "null" + ] + }, + "requiresShipping": { + "type": [ + "boolean", + "null" + ] + }, + "taxable": { + "type": [ + "boolean", + "null" + ] + }, + "custom": { + "type": [ + "boolean", + "null" + ] + }, + "isGiftCard": { + "type": [ + "boolean", + "null" + ] + }, + "vendor": { + "type": [ + "string", + "null" + ] + }, + "variant": { + "type": [ + "object", + "null" + ], + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "title": { + "type": [ + "string", + "null" + ] + } + } + }, + "weight": { + "type": [ + "object", + "null" + ], + "properties": { + "value": { + "type": [ + "number", + "null" + ] + }, + "unit": { + "type": [ + "string", + "null" + ] + } + } + }, + "originalUnitPriceWithCurrency": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currencyCode": { + "type": [ + "string", + "null" + ] + } + } + }, + "product": { + "type": [ + "object", + "null" + ], + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "title": { + "type": [ + "string", + "null" + ] + } + } + }, + "taxLines": { + "type": [ + "array", + "null" + ], + "items": { + "type": "object", + "properties": { + "title": { + "type": [ + "string", + "null" + ] + }, + "source": { + "type": [ + "string", + "null" + ] + }, + "rate": { + "type": [ + "number", + "null" + ] + }, + "ratePercentage": { + "type": [ + "number", + "null" + ] + }, + "priceSet": { + "type": [ + "object", + "null" + ], + "properties": { + "presentmentMoney": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currencyCode": { + "type": [ + "string", + "null" + ] + } + } + }, + "shopMoney": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currencyCode": { + "type": [ + "string", + "null" + ] + } + } + } + } + } + } + } + } + } + } + } + } + } + } + }, + "shippingAddress": { + "type": [ + "object", + "null" + ], + "properties": { + "address1": { + "type": [ + "string", + "null" + ] + }, + "address2": { + "type": [ + "string", + "null" + ] + }, + "city": { + "type": [ + "string", + "null" + ] + }, + "company": { + "type": [ + "string", + "null" + ] + }, + "country": { + "type": [ + "string", + "null" + ] + }, + "countryCodeV2": { + "type": [ + "string", + "null" + ] + }, + "firstName": { + "type": [ + "string", + "null" + ] + }, + "lastName": { + "type": [ + "string", + "null" + ] + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "latitude": { + "type": [ + "number", + "null" + ] + }, + "longitude": { + "type": [ + "number", + "null" + ] + }, + "phone": { + "type": [ + "string", + "null" + ] + }, + "province": { + "type": [ + "string", + "null" + ] + }, + "provinceCode": { + "type": [ + "string", + "null" + ] + }, + "zip": { + "type": [ + "string", + "null" + ] + } + } + }, + "shippingLine": { + "type": [ + "object", + "null" + ], + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "title": { + "type": [ + "string", + "null" + ] + }, + "carrierIdentifier": { + "type": [ + "string", + "null" + ] + }, + "custom": { + "type": [ + "boolean", + "null" + ] + }, + "code": { + "type": [ + "string", + "null" + ] + }, + "deliveryCategory": { + "type": [ + "string", + "null" + ] + }, + "source": { + "type": [ + "string", + "null" + ] + }, + "phone": { + "type": [ + "string", + "null" + ] + }, + "discountedPriceSet": { + "type": [ + "object", + "null" + ], + "properties": { + "presentmentMoney": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currencyCode": { + "type": [ + "string", + "null" + ] + } + } + }, + "shopMoney": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currencyCode": { + "type": [ + "string", + "null" + ] + } + } + } + } + } + } + }, + "taxLines": { + "type": [ + "array", + "null" + ], + "items": { + "type": "object", + "properties": { + "channelLiable": { + "type": [ + "boolean", + "null" + ] + }, + "rate": { + "type": [ + "number", + "null" + ] + }, + "ratePercentage": { + "type": [ + "number", + "null" + ] + }, + "source": { + "type": [ + "string", + "null" + ] + }, + "title": { + "type": [ + "string", + "null" + ] + }, + "priceSet": { + "type": [ + "object", + "null" + ], + "properties": { + "presentmentMoney": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currencyCode": { + "type": [ + "string", + "null" + ] + } + } + }, + "shopMoney": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currencyCode": { + "type": [ + "string", + "null" + ] + } + } + } + } + } + } + } + }, + "customer": { + "type": [ + "object", + "null" + ], + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "createdAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "lastName": { + "type": [ + "string", + "null" + ] + }, + "firstName": { + "type": [ + "string", + "null" + ] + }, + "taxExempt": { + "type": [ + "boolean", + "null" + ] + }, + "tags": { + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + } + }, + "defaultAddress": { + "type": [ + "object", + "null" + ], + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "address1": { + "type": [ + "string", + "null" + ] + }, + "address2": { + "type": [ + "string", + "null" + ] + }, + "city": { + "type": [ + "string", + "null" + ] + }, + "company": { + "type": [ + "string", + "null" + ] + }, + "country": { + "type": [ + "string", + "null" + ] + }, + "countryCodeV2": { + "type": [ + "string", + "null" + ] + }, + "firstName": { + "type": [ + "string", + "null" + ] + }, + "lastName": { + "type": [ + "string", + "null" + ] + }, + "longitude": { + "type": [ + "number", + "null" + ] + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "phone": { + "type": [ + "string", + "null" + ] + }, + "province": { + "type": [ + "string", + "null" + ] + }, + "provinceCode": { + "type": [ + "string", + "null" + ] + }, + "zip": { + "type": [ + "string", + "null" + ] + } + } + }, + "defaultEmailAddress": { + "type": [ + "object", + "null" + ], + "properties": { + "emailAddress": { + "type": [ + "string", + "null" + ] + } + } + }, + "lastOrder": { + "type": [ + "object", + "null" + ], + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "currencyCode": { + "type": [ + "string", + "null" + ] + } + } + } + } + } + } +} diff --git a/tap_shopify/schemas/fulfillment_services.json b/tap_shopify/schemas/fulfillment_services.json new file mode 100644 index 00000000..69b61d7b --- /dev/null +++ b/tap_shopify/schemas/fulfillment_services.json @@ -0,0 +1,59 @@ +{ + "type": "object", + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "handle": { + "type": [ + "string", + "null" + ] + }, + "callbackUrl": { + "type": [ + "string", + "null" + ] + }, + "inventoryManagement": { + "type": [ + "boolean", + "null" + ] + }, + "permitsSkuSharing": { + "type": [ + "boolean", + "null" + ] + }, + "requiresShippingMethod": { + "type": [ + "boolean", + "null" + ] + }, + "serviceName": { + "type": [ + "string", + "null" + ] + }, + "trackingSupport": { + "type": [ + "boolean", + "null" + ] + }, + "type": { + "type": [ + "string", + "null" + ] + } + } +} diff --git a/tap_shopify/schemas/marketing_events.json b/tap_shopify/schemas/marketing_events.json new file mode 100644 index 00000000..2c1ab696 --- /dev/null +++ b/tap_shopify/schemas/marketing_events.json @@ -0,0 +1,110 @@ +{ + "type": "object", + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "type": { + "type": [ + "string", + "null" + ] + }, + "remoteId": { + "type": [ + "string", + "null" + ] + }, + "startedAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "endedAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "scheduledToEndAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "manageUrl": { + "type": [ + "string", + "null" + ] + }, + "previewUrl": { + "type": [ + "string", + "null" + ] + }, + "utmCampaign": { + "type": [ + "string", + "null" + ] + }, + "utmMedium": { + "type": [ + "string", + "null" + ] + }, + "utmSource": { + "type": [ + "string", + "null" + ] + }, + "description": { + "type": [ + "string", + "null" + ] + }, + "marketingChannelType": { + "type": [ + "string", + "null" + ] + }, + "sourceAndMedium": { + "type": [ + "string", + "null" + ] + }, + "targetTypeDisplayText": { + "type": [ + "string", + "null" + ] + }, + "channel": { + "type": [ + "string", + "null" + ] + }, + "channelHandle": { + "type": [ + "string", + "null" + ] + } + } +} diff --git a/tap_shopify/schemas/pages.json b/tap_shopify/schemas/pages.json new file mode 100644 index 00000000..f9fda3fa --- /dev/null +++ b/tap_shopify/schemas/pages.json @@ -0,0 +1,68 @@ +{ + "type": "object", + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "updatedAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "handle": { + "type": [ + "string", + "null" + ] + }, + "createdAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "bodySummary": { + "type": [ + "string", + "null" + ] + }, + "body": { + "type": [ + "string", + "null" + ] + }, + "isPublished": { + "type": [ + "boolean", + "null" + ] + }, + "publishedAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "templateSuffix": { + "type": [ + "string", + "null" + ] + }, + "title": { + "type": [ + "string", + "null" + ] + } + } +} diff --git a/tap_shopify/schemas/policies.json b/tap_shopify/schemas/policies.json new file mode 100644 index 00000000..830d23e5 --- /dev/null +++ b/tap_shopify/schemas/policies.json @@ -0,0 +1,43 @@ +{ + "type": "object", + "properties": { + "createdAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "id": { + "type": [ + "string", + "null" + ] + }, + "title": { + "type": [ + "string", + "null" + ] + }, + "type": { + "type": [ + "string", + "null" + ] + }, + "updatedAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "url": { + "type": [ + "string", + "null" + ] + } + } +} diff --git a/tap_shopify/schemas/price_rules.json b/tap_shopify/schemas/price_rules.json new file mode 100644 index 00000000..e367d413 --- /dev/null +++ b/tap_shopify/schemas/price_rules.json @@ -0,0 +1,103 @@ +{ + "type": "object", + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "discount": { + "type": [ + "object", + "null" + ], + "properties": { + "__typename": { + "type": [ + "string", + "null" + ] + }, + "title": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": [ + "string", + "null" + ] + }, + "createdAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "updatedAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "startsAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "endsAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "usageLimit": { + "type": [ + "integer", + "null" + ] + }, + "usesPerOrderLimit": { + "type": [ + "integer", + "null" + ] + }, + "combinesWith": { + "type": [ + "object", + "null" + ], + "properties": { + "orderDiscounts": { + "type": [ + "boolean", + "null" + ] + }, + "productDiscounts": { + "type": [ + "boolean", + "null" + ] + }, + "shippingDiscounts": { + "type": [ + "boolean", + "null" + ] + } + } + } + } + } + } +} diff --git a/tap_shopify/schemas/recurring_application_charges.json b/tap_shopify/schemas/recurring_application_charges.json new file mode 100644 index 00000000..cb935c7b --- /dev/null +++ b/tap_shopify/schemas/recurring_application_charges.json @@ -0,0 +1,186 @@ +{ + "type": "object", + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "createdAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "currentPeriodEnd": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "returnUrl": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": [ + "string", + "null" + ] + }, + "test": { + "type": [ + "boolean", + "null" + ] + }, + "trialDays": { + "type": [ + "integer", + "null" + ] + }, + "lineItems": { + "type": [ + "array", + "null" + ], + "items": { + "type": "object", + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "plan": { + "type": [ + "object", + "null" + ], + "properties": { + "pricingDetails": { + "type": [ + "object", + "null" + ], + "properties": { + "__typename": { + "type": [ + "string", + "null" + ] + }, + "planHandle": { + "type": [ + "string", + "null" + ] + }, + "interval": { + "type": [ + "string", + "null" + ] + }, + "discount": { + "type": [ + "object", + "null" + ], + "properties": { + "durationLimitInIntervals": { + "type": [ + "integer", + "null" + ] + }, + "remainingDurationInIntervals": { + "type": [ + "integer", + "null" + ] + }, + "priceAfterDiscount": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "number", + "null" + ] + }, + "currencyCode": { + "type": [ + "string", + "null" + ] + } + } + }, + "value": { + "type": [ + "object", + "null" + ], + "properties": { + "__typename": { + "type": [ + "string", + "null" + ] + }, + "percentage": { + "type": [ + "number", + "null" + ] + }, + "amount": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "number", + "null" + ] + }, + "currencyCode": { + "type": [ + "string", + "null" + ] + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } +} diff --git a/tap_shopify/schemas/redirects.json b/tap_shopify/schemas/redirects.json new file mode 100644 index 00000000..19dd5475 --- /dev/null +++ b/tap_shopify/schemas/redirects.json @@ -0,0 +1,30 @@ +{ + "type": "object", + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "path": { + "type": [ + "string", + "null" + ] + }, + "target": { + "type": [ + "string", + "null" + ] + }, + "createdAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + } + } +} diff --git a/tap_shopify/schemas/resource_feedback.json b/tap_shopify/schemas/resource_feedback.json new file mode 100644 index 00000000..11be9b0e --- /dev/null +++ b/tap_shopify/schemas/resource_feedback.json @@ -0,0 +1,88 @@ +{ + "type": "object", + "properties": { + "app_id": { + "type": [ + "string", + "null" + ] + }, + "title": { + "type": [ + "string", + "null" + ] + }, + "feedback_generated_at": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "feedback": { + "type": [ + "object", + "null" + ], + "properties": { + "messages": { + "type": [ + "array", + "null" + ], + "items": { + "type": "object", + "properties": { + "field": { + "type": [ + "string", + "null" + ] + }, + "message": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "state": { + "type": [ + "string", + "null" + ] + }, + "feedbackGeneratedAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "link": { + "type": [ + "object", + "null" + ], + "properties": { + "label": { + "type": [ + "string", + "null" + ] + }, + "url": { + "type": [ + "string", + "null" + ] + } + } + } + } + } + } +} \ No newline at end of file diff --git a/tap_shopify/schemas/script_tags.json b/tap_shopify/schemas/script_tags.json new file mode 100644 index 00000000..9ee46a82 --- /dev/null +++ b/tap_shopify/schemas/script_tags.json @@ -0,0 +1,49 @@ +{ + "type": "object", + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "cache": { + "type": [ + "boolean", + "null" + ] + }, + "createdAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "updatedAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "displayScope": { + "type": [ + "string", + "null" + ] + }, + "legacyResourceId": { + "type": [ + "string", + "null" + ] + }, + "src": { + "type": [ + "string", + "null" + ] + } + } +} diff --git a/tap_shopify/schemas/shipping_zones.json b/tap_shopify/schemas/shipping_zones.json new file mode 100644 index 00000000..89f2bec9 --- /dev/null +++ b/tap_shopify/schemas/shipping_zones.json @@ -0,0 +1,243 @@ +{ + "type": "object", + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "profileLocationGroups": { + "type": [ + "array", + "null" + ], + "items": { + "type": "object", + "properties": { + "locationGroup": { + "type": [ + "object", + "null" + ], + "properties": { + "id": { + "type": [ + "string", + "null" + ] + } + } + }, + "locationGroupZones": { + "type": [ + "object", + "null" + ], + "properties": { + "edges": { + "type": [ + "array", + "null" + ], + "items": { + "type": "object", + "properties": { + "node": { + "type": [ + "object", + "null" + ], + "properties": { + "zone": { + "type": [ + "object", + "null" + ], + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "countries": { + "type": [ + "array", + "null" + ], + "items": { + "type": "object", + "properties": { + "code": { + "type": [ + "object", + "null" + ], + "properties": { + "countryCode": { + "type": [ + "string", + "null" + ] + }, + "restOfWorld": { + "type": [ + "boolean", + "null" + ] + } + } + }, + "provinces": { + "type": [ + "array", + "null" + ], + "items": { + "type": "object", + "properties": { + "name": { + "type": [ + "string", + "null" + ] + }, + "code": { + "type": [ + "string", + "null" + ] + } + } + } + } + } + } + } + } + }, + "methodDefinitions": { + "type": [ + "object", + "null" + ], + "properties": { + "edges": { + "type": [ + "array", + "null" + ], + "items": { + "type": "object", + "properties": { + "node": { + "type": [ + "object", + "null" + ], + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "active": { + "type": [ + "boolean", + "null" + ] + }, + "description": { + "type": [ + "string", + "null" + ] + }, + "methodConditions": { + "type": [ + "array", + "null" + ], + "items": { + "type": "object", + "properties": { + "field": { + "type": [ + "string", + "null" + ] + }, + "operator": { + "type": [ + "string", + "null" + ] + }, + "conditionCriteria": { + "type": [ + "object", + "null" + ], + "properties": { + "__typename": { + "type": [ + "string", + "null" + ] + }, + "amount": { + "type": [ + "string", + "null" + ] + }, + "currencyCode": { + "type": [ + "string", + "null" + ] + }, + "unit": { + "type": [ + "string", + "null" + ] + }, + "value": { + "type": [ + "number", + "null" + ] + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } +} diff --git a/tap_shopify/schemas/shop.json b/tap_shopify/schemas/shop.json new file mode 100644 index 00000000..27328531 --- /dev/null +++ b/tap_shopify/schemas/shop.json @@ -0,0 +1,415 @@ +{ + "type": "object", + "properties": { + "alerts": { + "type": [ + "array", + "null" + ], + "items": { + "type": "object", + "properties": { + "action": { + "type": [ + "object", + "null" + ], + "properties": { + "title": { + "type": [ + "string", + "null" + ] + }, + "url": { + "type": [ + "string", + "null" + ] + } + } + }, + "description": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "billingAddress": { + "type": [ + "object", + "null" + ], + "properties": { + "address1": { + "type": [ + "string", + "null" + ] + }, + "address2": { + "type": [ + "string", + "null" + ] + }, + "city": { + "type": [ + "string", + "null" + ] + }, + "company": { + "type": [ + "string", + "null" + ] + }, + "country": { + "type": [ + "string", + "null" + ] + }, + "countryCodeV2": { + "type": [ + "string", + "null" + ] + }, + "latitude": { + "type": [ + "number", + "null" + ] + }, + "longitude": { + "type": [ + "number", + "null" + ] + }, + "phone": { + "type": [ + "string", + "null" + ] + }, + "province": { + "type": [ + "string", + "null" + ] + }, + "provinceCode": { + "type": [ + "string", + "null" + ] + }, + "zip": { + "type": [ + "string", + "null" + ] + } + } + }, + "countriesInShippingZones": { + "type": [ + "object", + "null" + ], + "properties": { + "countryCodes": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "string", + "null" + ] + } + }, + "includeRestOfWorld": { + "type": [ + "boolean", + "null" + ] + } + } + }, + "checkoutApiSupported": { + "type": [ + "boolean", + "null" + ] + }, + "contactEmail": { + "type": [ + "string", + "null" + ] + }, + "createdAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "currencyCode": { + "type": [ + "string", + "null" + ] + }, + "currencyFormats": { + "type": [ + "object", + "null" + ], + "properties": { + "moneyFormat": { + "type": [ + "string", + "null" + ] + }, + "moneyInEmailsFormat": { + "type": [ + "string", + "null" + ] + }, + "moneyWithCurrencyFormat": { + "type": [ + "string", + "null" + ] + }, + "moneyWithCurrencyInEmailsFormat": { + "type": [ + "string", + "null" + ] + } + } + }, + "customerAccounts": { + "type": [ + "string", + "null" + ] + }, + "description": { + "type": [ + "string", + "null" + ] + }, + "email": { + "type": [ + "string", + "null" + ] + }, + "enabledPresentmentCurrencies": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "string", + "null" + ] + } + }, + "fulfillmentServices": { + "type": [ + "array", + "null" + ], + "items": { + "type": "object", + "properties": { + "handle": { + "type": [ + "string", + "null" + ] + }, + "serviceName": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "ianaTimezone": { + "type": [ + "string", + "null" + ] + }, + "id": { + "type": [ + "string", + "null" + ] + }, + "marketingSmsConsentEnabledAtCheckout": { + "type": [ + "boolean", + "null" + ] + }, + "myshopifyDomain": { + "type": [ + "string", + "null" + ] + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "paymentSettings": { + "type": [ + "object", + "null" + ], + "properties": { + "supportedDigitalWallets": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "plan": { + "type": [ + "object", + "null" + ], + "properties": { + "partnerDevelopment": { + "type": [ + "boolean", + "null" + ] + }, + "shopifyPlus": { + "type": [ + "boolean", + "null" + ] + }, + "publicDisplayName": { + "type": [ + "string", + "null" + ] + } + } + }, + "primaryDomain": { + "type": [ + "object", + "null" + ], + "properties": { + "host": { + "type": [ + "string", + "null" + ] + }, + "id": { + "type": [ + "string", + "null" + ] + } + } + }, + "setupRequired": { + "type": [ + "boolean", + "null" + ] + }, + "shipsToCountries": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "string", + "null" + ] + } + }, + "taxesIncluded": { + "type": [ + "boolean", + "null" + ] + }, + "taxShipping": { + "type": [ + "boolean", + "null" + ] + }, + "timezoneAbbreviation": { + "type": [ + "string", + "null" + ] + }, + "transactionalSmsDisabled": { + "type": [ + "boolean", + "null" + ] + }, + "updatedAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "url": { + "type": [ + "string", + "null" + ] + }, + "weightUnit": { + "type": [ + "string", + "null" + ] + } + } +} diff --git a/tap_shopify/schemas/smart_collections.json b/tap_shopify/schemas/smart_collections.json new file mode 100644 index 00000000..846e458a --- /dev/null +++ b/tap_shopify/schemas/smart_collections.json @@ -0,0 +1,117 @@ +{ + "type": "object", + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "handle": { + "type": [ + "string", + "null" + ] + }, + "title": { + "type": [ + "string", + "null" + ] + }, + "updatedAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "description": { + "type": [ + "string", + "null" + ] + }, + "descriptionHtml": { + "type": [ + "string", + "null" + ] + }, + "sortOrder": { + "type": [ + "string", + "null" + ] + }, + "templateSuffix": { + "type": [ + "string", + "null" + ] + }, + "ruleSet": { + "type": [ + "object", + "null" + ], + "properties": { + "appliedDisjunctively": { + "type": [ + "boolean", + "null" + ] + }, + "rules": { + "type": [ + "array", + "null" + ], + "items": { + "type": "object", + "properties": { + "column": { + "type": [ + "string", + "null" + ] + }, + "relation": { + "type": [ + "string", + "null" + ] + }, + "condition": { + "type": [ + "string", + "null" + ] + } + } + } + } + } + }, + "seo": { + "type": [ + "object", + "null" + ], + "properties": { + "title": { + "type": [ + "string", + "null" + ] + }, + "description": { + "type": [ + "string", + "null" + ] + } + } + } + } +} diff --git a/tap_shopify/schemas/tender_transactions.json b/tap_shopify/schemas/tender_transactions.json new file mode 100644 index 00000000..52648699 --- /dev/null +++ b/tap_shopify/schemas/tender_transactions.json @@ -0,0 +1,90 @@ +{ + "type": "object", + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "paymentMethod": { + "type": [ + "string", + "null" + ] + }, + "processedAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "remoteReference": { + "type": [ + "string", + "null" + ] + }, + "test": { + "type": [ + "boolean", + "null" + ] + }, + "transactionDetails": { + "type": [ + "object", + "null" + ], + "properties": { + "creditCardCompany": { + "type": [ + "string", + "null" + ] + }, + "creditCardNumber": { + "type": [ + "string", + "null" + ] + } + } + }, + "amount": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currencyCode": { + "type": [ + "string", + "null" + ] + } + } + }, + "order": { + "type": [ + "object", + "null" + ], + "properties": { + "id": { + "type": [ + "string", + "null" + ] + } + } + } + } +} diff --git a/tap_shopify/schemas/themes.json b/tap_shopify/schemas/themes.json new file mode 100644 index 00000000..9194f8be --- /dev/null +++ b/tap_shopify/schemas/themes.json @@ -0,0 +1,61 @@ +{ + "type": "object", + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "prefix": { + "type": [ + "string", + "null" + ] + }, + "processing": { + "type": [ + "boolean", + "null" + ] + }, + "processingFailed": { + "type": [ + "boolean", + "null" + ] + }, + "role": { + "type": [ + "string", + "null" + ] + }, + "themeStoreId": { + "type": [ + "integer", + "null" + ] + }, + "updatedAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "createdAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + } + } +} diff --git a/tap_shopify/schemas/webhooks.json b/tap_shopify/schemas/webhooks.json new file mode 100644 index 00000000..212cc27c --- /dev/null +++ b/tap_shopify/schemas/webhooks.json @@ -0,0 +1,143 @@ +{ + "type": "object", + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "format": { + "type": [ + "string", + "null" + ] + }, + "filter": { + "type": [ + "string", + "null" + ] + }, + "includeFields": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "string", + "null" + ] + } + }, + "legacyResourceId": { + "type": [ + "string", + "null" + ] + }, + "metafieldNamespaces": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "string", + "null" + ] + } + }, + "topic": { + "type": [ + "string", + "null" + ] + }, + "createdAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "updatedAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "callbackUrl": { + "type": [ + "string", + "null" + ] + }, + "apiVersion": { + "type": [ + "object", + "null" + ], + "properties": { + "displayName": { + "type": [ + "string", + "null" + ] + }, + "handle": { + "type": [ + "string", + "null" + ] + }, + "supported": { + "type": [ + "boolean", + "null" + ] + } + } + }, + "endpoint": { + "type": [ + "object", + "null" + ], + "properties": { + "__typename": { + "type": [ + "string", + "null" + ] + }, + "arn": { + "type": [ + "string", + "null" + ] + }, + "callbackUrl": { + "type": [ + "string", + "null" + ] + }, + "pubSubProject": { + "type": [ + "string", + "null" + ] + }, + "pubSubTopic": { + "type": [ + "string", + "null" + ] + } + } + } + } +} diff --git a/tap_shopify/streams/__init__.py b/tap_shopify/streams/__init__.py index b647db5c..acb0f389 100644 --- a/tap_shopify/streams/__init__.py +++ b/tap_shopify/streams/__init__.py @@ -16,3 +16,28 @@ import tap_shopify.streams.collections import tap_shopify.streams.fulfillment_orders import tap_shopify.streams.order_shipping_lines +import tap_shopify.streams.application_charges +import tap_shopify.streams.application_credits +import tap_shopify.streams.article_authors +import tap_shopify.streams.article_tags +import tap_shopify.streams.blogs +import tap_shopify.streams.carrier_services +import tap_shopify.streams.comments +import tap_shopify.streams.currencies +import tap_shopify.streams.custom_collections +import tap_shopify.streams.draft_orders +import tap_shopify.streams.fulfillment_services +import tap_shopify.streams.marketing_events +import tap_shopify.streams.pages +import tap_shopify.streams.policies +import tap_shopify.streams.price_rules +import tap_shopify.streams.script_tags +import tap_shopify.streams.shipping_zones +import tap_shopify.streams.shop +import tap_shopify.streams.smart_collections +import tap_shopify.streams.tender_transactions +import tap_shopify.streams.themes +import tap_shopify.streams.redirects +import tap_shopify.streams.resource_feedback +import tap_shopify.streams.recurring_application_charges +import tap_shopify.streams.webhooks diff --git a/tap_shopify/streams/abandoned_checkouts.py b/tap_shopify/streams/abandoned_checkouts.py index a1712ff8..d088d144 100644 --- a/tap_shopify/streams/abandoned_checkouts.py +++ b/tap_shopify/streams/abandoned_checkouts.py @@ -7,6 +7,7 @@ class AbandonedCheckouts(Stream): name = "abandoned_checkouts" data_key = "abandonedCheckouts" replication_key = "updatedAt" + access_scope = ["read_orders"] @classmethod def process_sub_entities(cls, data, entity_name): @@ -26,13 +27,13 @@ def process_sub_entities(cls, data, entity_name): sub_entities.append(node) return sub_entities - def transform_object(self, obj): + def transform_object(self, obj, **_kwargs): """ Transforms the object by processing its sub-entities. Args: obj (dict): Object to transform. - + **_kwargs: Optional additional parameters. Returns: dict: Transformed object. """ diff --git a/tap_shopify/streams/application_charges.py b/tap_shopify/streams/application_charges.py new file mode 100644 index 00000000..d5a94476 --- /dev/null +++ b/tap_shopify/streams/application_charges.py @@ -0,0 +1,48 @@ +from tap_shopify.context import Context +from tap_shopify.streams.base import FullTableStream + + +class ApplicationCharges(FullTableStream): + """Stream class for Application Charges in Shopify.""" + name = "application_charges" + data_key = "currentAppInstallation" + + def call_api(self, query_params, query=None, data_key=None): + """ + Overriding call_api method to extract data from nested data_key. + """ + root_data = super().call_api(query_params, query=query, data_key=data_key) + data = ( + root_data + .get("oneTimePurchases", {}) + ) + return data + + def get_query(self): + return """ + query ApplicationCharges($first: Int!, $after: String) { + currentAppInstallation { + oneTimePurchases(first: $first, after: $after) { + edges { + node { + createdAt + id + name + price { + amount + currencyCode + } + status + test + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + } + """ + +Context.stream_objects["application_charges"] = ApplicationCharges diff --git a/tap_shopify/streams/application_credits.py b/tap_shopify/streams/application_credits.py new file mode 100644 index 00000000..d075cc0f --- /dev/null +++ b/tap_shopify/streams/application_credits.py @@ -0,0 +1,47 @@ +from tap_shopify.context import Context +from tap_shopify.streams.base import FullTableStream + + +class ApplicationCredits(FullTableStream): + """Stream class for Application Credits in Shopify.""" + name = "application_credits" + data_key = "appInstallation" + + def call_api(self, query_params, query=None, data_key=None): + """ + Overriding call_api method to extract data from nested data_key. + """ + root_data = super().call_api(query_params, query=query, data_key=data_key) + data = ( + root_data + .get("credits", {}) + ) + return data + + def get_query(self): + return """ + query ApplicationCredits($first: Int!, $after: String) { + appInstallation { + credits(first: $first, after: $after) { + edges { + node { + amount { + amount + currencyCode + } + createdAt + description + id + test + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + } + """ + +Context.stream_objects["application_credits"] = ApplicationCredits diff --git a/tap_shopify/streams/article_authors.py b/tap_shopify/streams/article_authors.py new file mode 100644 index 00000000..39bdb01b --- /dev/null +++ b/tap_shopify/streams/article_authors.py @@ -0,0 +1,29 @@ +from tap_shopify.context import Context +from tap_shopify.streams.base import FullTableStream + + +class ArticleAuthors(FullTableStream): + """Stream class for Article Authors in Shopify.""" + name = "article_authors" + data_key = "articleAuthors" + key_properties = ["name"] + access_scope = ["read_content" ,"read_online_store_pages"] + + def get_query(self): + return """ + query ArticleAuthors($first: Int!, $after: String) { + articleAuthors(first: $first, after: $after) { + edges { + node { + name + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + +Context.stream_objects["article_authors"] = ArticleAuthors diff --git a/tap_shopify/streams/article_tags.py b/tap_shopify/streams/article_tags.py new file mode 100644 index 00000000..5241f9f1 --- /dev/null +++ b/tap_shopify/streams/article_tags.py @@ -0,0 +1,60 @@ +import singer +from singer import metrics +from tap_shopify.context import Context +from tap_shopify.streams.base import FullTableStream + +LOGGER = singer.get_logger() + + +class ArticleTags(FullTableStream): + """Stream class for Article Tags in Shopify.""" + name = "article_tags" + data_key = "articleTags" + key_properties = ["article_tag"] + access_scope = ["read_content" ,"read_online_store_pages"] + + # pylint: disable=W0221 + def get_query_params(self): + """ + Construct query parameters for GraphQL requests. + Returns: + dict: Dictionary of query parameters. + """ + params = { + "limit": 250, + } + return params + + def transform_object(self, obj, **_kwargs): + """ + Modify this to perform custom transformation on each object + """ + return {"article_tag": obj} + + # pylint: disable=too-many-locals + def get_objects(self): + """ + Returns: + - Yields list of objects for the stream + Performs: + - Transformation + """ + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) + LOGGER.info("GraphQL query for stream '%s': %s", self.name, ' '.join(query.split())) + + query_params = self.get_query_params() + + with metrics.http_request_timer(self.name): + data = self.call_api(query_params, query=query) + for record in data: + obj = self.transform_object(record) + yield obj + + def get_query(self): + return """ + query ArticleTags($limit: Int!) { + articleTags(limit: $limit) + } + """ + +Context.stream_objects["article_tags"] = ArticleTags diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index ada0aa81..28843445 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -137,6 +137,7 @@ class Stream(): date_window_size = None data_key = None results_per_page = None + access_scope = None def __init__(self): self.results_per_page = Context.get_results_per_page(RESULTS_PER_PAGE) @@ -152,7 +153,7 @@ def get_query(self): """ raise NotImplementedError("Function Not Implemented") - def transform_object(self, obj): + def transform_object(self, obj, **_kwargs): """ Modify this to perform custom transformation on each object """ @@ -352,7 +353,7 @@ def get_objects(self): data = self.call_api(query_params, query=query) for edge in data.get("edges"): - obj = self.transform_object(edge.get("node")) + obj = self.transform_object(edge.get("node"), last_updated_at=last_updated_at) replication_value = utils.strptime_to_utc(obj[self.replication_key]) current_bookmark = max(current_bookmark, replication_value) yield obj @@ -370,3 +371,58 @@ def sync(self): Default implementation for sync method """ yield from self.get_objects() + + +class FullTableStream(Stream): + replication_method = 'FULL_TABLE' + + def get_query(self): + """ + Provides GraphQL query + """ + raise NotImplementedError("Function Not Implemented") + + # pylint: disable=W0221 + def get_query_params(self, cursor=None): + """ + Construct query parameters for GraphQL requests. + + Args: + cursor (str): Pagination cursor, if any. + + Returns: + dict: Dictionary of query parameters. + """ + params = { + "first": self.results_per_page, + } + + if cursor: + params["after"] = cursor + return params + + # pylint: disable=too-many-locals + def get_objects(self): + """ + Returns: + - Yields list of objects for the stream + Performs: + - Pagination & Filtering of stream + - Transformation + """ + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) + LOGGER.info("GraphQL query for stream '%s': %s", self.name, ' '.join(query.split())) + + has_next_page, cursor = True, None + + while has_next_page: + query_params = self.get_query_params(cursor=cursor) + with metrics.http_request_timer(self.name): + data = self.call_api(query_params, query=query) + + for edge in data.get("edges", []): + obj = self.transform_object(edge.get("node")) + yield obj + + page_info = data.get("pageInfo") + cursor , has_next_page = page_info.get("endCursor"), page_info.get("hasNextPage") diff --git a/tap_shopify/streams/blogs.py b/tap_shopify/streams/blogs.py new file mode 100644 index 00000000..4c9b7e8a --- /dev/null +++ b/tap_shopify/streams/blogs.py @@ -0,0 +1,54 @@ +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream + + +class Blogs(Stream): + """Stream class for Blogs in Shopify.""" + name = "blogs" + data_key = "blogs" + replication_key = "updatedAt" + access_scope = ["read_content" ,"read_online_store_pages"] + + def get_query(self): + return """ + query Blogs($first: Int!, $after: String, $query: String) { + blogs(first: $first, after: $after, query: $query) { + edges { + node { + id + handle + commentPolicy + createdAt + tags + updatedAt + title + templateSuffix + articles(first: 250) { + edges { + node { + body + id + title + updatedAt + publishedAt + tags + isPublished + createdAt + } + } + } + feed { + location + path + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + +Context.stream_objects["blogs"] = Blogs diff --git a/tap_shopify/streams/carrier_services.py b/tap_shopify/streams/carrier_services.py new file mode 100644 index 00000000..c222d3d3 --- /dev/null +++ b/tap_shopify/streams/carrier_services.py @@ -0,0 +1,33 @@ +from tap_shopify.context import Context +from tap_shopify.streams.base import FullTableStream + + +class CarrierServices(FullTableStream): + """Stream class for Carrier Services in Shopify.""" + name = "carrier_services" + data_key = "carrierServices" + access_scope = ["read_shipping"] + + def get_query(self): + return """ + query CarrierServices($first: Int!, $after: String, $query: String) { + carrierServices(first: $first, after: $after, query: $query) { + edges { + node { + id + name + active + callbackUrl + formattedName + supportsServiceDiscovery + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + +Context.stream_objects["carrier_services"] = CarrierServices diff --git a/tap_shopify/streams/collections.py b/tap_shopify/streams/collections.py index dc584207..69785047 100644 --- a/tap_shopify/streams/collections.py +++ b/tap_shopify/streams/collections.py @@ -7,6 +7,7 @@ class Collections(Stream): name = "collections" data_key = "collections" replication_key = "updatedAt" + access_scope = ["read_products"] def transform_products(self, data): """ @@ -47,13 +48,13 @@ def transform_products(self, data): return product_ids - def transform_object(self, obj): + def transform_object(self, obj, **_kwargs): """ Transforms a collection object. Args: obj (dict): Collection object. - + **_kwargs: Optional additional parameters. Returns: dict: Transformed collection object. """ diff --git a/tap_shopify/streams/comments.py b/tap_shopify/streams/comments.py new file mode 100644 index 00000000..bcc04fcf --- /dev/null +++ b/tap_shopify/streams/comments.py @@ -0,0 +1,51 @@ +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream + + +class Comments(Stream): + """Stream class for Comments in Shopify.""" + name = "comments" + data_key = "comments" + replication_key = "updatedAt" + access_scope = ["read_content" ,"read_online_store_pages"] + + def get_query(self): + return """ + query Comments($first: Int!, $after: String, $query: String) { + comments(first: $first, after: $after, query: $query) { + edges { + node { + id + body + bodyHtml + author { + email + name + } + createdAt + updatedAt + userAgent + status + publishedAt + isPublished + ip + article { + id + title + tags + publishedAt + isPublished + createdAt + body + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + +Context.stream_objects["comments"] = Comments diff --git a/tap_shopify/streams/currencies.py b/tap_shopify/streams/currencies.py new file mode 100644 index 00000000..5966c35d --- /dev/null +++ b/tap_shopify/streams/currencies.py @@ -0,0 +1,45 @@ +from tap_shopify.context import Context +from tap_shopify.streams.base import FullTableStream + + +class Currencies(FullTableStream): + """Stream class for Currencies in Shopify.""" + name = "currencies" + data_key = "shop" + key_properties = ["currencyCode"] + + def call_api(self, query_params, query=None, data_key=None): + """ + Overriding call_api method to extract data from nested data_key. + """ + root_data = super().call_api(query_params, query=query, data_key=data_key) + data = ( + root_data + .get("currencySettings", {}) + ) + return data + + def get_query(self): + return """ + query Currencies($first: Int!, $after: String) { + shop { + currencySettings(first: $first, after: $after) { + edges { + node { + currencyCode + currencyName + enabled + manualRate + rateUpdatedAt + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + } + """ + +Context.stream_objects["currencies"] = Currencies diff --git a/tap_shopify/streams/custom_collections.py b/tap_shopify/streams/custom_collections.py new file mode 100644 index 00000000..0a62dcf1 --- /dev/null +++ b/tap_shopify/streams/custom_collections.py @@ -0,0 +1,64 @@ +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream + + +class CustomCollections(Stream): + """Stream class for Custom Collections in Shopify.""" + name = "custom_collections" + data_key = "collections" + replication_key = "updatedAt" + access_scope = ["read_products"] + + def get_query_params(self, updated_at_min, updated_at_max, cursor=None): + """ + Extend base query params by adding `collection_type:custom` to the query filter. + + This keeps the updated_at filtering from the base class and adds + the supported filter for custom collections. + + Args: + updated_at_min (str): Lower bound for updated_at filtering. + updated_at_max (str): Upper bound for updated_at filtering. + cursor (str): Pagination cursor. + + Returns: + dict: Modified query parameters with only supported filters. + """ + params = super().get_query_params(updated_at_min, updated_at_max, cursor) + base_query = params.get("query", "") + extended_query = f"{base_query} AND collection_type:custom" + params["query"] = extended_query + + return params + + def get_query(self): + return """ + query CustomCollections($first: Int!, $after: String, $query: String) { + collections(first: $first, after: $after, query: $query) { + edges { + node { + id + description + title + updatedAt + handle + descriptionHtml + sortOrder + templateSuffix + image { + height + width + altText + url + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + +Context.stream_objects["custom_collections"] = CustomCollections diff --git a/tap_shopify/streams/customers.py b/tap_shopify/streams/customers.py index f589abd6..3bd3210a 100644 --- a/tap_shopify/streams/customers.py +++ b/tap_shopify/streams/customers.py @@ -7,6 +7,7 @@ class Customers(Stream): name = "customers" data_key = "customers" replication_key = "updatedAt" + access_scope = ["read_customers"] def get_query(self): """ diff --git a/tap_shopify/streams/draft_orders.py b/tap_shopify/streams/draft_orders.py new file mode 100644 index 00000000..30723b61 --- /dev/null +++ b/tap_shopify/streams/draft_orders.py @@ -0,0 +1,218 @@ +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream + + +class DraftOrders(Stream): + """Stream class for Draft Orders in Shopify.""" + name = "draft_orders" + data_key = "draftOrders" + replication_key = "updatedAt" + access_scope = ["read_draft_orders"] + + def get_query(self): + return """ + query DraftOrders($first: Int!, $after: String, $query: String) { + draftOrders(first: $first, after: $after, query: $query) { + edges { + node { + id + note2 + email + taxesIncluded + currencyCode + invoiceSentAt + createdAt + updatedAt + taxExempt + completedAt + name + status + invoiceUrl + tags + phone + totalShippingPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + subtotalPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + totalTaxSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + lineItems(first: 250) { + edges { + node { + id + name + quantity + sku + title + requiresShipping + taxable + custom + variant { + id + title + } + weight { + value + unit + } + vendor + taxLines { + title + source + rate + ratePercentage + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + } + product { + id + title + } + isGiftCard + originalUnitPriceWithCurrency { + amount + currencyCode + } + } + } + } + shippingAddress { + address1 + address2 + city + company + country + countryCodeV2 + firstName + latitude + longitude + lastName + name + phone + provinceCode + province + zip + } + shippingLine { + id + title + carrierIdentifier + custom + code + deliveryCategory + source + discountedPriceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + phone + } + taxLines { + channelLiable + rate + ratePercentage + source + title + priceSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + } + customer { + id + createdAt + lastName + firstName + defaultAddress { + address1 + address2 + city + company + country + countryCodeV2 + firstName + id + lastName + longitude + name + phone + province + provinceCode + zip + } + taxExempt + tags + defaultEmailAddress { + emailAddress + } + lastOrder { + id + name + currencyCode + } + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + +Context.stream_objects["draft_orders"] = DraftOrders diff --git a/tap_shopify/streams/fulfillment_orders.py b/tap_shopify/streams/fulfillment_orders.py index e42344ad..2a3166d7 100644 --- a/tap_shopify/streams/fulfillment_orders.py +++ b/tap_shopify/streams/fulfillment_orders.py @@ -11,6 +11,12 @@ class FulfillmentOrders(Stream): name = "fulfillment_orders" data_key = "fulfillmentOrders" replication_key = "updatedAt" + access_scope = [ + "read_assigned_fulfillment_orders", + "read_merchant_managed_fulfillment_orders", + "read_third_party_fulfillment_orders", + "read_marketplace_fulfillment_orders" + ] # pylint: disable=W0221,fixme def get_query_params(self, updated_at_min, updated_at_max, cursor=None): @@ -65,11 +71,12 @@ def transform_childitems(self, data, parent_id, key, next_page_key): return child_records - def transform_object(self, obj): + def transform_object(self, obj, **_kwargs): """ Transforms a collection object. Args: obj (dict): Collection object. + **_kwargs: Optional additional parameters. Returns: dict: Transformed collection object. """ diff --git a/tap_shopify/streams/fulfillment_services.py b/tap_shopify/streams/fulfillment_services.py new file mode 100644 index 00000000..0762cb00 --- /dev/null +++ b/tap_shopify/streams/fulfillment_services.py @@ -0,0 +1,65 @@ +import singer +from singer import metrics +from tap_shopify.context import Context +from tap_shopify.streams.base import FullTableStream + +LOGGER = singer.get_logger() + + +class FulfillmentServices(FullTableStream): + """Stream class for Fulfillment Services in Shopify.""" + name = "fulfillment_services" + data_key = "shop" + access_scope = ["read_fulfillments"] + + def call_api(self, query_params, query=None, data_key=None): + """ + Overriding call_api method to extract data from nested data_key. + """ + root_data = super().call_api(query_params, query=query, data_key=data_key) + data = ( + root_data + .get("fulfillmentServices", {}) + ) + return data + + # pylint: disable=too-many-locals + def get_objects(self): + """ + Returns: + - Yields list of objects for the stream + Performs: + - Transformation + """ + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) + LOGGER.info("GraphQL query for stream '%s': %s", self.name, ' '.join(query.split())) + + query_params = {} + + with metrics.http_request_timer(self.name): + data = self.call_api(query_params, query=query) + + for record in data: + obj = self.transform_object(record) + yield obj + + def get_query(self): + return """ + query FulfillmentServices { + shop { + fulfillmentServices { + id + handle + callbackUrl + inventoryManagement + permitsSkuSharing + requiresShippingMethod + serviceName + trackingSupport + type + } + } + } + """ + +Context.stream_objects["fulfillment_services"] = FulfillmentServices diff --git a/tap_shopify/streams/inventory_items.py b/tap_shopify/streams/inventory_items.py index dde493b4..d9d06965 100644 --- a/tap_shopify/streams/inventory_items.py +++ b/tap_shopify/streams/inventory_items.py @@ -8,14 +8,15 @@ class InventoryItems(Stream): name = "inventory_items" data_key = "inventoryItems" replication_key = "updatedAt" + access_scope = ["read_inventory"] - def transform_object(self, obj): + def transform_object(self, obj, **_kwargs): """ Transforms the object by extracting country harmonized system codes. Args: obj (dict): The object to transform. - + **_kwargs: Optional additional parameters. Returns: dict: Transformed object. """ diff --git a/tap_shopify/streams/inventory_levels.py b/tap_shopify/streams/inventory_levels.py index eb12bc6d..081fda2f 100644 --- a/tap_shopify/streams/inventory_levels.py +++ b/tap_shopify/streams/inventory_levels.py @@ -11,6 +11,7 @@ class InventoryLevels(Stream): data_key = "locations" child_data_key = "inventoryLevels" replication_key = "updatedAt" + access_scope = ["read_inventory"] def get_next_page_child(self, parent_id, cursor, child_query): """ diff --git a/tap_shopify/streams/locations.py b/tap_shopify/streams/locations.py index 67b60ec1..73b06416 100644 --- a/tap_shopify/streams/locations.py +++ b/tap_shopify/streams/locations.py @@ -9,6 +9,7 @@ class Locations(Stream): # Currently, the replication key is set to 'createdAt' because the Shopify # locations graphql endpoint doesn't allow the filter on 'updatedAt' field. replication_key = "createdAt" + access_scope = ["read_locations"] def get_query(self): """ diff --git a/tap_shopify/streams/marketing_events.py b/tap_shopify/streams/marketing_events.py new file mode 100644 index 00000000..1da14da2 --- /dev/null +++ b/tap_shopify/streams/marketing_events.py @@ -0,0 +1,43 @@ +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream + + +class MarketingEvents(Stream): + """Stream class for Marketing Events in Shopify.""" + name = "marketing_events" + data_key = "marketingEvents" + replication_key = "startedAt" + access_scope = ["read_marketing_events"] + + def get_query(self): + return """ + query MarketingEvents($first: Int!, $after: String, $query: String) { + marketingEvents(first: $first, after: $after, query: $query) { + edges { + node { + id + type + remoteId + startedAt + endedAt + scheduledToEndAt + manageUrl + previewUrl + utmCampaign + utmMedium + utmSource + description + marketingChannelType + sourceAndMedium + channelHandle + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + +Context.stream_objects["marketing_events"] = MarketingEvents diff --git a/tap_shopify/streams/metafields.py b/tap_shopify/streams/metafields.py index cc830fdf..39a730af 100644 --- a/tap_shopify/streams/metafields.py +++ b/tap_shopify/streams/metafields.py @@ -21,7 +21,7 @@ class Metafields(Stream, ABC): def get_query(self): """Placeholder for get_query method.""" - def transform_object(self, obj): + def transform_object(self, obj, **_kwargs): """ Transforms a metafield object for output. """ diff --git a/tap_shopify/streams/metafields_collections.py b/tap_shopify/streams/metafields_collections.py index 1b94cd4e..ccd564e5 100644 --- a/tap_shopify/streams/metafields_collections.py +++ b/tap_shopify/streams/metafields_collections.py @@ -9,6 +9,7 @@ class MetafieldsCollections(Metafields): name = "metafields_collections" data_key = "collections" + access_scope = ["read_products"] def get_query(self): """Return the GraphQL query for fetching collection metafields.""" diff --git a/tap_shopify/streams/metafields_customers.py b/tap_shopify/streams/metafields_customers.py index e3290c78..e68ea553 100644 --- a/tap_shopify/streams/metafields_customers.py +++ b/tap_shopify/streams/metafields_customers.py @@ -9,6 +9,7 @@ class MetafieldsCustomers(Metafields): name = "metafields_customers" data_key = "customers" + access_scope = ["read_customers"] def get_query(self): """Return the GraphQL query for fetching customer metafields.""" diff --git a/tap_shopify/streams/metafields_orders.py b/tap_shopify/streams/metafields_orders.py index 2c7f0d36..416190dd 100644 --- a/tap_shopify/streams/metafields_orders.py +++ b/tap_shopify/streams/metafields_orders.py @@ -8,6 +8,7 @@ class MetafieldsOrders(Metafields): """Stream class for metafields associated with orders.""" name = "metafields_orders" data_key = "orders" + access_scope = ["read_orders"] def get_query(self): """Returns the GraphQL query for fetching order metafields.""" diff --git a/tap_shopify/streams/metafields_products.py b/tap_shopify/streams/metafields_products.py index c4c5e6e3..644fa830 100644 --- a/tap_shopify/streams/metafields_products.py +++ b/tap_shopify/streams/metafields_products.py @@ -8,6 +8,7 @@ class MetafieldsProducts(Metafields): """Stream class for product metafields.""" name = "metafields_products" data_key = "products" + access_scope = ["read_products"] def get_query(self): """Return the GraphQL query for product metafields.""" diff --git a/tap_shopify/streams/order_refunds.py b/tap_shopify/streams/order_refunds.py index 1b11cd16..7570c331 100644 --- a/tap_shopify/streams/order_refunds.py +++ b/tap_shopify/streams/order_refunds.py @@ -11,6 +11,7 @@ class OrderRefunds(Stream): data_key = "orders" child_data_key = "refunds" replication_key = "updatedAt" + access_scope = ["read_orders"] automatic_keys = ["order"] # pylint: disable=too-many-locals @@ -145,13 +146,13 @@ def transform_orderadjustments(self, data): return orderadjustments - def transform_object(self, obj): + def transform_object(self, obj, **_kwargs): """ Transform refund objects by extracting refund line items from edges. Args: obj (dict): Refund object. - + **_kwargs: Optional additional parameters. Returns: dict: Transformed refund object. """ diff --git a/tap_shopify/streams/order_shipping_lines.py b/tap_shopify/streams/order_shipping_lines.py index 6786b35e..62d4769a 100644 --- a/tap_shopify/streams/order_shipping_lines.py +++ b/tap_shopify/streams/order_shipping_lines.py @@ -11,6 +11,7 @@ class OrderShippingLines(Stream): data_key = "orders" child_data_key = "shippingLines" replication_key = "updatedAt" + access_scope = ["read_orders"] # pylint: disable=too-many-locals def get_objects(self): diff --git a/tap_shopify/streams/orders.py b/tap_shopify/streams/orders.py index d6137c0e..43e629e5 100644 --- a/tap_shopify/streams/orders.py +++ b/tap_shopify/streams/orders.py @@ -16,6 +16,7 @@ class Orders(Stream): name = "orders" data_key = "orders" replication_key = "updatedAt" + access_scope = ["read_orders"] def get_query(self): """ @@ -1185,7 +1186,7 @@ def parse_bulk_jsonl(self, url): current_order["lineItems"] = current_line_items yield current_order - def transform_object(self, obj): + def transform_object(self, obj, **_kwargs): if obj.get("lineItems", {}).get("edges"): obj["lineItems"] = [item["node"] for item in obj["lineItems"]["edges"]] return obj diff --git a/tap_shopify/streams/pages.py b/tap_shopify/streams/pages.py new file mode 100644 index 00000000..ddb36f26 --- /dev/null +++ b/tap_shopify/streams/pages.py @@ -0,0 +1,37 @@ +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream + + +class Pages(Stream): + """Stream class for Pages in Shopify.""" + name = "pages" + data_key = "pages" + access_scope = ["read_content" ,"read_online_store_pages"] + + def get_query(self): + return """ + query Pages($first: Int!, $after: String, $query: String) { + pages(first: $first, after: $after, query: $query) { + edges { + node { + id + updatedAt + handle + createdAt + bodySummary + body + isPublished + publishedAt + templateSuffix + title + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + +Context.stream_objects["pages"] = Pages diff --git a/tap_shopify/streams/policies.py b/tap_shopify/streams/policies.py new file mode 100644 index 00000000..50d2d54e --- /dev/null +++ b/tap_shopify/streams/policies.py @@ -0,0 +1,88 @@ +import singer +from singer import metrics, utils +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream + +LOGGER = singer.get_logger() + + +class Policies(Stream): + """Stream class for Policies in Shopify.""" + name = "policies" + data_key = "shop" + access_scope = ["read_legal_policies"] + + def call_api(self, query_params, query=None, data_key=None): + """ + Overriding call_api method to extract data from nested data_key. + """ + root_data = super().call_api(query_params, query=query, data_key=data_key) + data = ( + root_data + .get("shopPolicies", {}) + ) + return data + + def transform_object(self, obj, **_kwargs): + """ + If the replication key is missing in the input node, a fallback timestamp (current UTC time) + is injected to allow the pipeline to continue operating in incremental sync mode. + Ensures the replication key is returned as an ISO string. + + Args: + obj (dict): Product object. + **_kwargs: Optional additional parameters. + Returns: + dict: Transformed product object. + """ + if self.replication_key not in obj or not obj[self.replication_key]: + last_updated_at = _kwargs.get( + "last_updated_at", + utils.now().replace(microsecond=0) + ) + obj[self.replication_key] = last_updated_at.isoformat() + return obj + + # pylint: disable=too-many-locals + def get_objects(self): + """ + Returns: + - Yields list of objects for the stream + Performs: + - Transformation and bookmarking + """ + last_updated_at = self.get_bookmark() + current_bookmark = last_updated_at + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) + LOGGER.info("GraphQL query for stream '%s': %s", self.name, ' '.join(query.split())) + query_params = {} + + with metrics.http_request_timer(self.name): + data = self.call_api(query_params, query=query) + + for record in data: + obj = self.transform_object(record, last_updated_at=last_updated_at) + replication_value = utils.strptime_to_utc(obj[self.replication_key]) + if replication_value >= current_bookmark: + yield obj + + current_bookmark = max(current_bookmark, replication_value) + self.update_bookmark(utils.strftime(current_bookmark)) + + def get_query(self): + return """ + query Policies { + shop { + shopPolicies { + createdAt + id + title + type + updatedAt + url + } + } + } + """ + +Context.stream_objects["policies"] = Policies diff --git a/tap_shopify/streams/price_rules.py b/tap_shopify/streams/price_rules.py new file mode 100644 index 00000000..e2763e36 --- /dev/null +++ b/tap_shopify/streams/price_rules.py @@ -0,0 +1,148 @@ +from tap_shopify.context import Context +from tap_shopify.streams.base import FullTableStream + + +class PriceRules(FullTableStream): + """Stream class for Price Rules in Shopify.""" + name = "price_rules" + data_key = "discountNodes" + access_scope = ["read_discounts"] + + def get_query(self): + return """ + query PriceRules($first: Int!, $after: String, $query: String) { + discountNodes(first: $first, after: $after, query: $query) { + edges { + node { + id + discount { + ... on DiscountAutomaticApp { + __typename + title + status + createdAt + startsAt + updatedAt + endsAt + combinesWith { + orderDiscounts + productDiscounts + shippingDiscounts + } + } + ... on DiscountAutomaticBasic { + __typename + title + status + createdAt + startsAt + updatedAt + endsAt + combinesWith { + orderDiscounts + productDiscounts + shippingDiscounts + } + } + ... on DiscountAutomaticBxgy { + __typename + title + status + createdAt + startsAt + updatedAt + endsAt + usesPerOrderLimit + combinesWith { + orderDiscounts + productDiscounts + shippingDiscounts + } + } + ... on DiscountAutomaticFreeShipping { + __typename + title + status + createdAt + startsAt + updatedAt + endsAt + combinesWith { + orderDiscounts + productDiscounts + shippingDiscounts + } + } + ... on DiscountCodeApp { + __typename + title + status + createdAt + startsAt + updatedAt + endsAt + usageLimit + combinesWith { + orderDiscounts + productDiscounts + shippingDiscounts + } + } + ... on DiscountCodeBasic { + __typename + title + status + createdAt + startsAt + updatedAt + endsAt + usageLimit + combinesWith { + orderDiscounts + productDiscounts + shippingDiscounts + } + } + ... on DiscountCodeBxgy { + __typename + title + status + createdAt + startsAt + updatedAt + endsAt + usageLimit + usesPerOrderLimit + combinesWith { + orderDiscounts + productDiscounts + shippingDiscounts + } + } + ... on DiscountCodeFreeShipping { + __typename + title + status + createdAt + startsAt + updatedAt + endsAt + usageLimit + combinesWith { + orderDiscounts + productDiscounts + shippingDiscounts + } + } + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + +Context.stream_objects["price_rules"] = PriceRules diff --git a/tap_shopify/streams/product_variants.py b/tap_shopify/streams/product_variants.py index 4f18cbe8..b8ab498b 100644 --- a/tap_shopify/streams/product_variants.py +++ b/tap_shopify/streams/product_variants.py @@ -7,14 +7,15 @@ class ProductVariants(Stream): name = "product_variants" data_key = "productVariants" replication_key = "updatedAt" + access_scope = ["read_products"] - def transform_object(self, obj): + def transform_object(self, obj, **_kwargs): """ Transforms the object if needed. Args: obj (dict): The object to transform. - + **_kwargs: Optional additional parameters. Returns: dict: The transformed object. """ diff --git a/tap_shopify/streams/products.py b/tap_shopify/streams/products.py index e8c564c2..0ef543ff 100644 --- a/tap_shopify/streams/products.py +++ b/tap_shopify/streams/products.py @@ -8,14 +8,15 @@ class Products(Stream): name = "products" data_key = "products" replication_key = "updatedAt" + access_scope = ["read_products"] - def transform_object(self, obj): + def transform_object(self, obj, **_kwargs): """ Transforms the product object by extracting media information. Args: obj (dict): Product object. - + **_kwargs: Optional additional parameters. Returns: dict: Transformed product object. """ diff --git a/tap_shopify/streams/recurring_application_charges.py b/tap_shopify/streams/recurring_application_charges.py new file mode 100644 index 00000000..6012e108 --- /dev/null +++ b/tap_shopify/streams/recurring_application_charges.py @@ -0,0 +1,97 @@ +import singer +from singer import metrics +from tap_shopify.context import Context +from tap_shopify.streams.base import FullTableStream + +LOGGER = singer.get_logger() + + +class RecurringApplicationCharges(FullTableStream): + """Stream class for Recurring Application Charges in Shopify.""" + name = "recurring_application_charges" + data_key = "currentAppInstallation" + + def call_api(self, query_params, query=None, data_key=None): + """ + Overriding call_api method to extract data from nested data_key. + """ + root_data = super().call_api(query_params, query=query, data_key=data_key) + data = ( + root_data + .get("activeSubscriptions", {}) + ) + return data + + # pylint: disable=too-many-locals + def get_objects(self): + """ + Returns: + - Yields list of objects for the stream + Performs: + - Transformation + """ + + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) + LOGGER.info("GraphQL query for stream '%s': %s", self.name, ' '.join(query.split())) + + query_params = {} + + with metrics.http_request_timer(self.name): + data = self.call_api(query_params, query=query) + + for record in data: + obj = self.transform_object(record) + yield obj + + def get_query(self): + return """ + query RecurringApplicationCharges { + currentAppInstallation { + activeSubscriptions { + createdAt + currentPeriodEnd + id + name + returnUrl + status + test + trialDays + lineItems { + id + plan { + pricingDetails { + __typename + ... on AppRecurringPricing { + planHandle + interval + discount { + durationLimitInIntervals + remainingDurationInIntervals + value { + ... on AppSubscriptionDiscountAmount { + __typename + amount { + amount + currencyCode + } + } + ... on AppSubscriptionDiscountPercentage { + __typename + percentage + } + } + priceAfterDiscount { + amount + currencyCode + } + } + } + } + } + } + } + } + } + """ + +Context.stream_objects["recurring_application_charges"] = RecurringApplicationCharges diff --git a/tap_shopify/streams/redirects.py b/tap_shopify/streams/redirects.py new file mode 100644 index 00000000..66349690 --- /dev/null +++ b/tap_shopify/streams/redirects.py @@ -0,0 +1,52 @@ +from singer import utils +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream + + +class Redirects(Stream): + """Stream class for Redirects in Shopify.""" + name = "redirects" + data_key = "urlRedirects" + replication_key = "createdAt" + access_scope = ["read_online_store_navigation"] + + def transform_object(self, obj, **_kwargs): + """ + If the replication key is missing in the input node, a fallback timestamp (current UTC time) + is injected to allow the pipeline to continue operating in incremental sync mode. + Ensures the replication key is returned as an ISO string. + + Args: + obj (dict): Product object. + **_kwargs: Optional additional parameters. + Returns: + dict: Transformed product object. + """ + if self.replication_key not in obj or not obj[self.replication_key]: + last_updated_at = _kwargs.get( + "last_updated_at", + utils.now().replace(microsecond=0) + ) + obj[self.replication_key] = last_updated_at.isoformat() + return obj + + def get_query(self): + return """ + query Redirects($first: Int!, $after: String, $query: String) { + urlRedirects(first: $first, after: $after, query: $query) { + edges { + node { + id + path + target + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + +Context.stream_objects["redirects"] = Redirects diff --git a/tap_shopify/streams/resource_feedback.py b/tap_shopify/streams/resource_feedback.py new file mode 100644 index 00000000..7230c6b1 --- /dev/null +++ b/tap_shopify/streams/resource_feedback.py @@ -0,0 +1,89 @@ +import singer +from singer import metrics, utils +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream + +LOGGER = singer.get_logger() + + +class ResourceFeedback(Stream): + """Stream class for Resource Feedback in Shopify.""" + name = "resource_feedback" + data_key = "app" + replication_key = "feedback_generated_at" + key_properties = ["app_id"] + access_scope = ["read_products"] + + def transform_object(self, obj, **_kwargs): + """ + If the replication key is missing in the input node, a fallback timestamp (current UTC time) + is injected to allow the pipeline to continue operating in incremental sync mode. + Ensures the replication key is returned as an ISO string. + + Args: + obj (dict): object. + **_kwargs: Optional additional parameters. + Returns: + dict: Transformed product object. + """ + if "id" in obj: + obj["app_id"] = obj.pop("id") + + feedback = obj.get("feedback") + feedback_generated_at = feedback.get("feedbackGeneratedAt") if feedback else None + + if feedback_generated_at: + obj[self.replication_key] = feedback_generated_at + elif not obj.get(self.replication_key): + last_updated_at = _kwargs["last_updated_at"] + obj[self.replication_key] = last_updated_at.isoformat() + return obj + + # pylint: disable=too-many-locals + def get_objects(self): + """ + Returns: + - Yields list of objects for the stream + Performs: + - Transformation and Bookmarking + """ + last_updated_at = self.get_bookmark() + current_bookmark = last_updated_at + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) + LOGGER.info("GraphQL query for stream '%s': %s", self.name, ' '.join(query.split())) + query_params = {} + + with metrics.http_request_timer(self.name): + data = self.call_api(query_params, query=query) + + obj = self.transform_object(data, last_updated_at=last_updated_at) + replication_value = utils.strptime_to_utc(obj[self.replication_key]) + if replication_value >= current_bookmark: + yield obj + + current_bookmark = max(current_bookmark, replication_value) + self.update_bookmark(utils.strftime(current_bookmark)) + + def get_query(self): + return """ + query Redirects { + app { + id + title + feedback { + messages { + field + message + } + state + feedbackGeneratedAt + link { + label + url + } + } + } + } + """ + +Context.stream_objects["resource_feedback"] = ResourceFeedback diff --git a/tap_shopify/streams/script_tags.py b/tap_shopify/streams/script_tags.py new file mode 100644 index 00000000..3437e724 --- /dev/null +++ b/tap_shopify/streams/script_tags.py @@ -0,0 +1,35 @@ +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream + + +class ScriptTags(Stream): + """Stream class for Script Tags in Shopify.""" + name = "script_tags" + data_key = "scriptTags" + replication_key = "updatedAt" + access_scope = ["read_script_tags"] + + def get_query(self): + return """ + query ScriptTags($first: Int!, $after: String, $query: String) { + scriptTags(first: $first, after: $after, query: $query) { + edges { + node { + id + cache + createdAt + updatedAt + displayScope + legacyResourceId + src + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + +Context.stream_objects["script_tags"] = ScriptTags diff --git a/tap_shopify/streams/shipping_zones.py b/tap_shopify/streams/shipping_zones.py new file mode 100644 index 00000000..c01a29f4 --- /dev/null +++ b/tap_shopify/streams/shipping_zones.py @@ -0,0 +1,82 @@ +from tap_shopify.context import Context +from tap_shopify.streams.base import FullTableStream + +RESULTS_PER_PAGE = 10 + +class ShippingZones(FullTableStream): + """Stream class for Shipping Zones in Shopify.""" + name = "shipping_zones" + data_key = "deliveryProfiles" + access_scope = ["read_shipping"] + + def __init__(self): + super().__init__() + self.results_per_page = RESULTS_PER_PAGE + + def get_query(self): + return """ + query ShippingZones($first: Int!, $after: String) { + deliveryProfiles(first: $first, after: $after) { + edges { + node { + id + profileLocationGroups { + locationGroup { + id + } + locationGroupZones(first: 50) { + edges { + node { + zone { + id + name + countries { + code { + countryCode + restOfWorld + } + provinces { + name + code + } + } + } + methodDefinitions(first: 50) { + edges { + node { + id + active + description + methodConditions { + field + operator + conditionCriteria { + __typename + ... on MoneyV2 { + amount + currencyCode + } + ... on Weight { + unit + value + } + } + } + } + } + } + } + } + } + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + +Context.stream_objects["shipping_zones"] = ShippingZones diff --git a/tap_shopify/streams/shop.py b/tap_shopify/streams/shop.py new file mode 100644 index 00000000..f9ec1227 --- /dev/null +++ b/tap_shopify/streams/shop.py @@ -0,0 +1,116 @@ +import singer +from singer import metrics, utils +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream + +LOGGER = singer.get_logger() + + +class Shop(Stream): + """Stream class for Shop in Shopify.""" + name = "shop" + data_key = "shop" + + # pylint: disable=too-many-locals + def get_objects(self): + """ + Returns: + - Yields list of objects for the stream + Performs: + - Transformation and Bookmarking + """ + last_updated_at = self.get_bookmark() + current_bookmark = last_updated_at + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) + LOGGER.info("GraphQL query for stream '%s': %s", self.name, ' '.join(query.split())) + query_params = {} + + with metrics.http_request_timer(self.name): + data = self.call_api(query_params, query=query) + + obj = self.transform_object(data) + replication_value = utils.strptime_to_utc(obj[self.replication_key]) + if replication_value >= current_bookmark: + yield obj + + current_bookmark = max(current_bookmark, replication_value) + self.update_bookmark(utils.strftime(current_bookmark)) + + def get_query(self): + return """ + query ShopShow { + shop { + alerts { + action { + title + url + } + description + } + billingAddress { + address1 + address2 + city + company + country + countryCodeV2 + latitude + longitude + phone + province + provinceCode + zip + } + countriesInShippingZones { + countryCodes + includeRestOfWorld + } + checkoutApiSupported + contactEmail + createdAt + currencyCode + currencyFormats { + moneyFormat + moneyInEmailsFormat + moneyWithCurrencyFormat + moneyWithCurrencyInEmailsFormat + } + customerAccounts + description + email + enabledPresentmentCurrencies + fulfillmentServices { + handle + serviceName + } + ianaTimezone + id + marketingSmsConsentEnabledAtCheckout + myshopifyDomain + name + paymentSettings { + supportedDigitalWallets + } + plan { + partnerDevelopment + shopifyPlus + publicDisplayName + } + primaryDomain { + host + id + } + setupRequired + shipsToCountries + taxesIncluded + taxShipping + timezoneAbbreviation + transactionalSmsDisabled + updatedAt + url + weightUnit + } + } + """ + +Context.stream_objects["shop"] = Shop diff --git a/tap_shopify/streams/smart_collections.py b/tap_shopify/streams/smart_collections.py new file mode 100644 index 00000000..4a727642 --- /dev/null +++ b/tap_shopify/streams/smart_collections.py @@ -0,0 +1,70 @@ +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream + + +class SmartCollections(Stream): + """Stream class for Smart Collections in Shopify.""" + name = "smart_collections" + data_key = "collections" + replication_key = "updatedAt" + access_scope = ["read_products"] + + def get_query_params(self, updated_at_min, updated_at_max, cursor=None): + """ + Extend base query params by adding `collection_type:custom` to the query filter. + + This keeps the updated_at filtering from the base class and adds + the supported filter for custom collections. + + Args: + updated_at_min (str): Lower bound for updated_at filtering. + updated_at_max (str): Upper bound for updated_at filtering. + cursor (str): Pagination cursor. + + Returns: + dict: Modified query parameters with only supported filters. + """ + params = super().get_query_params(updated_at_min, updated_at_max, cursor) + base_query = params.get("query", "") + extended_query = f"{base_query} AND collection_type:smart" + params["query"] = extended_query + + return params + + def get_query(self): + return """ + query SmartCollections($first: Int!, $after: String, $query: String) { + collections(first: $first, after: $after, query: $query) { + edges { + node { + id + handle + title + updatedAt + description + descriptionHtml + sortOrder + templateSuffix + ruleSet { + appliedDisjunctively + rules { + column + relation + condition + } + } + seo { + description + title + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + +Context.stream_objects["smart_collections"] = SmartCollections diff --git a/tap_shopify/streams/tender_transactions.py b/tap_shopify/streams/tender_transactions.py new file mode 100644 index 00000000..3bd2c4b3 --- /dev/null +++ b/tap_shopify/streams/tender_transactions.py @@ -0,0 +1,46 @@ +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream + + +class TenderTransactions(Stream): + """Stream class for Tender Transactions in Shopify.""" + name = "tender_transactions" + data_key = "tenderTransactions" + replication_key = "processedAt" + access_scope = ["read_orders"] + + def get_query(self): + return """ + query TenderTransactions($first: Int!, $after: String, $query: String) { + tenderTransactions(first: $first, after: $after, query: $query) { + edges { + node { + id + paymentMethod + processedAt + remoteReference + test + transactionDetails { + ... on TenderTransactionCreditCardDetails { + creditCardCompany + creditCardNumber + } + } + amount { + amount + currencyCode + } + order { + id + } + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + +Context.stream_objects["tender_transactions"] = TenderTransactions diff --git a/tap_shopify/streams/themes.py b/tap_shopify/streams/themes.py new file mode 100644 index 00000000..5304ad30 --- /dev/null +++ b/tap_shopify/streams/themes.py @@ -0,0 +1,92 @@ +import singer +from singer import metrics, utils +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream + +LOGGER = singer.get_logger() + + +class Themes(Stream): + """Stream class for Themes in Shopify.""" + name = "themes" + data_key = "themes" + replication_key = "updatedAt" + access_scope = ["read_themes"] + + # pylint: disable=W0221 + def get_query_params(self, cursor=None): + """ + Construct query parameters for GraphQL requests. + + Args: + cursor (str): Pagination cursor, if any. + + Returns: + dict: Dictionary of query parameters. + """ + params = { + "first": self.results_per_page, + } + + if cursor: + params["after"] = cursor + return params + + # pylint: disable=too-many-locals + def get_objects(self): + """ + Returns: + - Yields list of objects for the stream + Performs: + - Pagination & Filtering of stream + - Transformation and bookmarking + """ + last_updated_at = self.get_bookmark() + current_bookmark = last_updated_at + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) + LOGGER.info("GraphQL query for stream '%s': %s", self.name, ' '.join(query.split())) + has_next_page, cursor = True, None + + while has_next_page: + query_params = self.get_query_params(cursor) + + with metrics.http_request_timer(self.name): + data = self.call_api(query_params, query=query) + + for edge in data.get("edges"): + obj = self.transform_object(edge.get("node")) + replication_value = utils.strptime_to_utc(obj[self.replication_key]) + if replication_value >= current_bookmark: + current_bookmark = max(current_bookmark, replication_value) + yield obj + + page_info = data.get("pageInfo") + cursor, has_next_page = page_info.get("endCursor"), page_info.get("hasNextPage") + self.update_bookmark(utils.strftime(current_bookmark)) + + def get_query(self): + return """ + query Themes($first: Int!, $after: String) { + themes(first: $first, after: $after) { + edges { + node { + id + name + prefix + processing + processingFailed + role + themeStoreId + updatedAt + createdAt + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + +Context.stream_objects["themes"] = Themes diff --git a/tap_shopify/streams/transactions.py b/tap_shopify/streams/transactions.py index fb3de78a..a29928b1 100644 --- a/tap_shopify/streams/transactions.py +++ b/tap_shopify/streams/transactions.py @@ -11,6 +11,7 @@ class Transactions(Stream): data_key = "orders" child_data_key = "transactions" replication_key = "createdAt" + access_scope = ["read_orders"] # pylint: disable=W0221 def get_query_params(self, updated_at_min, updated_at_max, cursor=None): diff --git a/tap_shopify/streams/webhooks.py b/tap_shopify/streams/webhooks.py new file mode 100644 index 00000000..ba19abb3 --- /dev/null +++ b/tap_shopify/streams/webhooks.py @@ -0,0 +1,56 @@ +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream + + +class Webhooks(Stream): + """Stream class for Webhooks in Shopify.""" + name = "webhooks" + data_key = "webhookSubscriptions" + + def get_query(self): + return """ + query Webhooks($first: Int!, $after: String, $query: String) { + webhookSubscriptions(first: $first, after: $after, query: $query) { + edges { + node { + id + format + filter + createdAt + apiVersion { + displayName + handle + supported + } + includeFields + legacyResourceId + metafieldNamespaces + topic + updatedAt + endpoint { + ... on WebhookEventBridgeEndpoint { + __typename + arn + } + ... on WebhookHttpEndpoint { + __typename + callbackUrl + } + ... on WebhookPubSubEndpoint { + __typename + pubSubProject + pubSubTopic + } + } + callbackUrl + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + """ + +Context.stream_objects["webhooks"] = Webhooks diff --git a/tests/base.py b/tests/base.py index ef414d2e..09bc2214 100644 --- a/tests/base.py +++ b/tests/base.py @@ -27,7 +27,7 @@ class BaseTapTest(BaseCase): FULL = "FULL_TABLE" START_DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" BOOKMARK_COMPARISON_FORMAT = "%Y-%m-%dT00:00:00+00:00" - DEFAULT_RESULTS_PER_PAGE = 250 + DEFAULT_RESULTS_PER_PAGE = 1 @staticmethod def tap_name(): @@ -43,7 +43,7 @@ def get_type(): def get_properties(self, original: bool = True): """Configuration properties required for the tap.""" return_value = { - 'start_date': '2017-07-01T00:00:00Z', + 'start_date': '2020-07-01T00:00:00Z', 'shop': 'stitchdatawearhouse', 'date_window_size': 180, # BUG: https://jira.talendforge.org/browse/TDL-13180 @@ -82,9 +82,15 @@ def expected_metadata(self): self.REPLICATION_METHOD: self.INCREMENTAL, self.API_LIMIT: self.DEFAULT_RESULTS_PER_PAGE} + full_default = { + self.PRIMARY_KEYS: {"id"}, + self.REPLICATION_METHOD: self.FULL, + self.API_LIMIT: 2 + } + meta = default.copy() meta[self.REPLICATION_KEYS] = {"updatedAt"} - meta[self.API_LIMIT] = 30 + meta[self.API_LIMIT] = 2 meta.update({self.FOREIGN_KEYS: {"owner", "ownerType"}}) return { @@ -93,7 +99,7 @@ def expected_metadata(self): self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.INCREMENTAL, # BUG: https://qlik-dev.atlassian.net/browse/TDL-13180 - self.API_LIMIT: 50}, + self.API_LIMIT: 2}, "collections": default, "customers": default, "orders": default, @@ -101,13 +107,13 @@ def expected_metadata(self): self.REPLICATION_KEYS: {"updatedAt"}, self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.INCREMENTAL, - self.API_LIMIT: 30 + self.API_LIMIT: 2 }, "order_shipping_lines": { self.REPLICATION_KEYS: {"updatedAt"}, self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.INCREMENTAL, - self.API_LIMIT: 15 + self.API_LIMIT: 2 }, "order_refunds": default, "products": default, @@ -122,7 +128,7 @@ def expected_metadata(self): self.PRIMARY_KEYS: {"id"}, self.FOREIGN_KEYS: {"order"}, self.REPLICATION_METHOD: self.INCREMENTAL, - self.API_LIMIT: 150}, + self.API_LIMIT: 2}, "locations": { self.REPLICATION_KEYS: {"createdAt"}, self.PRIMARY_KEYS: {"id"}, @@ -134,18 +140,103 @@ def expected_metadata(self): self.REPLICATION_KEYS: {"createdAt"}, self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.INCREMENTAL, - self.API_LIMIT: 50 - } + self.API_LIMIT: 2 + }, + "application_charges": full_default, + "application_credits": full_default, + "article_authors": { + self.PRIMARY_KEYS: {"name"}, + self.REPLICATION_METHOD: self.FULL, + self.API_LIMIT: 2 + }, + "article_tags": { + self.PRIMARY_KEYS: {"article_tag"}, + self.REPLICATION_METHOD: self.FULL, + self.API_LIMIT: 2 + }, + "blogs": default, + "carrier_services": full_default, + "comments": default, + "currencies": { + self.PRIMARY_KEYS: {"currencyCode"}, + self.REPLICATION_METHOD: self.FULL, + self.API_LIMIT: 2 + }, + "custom_collections": default, + "draft_orders": default, + "fulfillment_services": full_default, + "marketing_events": { + self.REPLICATION_KEYS: {"startedAt"}, + self.PRIMARY_KEYS: {"id"}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.API_LIMIT: 2 + }, + "pages": default, + "policies": default, + "price_rules": full_default, + "recurring_application_charges": full_default, + "redirects": { + self.REPLICATION_KEYS: {"createdAt"}, + self.PRIMARY_KEYS: {"id"}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.API_LIMIT: 2 + }, + "resource_feedback": { + self.REPLICATION_KEYS: {"feedback_generated_at"}, + self.PRIMARY_KEYS: {"app_id"}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.API_LIMIT: 2 + }, + "script_tags": default, + "shipping_zones": { + self.PRIMARY_KEYS: {"id"}, + self.REPLICATION_METHOD: self.FULL, + self.API_LIMIT: 2 + }, + "shop": default, + "smart_collections": default, + "tender_transactions": { + self.REPLICATION_KEYS: {"processedAt"}, + self.PRIMARY_KEYS: {"id"}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.API_LIMIT: 2 + }, + "themes": default, + "webhooks": default } def expected_streams(self): """A set of expected stream names""" + # excluding streams due to lack of data and permissions + streams_to_exclude = { + 'article_authors', + 'article_tags', + 'blogs', + 'carrier_services', + 'comments', + 'draft_orders', + 'fulfillment_services', + 'marketing_events', + 'pages', + 'policies', + 'price_rules', + 'redirects', + 'script_tags', + 'shipping_zones', + 'themes', + 'application_charges', + 'currencies', + 'application_credits', + 'webhooks', + 'recurring_application_charges', + 'abandoned_checkouts' + } # removed "abandoned_checkouts", as per the Doc: # https://help.shopify.com/en/manual/orders/abandoned-checkouts?st_source=admin&st_campaign=abandoned_checkouts_footer&utm_source=admin&utm_campaign=abandoned_checkouts_footer#review-your-abandoned-checkouts # abandoned checkouts are saved in the Shopify admin for three months. # Every Monday, abandoned checkouts that are older than three months are removed from your admin. # Also no POST call is available for this endpoint: https://shopify.dev/api/admin-rest/2022-01/resources/abandoned-checkouts - return set(self.expected_metadata().keys()) - {"abandoned_checkouts"} + return set(self.expected_metadata().keys()) - streams_to_exclude def child_streams(self): """ @@ -313,13 +404,34 @@ def select_all_streams_and_fields(conn_id, catalogs, select_all_fields: bool = T def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.start_date = self.get_properties().get("start_date") - self.store_1_streams = {'collections', 'orders', 'products', 'customers', 'locations', 'inventory_levels', 'inventory_items', 'fulfillment_orders'} + self.store_1_streams = { + 'collections', + 'orders', + 'products', + 'customers', + 'locations', + 'inventory_levels', + 'inventory_items', + 'fulfillment_orders', + 'events' + } # removed 'abandoned_checkouts' from store 2 streams, as per the Doc: # https://help.shopify.com/en/manual/orders/abandoned-checkouts?st_source=admin&st_campaign=abandoned_checkouts_footer&utm_source=admin&utm_campaign=abandoned_checkouts_footer#review-your-abandoned-checkouts # abandoned checkouts are saved in the Shopify admin for three months. # Every Monday, abandoned checkouts that are older than three months are removed from your admin. # Also no POST call is available for this endpoint: https://shopify.dev/api/admin-rest/2022-01/resources/abandoned-checkouts - self.store_2_streams = {'metafields_products', 'transactions', 'order_refunds', 'products', 'locations', 'inventory_levels', 'inventory_items', 'order_shipping_lines'} + self.store_2_streams = { + 'metafields_products', + 'transactions', + 'order_refunds', + 'products', + 'locations', + 'inventory_levels', + 'inventory_items', + 'order_shipping_lines', + 'product_variants', + 'tender_transactions' + } #modified this method to accommodate replication key in the current_state def calculated_states_by_stream(self, current_state): diff --git a/tests/test_all_fields.py b/tests/test_all_fields.py index e066a1c5..214a6e20 100644 --- a/tests/test_all_fields.py +++ b/tests/test_all_fields.py @@ -48,8 +48,31 @@ def test_run(self): - Verify no unexpected streams were replicated - Verify that more than just the automatic fields are replicated for each stream """ - - expected_streams = self.expected_streams() + # excluding streams due to lack of data and permissions + streams_to_exclude = { + 'article_authors', + 'article_tags', + 'blogs', + 'carrier_services', + 'comments', + 'draft_orders', + 'fulfillment_services', + 'marketing_events', + 'pages', + 'policies', + 'price_rules', + 'redirects', + 'script_tags', + 'shipping_zones', + 'themes', + 'application_charges', + 'custom_collections', + 'currencies', + 'application_credits', + 'webhooks', + 'recurring_application_charges' + } + expected_streams = self.expected_streams() - streams_to_exclude # instantiate connection conn_id = self.create_connection() diff --git a/tests/test_bookmarks.py b/tests/test_bookmarks.py index a28c3dd7..4b34e436 100644 --- a/tests/test_bookmarks.py +++ b/tests/test_bookmarks.py @@ -18,6 +18,41 @@ def name(): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.start_date = '2021-04-01T00:00:00Z' + # excluding streams which dont have data or permission + self.streams_to_exclude = { + 'article_authors', + 'article_tags', + 'blogs', + 'carrier_services', + 'comments', + 'draft_orders', + 'fulfillment_services', + 'marketing_events', + 'pages', + 'policies', + 'price_rules', + 'redirects', + 'script_tags', + 'shipping_zones', + 'themes', + 'application_charges', + 'custom_collections', + 'currencies', + 'application_credits', + 'webhooks', + 'recurring_application_charges', + 'metafields_collections', + 'metafields_orders', + 'customers', + 'smart_collections', + 'fulfillment_orders', + 'shop', + 'events', + 'orders', + 'metafields_customers', + 'resource_feedback', + 'collections' + } def max_bookmarks_by_stream(self, sync_records): """ @@ -77,7 +112,7 @@ def bookmarks_test(self, conn_id, testable_streams): # Select all streams and no fields within streams found_catalogs = menagerie.get_catalogs(conn_id) incremental_streams = {key for key, value in self.expected_replication_method().items() - if value == self.INCREMENTAL and key in testable_streams} + if value == self.INCREMENTAL and key in testable_streams} - self.streams_to_exclude incremental_streams = incremental_streams # Our test data sets for Shopify do not have any abandoned_checkouts @@ -99,7 +134,7 @@ def bookmarks_test(self, conn_id, testable_streams): first_max_bookmarks = self.max_bookmarks_by_stream(first_sync_records) first_min_bookmarks = self.min_bookmarks_by_stream(first_sync_records) #first_sync_bookmarks = menagerie.get_state(conn_id) - + ####################### # Update State between Syncs ####################### diff --git a/tests/test_discovery.py b/tests/test_discovery.py index 89aa4a49..9761b10f 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -33,7 +33,31 @@ def test_run(self): • verify that all other fields have inclusion of available (metadata and schema) """ conn_id = self.create_connection() - expected_streams = self.expected_streams() | {"abandoned_checkouts"} + streams_to_include = { + 'article_authors', + 'article_tags', + 'blogs', + 'carrier_services', + 'comments', + 'draft_orders', + 'fulfillment_services', + 'marketing_events', + 'pages', + 'policies', + 'price_rules', + 'redirects', + 'script_tags', + 'shipping_zones', + 'themes', + 'application_charges', + 'custom_collections', + 'currencies', + 'application_credits', + 'webhooks', + 'recurring_application_charges', + 'abandoned_checkouts' + } + expected_streams = self.expected_streams() | streams_to_include # Verify number of actual streams discovered match expected found_catalogs = menagerie.get_catalogs(conn_id) diff --git a/tests/test_full_replication.py b/tests/test_full_replication.py index f0924da9..c5c50f31 100644 --- a/tests/test_full_replication.py +++ b/tests/test_full_replication.py @@ -27,11 +27,24 @@ def test_run(self): different values for the replication key """ conn_id = self.create_connection() + # excluding streams due to lack of permssion and data + streams_to_exclude = { + "article_authors", + "article_tags", + "carrier_services", + "fulfillment_services", + "price_rules", + "recurring_application_charges", + "shipping_zones", + "application_credits", + "application_charges", + "currencies" + } # Select all streams and no fields within streams found_catalogs = menagerie.get_catalogs(conn_id) full_streams = {key for key, value in self.expected_replication_method().items() - if value == self.FULL} + if value == self.FULL} - streams_to_exclude our_catalogs = [catalog for catalog in found_catalogs if catalog.get('tap_stream_id') in full_streams] self.select_all_streams_and_fields(conn_id, our_catalogs, select_all_fields=True) diff --git a/tests/test_interrupted_sync.py b/tests/test_interrupted_sync.py index aa1ec44f..7c3c1b3b 100644 --- a/tests/test_interrupted_sync.py +++ b/tests/test_interrupted_sync.py @@ -40,7 +40,8 @@ def test_run(self): 'collections', 'orders', 'products', - 'transactions'} + 'transactions', + 'events'} # Select all streams and no fields within streams found_catalogs = menagerie.get_catalogs(conn_id) diff --git a/tests/test_pagination.py b/tests/test_pagination.py index f4b8ab2e..2dea8a36 100644 --- a/tests/test_pagination.py +++ b/tests/test_pagination.py @@ -11,7 +11,7 @@ class PaginationTest(BaseTapTest): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.start_date = '2021-04-01T00:00:00Z' + self.start_date = '2021-04-01T00:00:00Z' @staticmethod def name(): @@ -19,7 +19,7 @@ def name(): def get_properties(self, *args, **kwargs): props = super().get_properties(*args, **kwargs) - props['results_per_page'] = '30' + props['results_per_page'] = '3' return props def test_run(self): diff --git a/tests/test_shop_info_fields.py b/tests/test_shop_info_fields.py index 1aeb7102..a4639ece 100644 --- a/tests/test_shop_info_fields.py +++ b/tests/test_shop_info_fields.py @@ -44,7 +44,7 @@ def test_run(self): for stream in self.expected_streams(): with self.subTest(stream=stream): - + # Verify that every stream schema contains shop info fields catalog = next(iter([catalog for catalog in found_catalogs if catalog["stream_name"] == stream])) diff --git a/tests/test_start_date.py b/tests/test_start_date.py index d2284913..d02cae4e 100644 --- a/tests/test_start_date.py +++ b/tests/test_start_date.py @@ -53,6 +53,20 @@ def name(): def test_run(self): """Test we get a lot of data back based on the start date configured in base""" conn_id = self.create_connection() + streams_to_exclude = { + "abandoned_checkouts", + "orders", + "blogs", + "comments", + "marketing_events", + "pages", + "policies", + "redirects", + "script_tags", + "themes", + "draft_orders", + "webhooks" + } # Select all streams and all fields within streams found_catalogs = menagerie.get_catalogs(conn_id) @@ -62,10 +76,8 @@ def test_run(self): # Every Monday, abandoned checkouts that are older than three months are removed from your admin. # Also no POST call is available for this endpoint: https://shopify.dev/api/admin-rest/2022-01/resources/abandoned-checkouts expected_replication_method = self.expected_replication_method() - expected_replication_method.pop("abandoned_checkouts") - expected_replication_method.pop("orders") incremental_streams = {key for key, value in expected_replication_method.items() - if value == self.INCREMENTAL} + if value == self.INCREMENTAL} - streams_to_exclude # IF THERE ARE STREAMS THAT SHOULD NOT BE TESTED # REPLACE THE EMPTY SET BELOW WITH THOSE STREAMS diff --git a/tests/unittests/test_base.py b/tests/unittests/test_base.py index 203a762f..598880eb 100644 --- a/tests/unittests/test_base.py +++ b/tests/unittests/test_base.py @@ -102,7 +102,7 @@ def test_call_api_empty_response(self, mock_get_query, mock_graphql): @patch('shopify.GraphQL') @patch.object(Stream, 'get_query', return_value=mock_query()) - @patch.object(Stream, 'transform_object', side_effect=lambda x: x) + @patch.object(Stream, 'transform_object', side_effect=lambda x, **_kwargs: x) @patch('tap_shopify.streams.base.utils.now', return_value=datetime(2025, 2, 1, 0, 0, tzinfo=tzlocal())) def test_get_objects(self, mock_now, mock_transform_object, mock_get_query, mock_graphql): """Test get_objects with pagination and bookmarking.""" diff --git a/tests/unittests/test_date_window.py b/tests/unittests/test_date_window.py index 402dc8d7..7ebe907d 100644 --- a/tests/unittests/test_date_window.py +++ b/tests/unittests/test_date_window.py @@ -3,7 +3,7 @@ from tap_shopify.streams.base import Stream class TestShopifyDateWindowHandling(unittest.TestCase): - + def test_no_date_window_value(self): """Test that no value for date_window is handled correctly.""" Context.config = { diff --git a/tests/unittests/test_token_check_in_discover.py b/tests/unittests/test_token_check_in_discover.py index 0fe2d981..d2865dd7 100644 --- a/tests/unittests/test_token_check_in_discover.py +++ b/tests/unittests/test_token_check_in_discover.py @@ -25,15 +25,25 @@ def connection_error_raiser(): @mock.patch('tap_shopify.discover', side_effect=tap_shopify.discover) @mock.patch("builtins.print") @mock.patch("tap_shopify.has_read_users_access") +@mock.patch("tap_shopify.has_access_scope") class TestTokenInDiscoverMode(unittest.TestCase): @mock.patch('tap_shopify.initialize_shopify_client', side_effect=resource_not_found_raiser) - def test_resource_not_found(self, mocked_client, mocked_access, mocked_print, mocked_discover, mocked_args): + def test_resource_not_found( + self, + mocked_client, + mocked_access_scope, + mocked_access, + mocked_print, + mocked_discover, + mocked_args + ): ''' Verify exception is raised for ResourceNotFound with proper error message and - test that discover mode is called + test that discover mode is called ''' mocked_access.return_value = True + mocked_access_scope.return_value = True mocked_args.return_value = Args() try: tap_shopify.main() @@ -44,12 +54,21 @@ def test_resource_not_found(self, mocked_client, mocked_access, mocked_print, mo self.assertEqual(mocked_print.call_count, 0) @mock.patch('tap_shopify.initialize_shopify_client', side_effect=unauthorized_access_raiser) - def test_unauthorized_access(self, mocked_client, mocked_access, mocked_print, mocked_discover, mocked_args): + def test_unauthorized_access( + self, + mocked_client, + mocked_access_scope, + mocked_access, + mocked_print, + mocked_discover, + mocked_args + ): ''' Verify exception is raised for UnauthorizedAccess with proper error message and - test that discover mode is called + test that discover mode is called ''' mocked_access.return_value = True + mocked_access_scope.return_value = True mocked_args.return_value = Args() try: tap_shopify.main() @@ -60,12 +79,21 @@ def test_unauthorized_access(self, mocked_client, mocked_access, mocked_print, m self.assertEqual(mocked_print.call_count, 0) @mock.patch('tap_shopify.initialize_shopify_client', side_effect=connection_error_raiser) - def test_connection_error(self, mocked_client, mocked_access, mocked_print, mocked_discover, mocked_args): + def test_connection_error( + self, + mocked_client, + mocked_access_scope, + mocked_access, + mocked_print, + mocked_discover, + mocked_args + ): ''' Verify exception is raised for ConnectionError with proper error message and - test that discover mode is called + test that discover mode is called ''' mocked_access.return_value = True + mocked_access_scope.return_value = True mocked_args.return_value = Args() try: tap_shopify.main() @@ -76,12 +104,21 @@ def test_connection_error(self, mocked_client, mocked_access, mocked_print, mock self.assertEqual(mocked_print.call_count, 0) @mock.patch('tap_shopify.initialize_shopify_client') - def test_no_error(self, mocked_client, mocked_access, mocked_print, mocked_discover, mocked_args): + def test_no_error( + self, + mocked_client, + mocked_access_scope, + mocked_access, + mocked_print, + mocked_discover, + mocked_args + ): ''' Verify that if no error during discover then print should be called once for writing catalog ''' mocked_access.return_value = True + mocked_access_scope.return_value = True mocked_args.return_value = Args() tap_shopify.main() self.assertEqual(mocked_discover.call_count, 1) From af1d05ed2fa74ea1321ac058074b26eccdc16d21 Mon Sep 17 00:00:00 2001 From: Atul Tiwari Date: Thu, 30 Oct 2025 09:58:55 +0530 Subject: [PATCH 57/66] Revert "GL-Work (#242)" (#243) This reverts commit 313e366e0e835a3406888ae31270b687bab6c33c. --- CHANGELOG.md | 3 - README.md | 105 +- setup.py | 2 +- tap_shopify/__init__.py | 39 +- tap_shopify/schemas/application_charges.json | 56 - tap_shopify/schemas/application_credits.json | 50 - tap_shopify/schemas/article_authors.json | 11 - tap_shopify/schemas/article_tags.json | 11 - tap_shopify/schemas/blogs.json | 165 --- tap_shopify/schemas/carrier_services.json | 41 - tap_shopify/schemas/comments.json | 146 --- tap_shopify/schemas/currencies.json | 36 - tap_shopify/schemas/custom_collections.json | 92 -- tap_shopify/schemas/draft_orders.json | 1021 ----------------- tap_shopify/schemas/fulfillment_services.json | 59 - tap_shopify/schemas/marketing_events.json | 110 -- tap_shopify/schemas/pages.json | 68 -- tap_shopify/schemas/policies.json | 43 - tap_shopify/schemas/price_rules.json | 103 -- .../recurring_application_charges.json | 186 --- tap_shopify/schemas/redirects.json | 30 - tap_shopify/schemas/resource_feedback.json | 88 -- tap_shopify/schemas/script_tags.json | 49 - tap_shopify/schemas/shipping_zones.json | 243 ---- tap_shopify/schemas/shop.json | 415 ------- tap_shopify/schemas/smart_collections.json | 117 -- tap_shopify/schemas/tender_transactions.json | 90 -- tap_shopify/schemas/themes.json | 61 - tap_shopify/schemas/webhooks.json | 143 --- tap_shopify/streams/__init__.py | 25 - tap_shopify/streams/abandoned_checkouts.py | 5 +- tap_shopify/streams/application_charges.py | 48 - tap_shopify/streams/application_credits.py | 47 - tap_shopify/streams/article_authors.py | 29 - tap_shopify/streams/article_tags.py | 60 - tap_shopify/streams/base.py | 60 +- tap_shopify/streams/blogs.py | 54 - tap_shopify/streams/carrier_services.py | 33 - tap_shopify/streams/collections.py | 5 +- tap_shopify/streams/comments.py | 51 - tap_shopify/streams/currencies.py | 45 - tap_shopify/streams/custom_collections.py | 64 -- tap_shopify/streams/customers.py | 1 - tap_shopify/streams/draft_orders.py | 218 ---- tap_shopify/streams/fulfillment_orders.py | 9 +- tap_shopify/streams/fulfillment_services.py | 65 -- tap_shopify/streams/inventory_items.py | 5 +- tap_shopify/streams/inventory_levels.py | 1 - tap_shopify/streams/locations.py | 1 - tap_shopify/streams/marketing_events.py | 43 - tap_shopify/streams/metafields.py | 2 +- tap_shopify/streams/metafields_collections.py | 1 - tap_shopify/streams/metafields_customers.py | 1 - tap_shopify/streams/metafields_orders.py | 1 - tap_shopify/streams/metafields_products.py | 1 - tap_shopify/streams/order_refunds.py | 5 +- tap_shopify/streams/order_shipping_lines.py | 1 - tap_shopify/streams/orders.py | 3 +- tap_shopify/streams/pages.py | 37 - tap_shopify/streams/policies.py | 88 -- tap_shopify/streams/price_rules.py | 148 --- tap_shopify/streams/product_variants.py | 5 +- tap_shopify/streams/products.py | 5 +- .../streams/recurring_application_charges.py | 97 -- tap_shopify/streams/redirects.py | 52 - tap_shopify/streams/resource_feedback.py | 89 -- tap_shopify/streams/script_tags.py | 35 - tap_shopify/streams/shipping_zones.py | 82 -- tap_shopify/streams/shop.py | 116 -- tap_shopify/streams/smart_collections.py | 70 -- tap_shopify/streams/tender_transactions.py | 46 - tap_shopify/streams/themes.py | 92 -- tap_shopify/streams/transactions.py | 1 - tap_shopify/streams/webhooks.py | 56 - tests/base.py | 136 +-- tests/test_all_fields.py | 27 +- tests/test_bookmarks.py | 39 +- tests/test_discovery.py | 26 +- tests/test_full_replication.py | 15 +- tests/test_interrupted_sync.py | 3 +- tests/test_pagination.py | 4 +- tests/test_shop_info_fields.py | 2 +- tests/test_start_date.py | 18 +- tests/unittests/test_base.py | 2 +- tests/unittests/test_date_window.py | 2 +- .../unittests/test_token_check_in_discover.py | 51 +- 86 files changed, 79 insertions(+), 5732 deletions(-) delete mode 100644 tap_shopify/schemas/application_charges.json delete mode 100644 tap_shopify/schemas/application_credits.json delete mode 100644 tap_shopify/schemas/article_authors.json delete mode 100644 tap_shopify/schemas/article_tags.json delete mode 100644 tap_shopify/schemas/blogs.json delete mode 100644 tap_shopify/schemas/carrier_services.json delete mode 100644 tap_shopify/schemas/comments.json delete mode 100644 tap_shopify/schemas/currencies.json delete mode 100644 tap_shopify/schemas/custom_collections.json delete mode 100644 tap_shopify/schemas/draft_orders.json delete mode 100644 tap_shopify/schemas/fulfillment_services.json delete mode 100644 tap_shopify/schemas/marketing_events.json delete mode 100644 tap_shopify/schemas/pages.json delete mode 100644 tap_shopify/schemas/policies.json delete mode 100644 tap_shopify/schemas/price_rules.json delete mode 100644 tap_shopify/schemas/recurring_application_charges.json delete mode 100644 tap_shopify/schemas/redirects.json delete mode 100644 tap_shopify/schemas/resource_feedback.json delete mode 100644 tap_shopify/schemas/script_tags.json delete mode 100644 tap_shopify/schemas/shipping_zones.json delete mode 100644 tap_shopify/schemas/shop.json delete mode 100644 tap_shopify/schemas/smart_collections.json delete mode 100644 tap_shopify/schemas/tender_transactions.json delete mode 100644 tap_shopify/schemas/themes.json delete mode 100644 tap_shopify/schemas/webhooks.json delete mode 100644 tap_shopify/streams/application_charges.py delete mode 100644 tap_shopify/streams/application_credits.py delete mode 100644 tap_shopify/streams/article_authors.py delete mode 100644 tap_shopify/streams/article_tags.py delete mode 100644 tap_shopify/streams/blogs.py delete mode 100644 tap_shopify/streams/carrier_services.py delete mode 100644 tap_shopify/streams/comments.py delete mode 100644 tap_shopify/streams/currencies.py delete mode 100644 tap_shopify/streams/custom_collections.py delete mode 100644 tap_shopify/streams/draft_orders.py delete mode 100644 tap_shopify/streams/fulfillment_services.py delete mode 100644 tap_shopify/streams/marketing_events.py delete mode 100644 tap_shopify/streams/pages.py delete mode 100644 tap_shopify/streams/policies.py delete mode 100644 tap_shopify/streams/price_rules.py delete mode 100644 tap_shopify/streams/recurring_application_charges.py delete mode 100644 tap_shopify/streams/redirects.py delete mode 100644 tap_shopify/streams/resource_feedback.py delete mode 100644 tap_shopify/streams/script_tags.py delete mode 100644 tap_shopify/streams/shipping_zones.py delete mode 100644 tap_shopify/streams/shop.py delete mode 100644 tap_shopify/streams/smart_collections.py delete mode 100644 tap_shopify/streams/tender_transactions.py delete mode 100644 tap_shopify/streams/themes.py delete mode 100644 tap_shopify/streams/webhooks.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 937ea295..a0a117ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,5 @@ # Changelog -### 3.8.0 - * Introduced new streams [#242](https://github.com/singer-io/tap-shopify/pull/242) - ### 3.7.2 * New automatic_keys Support and Bulk State Persistence Logic [#237](https://github.com/singer-io/tap-shopify/pull/237) diff --git a/README.md b/README.md index 122f666e..f44dc09a 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # tap-shopify This is a [Singer](https://singer.io) tap that produces JSON-formatted data -following the [Singer spec](https://github.com/singer-io/getting-started/blob/master/SPEC.md). +following the [Singer +spec](https://github.com/singer-io/getting-started/blob/master/SPEC.md). This tap: @@ -23,96 +24,34 @@ This tap: - [Locations](https://shopify.dev/docs/api/admin-graphql/latest/queries/locations) - [Inventory Levels](https://shopify.dev/docs/api/admin-graphql/latest/queries/inventorylevel) - [Inventory Item](https://shopify.dev/docs/api/admin-graphql/latest/queries/inventoryitems) - - [Application Charges](https://shopify.dev/docs/api/admin-graphql/latest/queries/currentAppInstallation?example=retrieves-a-list-of-application-charges) - - [Application Credits](https://shopify.dev/docs/api/admin-graphql/latest/queries/appInstallation?example=retrieves-all-application-credits) - - [Article Authors](https://shopify.dev/docs/api/admin-rest/latest/resources/article#get-articles-authors) - - [Article Tags](https://shopify.dev/docs/api/admin-rest/latest/resources/article#get-articles-tags) - - [Blogs](https://shopify.dev/docs/api/admin-graphql/latest/queries/blogs?example=retrieve-a-list-of-all-blogs) - - [Carrier Services](https://shopify.dev/docs/api/admin-rest/latest/resources/carrierservice#get-carrier-services) - - [Comments](https://shopify.dev/docs/api/admin-graphql/latest/queries/comments?example=retrieves-a-list-of-comments) - - [Currencies](https://shopify.dev/docs/api/admin-graphql/latest/queries/shop?example=retrieves-a-list-of-currencies-enabled-on-a-shop) - - [Custom Collections](https://shopify.dev/docs/api/admin-graphql/latest/queries/collections?example=retrieve-a-list-of-custom-collections) - - [Draft Orders](https://shopify.dev/docs/api/admin-graphql/latest/queries/draftOrders?example=retrieves-a-list-of-draft-orders) - - [Fulfillment Services](https://shopify.dev/docs/api/admin-graphql/latest/queries/shop?example=receive-a-list-of-all-fulfillmentservices) - - [Marketing Events](https://shopify.dev/docs/api/admin-graphql/latest/queries/marketingEvents?example=retrieves-a-list-of-all-marketing-events) - - [Pages](https://shopify.dev/docs/api/admin-graphql/latest/queries/pages?example=retrieves-a-list-of-pages) - - [Policies](https://shopify.dev/docs/api/admin-graphql/latest/queries/shop?example=retrieves-a-list-of-the-shops-policies) - - [Price Rules](https://shopify.dev/docs/api/admin-graphql/latest/queries/discountNodes) - - [Recurring Application Charges](https://shopify.dev/docs/api/admin-graphql/latest/queries/currentAppInstallationexample=retrieves-a-list-of-recurring-application-charges) - - [Redirects](https://shopify.dev/docs/api/admin-graphql/latest/queries/urlRedirects?example=retrieves-a-list-of-url-redirects) - - [Resource Feedback](https://shopify.dev/docs/api/admin-graphql/latest/queries/app?example=get-the-feedback-field) - - [Script Tags](https://shopify.dev/docs/api/admin-graphql/latest/queries/scriptTags?example=retrieves-a-list-of-all-script-tags) - - [Shipping Zones](https://shopify.dev/docs/api/admin-graphql/latest/queries/deliveryProfiles?example=receive-a-list-of-all-shippingzones) - - [Shop](https://shopify.dev/docs/api/admin-rest/latest/resources/shop#get-shop) - - [Smart Collections](https://shopify.dev/docs/api/admin-graphql/latest/queries/collections?example=retrieve-a-list-of-smart-collections) - - [Tender Transactions](https://shopify.dev/docs/api/admin-graphql/latest/queries/tenderTransactions?example=retrieves-a-list-of-tender-transactions) - - [Themes](https://shopify.dev/docs/api/admin-graphql/latest/queries/themes?example=retrieves-a-list-of-themes) - - [Webhooks](https://shopify.dev/docs/api/admin-graphql/latest/queries/webhookSubscriptions) - Outputs the schema for each resource - Incrementally pulls data based on the input state ## Stream Details -| Stream Name | Replication Key | Key Properties| -|------------------------|--------------------------|---------------| -| abandoned_checkouts | updatedAt | id | -| collections | updatedAt | id | -| customers | updatedAt | id | -| events | createdAt | id | -| inventory_items | updatedAt | id | -| inventory_levels | updatedAt | id | -| locations | createdAt | id | -| metafields_collections | updatedAt | id | -| metafields_customers | updatedAt | id | -| metafields_orders | updatedAt | id | -| metafields_products | updatedAt | id | -| order_refunds | updatedAt | id | -| orders | updatedAt | id | -| order_shipping_lines | updatedAt | id | -| fulfillment_orders | updatedAt | id | -| product_variants | updatedAt | id | -| products | updatedAt | id | -| transactions | createdAt | id | -| blogs | updatedAt | id | -| comments | updatedAt | id | -| custom_collections | updatedAt | id | -| draft_orders | updatedAt | id | -| marketing_events | started_at | id | -| pages | updatedAt | id | -| policies | updatedAt | id | -| redirects | createdAt | id | -| resource_feedback | feedback_generated_at | app_id | -| script_tags | updatedAt | id | -| shop | updatedAt | id | -| smart_collections | updatedAt | id | -| tender_transactions | processedAt | id | -| themes | updatedAt | id | -| webhooks | updatedAt | id | +| Stream Name | Replication Key | Key Properties | +|------------------------|----------------|---------------| +| abandoned_checkouts | updatedAt | id | +| collections | updatedAt | id | +| customers | updatedAt | id | +| events | createdAt | id | +| inventory_items | updatedAt | id | +| inventory_levels | updatedAt | id | +| locations | createdAt | id | +| metafields_collections | updatedAt | id | +| metafields_customers | updatedAt | id | +| metafields_orders | updatedAt | id | +| metafields_products | updatedAt | id | +| order_refunds | updatedAt | id | +| orders | updatedAt | id | +| order_shipping_lines | updatedAt | id | +| fulfillment_orders | updatedAt | id | +| product_variants | updatedAt | id | +| products | updatedAt | id | +| transactions | createdAt | id | Currently, `locations` graphql endpoint doesn't support querying on the `updatedAt`, therefore, `createdAt` is made the replication key. -## Required Account Access Scopes - -1. CarrierServices, ShippingZones - shipping -2. Comments, Pages, Redirect, Article, Blog - content -3. Collections, CustomCollection , SmartCollection, ProductVariants, Products, ResourceFeedback - products -5. DraftOrders - draft_orders -6. TenderTransaction, Transactions, AbandonedCheckouts, OrderRefund, OrderShippingLines - orders -7. FulfillmentService - fulfillments -8. Marketing Events - marketing_events -9. ResourceFeedback - resource_feedbacks -10. ScriptTags - script_tags -11. Themes - themes -12. PriceRules - price_rules -13. Customers - customers -14. FulfillmentOrders - fulfillments_orders -15. InventoryItems, InventoryLevels - inventory -16. Locations - location -17. MarketingEvents - marketing_events -18. Policies - legal_policies -19. PriceRules - discount -20. Redirects - online_store_navigation - ## Quick Start 1. Install diff --git a/setup.py b/setup.py index 808eebae..19fa631f 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="3.8.0", + version="3.7.2", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index be1855ec..0544dad6 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -59,19 +59,6 @@ def has_read_users_access(): return False return True -def has_access_scope(stream, scopes): - """If the app does not have the required scope, return False""" - app_access_scopes = fetch_app_scopes() - - if not any(access_scope in scopes for access_scope in app_access_scopes): - message = "The account credentials supplied do not have '{0}' access scope " \ - "to the following stream: {1}. The data for these streams would not be " \ - "collected due to lack of required permission.".format(", ".join(scopes), stream) - LOGGER.warning(message) - return False - - return True - def get_abs_path(path): return os.path.join(os.path.dirname(os.path.realpath(__file__)), path) @@ -95,7 +82,7 @@ def get_discovery_metadata(stream, schema): mdata = metadata.write(mdata, (), 'table-key-properties', stream.key_properties) mdata = metadata.write(mdata, (), 'forced-replication-method', stream.replication_method) - if stream.replication_method=="INCREMENTAL" and stream.replication_key: + if stream.replication_key: mdata = metadata.write(mdata, (), 'valid-replication-keys', [stream.replication_key]) for field_name in schema['properties'].keys(): @@ -119,15 +106,12 @@ def discover(): raw_schemas = load_schemas() streams = [] - error_list = [] for schema_name, schema in raw_schemas.items(): if schema_name not in Context.stream_objects: continue stream = Context.stream_objects[schema_name]() - if stream.access_scope and not has_access_scope(stream.name, stream.access_scope): - error_list.append({stream.name: stream.access_scope}) catalog_schema = add_synthetic_key_to_schema(schema) # create and add catalog entry @@ -137,25 +121,11 @@ def discover(): 'schema': catalog_schema, 'metadata': get_discovery_metadata(stream, schema), 'key_properties': stream.key_properties, + 'replication_key': stream.replication_key, + 'replication_method': stream.replication_method } streams.append(catalog_entry) - if error_list: - total_stream = len(raw_schemas.keys()) - missing_scopes_msg = [ - f"{stream}: {', '.join(scopes)}" - for item in error_list - for stream, scopes in item.items() - ] - message = "The account credentials supplied do not have access to the following " \ - "stream(s): {}. The data for these streams would not be collected due to lack of " \ - "required permission.".format(", ".join(missing_scopes_msg)) - - if len(error_list) != total_stream: - LOGGER.warning(message) - else: - raise ShopifyAPIError(message) - return {'streams': streams} def shuffle_streams(stream_name): @@ -188,7 +158,8 @@ def sync(): if Context.is_selected(stream["tap_stream_id"]): singer.write_schema(stream["tap_stream_id"], stream["schema"], - stream["key_properties"]) + stream["key_properties"], + bookmark_properties=stream["replication_key"]) Context.counts[stream["tap_stream_id"]] = 0 # Loop over streams in catalog diff --git a/tap_shopify/schemas/application_charges.json b/tap_shopify/schemas/application_charges.json deleted file mode 100644 index 1f735f0c..00000000 --- a/tap_shopify/schemas/application_charges.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "type": "object", - "properties": { - "createdAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "id": { - "type": [ - "string", - "null" - ] - }, - "name": { - "type": [ - "string", - "null" - ] - }, - "price": { - "type": [ - "object", - "null" - ], - "properties": { - "amount": { - "type": [ - "number", - "null" - ] - }, - "currencyCode": { - "type": [ - "string", - "null" - ] - } - } - }, - "status": { - "type": [ - "string", - "null" - ] - }, - "test": { - "type": [ - "boolean", - "null" - ] - } - } -} diff --git a/tap_shopify/schemas/application_credits.json b/tap_shopify/schemas/application_credits.json deleted file mode 100644 index ab45aaf7..00000000 --- a/tap_shopify/schemas/application_credits.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "createdAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "description": { - "type": [ - "string", - "null" - ] - }, - "test": { - "type": [ - "boolean", - "null" - ] - }, - "amount": { - "type": [ - "object", - "null" - ], - "properties": { - "amount": { - "type": [ - "number", - "null" - ] - }, - "currencyCode": { - "type": [ - "string", - "null" - ] - } - } - } - } -} diff --git a/tap_shopify/schemas/article_authors.json b/tap_shopify/schemas/article_authors.json deleted file mode 100644 index f11bd16a..00000000 --- a/tap_shopify/schemas/article_authors.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "type": "object", - "properties": { - "name": { - "type": [ - "string", - "null" - ] - } - } -} diff --git a/tap_shopify/schemas/article_tags.json b/tap_shopify/schemas/article_tags.json deleted file mode 100644 index 4327dd00..00000000 --- a/tap_shopify/schemas/article_tags.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "type": "object", - "properties": { - "article_tag": { - "type": [ - "string", - "null" - ] - } - } -} diff --git a/tap_shopify/schemas/blogs.json b/tap_shopify/schemas/blogs.json deleted file mode 100644 index 8e9067bf..00000000 --- a/tap_shopify/schemas/blogs.json +++ /dev/null @@ -1,165 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "handle": { - "type": [ - "string", - "null" - ] - }, - "commentPolicy": { - "type": [ - "string", - "null" - ] - }, - "createdAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "updatedAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "tags": { - "type": [ - "array", - "null" - ], - "items": { - "type": [ - "string", - "null" - ] - } - }, - "title": { - "type": [ - "string", - "null" - ] - }, - "templateSuffix": { - "type": [ - "string", - "null" - ] - }, - "articles": { - "type": [ - "object", - "null" - ], - "properties": { - "edges": { - "type": [ - "array", - "null" - ], - "items": { - "type": "object", - "properties": { - "node": { - "type": [ - "object", - "null" - ], - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "title": { - "type": [ - "string", - "null" - ] - }, - "body": { - "type": [ - "string", - "null" - ] - }, - "createdAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "updatedAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "publishedAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "isPublished": { - "type": [ - "boolean", - "null" - ] - }, - "tags": { - "type": [ - "array", - "null" - ], - "items": { - "type": [ - "string", - "null" - ] - } - } - } - } - } - } - } - } - }, - "feed": { - "type": [ - "object", - "null" - ], - "properties": { - "location": { - "type": [ - "string", - "null" - ] - }, - "path": { - "type": [ - "string", - "null" - ] - } - } - } - } -} diff --git a/tap_shopify/schemas/carrier_services.json b/tap_shopify/schemas/carrier_services.json deleted file mode 100644 index 6bcf5434..00000000 --- a/tap_shopify/schemas/carrier_services.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "name": { - "type": [ - "string", - "null" - ] - }, - "active": { - "type": [ - "boolean", - "null" - ] - }, - "callbackUrl": { - "type": [ - "string", - "null" - ] - }, - "formattedName": { - "type": [ - "string", - "null" - ] - }, - "supportsServiceDiscovery": { - "type": [ - "boolean", - "null" - ] - } - } -} diff --git a/tap_shopify/schemas/comments.json b/tap_shopify/schemas/comments.json deleted file mode 100644 index 01ffd525..00000000 --- a/tap_shopify/schemas/comments.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "body": { - "type": [ - "string", - "null" - ] - }, - "bodyHtml": { - "type": [ - "string", - "null" - ] - }, - "author": { - "type": [ - "object", - "null" - ], - "properties": { - "email": { - "type": [ - "string", - "null" - ] - }, - "name": { - "type": [ - "string", - "null" - ] - } - } - }, - "createdAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "updatedAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "userAgent": { - "type": [ - "string", - "null" - ] - }, - "status": { - "type": [ - "string", - "null" - ] - }, - "publishedAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "isPublished": { - "type": [ - "boolean", - "null" - ] - }, - "ip": { - "type": [ - "string", - "null" - ] - }, - "article": { - "type": [ - "object", - "null" - ], - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "title": { - "type": [ - "string", - "null" - ] - }, - "tags": { - "type": [ - "array", - "null" - ], - "items": { - "type": [ - "string", - "null" - ] - } - }, - "publishedAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "isPublished": { - "type": [ - "boolean", - "null" - ] - }, - "createdAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "body": { - "type": [ - "string", - "null" - ] - } - } - } - } -} diff --git a/tap_shopify/schemas/currencies.json b/tap_shopify/schemas/currencies.json deleted file mode 100644 index 8d81499d..00000000 --- a/tap_shopify/schemas/currencies.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "type": "object", - "properties": { - "currencyCode": { - "type": [ - "string", - "null" - ] - }, - "currencyName": { - "type": [ - "string", - "null" - ] - }, - "enabled": { - "type": [ - "boolean", - "null" - ] - }, - "manualRate": { - "type": [ - "number", - "null" - ] - }, - "rateUpdatedAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - } - } -} diff --git a/tap_shopify/schemas/custom_collections.json b/tap_shopify/schemas/custom_collections.json deleted file mode 100644 index e842c5f9..00000000 --- a/tap_shopify/schemas/custom_collections.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "title": { - "type": [ - "string", - "null" - ] - }, - "description": { - "type": [ - "string", - "null" - ] - }, - "descriptionHtml": { - "type": [ - "string", - "null" - ] - }, - "updatedAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "handle": { - "type": [ - "string", - "null" - ] - }, - "publishedOnCurrentPublication": { - "type": [ - "boolean", - "null" - ] - }, - "sortOrder": { - "type": [ - "string", - "null" - ] - }, - "templateSuffix": { - "type": [ - "string", - "null" - ] - }, - "image": { - "type": [ - "object", - "null" - ], - "properties": { - "url": { - "type": [ - "string", - "null" - ] - }, - "altText": { - "type": [ - "string", - "null" - ] - }, - "height": { - "type": [ - "integer", - "null" - ] - }, - "width": { - "type": [ - "integer", - "null" - ] - } - } - } - } -} diff --git a/tap_shopify/schemas/draft_orders.json b/tap_shopify/schemas/draft_orders.json deleted file mode 100644 index 75bc91b6..00000000 --- a/tap_shopify/schemas/draft_orders.json +++ /dev/null @@ -1,1021 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "note2": { - "type": [ - "string", - "null" - ] - }, - "email": { - "type": [ - "string", - "null" - ] - }, - "taxesIncluded": { - "type": [ - "boolean", - "null" - ] - }, - "currencyCode": { - "type": [ - "string", - "null" - ] - }, - "invoiceSentAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "createdAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "updatedAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "taxExempt": { - "type": [ - "boolean", - "null" - ] - }, - "completedAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "name": { - "type": [ - "string", - "null" - ] - }, - "status": { - "type": [ - "string", - "null" - ] - }, - "invoiceUrl": { - "type": [ - "string", - "null" - ] - }, - "tags": { - "type": [ - "array", - "null" - ], - "items": { - "type": "string" - } - }, - "phone": { - "type": [ - "string", - "null" - ] - }, - "totalShippingPriceSet": { - "type": [ - "object", - "null" - ], - "properties": { - "presentmentMoney": { - "type": [ - "object", - "null" - ], - "properties": { - "amount": { - "type": [ - "string", - "null" - ] - }, - "currencyCode": { - "type": [ - "string", - "null" - ] - } - } - }, - "shopMoney": { - "type": [ - "object", - "null" - ], - "properties": { - "amount": { - "type": [ - "string", - "null" - ] - }, - "currencyCode": { - "type": [ - "string", - "null" - ] - } - } - } - } - }, - "totalPriceSet": { - "type": [ - "object", - "null" - ], - "properties": { - "presentmentMoney": { - "type": [ - "object", - "null" - ], - "properties": { - "amount": { - "type": [ - "string", - "null" - ] - }, - "currencyCode": { - "type": [ - "string", - "null" - ] - } - } - }, - "shopMoney": { - "type": [ - "object", - "null" - ], - "properties": { - "amount": { - "type": [ - "string", - "null" - ] - }, - "currencyCode": { - "type": [ - "string", - "null" - ] - } - } - } - } - }, - "subtotalPriceSet": { - "type": [ - "object", - "null" - ], - "properties": { - "presentmentMoney": { - "type": [ - "object", - "null" - ], - "properties": { - "amount": { - "type": [ - "string", - "null" - ] - }, - "currencyCode": { - "type": [ - "string", - "null" - ] - } - } - }, - "shopMoney": { - "type": [ - "object", - "null" - ], - "properties": { - "amount": { - "type": [ - "string", - "null" - ] - }, - "currencyCode": { - "type": [ - "string", - "null" - ] - } - } - } - } - }, - "totalTaxSet": { - "type": [ - "object", - "null" - ], - "properties": { - "presentmentMoney": { - "type": [ - "object", - "null" - ], - "properties": { - "amount": { - "type": [ - "string", - "null" - ] - }, - "currencyCode": { - "type": [ - "string", - "null" - ] - } - } - }, - "shopMoney": { - "type": [ - "object", - "null" - ], - "properties": { - "amount": { - "type": [ - "string", - "null" - ] - }, - "currencyCode": { - "type": [ - "string", - "null" - ] - } - } - } - } - }, - "lineItems": { - "type": [ - "object", - "null" - ], - "properties": { - "edges": { - "type": [ - "array", - "null" - ], - "items": { - "type": "object", - "properties": { - "node": { - "type": [ - "object", - "null" - ], - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "name": { - "type": [ - "string", - "null" - ] - }, - "quantity": { - "type": [ - "integer", - "null" - ] - }, - "sku": { - "type": [ - "string", - "null" - ] - }, - "title": { - "type": [ - "string", - "null" - ] - }, - "requiresShipping": { - "type": [ - "boolean", - "null" - ] - }, - "taxable": { - "type": [ - "boolean", - "null" - ] - }, - "custom": { - "type": [ - "boolean", - "null" - ] - }, - "isGiftCard": { - "type": [ - "boolean", - "null" - ] - }, - "vendor": { - "type": [ - "string", - "null" - ] - }, - "variant": { - "type": [ - "object", - "null" - ], - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "title": { - "type": [ - "string", - "null" - ] - } - } - }, - "weight": { - "type": [ - "object", - "null" - ], - "properties": { - "value": { - "type": [ - "number", - "null" - ] - }, - "unit": { - "type": [ - "string", - "null" - ] - } - } - }, - "originalUnitPriceWithCurrency": { - "type": [ - "object", - "null" - ], - "properties": { - "amount": { - "type": [ - "string", - "null" - ] - }, - "currencyCode": { - "type": [ - "string", - "null" - ] - } - } - }, - "product": { - "type": [ - "object", - "null" - ], - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "title": { - "type": [ - "string", - "null" - ] - } - } - }, - "taxLines": { - "type": [ - "array", - "null" - ], - "items": { - "type": "object", - "properties": { - "title": { - "type": [ - "string", - "null" - ] - }, - "source": { - "type": [ - "string", - "null" - ] - }, - "rate": { - "type": [ - "number", - "null" - ] - }, - "ratePercentage": { - "type": [ - "number", - "null" - ] - }, - "priceSet": { - "type": [ - "object", - "null" - ], - "properties": { - "presentmentMoney": { - "type": [ - "object", - "null" - ], - "properties": { - "amount": { - "type": [ - "string", - "null" - ] - }, - "currencyCode": { - "type": [ - "string", - "null" - ] - } - } - }, - "shopMoney": { - "type": [ - "object", - "null" - ], - "properties": { - "amount": { - "type": [ - "string", - "null" - ] - }, - "currencyCode": { - "type": [ - "string", - "null" - ] - } - } - } - } - } - } - } - } - } - } - } - } - } - } - }, - "shippingAddress": { - "type": [ - "object", - "null" - ], - "properties": { - "address1": { - "type": [ - "string", - "null" - ] - }, - "address2": { - "type": [ - "string", - "null" - ] - }, - "city": { - "type": [ - "string", - "null" - ] - }, - "company": { - "type": [ - "string", - "null" - ] - }, - "country": { - "type": [ - "string", - "null" - ] - }, - "countryCodeV2": { - "type": [ - "string", - "null" - ] - }, - "firstName": { - "type": [ - "string", - "null" - ] - }, - "lastName": { - "type": [ - "string", - "null" - ] - }, - "name": { - "type": [ - "string", - "null" - ] - }, - "latitude": { - "type": [ - "number", - "null" - ] - }, - "longitude": { - "type": [ - "number", - "null" - ] - }, - "phone": { - "type": [ - "string", - "null" - ] - }, - "province": { - "type": [ - "string", - "null" - ] - }, - "provinceCode": { - "type": [ - "string", - "null" - ] - }, - "zip": { - "type": [ - "string", - "null" - ] - } - } - }, - "shippingLine": { - "type": [ - "object", - "null" - ], - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "title": { - "type": [ - "string", - "null" - ] - }, - "carrierIdentifier": { - "type": [ - "string", - "null" - ] - }, - "custom": { - "type": [ - "boolean", - "null" - ] - }, - "code": { - "type": [ - "string", - "null" - ] - }, - "deliveryCategory": { - "type": [ - "string", - "null" - ] - }, - "source": { - "type": [ - "string", - "null" - ] - }, - "phone": { - "type": [ - "string", - "null" - ] - }, - "discountedPriceSet": { - "type": [ - "object", - "null" - ], - "properties": { - "presentmentMoney": { - "type": [ - "object", - "null" - ], - "properties": { - "amount": { - "type": [ - "string", - "null" - ] - }, - "currencyCode": { - "type": [ - "string", - "null" - ] - } - } - }, - "shopMoney": { - "type": [ - "object", - "null" - ], - "properties": { - "amount": { - "type": [ - "string", - "null" - ] - }, - "currencyCode": { - "type": [ - "string", - "null" - ] - } - } - } - } - } - } - }, - "taxLines": { - "type": [ - "array", - "null" - ], - "items": { - "type": "object", - "properties": { - "channelLiable": { - "type": [ - "boolean", - "null" - ] - }, - "rate": { - "type": [ - "number", - "null" - ] - }, - "ratePercentage": { - "type": [ - "number", - "null" - ] - }, - "source": { - "type": [ - "string", - "null" - ] - }, - "title": { - "type": [ - "string", - "null" - ] - }, - "priceSet": { - "type": [ - "object", - "null" - ], - "properties": { - "presentmentMoney": { - "type": [ - "object", - "null" - ], - "properties": { - "amount": { - "type": [ - "string", - "null" - ] - }, - "currencyCode": { - "type": [ - "string", - "null" - ] - } - } - }, - "shopMoney": { - "type": [ - "object", - "null" - ], - "properties": { - "amount": { - "type": [ - "string", - "null" - ] - }, - "currencyCode": { - "type": [ - "string", - "null" - ] - } - } - } - } - } - } - } - }, - "customer": { - "type": [ - "object", - "null" - ], - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "createdAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "lastName": { - "type": [ - "string", - "null" - ] - }, - "firstName": { - "type": [ - "string", - "null" - ] - }, - "taxExempt": { - "type": [ - "boolean", - "null" - ] - }, - "tags": { - "type": [ - "array", - "null" - ], - "items": { - "type": "string" - } - }, - "defaultAddress": { - "type": [ - "object", - "null" - ], - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "address1": { - "type": [ - "string", - "null" - ] - }, - "address2": { - "type": [ - "string", - "null" - ] - }, - "city": { - "type": [ - "string", - "null" - ] - }, - "company": { - "type": [ - "string", - "null" - ] - }, - "country": { - "type": [ - "string", - "null" - ] - }, - "countryCodeV2": { - "type": [ - "string", - "null" - ] - }, - "firstName": { - "type": [ - "string", - "null" - ] - }, - "lastName": { - "type": [ - "string", - "null" - ] - }, - "longitude": { - "type": [ - "number", - "null" - ] - }, - "name": { - "type": [ - "string", - "null" - ] - }, - "phone": { - "type": [ - "string", - "null" - ] - }, - "province": { - "type": [ - "string", - "null" - ] - }, - "provinceCode": { - "type": [ - "string", - "null" - ] - }, - "zip": { - "type": [ - "string", - "null" - ] - } - } - }, - "defaultEmailAddress": { - "type": [ - "object", - "null" - ], - "properties": { - "emailAddress": { - "type": [ - "string", - "null" - ] - } - } - }, - "lastOrder": { - "type": [ - "object", - "null" - ], - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "name": { - "type": [ - "string", - "null" - ] - }, - "currencyCode": { - "type": [ - "string", - "null" - ] - } - } - } - } - } - } -} diff --git a/tap_shopify/schemas/fulfillment_services.json b/tap_shopify/schemas/fulfillment_services.json deleted file mode 100644 index 69b61d7b..00000000 --- a/tap_shopify/schemas/fulfillment_services.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "handle": { - "type": [ - "string", - "null" - ] - }, - "callbackUrl": { - "type": [ - "string", - "null" - ] - }, - "inventoryManagement": { - "type": [ - "boolean", - "null" - ] - }, - "permitsSkuSharing": { - "type": [ - "boolean", - "null" - ] - }, - "requiresShippingMethod": { - "type": [ - "boolean", - "null" - ] - }, - "serviceName": { - "type": [ - "string", - "null" - ] - }, - "trackingSupport": { - "type": [ - "boolean", - "null" - ] - }, - "type": { - "type": [ - "string", - "null" - ] - } - } -} diff --git a/tap_shopify/schemas/marketing_events.json b/tap_shopify/schemas/marketing_events.json deleted file mode 100644 index 2c1ab696..00000000 --- a/tap_shopify/schemas/marketing_events.json +++ /dev/null @@ -1,110 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "type": { - "type": [ - "string", - "null" - ] - }, - "remoteId": { - "type": [ - "string", - "null" - ] - }, - "startedAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "endedAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "scheduledToEndAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "manageUrl": { - "type": [ - "string", - "null" - ] - }, - "previewUrl": { - "type": [ - "string", - "null" - ] - }, - "utmCampaign": { - "type": [ - "string", - "null" - ] - }, - "utmMedium": { - "type": [ - "string", - "null" - ] - }, - "utmSource": { - "type": [ - "string", - "null" - ] - }, - "description": { - "type": [ - "string", - "null" - ] - }, - "marketingChannelType": { - "type": [ - "string", - "null" - ] - }, - "sourceAndMedium": { - "type": [ - "string", - "null" - ] - }, - "targetTypeDisplayText": { - "type": [ - "string", - "null" - ] - }, - "channel": { - "type": [ - "string", - "null" - ] - }, - "channelHandle": { - "type": [ - "string", - "null" - ] - } - } -} diff --git a/tap_shopify/schemas/pages.json b/tap_shopify/schemas/pages.json deleted file mode 100644 index f9fda3fa..00000000 --- a/tap_shopify/schemas/pages.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "updatedAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "handle": { - "type": [ - "string", - "null" - ] - }, - "createdAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "bodySummary": { - "type": [ - "string", - "null" - ] - }, - "body": { - "type": [ - "string", - "null" - ] - }, - "isPublished": { - "type": [ - "boolean", - "null" - ] - }, - "publishedAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "templateSuffix": { - "type": [ - "string", - "null" - ] - }, - "title": { - "type": [ - "string", - "null" - ] - } - } -} diff --git a/tap_shopify/schemas/policies.json b/tap_shopify/schemas/policies.json deleted file mode 100644 index 830d23e5..00000000 --- a/tap_shopify/schemas/policies.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "type": "object", - "properties": { - "createdAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "id": { - "type": [ - "string", - "null" - ] - }, - "title": { - "type": [ - "string", - "null" - ] - }, - "type": { - "type": [ - "string", - "null" - ] - }, - "updatedAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "url": { - "type": [ - "string", - "null" - ] - } - } -} diff --git a/tap_shopify/schemas/price_rules.json b/tap_shopify/schemas/price_rules.json deleted file mode 100644 index e367d413..00000000 --- a/tap_shopify/schemas/price_rules.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "discount": { - "type": [ - "object", - "null" - ], - "properties": { - "__typename": { - "type": [ - "string", - "null" - ] - }, - "title": { - "type": [ - "string", - "null" - ] - }, - "status": { - "type": [ - "string", - "null" - ] - }, - "createdAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "updatedAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "startsAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "endsAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "usageLimit": { - "type": [ - "integer", - "null" - ] - }, - "usesPerOrderLimit": { - "type": [ - "integer", - "null" - ] - }, - "combinesWith": { - "type": [ - "object", - "null" - ], - "properties": { - "orderDiscounts": { - "type": [ - "boolean", - "null" - ] - }, - "productDiscounts": { - "type": [ - "boolean", - "null" - ] - }, - "shippingDiscounts": { - "type": [ - "boolean", - "null" - ] - } - } - } - } - } - } -} diff --git a/tap_shopify/schemas/recurring_application_charges.json b/tap_shopify/schemas/recurring_application_charges.json deleted file mode 100644 index cb935c7b..00000000 --- a/tap_shopify/schemas/recurring_application_charges.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "name": { - "type": [ - "string", - "null" - ] - }, - "createdAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "currentPeriodEnd": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "returnUrl": { - "type": [ - "string", - "null" - ] - }, - "status": { - "type": [ - "string", - "null" - ] - }, - "test": { - "type": [ - "boolean", - "null" - ] - }, - "trialDays": { - "type": [ - "integer", - "null" - ] - }, - "lineItems": { - "type": [ - "array", - "null" - ], - "items": { - "type": "object", - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "plan": { - "type": [ - "object", - "null" - ], - "properties": { - "pricingDetails": { - "type": [ - "object", - "null" - ], - "properties": { - "__typename": { - "type": [ - "string", - "null" - ] - }, - "planHandle": { - "type": [ - "string", - "null" - ] - }, - "interval": { - "type": [ - "string", - "null" - ] - }, - "discount": { - "type": [ - "object", - "null" - ], - "properties": { - "durationLimitInIntervals": { - "type": [ - "integer", - "null" - ] - }, - "remainingDurationInIntervals": { - "type": [ - "integer", - "null" - ] - }, - "priceAfterDiscount": { - "type": [ - "object", - "null" - ], - "properties": { - "amount": { - "type": [ - "number", - "null" - ] - }, - "currencyCode": { - "type": [ - "string", - "null" - ] - } - } - }, - "value": { - "type": [ - "object", - "null" - ], - "properties": { - "__typename": { - "type": [ - "string", - "null" - ] - }, - "percentage": { - "type": [ - "number", - "null" - ] - }, - "amount": { - "type": [ - "object", - "null" - ], - "properties": { - "amount": { - "type": [ - "number", - "null" - ] - }, - "currencyCode": { - "type": [ - "string", - "null" - ] - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } -} diff --git a/tap_shopify/schemas/redirects.json b/tap_shopify/schemas/redirects.json deleted file mode 100644 index 19dd5475..00000000 --- a/tap_shopify/schemas/redirects.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "path": { - "type": [ - "string", - "null" - ] - }, - "target": { - "type": [ - "string", - "null" - ] - }, - "createdAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - } - } -} diff --git a/tap_shopify/schemas/resource_feedback.json b/tap_shopify/schemas/resource_feedback.json deleted file mode 100644 index 11be9b0e..00000000 --- a/tap_shopify/schemas/resource_feedback.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "type": "object", - "properties": { - "app_id": { - "type": [ - "string", - "null" - ] - }, - "title": { - "type": [ - "string", - "null" - ] - }, - "feedback_generated_at": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "feedback": { - "type": [ - "object", - "null" - ], - "properties": { - "messages": { - "type": [ - "array", - "null" - ], - "items": { - "type": "object", - "properties": { - "field": { - "type": [ - "string", - "null" - ] - }, - "message": { - "type": [ - "string", - "null" - ] - } - } - } - }, - "state": { - "type": [ - "string", - "null" - ] - }, - "feedbackGeneratedAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "link": { - "type": [ - "object", - "null" - ], - "properties": { - "label": { - "type": [ - "string", - "null" - ] - }, - "url": { - "type": [ - "string", - "null" - ] - } - } - } - } - } - } -} \ No newline at end of file diff --git a/tap_shopify/schemas/script_tags.json b/tap_shopify/schemas/script_tags.json deleted file mode 100644 index 9ee46a82..00000000 --- a/tap_shopify/schemas/script_tags.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "cache": { - "type": [ - "boolean", - "null" - ] - }, - "createdAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "updatedAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "displayScope": { - "type": [ - "string", - "null" - ] - }, - "legacyResourceId": { - "type": [ - "string", - "null" - ] - }, - "src": { - "type": [ - "string", - "null" - ] - } - } -} diff --git a/tap_shopify/schemas/shipping_zones.json b/tap_shopify/schemas/shipping_zones.json deleted file mode 100644 index 89f2bec9..00000000 --- a/tap_shopify/schemas/shipping_zones.json +++ /dev/null @@ -1,243 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "profileLocationGroups": { - "type": [ - "array", - "null" - ], - "items": { - "type": "object", - "properties": { - "locationGroup": { - "type": [ - "object", - "null" - ], - "properties": { - "id": { - "type": [ - "string", - "null" - ] - } - } - }, - "locationGroupZones": { - "type": [ - "object", - "null" - ], - "properties": { - "edges": { - "type": [ - "array", - "null" - ], - "items": { - "type": "object", - "properties": { - "node": { - "type": [ - "object", - "null" - ], - "properties": { - "zone": { - "type": [ - "object", - "null" - ], - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "name": { - "type": [ - "string", - "null" - ] - }, - "countries": { - "type": [ - "array", - "null" - ], - "items": { - "type": "object", - "properties": { - "code": { - "type": [ - "object", - "null" - ], - "properties": { - "countryCode": { - "type": [ - "string", - "null" - ] - }, - "restOfWorld": { - "type": [ - "boolean", - "null" - ] - } - } - }, - "provinces": { - "type": [ - "array", - "null" - ], - "items": { - "type": "object", - "properties": { - "name": { - "type": [ - "string", - "null" - ] - }, - "code": { - "type": [ - "string", - "null" - ] - } - } - } - } - } - } - } - } - }, - "methodDefinitions": { - "type": [ - "object", - "null" - ], - "properties": { - "edges": { - "type": [ - "array", - "null" - ], - "items": { - "type": "object", - "properties": { - "node": { - "type": [ - "object", - "null" - ], - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "active": { - "type": [ - "boolean", - "null" - ] - }, - "description": { - "type": [ - "string", - "null" - ] - }, - "methodConditions": { - "type": [ - "array", - "null" - ], - "items": { - "type": "object", - "properties": { - "field": { - "type": [ - "string", - "null" - ] - }, - "operator": { - "type": [ - "string", - "null" - ] - }, - "conditionCriteria": { - "type": [ - "object", - "null" - ], - "properties": { - "__typename": { - "type": [ - "string", - "null" - ] - }, - "amount": { - "type": [ - "string", - "null" - ] - }, - "currencyCode": { - "type": [ - "string", - "null" - ] - }, - "unit": { - "type": [ - "string", - "null" - ] - }, - "value": { - "type": [ - "number", - "null" - ] - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } -} diff --git a/tap_shopify/schemas/shop.json b/tap_shopify/schemas/shop.json deleted file mode 100644 index 27328531..00000000 --- a/tap_shopify/schemas/shop.json +++ /dev/null @@ -1,415 +0,0 @@ -{ - "type": "object", - "properties": { - "alerts": { - "type": [ - "array", - "null" - ], - "items": { - "type": "object", - "properties": { - "action": { - "type": [ - "object", - "null" - ], - "properties": { - "title": { - "type": [ - "string", - "null" - ] - }, - "url": { - "type": [ - "string", - "null" - ] - } - } - }, - "description": { - "type": [ - "string", - "null" - ] - } - } - } - }, - "billingAddress": { - "type": [ - "object", - "null" - ], - "properties": { - "address1": { - "type": [ - "string", - "null" - ] - }, - "address2": { - "type": [ - "string", - "null" - ] - }, - "city": { - "type": [ - "string", - "null" - ] - }, - "company": { - "type": [ - "string", - "null" - ] - }, - "country": { - "type": [ - "string", - "null" - ] - }, - "countryCodeV2": { - "type": [ - "string", - "null" - ] - }, - "latitude": { - "type": [ - "number", - "null" - ] - }, - "longitude": { - "type": [ - "number", - "null" - ] - }, - "phone": { - "type": [ - "string", - "null" - ] - }, - "province": { - "type": [ - "string", - "null" - ] - }, - "provinceCode": { - "type": [ - "string", - "null" - ] - }, - "zip": { - "type": [ - "string", - "null" - ] - } - } - }, - "countriesInShippingZones": { - "type": [ - "object", - "null" - ], - "properties": { - "countryCodes": { - "type": [ - "array", - "null" - ], - "items": { - "type": [ - "string", - "null" - ] - } - }, - "includeRestOfWorld": { - "type": [ - "boolean", - "null" - ] - } - } - }, - "checkoutApiSupported": { - "type": [ - "boolean", - "null" - ] - }, - "contactEmail": { - "type": [ - "string", - "null" - ] - }, - "createdAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "currencyCode": { - "type": [ - "string", - "null" - ] - }, - "currencyFormats": { - "type": [ - "object", - "null" - ], - "properties": { - "moneyFormat": { - "type": [ - "string", - "null" - ] - }, - "moneyInEmailsFormat": { - "type": [ - "string", - "null" - ] - }, - "moneyWithCurrencyFormat": { - "type": [ - "string", - "null" - ] - }, - "moneyWithCurrencyInEmailsFormat": { - "type": [ - "string", - "null" - ] - } - } - }, - "customerAccounts": { - "type": [ - "string", - "null" - ] - }, - "description": { - "type": [ - "string", - "null" - ] - }, - "email": { - "type": [ - "string", - "null" - ] - }, - "enabledPresentmentCurrencies": { - "type": [ - "array", - "null" - ], - "items": { - "type": [ - "string", - "null" - ] - } - }, - "fulfillmentServices": { - "type": [ - "array", - "null" - ], - "items": { - "type": "object", - "properties": { - "handle": { - "type": [ - "string", - "null" - ] - }, - "serviceName": { - "type": [ - "string", - "null" - ] - } - } - } - }, - "ianaTimezone": { - "type": [ - "string", - "null" - ] - }, - "id": { - "type": [ - "string", - "null" - ] - }, - "marketingSmsConsentEnabledAtCheckout": { - "type": [ - "boolean", - "null" - ] - }, - "myshopifyDomain": { - "type": [ - "string", - "null" - ] - }, - "name": { - "type": [ - "string", - "null" - ] - }, - "paymentSettings": { - "type": [ - "object", - "null" - ], - "properties": { - "supportedDigitalWallets": { - "type": [ - "array", - "null" - ], - "items": { - "type": [ - "string", - "null" - ] - } - } - } - }, - "plan": { - "type": [ - "object", - "null" - ], - "properties": { - "partnerDevelopment": { - "type": [ - "boolean", - "null" - ] - }, - "shopifyPlus": { - "type": [ - "boolean", - "null" - ] - }, - "publicDisplayName": { - "type": [ - "string", - "null" - ] - } - } - }, - "primaryDomain": { - "type": [ - "object", - "null" - ], - "properties": { - "host": { - "type": [ - "string", - "null" - ] - }, - "id": { - "type": [ - "string", - "null" - ] - } - } - }, - "setupRequired": { - "type": [ - "boolean", - "null" - ] - }, - "shipsToCountries": { - "type": [ - "array", - "null" - ], - "items": { - "type": [ - "string", - "null" - ] - } - }, - "taxesIncluded": { - "type": [ - "boolean", - "null" - ] - }, - "taxShipping": { - "type": [ - "boolean", - "null" - ] - }, - "timezoneAbbreviation": { - "type": [ - "string", - "null" - ] - }, - "transactionalSmsDisabled": { - "type": [ - "boolean", - "null" - ] - }, - "updatedAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "url": { - "type": [ - "string", - "null" - ] - }, - "weightUnit": { - "type": [ - "string", - "null" - ] - } - } -} diff --git a/tap_shopify/schemas/smart_collections.json b/tap_shopify/schemas/smart_collections.json deleted file mode 100644 index 846e458a..00000000 --- a/tap_shopify/schemas/smart_collections.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "handle": { - "type": [ - "string", - "null" - ] - }, - "title": { - "type": [ - "string", - "null" - ] - }, - "updatedAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "description": { - "type": [ - "string", - "null" - ] - }, - "descriptionHtml": { - "type": [ - "string", - "null" - ] - }, - "sortOrder": { - "type": [ - "string", - "null" - ] - }, - "templateSuffix": { - "type": [ - "string", - "null" - ] - }, - "ruleSet": { - "type": [ - "object", - "null" - ], - "properties": { - "appliedDisjunctively": { - "type": [ - "boolean", - "null" - ] - }, - "rules": { - "type": [ - "array", - "null" - ], - "items": { - "type": "object", - "properties": { - "column": { - "type": [ - "string", - "null" - ] - }, - "relation": { - "type": [ - "string", - "null" - ] - }, - "condition": { - "type": [ - "string", - "null" - ] - } - } - } - } - } - }, - "seo": { - "type": [ - "object", - "null" - ], - "properties": { - "title": { - "type": [ - "string", - "null" - ] - }, - "description": { - "type": [ - "string", - "null" - ] - } - } - } - } -} diff --git a/tap_shopify/schemas/tender_transactions.json b/tap_shopify/schemas/tender_transactions.json deleted file mode 100644 index 52648699..00000000 --- a/tap_shopify/schemas/tender_transactions.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "paymentMethod": { - "type": [ - "string", - "null" - ] - }, - "processedAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "remoteReference": { - "type": [ - "string", - "null" - ] - }, - "test": { - "type": [ - "boolean", - "null" - ] - }, - "transactionDetails": { - "type": [ - "object", - "null" - ], - "properties": { - "creditCardCompany": { - "type": [ - "string", - "null" - ] - }, - "creditCardNumber": { - "type": [ - "string", - "null" - ] - } - } - }, - "amount": { - "type": [ - "object", - "null" - ], - "properties": { - "amount": { - "type": [ - "string", - "null" - ] - }, - "currencyCode": { - "type": [ - "string", - "null" - ] - } - } - }, - "order": { - "type": [ - "object", - "null" - ], - "properties": { - "id": { - "type": [ - "string", - "null" - ] - } - } - } - } -} diff --git a/tap_shopify/schemas/themes.json b/tap_shopify/schemas/themes.json deleted file mode 100644 index 9194f8be..00000000 --- a/tap_shopify/schemas/themes.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "name": { - "type": [ - "string", - "null" - ] - }, - "prefix": { - "type": [ - "string", - "null" - ] - }, - "processing": { - "type": [ - "boolean", - "null" - ] - }, - "processingFailed": { - "type": [ - "boolean", - "null" - ] - }, - "role": { - "type": [ - "string", - "null" - ] - }, - "themeStoreId": { - "type": [ - "integer", - "null" - ] - }, - "updatedAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "createdAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - } - } -} diff --git a/tap_shopify/schemas/webhooks.json b/tap_shopify/schemas/webhooks.json deleted file mode 100644 index 212cc27c..00000000 --- a/tap_shopify/schemas/webhooks.json +++ /dev/null @@ -1,143 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": [ - "string", - "null" - ] - }, - "format": { - "type": [ - "string", - "null" - ] - }, - "filter": { - "type": [ - "string", - "null" - ] - }, - "includeFields": { - "type": [ - "array", - "null" - ], - "items": { - "type": [ - "string", - "null" - ] - } - }, - "legacyResourceId": { - "type": [ - "string", - "null" - ] - }, - "metafieldNamespaces": { - "type": [ - "array", - "null" - ], - "items": { - "type": [ - "string", - "null" - ] - } - }, - "topic": { - "type": [ - "string", - "null" - ] - }, - "createdAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "updatedAt": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "callbackUrl": { - "type": [ - "string", - "null" - ] - }, - "apiVersion": { - "type": [ - "object", - "null" - ], - "properties": { - "displayName": { - "type": [ - "string", - "null" - ] - }, - "handle": { - "type": [ - "string", - "null" - ] - }, - "supported": { - "type": [ - "boolean", - "null" - ] - } - } - }, - "endpoint": { - "type": [ - "object", - "null" - ], - "properties": { - "__typename": { - "type": [ - "string", - "null" - ] - }, - "arn": { - "type": [ - "string", - "null" - ] - }, - "callbackUrl": { - "type": [ - "string", - "null" - ] - }, - "pubSubProject": { - "type": [ - "string", - "null" - ] - }, - "pubSubTopic": { - "type": [ - "string", - "null" - ] - } - } - } - } -} diff --git a/tap_shopify/streams/__init__.py b/tap_shopify/streams/__init__.py index acb0f389..b647db5c 100644 --- a/tap_shopify/streams/__init__.py +++ b/tap_shopify/streams/__init__.py @@ -16,28 +16,3 @@ import tap_shopify.streams.collections import tap_shopify.streams.fulfillment_orders import tap_shopify.streams.order_shipping_lines -import tap_shopify.streams.application_charges -import tap_shopify.streams.application_credits -import tap_shopify.streams.article_authors -import tap_shopify.streams.article_tags -import tap_shopify.streams.blogs -import tap_shopify.streams.carrier_services -import tap_shopify.streams.comments -import tap_shopify.streams.currencies -import tap_shopify.streams.custom_collections -import tap_shopify.streams.draft_orders -import tap_shopify.streams.fulfillment_services -import tap_shopify.streams.marketing_events -import tap_shopify.streams.pages -import tap_shopify.streams.policies -import tap_shopify.streams.price_rules -import tap_shopify.streams.script_tags -import tap_shopify.streams.shipping_zones -import tap_shopify.streams.shop -import tap_shopify.streams.smart_collections -import tap_shopify.streams.tender_transactions -import tap_shopify.streams.themes -import tap_shopify.streams.redirects -import tap_shopify.streams.resource_feedback -import tap_shopify.streams.recurring_application_charges -import tap_shopify.streams.webhooks diff --git a/tap_shopify/streams/abandoned_checkouts.py b/tap_shopify/streams/abandoned_checkouts.py index d088d144..a1712ff8 100644 --- a/tap_shopify/streams/abandoned_checkouts.py +++ b/tap_shopify/streams/abandoned_checkouts.py @@ -7,7 +7,6 @@ class AbandonedCheckouts(Stream): name = "abandoned_checkouts" data_key = "abandonedCheckouts" replication_key = "updatedAt" - access_scope = ["read_orders"] @classmethod def process_sub_entities(cls, data, entity_name): @@ -27,13 +26,13 @@ def process_sub_entities(cls, data, entity_name): sub_entities.append(node) return sub_entities - def transform_object(self, obj, **_kwargs): + def transform_object(self, obj): """ Transforms the object by processing its sub-entities. Args: obj (dict): Object to transform. - **_kwargs: Optional additional parameters. + Returns: dict: Transformed object. """ diff --git a/tap_shopify/streams/application_charges.py b/tap_shopify/streams/application_charges.py deleted file mode 100644 index d5a94476..00000000 --- a/tap_shopify/streams/application_charges.py +++ /dev/null @@ -1,48 +0,0 @@ -from tap_shopify.context import Context -from tap_shopify.streams.base import FullTableStream - - -class ApplicationCharges(FullTableStream): - """Stream class for Application Charges in Shopify.""" - name = "application_charges" - data_key = "currentAppInstallation" - - def call_api(self, query_params, query=None, data_key=None): - """ - Overriding call_api method to extract data from nested data_key. - """ - root_data = super().call_api(query_params, query=query, data_key=data_key) - data = ( - root_data - .get("oneTimePurchases", {}) - ) - return data - - def get_query(self): - return """ - query ApplicationCharges($first: Int!, $after: String) { - currentAppInstallation { - oneTimePurchases(first: $first, after: $after) { - edges { - node { - createdAt - id - name - price { - amount - currencyCode - } - status - test - } - } - pageInfo { - endCursor - hasNextPage - } - } - } - } - """ - -Context.stream_objects["application_charges"] = ApplicationCharges diff --git a/tap_shopify/streams/application_credits.py b/tap_shopify/streams/application_credits.py deleted file mode 100644 index d075cc0f..00000000 --- a/tap_shopify/streams/application_credits.py +++ /dev/null @@ -1,47 +0,0 @@ -from tap_shopify.context import Context -from tap_shopify.streams.base import FullTableStream - - -class ApplicationCredits(FullTableStream): - """Stream class for Application Credits in Shopify.""" - name = "application_credits" - data_key = "appInstallation" - - def call_api(self, query_params, query=None, data_key=None): - """ - Overriding call_api method to extract data from nested data_key. - """ - root_data = super().call_api(query_params, query=query, data_key=data_key) - data = ( - root_data - .get("credits", {}) - ) - return data - - def get_query(self): - return """ - query ApplicationCredits($first: Int!, $after: String) { - appInstallation { - credits(first: $first, after: $after) { - edges { - node { - amount { - amount - currencyCode - } - createdAt - description - id - test - } - } - pageInfo { - endCursor - hasNextPage - } - } - } - } - """ - -Context.stream_objects["application_credits"] = ApplicationCredits diff --git a/tap_shopify/streams/article_authors.py b/tap_shopify/streams/article_authors.py deleted file mode 100644 index 39bdb01b..00000000 --- a/tap_shopify/streams/article_authors.py +++ /dev/null @@ -1,29 +0,0 @@ -from tap_shopify.context import Context -from tap_shopify.streams.base import FullTableStream - - -class ArticleAuthors(FullTableStream): - """Stream class for Article Authors in Shopify.""" - name = "article_authors" - data_key = "articleAuthors" - key_properties = ["name"] - access_scope = ["read_content" ,"read_online_store_pages"] - - def get_query(self): - return """ - query ArticleAuthors($first: Int!, $after: String) { - articleAuthors(first: $first, after: $after) { - edges { - node { - name - } - } - pageInfo { - endCursor - hasNextPage - } - } - } - """ - -Context.stream_objects["article_authors"] = ArticleAuthors diff --git a/tap_shopify/streams/article_tags.py b/tap_shopify/streams/article_tags.py deleted file mode 100644 index 5241f9f1..00000000 --- a/tap_shopify/streams/article_tags.py +++ /dev/null @@ -1,60 +0,0 @@ -import singer -from singer import metrics -from tap_shopify.context import Context -from tap_shopify.streams.base import FullTableStream - -LOGGER = singer.get_logger() - - -class ArticleTags(FullTableStream): - """Stream class for Article Tags in Shopify.""" - name = "article_tags" - data_key = "articleTags" - key_properties = ["article_tag"] - access_scope = ["read_content" ,"read_online_store_pages"] - - # pylint: disable=W0221 - def get_query_params(self): - """ - Construct query parameters for GraphQL requests. - Returns: - dict: Dictionary of query parameters. - """ - params = { - "limit": 250, - } - return params - - def transform_object(self, obj, **_kwargs): - """ - Modify this to perform custom transformation on each object - """ - return {"article_tag": obj} - - # pylint: disable=too-many-locals - def get_objects(self): - """ - Returns: - - Yields list of objects for the stream - Performs: - - Transformation - """ - query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) - LOGGER.info("GraphQL query for stream '%s': %s", self.name, ' '.join(query.split())) - - query_params = self.get_query_params() - - with metrics.http_request_timer(self.name): - data = self.call_api(query_params, query=query) - for record in data: - obj = self.transform_object(record) - yield obj - - def get_query(self): - return """ - query ArticleTags($limit: Int!) { - articleTags(limit: $limit) - } - """ - -Context.stream_objects["article_tags"] = ArticleTags diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index 28843445..ada0aa81 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -137,7 +137,6 @@ class Stream(): date_window_size = None data_key = None results_per_page = None - access_scope = None def __init__(self): self.results_per_page = Context.get_results_per_page(RESULTS_PER_PAGE) @@ -153,7 +152,7 @@ def get_query(self): """ raise NotImplementedError("Function Not Implemented") - def transform_object(self, obj, **_kwargs): + def transform_object(self, obj): """ Modify this to perform custom transformation on each object """ @@ -353,7 +352,7 @@ def get_objects(self): data = self.call_api(query_params, query=query) for edge in data.get("edges"): - obj = self.transform_object(edge.get("node"), last_updated_at=last_updated_at) + obj = self.transform_object(edge.get("node")) replication_value = utils.strptime_to_utc(obj[self.replication_key]) current_bookmark = max(current_bookmark, replication_value) yield obj @@ -371,58 +370,3 @@ def sync(self): Default implementation for sync method """ yield from self.get_objects() - - -class FullTableStream(Stream): - replication_method = 'FULL_TABLE' - - def get_query(self): - """ - Provides GraphQL query - """ - raise NotImplementedError("Function Not Implemented") - - # pylint: disable=W0221 - def get_query_params(self, cursor=None): - """ - Construct query parameters for GraphQL requests. - - Args: - cursor (str): Pagination cursor, if any. - - Returns: - dict: Dictionary of query parameters. - """ - params = { - "first": self.results_per_page, - } - - if cursor: - params["after"] = cursor - return params - - # pylint: disable=too-many-locals - def get_objects(self): - """ - Returns: - - Yields list of objects for the stream - Performs: - - Pagination & Filtering of stream - - Transformation - """ - query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) - LOGGER.info("GraphQL query for stream '%s': %s", self.name, ' '.join(query.split())) - - has_next_page, cursor = True, None - - while has_next_page: - query_params = self.get_query_params(cursor=cursor) - with metrics.http_request_timer(self.name): - data = self.call_api(query_params, query=query) - - for edge in data.get("edges", []): - obj = self.transform_object(edge.get("node")) - yield obj - - page_info = data.get("pageInfo") - cursor , has_next_page = page_info.get("endCursor"), page_info.get("hasNextPage") diff --git a/tap_shopify/streams/blogs.py b/tap_shopify/streams/blogs.py deleted file mode 100644 index 4c9b7e8a..00000000 --- a/tap_shopify/streams/blogs.py +++ /dev/null @@ -1,54 +0,0 @@ -from tap_shopify.context import Context -from tap_shopify.streams.base import Stream - - -class Blogs(Stream): - """Stream class for Blogs in Shopify.""" - name = "blogs" - data_key = "blogs" - replication_key = "updatedAt" - access_scope = ["read_content" ,"read_online_store_pages"] - - def get_query(self): - return """ - query Blogs($first: Int!, $after: String, $query: String) { - blogs(first: $first, after: $after, query: $query) { - edges { - node { - id - handle - commentPolicy - createdAt - tags - updatedAt - title - templateSuffix - articles(first: 250) { - edges { - node { - body - id - title - updatedAt - publishedAt - tags - isPublished - createdAt - } - } - } - feed { - location - path - } - } - } - pageInfo { - endCursor - hasNextPage - } - } - } - """ - -Context.stream_objects["blogs"] = Blogs diff --git a/tap_shopify/streams/carrier_services.py b/tap_shopify/streams/carrier_services.py deleted file mode 100644 index c222d3d3..00000000 --- a/tap_shopify/streams/carrier_services.py +++ /dev/null @@ -1,33 +0,0 @@ -from tap_shopify.context import Context -from tap_shopify.streams.base import FullTableStream - - -class CarrierServices(FullTableStream): - """Stream class for Carrier Services in Shopify.""" - name = "carrier_services" - data_key = "carrierServices" - access_scope = ["read_shipping"] - - def get_query(self): - return """ - query CarrierServices($first: Int!, $after: String, $query: String) { - carrierServices(first: $first, after: $after, query: $query) { - edges { - node { - id - name - active - callbackUrl - formattedName - supportsServiceDiscovery - } - } - pageInfo { - endCursor - hasNextPage - } - } - } - """ - -Context.stream_objects["carrier_services"] = CarrierServices diff --git a/tap_shopify/streams/collections.py b/tap_shopify/streams/collections.py index 69785047..dc584207 100644 --- a/tap_shopify/streams/collections.py +++ b/tap_shopify/streams/collections.py @@ -7,7 +7,6 @@ class Collections(Stream): name = "collections" data_key = "collections" replication_key = "updatedAt" - access_scope = ["read_products"] def transform_products(self, data): """ @@ -48,13 +47,13 @@ def transform_products(self, data): return product_ids - def transform_object(self, obj, **_kwargs): + def transform_object(self, obj): """ Transforms a collection object. Args: obj (dict): Collection object. - **_kwargs: Optional additional parameters. + Returns: dict: Transformed collection object. """ diff --git a/tap_shopify/streams/comments.py b/tap_shopify/streams/comments.py deleted file mode 100644 index bcc04fcf..00000000 --- a/tap_shopify/streams/comments.py +++ /dev/null @@ -1,51 +0,0 @@ -from tap_shopify.context import Context -from tap_shopify.streams.base import Stream - - -class Comments(Stream): - """Stream class for Comments in Shopify.""" - name = "comments" - data_key = "comments" - replication_key = "updatedAt" - access_scope = ["read_content" ,"read_online_store_pages"] - - def get_query(self): - return """ - query Comments($first: Int!, $after: String, $query: String) { - comments(first: $first, after: $after, query: $query) { - edges { - node { - id - body - bodyHtml - author { - email - name - } - createdAt - updatedAt - userAgent - status - publishedAt - isPublished - ip - article { - id - title - tags - publishedAt - isPublished - createdAt - body - } - } - } - pageInfo { - endCursor - hasNextPage - } - } - } - """ - -Context.stream_objects["comments"] = Comments diff --git a/tap_shopify/streams/currencies.py b/tap_shopify/streams/currencies.py deleted file mode 100644 index 5966c35d..00000000 --- a/tap_shopify/streams/currencies.py +++ /dev/null @@ -1,45 +0,0 @@ -from tap_shopify.context import Context -from tap_shopify.streams.base import FullTableStream - - -class Currencies(FullTableStream): - """Stream class for Currencies in Shopify.""" - name = "currencies" - data_key = "shop" - key_properties = ["currencyCode"] - - def call_api(self, query_params, query=None, data_key=None): - """ - Overriding call_api method to extract data from nested data_key. - """ - root_data = super().call_api(query_params, query=query, data_key=data_key) - data = ( - root_data - .get("currencySettings", {}) - ) - return data - - def get_query(self): - return """ - query Currencies($first: Int!, $after: String) { - shop { - currencySettings(first: $first, after: $after) { - edges { - node { - currencyCode - currencyName - enabled - manualRate - rateUpdatedAt - } - } - pageInfo { - endCursor - hasNextPage - } - } - } - } - """ - -Context.stream_objects["currencies"] = Currencies diff --git a/tap_shopify/streams/custom_collections.py b/tap_shopify/streams/custom_collections.py deleted file mode 100644 index 0a62dcf1..00000000 --- a/tap_shopify/streams/custom_collections.py +++ /dev/null @@ -1,64 +0,0 @@ -from tap_shopify.context import Context -from tap_shopify.streams.base import Stream - - -class CustomCollections(Stream): - """Stream class for Custom Collections in Shopify.""" - name = "custom_collections" - data_key = "collections" - replication_key = "updatedAt" - access_scope = ["read_products"] - - def get_query_params(self, updated_at_min, updated_at_max, cursor=None): - """ - Extend base query params by adding `collection_type:custom` to the query filter. - - This keeps the updated_at filtering from the base class and adds - the supported filter for custom collections. - - Args: - updated_at_min (str): Lower bound for updated_at filtering. - updated_at_max (str): Upper bound for updated_at filtering. - cursor (str): Pagination cursor. - - Returns: - dict: Modified query parameters with only supported filters. - """ - params = super().get_query_params(updated_at_min, updated_at_max, cursor) - base_query = params.get("query", "") - extended_query = f"{base_query} AND collection_type:custom" - params["query"] = extended_query - - return params - - def get_query(self): - return """ - query CustomCollections($first: Int!, $after: String, $query: String) { - collections(first: $first, after: $after, query: $query) { - edges { - node { - id - description - title - updatedAt - handle - descriptionHtml - sortOrder - templateSuffix - image { - height - width - altText - url - } - } - } - pageInfo { - endCursor - hasNextPage - } - } - } - """ - -Context.stream_objects["custom_collections"] = CustomCollections diff --git a/tap_shopify/streams/customers.py b/tap_shopify/streams/customers.py index 3bd3210a..f589abd6 100644 --- a/tap_shopify/streams/customers.py +++ b/tap_shopify/streams/customers.py @@ -7,7 +7,6 @@ class Customers(Stream): name = "customers" data_key = "customers" replication_key = "updatedAt" - access_scope = ["read_customers"] def get_query(self): """ diff --git a/tap_shopify/streams/draft_orders.py b/tap_shopify/streams/draft_orders.py deleted file mode 100644 index 30723b61..00000000 --- a/tap_shopify/streams/draft_orders.py +++ /dev/null @@ -1,218 +0,0 @@ -from tap_shopify.context import Context -from tap_shopify.streams.base import Stream - - -class DraftOrders(Stream): - """Stream class for Draft Orders in Shopify.""" - name = "draft_orders" - data_key = "draftOrders" - replication_key = "updatedAt" - access_scope = ["read_draft_orders"] - - def get_query(self): - return """ - query DraftOrders($first: Int!, $after: String, $query: String) { - draftOrders(first: $first, after: $after, query: $query) { - edges { - node { - id - note2 - email - taxesIncluded - currencyCode - invoiceSentAt - createdAt - updatedAt - taxExempt - completedAt - name - status - invoiceUrl - tags - phone - totalShippingPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - totalPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - subtotalPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - totalTaxSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - lineItems(first: 250) { - edges { - node { - id - name - quantity - sku - title - requiresShipping - taxable - custom - variant { - id - title - } - weight { - value - unit - } - vendor - taxLines { - title - source - rate - ratePercentage - priceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - } - product { - id - title - } - isGiftCard - originalUnitPriceWithCurrency { - amount - currencyCode - } - } - } - } - shippingAddress { - address1 - address2 - city - company - country - countryCodeV2 - firstName - latitude - longitude - lastName - name - phone - provinceCode - province - zip - } - shippingLine { - id - title - carrierIdentifier - custom - code - deliveryCategory - source - discountedPriceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - phone - } - taxLines { - channelLiable - rate - ratePercentage - source - title - priceSet { - presentmentMoney { - amount - currencyCode - } - shopMoney { - amount - currencyCode - } - } - } - customer { - id - createdAt - lastName - firstName - defaultAddress { - address1 - address2 - city - company - country - countryCodeV2 - firstName - id - lastName - longitude - name - phone - province - provinceCode - zip - } - taxExempt - tags - defaultEmailAddress { - emailAddress - } - lastOrder { - id - name - currencyCode - } - } - } - } - pageInfo { - endCursor - hasNextPage - } - } - } - """ - -Context.stream_objects["draft_orders"] = DraftOrders diff --git a/tap_shopify/streams/fulfillment_orders.py b/tap_shopify/streams/fulfillment_orders.py index 2a3166d7..e42344ad 100644 --- a/tap_shopify/streams/fulfillment_orders.py +++ b/tap_shopify/streams/fulfillment_orders.py @@ -11,12 +11,6 @@ class FulfillmentOrders(Stream): name = "fulfillment_orders" data_key = "fulfillmentOrders" replication_key = "updatedAt" - access_scope = [ - "read_assigned_fulfillment_orders", - "read_merchant_managed_fulfillment_orders", - "read_third_party_fulfillment_orders", - "read_marketplace_fulfillment_orders" - ] # pylint: disable=W0221,fixme def get_query_params(self, updated_at_min, updated_at_max, cursor=None): @@ -71,12 +65,11 @@ def transform_childitems(self, data, parent_id, key, next_page_key): return child_records - def transform_object(self, obj, **_kwargs): + def transform_object(self, obj): """ Transforms a collection object. Args: obj (dict): Collection object. - **_kwargs: Optional additional parameters. Returns: dict: Transformed collection object. """ diff --git a/tap_shopify/streams/fulfillment_services.py b/tap_shopify/streams/fulfillment_services.py deleted file mode 100644 index 0762cb00..00000000 --- a/tap_shopify/streams/fulfillment_services.py +++ /dev/null @@ -1,65 +0,0 @@ -import singer -from singer import metrics -from tap_shopify.context import Context -from tap_shopify.streams.base import FullTableStream - -LOGGER = singer.get_logger() - - -class FulfillmentServices(FullTableStream): - """Stream class for Fulfillment Services in Shopify.""" - name = "fulfillment_services" - data_key = "shop" - access_scope = ["read_fulfillments"] - - def call_api(self, query_params, query=None, data_key=None): - """ - Overriding call_api method to extract data from nested data_key. - """ - root_data = super().call_api(query_params, query=query, data_key=data_key) - data = ( - root_data - .get("fulfillmentServices", {}) - ) - return data - - # pylint: disable=too-many-locals - def get_objects(self): - """ - Returns: - - Yields list of objects for the stream - Performs: - - Transformation - """ - query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) - LOGGER.info("GraphQL query for stream '%s': %s", self.name, ' '.join(query.split())) - - query_params = {} - - with metrics.http_request_timer(self.name): - data = self.call_api(query_params, query=query) - - for record in data: - obj = self.transform_object(record) - yield obj - - def get_query(self): - return """ - query FulfillmentServices { - shop { - fulfillmentServices { - id - handle - callbackUrl - inventoryManagement - permitsSkuSharing - requiresShippingMethod - serviceName - trackingSupport - type - } - } - } - """ - -Context.stream_objects["fulfillment_services"] = FulfillmentServices diff --git a/tap_shopify/streams/inventory_items.py b/tap_shopify/streams/inventory_items.py index d9d06965..dde493b4 100644 --- a/tap_shopify/streams/inventory_items.py +++ b/tap_shopify/streams/inventory_items.py @@ -8,15 +8,14 @@ class InventoryItems(Stream): name = "inventory_items" data_key = "inventoryItems" replication_key = "updatedAt" - access_scope = ["read_inventory"] - def transform_object(self, obj, **_kwargs): + def transform_object(self, obj): """ Transforms the object by extracting country harmonized system codes. Args: obj (dict): The object to transform. - **_kwargs: Optional additional parameters. + Returns: dict: Transformed object. """ diff --git a/tap_shopify/streams/inventory_levels.py b/tap_shopify/streams/inventory_levels.py index 081fda2f..eb12bc6d 100644 --- a/tap_shopify/streams/inventory_levels.py +++ b/tap_shopify/streams/inventory_levels.py @@ -11,7 +11,6 @@ class InventoryLevels(Stream): data_key = "locations" child_data_key = "inventoryLevels" replication_key = "updatedAt" - access_scope = ["read_inventory"] def get_next_page_child(self, parent_id, cursor, child_query): """ diff --git a/tap_shopify/streams/locations.py b/tap_shopify/streams/locations.py index 73b06416..67b60ec1 100644 --- a/tap_shopify/streams/locations.py +++ b/tap_shopify/streams/locations.py @@ -9,7 +9,6 @@ class Locations(Stream): # Currently, the replication key is set to 'createdAt' because the Shopify # locations graphql endpoint doesn't allow the filter on 'updatedAt' field. replication_key = "createdAt" - access_scope = ["read_locations"] def get_query(self): """ diff --git a/tap_shopify/streams/marketing_events.py b/tap_shopify/streams/marketing_events.py deleted file mode 100644 index 1da14da2..00000000 --- a/tap_shopify/streams/marketing_events.py +++ /dev/null @@ -1,43 +0,0 @@ -from tap_shopify.context import Context -from tap_shopify.streams.base import Stream - - -class MarketingEvents(Stream): - """Stream class for Marketing Events in Shopify.""" - name = "marketing_events" - data_key = "marketingEvents" - replication_key = "startedAt" - access_scope = ["read_marketing_events"] - - def get_query(self): - return """ - query MarketingEvents($first: Int!, $after: String, $query: String) { - marketingEvents(first: $first, after: $after, query: $query) { - edges { - node { - id - type - remoteId - startedAt - endedAt - scheduledToEndAt - manageUrl - previewUrl - utmCampaign - utmMedium - utmSource - description - marketingChannelType - sourceAndMedium - channelHandle - } - } - pageInfo { - endCursor - hasNextPage - } - } - } - """ - -Context.stream_objects["marketing_events"] = MarketingEvents diff --git a/tap_shopify/streams/metafields.py b/tap_shopify/streams/metafields.py index 39a730af..cc830fdf 100644 --- a/tap_shopify/streams/metafields.py +++ b/tap_shopify/streams/metafields.py @@ -21,7 +21,7 @@ class Metafields(Stream, ABC): def get_query(self): """Placeholder for get_query method.""" - def transform_object(self, obj, **_kwargs): + def transform_object(self, obj): """ Transforms a metafield object for output. """ diff --git a/tap_shopify/streams/metafields_collections.py b/tap_shopify/streams/metafields_collections.py index ccd564e5..1b94cd4e 100644 --- a/tap_shopify/streams/metafields_collections.py +++ b/tap_shopify/streams/metafields_collections.py @@ -9,7 +9,6 @@ class MetafieldsCollections(Metafields): name = "metafields_collections" data_key = "collections" - access_scope = ["read_products"] def get_query(self): """Return the GraphQL query for fetching collection metafields.""" diff --git a/tap_shopify/streams/metafields_customers.py b/tap_shopify/streams/metafields_customers.py index e68ea553..e3290c78 100644 --- a/tap_shopify/streams/metafields_customers.py +++ b/tap_shopify/streams/metafields_customers.py @@ -9,7 +9,6 @@ class MetafieldsCustomers(Metafields): name = "metafields_customers" data_key = "customers" - access_scope = ["read_customers"] def get_query(self): """Return the GraphQL query for fetching customer metafields.""" diff --git a/tap_shopify/streams/metafields_orders.py b/tap_shopify/streams/metafields_orders.py index 416190dd..2c7f0d36 100644 --- a/tap_shopify/streams/metafields_orders.py +++ b/tap_shopify/streams/metafields_orders.py @@ -8,7 +8,6 @@ class MetafieldsOrders(Metafields): """Stream class for metafields associated with orders.""" name = "metafields_orders" data_key = "orders" - access_scope = ["read_orders"] def get_query(self): """Returns the GraphQL query for fetching order metafields.""" diff --git a/tap_shopify/streams/metafields_products.py b/tap_shopify/streams/metafields_products.py index 644fa830..c4c5e6e3 100644 --- a/tap_shopify/streams/metafields_products.py +++ b/tap_shopify/streams/metafields_products.py @@ -8,7 +8,6 @@ class MetafieldsProducts(Metafields): """Stream class for product metafields.""" name = "metafields_products" data_key = "products" - access_scope = ["read_products"] def get_query(self): """Return the GraphQL query for product metafields.""" diff --git a/tap_shopify/streams/order_refunds.py b/tap_shopify/streams/order_refunds.py index 7570c331..1b11cd16 100644 --- a/tap_shopify/streams/order_refunds.py +++ b/tap_shopify/streams/order_refunds.py @@ -11,7 +11,6 @@ class OrderRefunds(Stream): data_key = "orders" child_data_key = "refunds" replication_key = "updatedAt" - access_scope = ["read_orders"] automatic_keys = ["order"] # pylint: disable=too-many-locals @@ -146,13 +145,13 @@ def transform_orderadjustments(self, data): return orderadjustments - def transform_object(self, obj, **_kwargs): + def transform_object(self, obj): """ Transform refund objects by extracting refund line items from edges. Args: obj (dict): Refund object. - **_kwargs: Optional additional parameters. + Returns: dict: Transformed refund object. """ diff --git a/tap_shopify/streams/order_shipping_lines.py b/tap_shopify/streams/order_shipping_lines.py index 62d4769a..6786b35e 100644 --- a/tap_shopify/streams/order_shipping_lines.py +++ b/tap_shopify/streams/order_shipping_lines.py @@ -11,7 +11,6 @@ class OrderShippingLines(Stream): data_key = "orders" child_data_key = "shippingLines" replication_key = "updatedAt" - access_scope = ["read_orders"] # pylint: disable=too-many-locals def get_objects(self): diff --git a/tap_shopify/streams/orders.py b/tap_shopify/streams/orders.py index 43e629e5..d6137c0e 100644 --- a/tap_shopify/streams/orders.py +++ b/tap_shopify/streams/orders.py @@ -16,7 +16,6 @@ class Orders(Stream): name = "orders" data_key = "orders" replication_key = "updatedAt" - access_scope = ["read_orders"] def get_query(self): """ @@ -1186,7 +1185,7 @@ def parse_bulk_jsonl(self, url): current_order["lineItems"] = current_line_items yield current_order - def transform_object(self, obj, **_kwargs): + def transform_object(self, obj): if obj.get("lineItems", {}).get("edges"): obj["lineItems"] = [item["node"] for item in obj["lineItems"]["edges"]] return obj diff --git a/tap_shopify/streams/pages.py b/tap_shopify/streams/pages.py deleted file mode 100644 index ddb36f26..00000000 --- a/tap_shopify/streams/pages.py +++ /dev/null @@ -1,37 +0,0 @@ -from tap_shopify.context import Context -from tap_shopify.streams.base import Stream - - -class Pages(Stream): - """Stream class for Pages in Shopify.""" - name = "pages" - data_key = "pages" - access_scope = ["read_content" ,"read_online_store_pages"] - - def get_query(self): - return """ - query Pages($first: Int!, $after: String, $query: String) { - pages(first: $first, after: $after, query: $query) { - edges { - node { - id - updatedAt - handle - createdAt - bodySummary - body - isPublished - publishedAt - templateSuffix - title - } - } - pageInfo { - endCursor - hasNextPage - } - } - } - """ - -Context.stream_objects["pages"] = Pages diff --git a/tap_shopify/streams/policies.py b/tap_shopify/streams/policies.py deleted file mode 100644 index 50d2d54e..00000000 --- a/tap_shopify/streams/policies.py +++ /dev/null @@ -1,88 +0,0 @@ -import singer -from singer import metrics, utils -from tap_shopify.context import Context -from tap_shopify.streams.base import Stream - -LOGGER = singer.get_logger() - - -class Policies(Stream): - """Stream class for Policies in Shopify.""" - name = "policies" - data_key = "shop" - access_scope = ["read_legal_policies"] - - def call_api(self, query_params, query=None, data_key=None): - """ - Overriding call_api method to extract data from nested data_key. - """ - root_data = super().call_api(query_params, query=query, data_key=data_key) - data = ( - root_data - .get("shopPolicies", {}) - ) - return data - - def transform_object(self, obj, **_kwargs): - """ - If the replication key is missing in the input node, a fallback timestamp (current UTC time) - is injected to allow the pipeline to continue operating in incremental sync mode. - Ensures the replication key is returned as an ISO string. - - Args: - obj (dict): Product object. - **_kwargs: Optional additional parameters. - Returns: - dict: Transformed product object. - """ - if self.replication_key not in obj or not obj[self.replication_key]: - last_updated_at = _kwargs.get( - "last_updated_at", - utils.now().replace(microsecond=0) - ) - obj[self.replication_key] = last_updated_at.isoformat() - return obj - - # pylint: disable=too-many-locals - def get_objects(self): - """ - Returns: - - Yields list of objects for the stream - Performs: - - Transformation and bookmarking - """ - last_updated_at = self.get_bookmark() - current_bookmark = last_updated_at - query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) - LOGGER.info("GraphQL query for stream '%s': %s", self.name, ' '.join(query.split())) - query_params = {} - - with metrics.http_request_timer(self.name): - data = self.call_api(query_params, query=query) - - for record in data: - obj = self.transform_object(record, last_updated_at=last_updated_at) - replication_value = utils.strptime_to_utc(obj[self.replication_key]) - if replication_value >= current_bookmark: - yield obj - - current_bookmark = max(current_bookmark, replication_value) - self.update_bookmark(utils.strftime(current_bookmark)) - - def get_query(self): - return """ - query Policies { - shop { - shopPolicies { - createdAt - id - title - type - updatedAt - url - } - } - } - """ - -Context.stream_objects["policies"] = Policies diff --git a/tap_shopify/streams/price_rules.py b/tap_shopify/streams/price_rules.py deleted file mode 100644 index e2763e36..00000000 --- a/tap_shopify/streams/price_rules.py +++ /dev/null @@ -1,148 +0,0 @@ -from tap_shopify.context import Context -from tap_shopify.streams.base import FullTableStream - - -class PriceRules(FullTableStream): - """Stream class for Price Rules in Shopify.""" - name = "price_rules" - data_key = "discountNodes" - access_scope = ["read_discounts"] - - def get_query(self): - return """ - query PriceRules($first: Int!, $after: String, $query: String) { - discountNodes(first: $first, after: $after, query: $query) { - edges { - node { - id - discount { - ... on DiscountAutomaticApp { - __typename - title - status - createdAt - startsAt - updatedAt - endsAt - combinesWith { - orderDiscounts - productDiscounts - shippingDiscounts - } - } - ... on DiscountAutomaticBasic { - __typename - title - status - createdAt - startsAt - updatedAt - endsAt - combinesWith { - orderDiscounts - productDiscounts - shippingDiscounts - } - } - ... on DiscountAutomaticBxgy { - __typename - title - status - createdAt - startsAt - updatedAt - endsAt - usesPerOrderLimit - combinesWith { - orderDiscounts - productDiscounts - shippingDiscounts - } - } - ... on DiscountAutomaticFreeShipping { - __typename - title - status - createdAt - startsAt - updatedAt - endsAt - combinesWith { - orderDiscounts - productDiscounts - shippingDiscounts - } - } - ... on DiscountCodeApp { - __typename - title - status - createdAt - startsAt - updatedAt - endsAt - usageLimit - combinesWith { - orderDiscounts - productDiscounts - shippingDiscounts - } - } - ... on DiscountCodeBasic { - __typename - title - status - createdAt - startsAt - updatedAt - endsAt - usageLimit - combinesWith { - orderDiscounts - productDiscounts - shippingDiscounts - } - } - ... on DiscountCodeBxgy { - __typename - title - status - createdAt - startsAt - updatedAt - endsAt - usageLimit - usesPerOrderLimit - combinesWith { - orderDiscounts - productDiscounts - shippingDiscounts - } - } - ... on DiscountCodeFreeShipping { - __typename - title - status - createdAt - startsAt - updatedAt - endsAt - usageLimit - combinesWith { - orderDiscounts - productDiscounts - shippingDiscounts - } - } - } - } - } - pageInfo { - endCursor - hasNextPage - } - } - } - """ - -Context.stream_objects["price_rules"] = PriceRules diff --git a/tap_shopify/streams/product_variants.py b/tap_shopify/streams/product_variants.py index b8ab498b..4f18cbe8 100644 --- a/tap_shopify/streams/product_variants.py +++ b/tap_shopify/streams/product_variants.py @@ -7,15 +7,14 @@ class ProductVariants(Stream): name = "product_variants" data_key = "productVariants" replication_key = "updatedAt" - access_scope = ["read_products"] - def transform_object(self, obj, **_kwargs): + def transform_object(self, obj): """ Transforms the object if needed. Args: obj (dict): The object to transform. - **_kwargs: Optional additional parameters. + Returns: dict: The transformed object. """ diff --git a/tap_shopify/streams/products.py b/tap_shopify/streams/products.py index 0ef543ff..e8c564c2 100644 --- a/tap_shopify/streams/products.py +++ b/tap_shopify/streams/products.py @@ -8,15 +8,14 @@ class Products(Stream): name = "products" data_key = "products" replication_key = "updatedAt" - access_scope = ["read_products"] - def transform_object(self, obj, **_kwargs): + def transform_object(self, obj): """ Transforms the product object by extracting media information. Args: obj (dict): Product object. - **_kwargs: Optional additional parameters. + Returns: dict: Transformed product object. """ diff --git a/tap_shopify/streams/recurring_application_charges.py b/tap_shopify/streams/recurring_application_charges.py deleted file mode 100644 index 6012e108..00000000 --- a/tap_shopify/streams/recurring_application_charges.py +++ /dev/null @@ -1,97 +0,0 @@ -import singer -from singer import metrics -from tap_shopify.context import Context -from tap_shopify.streams.base import FullTableStream - -LOGGER = singer.get_logger() - - -class RecurringApplicationCharges(FullTableStream): - """Stream class for Recurring Application Charges in Shopify.""" - name = "recurring_application_charges" - data_key = "currentAppInstallation" - - def call_api(self, query_params, query=None, data_key=None): - """ - Overriding call_api method to extract data from nested data_key. - """ - root_data = super().call_api(query_params, query=query, data_key=data_key) - data = ( - root_data - .get("activeSubscriptions", {}) - ) - return data - - # pylint: disable=too-many-locals - def get_objects(self): - """ - Returns: - - Yields list of objects for the stream - Performs: - - Transformation - """ - - query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) - LOGGER.info("GraphQL query for stream '%s': %s", self.name, ' '.join(query.split())) - - query_params = {} - - with metrics.http_request_timer(self.name): - data = self.call_api(query_params, query=query) - - for record in data: - obj = self.transform_object(record) - yield obj - - def get_query(self): - return """ - query RecurringApplicationCharges { - currentAppInstallation { - activeSubscriptions { - createdAt - currentPeriodEnd - id - name - returnUrl - status - test - trialDays - lineItems { - id - plan { - pricingDetails { - __typename - ... on AppRecurringPricing { - planHandle - interval - discount { - durationLimitInIntervals - remainingDurationInIntervals - value { - ... on AppSubscriptionDiscountAmount { - __typename - amount { - amount - currencyCode - } - } - ... on AppSubscriptionDiscountPercentage { - __typename - percentage - } - } - priceAfterDiscount { - amount - currencyCode - } - } - } - } - } - } - } - } - } - """ - -Context.stream_objects["recurring_application_charges"] = RecurringApplicationCharges diff --git a/tap_shopify/streams/redirects.py b/tap_shopify/streams/redirects.py deleted file mode 100644 index 66349690..00000000 --- a/tap_shopify/streams/redirects.py +++ /dev/null @@ -1,52 +0,0 @@ -from singer import utils -from tap_shopify.context import Context -from tap_shopify.streams.base import Stream - - -class Redirects(Stream): - """Stream class for Redirects in Shopify.""" - name = "redirects" - data_key = "urlRedirects" - replication_key = "createdAt" - access_scope = ["read_online_store_navigation"] - - def transform_object(self, obj, **_kwargs): - """ - If the replication key is missing in the input node, a fallback timestamp (current UTC time) - is injected to allow the pipeline to continue operating in incremental sync mode. - Ensures the replication key is returned as an ISO string. - - Args: - obj (dict): Product object. - **_kwargs: Optional additional parameters. - Returns: - dict: Transformed product object. - """ - if self.replication_key not in obj or not obj[self.replication_key]: - last_updated_at = _kwargs.get( - "last_updated_at", - utils.now().replace(microsecond=0) - ) - obj[self.replication_key] = last_updated_at.isoformat() - return obj - - def get_query(self): - return """ - query Redirects($first: Int!, $after: String, $query: String) { - urlRedirects(first: $first, after: $after, query: $query) { - edges { - node { - id - path - target - } - } - pageInfo { - endCursor - hasNextPage - } - } - } - """ - -Context.stream_objects["redirects"] = Redirects diff --git a/tap_shopify/streams/resource_feedback.py b/tap_shopify/streams/resource_feedback.py deleted file mode 100644 index 7230c6b1..00000000 --- a/tap_shopify/streams/resource_feedback.py +++ /dev/null @@ -1,89 +0,0 @@ -import singer -from singer import metrics, utils -from tap_shopify.context import Context -from tap_shopify.streams.base import Stream - -LOGGER = singer.get_logger() - - -class ResourceFeedback(Stream): - """Stream class for Resource Feedback in Shopify.""" - name = "resource_feedback" - data_key = "app" - replication_key = "feedback_generated_at" - key_properties = ["app_id"] - access_scope = ["read_products"] - - def transform_object(self, obj, **_kwargs): - """ - If the replication key is missing in the input node, a fallback timestamp (current UTC time) - is injected to allow the pipeline to continue operating in incremental sync mode. - Ensures the replication key is returned as an ISO string. - - Args: - obj (dict): object. - **_kwargs: Optional additional parameters. - Returns: - dict: Transformed product object. - """ - if "id" in obj: - obj["app_id"] = obj.pop("id") - - feedback = obj.get("feedback") - feedback_generated_at = feedback.get("feedbackGeneratedAt") if feedback else None - - if feedback_generated_at: - obj[self.replication_key] = feedback_generated_at - elif not obj.get(self.replication_key): - last_updated_at = _kwargs["last_updated_at"] - obj[self.replication_key] = last_updated_at.isoformat() - return obj - - # pylint: disable=too-many-locals - def get_objects(self): - """ - Returns: - - Yields list of objects for the stream - Performs: - - Transformation and Bookmarking - """ - last_updated_at = self.get_bookmark() - current_bookmark = last_updated_at - query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) - LOGGER.info("GraphQL query for stream '%s': %s", self.name, ' '.join(query.split())) - query_params = {} - - with metrics.http_request_timer(self.name): - data = self.call_api(query_params, query=query) - - obj = self.transform_object(data, last_updated_at=last_updated_at) - replication_value = utils.strptime_to_utc(obj[self.replication_key]) - if replication_value >= current_bookmark: - yield obj - - current_bookmark = max(current_bookmark, replication_value) - self.update_bookmark(utils.strftime(current_bookmark)) - - def get_query(self): - return """ - query Redirects { - app { - id - title - feedback { - messages { - field - message - } - state - feedbackGeneratedAt - link { - label - url - } - } - } - } - """ - -Context.stream_objects["resource_feedback"] = ResourceFeedback diff --git a/tap_shopify/streams/script_tags.py b/tap_shopify/streams/script_tags.py deleted file mode 100644 index 3437e724..00000000 --- a/tap_shopify/streams/script_tags.py +++ /dev/null @@ -1,35 +0,0 @@ -from tap_shopify.context import Context -from tap_shopify.streams.base import Stream - - -class ScriptTags(Stream): - """Stream class for Script Tags in Shopify.""" - name = "script_tags" - data_key = "scriptTags" - replication_key = "updatedAt" - access_scope = ["read_script_tags"] - - def get_query(self): - return """ - query ScriptTags($first: Int!, $after: String, $query: String) { - scriptTags(first: $first, after: $after, query: $query) { - edges { - node { - id - cache - createdAt - updatedAt - displayScope - legacyResourceId - src - } - } - pageInfo { - endCursor - hasNextPage - } - } - } - """ - -Context.stream_objects["script_tags"] = ScriptTags diff --git a/tap_shopify/streams/shipping_zones.py b/tap_shopify/streams/shipping_zones.py deleted file mode 100644 index c01a29f4..00000000 --- a/tap_shopify/streams/shipping_zones.py +++ /dev/null @@ -1,82 +0,0 @@ -from tap_shopify.context import Context -from tap_shopify.streams.base import FullTableStream - -RESULTS_PER_PAGE = 10 - -class ShippingZones(FullTableStream): - """Stream class for Shipping Zones in Shopify.""" - name = "shipping_zones" - data_key = "deliveryProfiles" - access_scope = ["read_shipping"] - - def __init__(self): - super().__init__() - self.results_per_page = RESULTS_PER_PAGE - - def get_query(self): - return """ - query ShippingZones($first: Int!, $after: String) { - deliveryProfiles(first: $first, after: $after) { - edges { - node { - id - profileLocationGroups { - locationGroup { - id - } - locationGroupZones(first: 50) { - edges { - node { - zone { - id - name - countries { - code { - countryCode - restOfWorld - } - provinces { - name - code - } - } - } - methodDefinitions(first: 50) { - edges { - node { - id - active - description - methodConditions { - field - operator - conditionCriteria { - __typename - ... on MoneyV2 { - amount - currencyCode - } - ... on Weight { - unit - value - } - } - } - } - } - } - } - } - } - } - } - } - pageInfo { - endCursor - hasNextPage - } - } - } - """ - -Context.stream_objects["shipping_zones"] = ShippingZones diff --git a/tap_shopify/streams/shop.py b/tap_shopify/streams/shop.py deleted file mode 100644 index f9ec1227..00000000 --- a/tap_shopify/streams/shop.py +++ /dev/null @@ -1,116 +0,0 @@ -import singer -from singer import metrics, utils -from tap_shopify.context import Context -from tap_shopify.streams.base import Stream - -LOGGER = singer.get_logger() - - -class Shop(Stream): - """Stream class for Shop in Shopify.""" - name = "shop" - data_key = "shop" - - # pylint: disable=too-many-locals - def get_objects(self): - """ - Returns: - - Yields list of objects for the stream - Performs: - - Transformation and Bookmarking - """ - last_updated_at = self.get_bookmark() - current_bookmark = last_updated_at - query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) - LOGGER.info("GraphQL query for stream '%s': %s", self.name, ' '.join(query.split())) - query_params = {} - - with metrics.http_request_timer(self.name): - data = self.call_api(query_params, query=query) - - obj = self.transform_object(data) - replication_value = utils.strptime_to_utc(obj[self.replication_key]) - if replication_value >= current_bookmark: - yield obj - - current_bookmark = max(current_bookmark, replication_value) - self.update_bookmark(utils.strftime(current_bookmark)) - - def get_query(self): - return """ - query ShopShow { - shop { - alerts { - action { - title - url - } - description - } - billingAddress { - address1 - address2 - city - company - country - countryCodeV2 - latitude - longitude - phone - province - provinceCode - zip - } - countriesInShippingZones { - countryCodes - includeRestOfWorld - } - checkoutApiSupported - contactEmail - createdAt - currencyCode - currencyFormats { - moneyFormat - moneyInEmailsFormat - moneyWithCurrencyFormat - moneyWithCurrencyInEmailsFormat - } - customerAccounts - description - email - enabledPresentmentCurrencies - fulfillmentServices { - handle - serviceName - } - ianaTimezone - id - marketingSmsConsentEnabledAtCheckout - myshopifyDomain - name - paymentSettings { - supportedDigitalWallets - } - plan { - partnerDevelopment - shopifyPlus - publicDisplayName - } - primaryDomain { - host - id - } - setupRequired - shipsToCountries - taxesIncluded - taxShipping - timezoneAbbreviation - transactionalSmsDisabled - updatedAt - url - weightUnit - } - } - """ - -Context.stream_objects["shop"] = Shop diff --git a/tap_shopify/streams/smart_collections.py b/tap_shopify/streams/smart_collections.py deleted file mode 100644 index 4a727642..00000000 --- a/tap_shopify/streams/smart_collections.py +++ /dev/null @@ -1,70 +0,0 @@ -from tap_shopify.context import Context -from tap_shopify.streams.base import Stream - - -class SmartCollections(Stream): - """Stream class for Smart Collections in Shopify.""" - name = "smart_collections" - data_key = "collections" - replication_key = "updatedAt" - access_scope = ["read_products"] - - def get_query_params(self, updated_at_min, updated_at_max, cursor=None): - """ - Extend base query params by adding `collection_type:custom` to the query filter. - - This keeps the updated_at filtering from the base class and adds - the supported filter for custom collections. - - Args: - updated_at_min (str): Lower bound for updated_at filtering. - updated_at_max (str): Upper bound for updated_at filtering. - cursor (str): Pagination cursor. - - Returns: - dict: Modified query parameters with only supported filters. - """ - params = super().get_query_params(updated_at_min, updated_at_max, cursor) - base_query = params.get("query", "") - extended_query = f"{base_query} AND collection_type:smart" - params["query"] = extended_query - - return params - - def get_query(self): - return """ - query SmartCollections($first: Int!, $after: String, $query: String) { - collections(first: $first, after: $after, query: $query) { - edges { - node { - id - handle - title - updatedAt - description - descriptionHtml - sortOrder - templateSuffix - ruleSet { - appliedDisjunctively - rules { - column - relation - condition - } - } - seo { - description - title - } - } - } - pageInfo { - endCursor - hasNextPage - } - } - } - """ - -Context.stream_objects["smart_collections"] = SmartCollections diff --git a/tap_shopify/streams/tender_transactions.py b/tap_shopify/streams/tender_transactions.py deleted file mode 100644 index 3bd2c4b3..00000000 --- a/tap_shopify/streams/tender_transactions.py +++ /dev/null @@ -1,46 +0,0 @@ -from tap_shopify.context import Context -from tap_shopify.streams.base import Stream - - -class TenderTransactions(Stream): - """Stream class for Tender Transactions in Shopify.""" - name = "tender_transactions" - data_key = "tenderTransactions" - replication_key = "processedAt" - access_scope = ["read_orders"] - - def get_query(self): - return """ - query TenderTransactions($first: Int!, $after: String, $query: String) { - tenderTransactions(first: $first, after: $after, query: $query) { - edges { - node { - id - paymentMethod - processedAt - remoteReference - test - transactionDetails { - ... on TenderTransactionCreditCardDetails { - creditCardCompany - creditCardNumber - } - } - amount { - amount - currencyCode - } - order { - id - } - } - } - pageInfo { - endCursor - hasNextPage - } - } - } - """ - -Context.stream_objects["tender_transactions"] = TenderTransactions diff --git a/tap_shopify/streams/themes.py b/tap_shopify/streams/themes.py deleted file mode 100644 index 5304ad30..00000000 --- a/tap_shopify/streams/themes.py +++ /dev/null @@ -1,92 +0,0 @@ -import singer -from singer import metrics, utils -from tap_shopify.context import Context -from tap_shopify.streams.base import Stream - -LOGGER = singer.get_logger() - - -class Themes(Stream): - """Stream class for Themes in Shopify.""" - name = "themes" - data_key = "themes" - replication_key = "updatedAt" - access_scope = ["read_themes"] - - # pylint: disable=W0221 - def get_query_params(self, cursor=None): - """ - Construct query parameters for GraphQL requests. - - Args: - cursor (str): Pagination cursor, if any. - - Returns: - dict: Dictionary of query parameters. - """ - params = { - "first": self.results_per_page, - } - - if cursor: - params["after"] = cursor - return params - - # pylint: disable=too-many-locals - def get_objects(self): - """ - Returns: - - Yields list of objects for the stream - Performs: - - Pagination & Filtering of stream - - Transformation and bookmarking - """ - last_updated_at = self.get_bookmark() - current_bookmark = last_updated_at - query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) - LOGGER.info("GraphQL query for stream '%s': %s", self.name, ' '.join(query.split())) - has_next_page, cursor = True, None - - while has_next_page: - query_params = self.get_query_params(cursor) - - with metrics.http_request_timer(self.name): - data = self.call_api(query_params, query=query) - - for edge in data.get("edges"): - obj = self.transform_object(edge.get("node")) - replication_value = utils.strptime_to_utc(obj[self.replication_key]) - if replication_value >= current_bookmark: - current_bookmark = max(current_bookmark, replication_value) - yield obj - - page_info = data.get("pageInfo") - cursor, has_next_page = page_info.get("endCursor"), page_info.get("hasNextPage") - self.update_bookmark(utils.strftime(current_bookmark)) - - def get_query(self): - return """ - query Themes($first: Int!, $after: String) { - themes(first: $first, after: $after) { - edges { - node { - id - name - prefix - processing - processingFailed - role - themeStoreId - updatedAt - createdAt - } - } - pageInfo { - endCursor - hasNextPage - } - } - } - """ - -Context.stream_objects["themes"] = Themes diff --git a/tap_shopify/streams/transactions.py b/tap_shopify/streams/transactions.py index a29928b1..fb3de78a 100644 --- a/tap_shopify/streams/transactions.py +++ b/tap_shopify/streams/transactions.py @@ -11,7 +11,6 @@ class Transactions(Stream): data_key = "orders" child_data_key = "transactions" replication_key = "createdAt" - access_scope = ["read_orders"] # pylint: disable=W0221 def get_query_params(self, updated_at_min, updated_at_max, cursor=None): diff --git a/tap_shopify/streams/webhooks.py b/tap_shopify/streams/webhooks.py deleted file mode 100644 index ba19abb3..00000000 --- a/tap_shopify/streams/webhooks.py +++ /dev/null @@ -1,56 +0,0 @@ -from tap_shopify.context import Context -from tap_shopify.streams.base import Stream - - -class Webhooks(Stream): - """Stream class for Webhooks in Shopify.""" - name = "webhooks" - data_key = "webhookSubscriptions" - - def get_query(self): - return """ - query Webhooks($first: Int!, $after: String, $query: String) { - webhookSubscriptions(first: $first, after: $after, query: $query) { - edges { - node { - id - format - filter - createdAt - apiVersion { - displayName - handle - supported - } - includeFields - legacyResourceId - metafieldNamespaces - topic - updatedAt - endpoint { - ... on WebhookEventBridgeEndpoint { - __typename - arn - } - ... on WebhookHttpEndpoint { - __typename - callbackUrl - } - ... on WebhookPubSubEndpoint { - __typename - pubSubProject - pubSubTopic - } - } - callbackUrl - } - } - pageInfo { - endCursor - hasNextPage - } - } - } - """ - -Context.stream_objects["webhooks"] = Webhooks diff --git a/tests/base.py b/tests/base.py index 09bc2214..ef414d2e 100644 --- a/tests/base.py +++ b/tests/base.py @@ -27,7 +27,7 @@ class BaseTapTest(BaseCase): FULL = "FULL_TABLE" START_DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" BOOKMARK_COMPARISON_FORMAT = "%Y-%m-%dT00:00:00+00:00" - DEFAULT_RESULTS_PER_PAGE = 1 + DEFAULT_RESULTS_PER_PAGE = 250 @staticmethod def tap_name(): @@ -43,7 +43,7 @@ def get_type(): def get_properties(self, original: bool = True): """Configuration properties required for the tap.""" return_value = { - 'start_date': '2020-07-01T00:00:00Z', + 'start_date': '2017-07-01T00:00:00Z', 'shop': 'stitchdatawearhouse', 'date_window_size': 180, # BUG: https://jira.talendforge.org/browse/TDL-13180 @@ -82,15 +82,9 @@ def expected_metadata(self): self.REPLICATION_METHOD: self.INCREMENTAL, self.API_LIMIT: self.DEFAULT_RESULTS_PER_PAGE} - full_default = { - self.PRIMARY_KEYS: {"id"}, - self.REPLICATION_METHOD: self.FULL, - self.API_LIMIT: 2 - } - meta = default.copy() meta[self.REPLICATION_KEYS] = {"updatedAt"} - meta[self.API_LIMIT] = 2 + meta[self.API_LIMIT] = 30 meta.update({self.FOREIGN_KEYS: {"owner", "ownerType"}}) return { @@ -99,7 +93,7 @@ def expected_metadata(self): self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.INCREMENTAL, # BUG: https://qlik-dev.atlassian.net/browse/TDL-13180 - self.API_LIMIT: 2}, + self.API_LIMIT: 50}, "collections": default, "customers": default, "orders": default, @@ -107,13 +101,13 @@ def expected_metadata(self): self.REPLICATION_KEYS: {"updatedAt"}, self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.INCREMENTAL, - self.API_LIMIT: 2 + self.API_LIMIT: 30 }, "order_shipping_lines": { self.REPLICATION_KEYS: {"updatedAt"}, self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.INCREMENTAL, - self.API_LIMIT: 2 + self.API_LIMIT: 15 }, "order_refunds": default, "products": default, @@ -128,7 +122,7 @@ def expected_metadata(self): self.PRIMARY_KEYS: {"id"}, self.FOREIGN_KEYS: {"order"}, self.REPLICATION_METHOD: self.INCREMENTAL, - self.API_LIMIT: 2}, + self.API_LIMIT: 150}, "locations": { self.REPLICATION_KEYS: {"createdAt"}, self.PRIMARY_KEYS: {"id"}, @@ -140,103 +134,18 @@ def expected_metadata(self): self.REPLICATION_KEYS: {"createdAt"}, self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.INCREMENTAL, - self.API_LIMIT: 2 - }, - "application_charges": full_default, - "application_credits": full_default, - "article_authors": { - self.PRIMARY_KEYS: {"name"}, - self.REPLICATION_METHOD: self.FULL, - self.API_LIMIT: 2 - }, - "article_tags": { - self.PRIMARY_KEYS: {"article_tag"}, - self.REPLICATION_METHOD: self.FULL, - self.API_LIMIT: 2 - }, - "blogs": default, - "carrier_services": full_default, - "comments": default, - "currencies": { - self.PRIMARY_KEYS: {"currencyCode"}, - self.REPLICATION_METHOD: self.FULL, - self.API_LIMIT: 2 - }, - "custom_collections": default, - "draft_orders": default, - "fulfillment_services": full_default, - "marketing_events": { - self.REPLICATION_KEYS: {"startedAt"}, - self.PRIMARY_KEYS: {"id"}, - self.REPLICATION_METHOD: self.INCREMENTAL, - self.API_LIMIT: 2 - }, - "pages": default, - "policies": default, - "price_rules": full_default, - "recurring_application_charges": full_default, - "redirects": { - self.REPLICATION_KEYS: {"createdAt"}, - self.PRIMARY_KEYS: {"id"}, - self.REPLICATION_METHOD: self.INCREMENTAL, - self.API_LIMIT: 2 - }, - "resource_feedback": { - self.REPLICATION_KEYS: {"feedback_generated_at"}, - self.PRIMARY_KEYS: {"app_id"}, - self.REPLICATION_METHOD: self.INCREMENTAL, - self.API_LIMIT: 2 - }, - "script_tags": default, - "shipping_zones": { - self.PRIMARY_KEYS: {"id"}, - self.REPLICATION_METHOD: self.FULL, - self.API_LIMIT: 2 - }, - "shop": default, - "smart_collections": default, - "tender_transactions": { - self.REPLICATION_KEYS: {"processedAt"}, - self.PRIMARY_KEYS: {"id"}, - self.REPLICATION_METHOD: self.INCREMENTAL, - self.API_LIMIT: 2 - }, - "themes": default, - "webhooks": default + self.API_LIMIT: 50 + } } def expected_streams(self): """A set of expected stream names""" - # excluding streams due to lack of data and permissions - streams_to_exclude = { - 'article_authors', - 'article_tags', - 'blogs', - 'carrier_services', - 'comments', - 'draft_orders', - 'fulfillment_services', - 'marketing_events', - 'pages', - 'policies', - 'price_rules', - 'redirects', - 'script_tags', - 'shipping_zones', - 'themes', - 'application_charges', - 'currencies', - 'application_credits', - 'webhooks', - 'recurring_application_charges', - 'abandoned_checkouts' - } # removed "abandoned_checkouts", as per the Doc: # https://help.shopify.com/en/manual/orders/abandoned-checkouts?st_source=admin&st_campaign=abandoned_checkouts_footer&utm_source=admin&utm_campaign=abandoned_checkouts_footer#review-your-abandoned-checkouts # abandoned checkouts are saved in the Shopify admin for three months. # Every Monday, abandoned checkouts that are older than three months are removed from your admin. # Also no POST call is available for this endpoint: https://shopify.dev/api/admin-rest/2022-01/resources/abandoned-checkouts - return set(self.expected_metadata().keys()) - streams_to_exclude + return set(self.expected_metadata().keys()) - {"abandoned_checkouts"} def child_streams(self): """ @@ -404,34 +313,13 @@ def select_all_streams_and_fields(conn_id, catalogs, select_all_fields: bool = T def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.start_date = self.get_properties().get("start_date") - self.store_1_streams = { - 'collections', - 'orders', - 'products', - 'customers', - 'locations', - 'inventory_levels', - 'inventory_items', - 'fulfillment_orders', - 'events' - } + self.store_1_streams = {'collections', 'orders', 'products', 'customers', 'locations', 'inventory_levels', 'inventory_items', 'fulfillment_orders'} # removed 'abandoned_checkouts' from store 2 streams, as per the Doc: # https://help.shopify.com/en/manual/orders/abandoned-checkouts?st_source=admin&st_campaign=abandoned_checkouts_footer&utm_source=admin&utm_campaign=abandoned_checkouts_footer#review-your-abandoned-checkouts # abandoned checkouts are saved in the Shopify admin for three months. # Every Monday, abandoned checkouts that are older than three months are removed from your admin. # Also no POST call is available for this endpoint: https://shopify.dev/api/admin-rest/2022-01/resources/abandoned-checkouts - self.store_2_streams = { - 'metafields_products', - 'transactions', - 'order_refunds', - 'products', - 'locations', - 'inventory_levels', - 'inventory_items', - 'order_shipping_lines', - 'product_variants', - 'tender_transactions' - } + self.store_2_streams = {'metafields_products', 'transactions', 'order_refunds', 'products', 'locations', 'inventory_levels', 'inventory_items', 'order_shipping_lines'} #modified this method to accommodate replication key in the current_state def calculated_states_by_stream(self, current_state): diff --git a/tests/test_all_fields.py b/tests/test_all_fields.py index 214a6e20..e066a1c5 100644 --- a/tests/test_all_fields.py +++ b/tests/test_all_fields.py @@ -48,31 +48,8 @@ def test_run(self): - Verify no unexpected streams were replicated - Verify that more than just the automatic fields are replicated for each stream """ - # excluding streams due to lack of data and permissions - streams_to_exclude = { - 'article_authors', - 'article_tags', - 'blogs', - 'carrier_services', - 'comments', - 'draft_orders', - 'fulfillment_services', - 'marketing_events', - 'pages', - 'policies', - 'price_rules', - 'redirects', - 'script_tags', - 'shipping_zones', - 'themes', - 'application_charges', - 'custom_collections', - 'currencies', - 'application_credits', - 'webhooks', - 'recurring_application_charges' - } - expected_streams = self.expected_streams() - streams_to_exclude + + expected_streams = self.expected_streams() # instantiate connection conn_id = self.create_connection() diff --git a/tests/test_bookmarks.py b/tests/test_bookmarks.py index 4b34e436..a28c3dd7 100644 --- a/tests/test_bookmarks.py +++ b/tests/test_bookmarks.py @@ -18,41 +18,6 @@ def name(): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.start_date = '2021-04-01T00:00:00Z' - # excluding streams which dont have data or permission - self.streams_to_exclude = { - 'article_authors', - 'article_tags', - 'blogs', - 'carrier_services', - 'comments', - 'draft_orders', - 'fulfillment_services', - 'marketing_events', - 'pages', - 'policies', - 'price_rules', - 'redirects', - 'script_tags', - 'shipping_zones', - 'themes', - 'application_charges', - 'custom_collections', - 'currencies', - 'application_credits', - 'webhooks', - 'recurring_application_charges', - 'metafields_collections', - 'metafields_orders', - 'customers', - 'smart_collections', - 'fulfillment_orders', - 'shop', - 'events', - 'orders', - 'metafields_customers', - 'resource_feedback', - 'collections' - } def max_bookmarks_by_stream(self, sync_records): """ @@ -112,7 +77,7 @@ def bookmarks_test(self, conn_id, testable_streams): # Select all streams and no fields within streams found_catalogs = menagerie.get_catalogs(conn_id) incremental_streams = {key for key, value in self.expected_replication_method().items() - if value == self.INCREMENTAL and key in testable_streams} - self.streams_to_exclude + if value == self.INCREMENTAL and key in testable_streams} incremental_streams = incremental_streams # Our test data sets for Shopify do not have any abandoned_checkouts @@ -134,7 +99,7 @@ def bookmarks_test(self, conn_id, testable_streams): first_max_bookmarks = self.max_bookmarks_by_stream(first_sync_records) first_min_bookmarks = self.min_bookmarks_by_stream(first_sync_records) #first_sync_bookmarks = menagerie.get_state(conn_id) - + ####################### # Update State between Syncs ####################### diff --git a/tests/test_discovery.py b/tests/test_discovery.py index 9761b10f..89aa4a49 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -33,31 +33,7 @@ def test_run(self): • verify that all other fields have inclusion of available (metadata and schema) """ conn_id = self.create_connection() - streams_to_include = { - 'article_authors', - 'article_tags', - 'blogs', - 'carrier_services', - 'comments', - 'draft_orders', - 'fulfillment_services', - 'marketing_events', - 'pages', - 'policies', - 'price_rules', - 'redirects', - 'script_tags', - 'shipping_zones', - 'themes', - 'application_charges', - 'custom_collections', - 'currencies', - 'application_credits', - 'webhooks', - 'recurring_application_charges', - 'abandoned_checkouts' - } - expected_streams = self.expected_streams() | streams_to_include + expected_streams = self.expected_streams() | {"abandoned_checkouts"} # Verify number of actual streams discovered match expected found_catalogs = menagerie.get_catalogs(conn_id) diff --git a/tests/test_full_replication.py b/tests/test_full_replication.py index c5c50f31..f0924da9 100644 --- a/tests/test_full_replication.py +++ b/tests/test_full_replication.py @@ -27,24 +27,11 @@ def test_run(self): different values for the replication key """ conn_id = self.create_connection() - # excluding streams due to lack of permssion and data - streams_to_exclude = { - "article_authors", - "article_tags", - "carrier_services", - "fulfillment_services", - "price_rules", - "recurring_application_charges", - "shipping_zones", - "application_credits", - "application_charges", - "currencies" - } # Select all streams and no fields within streams found_catalogs = menagerie.get_catalogs(conn_id) full_streams = {key for key, value in self.expected_replication_method().items() - if value == self.FULL} - streams_to_exclude + if value == self.FULL} our_catalogs = [catalog for catalog in found_catalogs if catalog.get('tap_stream_id') in full_streams] self.select_all_streams_and_fields(conn_id, our_catalogs, select_all_fields=True) diff --git a/tests/test_interrupted_sync.py b/tests/test_interrupted_sync.py index 7c3c1b3b..aa1ec44f 100644 --- a/tests/test_interrupted_sync.py +++ b/tests/test_interrupted_sync.py @@ -40,8 +40,7 @@ def test_run(self): 'collections', 'orders', 'products', - 'transactions', - 'events'} + 'transactions'} # Select all streams and no fields within streams found_catalogs = menagerie.get_catalogs(conn_id) diff --git a/tests/test_pagination.py b/tests/test_pagination.py index 2dea8a36..f4b8ab2e 100644 --- a/tests/test_pagination.py +++ b/tests/test_pagination.py @@ -11,7 +11,7 @@ class PaginationTest(BaseTapTest): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.start_date = '2021-04-01T00:00:00Z' + self.start_date = '2021-04-01T00:00:00Z' @staticmethod def name(): @@ -19,7 +19,7 @@ def name(): def get_properties(self, *args, **kwargs): props = super().get_properties(*args, **kwargs) - props['results_per_page'] = '3' + props['results_per_page'] = '30' return props def test_run(self): diff --git a/tests/test_shop_info_fields.py b/tests/test_shop_info_fields.py index a4639ece..1aeb7102 100644 --- a/tests/test_shop_info_fields.py +++ b/tests/test_shop_info_fields.py @@ -44,7 +44,7 @@ def test_run(self): for stream in self.expected_streams(): with self.subTest(stream=stream): - + # Verify that every stream schema contains shop info fields catalog = next(iter([catalog for catalog in found_catalogs if catalog["stream_name"] == stream])) diff --git a/tests/test_start_date.py b/tests/test_start_date.py index d02cae4e..d2284913 100644 --- a/tests/test_start_date.py +++ b/tests/test_start_date.py @@ -53,20 +53,6 @@ def name(): def test_run(self): """Test we get a lot of data back based on the start date configured in base""" conn_id = self.create_connection() - streams_to_exclude = { - "abandoned_checkouts", - "orders", - "blogs", - "comments", - "marketing_events", - "pages", - "policies", - "redirects", - "script_tags", - "themes", - "draft_orders", - "webhooks" - } # Select all streams and all fields within streams found_catalogs = menagerie.get_catalogs(conn_id) @@ -76,8 +62,10 @@ def test_run(self): # Every Monday, abandoned checkouts that are older than three months are removed from your admin. # Also no POST call is available for this endpoint: https://shopify.dev/api/admin-rest/2022-01/resources/abandoned-checkouts expected_replication_method = self.expected_replication_method() + expected_replication_method.pop("abandoned_checkouts") + expected_replication_method.pop("orders") incremental_streams = {key for key, value in expected_replication_method.items() - if value == self.INCREMENTAL} - streams_to_exclude + if value == self.INCREMENTAL} # IF THERE ARE STREAMS THAT SHOULD NOT BE TESTED # REPLACE THE EMPTY SET BELOW WITH THOSE STREAMS diff --git a/tests/unittests/test_base.py b/tests/unittests/test_base.py index 598880eb..203a762f 100644 --- a/tests/unittests/test_base.py +++ b/tests/unittests/test_base.py @@ -102,7 +102,7 @@ def test_call_api_empty_response(self, mock_get_query, mock_graphql): @patch('shopify.GraphQL') @patch.object(Stream, 'get_query', return_value=mock_query()) - @patch.object(Stream, 'transform_object', side_effect=lambda x, **_kwargs: x) + @patch.object(Stream, 'transform_object', side_effect=lambda x: x) @patch('tap_shopify.streams.base.utils.now', return_value=datetime(2025, 2, 1, 0, 0, tzinfo=tzlocal())) def test_get_objects(self, mock_now, mock_transform_object, mock_get_query, mock_graphql): """Test get_objects with pagination and bookmarking.""" diff --git a/tests/unittests/test_date_window.py b/tests/unittests/test_date_window.py index 7ebe907d..402dc8d7 100644 --- a/tests/unittests/test_date_window.py +++ b/tests/unittests/test_date_window.py @@ -3,7 +3,7 @@ from tap_shopify.streams.base import Stream class TestShopifyDateWindowHandling(unittest.TestCase): - + def test_no_date_window_value(self): """Test that no value for date_window is handled correctly.""" Context.config = { diff --git a/tests/unittests/test_token_check_in_discover.py b/tests/unittests/test_token_check_in_discover.py index d2865dd7..0fe2d981 100644 --- a/tests/unittests/test_token_check_in_discover.py +++ b/tests/unittests/test_token_check_in_discover.py @@ -25,25 +25,15 @@ def connection_error_raiser(): @mock.patch('tap_shopify.discover', side_effect=tap_shopify.discover) @mock.patch("builtins.print") @mock.patch("tap_shopify.has_read_users_access") -@mock.patch("tap_shopify.has_access_scope") class TestTokenInDiscoverMode(unittest.TestCase): @mock.patch('tap_shopify.initialize_shopify_client', side_effect=resource_not_found_raiser) - def test_resource_not_found( - self, - mocked_client, - mocked_access_scope, - mocked_access, - mocked_print, - mocked_discover, - mocked_args - ): + def test_resource_not_found(self, mocked_client, mocked_access, mocked_print, mocked_discover, mocked_args): ''' Verify exception is raised for ResourceNotFound with proper error message and - test that discover mode is called + test that discover mode is called ''' mocked_access.return_value = True - mocked_access_scope.return_value = True mocked_args.return_value = Args() try: tap_shopify.main() @@ -54,21 +44,12 @@ def test_resource_not_found( self.assertEqual(mocked_print.call_count, 0) @mock.patch('tap_shopify.initialize_shopify_client', side_effect=unauthorized_access_raiser) - def test_unauthorized_access( - self, - mocked_client, - mocked_access_scope, - mocked_access, - mocked_print, - mocked_discover, - mocked_args - ): + def test_unauthorized_access(self, mocked_client, mocked_access, mocked_print, mocked_discover, mocked_args): ''' Verify exception is raised for UnauthorizedAccess with proper error message and - test that discover mode is called + test that discover mode is called ''' mocked_access.return_value = True - mocked_access_scope.return_value = True mocked_args.return_value = Args() try: tap_shopify.main() @@ -79,21 +60,12 @@ def test_unauthorized_access( self.assertEqual(mocked_print.call_count, 0) @mock.patch('tap_shopify.initialize_shopify_client', side_effect=connection_error_raiser) - def test_connection_error( - self, - mocked_client, - mocked_access_scope, - mocked_access, - mocked_print, - mocked_discover, - mocked_args - ): + def test_connection_error(self, mocked_client, mocked_access, mocked_print, mocked_discover, mocked_args): ''' Verify exception is raised for ConnectionError with proper error message and - test that discover mode is called + test that discover mode is called ''' mocked_access.return_value = True - mocked_access_scope.return_value = True mocked_args.return_value = Args() try: tap_shopify.main() @@ -104,21 +76,12 @@ def test_connection_error( self.assertEqual(mocked_print.call_count, 0) @mock.patch('tap_shopify.initialize_shopify_client') - def test_no_error( - self, - mocked_client, - mocked_access_scope, - mocked_access, - mocked_print, - mocked_discover, - mocked_args - ): + def test_no_error(self, mocked_client, mocked_access, mocked_print, mocked_discover, mocked_args): ''' Verify that if no error during discover then print should be called once for writing catalog ''' mocked_access.return_value = True - mocked_access_scope.return_value = True mocked_args.return_value = Args() tap_shopify.main() self.assertEqual(mocked_discover.call_count, 1) From ad55af4d1d630ef110d3d6face6495292d5e8537 Mon Sep 17 00:00:00 2001 From: Sourabh Gandhi <105213416+sgandhi1311@users.noreply.github.com> Date: Mon, 10 Nov 2025 16:56:32 +0530 Subject: [PATCH 58/66] Cleanup state file for failed Bulk operations (#244) * clear state file for the failed bulk operation * update setup and changelog --- CHANGELOG.md | 3 +++ setup.py | 2 +- tap_shopify/streams/orders.py | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0a117ad..4a6590ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +### 3.7.3 + * Cleanup state file for failed Bulk operations [#244](https://github.com/singer-io/tap-shopify/pull/244) + ### 3.7.2 * New automatic_keys Support and Bulk State Persistence Logic [#237](https://github.com/singer-io/tap-shopify/pull/237) diff --git a/setup.py b/setup.py index 19fa631f..b407a819 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="3.7.2", + version="3.7.3", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/streams/orders.py b/tap_shopify/streams/orders.py index d6137c0e..4aa57266 100644 --- a/tap_shopify/streams/orders.py +++ b/tap_shopify/streams/orders.py @@ -1130,6 +1130,7 @@ def fetch_bulk_operation(op_id): return op.get("url") if current_status in ["FAILED", "CANCELED"]: + self.clear_bulk_operation_state() raise ShopifyAPIError(f"Bulk operation failed: {op.get('errorCode')}") time.sleep(60) From 2b61a0d75e3887187e3624d09fca0ca51a8faf44 Mon Sep 17 00:00:00 2001 From: Sourabh Gandhi <105213416+sgandhi1311@users.noreply.github.com> Date: Tue, 16 Dec 2025 01:22:23 +0530 Subject: [PATCH 59/66] Add exponential backoff retry logic for Shopify bulk operations in progress (#246) * retry if the bulk operation is already in progress * add factor as 5 * update backoff logic * fix pylint * fixes as per co-pilot * update setup and changelog --- CHANGELOG.md | 3 + setup.py | 2 +- tap_shopify/exceptions.py | 6 ++ tap_shopify/streams/orders.py | 123 +++++++++++++++++++++------------- 4 files changed, 85 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a6590ef..3bb514e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +### 3.8.0 + * Exponential backoff for Shopify bulk operations in progress [#246](https://github.com/singer-io/tap-shopify/pull/246) + ### 3.7.3 * Cleanup state file for failed Bulk operations [#244](https://github.com/singer-io/tap-shopify/pull/244) diff --git a/setup.py b/setup.py index b407a819..808eebae 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="3.7.3", + version="3.8.0", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/exceptions.py b/tap_shopify/exceptions.py index 417a01c6..14cbd154 100644 --- a/tap_shopify/exceptions.py +++ b/tap_shopify/exceptions.py @@ -4,3 +4,9 @@ def __init__(self, error, msg=''): class ShopifyAPIError(Exception): """Raised for any unexpected api error without a valid status code""" + +class BulkOperationInProgressError(Exception): + """Raised when a bulk operation is already in progress""" + def __init__(self, message, bulk_op_id=None): + super().__init__(message) + self.bulk_op_id = bulk_op_id diff --git a/tap_shopify/streams/orders.py b/tap_shopify/streams/orders.py index 4aa57266..b437df8e 100644 --- a/tap_shopify/streams/orders.py +++ b/tap_shopify/streams/orders.py @@ -2,13 +2,14 @@ import json import time import re +import backoff import requests import shopify import singer from singer import metrics, utils from tap_shopify.context import Context from tap_shopify.streams.base import Stream -from tap_shopify.exceptions import ShopifyAPIError +from tap_shopify.exceptions import ShopifyAPIError, BulkOperationInProgressError LOGGER = singer.get_logger() @@ -1238,53 +1239,12 @@ def get_objects(self): query_end = min(sync_start, date_window_end) if not existing_url: - with metrics.http_request_timer(self.name): - query_filter = self.build_query_filter( - utils.strftime(last_updated_at), - utils.strftime(query_end) - ) - query = query_template % query_filter - LOGGER.info("Fetching records in date range: %s", query_filter) - - bulk_op_data = self.submit_bulk_query(query) - - user_errors = ( - bulk_op_data.get("data", {}) - .get("bulkOperationRunQuery", {}) - .get("userErrors") - ) - if user_errors: - for error in user_errors: - message = error.get("message", "") - if ( - "bulk query operation for this app and shop is already in progress" - in message - ): - # Try to extract BulkOperation ID using regex - match = re.search( - r"gid://shopify/BulkOperation/\d+", message - ) - bulk_op_id = match.group(0) if match else "UNKNOWN" - raise ShopifyAPIError( - "Shopify limitation: Only one bulk operation of this type " - "is allowed at a time. A bulk operation is already in progress " - f"(ID: {bulk_op_id}). To proceed, please adjust the anchor time" - " to avoid overlapping operations." - ) - raise ShopifyAPIError("Bulk query error: {}".format(user_errors)) - - bulk_operation = ( - bulk_op_data.get("data", {}) - .get("bulkOperationRunQuery", {}) - .get("bulkOperation") - ) - bulk_op_id = bulk_operation.get("id") if bulk_operation else None - if not bulk_op_id: - raise ShopifyAPIError("Invalid bulk operation response: {}". - format(bulk_op_data)) - - existing_url = self.poll_bulk_completion(current_bookmark, bulk_op_id) - + existing_url = self.submit_and_poll_bulk_query( + query_template, + last_updated_at, + query_end, + current_bookmark + ) if existing_url: for obj in self.parse_bulk_jsonl(existing_url): replication_value = utils.strptime_to_utc(obj[self.replication_key]) @@ -1301,4 +1261,71 @@ def get_objects(self): self.update_bookmark(utils.strftime(max_bookmark_value)) existing_url = None + @backoff.on_exception( + backoff.expo, + BulkOperationInProgressError, + max_tries=7, + factor=10, + jitter=None, + on_backoff=lambda details: LOGGER.warning( + "Bulk operation already in progress (ID: %s). " + "Retry attempt %d after %.2f seconds. Total elapsed: %.2f seconds.", + getattr(details['exception'], 'bulk_op_id', 'UNKNOWN'), + details['tries'], + details['wait'], + details['elapsed'] + ) + ) + def submit_and_poll_bulk_query( + self, query_template, last_updated_at, query_end, current_bookmark + ): + """Submit bulk query and poll for completion with automatic retry on conflicts""" + with metrics.http_request_timer(self.name): + query_filter = self.build_query_filter( + utils.strftime(last_updated_at), + utils.strftime(query_end) + ) + query = query_template % query_filter + LOGGER.info("Fetching records in date range: %s", query_filter) + + bulk_op_data = self.submit_bulk_query(query) + + user_errors = ( + bulk_op_data.get("data", {}) + .get("bulkOperationRunQuery", {}) + .get("userErrors") + ) + + if user_errors: + for error in user_errors: + message = error.get("message", "") + if ( + "bulk query operation for this app and shop is already in progress" + in message + ): + # Extract BulkOperation ID using regex + match = re.search(r"gid://shopify/BulkOperation/\d+", message) + bulk_op_id = match.group(0) if match else None + + LOGGER.info("Detected concurrent bulk operation (ID: %s)", bulk_op_id) + raise BulkOperationInProgressError( + f"Bulk operation already in progress: {bulk_op_id}", + bulk_op_id=bulk_op_id + ) + + # Handle other user errors + raise ShopifyAPIError("Bulk query error: {}".format(user_errors)) + + bulk_operation = ( + bulk_op_data.get("data", {}) + .get("bulkOperationRunQuery", {}) + .get("bulkOperation") + ) + bulk_op_id = bulk_operation.get("id") if bulk_operation else None + if not bulk_op_id: + raise ShopifyAPIError("Invalid bulk operation response: {}".format(bulk_op_data)) + + return self.poll_bulk_completion(current_bookmark, bulk_op_id) + + Context.stream_objects["orders"] = Orders From 859af9b0d29482e6d708f4a8739cdcfc3903d97c Mon Sep 17 00:00:00 2001 From: Sourabh Gandhi <105213416+sgandhi1311@users.noreply.github.com> Date: Tue, 6 Jan 2026 17:15:32 +0530 Subject: [PATCH 60/66] update the bookmark value even if the record is not retrieved (#247) * update the bookmark value even if the record is not retrieved * setup and changelog update * add discount applications in the output * fix the test * fix copilot review comments --- CHANGELOG.md | 3 +++ setup.py | 2 +- tap_shopify/streams/orders.py | 22 ++++++++++++++++++++-- tests/test_all_fields.py | 3 +-- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bb514e1..5a328e93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +### 3.8.1 + * Update the bookmark value even if the record is not retrieved [#247](https://github.com/singer-io/tap-shopify/pull/247) + ### 3.8.0 * Exponential backoff for Shopify bulk operations in progress [#246](https://github.com/singer-io/tap-shopify/pull/246) diff --git a/setup.py b/setup.py index 808eebae..6de4dc5c 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="3.8.0", + version="3.8.1", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/streams/orders.py b/tap_shopify/streams/orders.py index b437df8e..88e7ef71 100644 --- a/tap_shopify/streams/orders.py +++ b/tap_shopify/streams/orders.py @@ -1004,6 +1004,7 @@ def get_query(self): code } ... on ManualDiscountApplication { + __typename title description } @@ -1020,6 +1021,14 @@ def get_query(self): } """ + def is_discount_application(self, rec): + if '__typename' in rec and rec['__typename'] in ['AutomaticDiscountApplication', + 'DiscountCodeApplication', + 'ManualDiscountApplication', + 'ScriptDiscountApplication']: + return True + return False + def update_bookmark(self, bookmark_value, bookmark_key=None, bulk_op_metadata=None): # Standard Singer bookmark singer.write_bookmark( @@ -1163,6 +1172,7 @@ def parse_bulk_jsonl(self, url): resp = requests.get(url, stream=True, timeout=60) current_order = None current_line_items = [] + current_discount_applications = [] for line in resp.iter_lines(): if not line: @@ -1173,18 +1183,25 @@ def parse_bulk_jsonl(self, url): continue # Detect line item (child) or order (parent) if '__parentId' in rec: - # It's a line item belonging to current_order - current_line_items.append(rec) + if self.is_discount_application(rec): + # It's a discount application belonging to current_order + current_discount_applications.append(rec) + else: + # It's a line item belonging to current_order + current_line_items.append(rec) else: if current_order: current_order["lineItems"] = current_line_items + current_order["discountApplications"] = current_discount_applications yield current_order # Start tracking new parent group current_order = rec current_line_items = [] + current_discount_applications = [] # Yield the last parent group (if exists) if current_order: current_order["lineItems"] = current_line_items + current_order["discountApplications"] = current_discount_applications yield current_order def transform_object(self, obj): @@ -1254,6 +1271,7 @@ def get_objects(self): else: LOGGER.info("No data returned for the date range: %s to %s", last_updated_at, query_end) + current_bookmark = max(current_bookmark, query_end) self.clear_bulk_operation_state() last_updated_at = query_end diff --git a/tests/test_all_fields.py b/tests/test_all_fields.py index e066a1c5..ff1e8179 100644 --- a/tests/test_all_fields.py +++ b/tests/test_all_fields.py @@ -12,8 +12,7 @@ 'canDelete', 'rawMessage', 'canEdit', - }, - 'orders': {'discountApplications'} + } } class AllFieldsTest(BaseTapTest): From 2a38165cbca7f6cf055439241a3e64e99367c2a9 Mon Sep 17 00:00:00 2001 From: Sourabh Gandhi <105213416+sgandhi1311@users.noreply.github.com> Date: Tue, 27 Jan 2026 19:00:18 +0530 Subject: [PATCH 61/66] Add pagination support for fulfillment line items (#248) * add pagination for fulfillment line items * update page size * copilot comments * update setup and changelog --- CHANGELOG.md | 3 + setup.py | 2 +- tap_shopify/streams/fulfillment_orders.py | 80 ++++++++++++++++++++++- 3 files changed, 82 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a328e93..3a897bc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +### 3.9.0 + * Add pagination support for fulfillment line items [#248](https://github.com/singer-io/tap-shopify/pull/248) + ### 3.8.1 * Update the bookmark value even if the record is not retrieved [#247](https://github.com/singer-io/tap-shopify/pull/247) diff --git a/setup.py b/setup.py index 6de4dc5c..65f06144 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="3.8.1", + version="3.9.0", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/streams/fulfillment_orders.py b/tap_shopify/streams/fulfillment_orders.py index e42344ad..4032ddb3 100644 --- a/tap_shopify/streams/fulfillment_orders.py +++ b/tap_shopify/streams/fulfillment_orders.py @@ -65,6 +65,71 @@ def transform_childitems(self, data, parent_id, key, next_page_key): return child_records + def get_fulfillment_line_items(self, fulfillment_id, next_page=None): + """ + Fetch all fulfillment line items for a given fulfillment ID. + """ + fulfillment_line_items = [] + query = """ + query FulfillmentLineItems($fulfillmentId: ID!, $next_page: String) { + fulfillment(id: $fulfillmentId) { + fulfillmentLineItems(first: 100, after: $next_page) { + pageInfo { + hasNextPage + endCursor + } + nodes { + id + quantity + originalTotalSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + discountedTotalSet { + presentmentMoney { + amount + currencyCode + } + shopMoney { + amount + currencyCode + } + } + lineItem { + id + } + } + } + } + } + """ + + params = { + "fulfillmentId": fulfillment_id, + } + if next_page: + params["next_page"] = next_page + + while True: + response = self.call_api(params, query=query, data_key="fulfillment") + fulfillment_line_items.extend( + response.get("fulfillmentLineItems", {}).get("nodes", []) + ) + + page_info = response.get("fulfillmentLineItems", {}).get("pageInfo", {}) + if not page_info.get("hasNextPage"): + break + + params["next_page"] = page_info.get("endCursor") + + return fulfillment_line_items + def transform_object(self, obj): """ Transforms a collection object. @@ -98,8 +163,16 @@ def transform_object(self, obj): ) for item in obj["fulfillments"]: item["fulfillmentOrders"] = item["fulfillmentOrders"]["nodes"] - item["fulfillmentLineItems"] = item["fulfillmentLineItems"]["nodes"] item["events"] = item["events"]["nodes"] + initial_nodes = item["fulfillmentLineItems"]["nodes"] + if item["fulfillmentLineItems"]["pageInfo"]["hasNextPage"]: + more_nodes = self.get_fulfillment_line_items( + fulfillment_id=item["id"], + next_page=item["fulfillmentLineItems"]["pageInfo"]["endCursor"] + ) + item["fulfillmentLineItems"] = initial_nodes + more_nodes + else: + item["fulfillmentLineItems"] = initial_nodes if obj.get("fulfillmentOrdersForMerge"): obj["fulfillmentOrdersForMerge"] = obj["fulfillmentOrdersForMerge"]["nodes"] @@ -286,6 +359,10 @@ def get_query(self): } } fulfillmentLineItems(first: 3) { + pageInfo { + hasNextPage + endCursor + } nodes { id quantity @@ -313,7 +390,6 @@ def get_query(self): id } } - } legacyResourceId order { From fe62f966c50a6cc777fbb2a99a0faa604ac58c9f Mon Sep 17 00:00:00 2001 From: Nithin <149024913+nk7up@users.noreply.github.com> Date: Fri, 17 Apr 2026 12:48:33 -0400 Subject: [PATCH 62/66] SAC-30546: Fix current bookmark value for no data time window (#251) * adding events to expected streams * removing current_bookmark when no data available, uses previously updated * putting back the stream with the fix * removing events as expansion of coverage is not necessary --- tap_shopify/streams/orders.py | 1 - tests/test_interrupted_sync.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_shopify/streams/orders.py b/tap_shopify/streams/orders.py index 88e7ef71..088b1c48 100644 --- a/tap_shopify/streams/orders.py +++ b/tap_shopify/streams/orders.py @@ -1271,7 +1271,6 @@ def get_objects(self): else: LOGGER.info("No data returned for the date range: %s to %s", last_updated_at, query_end) - current_bookmark = max(current_bookmark, query_end) self.clear_bulk_operation_state() last_updated_at = query_end diff --git a/tests/test_interrupted_sync.py b/tests/test_interrupted_sync.py index aa1ec44f..3ab6473f 100644 --- a/tests/test_interrupted_sync.py +++ b/tests/test_interrupted_sync.py @@ -90,6 +90,7 @@ def test_run(self): {'currently_sync_stream': currently_syncing_stream, 'customers': first_sync_state.get('bookmarks').get('customers'), 'orders': first_sync_state.get('bookmarks').get('orders'), + 'collections': first_sync_state.get('bookmarks').get('collections'), 'products': first_sync_state.get('bookmarks').get('products'), 'transactions': first_sync_state.get('bookmarks').get('transactions') }} From 30333421835d1145e214f8ecccab8b893e68ae01 Mon Sep 17 00:00:00 2001 From: Prijen Khokhani <88327452+prijendev@users.noreply.github.com> Date: Tue, 21 Apr 2026 20:56:58 +0530 Subject: [PATCH 63/66] Add support of client_cred grant type auth (#249) * Add support of client_cred grant type auth * Fix pylint and unit tests * Fix pylint * update integration test configuration * execute without access_token and expires config variables * Added better error handling * Fix order stream sync * Remove the expiry token variable * run test in memory * make pylint happy * fix the integration tests * Add few more unit test cases * add better logging * update setup and changelog --------- Co-authored-by: Sourabh Gandhi --- .circleci/config.yml | 1 + CHANGELOG.md | 4 + setup.py | 2 +- tap_shopify/__init__.py | 22 +- tap_shopify/client.py | 110 +++ tap_shopify/context.py | 1 + tap_shopify/exceptions.py | 4 + tap_shopify/streams/base.py | 20 +- tap_shopify/streams/orders.py | 5 +- tests/base.py | 17 +- tests/test_all_fields.py | 8 - tests/test_start_date.py | 6 - tests/unittests/test_client.py | 703 ++++++++++++++++++ .../unittests/test_token_check_in_discover.py | 1 + 14 files changed, 872 insertions(+), 32 deletions(-) create mode 100644 tap_shopify/client.py create mode 100644 tests/unittests/test_client.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 1aa31125..b66734ac 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -83,6 +83,7 @@ jobs: source /usr/local/share/virtualenvs/tap-shopify/bin/activate aws s3 cp s3://com-stitchdata-dev-deployment-assets/environments/tap-tester/tap_tester_sandbox dev_env.sh source dev_env.sh + unset USE_STITCH_BACKEND mkdir /tmp/${CIRCLE_PROJECT_REPONAME} export STITCH_CONFIG_DIR=/tmp/${CIRCLE_PROJECT_REPONAME} source /usr/local/share/virtualenvs/tap-tester/bin/activate diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a897bc5..5306cdd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +### 3.10.0 + * Add support of client_cred grant type auth [#249](https://github.com/singer-io/tap-shopify/pull/249) + * Fix current bookmark value for no data time window [#251](https://github.com/singer-io/tap-shopify/pull/251) + ### 3.9.0 * Add pagination support for fulfillment line items [#248](https://github.com/singer-io/tap-shopify/pull/248) diff --git a/setup.py b/setup.py index 65f06144..203b06a6 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="3.9.0", + version="3.10.0", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index 0544dad6..97ca093b 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -13,17 +13,18 @@ from singer import metadata from singer import Transformer from tap_shopify.context import Context -from tap_shopify.exceptions import ShopifyError, ShopifyAPIError +from tap_shopify.client import ShopifyClient +from tap_shopify.exceptions import ShopifyError, ShopifyAPIError, ShopifyUnauthorizedError from tap_shopify.streams.base import shopify_error_handling, get_request_timeout -REQUIRED_CONFIG_KEYS = ["shop", "api_key"] +REQUIRED_CONFIG_KEYS = ["shop"] LOGGER = singer.get_logger() SDC_KEYS = {'id': 'integer', 'name': 'string', 'myshopify_domain': 'string'} UNSUPPORTED_FIELDS = {"author"} @shopify_error_handling def initialize_shopify_client(): - api_key = Context.config['api_key'] + api_key = Context.config.get('access_token') or Context.config.get('api_key') shop = Context.config['shop'] version = '2025-07' session = shopify.Session(shop, version, api_key) @@ -33,7 +34,10 @@ def initialize_shopify_client(): shopify.Shop.set_timeout(get_request_timeout()) # Shop.current() makes a call for shop details with provided shop and api_key - return shopify.Shop.current().attributes + try: + return shopify.Shop.current().attributes + except pyactiveresource.connection.UnauthorizedAccess as exc: + raise ShopifyUnauthorizedError(exc, "Invalid access token") from exc # Add helper def fetch_app_scopes(): @@ -219,6 +223,14 @@ def main(): Context.config = args.config Context.state = args.state + if 'client_id' in Context.config: + # Initialize the ShopifyClient for token management (client credentials mode) + Context.client = ShopifyClient( + config_path=args.config_path, + config=Context.config + ) + Context.config['access_token'] = Context.client.access_token + # If discover flag was passed, run discovery mode and dump output to stdout if args.discover: catalog = discover() @@ -245,6 +257,8 @@ def main(): msg = body.get('errors') finally: raise ShopifyError(exc, msg) from exc + except ShopifyUnauthorizedError as error: + raise error except ShopifyError as error: raise error except ShopifyAPIError as error: diff --git a/tap_shopify/client.py b/tap_shopify/client.py new file mode 100644 index 00000000..fadca6e6 --- /dev/null +++ b/tap_shopify/client.py @@ -0,0 +1,110 @@ +import json +import urllib.error +import backoff +import requests +import shopify +import singer +from tap_shopify.streams.base import get_request_timeout +from tap_shopify.exceptions import ShopifyError + +LOGGER = singer.get_logger() + +SHOPIFY_API_VERSION = '2025-07' + +class ShopifyClient: + """ + Handles Shopify authentication via client credentials grant type. + + - Fetches an access token on startup if one is not already present + - Re-fetches the token when the API returns a 401 (token expired or revoked) + - Saves refreshed tokens back to the config file + """ + + def __init__(self, config_path, config): + self.config_path = config_path + self.config = config + self.access_token = config.get('access_token') + + if not self.access_token: + self._refresh_access_token() + + + # pylint: disable=broad-exception-caught + @backoff.on_exception(backoff.expo, + requests.exceptions.RequestException, + max_tries=3, + factor=2) + def _refresh_access_token(self): + """Generate a new access token using client credentials grant type.""" + shop = self.config['shop'] + client_id = self.config['client_id'] + client_secret = self.config['client_secret'] + + token_url = f"https://{shop}.myshopify.com/admin/oauth/access_token" + payload = { + "client_id": client_id, + "client_secret": client_secret, + "grant_type": "client_credentials" + } + + LOGGER.info("Requesting new access token via client credentials grant") + response = requests.post(token_url, json=payload, + headers={"Accept": "application/json"}, + timeout=30) + + if response.status_code != 200: + try: + error_data = response.json() + error_detail = ( + error_data.get('error_description') + or error_data.get('error') + or response.text + ) + except Exception: + error_detail = response.text + + raise ShopifyError( + urllib.error.HTTPError( + url=token_url, + code=response.status_code, + msg=error_detail, + hdrs={}, + fp=None + ), + f"Failed to obtain access token. " + f"Status: {response.status_code}. {error_detail}" + ) + + token_data = response.json() + self.access_token = token_data['access_token'] + + # Update config in memory + self.config['access_token'] = self.access_token + + # Write back to config file + self._write_config() + + def _write_config(self): + """Save updated config (with new token) back to config file.""" + LOGGER.info("Saving credentials back to config") + + with open(self.config_path, 'r', encoding='utf-8') as f: + config = json.load(f) + + config['access_token'] = self.config['access_token'] + + with open(self.config_path, 'w', encoding='utf-8') as f: + json.dump(config, f, indent=2) + + def refresh_token(self): + """Force a token refresh. Called when the API returns a 401 (token expired or revoked).""" + self._refresh_access_token() + + def reinitialize_session(self): + """Reinitialize the Shopify session with the current access token.""" + session = shopify.Session(self.config['shop'], SHOPIFY_API_VERSION, self.access_token) + shopify.ShopifyResource.activate_session(session) + + # set request timeout + shopify.Shop.set_timeout(get_request_timeout()) + LOGGER.info("Shopify session reinitialized with refreshed token") diff --git a/tap_shopify/context.py b/tap_shopify/context.py index dca79f79..a2b72cc0 100644 --- a/tap_shopify/context.py +++ b/tap_shopify/context.py @@ -10,6 +10,7 @@ class Context(): stream_map = {} stream_objects = {} counts = {} + client = None # ShopifyClient instance for token management @classmethod def get_catalog_entry(cls, stream_name): diff --git a/tap_shopify/exceptions.py b/tap_shopify/exceptions.py index 14cbd154..0c244212 100644 --- a/tap_shopify/exceptions.py +++ b/tap_shopify/exceptions.py @@ -2,6 +2,10 @@ class ShopifyError(Exception): def __init__(self, error, msg=''): super().__init__('{}\n{}'.format(error.__class__.__name__, msg)) +class ShopifyUnauthorizedError(Exception): + def __init__(self, error, msg=''): + super().__init__('{}\n{}'.format(error.__class__.__name__, msg)) + class ShopifyAPIError(Exception): """Raised for any unexpected api error without a valid status code""" diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index ada0aa81..d80f9965 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -16,7 +16,7 @@ from graphql import parse, print_ast, visit from graphql.language import Visitor, FieldNode, SelectionSetNode, OperationDefinitionNode, NameNode from tap_shopify.context import Context -from tap_shopify.exceptions import ShopifyError, ShopifyAPIError +from tap_shopify.exceptions import ShopifyError, ShopifyAPIError, ShopifyUnauthorizedError LOGGER = singer.get_logger() @@ -76,6 +76,14 @@ def leaky_bucket_handler(details): LOGGER.info("Received 429 -- sleeping for %s seconds", details['wait']) +def retry_401_handler(_details): + if Context.client: + LOGGER.info("Received 401 Unauthorized - attempting token refresh and retry") + Context.client.refresh_token() + Context.client.reinitialize_session() + else: + LOGGER.info("Received 401 Unauthorized - no client available for token refresh") + def retry_handler(details): LOGGER.info("Received 500 or retryable error -- Retry %s/%s", details['tries'], MAX_RETRIES) @@ -115,6 +123,10 @@ def shopify_error_handling(fnc): giveup=is_not_status_code_fn([404]), on_backoff=retry_handler, max_tries=MAX_RETRIES) + @backoff.on_exception(backoff.expo, + ShopifyUnauthorizedError, + on_backoff=retry_401_handler, + max_tries=2) # Only retry once on 401 for refreshing token @backoff.on_exception(backoff.expo, pyactiveresource.connection.ClientError, giveup=is_not_status_code_fn([429]), @@ -292,6 +304,12 @@ def call_api(self, query_params, query=None, data_key=None): if error_body else http_error.reason ) + if http_error.code == 401: + raise ShopifyUnauthorizedError(http_error, + f"Unauthorized access - token may have expired with status {http_error.code} " + f"and X-Request-ID '{request_id or 'N/A'}', Reason: {error_message}." + ) from http_error + raise ShopifyError(http_error, f"GraphQL request failed for stream '{self.name}' with status {http_error.code} " f"and X-Request-ID '{request_id or 'N/A'}', Reason: {error_message}." diff --git a/tap_shopify/streams/orders.py b/tap_shopify/streams/orders.py index 088b1c48..ca591221 100644 --- a/tap_shopify/streams/orders.py +++ b/tap_shopify/streams/orders.py @@ -1052,7 +1052,10 @@ def submit_bulk_query(self, query_string): url = f"https://{Context.config.get('shop')}.myshopify.com/admin/api/2025-07/graphql.json" headers = { "Content-Type": "application/json", - "X-Shopify-Access-Token": Context.config.get("api_key"), + "X-Shopify-Access-Token": ( + Context.config.get("access_token") + or Context.config.get("api_key") + ), } operation = { "query": """ diff --git a/tests/base.py b/tests/base.py index ef414d2e..c7e6d800 100644 --- a/tests/base.py +++ b/tests/base.py @@ -38,16 +38,15 @@ def tap_name(): def get_type(): """the expected url route ending""" # return "platform.shopify-tba" # moved to alpha Oct 13, 2023 - return "platform.shopify" # new connections after Oct 13th + # return "platform.shopify" # new connections after Oct 13th + return "platform.shopify-byoa" def get_properties(self, original: bool = True): """Configuration properties required for the tap.""" return_value = { 'start_date': '2017-07-01T00:00:00Z', 'shop': 'stitchdatawearhouse', - 'date_window_size': 180, - # BUG: https://jira.talendforge.org/browse/TDL-13180 - # 'results_per_page': '50' + 'date_window_size': 180 } if original: @@ -61,16 +60,12 @@ def get_properties(self, original: bool = True): return return_value @staticmethod - def get_credentials(original_credentials: bool = True): + def get_credentials(original_credentials=True): """Authentication information for the test account""" - if original_credentials: - return { - 'api_key': os.getenv('TAP_SHOPIFY_API_KEY_STITCHDATAWEARHOUSE') - } - return { - 'api_key': os.getenv('TAP_SHOPIFY_API_KEY_TALENDDATAWEARHOUSE') + 'client_id': os.getenv('TAP_SHOPIFY_CLIENT_ID'), + 'client_secret': os.getenv('TAP_SHOPIFY_CLIENT_SECRET') } def expected_metadata(self): diff --git a/tests/test_all_fields.py b/tests/test_all_fields.py index ff1e8179..2c6d1de3 100644 --- a/tests/test_all_fields.py +++ b/tests/test_all_fields.py @@ -32,14 +32,6 @@ def get_properties(self, original: bool = True): return return_value - @staticmethod - def get_credentials(original_credentials: bool = True): - """Authentication information for the test account""" - - return { - 'api_key': os.getenv('TAP_SHOPIFY_API_KEY_TALENDDATAWEARHOUSE') - } - def test_run(self): """ Ensure running the tap with all streams and fields selected results in the diff --git a/tests/test_start_date.py b/tests/test_start_date.py index d2284913..7ecad59b 100644 --- a/tests/test_start_date.py +++ b/tests/test_start_date.py @@ -40,12 +40,6 @@ def get_properties(self, original: bool = True): return_value["start_date"] = '2021-09-15T00:00:00Z' return return_value - @staticmethod - def get_credentials(original_credentials: bool = True): - return { - 'api_key': os.getenv('TAP_SHOPIFY_API_KEY_TALENDDATAWEARHOUSE') - } - @staticmethod def name(): return "tap_tester_shopify_start_date_test" diff --git a/tests/unittests/test_client.py b/tests/unittests/test_client.py new file mode 100644 index 00000000..a24b68b1 --- /dev/null +++ b/tests/unittests/test_client.py @@ -0,0 +1,703 @@ +import json +import os +import tempfile +import unittest +from unittest.mock import patch, MagicMock + +import requests + +from tap_shopify.client import ( + ShopifyClient, + SHOPIFY_API_VERSION, +) +from tap_shopify.context import Context +from tap_shopify.exceptions import ShopifyError, ShopifyUnauthorizedError + +class TestShopifyClientInit(unittest.TestCase): + """Tests for ShopifyClient initialization.""" + + def _make_config(self, **overrides): + base = { + "shop": "test-shop", + "client_id": "cid", + "client_secret": "csecret", + "access_token": "existing_token", + "start_date": "2025-01-01T00:00:00Z", + } + base.update(overrides) + return base + + def _write_config_file(self, config): + """Write config dict to a temp file and return its path.""" + tmp = tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) + json.dump(config, tmp) + tmp.close() + return tmp.name + + @patch('tap_shopify.client.requests.post') + def test_init_uses_existing_token_without_refresh(self, mock_post): + """When an access_token is already in config, no refresh should be triggered.""" + config = self._make_config() # has access_token='existing_token' + path = self._write_config_file(config) + try: + client = ShopifyClient(path, config) + mock_post.assert_not_called() + self.assertEqual(client.access_token, "existing_token") + finally: + os.unlink(path) + + @patch('tap_shopify.client.requests.post') + def test_init_fetches_token_when_missing(self, mock_post): + """First run: no access_token in config, should fetch one via client credentials.""" + config = self._make_config() + del config['access_token'] + path = self._write_config_file(config) + try: + mock_post.return_value = MagicMock( + status_code=200, + json=MagicMock(return_value={"access_token": "fetched_token"}), + ) + client = ShopifyClient(path, config) + mock_post.assert_called_once() + self.assertEqual(client.access_token, "fetched_token") + self.assertEqual(config['access_token'], "fetched_token") + finally: + os.unlink(path) + + + +class TestRefreshAccessToken(unittest.TestCase): + """Tests for ShopifyClient._refresh_access_token.""" + + def _create_client_skip_init(self, config, config_path="/tmp/dummy.json"): + client = object.__new__(ShopifyClient) + client.config = config + client.config_path = config_path + return client + + @patch('tap_shopify.client.requests.post') + def test_successful_refresh(self, mock_post): + """Successful token refresh should update config and access_token.""" + config = { + "shop": "test-shop", + "client_id": "cid", + "client_secret": "csecret", + } + tmp = tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) + json.dump(config, tmp) + tmp.close() + + try: + client = self._create_client_skip_init(config, tmp.name) + mock_post.return_value = MagicMock( + status_code=200, + json=MagicMock(return_value={ + "access_token": "refreshed_token", + "expires_in": 86400, + }), + ) + client._refresh_access_token() + + self.assertEqual(client.access_token, "refreshed_token") + self.assertEqual(config['access_token'], "refreshed_token") + + # Verify the correct endpoint was called (shop + .myshopify.com) + mock_post.assert_called_once_with( + "https://test-shop.myshopify.com/admin/oauth/access_token", + json={ + "client_id": "cid", + "client_secret": "csecret", + "grant_type": "client_credentials", + }, + headers={"Accept": "application/json"}, + timeout=30, + ) + + # Verify config file was updated (access_token only, no expiry fields) + with open(tmp.name, 'r') as f: + saved = json.load(f) + self.assertEqual(saved['access_token'], "refreshed_token") + finally: + os.unlink(tmp.name) + + @patch('tap_shopify.client.requests.post') + def test_refresh_failure_raises(self, mock_post): + """Non-200 response from token endpoint should raise.""" + config = { + "shop": "test-shop", + "client_id": "cid", + "client_secret": "csecret", + } + tmp = tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) + json.dump(config, tmp) + tmp.close() + + try: + client = self._create_client_skip_init(config, tmp.name) + mock_post.return_value = MagicMock( + status_code=401, + text="Unauthorized", + ) + with self.assertRaises(ShopifyError) as ctx: + client._refresh_access_token() + self.assertIn("Failed to obtain access token", str(ctx.exception)) + finally: + os.unlink(tmp.name) + + +class TestWriteConfig(unittest.TestCase): + """Tests for ShopifyClient._write_config.""" + + def test_write_config_updates_file(self): + """Config file should be updated with new access_token (token_expires_at not persisted).""" + original = { + "shop": "test-shop", + "client_id": "cid", + "client_secret": "csecret", + "start_date": "2025-01-01T00:00:00Z", + } + tmp = tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) + json.dump(original, tmp) + tmp.close() + + try: + config = { + **original, + "access_token": "new_token", + } + client = object.__new__(ShopifyClient) + client.config = config + client.config_path = tmp.name + + client._write_config() + + with open(tmp.name, 'r') as f: + saved = json.load(f) + + self.assertEqual(saved['access_token'], "new_token") + self.assertNotIn('token_expires_at', saved) + # Original keys preserved + self.assertEqual(saved['shop'], "test-shop") + self.assertEqual(saved['client_id'], "cid") + finally: + os.unlink(tmp.name) + + def test_write_config_preserves_extra_keys(self): + """Extra keys in the config file should not be removed.""" + original = { + "shop": "test-shop", + "client_id": "cid", + "client_secret": "csecret", + "custom_setting": "keep_me", + } + tmp = tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) + json.dump(original, tmp) + tmp.close() + + try: + config = { + **original, + "access_token": "tok", + } + client = object.__new__(ShopifyClient) + client.config = config + client.config_path = tmp.name + + client._write_config() + + with open(tmp.name, 'r') as f: + saved = json.load(f) + self.assertEqual(saved['custom_setting'], "keep_me") + finally: + os.unlink(tmp.name) + + + +class TestRefreshToken(unittest.TestCase): + """Tests for ShopifyClient.refresh_token (reactive 401 refresh).""" + + @patch.object(ShopifyClient, '_refresh_access_token') + def test_refresh_token_calls_underlying_refresh(self, mock_refresh): + """refresh_token should always call _refresh_access_token.""" + client = object.__new__(ShopifyClient) + client.config = {"shop": "test-shop", "access_token": "tok"} + client.config_path = "/tmp/dummy.json" + client.access_token = "tok" + client.refresh_token() + mock_refresh.assert_called_once() + + +class TestReinitializeSession(unittest.TestCase): + """Tests for ShopifyClient.reinitialize_session.""" + + @patch('tap_shopify.client.shopify.Shop.set_timeout') + @patch('tap_shopify.client.shopify.ShopifyResource.activate_session') + @patch('tap_shopify.client.shopify.Session') + def test_reinitialize_session(self, mock_session_cls, mock_activate, mock_set_timeout): + """reinitialize_session should create a new session, activate it, and set timeout.""" + client = object.__new__(ShopifyClient) + client.config = {"shop": "test-shop"} + client.access_token = "my_token" + client.config_path = "/tmp/dummy.json" + + mock_session_instance = MagicMock() + mock_session_cls.return_value = mock_session_instance + + client.reinitialize_session() + + mock_session_cls.assert_called_once_with( + "test-shop", + SHOPIFY_API_VERSION, + "my_token", + ) + mock_activate.assert_called_once_with(mock_session_instance) + mock_set_timeout.assert_called_once() + + +class TestRetry401Handler(unittest.TestCase): + """Tests for the retry_401_handler function used in backoff decorator.""" + + def setUp(self): + self.original_client = Context.client + + def tearDown(self): + Context.client = self.original_client + + def test_retry_401_handler_refreshes_and_reinitializes(self): + """retry_401_handler should call refresh_token and reinitialize_session.""" + from tap_shopify.streams.base import retry_401_handler + + mock_client = MagicMock() + Context.client = mock_client + + retry_401_handler({'wait': 1, 'tries': 1}) + + mock_client.refresh_token.assert_called_once() + mock_client.reinitialize_session.assert_called_once() + + def test_retry_401_handler_no_client(self): + """retry_401_handler should do nothing if Context.client is None.""" + from tap_shopify.streams.base import retry_401_handler + + Context.client = None + # Should not raise + retry_401_handler({'wait': 1, 'tries': 1}) + + @patch('tap_shopify.client.requests.post') + def test_retry_401_handler_updates_context_config_access_token(self, mock_post): + """After retry_401_handler fires, Context.config['access_token'] must reflect + the newly fetched token. + + This relies on ShopifyClient.config being the *same dict object* as + Context.config (passed by reference in main()), so that + _refresh_access_token()'s `self.config['access_token'] = ...` + is immediately visible via Context.config['access_token']. + """ + import tempfile, os + from tap_shopify.streams.base import retry_401_handler + + shared_config = { + "shop": "test-shop", + "client_id": "cid", + "client_secret": "csecret", + "access_token": "old_token", + "start_date": "2025-01-01T00:00:00Z", + } + + tmp = tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) + json.dump(shared_config, tmp) + tmp.close() + + try: + # Wire up a real ShopifyClient sharing the same config dict as Context + client = object.__new__(ShopifyClient) + client.config = shared_config # same object as Context.config below + client.config_path = tmp.name + client.access_token = "old_token" + + Context.config = shared_config # same dict reference + Context.client = client + + mock_post.return_value = MagicMock( + status_code=200, + json=MagicMock(return_value={"access_token": "new_token"}), + ) + + with patch('shopify.Session'), \ + patch('shopify.ShopifyResource.activate_session'), \ + patch('shopify.Shop.set_timeout'): + retry_401_handler({'wait': 1, 'tries': 1}) + + # The token update in ShopifyClient must be visible through Context.config + self.assertEqual(Context.config['access_token'], "new_token") + self.assertEqual(client.access_token, "new_token") + finally: + os.unlink(tmp.name) + + +class TestCallApiWithTokenRefresh(unittest.TestCase): + """Tests for call_api behaviour regarding token handling.""" + + def setUp(self): + self.original_config = Context.config + self.original_client = Context.client + Context.config = { + "start_date": "2025-01-01T00:00:00Z", + "date_window_size": 30, + } + Context.catalog = { + "streams": [{ + "tap_stream_id": "products", + "schema": {"properties": {"id": {"type": "string"}, "updatedAt": {"type": "string"}}}, + "metadata": [], + }], + } + + def tearDown(self): + Context.config = self.original_config + Context.client = self.original_client + + def _make_stream(self): + from tap_shopify.streams.base import Stream + stream = Stream() + stream.name = "products" + stream.data_key = "products" + return stream + + @patch('shopify.GraphQL') + @patch('tap_shopify.streams.base.Stream.get_query', return_value='{ products { edges { node { id } } } }') + def test_call_api_success(self, mock_get_query, mock_graphql): + """call_api should return data on a successful GraphQL response.""" + Context.client = None + + mock_response = { + "data": {"products": {"edges": [], "pageInfo": {"endCursor": None, "hasNextPage": False}}} + } + mock_graphql.return_value.execute.return_value = json.dumps(mock_response) + + stream = self._make_stream() + result = stream.call_api({"query": "test", "first": 10}) + self.assertEqual(result, mock_response["data"]["products"]) + + @patch('shopify.GraphQL') + @patch('tap_shopify.streams.base.Stream.get_query', return_value='{ products { edges { node { id } } } }') + def test_call_api_without_client(self, mock_get_query, mock_graphql): + """call_api should work even if Context.client is None (backward compat).""" + Context.client = None + + mock_response = { + "data": {"products": {"edges": [], "pageInfo": {"endCursor": None, "hasNextPage": False}}} + } + mock_graphql.return_value.execute.return_value = json.dumps(mock_response) + + stream = self._make_stream() + result = stream.call_api({"query": "test", "first": 10}) + self.assertEqual(result, mock_response["data"]["products"]) + + @patch('shopify.GraphQL') + @patch('tap_shopify.streams.base.Stream.get_query', return_value='{ products { edges { node { id } } } }') + def test_non_401_http_error_raises_shopify_error(self, mock_get_query, mock_graphql): + """Non-401 HTTPError should raise ShopifyError.""" + import urllib.error + from tap_shopify.exceptions import ShopifyError + + Context.client = None + + http_error = urllib.error.HTTPError( + url="https://test-shop.myshopify.com/admin/api/graphql.json", + code=500, + msg="Internal Server Error", + hdrs=MagicMock(**{"get.return_value": "req-456"}), + fp=None, + ) + mock_graphql.return_value.execute.side_effect = http_error + + stream = self._make_stream() + + with self.assertRaises(ShopifyError): + stream.call_api({"query": "test", "first": 10}) + + @patch('shopify.GraphQL') + @patch('tap_shopify.streams.base.Stream.get_query', return_value='{ products { edges { node { id } } } }') + def test_401_http_error_raises_shopify_unauthorized_error(self, mock_get_query, mock_graphql): + """401 HTTPError should raise ShopifyUnauthorizedError (not ShopifyError).""" + import urllib.error + from tap_shopify.exceptions import ShopifyUnauthorizedError + + # Must provide a mock client so the retry_401_handler doesn't blow up + mock_client = MagicMock() + mock_client.refresh_token.return_value = False + Context.client = mock_client + + http_error = urllib.error.HTTPError( + url="https://test-shop.myshopify.com/admin/api/graphql.json", + code=401, + msg="Unauthorized", + hdrs=MagicMock(**{"get.return_value": "req-789"}), + fp=None, + ) + mock_graphql.return_value.execute.side_effect = http_error + + stream = self._make_stream() + + with self.assertRaises(ShopifyUnauthorizedError): + stream.call_api({"query": "test", "first": 10}) + + +class TestMainClient(unittest.TestCase): + """Tests for ShopifyClient wiring in main().""" + + @patch('tap_shopify.ShopifyClient') + @patch('tap_shopify.discover') + @patch('singer.utils.parse_args') + def test_main_stores_client_in_context(self, mock_parse_args, mock_discover, mock_client_cls): + """main() should store the ShopifyClient instance in Context.client.""" + from tap_shopify import main + + mock_args = MagicMock() + mock_args.config = { + "shop": "test-shop", + "client_id": "cid", + "client_secret": "csecret", + "access_token": "tok", + "start_date": "2025-01-01T00:00:00Z", + } + mock_args.state = {} + mock_args.dev = False + mock_args.discover = True + mock_args.config_path = "/tmp/config.json" + mock_args.catalog = None + mock_parse_args.return_value = mock_args + + mock_client_instance = MagicMock() + mock_client_instance.access_token = "tok" + mock_client_cls.return_value = mock_client_instance + mock_discover.return_value = {"streams": []} + + try: + main() + except SystemExit: + pass + + self.assertEqual(Context.client, mock_client_instance) + + @patch('tap_shopify.discover') + @patch('singer.utils.parse_args') + def test_main_no_client_when_api_key_present(self, mock_parse_args, mock_discover): + """main() should NOT create ShopifyClient when api_key is in config (legacy auth).""" + from tap_shopify import main + + mock_args = MagicMock() + mock_args.config = { + "shop": "test-shop", + "api_key": "legacy_key", + "start_date": "2025-01-01T00:00:00Z", + } + mock_args.state = {} + mock_args.discover = True + mock_args.config_path = "/tmp/config.json" + mock_args.catalog = None + mock_parse_args.return_value = mock_args + + mock_discover.return_value = {"streams": []} + + original_client = Context.client + Context.client = None + try: + main() + except SystemExit: + pass + + self.assertIsNone(Context.client) + Context.client = original_client + + @patch('tap_shopify.ShopifyClient') + @patch('tap_shopify.discover') + @patch('singer.utils.parse_args') + def test_main_updates_config_access_token(self, mock_parse_args, mock_discover, mock_client_cls): + """main() should update Context.config['access_token'] from client.access_token.""" + from tap_shopify import main + + mock_args = MagicMock() + mock_args.config = { + "shop": "test-shop", + "client_id": "cid", + "client_secret": "csecret", + "access_token": "old_tok", + "start_date": "2025-01-01T00:00:00Z", + } + mock_args.state = {} + mock_args.dev = False + mock_args.discover = True + mock_args.config_path = "/tmp/config.json" + mock_args.catalog = None + mock_parse_args.return_value = mock_args + + mock_client_instance = MagicMock() + mock_client_instance.access_token = "refreshed_tok" + mock_client_cls.return_value = mock_client_instance + mock_discover.return_value = {"streams": []} + + try: + main() + except SystemExit: + pass + + self.assertEqual(Context.config.get('access_token'), "refreshed_tok") + + def _make_discover_args(self, extra_config=None): + mock_args = MagicMock() + mock_args.config = { + "shop": "test-shop", + "client_id": "cid", + "client_secret": "csecret", + "access_token": "tok", + "start_date": "2025-01-01T00:00:00Z", + } + if extra_config: + mock_args.config.update(extra_config) + mock_args.state = {} + mock_args.dev = False + mock_args.discover = True + mock_args.config_path = "/tmp/config.json" + mock_args.catalog = None + return mock_args + + @patch('tap_shopify.ShopifyClient') + @patch('tap_shopify.discover') + @patch('singer.utils.parse_args') + def test_main_propagates_shopify_unauthorized_error( + self, mock_parse_args, mock_discover, mock_client_cls + ): + """main() must re-raise ShopifyUnauthorizedError as-is, not wrap it in ShopifyError. + + Before the fix, ShopifyUnauthorizedError fell into `except Exception` and was + re-raised as ShopifyError(exc) with an empty message, losing the original context. + After the fix, a dedicated `except ShopifyUnauthorizedError` branch re-raises it + directly so callers receive the correct type and message. + """ + from tap_shopify import main + + mock_parse_args.return_value = self._make_discover_args() + + mock_client_instance = MagicMock() + mock_client_instance.access_token = "tok" + mock_client_cls.return_value = mock_client_instance + + original_error = ShopifyUnauthorizedError( + Exception("UnauthorizedAccess"), "Invalid access token" + ) + mock_discover.side_effect = original_error + + with self.assertRaises(ShopifyUnauthorizedError) as ctx: + main() + + # The exception must be the original instance (not a wrapped ShopifyError) + self.assertIsInstance(ctx.exception, ShopifyUnauthorizedError) + self.assertNotIsInstance(ctx.exception, ShopifyError) + self.assertIn("Invalid access token", str(ctx.exception)) + + @patch('tap_shopify.ShopifyClient') + @patch('tap_shopify.discover') + @patch('singer.utils.parse_args') + def test_main_unauthorized_error_message_preserved( + self, mock_parse_args, mock_discover, mock_client_cls + ): + """ShopifyUnauthorizedError message must survive propagation through main(). + + Previously the message was lost because the error was caught by the generic + `except Exception` handler and re-raised as ShopifyError(exc, msg=''). + """ + from tap_shopify import main + + mock_parse_args.return_value = self._make_discover_args() + + mock_client_instance = MagicMock() + mock_client_instance.access_token = "tok" + mock_client_cls.return_value = mock_client_instance + + expected_message = "Invalid access token" + mock_discover.side_effect = ShopifyUnauthorizedError( + Exception("UnauthorizedAccess"), expected_message + ) + + try: + main() + self.fail("Expected ShopifyUnauthorizedError to be raised") + except ShopifyUnauthorizedError as exc: + self.assertIn(expected_message, str(exc)) + except ShopifyError: + self.fail( + "ShopifyUnauthorizedError was incorrectly wrapped as ShopifyError, " + "losing the original message" + ) + + +class TestBackoffOnRefresh(unittest.TestCase): + """Tests for backoff/retry on token refresh failures.""" + + @patch('tap_shopify.client.requests.post') + def test_refresh_retries_on_connection_error(self, mock_post): + """_refresh_access_token should retry on RequestException.""" + config = { + "shop": "test-shop", + "client_id": "cid", + "client_secret": "csecret", + } + tmp = tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) + json.dump(config, tmp) + tmp.close() + + try: + client = object.__new__(ShopifyClient) + client.config = config + client.config_path = tmp.name + + # Fail twice then succeed + mock_post.side_effect = [ + requests.exceptions.ConnectionError("Connection refused"), + requests.exceptions.ConnectionError("Connection refused"), + MagicMock( + status_code=200, + json=MagicMock(return_value={ + "access_token": "recovered_token", + "expires_in": 86400, + }), + ), + ] + client._refresh_access_token() + self.assertEqual(client.access_token, "recovered_token") + self.assertEqual(mock_post.call_count, 3) # backoff retried twice then succeeded + finally: + os.unlink(tmp.name) + + @patch('tap_shopify.client.requests.post') + def test_refresh_gives_up_after_max_retries(self, mock_post): + """_refresh_access_token should give up after max retries.""" + config = { + "shop": "test-shop", + "client_id": "cid", + "client_secret": "csecret", + } + tmp = tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) + json.dump(config, tmp) + tmp.close() + + try: + client = object.__new__(ShopifyClient) + client.config = config + client.config_path = tmp.name + + mock_post.side_effect = requests.exceptions.ConnectionError("Connection refused") + + with self.assertRaises(requests.exceptions.ConnectionError): + client._refresh_access_token() + + # backoff max_tries=3 + self.assertEqual(mock_post.call_count, 3) + finally: + os.unlink(tmp.name) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/unittests/test_token_check_in_discover.py b/tests/unittests/test_token_check_in_discover.py index 0fe2d981..5bbbd1c2 100644 --- a/tests/unittests/test_token_check_in_discover.py +++ b/tests/unittests/test_token_check_in_discover.py @@ -11,6 +11,7 @@ def __init__(self): self.catalog = False self.config = {'api_key': 'test', 'shop': 'shop'} self.state = False + self.dev = False def resource_not_found_raiser(): raise pyactiveresource.connection.ResourceNotFound From 802fcc501d35ac3b8a00617ec18724fecd9d01d8 Mon Sep 17 00:00:00 2001 From: Sourabh Gandhi <105213416+sgandhi1311@users.noreply.github.com> Date: Thu, 28 May 2026 16:22:52 +0530 Subject: [PATCH 64/66] Filter GraphQL field pruning to top-level record selection sets (#250) * Only remove unselected fields at the top-level stream node * Changed the depth check from == 3 to == 2 * make pylint happy * update filtering logic for all the streams * fix: suppress W0613 false positive in FieldRemover.leave_field Rename unused `node` parameter to `_node` to signal intentional non-use per Python convention. The graphql-core Visitor interface requires the positional argument, but leave_field only needs to pop the field stack. No behavior change. * fix: wire field pruning into OrderShippingLines and bump singer-python to 6.8.0 * upgrade the requests library from 2.32.4 to 2.34.2 --- CHANGELOG.md | 3 + setup.py | 8 +- tap_shopify/streams/base.py | 38 +++- tap_shopify/streams/inventory_levels.py | 5 + tap_shopify/streams/metafields.py | 5 + tap_shopify/streams/order_refunds.py | 5 + tap_shopify/streams/order_shipping_lines.py | 15 +- tap_shopify/streams/transactions.py | 5 + tests/unittests/test_schema_validation.py | 213 ++++++++++++++++++++ 9 files changed, 286 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5306cdd5..893913fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +### 3.11.0 + * Filter GraphQL field pruning to top-level record selection sets [#250](https://github.com/singer-io/tap-shopify/pull/250) + ### 3.10.0 * Add support of client_cred grant type auth [#249](https://github.com/singer-io/tap-shopify/pull/249) * Fix current bookmark value for no data time window [#251](https://github.com/singer-io/tap-shopify/pull/251) diff --git a/setup.py b/setup.py index 203b06a6..50d0275c 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="3.10.0", + version="3.11.0", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", @@ -14,15 +14,15 @@ install_requires=[ # Important: review the monkey-patched method in the GraphQL client when upgrading this dependency. "ShopifyAPI==12.7.0", - "singer-python==6.1.1", + "singer-python==6.8.0", "graphql-core==3.2.6", - 'requests==2.32.4', + 'requests==2.34.2', ], extras_require={ 'dev': [ 'pylint==3.3.6', 'ipdb', - 'requests==2.32.4', + 'requests==2.34.2', 'nose', ] }, diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index d80f9965..631fa758 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -150,6 +150,16 @@ class Stream(): data_key = None results_per_page = None + def _get_record_node_path(self): + """ + Returns the exact tuple of FieldNode names leading from the query root + to the selection set that holds the stream's top-level record fields. + + The default covers the standard ``{ stream { edges { node { FIELDS } } } }`` + shape; subclasses whose records sit at a different depth must override. + """ + return (self.data_key, "edges", "node") + def __init__(self): self.results_per_page = Context.get_results_per_page(RESULTS_PER_PAGE) self.date_window_size = float(Context.config.get("date_window_size") or @@ -232,15 +242,37 @@ def update_bookmark(self, bookmark_value, bookmark_key=None): def remove_fields_from_query(self, fields_to_remove: list) -> str: ast = parse(self.get_query()) used_variable_names = set() + # O(1) membership checks. + fields_to_remove_set = set(fields_to_remove) + # Exact full path from the query root to the record-level selection set. + # Exact matching (not suffix) avoids false positives when a nested + # connection (e.g. lineItems.edges.node) ends with the same field names. + record_path = self._get_record_node_path() class FieldRemover(Visitor): - def enter_selection_set(self, node, _key, _parent, _path, _ancestors): + def __init__(self): + super().__init__() + # Incremental stack of FieldNode names maintained via + # enter_field / leave_field — O(1) push/pop per visit. + self._field_stack = [] + + def enter_field(self, node, *_): + self._field_stack.append(node.name.value) + + def leave_field(self, _node, *_): + self._field_stack.pop() + + def enter_selection_set(self, node, *_): + # Exact match against the full path: only the one selection set + # that corresponds to the stream's record node is pruned. + at_record_node = tuple(self._field_stack) == record_path + new_selections = [] for selection in node.selections: if isinstance(selection, FieldNode): - if selection.name.value in fields_to_remove: + if at_record_node and selection.name.value in fields_to_remove_set: continue - # Check field arguments for variable usage + # Track variable names used in field arguments. for arg in selection.arguments or []: if hasattr(arg.value, "name") and isinstance(arg.value.name, NameNode): used_variable_names.add(arg.value.name.value) diff --git a/tap_shopify/streams/inventory_levels.py b/tap_shopify/streams/inventory_levels.py index eb12bc6d..44a79862 100644 --- a/tap_shopify/streams/inventory_levels.py +++ b/tap_shopify/streams/inventory_levels.py @@ -12,6 +12,11 @@ class InventoryLevels(Stream): child_data_key = "inventoryLevels" replication_key = "updatedAt" + def _get_record_node_path(self): + # Inventory-level records live at + # locations.edges.node.inventoryLevels.edges.node { FIELDS }. + return ("locations", "edges", "node", "inventoryLevels", "edges", "node") + def get_next_page_child(self, parent_id, cursor, child_query): """ Gets all child objects efficiently with pagination. diff --git a/tap_shopify/streams/metafields.py b/tap_shopify/streams/metafields.py index cc830fdf..30665c43 100644 --- a/tap_shopify/streams/metafields.py +++ b/tap_shopify/streams/metafields.py @@ -17,6 +17,11 @@ class Metafields(Stream, ABC): child_data_key = "metafields" replication_key = "updatedAt" + def _get_record_node_path(self): + # Metafield records live at {data_key}.edges.node.metafields.edges.node. + # data_key is defined by each concrete subclass. + return (self.data_key, "edges", "node", "metafields", "edges", "node") + @abstractmethod def get_query(self): """Placeholder for get_query method.""" diff --git a/tap_shopify/streams/order_refunds.py b/tap_shopify/streams/order_refunds.py index 1b11cd16..b4d34106 100644 --- a/tap_shopify/streams/order_refunds.py +++ b/tap_shopify/streams/order_refunds.py @@ -13,6 +13,11 @@ class OrderRefunds(Stream): replication_key = "updatedAt" automatic_keys = ["order"] + def _get_record_node_path(self): + # Refund records live at orders.edges.node.refunds { FIELDS }, + # not at the standard edges.node depth. + return ("orders", "edges", "node", "refunds") + # pylint: disable=too-many-locals def get_objects(self): """ diff --git a/tap_shopify/streams/order_shipping_lines.py b/tap_shopify/streams/order_shipping_lines.py index 6786b35e..8b50b50b 100644 --- a/tap_shopify/streams/order_shipping_lines.py +++ b/tap_shopify/streams/order_shipping_lines.py @@ -12,6 +12,11 @@ class OrderShippingLines(Stream): child_data_key = "shippingLines" replication_key = "updatedAt" + def _get_record_node_path(self): + # Shipping-line records live at + # orders.edges.node.shippingLines.edges.node { FIELDS }. + return ("orders", "edges", "node", "shippingLines", "edges", "node") + # pylint: disable=too-many-locals def get_objects(self): """ @@ -27,6 +32,7 @@ def get_objects(self): last_updated_at = self.get_bookmark() - timedelta(minutes=1) current_bookmark = self.get_bookmark() sync_start = utils.now().replace(microsecond=0) + query = self.remove_fields_from_query(Context.get_unselected_fields(self.name)) # Process each date window while last_updated_at < sync_start: @@ -38,14 +44,14 @@ def get_objects(self): query_params = self.get_query_params(last_updated_at, query_end, cursor) with metrics.http_request_timer(self.name): - data = self.call_api(query_params) + data = self.call_api(query_params, query=query) # Process parent objects and their shippinglines edges = data.get("edges", []) for edge in edges: node = edge.get("node", {}) - for shipping_line in self.paginate_shipping_lines(node): + for shipping_line in self.paginate_shipping_lines(node, query): shipping_line["orderId"] = node["id"].split("/")[-1] shipping_line["updatedAt"] = node["updatedAt"] @@ -66,12 +72,13 @@ def get_objects(self): max_bookmark_value = min(sync_start, current_bookmark) self.update_bookmark(utils.strftime(max_bookmark_value)) - def paginate_shipping_lines(self, data): + def paginate_shipping_lines(self, data, query): """ Transforms the shippingLines data by handling pagination. Args: data (dict): Order data. + query (str): Pruned GraphQL query string. Returns: list: List of shippingLines. @@ -93,7 +100,7 @@ def paginate_shipping_lines(self, data): } # Fetch the next page of data - response = self.call_api(params) + response = self.call_api(params, query=query) node = response.get("edges", [])[0].get("node", {}) shipping_lines_data = node.get("shippingLines") for item in shipping_lines_data["edges"]: diff --git a/tap_shopify/streams/transactions.py b/tap_shopify/streams/transactions.py index fb3de78a..ef59593f 100644 --- a/tap_shopify/streams/transactions.py +++ b/tap_shopify/streams/transactions.py @@ -12,6 +12,11 @@ class Transactions(Stream): child_data_key = "transactions" replication_key = "createdAt" + def _get_record_node_path(self): + # Transaction records live at orders.edges.node.transactions { FIELDS }, + # not at the standard edges.node depth. + return ("orders", "edges", "node", "transactions") + # pylint: disable=W0221 def get_query_params(self, updated_at_min, updated_at_max, cursor=None): """ diff --git a/tests/unittests/test_schema_validation.py b/tests/unittests/test_schema_validation.py index a0b9df6b..ab5384d6 100644 --- a/tests/unittests/test_schema_validation.py +++ b/tests/unittests/test_schema_validation.py @@ -4,6 +4,8 @@ from graphql import parse, FieldNode, SelectionSetNode from pathlib import Path from tap_shopify.streams.orders import Orders +from tap_shopify.streams.transactions import Transactions +from tap_shopify.streams.inventory_levels import InventoryLevels from tap_shopify.context import Context from tap_shopify.streams.base import Stream from textwrap import dedent @@ -101,6 +103,8 @@ def test_get_unselected_fields(self, mock_get_catalog_entry, mock_to_map): def setUp(self): class TestClass(Stream): + data_key = "products" + def get_query(self): return dedent(""" query GetProducts($first: Int!, $after: String, $query: String) { @@ -242,3 +246,212 @@ def test_remove_fields_from_query(self): ''.join(result.split()), ''.join(expected_query.split()) ) + + def test_remove_fields_preserves_nested_same_name_fields(self): + """Pruning a top-level field must not remove a same-named field that + appears inside a nested connection (e.g. lineItems.edges.node). + + This is the regression guard for the original customAttributes bug: + the top-level ``customAttributes`` field on the order record and the + ``customAttributes`` field inside ``lineItems.edges.node`` share a + name. If only the order-level one is unselected it must be removed + from ``orders.edges.node`` but left intact inside lineItems. + """ + class OrderLikeStream(Stream): + data_key = "orders" + + def get_query(self): + return dedent(""" + query GetOrders($first: Int!) { + orders(first: $first) { + edges { + node { + id + customAttributes { + key + value + } + lineItems { + edges { + node { + id + customAttributes { + key + value + } + } + } + } + } + } + } + } + """) + + stream = OrderLikeStream() + result = stream.remove_fields_from_query(["customAttributes"]) + + # Top-level customAttributes should be gone; lineItems and its nested + # customAttributes should remain intact. + parsed = parse(result) + + def get_selection_names(selection_set): + return {s.name.value for s in selection_set.selections if isinstance(s, FieldNode)} + + # Navigate: orders → edges → node + orders_node = parsed.definitions[0].selection_set.selections[0] + edges_node = orders_node.selection_set.selections[0] + node_fields = edges_node.selection_set.selections[0] + top_level_names = get_selection_names(node_fields.selection_set) + + self.assertNotIn( + "customAttributes", top_level_names, + "Top-level customAttributes should be pruned from orders.edges.node" + ) + self.assertIn( + "lineItems", top_level_names, + "lineItems should still be present after pruning" + ) + + # Navigate into lineItems → edges → node + line_items_field = next( + s for s in node_fields.selection_set.selections + if isinstance(s, FieldNode) and s.name.value == "lineItems" + ) + line_item_edges = line_items_field.selection_set.selections[0] + line_item_node = line_item_edges.selection_set.selections[0] + line_item_names = get_selection_names(line_item_node.selection_set) + + self.assertIn( + "customAttributes", line_item_names, + "customAttributes inside lineItems.edges.node must NOT be pruned" + ) + + +class TestRemoveFieldsTransactions(unittest.TestCase): + """remove_fields_from_query correctly prunes fields from the + ``transactions { }`` direct-list pattern used by the Transactions stream + (orders.edges.node.transactions { FIELDS }, no nested edges/node wrapper). + """ + + def setUp(self): + self.stream = Transactions() + + def _parse_transaction_fields(self, query_str): + """Return the set of field names directly inside ``transactions { }``.""" + parsed = parse(query_str) + op = parsed.definitions[0] + # orders → edges → node → transactions + orders = op.selection_set.selections[0] # orders + edges = orders.selection_set.selections[0] # edges + node = edges.selection_set.selections[0] # node + transactions = next( + s for s in node.selection_set.selections + if isinstance(s, FieldNode) and s.name.value == "transactions" + ) + return { + s.name.value + for s in transactions.selection_set.selections + if isinstance(s, FieldNode) + } + + def test_unselected_fields_pruned_from_transactions(self): + """Fields that are unselected should be removed from transactions { }.""" + result = self.stream.remove_fields_from_query(["gateway", "errorCode"]) + remaining = self._parse_transaction_fields(result) + + self.assertNotIn("gateway", remaining) + self.assertNotIn("errorCode", remaining) + + def test_selected_fields_kept_in_transactions(self): + """Fields that ARE selected must remain in transactions { }.""" + result = self.stream.remove_fields_from_query(["gateway"]) + remaining = self._parse_transaction_fields(result) + + self.assertIn("id", remaining) + self.assertIn("status", remaining) + self.assertIn("createdAt", remaining) + + def test_empty_fields_to_remove_leaves_query_intact(self): + """Passing an empty list must not alter the transactions fields.""" + original_fields = self._parse_transaction_fields(self.stream.get_query()) + result = self.stream.remove_fields_from_query([]) + pruned_fields = self._parse_transaction_fields(result) + + self.assertEqual(original_fields, pruned_fields) + + +class TestRemoveFieldsInventoryLevels(unittest.TestCase): + """remove_fields_from_query correctly targets the inner + ``inventoryLevels.edges.node { FIELDS }`` selection set and does NOT + prune from the outer ``locations.edges.node`` selection set. + """ + + def setUp(self): + self.stream = InventoryLevels() + + def _parse_outer_location_fields(self, query_str): + """Return field names directly inside ``locations.edges.node { }``.""" + parsed = parse(query_str) + op = parsed.definitions[0] + locations = op.selection_set.selections[0] # locations + edges = locations.selection_set.selections[0] # edges + node = edges.selection_set.selections[0] # outer node + return { + s.name.value + for s in node.selection_set.selections + if isinstance(s, FieldNode) + } + + def _parse_inner_inventory_level_fields(self, query_str): + """Return field names directly inside ``inventoryLevels.edges.node { }``.""" + parsed = parse(query_str) + op = parsed.definitions[0] + locations = op.selection_set.selections[0] + edges = locations.selection_set.selections[0] + outer_node = edges.selection_set.selections[0] + inventory_levels = next( + s for s in outer_node.selection_set.selections + if isinstance(s, FieldNode) and s.name.value == "inventoryLevels" + ) + inner_edges = next( + s for s in inventory_levels.selection_set.selections + if isinstance(s, FieldNode) and s.name.value == "edges" + ) + inner_node = inner_edges.selection_set.selections[0] + return { + s.name.value + for s in inner_node.selection_set.selections + if isinstance(s, FieldNode) + } + + def test_unselected_fields_pruned_from_inner_node(self): + """Unselected inventory-level fields are removed from the inner node.""" + result = self.stream.remove_fields_from_query(["canDeactivate", "deactivationAlert"]) + inner_fields = self._parse_inner_inventory_level_fields(result) + + self.assertNotIn("canDeactivate", inner_fields) + self.assertNotIn("deactivationAlert", inner_fields) + + def test_outer_location_node_is_not_pruned(self): + """The outer locations.edges.node must never be touched by pruning.""" + original_outer = self._parse_outer_location_fields(self.stream.get_query()) + result = self.stream.remove_fields_from_query(["canDeactivate", "id", "updatedAt"]) + pruned_outer = self._parse_outer_location_fields(result) + + # The outer node must be unchanged regardless of what fields_to_remove contains. + self.assertEqual( + original_outer, pruned_outer, + "Pruning inventory-level schema fields must not affect the outer " + "locations.edges.node selection set" + ) + + def test_selected_fields_kept_in_inner_node(self): + """Fields that ARE selected must remain in the inner inventory-level node.""" + result = self.stream.remove_fields_from_query(["canDeactivate"]) + inner_fields = self._parse_inner_inventory_level_fields(result) + + self.assertIn("id", inner_fields) + self.assertIn("updatedAt", inner_fields) + self.assertIn("item", inner_fields) + From 54c5c736b49e8aaad7e056716d4c77f2f1f1e5f0 Mon Sep 17 00:00:00 2001 From: Sourabh Gandhi <105213416+sgandhi1311@users.noreply.github.com> Date: Mon, 3 Aug 2026 19:52:33 +0530 Subject: [PATCH 65/66] fix: Add token refresh and retry logic for 401 errors during bulk operation polling (#252) * fix(tap-shopify): refresh token and retry on 401 during bulk operation polling * update according to the copilot comments * fix pylint * update setup and changelog * SAC-31774: fix Serialize metafield value to JSON string to prevent target type errors * Revert "SAC-31774: fix Serialize metafield value to JSON string to prevent target type errors" This reverts commit b80f37e57ece4911feccb861d6868784107bf139. --------- Co-authored-by: atttiwari --- CHANGELOG.md | 3 + setup.py | 2 +- tap_shopify/streams/orders.py | 25 +- .../test_orders_poll_bulk_completion.py | 308 ++++++++++++++++++ 4 files changed, 336 insertions(+), 2 deletions(-) create mode 100644 tests/unittests/test_orders_poll_bulk_completion.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 893913fd..26b43d0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +### 3.11.1 + * Add token refresh and retry logic for 401 errors during bulk operation polling [#252](https://github.com/singer-io/tap-shopify/pull/252) + ### 3.11.0 * Filter GraphQL field pruning to top-level record selection sets [#250](https://github.com/singer-io/tap-shopify/pull/250) diff --git a/setup.py b/setup.py index 50d0275c..2f720683 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="3.11.0", + version="3.11.1", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/streams/orders.py b/tap_shopify/streams/orders.py index ca591221..afdf7a1e 100644 --- a/tap_shopify/streams/orders.py +++ b/tap_shopify/streams/orders.py @@ -2,6 +2,7 @@ import json import time import re +import urllib.error import backoff import requests import shopify @@ -1100,7 +1101,29 @@ def fetch_bulk_operation(op_id): }} }} """ - response = json.loads(shopify.GraphQL().execute(query=query)) + # pylint: disable=unexpected-keyword-arg + # execute() is monkey-patched in base.py (execute_gql) to accept timeout + try: + response = json.loads(shopify.GraphQL().execute( + query=query, timeout=self.request_timeout)) + except urllib.error.HTTPError as http_error: + if http_error.code == 401: + LOGGER.warning("Received 401 Unauthorized during bulk operation polling.") + if Context.client: + Context.client.refresh_token() + Context.client.reinitialize_session() + Context.config['access_token'] = Context.client.access_token + else: + raise ShopifyAPIError( + "Received 401 Unauthorized during bulk operation polling " + "but no client is available to refresh the token." + ) from http_error + # Retry once with the refreshed token + response = json.loads(shopify.GraphQL().execute( + query=query, timeout=self.request_timeout)) + else: + raise + # pylint: enable=unexpected-keyword-arg if not isinstance(response, dict): raise ShopifyAPIError(f"Unexpected GraphQL response: {response}") return response.get("data", {}).get("node") diff --git a/tests/unittests/test_orders_poll_bulk_completion.py b/tests/unittests/test_orders_poll_bulk_completion.py new file mode 100644 index 00000000..c0f86fab --- /dev/null +++ b/tests/unittests/test_orders_poll_bulk_completion.py @@ -0,0 +1,308 @@ +""" +Unit tests for the 401 token-refresh-and-retry logic in +Orders.poll_bulk_completion (tap_shopify/streams/orders.py). + +Scenario under test +------------------- +During a long-running GraphQL Bulk Operation poll the Shopify access token can +expire. When that happens shopify.GraphQL().execute() raises a +urllib.error.HTTPError with status 401. The connector must: + + 1. Refresh the access token via Context.client. + 2. Reinitialize the Shopify session so subsequent calls use the new token. + 3. Update Context.config['access_token'] with the refreshed token. + 4. Retry the *same* status-check request (no bulk-operation re-submission). + +If no Context.client is present (non-OAuth flow) a ShopifyAPIError is raised. +Non-401 HTTP errors must propagate unchanged. +""" + +import json +import unittest +import urllib.error +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch + +from tap_shopify.context import Context +from tap_shopify.exceptions import ShopifyAPIError +from tap_shopify.streams.orders import Orders + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_BULK_OP_ID = "gid://shopify/BulkOperation/123" +_RESULT_URL = "https://storage.example.com/bulk_result.jsonl" + + +def _op_response(status, url=_RESULT_URL): + """Build a JSON-encoded fake bulk-operation GraphQL response.""" + return json.dumps({ + "data": { + "node": { + "id": _BULK_OP_ID, + "status": status, + "errorCode": None, + "createdAt": "2026-01-01T00:00:00Z", + "completedAt": "2026-01-01T01:00:00Z" if status == "COMPLETED" else None, + "objectCount": "10", + "fileSize": "1024", + "url": url if status == "COMPLETED" else None, + } + } + }) + + +def _http_error(code): + """Create a urllib.error.HTTPError with the given HTTP status code.""" + return urllib.error.HTTPError( + url="https://test.myshopify.com/admin/api/2025-07/graphql.json", + code=code, + msg="Unauthorized" if code == 401 else "Server Error", + hdrs=MagicMock(get=MagicMock(return_value=None)), + fp=None, + ) + + +def _make_stream(): + """Return an Orders instance with bookmarking helpers pre-mocked.""" + stream = Orders() + stream.date_window_size = 30 + stream.update_bookmark = MagicMock() + stream.clear_bulk_operation_state = MagicMock() + return stream + + +_BOOKMARK = datetime(2026, 1, 1, tzinfo=timezone.utc) + + +def _mock_client(new_token="refreshed_token"): + """Return a mock ShopifyClient whose access_token is *new_token*.""" + client = MagicMock() + client.access_token = new_token + return client + + +# --------------------------------------------------------------------------- +# Test class +# --------------------------------------------------------------------------- + +class TestPollBulkCompletion401Handling(unittest.TestCase): + + def setUp(self): + self._orig_config = Context.config + self._orig_client = Context.client + Context.config = { + "access_token": "original_token", + "start_date": "2025-01-01T00:00:00Z", + } + Context.client = None + + def tearDown(self): + Context.config = self._orig_config + Context.client = self._orig_client + + # ------------------------------------------------------------------ + # Happy-path baseline + # ------------------------------------------------------------------ + + @patch("tap_shopify.streams.orders.time") + @patch("shopify.GraphQL") + def test_completed_status_returns_result_url(self, mock_graphql, mock_time): + """Immediate COMPLETED response returns the bulk result URL.""" + mock_time.time.side_effect = [0, 1] + mock_graphql.return_value.execute.return_value = _op_response("COMPLETED") + + stream = _make_stream() + result = stream.poll_bulk_completion( + current_bookmark=_BOOKMARK, + bulk_op_id=_BULK_OP_ID, + ) + + self.assertEqual(result, _RESULT_URL) + stream.update_bookmark.assert_called_once() + + @patch("tap_shopify.streams.orders.time") + @patch("shopify.GraphQL") + def test_running_then_completed_polls_same_operation(self, mock_graphql, mock_time): + """Connector loops through RUNNING before COMPLETED, calling sleep between iterations.""" + mock_time.time.side_effect = [0, 1, 62] + mock_graphql.return_value.execute.side_effect = [ + _op_response("RUNNING"), + _op_response("COMPLETED"), + ] + + stream = _make_stream() + result = stream.poll_bulk_completion( + current_bookmark=_BOOKMARK, + bulk_op_id=_BULK_OP_ID, + ) + + self.assertEqual(result, _RESULT_URL) + self.assertEqual(mock_graphql.return_value.execute.call_count, 2) + mock_time.sleep.assert_called_once_with(60) + + @patch("tap_shopify.streams.orders.time") + @patch("shopify.GraphQL") + def test_failed_status_raises_shopify_api_error(self, mock_graphql, mock_time): + """A FAILED bulk operation raises ShopifyAPIError and clears state.""" + mock_time.time.side_effect = [0, 1] + mock_graphql.return_value.execute.return_value = _op_response("FAILED") + + stream = _make_stream() + with self.assertRaises(ShopifyAPIError): + stream.poll_bulk_completion( + current_bookmark=_BOOKMARK, + bulk_op_id=_BULK_OP_ID, + ) + + stream.clear_bulk_operation_state.assert_called_once() + + # ------------------------------------------------------------------ + # 401 handling — core feature under test + # ------------------------------------------------------------------ + + @patch("tap_shopify.streams.orders.time") + @patch("shopify.GraphQL") + def test_401_refreshes_token_and_retries_same_bulk_op(self, mock_graphql, mock_time): + """ + On a 401 the connector refreshes the token, reinitializes the session, + updates Context.config, and retries the status check for the SAME + bulk operation — no re-submission of the bulk query. + """ + mock_time.time.side_effect = [0, 1] + mock_graphql.return_value.execute.side_effect = [ + _http_error(401), # first call → 401 + _op_response("COMPLETED"), # retry → success + ] + + client = _mock_client("refreshed_token") + Context.client = client + + stream = _make_stream() + result = stream.poll_bulk_completion( + current_bookmark=_BOOKMARK, + bulk_op_id=_BULK_OP_ID, + ) + + # Token refresh and session reinit were called exactly once + client.refresh_token.assert_called_once() + client.reinitialize_session.assert_called_once() + + # Context.config carries the new token for subsequent calls + self.assertEqual(Context.config["access_token"], "refreshed_token") + + # Result URL returned — polling continued on the existing bulk op + self.assertEqual(result, _RESULT_URL) + + # execute called twice: 401 probe + successful retry + self.assertEqual(mock_graphql.return_value.execute.call_count, 2) + + @patch("tap_shopify.streams.orders.time") + @patch("shopify.GraphQL") + def test_401_mid_poll_loop_continues_polling_after_refresh(self, mock_graphql, mock_time): + """ + 401 can appear in any poll iteration. After refresh the loop continues + and eventually reaches COMPLETED. + """ + mock_time.time.side_effect = [0, 1, 62] + mock_graphql.return_value.execute.side_effect = [ + _op_response("RUNNING"), # first poll — ok + _http_error(401), # second poll — token expired + _op_response("COMPLETED"), # retry of second poll — success + ] + + client = _mock_client("refreshed_token") + Context.client = client + + stream = _make_stream() + result = stream.poll_bulk_completion( + current_bookmark=_BOOKMARK, + bulk_op_id=_BULK_OP_ID, + ) + + client.refresh_token.assert_called_once() + client.reinitialize_session.assert_called_once() + self.assertEqual(Context.config["access_token"], "refreshed_token") + self.assertEqual(result, _RESULT_URL) + self.assertEqual(mock_graphql.return_value.execute.call_count, 3) + mock_time.sleep.assert_called_once_with(60) + + @patch("tap_shopify.streams.orders.time") + @patch("shopify.GraphQL") + def test_401_without_client_raises_shopify_api_error(self, mock_graphql, mock_time): + """ + If Context.client is None (non-OAuth / api_key flow) and a 401 is + received, ShopifyAPIError is raised with a descriptive message. + """ + mock_time.time.side_effect = [0, 1] + mock_graphql.return_value.execute.side_effect = _http_error(401) + Context.client = None # no client available + + stream = _make_stream() + with self.assertRaises(ShopifyAPIError) as ctx: + stream.poll_bulk_completion( + current_bookmark=_BOOKMARK, + bulk_op_id=_BULK_OP_ID, + ) + + self.assertIn("no client is available", str(ctx.exception)) + + @patch("tap_shopify.streams.orders.time") + @patch("shopify.GraphQL") + def test_401_persists_after_retry_propagates_http_error(self, mock_graphql, mock_time): + """ + If the retry after token refresh also returns 401, the HTTPError + propagates to the caller — no infinite retry loop. + """ + mock_time.time.side_effect = [0, 1] + mock_graphql.return_value.execute.side_effect = [ + _http_error(401), # initial call + _http_error(401), # retry — still 401 + ] + + client = _mock_client("refreshed_token") + Context.client = client + + stream = _make_stream() + with self.assertRaises(urllib.error.HTTPError) as ctx: + stream.poll_bulk_completion( + current_bookmark=_BOOKMARK, + bulk_op_id=_BULK_OP_ID, + ) + + self.assertEqual(ctx.exception.code, 401) + # Refresh was still attempted once + client.refresh_token.assert_called_once() + client.reinitialize_session.assert_called_once() + # execute called twice: original + one retry + self.assertEqual(mock_graphql.return_value.execute.call_count, 2) + + # ------------------------------------------------------------------ + # Non-401 HTTP errors + # ------------------------------------------------------------------ + + @patch("tap_shopify.streams.orders.time") + @patch("shopify.GraphQL") + def test_non_401_http_error_propagates_without_token_refresh(self, mock_graphql, mock_time): + """HTTP errors other than 401 (e.g. 500) are re-raised without any token refresh.""" + mock_time.time.side_effect = [0, 1] + mock_graphql.return_value.execute.side_effect = _http_error(500) + + client = _mock_client() + Context.client = client + + stream = _make_stream() + with self.assertRaises(urllib.error.HTTPError) as ctx: + stream.poll_bulk_completion( + current_bookmark=_BOOKMARK, + bulk_op_id=_BULK_OP_ID, + ) + + self.assertEqual(ctx.exception.code, 500) + client.refresh_token.assert_not_called() + client.reinitialize_session.assert_not_called() + + +if __name__ == "__main__": + unittest.main() From 33344b0a28f83ca0c2bf6487769a18f767ed72be Mon Sep 17 00:00:00 2001 From: Atul Tiwari Date: Tue, 4 Aug 2026 20:21:08 +0530 Subject: [PATCH 66/66] fix: Serialize metafield value to JSON string to prevent target type errors (#253) * fix: normalize metafields serialization, schema typing, and user_agent metafields support --------- Co-authored-by: Sourabh Gandhi --- CHANGELOG.md | 4 ++++ setup.py | 2 +- tap_shopify/__init__.py | 11 +++++++++++ tap_shopify/streams/metafields.py | 7 +++++-- 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26b43d0c..4d2539a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +### 3.12.0 + * Re-stringify JSON values in transform_object to prevent SQL error 22P02 for QTC. [#253](https://github.com/singer-io/tap-shopify/pull/253) + * Fix metafield schema SQL type conflicts. + ### 3.11.1 * Add token refresh and retry logic for 401 errors during bulk operation polling [#252](https://github.com/singer-io/tap-shopify/pull/252) diff --git a/setup.py b/setup.py index 2f720683..ab907973 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-shopify", - version="3.11.1", + version="3.12.0", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index 97ca093b..1ff82b34 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -110,6 +110,7 @@ def discover(): raw_schemas = load_schemas() streams = [] + user_agent = Context.config.get("user_agent") for schema_name, schema in raw_schemas.items(): if schema_name not in Context.stream_objects: @@ -118,6 +119,16 @@ def discover(): stream = Context.stream_objects[schema_name]() catalog_schema = add_synthetic_key_to_schema(schema) + # For metafield streams, remove 'integer' and 'object' from the value field's + # type array to avoid SQL type conflicts in targets + if user_agent and schema_name.startswith('metafields_'): + value_prop = catalog_schema.get('properties', {}).get('value', {}) + if isinstance(value_prop.get('type'), list): + value_prop['type'] = [t for t in value_prop['type'] + if t not in ('integer', 'object')] + if 'properties' in value_prop and value_prop['properties'] == {}: + del value_prop['properties'] + # create and add catalog entry catalog_entry = { 'stream': schema_name, diff --git a/tap_shopify/streams/metafields.py b/tap_shopify/streams/metafields.py index 30665c43..e620cd80 100644 --- a/tap_shopify/streams/metafields.py +++ b/tap_shopify/streams/metafields.py @@ -30,15 +30,18 @@ def transform_object(self, obj): """ Transforms a metafield object for output. """ + user_agent = Context.config.get("user_agent") obj["value_type"] = obj.get("type") or None obj["updated_at"] = obj.get("updatedAt") - if obj["value_type"] in ["json", "weight", "volume", "dimension", "rating"]: + if user_agent: + if isinstance(obj.get("value"), (dict, list)): + obj["value"] = json.dumps(obj["value"]) + elif obj["value_type"] in ["json", "weight", "volume", "dimension", "rating"]: value = obj.get("value") try: obj["value"] = json.loads(value) if value is not None else value except json.decoder.JSONDecodeError: LOGGER.info("Failed to decode JSON value for obj %s", obj.get("id")) - obj["value"] = value return obj def fetch_paginated_child_data(self, initial_child_data, parent_id):