From a482da58afbdc54865ace67b49f6dfcf4c240011 Mon Sep 17 00:00:00 2001 From: Tom McKeesick Date: Tue, 2 Jan 2024 15:08:32 +1100 Subject: [PATCH 1/5] update shopify schemas These are lifted from the tap-shopify PR here: https://github.com/singer-io/tap-shopify/pull/178/files --- setup.py | 2 +- tap_shopify/schemas/orders.json | 284 ++++++++++++++++++++------ tap_shopify/schemas/transactions.json | 84 +++++++- 3 files changed, 309 insertions(+), 61 deletions(-) diff --git a/setup.py b/setup.py index fe55a36b..c3934b47 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.6.0", + version="1.6.1", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/schemas/orders.json b/tap_shopify/schemas/orders.json index dc87ea7e..8d4afe8e 100644 --- a/tap_shopify/schemas/orders.json +++ b/tap_shopify/schemas/orders.json @@ -12,6 +12,42 @@ "total_price_set": {}, "total_shipping_price_set": {}, "total_tax_set": {}, + "current_subtotal_price_set": { + "type": [ + "null", + "object" + ] + }, + "current_total_duties_set": { + "type": [ + "null", + "object" + ] + }, + "current_total_discounts_set": { + "type": [ + "null", + "object" + ] + }, + "current_total_price_set": { + "type": [ + "null", + "object" + ] + }, + "current_total_tax_set": { + "type": [ + "null", + "object" + ] + }, + "original_total_duties_set": { + "type": [ + "null", + "object" + ] + }, "total_price": { "type": [ "null", @@ -22,12 +58,6 @@ "line_items": { "$ref": "definitions.json#/line_items" }, - "processing_method": { - "type": [ - "null", - "string" - ] - }, "order_number": { "type": [ "null", @@ -363,7 +393,7 @@ ] }, "customer": { - "$ref": "definitions.json#/customer" + "$ref": "definitions.json#/customer_orders" }, "test": { "type": [ @@ -378,44 +408,6 @@ ], "format": "singer.decimal" }, - "payment_details": { - "properties": { - "avs_result_code": { - "type": [ - "null", - "string" - ] - }, - "credit_card_company": { - "type": [ - "null", - "string" - ] - }, - "cvv_result_code": { - "type": [ - "null", - "string" - ] - }, - "credit_card_bin": { - "type": [ - "null", - "string" - ] - }, - "credit_card_number": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, "number": { "type": [ "null", @@ -538,13 +530,6 @@ "object" ] }, - "total_price_usd": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, "closed_at": { "type": [ "null", @@ -951,12 +936,6 @@ "integer" ] }, - "gateway": { - "type": [ - "null", - "string" - ] - }, "cart_token": { "type": [ "null", @@ -1109,7 +1088,194 @@ "null", "string" ] + }, + "current_subtotal_price": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "current_total_discounts": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "current_total_price": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "current_total_tax": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "total_outstanding": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "payment_terms": { + "type": [ + "null", + "object" + ], + "properties": { + "amount": { + "type": [ + "null", + "integer", + "string" + ] + }, + "currency": { + "type": [ + "null", + "string" + ] + }, + "due_in_days": { + "type": [ + "null", + "integer" + ] + }, + "payment_schedules": { + "type": [ + "null", + "array" + ], + "items": { + "properties": { + "amount": { + "type": [ + "null", + "integer", + "string" + ] + }, + "due_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "currency": { + "type": [ + "null", + "string" + ] + }, + "issued_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "completed_at": { + "type": [ + "null", + "string" + ] + }, + "expected_payment_method": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "payment_terms_name": { + "type": [ + "null", + "string" + ] + }, + "payment_terms_type": { + "type": [ + "null", + "string" + ] + } + } + }, + "estimated_taxes": { + "type": [ + "null", + "boolean" + ] + }, + "merchant_of_record_app_id": { + "type": ["null", "integer"] + }, + "current_total_additional_fees_set": { + "type": ["null", "object"], + "properties": { + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"], + "format": "singer.decimal"}, + "currency": { + "type": ["null", "string"]} + } + }, + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"], + "format": "singer.decimal"}, + "currency": { + "type": ["null", "string"]} + } + } + } + }, + "original_total_additional_fees_set": { + "type": ["null", "object"], + "properties": { + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"], + "format": "singer.decimal"}, + "currency": { + "type": ["null", "string"] + } + } + }, + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"], + "format": "singer.decimal"}, + "currency": { + "type": ["null", "string"]} + } + } + } } }, "type": "object" -} +} \ No newline at end of file diff --git a/tap_shopify/schemas/transactions.json b/tap_shopify/schemas/transactions.json index bd9d321b..f9725819 100644 --- a/tap_shopify/schemas/transactions.json +++ b/tap_shopify/schemas/transactions.json @@ -111,12 +111,36 @@ "string" ] }, + "credit_card_expiration_month": { + "type": [ + "null", + "integer" + ] + }, + "credit_card_expiration_year": { + "type": [ + "null", + "integer" + ] + }, + "credit_card_name": { + "type": [ + "null", + "string" + ] + }, "credit_card_number": { "type": [ "null", "string" ] }, + "credit_card_wallet": { + "type": [ + "null", + "string" + ] + }, "avs_result_code": { "type": [ "null", @@ -176,7 +200,65 @@ "null", "integer" ] + }, + "processed_at": { + "type": [ + "null", + "string" + ] + }, + "payments_refund_attributes": { + "type": [ + "null", + "object" + ], + "properties": { + "status": { + "type": [ + "null", + "string" + ] + }, + "acquirer_reference_number": { + "type": [ + "null", + "string" + ] + } + } + }, + "payment_id": { + "type": [ + "null", + "string" + ] + }, + "total_unsettled_set": { + "type": ["null", "object"], + "properties": { + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"], + "format": "singer.decimal"}, + "currency": { + "type": ["null", "string"] + } + } + }, + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"], + "format": "singer.decimal"}, + "currency": { + "type": ["null", "string"]} + } + } + } } }, "type": "object" -} +} \ No newline at end of file From 7532570cde973c776f25505f0b394d324213e9ce Mon Sep 17 00:00:00 2001 From: Tom McKeesick Date: Tue, 2 Jan 2024 16:38:49 +1100 Subject: [PATCH 2/5] update all schemas from tap-shopify master --- tap_shopify/schemas/abandoned_checkouts.json | 19 + tap_shopify/schemas/definitions.json | 525 ++++++++++++++++++- tap_shopify/schemas/events.json | 23 + tap_shopify/schemas/metafields.json | 6 + tap_shopify/schemas/order_refunds.json | 115 +++- tap_shopify/schemas/orders.json | 2 +- tap_shopify/schemas/products.json | 6 + tap_shopify/schemas/transactions.json | 2 +- 8 files changed, 694 insertions(+), 4 deletions(-) diff --git a/tap_shopify/schemas/abandoned_checkouts.json b/tap_shopify/schemas/abandoned_checkouts.json index ee59cfdd..d1d0e52d 100644 --- a/tap_shopify/schemas/abandoned_checkouts.json +++ b/tap_shopify/schemas/abandoned_checkouts.json @@ -591,6 +591,25 @@ "null", "boolean" ] + }, + "buyer_accepts_sms_marketing": { + "type": [ + "null", + "boolean" + ] + }, + "sms_marketing_phone": { + "type": [ + "null", + "string" + ] + }, + "total_duties": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" } } } diff --git a/tap_shopify/schemas/definitions.json b/tap_shopify/schemas/definitions.json index f2c8fe13..f8e080ff 100644 --- a/tap_shopify/schemas/definitions.json +++ b/tap_shopify/schemas/definitions.json @@ -2,6 +2,18 @@ "order_adjustments": { "items": { "properties": { + "amount_set": { + "type": [ + "null", + "object" + ] + }, + "tax_amount_set": { + "type": [ + "null", + "object" + ] + }, "order_id": { "type": [ "null", @@ -424,6 +436,508 @@ "string" ], "format": "date-time" + }, + "tax_exemptions": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "string" + ] + } + }, + "marketing_opt_in_level": { + "type": [ + "null", + "string" + ] + }, + "email_marketing_consent": { + "type": [ + "null", + "object" + ], + "properties": { + "state": { + "type": [ + "null", + "string" + ] + }, + "opt_in_level": { + "type": [ + "null", + "string" + ] + }, + "consent_updated_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + } + } + }, + "sms_marketing_consent": { + "type": [ + "null", + "object" + ], + "properties": { + "state": { + "type": [ + "null", + "string" + ] + }, + "opt_in_level": { + "type": [ + "null", + "string" + ] + }, + "consent_updated_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "consent_collected_from": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "customer_orders": { + "type": [ + "null", + "object" + ], + "properties": { + "currency": { + "type": [ + "null", + "string" + ] + }, + "email": { + "type": [ + "null", + "string" + ] + }, + "multipass_identifier": { + "type": [ + "null", + "string" + ] + }, + "default_address": { + "type": [ + "null", + "object" + ], + "properties": { + "city": { + "type": [ + "null", + "string" + ] + }, + "address1": { + "type": [ + "null", + "string" + ] + }, + "zip": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "integer" + ] + }, + "country_name": { + "type": [ + "null", + "string" + ] + }, + "province": { + "type": [ + "null", + "string" + ] + }, + "phone": { + "type": [ + "null", + "string" + ] + }, + "country": { + "type": [ + "null", + "string" + ] + }, + "first_name": { + "type": [ + "null", + "string" + ] + }, + "customer_id": { + "type": [ + "null", + "integer" + ] + }, + "default": { + "type": [ + "null", + "boolean" + ] + }, + "last_name": { + "type": [ + "null", + "string" + ] + }, + "country_code": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "province_code": { + "type": [ + "null", + "string" + ] + }, + "address2": { + "type": [ + "null", + "string" + ] + }, + "company": { + "type": [ + "null", + "string" + ] + } + } + }, + "state": { + "type": [ + "null", + "string" + ] + }, + "verified_email": { + "type": [ + "null", + "boolean" + ] + }, + "first_name": { + "type": [ + "null", + "string" + ] + }, + "updated_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "note": { + "type": [ + "null", + "string" + ] + }, + "phone": { + "type": [ + "null", + "string" + ] + }, + "admin_graphql_api_id": { + "type": [ + "null", + "string" + ] + }, + "addresses": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "city": { + "type": [ + "null", + "string" + ] + }, + "address1": { + "type": [ + "null", + "string" + ] + }, + "zip": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "integer" + ] + }, + "country_name": { + "type": [ + "null", + "string" + ] + }, + "province": { + "type": [ + "null", + "string" + ] + }, + "phone": { + "type": [ + "null", + "string" + ] + }, + "country": { + "type": [ + "null", + "string" + ] + }, + "first_name": { + "type": [ + "null", + "string" + ] + }, + "customer_id": { + "type": [ + "null", + "integer" + ] + }, + "default": { + "type": [ + "null", + "boolean" + ] + }, + "last_name": { + "type": [ + "null", + "string" + ] + }, + "country_code": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "province_code": { + "type": [ + "null", + "string" + ] + }, + "address2": { + "type": [ + "null", + "string" + ] + }, + "company": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "last_name": { + "type": [ + "null", + "string" + ] + }, + "tags": { + "type": [ + "null", + "string" + ] + }, + "tax_exempt": { + "type": [ + "null", + "boolean" + ] + }, + "id": { + "type": [ + "null", + "integer" + ] + }, + "accepts_marketing": { + "type": [ + "null", + "boolean" + ] + }, + "accepts_marketing_updated_at": { + "anyOf": [ + { + "type": "string" , + "format": "date-time" + }, + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "created_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "tax_exemptions": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "string" + ] + } + }, + "marketing_opt_in_level": { + "type": [ + "null", + "string" + ] + }, + "email_marketing_consent": { + "type": [ + "null", + "object" + ], + "properties": { + "state": { + "type": [ + "null", + "string" + ] + }, + "opt_in_level": { + "type": [ + "null", + "string" + ] + }, + "consent_updated_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + } + } + }, + "sms_marketing_consent": { + "type": [ + "null", + "object" + ], + "properties": { + "state": { + "type": [ + "null", + "string" + ] + }, + "opt_in_level": { + "type": [ + "null", + "string" + ] + }, + "consent_updated_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "consent_collected_from": { + "type": [ + "null", + "string" + ] + } + } } } }, @@ -806,8 +1320,10 @@ "null", "integer" ] - } + }, + "duties": {} }, + "patternProperties": {".+": {}}, "type": [ "null", "object" @@ -946,8 +1462,15 @@ "null", "string" ] + }, + "channel_liable": { + "type": [ + "null", + "boolean" + ] } }, + "patternProperties": {".+": {}}, "type": [ "null", "object" diff --git a/tap_shopify/schemas/events.json b/tap_shopify/schemas/events.json index fc52b5a2..e829fe3c 100644 --- a/tap_shopify/schemas/events.json +++ b/tap_shopify/schemas/events.json @@ -55,6 +55,29 @@ "null", "string" ] + }, + "description": { + "type": [ + "null", + "string" + ] + }, + "arguments": { + "anyOf": [ + { + "type": [ + "null", + "array" + ], + "items": {} + }, + { + "type": [ + "null", + "string" + ] + } + ] } } } diff --git a/tap_shopify/schemas/metafields.json b/tap_shopify/schemas/metafields.json index ba597731..6a2a73b6 100644 --- a/tap_shopify/schemas/metafields.json +++ b/tap_shopify/schemas/metafields.json @@ -70,6 +70,12 @@ "string" ], "format": "date-time" + }, + "type": { + "type": [ + "null", + "string" + ] } }, "type": "object" diff --git a/tap_shopify/schemas/order_refunds.json b/tap_shopify/schemas/order_refunds.json index d05a153d..d170e05f 100644 --- a/tap_shopify/schemas/order_refunds.json +++ b/tap_shopify/schemas/order_refunds.json @@ -271,6 +271,7 @@ ] } }, + "patternProperties": {".+": {}}, "type": [ "null", "object" @@ -642,6 +643,7 @@ ] } }, + "patternProperties": {".+": {}}, "type": [ "null", "object" @@ -665,6 +667,117 @@ "object" ] } - } + }, + "transactions": { + "type": ["null", "array"], + "items": { + "properties": { + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"], + "format": "singer.decimal" + }, + "authorization": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "error_code": { + "type": ["null", "string"] + }, + "device_id": { + "type": ["null", "integer"] + }, + "gateway": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "kind": { + "type": ["null", "string"] + }, + "location_id": { + "type": ["null", "integer"] + }, + "message": { + "type": ["null", "string"] + }, + "order_id": { + "type": ["null", "integer"] + }, + "parent_id": { + "type": ["null", "integer"] + }, + "payment_details": { + "properties": { + "cvv_result_code": { + "type": ["null", "string"] + }, + "credit_card_bin": { + "type": ["null", "string"] + }, + "credit_card_company": { + "type": ["null", "string"] + }, + "credit_card_number": { + "type": ["null", "string"] + }, + "avs_result_code": { + "type": ["null", "string"] + }, + "payments_refund_attributes": { + "type": ["null", "string"] + }, + "acquirer_reference_number": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "processed_at": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "test": { + "type": ["null", "boolean"] + }, + "source_name": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "receipt": {}, + "payments_refund_attributes": { + "properties": { + "status": { + "type": ["null", "string"] + }, + "acquirer_reference_number": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "processed_at": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "duties": {} } } diff --git a/tap_shopify/schemas/orders.json b/tap_shopify/schemas/orders.json index 8d4afe8e..26b386ac 100644 --- a/tap_shopify/schemas/orders.json +++ b/tap_shopify/schemas/orders.json @@ -1278,4 +1278,4 @@ } }, "type": "object" -} \ No newline at end of file +} diff --git a/tap_shopify/schemas/products.json b/tap_shopify/schemas/products.json index 09f9a4ae..34b8b3b7 100644 --- a/tap_shopify/schemas/products.json +++ b/tap_shopify/schemas/products.json @@ -303,6 +303,12 @@ "null", "string" ] + }, + "product_id": { + "type": [ + "null", + "integer" + ] } }, "type": [ diff --git a/tap_shopify/schemas/transactions.json b/tap_shopify/schemas/transactions.json index f9725819..24bfc2fd 100644 --- a/tap_shopify/schemas/transactions.json +++ b/tap_shopify/schemas/transactions.json @@ -261,4 +261,4 @@ } }, "type": "object" -} \ No newline at end of file +} From 2b9fb82219546b5e64ef315fd1ca809b613dd56b Mon Sep 17 00:00:00 2001 From: Tom McKeesick Date: Wed, 3 Jan 2024 10:55:55 +1100 Subject: [PATCH 3/5] update streams/base.py to tap-shopify master --- tap_shopify/streams/base.py | 89 ++++++++++++++++++++++++++++++++++--- 1 file changed, 82 insertions(+), 7 deletions(-) diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index 60a039c5..74a73b47 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -2,17 +2,17 @@ import functools import math import sys -import http - import socket +from urllib.error import URLError +import http import backoff import pyactiveresource import pyactiveresource.formats import simplejson import singer from singer import metrics, utils +from singer.utils import strptime_to_utc from tap_shopify.context import Context -import urllib LOGGER = singer.get_logger() @@ -34,6 +34,47 @@ # If not timeout is specified/can be parsed from the response, then wait for this amount DEFAULT_WAIT = 20 +# We have observed transactions with receipt objects that contain both: +# - `token` and `Token` +# - `version` and `Version` +# - `ack` and `Ack` +# keys on transactions where PayPal is the payment type. We reached out to +# PayPal support and they told us the values should be the same, so one +# can be safely ignored since its a duplicate. Example: The logic is to +# prefer `token` if both are present and equal, convert `Token` -> `token` +# if only `Token` is present, and throw an error if both are present and +# their values are not equal. +def canonicalize(transaction_dict, field_name): + field_name_upper = field_name.capitalize() + # Not all Shopify transactions have receipts. Facebook has been shown + # to push a null receipt through the transaction + receipt = transaction_dict.get('receipt', {}) + if receipt: + value_lower = receipt.get(field_name) + value_upper = receipt.get(field_name_upper) + if value_lower and value_upper: + if value_lower == value_upper: + LOGGER.info(( + "Transaction (id=%d) contains a receipt " + "that has `%s` and `%s` keys with the same " + "value. Removing the `%s` key."), + transaction_dict['id'], + field_name, + field_name_upper, + field_name_upper) + transaction_dict['receipt'].pop(field_name_upper) + else: + raise ValueError(( + "Found Transaction (id={}) with a receipt that has " + "`{}` and `{}` keys with the different " + "values. Contact Shopify/PayPal support.").format( + transaction_dict['id'], + field_name_upper, + field_name)) + elif value_upper: + # pylint: disable=line-too-long + transaction_dict["receipt"][field_name] = transaction_dict['receipt'].pop(field_name_upper) + # function to return request timeout def get_request_timeout(): @@ -86,6 +127,10 @@ def is_timeout_error(error_raised): return True def shopify_error_handling(fnc): + @backoff.on_exception(backoff.expo, + (http.client.IncompleteRead, ConnectionResetError), + max_tries=MAX_RETRIES, + factor=2) @backoff.on_exception(backoff.expo, # timeout error raise by Shopify (pyactiveresource.connection.Error, socket.timeout), giveup=is_timeout_error, @@ -99,13 +144,20 @@ def shopify_error_handling(fnc): http.client.IncompleteRead, ConnectionResetError, TimeoutError, - urllib.error.URLError, + URLError ), + giveup=is_not_status_code_fn(range(500, 599)), on_backoff=retry_handler, max_tries=MAX_RETRIES, factor=FACTOR) + @backoff.on_exception(backoff.expo, + pyactiveresource.connection.ResourceNotFound, + giveup=is_not_status_code_fn([404]), + on_backoff=retry_handler, + max_tries=MAX_RETRIES) @backoff.on_exception(retry_after_wait_gen, pyactiveresource.connection.ClientError, + giveup=is_not_status_code_fn([429]), on_backoff=leaky_bucket_handler, max_tries=MAX_RETRIES, # No jitter as we want a constant value @@ -154,6 +206,10 @@ def get_since_id(self): self.name, 'since_id') + def get_updated_at_max(self): + updated_at_max = Context.state.get('bookmarks', {}).get(self.name, {}).get('updated_at_max') + return utils.strptime_with_tz(updated_at_max) if updated_at_max else None + def update_bookmark(self, bookmark_value, bookmark_key=None): # NOTE: Bookmarking can never be updated to not get the most # recent thing it saw the next time you run, because the querying @@ -187,7 +243,9 @@ def get_query_params(self, since_id, status_key, updated_at_min, updated_at_max) } def get_objects(self): + last_sync_interrupted_at = self.get_updated_at_max() updated_at_min = self.get_bookmark() + max_bookmark = updated_at_min stop_time = singer.utils.now().replace(microsecond=0) date_window_size = float(Context.config.get("date_window_size", DATE_WINDOW_SIZE)) @@ -205,7 +263,14 @@ def get_objects(self): # think it has something to do with how the API treats # microseconds on its date windows. Maybe it's possible to # drop data due to rounding errors or something like that? - updated_at_max = updated_at_min + datetime.timedelta(days=date_window_size) + # If last sync was interrupted, set updated_at_max to + # updated_at_max bookmarked in the interrupted sync. + # This will make sure that records with lower id than since_id + # which got updated later won't be missed + updated_at_max = (last_sync_interrupted_at + or updated_at_min + datetime.timedelta(days=date_window_size)) + last_sync_interrupted_at = None + if updated_at_max > stop_time: updated_at_max = stop_time while True: @@ -225,6 +290,9 @@ def get_objects(self): # since_id parameter. raise OutOfOrderIdsError("obj.id < since_id: {} < {}".format( obj.id, since_id)) + replication_value = strptime_to_utc(getattr(obj, self.replication_key)) + if replication_value > max_bookmark: + max_bookmark = replication_value yield obj # You know you're at the end when the current page has @@ -233,7 +301,9 @@ def get_objects(self): # Save the updated_at_max as our bookmark as we've synced all rows up in our # window and can move forward. Also remove the since_id because we want to # restart at 1. - Context.state.get('bookmarks', {}).get(self.name, {}).pop('since_id', None) + stream_bookmarks = Context.state.get('bookmarks', {}).get(self.name, {}) + stream_bookmarks.pop('since_id', None) + stream_bookmarks.pop('updated_at_max', None) self.update_bookmark(utils.strftime(updated_at_max)) break @@ -245,10 +315,15 @@ def get_objects(self): objects[-1].id, max([o.id for o in objects]))) since_id = objects[-1].id - # Put since_id into the state. + # Put since_id and updated_at_max into the state. self.update_bookmark(since_id, bookmark_key='since_id') + self.update_bookmark(utils.strftime(updated_at_max), bookmark_key='updated_at_max') updated_at_min = updated_at_max + bookmark = max(min(stop_time, + max_bookmark), + (stop_time - datetime.timedelta(days=date_window_size))) + self.update_bookmark(utils.strftime(bookmark)) def sync(self): """Yield's processed SDK object dicts to the caller. From 6f979a903d43ba5cfc6488874178dbc9097c4e5d Mon Sep 17 00:00:00 2001 From: Tom McKeesick Date: Wed, 3 Jan 2024 10:57:23 +1100 Subject: [PATCH 4/5] update remaining py files to tap-shopify master --- tap_shopify/streams/locations.py | 16 ++++++++- tap_shopify/streams/metafields.py | 10 ++++-- tap_shopify/streams/order_refunds.py | 6 +++- tap_shopify/streams/transactions.py | 50 ++-------------------------- 4 files changed, 31 insertions(+), 51 deletions(-) diff --git a/tap_shopify/streams/locations.py b/tap_shopify/streams/locations.py index 77e3765c..c3bd3c6c 100644 --- a/tap_shopify/streams/locations.py +++ b/tap_shopify/streams/locations.py @@ -20,7 +20,21 @@ def get_locations_data(self): yield from location_page def sync(self): + bookmark = self.get_bookmark() + max_bookmark = bookmark + for location in self.get_locations_data(): - yield location.to_dict() + + location_dict = location.to_dict() + replication_value = utils.strptime_to_utc(location_dict[self.replication_key]) + + if replication_value >= bookmark: + yield location_dict + + # update max bookmark if "replication_value" of current location is greater + if replication_value > max_bookmark: + max_bookmark = replication_value + + self.update_bookmark(utils.strftime(max_bookmark)) Context.stream_objects['locations'] = Locations diff --git a/tap_shopify/streams/metafields.py b/tap_shopify/streams/metafields.py index 8dbd253b..755a2e2c 100644 --- a/tap_shopify/streams/metafields.py +++ b/tap_shopify/streams/metafields.py @@ -62,8 +62,14 @@ def sync(self): # Shop metafields for metafield in self.get_objects(): metafield = metafield.to_dict() - value_type = metafield.get("value_type") - if value_type and value_type == "json_string": + metafield_type = metafield.get("type") + # create "value_type" field in the record + metafield["value_type"] = metafield_type + # the json_string value in "value_type" field will be + # mapped to following "type" value in the new version + # Reference: https://shopify.dev/apps/metafields/types + if metafield_type and metafield_type in ["json", "weight", "volume", \ + "dimension", "rating"]: value = metafield.get("value") try: metafield["value"] = json.loads(value) if value is not None else value diff --git a/tap_shopify/streams/order_refunds.py b/tap_shopify/streams/order_refunds.py index 2b168146..23e13c8a 100644 --- a/tap_shopify/streams/order_refunds.py +++ b/tap_shopify/streams/order_refunds.py @@ -3,7 +3,8 @@ from tap_shopify.context import Context from tap_shopify.streams.base import (Stream, shopify_error_handling, - OutOfOrderIdsError) + OutOfOrderIdsError, + canonicalize) class OrderRefunds(Stream): name = 'order_refunds' @@ -48,6 +49,9 @@ def sync(self): refund_dict = refund.to_dict() replication_value = strptime_to_utc(refund_dict[self.replication_key]) if replication_value >= bookmark: + for transaction_dict in refund_dict.get("transactions",[]): + for field_name in ['token', 'version', 'ack', 'timestamp', 'build']: + canonicalize(transaction_dict, field_name) yield refund_dict if replication_value > max_bookmark: diff --git a/tap_shopify/streams/transactions.py b/tap_shopify/streams/transactions.py index 2a259c18..6615a7cd 100644 --- a/tap_shopify/streams/transactions.py +++ b/tap_shopify/streams/transactions.py @@ -3,7 +3,8 @@ from singer.utils import strftime, strptime_to_utc from tap_shopify.context import Context from tap_shopify.streams.base import (Stream, - shopify_error_handling) + shopify_error_handling, + canonicalize) LOGGER = singer.get_logger() @@ -11,47 +12,6 @@ # order can have no more than 100 transactions associated with it. TRANSACTIONS_RESULTS_PER_PAGE = 100 -# We have observed transactions with receipt objects that contain both: -# - `token` and `Token` -# - `version` and `Version` -# - `ack` and `Ack` -# keys transactions where PayPal is the payment type. We reached out to -# PayPal support and they told us the values should be the same, so one -# can be safely ignored since its a duplicate. Example: The logic is to -# prefer `token` if both are present and equal, convert `Token` -> `token` -# if only `Token` is present, and throw an error if both are present and -# their values are not equal -def canonicalize(transaction_dict, field_name): - field_name_upper = field_name.capitalize() - # Not all Shopify transactions have receipts. Facebook has been shown - # to push a null receipt through the transaction - receipt = transaction_dict.get('receipt', {}) - if receipt: - value_lower = receipt.get(field_name) - value_upper = receipt.get(field_name_upper) - if value_lower and value_upper: - if value_lower == value_upper: - LOGGER.info(( - "Transaction (id=%d) contains a receipt " - "that has `%s` and `%s` keys with the same " - "value. Removing the `%s` key."), - transaction_dict['id'], - field_name, - field_name_upper, - field_name_upper) - transaction_dict['receipt'].pop(field_name_upper) - else: - raise ValueError(( - "Found Transaction (id={}) with a receipt that has " - "`{}` and `{}` keys with the different " - "values. Contact Shopify/PayPal support.").format( - transaction_dict['id'], - field_name_upper, - field_name)) - elif value_upper: - # pylint: disable=line-too-long - transaction_dict["receipt"][field_name] = transaction_dict['receipt'].pop(field_name_upper) - class Transactions(Stream): name = 'transactions' @@ -71,10 +31,6 @@ def call_api_for_transactions(self, parent_object): order_id=parent_object.id, ) - @shopify_error_handling - def get_next_page(self, page): - return page.next_page() - def get_transactions(self, parent_object): # We do not need to support paging on this substream. If that # were to become untrue, reference Metafields. @@ -89,7 +45,7 @@ def get_transactions(self, parent_object): yield from page while page.has_next_page(): - page = self.get_next_page(page) + page = page.next_page() yield from page def get_objects(self): From a881b6bd151b770adb834fa787f26bc92ccf0a51 Mon Sep 17 00:00:00 2001 From: Tom McKeesick Date: Wed, 3 Jan 2024 11:40:08 +1100 Subject: [PATCH 5/5] update __init__.py to tap-shopify master --- tap_shopify/__init__.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index 930904a3..64803e03 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -137,6 +137,12 @@ def sync(): shop_attributes = initialize_shopify_client() sdc_fields = {"_sdc_shop_" + x: shop_attributes[x] for x in SDC_KEYS} + # If there is a currently syncing stream bookmark, shuffle the + # stream order so it gets sync'd first + currently_sync_stream_name = Context.state.get('bookmarks', {}).get('currently_sync_stream') + if currently_sync_stream_name: + shuffle_streams(currently_sync_stream_name) + # Emit all schemas first so we have them for child streams for stream in Context.catalog["streams"]: if Context.is_selected(stream["tap_stream_id"]): @@ -146,12 +152,6 @@ def sync(): bookmark_properties=stream["replication_key"]) Context.counts[stream["tap_stream_id"]] = 0 - # If there is a currently syncing stream bookmark, shuffle the - # stream order so it gets sync'd first - currently_sync_stream_name = Context.state.get('bookmarks', {}).get('currently_sync_stream') - if currently_sync_stream_name: - shuffle_streams(currently_sync_stream_name) - # Loop over streams in catalog for catalog_entry in Context.catalog['streams']: stream_id = catalog_entry['tap_stream_id'] @@ -167,7 +167,8 @@ def sync(): Context.state['bookmarks'] = {} Context.state['bookmarks']['currently_sync_stream'] = stream_id - with Transformer() as transformer: + # some fields have epoch-time as date, hence transform into UTC date + with Transformer(singer.UNIX_SECONDS_INTEGER_DATETIME_PARSING) as transformer: for rec in stream.sync(): extraction_time = singer.utils.now() record_schema = catalog_entry['schema']