diff --git a/docs/index.asciidoc b/docs/index.asciidoc index dcf3414..2c3005c 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -41,7 +41,7 @@ that will be used in the SOQL query. ==== HTTP proxy -If your infrastructure uses a HTTP proxy, you can set the `SALESFORCE_PROXY_URI` environment variable with the desired URI value (e.g `export SALESFORCE_PROXY_URI="http://proxy.example.com:123"`). +If your infrastructure uses an HTTP proxy, you can set the `SALESFORCE_PROXY_URI` environment variable with the desired URI value (e.g `export SALESFORCE_PROXY_URI="http://proxy.example.com:123"`). ==== Example This example prints all the Salesforce Opportunities to standard out @@ -75,6 +75,7 @@ This plugin supports the following configuration options plus the <> |<>|No +| <> |<>|No | <> |<>|Yes | <> |<>|Yes | <> |<>|No @@ -86,6 +87,8 @@ This plugin supports the following configuration options plus the <> |<>|Yes | <> |<>|No | <> |<>|No +| <> |<>|No +| <> |<>|No | <> |<>|No | <> |<>|No | <> |<>|Yes @@ -103,7 +106,29 @@ input plugins. * There is no default value for this setting. By default, this uses the default Restforce API version. -To override this, set this to something like "32.0" for example +To override this, set this to something like "32.0" for example. + +[id="plugins-{type}s-{plugin}-changed_data_filter"] +===== `changed_data_filter` + +* Value type is <> + * There is no default value for this setting. + +The filter to add to the Salesforce query when a previous tracking field value +was read from the <>. +The string can (and should) contain a placeholder `%+{last_tracking_field_value}+` that +will be substituted with the actual value read from the <>. + +This clause is combined with any <> +clause that is configured using the `AND` operator. + +The value should be properly quoted according to the SOQL rules for the field +type. + +**Examples:** + + "changed_data_filter" => "Number > '%{last_tracking_field_value}'" + "changed_data_filter" => "LastModifiedDate >= %{last_tracking_field_value}" [id="plugins-{type}s-{plugin}-client_id"] ===== `client_id` @@ -112,10 +137,10 @@ To override this, set this to something like "32.0" for example * Value type is <> * There is no default value for this setting. -Consumer Key for authentication. You must set up a new SFDC -connected app with oath to use this output. More information +Consumer Key for authentication. You must set up a new Salesforce +connected app with OAuth enabled to use this plugin. More information can be found here: -https://help.salesforce.com/apex/HTViewHelpDoc?id=connected_app_create.htm +https://help.salesforce.com/apex/HTViewHelpDoc?id=connected_app_create.htm. [id="plugins-{type}s-{plugin}-client_secret"] ===== `client_secret` @@ -124,7 +149,7 @@ https://help.salesforce.com/apex/HTViewHelpDoc?id=connected_app_create.htm * Value type is <> * There is no default value for this setting. -Consumer Secret from your oauth enabled connected app +Consumer secret from your OAuth enabled connected app. [id="plugins-{type}s-{plugin}-interval"] ===== `interval` @@ -150,21 +175,21 @@ If this property is not specified or is set to -1, the plugin will run once and * Value type is <> * There is no default value for this setting. -The password used to login to sfdc +The password used to log in to Salesforce. [id="plugins-{type}s-{plugin}-security_token"] -===== `security_token` +===== `security_token` - * This is a required setting. - * Value type is <> - * There is no default value for this setting. +* This is a required setting. +* Value type is <> +* There is no default value for this setting. The security token for this account. For more information about -generting a security token, see: -https://help.salesforce.com/apex/HTViewHelpDoc?id=user_security_token.htm +generating a security token, see: +https://help.salesforce.com/apex/HTViewHelpDoc?id=user_security_token.htm. [id="plugins-{type}s-{plugin}-sfdc_fields"] -===== `sfdc_fields` +===== `sfdc_fields` * Value type is <> * Default value is `[]` @@ -178,9 +203,9 @@ If this is empty, all fields are returned. * Value type is <> * Default value is `""` -These options will be added to the WHERE clause in the +These options will be added to the `WHERE` clause in the SOQL statement. Additional fields can be filtered on by -adding field1 = value1 AND field2 = value2 AND... +adding `field1 = value1 AND field2 = value2 AND...`. [id="plugins-{type}s-{plugin}-sfdc_instance_url"] ===== `sfdc_instance_url` @@ -202,7 +227,7 @@ but not both to configure the url to which the plugin connects to. * Value type is <> * There is no default value for this setting. -The name of the salesforce object you are creating or updating +The name of the Salesforce object you are creating or updating. [id="plugins-{type}s-{plugin}-timeout"] ===== `timeout` @@ -221,7 +246,59 @@ read, an error occurs. * Value type is <> * Default value is `false` -Setting this to true will convert SFDC's NamedFields__c to named_fields__c +Setting this to true will convert Salesforce's `++NamedFields__c++` to `++named_fields__c++`. + +[id="plugins-{type}s-{plugin}-tracking_field"] +===== `tracking_field` + + * Value type is <> + * There is no default value for this setting. + +The field to track for incremental data loads. This field will +be used in an `ORDER BY ... ASC` clause that is added to the Salesforce query. +This field _should_ also be used in the <> clause +to actually achieve incremental loading of data. + +The last value (which is the highest value if the query sorts by this field ascending) +value for this field will be saved to the file at the path configured by +<>, if specified. + +This field should ideally be strictly ascending for new records. An +autonumber field is ideal for this. + +The standard `LastModifiedDate` field can be used, but since it is not _strictly_ +ascending (multiple records can have the same `LastModifiedDate`, the +<> should account for this by using the `>=` +operator, and duplicates should be expected. + +Note that Salesforce does not guarantee that the standard `Id` field has ascending +values for new records (https://developer.salesforce.com/docs/atlas.en-us.apexcode.meta/apexcode/apex_testing_best_practices.htm). +Therefore, using `Id` as tracking field risks missing records and is not recommended. + +If this field is not already included in the <>, +it is added. + +[id="plugins-{type}s-{plugin}-tracking_field_value_file"] +===== `tracking_field_value_file` + +* Value type is <> +* There is no default value for this setting. + +The full path to the file from which the latest tracking field value from the previous +plugin invocation will be read, and to which the new latest tracking field value will be +written after the current plugin invocation. + +This keeps persistent track of the last seen value of the tracking field used for incremental +loading of data. + +The file should be readable and writable by the Logstash process. + +If the file exists and a <> is configured, +a changed data filter clause is added to the query (and combined with any <> +clause that is configured using the `AND` operator). + +If the result set is not empty, the value for `tracking_field` from the last row is +written to the file. [id="plugins-{type}s-{plugin}-use_test_sandbox"] ===== `use_test_sandbox` @@ -230,7 +307,7 @@ Setting this to true will convert SFDC's NamedFields__c to named_fields__c * Default value is `false` Set this to true to connect to a sandbox sfdc instance -logging in through test.salesforce.com +logging in through test.salesforce.com. Use either this or the `sfdc_instance_url` configuration option but not both to configure the url to which the plugin connects to. @@ -255,9 +332,9 @@ of elements of sfdc flows) and security health check risks. * Value type is <> * There is no default value for this setting. -A valid salesforce user name, usually your email address. +A valid Salesforce username, usually your email address. Used for authentication and will be the user all objects -are created or modified by +are created or modified by. diff --git a/lib/logstash/inputs/salesforce.rb b/lib/logstash/inputs/salesforce.rb index d0337f6..94377b5 100644 --- a/lib/logstash/inputs/salesforce.rb +++ b/lib/logstash/inputs/salesforce.rb @@ -111,6 +111,25 @@ class LogStash::Inputs::Salesforce < LogStash::Inputs::Base # Setting this to true will convert SFDC's NamedFields__c to named_fields__c config :to_underscores, :validate => :boolean, :default => false + # File that stores the tracking field's latest value. This is read before querying data to interpolate + # the tracking field value into the incremental_filter, and the latest value of the tracking field is written + # to it after all the query results have been read. + config :tracking_field_value_file, :validate => :string, :required => false + + # Filter clause to use for incremental retrieval and indexing of data that has changed since the last invodation + # of the plugin. This is combined with sfdc_filters using the AND operator, if tracking_field_value_path exists. + # String interpolation is applied to replace "%{last_tracking_field_value}" in this string with the value read + # from tracking_field_value_file. This would usually be something like "tracking_field > '%{last_tracking_field_value}'" + # where tracking_field is the API name of the actual tracking field set using the tracking_field configuration property, + # e.g. LastModifiedDate + config :changed_data_filter, :validate => :string, :required => false + + # The field from which the last value will be stored in the tracking_field_value_file and interpolated + # for "%{last_tracking_field_value}" in the changed_data_filter expression. This field will also be used in an ORDER BY + # clause added to the query, with sorting done ascending, so that the last value in the results is also the + # highest. + config :tracking_field, :validate => :string, :required => false + # Interval to run the command. Value is in seconds. If no interval is given, # this plugin only fetches data once. config :interval, :validate => :number, :required => false, :default => -1 @@ -128,6 +147,7 @@ def run(queue) while !stop? start = Time.now results = client.query(get_query()) + latest_tracking_field_value = nil if results && results.first results.each do |result| event = LogStash::Event.new() @@ -136,7 +156,7 @@ def run(queue) field_type = @sfdc_field_types[field] value = result.send(field) event_key = @to_underscores ? underscore(field) : field - if not value.nil? + unless value.nil? case field_type when 'datetime', 'date' event.set(event_key, format_time(value)) @@ -146,8 +166,20 @@ def run(queue) end end queue << event + unless @tracking_field.nil? + latest_tracking_field_value = result[@tracking_field] + end + end # loop sObjects + end + + unless @tracking_field_value_file.nil? + unless latest_tracking_field_value.nil? + @logger.debug("Writing latest tracking field value " + latest_tracking_field_value + " to " + @tracking_field_value_file) + File.write(@tracking_field_value_file, latest_tracking_field_value) + else + @logger.debug("No tracking field value found in result, not updating " + @tracking_field_value_file) end - end # loop sObjects + end if @interval == -1 break @@ -199,15 +231,43 @@ def client_options private def get_query() - query = ["SELECT", @sfdc_fields.join(','), + sfdc_fields = @sfdc_fields.dup + unless @tracking_field.nil? + unless sfdc_fields.include?(@tracking_field) + sfdc_fields << [@tracking_field] + end + end + query = ["SELECT", sfdc_fields.join(','), "FROM", @sfdc_object_name] - query << ["WHERE", @sfdc_filters] unless @sfdc_filters.empty? - query << "ORDER BY LastModifiedDate DESC" if @sfdc_fields.include?('LastModifiedDate') + where = [] + unless @sfdc_filters.empty? + append_to_where_clause(@sfdc_filters, where) + end + unless @changed_data_filter.nil? + if File.exist?(@tracking_field_value_file) + last_tracking_field_value = File.read(@tracking_field_value_file) + changed_data_filter_interpolated = @changed_data_filter % { :last_tracking_field_value => last_tracking_field_value } + append_to_where_clause(changed_data_filter_interpolated, where) + end + end + query << where + unless @tracking_field.nil? + query << ["ORDER BY", @tracking_field, "ASC"] + end query_str = query.flatten.join(" ") @logger.debug? && @logger.debug("SFDC Query", :query => query_str) return query_str end + def append_to_where_clause(changed_data_filter_interpolated, where) + if where.empty? + where << ["WHERE"] + else + where << ["AND"] + end + where << [changed_data_filter_interpolated] + end + private def get_field_types(obj_desc) field_types = {} diff --git a/spec/fixtures/vcr_cassettes/load_some_lead_objects_order_by_lastmodifieddate.yml b/spec/fixtures/vcr_cassettes/load_some_lead_objects_order_by_lastmodifieddate.yml new file mode 100644 index 0000000..9a5a67c --- /dev/null +++ b/spec/fixtures/vcr_cassettes/load_some_lead_objects_order_by_lastmodifieddate.yml @@ -0,0 +1,124 @@ +--- +http_interactions: +- request: + method: post + uri: https://login.salesforce.com/services/oauth2/token + body: + encoding: US-ASCII + string: grant_type=password&client_id=xxxx&client_secret=xxxx&username=xxxx&password=xxxx + headers: + User-Agent: + - Faraday v0.9.1 + Content-Type: + - application/x-www-form-urlencoded + Accept: + - '*/*' + response: + status: + code: 200 + message: OK + headers: + Date: + - Thu, 27 Aug 2015 23:57:42 GMT + Set-Cookie: + - BrowserId=xxxx;Path=/;Domain=.salesforce.com;Expires=Mon, 26-Oct-2015 23:57:42 GMT + Expires: + - Thu, 01 Jan 1970 00:00:00 GMT + Pragma: + - no-cache + Cache-Control: + - no-cache, no-store + Content-Type: + - application/json;charset=UTF-8 + Transfer-Encoding: + - chunked + body: + encoding: US-ASCII + string: '{"id":"https://login.salesforce.com/id/xxxx/xxxx","issued_at":"1440719862904","token_type":"Bearer","instance_url":"https://eu2.salesforce.com","signature":"xxxx","access_token":"xxxx"}' + http_version: + recorded_at: Thu, 27 Aug 2015 23:57:43 GMT +- request: + method: get + uri: https://eu2.salesforce.com/services/data/v26.0/sobjects/Lead/describe + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Faraday v0.9.1 + Authorization: + - OAuth xxxx + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - '*/*' + response: + status: + code: 200 + message: OK + headers: + Date: + - Thu, 27 Aug 2015 23:57:44 GMT + Set-Cookie: + - BrowserId=xxxx;Path=/;Domain=.salesforce.com;Expires=Mon, 26-Oct-2015 23:57:44 GMT + Expires: + - Thu, 01 Jan 1970 00:00:00 GMT + Sforce-Limit-Info: + - api-usage=211068/451000 + Org.eclipse.jetty.server.include.etag: + - 5fb54cb6 + Last-Modified: + - Thu, 27 Aug 2015 22:36:55 GMT + Content-Type: + - application/json;charset=UTF-8 + Etag: + - 5fb54cb-gzip" + Transfer-Encoding: + - chunked + body: + encoding: UTF-8 + string: '{"activateable":false,"childRelationships":[],"createable":true,"custom":false,"customSetting":false,"deletable":true,"deprecatedAndHidden":false,"feedEnabled":true,"fields":[{"autoNumber":false,"byteLength":18,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"externalId":false,"filterable":true,"groupable":true,"htmlFormatted":false,"idLookup":true,"inlineHelpText":null,"label":"Lead + ID","length":18,"name":"Id","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"precision":0,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"soapType":"tns:ID","sortable":true,"type":"id","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"externalId":false,"filterable":true,"groupable":true,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Deleted","length":0,"name":"IsDeleted","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"precision":0,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"soapType":"xsd:boolean","sortable":true,"type":"boolean","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"autoNumber":false,"byteLength":240,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"externalId":false,"filterable":true,"groupable":true,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Last + Name","length":80,"name":"LastName","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"precision":0,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"soapType":"xsd:string","sortable":true,"type":"string","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"autoNumber":false,"byteLength":120,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"externalId":false,"filterable":true,"groupable":true,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"First + Name","length":40,"name":"FirstName","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"precision":0,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"soapType":"xsd:string","sortable":true,"type":"string","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"autoNumber":false,"byteLength":120,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"externalId":false,"filterable":true,"groupable":true,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Salutation","length":40,"name":"Salutation","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[{"active":true,"defaultValue":false,"label":"Mr.","validFor":null,"value":"Mr."},{"active":true,"defaultValue":false,"label":"Ms.","validFor":null,"value":"Ms."},{"active":true,"defaultValue":false,"label":"Mrs.","validFor":null,"value":"Mrs."},{"active":true,"defaultValue":false,"label":"Dr.","validFor":null,"value":"Dr."},{"active":true,"defaultValue":false,"label":"Prof.","validFor":null,"value":"Prof."}],"precision":0,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"soapType":"xsd:string","sortable":true,"type":"picklist","unique":false,"updateable":true,"writeRequiresMasterRead":false}],"keyPrefix":"00Q","label":"Lead","labelPlural":"Leads","layoutable":true,"listviewable":null,"lookupLayoutable":null,"mergeable":true,"name":"Lead","queryable":true,"recordTypeInfos":[{"available":true,"defaultRecordTypeMapping":true,"name":"Marketing","recordTypeId":"xxxx"},{"available":true,"defaultRecordTypeMapping":false,"name":"Partner + Deal","recordTypeId":"xxxx"},{"available":true,"defaultRecordTypeMapping":false,"name":"Sales","recordTypeId":"xxxx"},{"available":true,"defaultRecordTypeMapping":false,"name":"Master","recordTypeId":"xxxx"}],"replicateable":true,"retrieveable":true,"searchLayoutable":null,"searchable":true,"triggerable":true,"undeletable":true,"updateable":true,"urls":{"uiEditTemplate":"https://xxx.salesforce.com/{ID}/e","sobject":"/services/data/v26.0/sobjects/Lead","uiDetailTemplate":"https://xxx.salesforce.com/{ID}","describe":"/services/data/v26.0/sobjects/Lead/describe","rowTemplate":"/services/data/v26.0/sobjects/Lead/{ID}","uiNewRecord":"https://xxx.salesforce.com/00Q/e"}}' + http_version: + recorded_at: Thu, 27 Aug 2015 23:57:44 GMT +- request: + method: get + uri: https://eu2.salesforce.com/services/data/v26.0/query?q=SELECT%20Id,IsDeleted,LastName,FirstName,Salutation,LastModifiedDate%20FROM%20Lead%20WHERE%20Email%20LIKE%20%27%25@elastic.co%27%20ORDER%20BY%20LastModifiedDate%20ASC + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Faraday v0.9.1 + Authorization: + - OAuth xxx + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - '*/*' + response: + status: + code: 200 + message: OK + headers: + Date: + - Thu, 27 Aug 2015 23:57:45 GMT + Set-Cookie: + - BrowserId=xxxx;Path=/;Domain=.salesforce.com;Expires=Mon, 26-Oct-2015 23:57:45 GMT + Expires: + - Thu, 01 Jan 1970 00:00:00 GMT + Sforce-Limit-Info: + - api-usage=211063/451000 + Content-Type: + - application/json;charset=UTF-8 + Transfer-Encoding: + - chunked + body: + encoding: UTF-8 + string: '{"totalSize":3,"done":true,"records":[{"attributes":{"type":"Lead","url":"/services/data/v26.0/sobjects/Lead/xxxx"},"Id":"xxxx","IsDeleted":false,"LastName":"Katz","FirstName":"Aaron","Salutation":"Mr.","LastModifiedDate":"2025-05-03T10:07:54Z"},{"attributes":{"type":"Lead","url":"/services/data/v26.0/sobjects/Lead/xxxx"},"Id":"xxxx","IsDeleted":false,"LastName":"Grand","FirstName":"Adrien","Salutation":"Dr.","LastModifiedDate":"2025-05-03T13:52:09Z"},{"attributes":{"type":"Lead","url":"/services/data/v26.0/sobjects/Lead/xxxx"},"Id":"xxxx","IsDeleted":false,"LastName":"Hardy","FirstName":"Alan","Salutation":"Overlord","LastModifiedDate":"2025-05-07T14:32:17Z"}]}' + http_version: + recorded_at: Thu, 27 Aug 2015 23:57:45 GMT +recorded_with: VCR 2.9.3 diff --git a/spec/fixtures/vcr_cassettes/load_some_lead_objects_with_lastmodifieddate_filter.yml b/spec/fixtures/vcr_cassettes/load_some_lead_objects_with_lastmodifieddate_filter.yml new file mode 100644 index 0000000..7ee31eb --- /dev/null +++ b/spec/fixtures/vcr_cassettes/load_some_lead_objects_with_lastmodifieddate_filter.yml @@ -0,0 +1,123 @@ +--- +http_interactions: +- request: + method: post + uri: https://login.salesforce.com/services/oauth2/token + body: + encoding: US-ASCII + string: grant_type=password&client_id=xxxx&client_secret=xxxx&username=xxxx&password=xxxx + headers: + User-Agent: + - Faraday v0.9.1 + Content-Type: + - application/x-www-form-urlencoded + Accept: + - '*/*' + response: + status: + code: 200 + message: OK + headers: + Date: + - Thu, 27 Aug 2015 23:57:42 GMT + Set-Cookie: + - BrowserId=xxxx;Path=/;Domain=.salesforce.com;Expires=Mon, 26-Oct-2015 23:57:42 GMT + Expires: + - Thu, 01 Jan 1970 00:00:00 GMT + Pragma: + - no-cache + Cache-Control: + - no-cache, no-store + Content-Type: + - application/json;charset=UTF-8 + Transfer-Encoding: + - chunked + body: + encoding: US-ASCII + string: '{"id":"https://login.salesforce.com/id/xxxx/xxxx","issued_at":"1440719862904","token_type":"Bearer","instance_url":"https://eu2.salesforce.com","signature":"xxxx","access_token":"xxxx"}' + http_version: + recorded_at: Thu, 27 Aug 2015 23:57:43 GMT +- request: + method: get + uri: https://eu2.salesforce.com/services/data/v26.0/sobjects/Lead/describe + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Faraday v0.9.1 + Authorization: + - OAuth xxxx + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - '*/*' + response: + status: + code: 200 + message: OK + headers: + Date: + - Thu, 27 Aug 2015 23:57:44 GMT + Set-Cookie: + - BrowserId=xxxx;Path=/;Domain=.salesforce.com;Expires=Mon, 26-Oct-2015 23:57:44 GMT + Expires: + - Thu, 01 Jan 1970 00:00:00 GMT + Sforce-Limit-Info: + - api-usage=211068/451000 + Org.eclipse.jetty.server.include.etag: + - 5fb54cb6 + Last-Modified: + - Thu, 27 Aug 2015 22:36:55 GMT + Content-Type: + - application/json;charset=UTF-8 + Etag: + - 5fb54cb-gzip" + Transfer-Encoding: + - chunked + body: + encoding: UTF-8 + string: '{"activateable":false,"childRelationships":[],"createable":true,"custom":false,"customSetting":false,"deletable":true,"deprecatedAndHidden":false,"feedEnabled":true,"fields":[{"autoNumber":false,"byteLength":18,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"externalId":false,"filterable":true,"groupable":true,"htmlFormatted":false,"idLookup":true,"inlineHelpText":null,"label":"Lead + ID","length":18,"name":"Id","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"precision":0,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"soapType":"tns:ID","sortable":true,"type":"id","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"externalId":false,"filterable":true,"groupable":true,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Deleted","length":0,"name":"IsDeleted","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"precision":0,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"soapType":"xsd:boolean","sortable":true,"type":"boolean","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"autoNumber":false,"byteLength":240,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"externalId":false,"filterable":true,"groupable":true,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Last + Name","length":80,"name":"LastName","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"precision":0,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"soapType":"xsd:string","sortable":true,"type":"string","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"autoNumber":false,"byteLength":120,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"externalId":false,"filterable":true,"groupable":true,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"First + Name","length":40,"name":"FirstName","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"precision":0,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"soapType":"xsd:string","sortable":true,"type":"string","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"autoNumber":false,"byteLength":120,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"externalId":false,"filterable":true,"groupable":true,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Salutation","length":40,"name":"Salutation","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[{"active":true,"defaultValue":false,"label":"Mr.","validFor":null,"value":"Mr."},{"active":true,"defaultValue":false,"label":"Ms.","validFor":null,"value":"Ms."},{"active":true,"defaultValue":false,"label":"Mrs.","validFor":null,"value":"Mrs."},{"active":true,"defaultValue":false,"label":"Dr.","validFor":null,"value":"Dr."},{"active":true,"defaultValue":false,"label":"Prof.","validFor":null,"value":"Prof."}],"precision":0,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"soapType":"xsd:string","sortable":true,"type":"picklist","unique":false,"updateable":true,"writeRequiresMasterRead":false}],"keyPrefix":"00Q","label":"Lead","labelPlural":"Leads","layoutable":true,"listviewable":null,"lookupLayoutable":null,"mergeable":true,"name":"Lead","queryable":true,"recordTypeInfos":[{"available":true,"defaultRecordTypeMapping":true,"name":"Marketing","recordTypeId":"xxxx"},{"available":true,"defaultRecordTypeMapping":false,"name":"Partner + Deal","recordTypeId":"xxxx"},{"available":true,"defaultRecordTypeMapping":false,"name":"Sales","recordTypeId":"xxxx"},{"available":true,"defaultRecordTypeMapping":false,"name":"Master","recordTypeId":"xxxx"}],"replicateable":true,"retrieveable":true,"searchLayoutable":null,"searchable":true,"triggerable":true,"undeletable":true,"updateable":true,"urls":{"uiEditTemplate":"https://xxx.salesforce.com/{ID}/e","sobject":"/services/data/v26.0/sobjects/Lead","uiDetailTemplate":"https://xxx.salesforce.com/{ID}","describe":"/services/data/v26.0/sobjects/Lead/describe","rowTemplate":"/services/data/v26.0/sobjects/Lead/{ID}","uiNewRecord":"https://xxx.salesforce.com/00Q/e"}}' + http_version: + recorded_at: Thu, 27 Aug 2015 23:57:44 GMT +- request: + method: get + uri: https://eu2.salesforce.com/services/data/v26.0/query?q=SELECT%20Id,IsDeleted,LastName,FirstName,Salutation%20FROM%20Lead%20WHERE%20Email%20LIKE%20%27%25@elastic.co%27%20AND%20LastModifiedDate%20%3E%202025-05-07T14:32:17Z + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Faraday v0.9.1 + Authorization: + - OAuth xxx + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - '*/*' + response: + status: + code: 200 + message: OK + headers: + Date: + - Thu, 27 Aug 2015 23:57:45 GMT + Set-Cookie: + - BrowserId=xxxx;Path=/;Domain=.salesforce.com;Expires=Mon, 26-Oct-2015 23:57:45 GMT + Expires: + - Thu, 01 Jan 1970 00:00:00 GMT + Sforce-Limit-Info: + - api-usage=211063/451000 + Content-Type: + - application/json;charset=UTF-8 + Transfer-Encoding: + - chunked + body: + encoding: UTF-8 + string: '{"totalSize":3,"done":true,"records":[{"attributes":{"type":"Lead","url":"/services/data/v26.0/sobjects/Lead/xxxx"},"Id":"xxxx","IsDeleted":false,"LastName":"Katz","FirstName":"Aaron","Salutation":"Mr."},{"attributes":{"type":"Lead","url":"/services/data/v26.0/sobjects/Lead/xxxx"},"Id":"xxxx","IsDeleted":false,"LastName":"Grand","FirstName":"Adrien","Salutation":"Dr."},{"attributes":{"type":"Lead","url":"/services/data/v26.0/sobjects/Lead/xxxx"},"Id":"xxxx","IsDeleted":false,"LastName":"Hardy","FirstName":"Alan","Salutation":"Overlord"}]}' + http_version: + recorded_at: Thu, 27 Aug 2015 23:57:45 GMT diff --git a/spec/fixtures/vcr_cassettes/load_some_lead_objects_with_lastmodifieddate_filter_order_by_lastmodifieddate.yml b/spec/fixtures/vcr_cassettes/load_some_lead_objects_with_lastmodifieddate_filter_order_by_lastmodifieddate.yml new file mode 100644 index 0000000..c378f7c --- /dev/null +++ b/spec/fixtures/vcr_cassettes/load_some_lead_objects_with_lastmodifieddate_filter_order_by_lastmodifieddate.yml @@ -0,0 +1,123 @@ +--- +http_interactions: +- request: + method: post + uri: https://login.salesforce.com/services/oauth2/token + body: + encoding: US-ASCII + string: grant_type=password&client_id=xxxx&client_secret=xxxx&username=xxxx&password=xxxx + headers: + User-Agent: + - Faraday v0.9.1 + Content-Type: + - application/x-www-form-urlencoded + Accept: + - '*/*' + response: + status: + code: 200 + message: OK + headers: + Date: + - Thu, 27 Aug 2015 23:57:42 GMT + Set-Cookie: + - BrowserId=xxxx;Path=/;Domain=.salesforce.com;Expires=Mon, 26-Oct-2015 23:57:42 GMT + Expires: + - Thu, 01 Jan 1970 00:00:00 GMT + Pragma: + - no-cache + Cache-Control: + - no-cache, no-store + Content-Type: + - application/json;charset=UTF-8 + Transfer-Encoding: + - chunked + body: + encoding: US-ASCII + string: '{"id":"https://login.salesforce.com/id/xxxx/xxxx","issued_at":"1440719862904","token_type":"Bearer","instance_url":"https://eu2.salesforce.com","signature":"xxxx","access_token":"xxxx"}' + http_version: + recorded_at: Thu, 27 Aug 2015 23:57:43 GMT +- request: + method: get + uri: https://eu2.salesforce.com/services/data/v26.0/sobjects/Lead/describe + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Faraday v0.9.1 + Authorization: + - OAuth xxxx + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - '*/*' + response: + status: + code: 200 + message: OK + headers: + Date: + - Thu, 27 Aug 2015 23:57:44 GMT + Set-Cookie: + - BrowserId=xxxx;Path=/;Domain=.salesforce.com;Expires=Mon, 26-Oct-2015 23:57:44 GMT + Expires: + - Thu, 01 Jan 1970 00:00:00 GMT + Sforce-Limit-Info: + - api-usage=211068/451000 + Org.eclipse.jetty.server.include.etag: + - 5fb54cb6 + Last-Modified: + - Thu, 27 Aug 2015 22:36:55 GMT + Content-Type: + - application/json;charset=UTF-8 + Etag: + - 5fb54cb-gzip" + Transfer-Encoding: + - chunked + body: + encoding: UTF-8 + string: '{"activateable":false,"childRelationships":[],"createable":true,"custom":false,"customSetting":false,"deletable":true,"deprecatedAndHidden":false,"feedEnabled":true,"fields":[{"autoNumber":false,"byteLength":18,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"externalId":false,"filterable":true,"groupable":true,"htmlFormatted":false,"idLookup":true,"inlineHelpText":null,"label":"Lead + ID","length":18,"name":"Id","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"precision":0,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"soapType":"tns:ID","sortable":true,"type":"id","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"externalId":false,"filterable":true,"groupable":true,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Deleted","length":0,"name":"IsDeleted","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"precision":0,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"soapType":"xsd:boolean","sortable":true,"type":"boolean","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"autoNumber":false,"byteLength":240,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"externalId":false,"filterable":true,"groupable":true,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Last + Name","length":80,"name":"LastName","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"precision":0,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"soapType":"xsd:string","sortable":true,"type":"string","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"autoNumber":false,"byteLength":120,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"externalId":false,"filterable":true,"groupable":true,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"First + Name","length":40,"name":"FirstName","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"precision":0,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"soapType":"xsd:string","sortable":true,"type":"string","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"autoNumber":false,"byteLength":120,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"externalId":false,"filterable":true,"groupable":true,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Salutation","length":40,"name":"Salutation","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[{"active":true,"defaultValue":false,"label":"Mr.","validFor":null,"value":"Mr."},{"active":true,"defaultValue":false,"label":"Ms.","validFor":null,"value":"Ms."},{"active":true,"defaultValue":false,"label":"Mrs.","validFor":null,"value":"Mrs."},{"active":true,"defaultValue":false,"label":"Dr.","validFor":null,"value":"Dr."},{"active":true,"defaultValue":false,"label":"Prof.","validFor":null,"value":"Prof."}],"precision":0,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"soapType":"xsd:string","sortable":true,"type":"picklist","unique":false,"updateable":true,"writeRequiresMasterRead":false}],"keyPrefix":"00Q","label":"Lead","labelPlural":"Leads","layoutable":true,"listviewable":null,"lookupLayoutable":null,"mergeable":true,"name":"Lead","queryable":true,"recordTypeInfos":[{"available":true,"defaultRecordTypeMapping":true,"name":"Marketing","recordTypeId":"xxxx"},{"available":true,"defaultRecordTypeMapping":false,"name":"Partner + Deal","recordTypeId":"xxxx"},{"available":true,"defaultRecordTypeMapping":false,"name":"Sales","recordTypeId":"xxxx"},{"available":true,"defaultRecordTypeMapping":false,"name":"Master","recordTypeId":"xxxx"}],"replicateable":true,"retrieveable":true,"searchLayoutable":null,"searchable":true,"triggerable":true,"undeletable":true,"updateable":true,"urls":{"uiEditTemplate":"https://xxx.salesforce.com/{ID}/e","sobject":"/services/data/v26.0/sobjects/Lead","uiDetailTemplate":"https://xxx.salesforce.com/{ID}","describe":"/services/data/v26.0/sobjects/Lead/describe","rowTemplate":"/services/data/v26.0/sobjects/Lead/{ID}","uiNewRecord":"https://xxx.salesforce.com/00Q/e"}}' + http_version: + recorded_at: Thu, 27 Aug 2015 23:57:44 GMT +- request: + method: get + uri: https://eu2.salesforce.com/services/data/v26.0/query?q=SELECT%20Id,IsDeleted,LastName,FirstName,Salutation,LastModifiedDate%20FROM%20Lead%20WHERE%20Email%20LIKE%20%27%25@elastic.co%27%20AND%20LastModifiedDate%20%3E%202025-05-07T14:32:17Z%20ORDER%20BY%20LastModifiedDate%20ASC + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Faraday v0.9.1 + Authorization: + - OAuth xxx + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - '*/*' + response: + status: + code: 200 + message: OK + headers: + Date: + - Thu, 27 Aug 2015 23:57:45 GMT + Set-Cookie: + - BrowserId=xxxx;Path=/;Domain=.salesforce.com;Expires=Mon, 26-Oct-2015 23:57:45 GMT + Expires: + - Thu, 01 Jan 1970 00:00:00 GMT + Sforce-Limit-Info: + - api-usage=211063/451000 + Content-Type: + - application/json;charset=UTF-8 + Transfer-Encoding: + - chunked + body: + encoding: UTF-8 + string: '{"totalSize":3,"done":true,"records":[{"attributes":{"type":"Lead","url":"/services/data/v26.0/sobjects/Lead/xxxx"},"Id":"xxxx","IsDeleted":false,"LastName":"Katz","FirstName":"Aaron","Salutation":"Mr."},{"attributes":{"type":"Lead","url":"/services/data/v26.0/sobjects/Lead/xxxx"},"Id":"xxxx","IsDeleted":false,"LastName":"Grand","FirstName":"Adrien","Salutation":"Dr."},{"attributes":{"type":"Lead","url":"/services/data/v26.0/sobjects/Lead/xxxx"},"Id":"xxxx","IsDeleted":false,"LastName":"Hardy","FirstName":"Alan","Salutation":"Overlord"}]}' + http_version: + recorded_at: Thu, 27 Aug 2015 23:57:45 GMT diff --git a/spec/fixtures/vcr_cassettes/load_some_lead_objects_with_lastmodifieddate_filter_order_by_lastmodifieddate_no_filters.yml b/spec/fixtures/vcr_cassettes/load_some_lead_objects_with_lastmodifieddate_filter_order_by_lastmodifieddate_no_filters.yml new file mode 100644 index 0000000..6f19f97 --- /dev/null +++ b/spec/fixtures/vcr_cassettes/load_some_lead_objects_with_lastmodifieddate_filter_order_by_lastmodifieddate_no_filters.yml @@ -0,0 +1,123 @@ +--- +http_interactions: +- request: + method: post + uri: https://login.salesforce.com/services/oauth2/token + body: + encoding: US-ASCII + string: grant_type=password&client_id=xxxx&client_secret=xxxx&username=xxxx&password=xxxx + headers: + User-Agent: + - Faraday v0.9.1 + Content-Type: + - application/x-www-form-urlencoded + Accept: + - '*/*' + response: + status: + code: 200 + message: OK + headers: + Date: + - Thu, 27 Aug 2015 23:57:42 GMT + Set-Cookie: + - BrowserId=xxxx;Path=/;Domain=.salesforce.com;Expires=Mon, 26-Oct-2015 23:57:42 GMT + Expires: + - Thu, 01 Jan 1970 00:00:00 GMT + Pragma: + - no-cache + Cache-Control: + - no-cache, no-store + Content-Type: + - application/json;charset=UTF-8 + Transfer-Encoding: + - chunked + body: + encoding: US-ASCII + string: '{"id":"https://login.salesforce.com/id/xxxx/xxxx","issued_at":"1440719862904","token_type":"Bearer","instance_url":"https://eu2.salesforce.com","signature":"xxxx","access_token":"xxxx"}' + http_version: + recorded_at: Thu, 27 Aug 2015 23:57:43 GMT +- request: + method: get + uri: https://eu2.salesforce.com/services/data/v26.0/sobjects/Lead/describe + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Faraday v0.9.1 + Authorization: + - OAuth xxxx + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - '*/*' + response: + status: + code: 200 + message: OK + headers: + Date: + - Thu, 27 Aug 2015 23:57:44 GMT + Set-Cookie: + - BrowserId=xxxx;Path=/;Domain=.salesforce.com;Expires=Mon, 26-Oct-2015 23:57:44 GMT + Expires: + - Thu, 01 Jan 1970 00:00:00 GMT + Sforce-Limit-Info: + - api-usage=211068/451000 + Org.eclipse.jetty.server.include.etag: + - 5fb54cb6 + Last-Modified: + - Thu, 27 Aug 2015 22:36:55 GMT + Content-Type: + - application/json;charset=UTF-8 + Etag: + - 5fb54cb-gzip" + Transfer-Encoding: + - chunked + body: + encoding: UTF-8 + string: '{"activateable":false,"childRelationships":[],"createable":true,"custom":false,"customSetting":false,"deletable":true,"deprecatedAndHidden":false,"feedEnabled":true,"fields":[{"autoNumber":false,"byteLength":18,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"externalId":false,"filterable":true,"groupable":true,"htmlFormatted":false,"idLookup":true,"inlineHelpText":null,"label":"Lead + ID","length":18,"name":"Id","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"precision":0,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"soapType":"tns:ID","sortable":true,"type":"id","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"autoNumber":false,"byteLength":0,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"controllerName":null,"createable":false,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":true,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"externalId":false,"filterable":true,"groupable":true,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Deleted","length":0,"name":"IsDeleted","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"precision":0,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"soapType":"xsd:boolean","sortable":true,"type":"boolean","unique":false,"updateable":false,"writeRequiresMasterRead":false},{"autoNumber":false,"byteLength":240,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"externalId":false,"filterable":true,"groupable":true,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Last + Name","length":80,"name":"LastName","nameField":false,"namePointing":false,"nillable":false,"permissionable":false,"picklistValues":[],"precision":0,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"soapType":"xsd:string","sortable":true,"type":"string","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"autoNumber":false,"byteLength":120,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"externalId":false,"filterable":true,"groupable":true,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"First + Name","length":40,"name":"FirstName","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[],"precision":0,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"soapType":"xsd:string","sortable":true,"type":"string","unique":false,"updateable":true,"writeRequiresMasterRead":false},{"autoNumber":false,"byteLength":120,"calculated":false,"calculatedFormula":null,"cascadeDelete":false,"caseSensitive":false,"controllerName":null,"createable":true,"custom":false,"defaultValue":null,"defaultValueFormula":null,"defaultedOnCreate":false,"dependentPicklist":false,"deprecatedAndHidden":false,"digits":0,"displayLocationInDecimal":false,"externalId":false,"filterable":true,"groupable":true,"htmlFormatted":false,"idLookup":false,"inlineHelpText":null,"label":"Salutation","length":40,"name":"Salutation","nameField":false,"namePointing":false,"nillable":true,"permissionable":false,"picklistValues":[{"active":true,"defaultValue":false,"label":"Mr.","validFor":null,"value":"Mr."},{"active":true,"defaultValue":false,"label":"Ms.","validFor":null,"value":"Ms."},{"active":true,"defaultValue":false,"label":"Mrs.","validFor":null,"value":"Mrs."},{"active":true,"defaultValue":false,"label":"Dr.","validFor":null,"value":"Dr."},{"active":true,"defaultValue":false,"label":"Prof.","validFor":null,"value":"Prof."}],"precision":0,"referenceTo":[],"relationshipName":null,"relationshipOrder":null,"restrictedDelete":false,"restrictedPicklist":false,"scale":0,"soapType":"xsd:string","sortable":true,"type":"picklist","unique":false,"updateable":true,"writeRequiresMasterRead":false}],"keyPrefix":"00Q","label":"Lead","labelPlural":"Leads","layoutable":true,"listviewable":null,"lookupLayoutable":null,"mergeable":true,"name":"Lead","queryable":true,"recordTypeInfos":[{"available":true,"defaultRecordTypeMapping":true,"name":"Marketing","recordTypeId":"xxxx"},{"available":true,"defaultRecordTypeMapping":false,"name":"Partner + Deal","recordTypeId":"xxxx"},{"available":true,"defaultRecordTypeMapping":false,"name":"Sales","recordTypeId":"xxxx"},{"available":true,"defaultRecordTypeMapping":false,"name":"Master","recordTypeId":"xxxx"}],"replicateable":true,"retrieveable":true,"searchLayoutable":null,"searchable":true,"triggerable":true,"undeletable":true,"updateable":true,"urls":{"uiEditTemplate":"https://xxx.salesforce.com/{ID}/e","sobject":"/services/data/v26.0/sobjects/Lead","uiDetailTemplate":"https://xxx.salesforce.com/{ID}","describe":"/services/data/v26.0/sobjects/Lead/describe","rowTemplate":"/services/data/v26.0/sobjects/Lead/{ID}","uiNewRecord":"https://xxx.salesforce.com/00Q/e"}}' + http_version: + recorded_at: Thu, 27 Aug 2015 23:57:44 GMT +- request: + method: get + uri: https://eu2.salesforce.com/services/data/v26.0/query?q=SELECT%20Id,IsDeleted,LastName,FirstName,Salutation,LastModifiedDate%20FROM%20Lead%20WHERE%20LastModifiedDate%20%3E%202025-05-07T14:32:17Z%20ORDER%20BY%20LastModifiedDate%20ASC + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Faraday v0.9.1 + Authorization: + - OAuth xxx + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - '*/*' + response: + status: + code: 200 + message: OK + headers: + Date: + - Thu, 27 Aug 2015 23:57:45 GMT + Set-Cookie: + - BrowserId=xxxx;Path=/;Domain=.salesforce.com;Expires=Mon, 26-Oct-2015 23:57:45 GMT + Expires: + - Thu, 01 Jan 1970 00:00:00 GMT + Sforce-Limit-Info: + - api-usage=211063/451000 + Content-Type: + - application/json;charset=UTF-8 + Transfer-Encoding: + - chunked + body: + encoding: UTF-8 + string: '{"totalSize":3,"done":true,"records":[{"attributes":{"type":"Lead","url":"/services/data/v26.0/sobjects/Lead/xxxx"},"Id":"xxxx","IsDeleted":false,"LastName":"Katz","FirstName":"Aaron","Salutation":"Mr."},{"attributes":{"type":"Lead","url":"/services/data/v26.0/sobjects/Lead/xxxx"},"Id":"xxxx","IsDeleted":false,"LastName":"Grand","FirstName":"Adrien","Salutation":"Dr."},{"attributes":{"type":"Lead","url":"/services/data/v26.0/sobjects/Lead/xxxx"},"Id":"xxxx","IsDeleted":false,"LastName":"Hardy","FirstName":"Alan","Salutation":"Overlord"}]}' + http_version: + recorded_at: Thu, 27 Aug 2015 23:57:45 GMT diff --git a/spec/inputs/salesforce_spec.rb b/spec/inputs/salesforce_spec.rb index 8ec5a99..86049c5 100644 --- a/spec/inputs/salesforce_spec.rb +++ b/spec/inputs/salesforce_spec.rb @@ -263,5 +263,219 @@ end end end + + context "run with last modified time from range field file" do + VCR.configure do |config| + config.cassette_library_dir = File.join(File.dirname(__FILE__), '..', 'fixtures', 'vcr_cassettes') + config.hook_into :webmock + config.before_record do |i| + if i.response.body.encoding.to_s == 'ASCII-8BIT' + # required because sfdc doesn't send back the content encoding and it + # confuses the yaml parser + json_body = JSON.load(i.response.body.encode("ASCII-8BIT").force_encoding("utf-8")) + i.response.body = json_body.to_json + i.response.update_content_length_header + end + end + end + let(:options) do + { + "client_id" => "", + "client_secret" => ::LogStash::Util::Password.new("secret-key"), + "username" => "", + "password" => ::LogStash::Util::Password.new("secret-password"), + "security_token" => ::LogStash::Util::Password.new("secret-token"), + "use_tooling_api" => false, + "sfdc_object_name" => "Lead", + "sfdc_fields" => ["Id", "IsDeleted", "LastName", "FirstName", "Salutation"], + "sfdc_filters" => "Email LIKE '%@elastic.co'", + "changed_data_filter" => "LastModifiedDate > %{last_tracking_field_value}", + "tracking_field_value_file" => "last_tracking_field_value.txt" + } + end + let(:input) { LogStash::Inputs::Salesforce.new(options) } + subject { input } + let(:queue) { [] } + it "loads some lead records modified after 2025-05-07 14:32:17 UTC" do + VCR.use_cassette("load some lead objects with lastmodifieddate filter", :decode_compressed_response => true) do + subject.register + File.write("last_tracking_field_value.txt", "2025-05-07T14:32:17Z") + subject.run(queue) + expect(queue.length).to eq(3) + end + end + end + + context "run with last modified time from range field file when file doesn't yet exist" do + VCR.configure do |config| + config.cassette_library_dir = File.join(File.dirname(__FILE__), '..', 'fixtures', 'vcr_cassettes') + config.hook_into :webmock + config.before_record do |i| + if i.response.body.encoding.to_s == 'ASCII-8BIT' + # required because sfdc doesn't send back the content encoding and it + # confuses the yaml parser + json_body = JSON.load(i.response.body.encode("ASCII-8BIT").force_encoding("utf-8")) + i.response.body = json_body.to_json + i.response.update_content_length_header + end + end + end + let(:options) do + { + "client_id" => "", + "client_secret" => ::LogStash::Util::Password.new("secret-key"), + "username" => "", + "password" => ::LogStash::Util::Password.new("secret-password"), + "security_token" => ::LogStash::Util::Password.new("secret-token"), + "use_tooling_api" => false, + "sfdc_object_name" => "Lead", + "sfdc_fields" => ["Id", "IsDeleted", "LastName", "FirstName", "Salutation"], + "sfdc_filters" => "Email LIKE '%@elastic.co'", + "changed_data_filter" => "LastModifiedDate > %{last_tracking_field_value}", + "tracking_field_value_file" => "last_tracking_field_value.txt" + } + end + let(:input) { LogStash::Inputs::Salesforce.new(options) } + subject { input } + let(:queue) { [] } + it "loads some lead records" do + VCR.use_cassette("load some lead objects", :decode_compressed_response => true) do + subject.register + File.exist?("last_tracking_field_value.txt") && File.delete("last_tracking_field_value.txt") + subject.run(queue) + expect(queue.length).to eq(3) + end + end + end + + context "run with last modified time from range field file with range field" do + VCR.configure do |config| + config.cassette_library_dir = File.join(File.dirname(__FILE__), '..', 'fixtures', 'vcr_cassettes') + config.hook_into :webmock + config.before_record do |i| + if i.response.body.encoding.to_s == 'ASCII-8BIT' + # required because sfdc doesn't send back the content encoding and it + # confuses the yaml parser + json_body = JSON.load(i.response.body.encode("ASCII-8BIT").force_encoding("utf-8")) + i.response.body = json_body.to_json + i.response.update_content_length_header + end + end + end + let(:options) do + { + "client_id" => "", + "client_secret" => ::LogStash::Util::Password.new("secret-key"), + "username" => "", + "password" => ::LogStash::Util::Password.new("secret-password"), + "security_token" => ::LogStash::Util::Password.new("secret-token"), + "use_tooling_api" => false, + "sfdc_object_name" => "Lead", + "sfdc_fields" => ["Id", "IsDeleted", "LastName", "FirstName", "Salutation"], + "sfdc_filters" => "Email LIKE '%@elastic.co'", + "tracking_field" => "LastModifiedDate", + "changed_data_filter" => "LastModifiedDate > %{last_tracking_field_value}", + "tracking_field_value_file" => "last_tracking_field_value.txt" + } + end + let(:input) { LogStash::Inputs::Salesforce.new(options) } + subject { input } + let(:queue) { [] } + it "loads some lead records modified after 2025-05-07 14:32:17 UTC" do + VCR.use_cassette("load some lead objects with lastmodifieddate filter order by lastmodifieddate", :decode_compressed_response => true) do + subject.register + File.write("last_tracking_field_value.txt", "2025-05-07T14:32:17Z") + subject.run(queue) + expect(queue.length).to eq(3) + end + end + end + + context "run with last modified time from range field file with range field and no other filters" do + VCR.configure do |config| + config.cassette_library_dir = File.join(File.dirname(__FILE__), '..', 'fixtures', 'vcr_cassettes') + config.hook_into :webmock + config.before_record do |i| + if i.response.body.encoding.to_s == 'ASCII-8BIT' + # required because sfdc doesn't send back the content encoding and it + # confuses the yaml parser + json_body = JSON.load(i.response.body.encode("ASCII-8BIT").force_encoding("utf-8")) + i.response.body = json_body.to_json + i.response.update_content_length_header + end + end + end + let(:options) do + { + "client_id" => "", + "client_secret" => ::LogStash::Util::Password.new("secret-key"), + "username" => "", + "password" => ::LogStash::Util::Password.new("secret-password"), + "security_token" => ::LogStash::Util::Password.new("secret-token"), + "use_tooling_api" => false, + "sfdc_object_name" => "Lead", + "sfdc_fields" => ["Id", "IsDeleted", "LastName", "FirstName", "Salutation"], + "tracking_field" => "LastModifiedDate", + "changed_data_filter" => "LastModifiedDate > %{last_tracking_field_value}", + "tracking_field_value_file" => "last_tracking_field_value.txt" + } + end + let(:input) { LogStash::Inputs::Salesforce.new(options) } + subject { input } + let(:queue) { [] } + it "loads some lead records modified after 2025-05-07 14:32:17 UTC" do + VCR.use_cassette("load some lead objects with lastmodifieddate filter order by lastmodifieddate no filters", :decode_compressed_response => true) do + subject.register + File.write("last_tracking_field_value.txt", "2025-05-07T14:32:17Z") + subject.run(queue) + expect(queue.length).to eq(3) + end + end + end + + context "run with last modified time from range field file when file doesn't yet exist with range field" do + VCR.configure do |config| + config.cassette_library_dir = File.join(File.dirname(__FILE__), '..', 'fixtures', 'vcr_cassettes') + config.hook_into :webmock + config.before_record do |i| + if i.response.body.encoding.to_s == 'ASCII-8BIT' + # required because sfdc doesn't send back the content encoding and it + # confuses the yaml parser + json_body = JSON.load(i.response.body.encode("ASCII-8BIT").force_encoding("utf-8")) + i.response.body = json_body.to_json + i.response.update_content_length_header + end + end + end + let(:options) do + { + "client_id" => "", + "client_secret" => ::LogStash::Util::Password.new("secret-key"), + "username" => "", + "password" => ::LogStash::Util::Password.new("secret-password"), + "security_token" => ::LogStash::Util::Password.new("secret-token"), + "use_tooling_api" => false, + "sfdc_object_name" => "Lead", + "sfdc_fields" => ["Id", "IsDeleted", "LastName", "FirstName", "Salutation"], + "sfdc_filters" => "Email LIKE '%@elastic.co'", + "tracking_field" => "LastModifiedDate", + "changed_data_filter" => "LastModifiedDate > %{last_tracking_field_value}", + "tracking_field_value_file" => "last_tracking_field_value.txt" + } + end + let(:input) { LogStash::Inputs::Salesforce.new(options) } + subject { input } + let(:queue) { [] } + it "loads some lead records" do + VCR.use_cassette("load some lead objects order by lastmodifieddate", :decode_compressed_response => true) do + subject.register + File.exist?("last_tracking_field_value.txt") && File.delete("last_tracking_field_value.txt") + subject.run(queue) + expect(queue.length).to eq(3) + tracking_field_value = File.read("last_tracking_field_value.txt") + expect(tracking_field_value).to eq("2025-05-07T14:32:17Z") + end + end + end end end