[Salesforce] Add pagination support for login_rest and logout_rest data streams (#4866)

* Add pagination support for login_rest and logout_rest

* Add Changelog entry

Author: kush-elastic

packages/salesforce/changelog.yml

@@ -1,5 +1,10 @@
 # newer versions go on top
 
+- version: 0.2.1
+  changes:
+    - description: Add pagination support for "login_rest" and "logout_rest".
+      link: https://github.com/elastic/integrations/pull/4866
+      type: bugfix
 - version: 0.2.0
   changes:
     - description: Salesforce integration package with "logout_rest" data stream.

packages/salesforce/data_stream/login_rest/agent/stream/httpjson.yml.hbs

@@ -14,16 +14,19 @@ request.transforms:
       target: url.params.q
       value: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Login' AND LogDate > [[.cursor.last_published_login]] ORDER BY LogDate ASC NULLS FIRST"
       default: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Login' ORDER BY LogDate ASC NULLS FIRST"
-response.split:
-  target: body.records
+response.pagination:
+- set:
+    target: url.value
+    value: '[[if (ne .last_response.body.done true)]]{{instance_url}}[[.last_response.body.nextRecordsUrl]][[end]]'
+    fail_on_template_error: true
 chain:
   - step:
       request.url: {{instance_url}}/services/data/v54.0/sobjects/EventLogFile/$.records[:].Id/LogFile
       request.method: GET
       replace: $.records[:].Id
 cursor:
   last_published_login:
-    value: '[[.last_event.LogDate]]'
+    value: '[[(formatDate ((parseDate .last_event.TIMESTAMP_DERIVED "RFC3339").Add (parseDuration "-1h")))]]'
 tags:
 {{#if preserve_original_event}}
   - preserve_original_event

packages/salesforce/data_stream/logout_rest/agent/stream/httpjson.yml.hbs

@@ -14,16 +14,19 @@ request.transforms:
       target: url.params.q
       value: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Logout' AND LogDate > [[.cursor.last_published_logout]] ORDER BY CreatedDate ASC NULLS FIRST"
       default: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Logout' ORDER BY LogDate ASC NULLS FIRST"
-response.split:
-  target: body.records
+response.pagination:
+- set:
+    target: url.value
+    value: '[[if (ne .last_response.body.done true)]]{{instance_url}}[[.last_response.body.nextRecordsUrl]][[end]]'
+    fail_on_template_error: true
 chain:
   - step:
       request.url: {{instance_url}}/services/data/v54.0/sobjects/EventLogFile/$.records[:].Id/LogFile
       request.method: GET
       replace: $.records[:].Id
 cursor:
   last_published_logout:
-    value: '[[.last_event.LogDate]]'
+    value: '[[(formatDate ((parseDate .last_event.TIMESTAMP_DERIVED "RFC3339").Add (parseDuration "-1h")))]]'
 tags:
 {{#if preserve_original_event}}
   - preserve_original_event

packages/salesforce/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: salesforce
 title: Salesforce
-version: 0.2.0
+version: 0.2.1
 license: basic
 description: Collect logs from Salesforce with Elastic Agent.
 type: integration