Updated CEL program and fields.yml

Author: gauravneelwarna

packages/checkpoint_harmony_endpoint/data_stream/forensics/_dev/test/pipeline/test-access-sample.log

@@ -1,30 +1,109 @@
-config_version: 1
+config_version: 3
+{{#if enable_request_tracer}}
+{{/if}}
+resource.tracer.filename: "../../logs/cel/http-request-trace-*.ndjson"
+request.tracer.maxbackups: 5
 resource.url: {{server_url}}
-auth_request_body: '{"clientId":"{{client_id}}","accessKey":"{{secret_key}}"}'
+auth.oauth2:
+  client.id: {{client_id}}
+  token_url: {{server_url}}/auth/external
+  endpoint_params:
+    accessKey: {{secret_key}}
+state:
+  interval: {{interval}}
+  initial_interval: {{initial_interval}}  // Define initial interval
+  want_more: false
+  full_url: {{server_url}}/logs_query/
+  per_page: {{per_page}}
+  page_limit: {{page_limit}}
 program: |
-  (
-    request("GET", state.url.trim_right("/") + {{auth_uri}})
-    .with(
+  request("GET", state.full_url).with({
+    "Header": {"Content-Type": ["application/json"]},
+    "Body": {
       {
-        "Header": {"Content-Type": ["application/json"]},
-        "Body": auth_request_body
+        "filter": "product:\"Forensics\"",
+        "limit": state.per_page,
+        "pageLimit": state.page_limit,
+        "cloudService": "Harmony Endpoint",
+        "timeframe": {
+          //Is this correct way to use apply time?
+          "startTime": '"' + timestamp(now() - duration('"+' + state.initial_interval + '"')).format(time_layout.RFC3339) + '"',
+          "endTime": '"' + timestamp(now()).format(time_layout.RFC3339) + '"'
+        }
+       }
+    }
+  }).do_request().as(resp,
+    resp.StatusCode == 200
+    ?
+    bytes(resp.Body).decode_json().as(body, {
+      "events": body.success,
+      "want_more": false,
+      "cursor": {
+        "taskid": body.data.taskId
       }
-      )
-    .do_request().as(resp,
-      resp.StatusCode == 200 ?
-      (bytes(resp.Body).decode_json().as(body, {
-        "events": [body.data.token]
-      },
-      "url": {{server_url}},
-    )
-    )
+    })
     :
     {
-      "events": {
-        "error": string(resp.message)
-      }
+        "events": [{
+          "error": {
+            "message": "response code: "+resp.StatusCode
+          }
+        }]
     }
+    )
 
+  resource.retry.wait_min(20s)
+  request("GET", state.full_url+"/"+cursor.taskid).with({
+    "Header": {"Content-Type": ["application/json"]}
+  }).do_request.as(resp,
+    resp.StatusCode == 200
+    ?
+    byte(resp.body).decode_json().as(body, {
+      "events": body.success,
+      "pagetokens": body.data.pageTokens
+    })
+    :
+    {
+      "events: [{
+        "error: {
+          "message": "response code: "+resp.StatusCode
+        }"
+      }]"
+    }
   )
-  debug(resp)
+
+  // How to apply pagination based on 'nextPageToken' value from response body.
+  // As discussed in previous call, how to use 'last_seen_ts' to update 'startTime' for next iteration?
+  request("GET", state.full_url+"/retrieve").with({
+    "Header": {"Content-Type": ["application/json"]},
+    "Body": {
+      {
+        "taskid": cursor.taskid,
+        "pagetoken": cursor.pagetokens
+       }
+    }
+  }).do_request.as(resp,
+    resp.StatusCode == 200
+    ?
+    byte(resp.body).decode_json().as(body, {
+      "events": body.data.records,
+      "nextPageToken": body.data.nextPageToken
+      "last_seen_ts": body.data.records[-1].time
+    })
+    :
+    {
+      "events: [{
+        "error: {
+          "message": "response code: "+resp.StatusCode
+        }"
+      }]"
+    }
   )
+
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#if preserve_duplicate_custom_fields}}
+  - preserve_duplicate_custom_fields
+{{/if}}

packages/checkpoint_harmony_endpoint/data_stream/forensics/_dev/test/pipeline/test-access-sample.log-config.yml

@@ -1,9 +1,51 @@
-- name: ti_crowdstrike.intel
+- name: checkpoint_harmony_endpoint.forensics
   type: group
   fields:
     - name: confidence_level
       type: keyword
-      description: Confidence levek
+      description: Confidence level
+    - name: policy_date
+      type: date
+      description: Policy Date
     - name: policy_name
       type: keyword
-      description: Policy Name
+      description: Policy Name
+    - name: severity
+      type: keyword
+      description: Event severity
+    - name: time
+      type: date
+      description: Time
+    - name: action
+      type: keyword
+      description: Action
+    - name: protection_type
+      type: keyword
+      description: Protection Type
+    - name: attack_status
+      type: keyword
+      description: Attack Status
+    - name: detected_by
+      type: keyword
+      description: Detected By
+    - name: event_type
+      type: keyword
+      description: Event Type
+    - name: malware_action
+      type: flattened
+      description: Action Taken
+    - name: protection_name
+      type: keyword
+      description: Protection Name
+    - name: resource
+      type: flattened
+      description: Resource
+    - name: src
+      type: ip
+      description: Source IP
+    - name: src_machine_name
+      type: keyword
+      description: Source Host Name
+    - name: src_user_name
+      type: flattened
+      description: Source User Name

packages/checkpoint_harmony_endpoint/data_stream/forensics/_dev/test/pipeline/test-access-sample.log-expected.json

@@ -59,25 +59,32 @@ policy_templates:
             default: https://cloudinfra-gw.portal.checkpoint.com
             required: true
             show_user: true
-          - name: auth_uri
+          - name: interval
             type: text
-            title: Authentication URI
-            description: URI of the CheckPoint Harmony Endpoint API Authentication API. Defaults to /auth/external
-            default: /auth/external
+            title: Interval
+            description: Interval at which the logs will be pulled. The value must be between 2m and 1h. Supported units for this parameter are h/m/s.
+            default: 5m
             required: true
             show_user: true
-          - name: search_uri
+          - name: initial_interval
             type: text
-            title: Search URI
-            description: URI of the CheckPoint Harmony Endpoint API Search Query. Defaults to /logs_query/
-            default: /logs_query/
+            title: Initial Interval
+            description: Initial interval at which the logs will be pulled. Defaults to 30 days (720 hours). Max is 12960 hours (18 months). Supported units for this parameter are h/m/s.
+            default: 720h
             required: true
             show_user: true
-          - name: retrieve_results_uri
-            type: text
-            title: Retrive Search Results URI
-            description: URI of the CheckPoint Harmony Endpoint API to check progress of Search Query. Defaults to /logs_query/retrieve
-            default: /logs_query/retrieve
+          - name: per_page
+            type: integer
+            title: Results per page
+            description: Sets the number of results to return per page. If set this must be greater or equal to 1 and less than or equal to 1000. If not set, the API default is used. Per page should not be greated than 'Page Limit'.
+            default: 100
+            required: true
+            show_user: true
+          - name: page_limit
+            type: integer
+            title: Results per page
+            description: Sets the number of results to return per page. If set this must be greater or equal to 1 and less than or equal to 1000. If not set, the API default is used.
+            default: 100
             required: true
             show_user: true
 owner: