Update and Sync winlog input integrations

- expose `ignore_older` option
- change "Preserve Original Event` option to use `include_xml`
- remove check for preserve_original_event from ingest pipelines
- make ordering of options the same for all winlog inputs
- switch to `processors.length` check to handlebars templates
- make including of tags the same in handlebars templates
- for windows/forwarded remove requirement to specify event id, none
  means all, which should be default
- for windows/sysmon_operational remove requirement to specify event id, none
  means all, which should be default
- for winlog remove requirement to specify event id, none
  means all, which should be default

Closes #2472

Author: leehinman

packages/microsoft_sqlserver/changelog.yml

@@ -1,4 +1,15 @@
 # newer versions go on top
+- version: "0.3.0"
+  changes:
+    - description: Expose winlog input ignore_older option.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2542
+    - description: Fix preserve original event option
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/2542
+    - description: Make order of options consistent with other winlog based integrations.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2542
 - version: "0.2.0"
   changes:
     - description: Expose winlog input language option.

packages/microsoft_sqlserver/data_stream/audit/_dev/test/pipeline/test-events.json-expected.json

@@ -1,21 +1,22 @@
 name: {{channel}}
 condition: ${host.platform} == 'windows'
 event_id: {{event_id}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
 {{#if language}}
 language: {{language}}
 {{/if}}
 {{#if tags.length}}
 tags:
-{{else if preserve_original_event}}
-tags:
-{{/if}}
 {{#each tags as |tag i|}}
   - {{tag}}
 {{/each}}
+{{/if}}
 {{#if preserve_original_event}}
-  - preserve_original_event
+include_xml: true
 {{/if}}
-{{#if processors}}
+{{#if processors.length}}
 processors:
 {{processors}}
 {{/if}}

packages/microsoft_sqlserver/data_stream/audit/agent/stream/winlog.yml.hbs

@@ -1243,11 +1243,6 @@ processors:
       - sqlserver.audit.target_server_principal_name
       - sqlserver.audit.target_server_principal_sid
     ignore_missing: true
-- remove:
-    field: event.original
-    if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-    ignore_failure: true
-    ignore_missing: true
 - script:
     lang: painless
     description: This script processor iterates over the whole document to remove fields with null values.

packages/microsoft_sqlserver/data_stream/audit/elasticsearch/ingest_pipeline/default.yml

@@ -6,6 +6,15 @@ streams:
     description: Collect SQL Server audit events from the Windows event logs
     template_path: winlog.yml.hbs
     vars:
+      - name: preserve_original_event
+        required: true
+        show_user: true
+        title: Preserve original event
+        description: >-
+          Preserves a raw copy of the original XML event, added to the field `event.original`
+        type: bool
+        multi: false
+        default: false
       - name: event_id
         type: text
         title: Event ID
@@ -23,6 +32,14 @@ streams:
         required: true
         default: Security
         show_user: true
+      - name: ignore_older
+        type: text
+        title: Ignore events older than
+        default: 72h
+        required: false
+        show_user: false
+        description: >-
+          If this option is specified, events that are older than the specified amount of time are ignored. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
       - name: language
         type: text
         title: Language ID
@@ -31,23 +48,15 @@ streams:
         required: false
         show_user: false
         default: 0
-      - name: preserve_original_event
-        required: true
-        show_user: true
-        title: Preserve original event
-        description: Preserves a raw copy of the original event, added to the field `event.original`
-        type: bool
-        multi: false
-        default: false
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        show_user: false
       - name: processors
         type: yaml
         title: Processors
         multi: false
         required: false
         show_user: false
         description: "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. \nThis executes in the agent before the logs are parsed. \nSee [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n"
-      - name: tags
-        type: text
-        title: Tags
-        multi: true
-        show_user: false

packages/microsoft_sqlserver/data_stream/audit/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: microsoft_sqlserver
 title: "Microsoft SQL Server"
-version: 0.2.0
+version: 0.3.0
 license: basic
 description: Collect audit events from Microsoft SQL Server with Elastic Agent.
 type: integration

packages/microsoft_sqlserver/manifest.yml

@@ -1,4 +1,15 @@
 # newer versions go on top
+- version: "1.10.0"
+  changes:
+    - description: Expose winlog input ignore_older option.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2542
+    - description: Fix preserve original event option
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/2542
+    - description: Make order of Security, Application, System options consistent with other winlog based integrations.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2542
 - version: "1.9.0"
   changes:
     - description: Update to ECS 8.0

packages/system/changelog.yml

@@ -1,20 +1,24 @@
 name: Application
 condition: ${host.platform} == 'windows'
-ignore_older: 72h
 {{#if event_id}}
 event_id: {{event_id}}
 {{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
 {{#if language}}
 language: {{language}}
 {{/if}}
-{{#if processors}}
+{{#if preserve_original_event}}
+include_xml: true
+{{/if}}
+{{#if processors.length}}
 processors:
 {{processors}}
 {{/if}}
+{{#if tags.length}}
 tags:
-{{#if preserve_original_event}}
-  - preserve_original_event
-{{/if}}
 {{#each tags as |tag i|}}
   - {{tag}}
-{{/each}}
+{{/each}}
+{{/if}}

packages/system/data_stream/application/agent/stream/winlog.yml.hbs

@@ -7,11 +7,6 @@ processors:
   - set:
       field: ecs.version
       value: 8.0.0
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: "error.message"

packages/system/data_stream/application/elasticsearch/ingest_pipeline/default.yml

@@ -7,6 +7,15 @@ streams:
     title: Application
     description: 'Collect Windows application logs'
     vars:
+      - name: preserve_original_event
+        required: true
+        show_user: true
+        title: Preserve original event
+        description: >-
+          Preserves a raw copy of the original XML event, added to the field `event.original`
+        type: bool
+        multi: false
+        default: false
       - name: event_id
         type: text
         title: Event ID
@@ -15,19 +24,14 @@ streams:
         show_user: false
         description: >-
           A list of included and excluded (blocked) event IDs. The value is a comma-separated list. The accepted values are single event IDs to include (e.g. 4624), a range of event IDs to include (e.g. 4700-4800), and single event IDs to exclude (e.g. -4735).  Limit 22 IDs.
-      - name: processors
-        type: yaml
-        title: Processors
-        multi: false
+      - name: ignore_older
+        type: text
+        title: Ignore events older than
+        default: 72h
         required: false
         show_user: false
         description: >-
-          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
-      - name: tags
-        type: text
-        title: Tags
-        multi: true
-        show_user: false
+          If this option is specified, events that are older than the specified amount of time are ignored. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
       - name: language
         type: text
         title: Language ID
@@ -36,6 +40,19 @@ streams:
         required: false
         show_user: false
         default: 0
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        show_user: false
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >-
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
   - input: httpjson
     title: Windows Application Events via Splunk Enterprise REST API
     description: Collect Application Events via Splunk Enterprise REST API