From 94dd7b58dd65f132145636ce6a1f9bddc78d498b Mon Sep 17 00:00:00 2001
From: Rafael Lukas Maers <rafael@mnemonic.no>
Date: Fri, 27 Sep 2019 09:58:05 +0200
Subject: [PATCH 01/13] Improve response.models.CbChildProcEvent.process

Provide as much information as possible, which is particularly useful
for suppressed child processes.
---
 src/cbapi/response/models.py | 57 ++++++++++++++++--------------------
 1 file changed, 26 insertions(+), 31 deletions(-)

diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py
index 5e68920a..b3e48833 100644
--- a/src/cbapi/response/models.py
+++ b/src/cbapi/response/models.py
@@ -3155,37 +3155,32 @@ def __init__(self, parent_process, timestamp, sequence, event_data, is_suppresse
 
     @property
     def process(self):
-        proc_data = self.proc_data
-        md5sum = self.__dict__.get("md5", None)
-        if md5sum:
-            proc_data["process_md5"] = md5sum
-        pid = self.__dict__.get("pid", None)
-        if pid:
-            proc_data["pid"] = pid
-        path = self.__dict__.get("path", None)
-        if path:
-            proc_data["path"] = path
-
-        proc_data["parent_unique_id"] = self.parent._model_unique_id
-        proc_data["parent_id"] = self.parent.id
-
-        try:
-            (sensor_id, proc_pid, proc_createtime) = parse_process_guid(self.parent.id)
-            if "sensor_id" not in proc_data:
-                proc_data["sensor_id"] = sensor_id
-            if "start" not in proc_data:
-                proc_data["start"] = convert_to_solr(proc_createtime)
-        except Exception:
-            # silently fail if the GUID is not able to be parsed
-            pass
-
-        if isinstance(self.procguid, six.string_types):
-            # strip off the segment number since we're just looking for the parent process, not a specific event
-            child_unique_id = "-".join(self.procguid.split("-")[:5])
-        else:
-            child_unique_id = self.procguid
-
-        return self.parent._cb.select(Process, child_unique_id, initial_data=proc_data,
+        self.proc_data["path"] = self.path
+        self.proc_data["process_md5"] = self.md5
+        self.proc_data["process_pid"] = self.pid
+        self.proc_data["id"] = "-".join(self.procguid.split("-")[:5])
+        self.proc_data["unique_id"] = self.procguid
+        self.proc_data["terminated"] = self.terminated
+
+        self.proc_data["parent_id"] = self.parent.id
+        self.proc_data["parent_unique_id"] = self.parent._model_unique_id
+        self.proc_data["parent_md5"] = self.parent.process_md5
+        self.proc_data["parent_pid"] = self.parent.process_pid
+        self.proc_data["parent_name"] = self.parent.process_name
+        self.proc_data["group"] = self.parent._info["group"]
+        self.proc_data["os_type"] = self.parent._info["os_type"]
+        self.proc_data["hostname"] = self.parent._info["hostname"]
+        self.proc_data["interface_ip"] = self.parent._info["interface_ip"]
+
+        sensor_id, _, proc_createtime = parse_process_guid(self.procguid)
+        self.proc_data["start"] = convert_to_solr(proc_createtime)
+        self.proc_data["sensor_id"] = sensor_id
+
+        self.proc_data["last_update"] = self.proc_data["start"]
+        if self.terminated:
+            self.proc_data["last_update"] = convert_to_solr(self.timestamp)
+
+        return self.parent._cb.select(Process, self.procguid, initial_data=self.proc_data,
                                       suppressed_process=self.is_suppressed)
 
 

From fcf37f9ac52b8bab187c7308ceddfe2df54eaf62 Mon Sep 17 00:00:00 2001
From: Rafael Lukas Maers <rafael@mnemonic.no>
Date: Fri, 27 Sep 2019 12:32:04 +0200
Subject: [PATCH 02/13] Rewrite response.models.Process.require_events

The previous implementation was restricted to fetch the first 10k events
from the current segment. This implementation fetches all events limited
by the optional parameters for segment and update times.
---
 src/cbapi/response/models.py | 136 +++++++++++++++++++++++------------
 1 file changed, 92 insertions(+), 44 deletions(-)

diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py
index b3e48833..278d9cdd 100644
--- a/src/cbapi/response/models.py
+++ b/src/cbapi/response/models.py
@@ -2415,38 +2415,81 @@ def end(self):
         if self.get("terminated", False) and self.get("last_update") is not None:
             return convert_from_solr(self._attribute('last_update', -1))
 
-    def require_events(self):
-        event_key_list = ['filemod_complete', 'regmod_complete', 'modload_complete', 'netconn_complete',
-                          'crossproc_complete', 'childproc_complete']
+    def require_events(self, segment=0, min_last_update=None, max_last_update=None):
+        event_types = {
+                'filemod_count': 'filemod_complete',
+                'regmod_count': 'regmod_complete',
+                'modload_count': 'modload_complete',
+                'netconn_count': 'netconn_complete',
+                'crossproc_count': 'crossproc_complete',
+                'childproc_count': 'childproc_complete'
+                }
 
-        if not self.valid_process or self.suppressed_process:
+        if self._events_loaded:
+            return
+        if not self.valid_process:
+            return
+        if self.suppressed_process:
             return
 
-        if self.current_segment not in self._events:
-            self._events[self.current_segment] = {}
-            res = self._cb.get_object("/api/{0}/process/{1}/{2}/event".format(self._process_event_api, self.id,
-                                                                              self.current_segment)).get("process", {})
+        if segment not in self._events:
+            # Prepare
+            start = 0
+            count = 10000 # Backend default
+            base_url = '/api/{0}/process/{1}/{2}/event?'.format(self._process_event_api, self.id, segment)
+
+            if min_last_update:
+                base_url += 'cb.min_last_update={0:%Y-%m-%dT%H:%M:%SZ}&'.format(min_last_update)
+            if max_last_update:
+                base_url += 'cb.max_last_update={0:%Y-%m-%dT%H:%M:%SZ}&'.format(max_last_update)
+
+            events = {event_complete:[] for event_complete in event_types.values()}
+
+            # Fetch and store events
+            while True:
+                url = base_url
+                url += 'cb.event_start={0}&'.format(start)
+                url += 'cb.event_count={0}&'.format(count)
+
+                res = self._cb.get_object(url)
+                proc = res.get('process', {})
+
+                num_events = 0
+                for event_complete in event_types.values():
+                    proc_event_complete = proc.get(event_complete, [])
+                    num_events += len(proc_event_complete)
+                    events[event_complete].extend(proc_event_complete)
+
+                # Note that there might be more events even though we got less than we
+                # asked for. Thus, break out of loop once we don't recieve any more events.
+                if not num_events:
+                    break
 
-            # The key self.current_segment may have disappeared from self._events at this point. This likely indicates
-            # there are no events to get, but we won't take any chances; instead, we will simply reinitialize
-            # self._events[self.current_segment] and check for events anyway.
-            if self.current_segment not in self._events:
-                self._events[self.current_segment] = {}
+                start += count
 
-            for k in event_key_list:
-                self._events[self.current_segment][k] = res.get(k, [])
+            self._events[segment] = events
 
-            if not self._full_init:
-                for k in event_key_list:
-                    try:
-                        del res[k]
-                    except KeyError:
-                        pass
+            # Update process info
+            for event_count, event_complete in event_types.items():
+                if event_count in proc:
+                    del proc[event_count]
 
-                self._parse(res)
-                self._full_init = True
+                if event_complete in proc:
+                    del proc[event_complete]
+
+            self._parse(res)
+            #self._full_init = True
 
-            self._events_loaded = True
+            # Append events to process info
+            for event_count, event_complete in event_types.items():
+                self._info[event_complete] = self._info.get(event_complete, [])
+                self._info[event_count] = self._info.get(event_count, 0)
+
+                self._info[event_complete].extend(self._events[segment][event_complete])
+                self._info[event_count] += len(self._info[event_complete])
+
+        # All events loaded
+        self._events_loaded = segment == 0
 
     def refresh(self):
         # when refreshing a process, also zero out all the events
@@ -2465,12 +2508,13 @@ def modloads(self):
         """
         Generator that returns `:py:class:CbModLoadEvent` associated with this process
         """
-        self.require_events()
+        self.require_events(segment=self.current_segment)
 
         i = 0
-        for raw_modload in self._events.get(self.current_segment, {}).get('modload_complete', []):
-            yield self._event_parser.parse_modload(i, raw_modload)
-            i += 1
+        for segment in self._events:
+            for raw_modload in self._events[segment].get('modload_complete', []):
+                yield self._event_parser.parse_modload(i, raw_modload)
+                i += 1
 
     @property
     def unsigned_modloads(self):
@@ -2484,48 +2528,52 @@ def filemods(self):
         """
         Generator that returns :py:class:`CbFileModEvent` objects associated with this process
         """
-        self.require_events()
+        self.require_events(segment=self.current_segment)
 
         i = 0
-        for raw_filemod in self._events.get(self.current_segment, {}).get('filemod_complete', []):
-            yield self._event_parser.parse_filemod(i, raw_filemod)
-            i += 1
+        for segment in self._events:
+            for raw_filemod in self._events[segment].get('filemod_complete', []):
+                yield self._event_parser.parse_filemod(i, raw_filemod)
+                i += 1
 
     @property
     def netconns(self):
         """
         Generator that returns :py:class:`CbNetConnEvent` objects associated with this process
         """
-        self.require_events()
+        self.require_events(segment=self.current_segment)
 
         i = 0
-        for raw_netconn in self._events.get(self.current_segment, {}).get('netconn_complete', []):
-            yield self._event_parser.parse_netconn(i, raw_netconn)
-            i += 1
+        for segment in self._events:
+            for raw_netconn in self._events[segment].get('netconn_complete', []):
+                yield self._event_parser.parse_netconn(i, raw_netconn)
+                i += 1
 
     @property
     def regmods(self):
         """
         Generator that returns :py:class:`CbRegModEvent` objects associated with this process
         """
-        self.require_events()
+        self.require_events(segment=self.current_segment)
 
         i = 0
-        for raw_regmod in self._events.get(self.current_segment, {}).get('regmod_complete', []):
-            yield self._event_parser.parse_regmod(i, raw_regmod)
-            i += 1
+        for segment in self._events:
+            for raw_regmod in self._events[segment].get('regmod_complete', []):
+                yield self._event_parser.parse_regmod(i, raw_regmod)
+                i += 1
 
     @property
     def crossprocs(self):
         """
         Generator that returns :py:class:`CbCrossProcEvent` objects associated with this process
         """
-        self.require_events()
+        self.require_events(segment=self.current_segment)
 
         i = 0
-        for raw_crossproc in self._events.get(self.current_segment, {}).get('crossproc_complete', []):
-            yield self._event_parser.parse_crossproc(i, raw_crossproc)
-            i += 1
+        for segment in self._events:
+            for raw_crossproc in self._events[segment].get('crossproc_complete', []):
+                yield self._event_parser.parse_crossproc(i, raw_crossproc)
+                i += 1
 
     @property
     def parents(self):

From 1ca8c63ba51cd5d96f165bc12f6c5aa378dd551c Mon Sep 17 00:00:00 2001
From: Rafael Lukas Maers <rafael@mnemonic.no>
Date: Fri, 27 Sep 2019 12:35:09 +0200
Subject: [PATCH 03/13] Rewrite response.models.Process._parse

Update instead of overwriting information objects. Attempt to determine
some possibly missing fields from adjacent information objects. Ignore
invalid values and set default values for missing fields.
---
 src/cbapi/response/models.py | 339 ++++++++++++++++++++++++++++++++---
 1 file changed, 310 insertions(+), 29 deletions(-)

diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py
index 278d9cdd..65da60d1 100644
--- a/src/cbapi/response/models.py
+++ b/src/cbapi/response/models.py
@@ -2125,6 +2125,15 @@ class Process(TaggedModel):
     urlobject = '/api/v1/process'
     default_sort = 'last_update desc'
 
+    invalid_id = 'ffff-ffff-0000-000000000000'
+    invalid_pid = -1
+    invalid_name = '(unknown)'
+    invalid_start = -1
+
+    defaults = {                \
+        'process_md5': '0' * 32 \
+    }
+
     @classmethod
     def _query_implementation(cls, cb):
         return ProcessQuery(cls, cb)
@@ -2158,14 +2167,13 @@ def __init__(self, cb, procguid, segment=None, max_children=15, initial_data=Non
             log.debug("{0} is suppressed".format(procguid))
 
         self.valid_process = True
-        self._overrides = {}
 
         self._events_loaded = False
         self._events = {}
         self._segments = []
-        self.__parent_info = None
-        self.__children_info = None
-        self.__sibling_info = None
+        self.__parent_info = {}
+        self.__children_info = []
+        self.__sibling_info = []
 
         if cb.cb_server_version < LooseVersion('6.0.0'):
             self._default_segment = 1
@@ -2245,9 +2253,6 @@ def _sibling_info(self):
         return self.__sibling_info
 
     def _attribute(self, attrname, default=None):
-        if attrname in self._overrides:
-            return self._overrides[attrname]
-
         # workaround for Cb Response where parent_unique_id is returned as null
         # string as part of a query result. in this case we need to do a
         # full_init.
@@ -2277,32 +2282,308 @@ def _retrieve_cb_info(self, query_parameters=None):
             except KeyError:
                 pass
 
+    def _parse_id_vars(self, proc_info, parent_info, sibling_info, children_info):
+        INVALID_ID = self.__class__.invalid_id
+        INVALID_PID = self.__class__.invalid_pid
+        INVALID_START = self.__class__.invalid_start
+
+        # Each information object is uniquely identified by the value of unique_id, which is
+        # the concatenation of the process id and the segment id. The process id encodes the
+        # value of start and process_pid, which we decode and use if possible and necessary.
+
+        for info in chain([proc_info, parent_info], sibling_info, children_info):
+            # Verify id and unique_id
+            missing_or_invalid_id = INVALID_ID in info.get('id', INVALID_ID)
+            missing_or_invalid_uid = INVALID_ID in info.get('unique_id', INVALID_ID)
+
+            if missing_or_invalid_id and missing_or_invalid_uid:
+                continue
+
+            if missing_or_invalid_id:
+                #old_id = info.get('id')
+                info['id'] = info.get('unique_id')[:36]
+                #logging.debug('Replacing missing or invalid id {} with {} while parsing {}'.format(old_id, info.get('id'), self.id))
+
+            if missing_or_invalid_uid:
+                #old_uid = info.get('unique_id')
+                info['unique_id'] = '-'.join((info.get('id'), hex(self._default_segment)[2:].zfill(12)))
+                #logging.debug('Replacing missing or invalid uid {} with {} while parsing {}'.format(old_uid, info.get('unique_id'), self.id))
+
+            # Verify process_pid and start
+            missing_or_invalid_pid = info.get('process_pid', INVALID_PID) == INVALID_PID
+            missing_or_invalid_start = info.get('start', INVALID_START) == INVALID_START or info.get('start', INVALID_START) == None
+
+            if missing_or_invalid_pid or missing_or_invalid_start:
+                try:
+                    _, process_pid, start = parse_process_guid(info.get('id'))
+                except:
+                    pass
+                else:
+                    #old_pid = info.get('process_pid')
+                    #old_start = info.get('start')
+                    info['process_pid'] = process_pid
+                    info['start'] = convert_to_solr(start)
+                    #logging.debug('Replacing missing or invalid pid {} with {} while parsing {}'.format(old_pid, info.get('process_pid'), self.id))
+                    #logging.debug('Replacing missing or invalid start {} with {} while parsing {}'.format(old_start, info.get('start'), self.id))
+
+    def _parse_proc_vars(self, proc_info, parent_info, sibling_info, children_info):
+        INVALID_ID = self.__class__.invalid_id
+        INVALID_PID = self.__class__.invalid_pid
+        INVALID_NAME = self.__class__.invalid_name
+
+        # Certain values in the process information object are replicated in other information
+        # objects, e.g. the value of 'id' in the process information object equals the value of
+        # 'parent_id' in all of the children information objects. However, the data is not always
+        # consistent, resulting in incorrect results even though the correct data may be available
+        # somewhere else. This parsing function first retrieves and deduplicates all known values of
+        # different keys, deduplicates them, removes invalid ones and finally attempts to assert the
+        # correct value and write it back to all of the relevant information objects.
+
+        # Read subject process id, uid, pid and name from all possible objects
+        process_ids = set([self._info.get('id', INVALID_ID), proc_info.get('id', INVALID_ID)] + \
+                          [child.get('parent_id', INVALID_ID) for child in chain(self.__children_info, children_info)])
+        process_uids = set([self._info.get('unique_id', INVALID_ID), proc_info.get('unique_id', INVALID_ID)] + \
+                           [child.get('parent_unique_id', INVALID_ID) for child in chain(self.__children_info, children_info)])
+        process_pids = set([self._info.get('process_pid', INVALID_PID), proc_info.get('process_pid', INVALID_PID)] + \
+                           [child.get('parent_pid', INVALID_PID) for child in chain(self.__children_info, children_info)])
+        process_names = set([self._info.get('process_name', INVALID_NAME), proc_info.get('process_name', INVALID_NAME)] + \
+                            [child.get('parent_name', INVALID_NAME) for child in chain(self.__children_info, children_info)])
+
+        # Remove invalid values
+        process_ids = process_ids.difference(set([process_id for process_id in process_ids if INVALID_ID in process_id]))
+        process_uids = process_uids.difference(set([process_uid for process_uid in process_uids if INVALID_ID in process_uids]))
+        process_pids = process_pids.difference(set([INVALID_PID]))
+        process_names = process_names.difference(set([INVALID_NAME]))
+
+        # Set default values
+        process_id = proc_info.get('id', INVALID_ID)
+        process_uid = proc_info.get('unique_id', INVALID_ID)
+        process_pid = proc_info.get('process_pid', INVALID_PID)
+        process_name = proc_info.get('process_name', INVALID_NAME)
+
+        # We must have at least one ID, but no more than one valid
+        # process_id. Following is a table of acceptable combinations.
+        #
+        # OK | len(*_ids) | len(*_uids)
+        # N  |      0     |      0
+        # Y  |      0     |      1
+        # Y  |      0     |      2..
+        # Y  |      1     |      0
+        # Y  |      1     |      1
+        # Y  |      1     |      2..
+        # N  |      2..   |      0
+        # N  |      2..   |      1
+        # N  |      2..   |      2..
+
+        # Set valid values
+        if len(process_ids) == 1:
+            process_id = process_ids.pop()
+        elif not process_ids and process_uids:
+            process_id = process_uid.pop()[:36]
+        #else:
+        #    logging.debug('Missing, invalid or ambiguous process id(s) {} while parsing {}'.format(process_pids, self.id))
+
+        process_uid = '-'.join((process_id, hex(self._default_segment)[2:].zfill(12)))
+
+        if len(process_pids) == 1:
+            process_pid = process_pids.pop()
+        #else:
+        #    logging.debug('Missing, invalid or ambiguous process pid(s) {} while parsing {}'.format(process_pids, self.id))
+
+        if len(process_names) == 1:
+            process_name = process_names.pop()
+        #else:
+        #    logging.debug('Missing, invalid or ambiguous process name(s) {} while parsing {}'.format(process_names, self.id))
+
+        # Write values
+        for info in [self._info, proc_info]:
+            if not info:
+                continue
+
+            missing_or_invalid_id = INVALID_ID in info.get('id', INVALID_ID)
+            missing_or_invalid_uid = INVALID_ID in info.get('unique_id', INVALID_ID)
+            missing_or_invalid_pid = info.get('process_pid', INVALID_PID) == INVALID_PID
+            missing_or_invalid_name = info.get('process_name', INVALID_NAME) == INVALID_NAME
+
+            if missing_or_invalid_id:
+                #old_id = info.get('id')
+                info['id'] = process_id
+                #logging.debug('Replacing missing or invalid process id {} with {} while parsing {}'.format(old_id, process_id, self.id))
+
+            if missing_or_invalid_uid:
+                #old_uid = info.get('unique_id')
+                info['unique_id'] = process_uid
+                #logging.debug('Replacing missing or invalid process uid {} with {} while parsing {}'.format(old_uid, process_uid, self.id))
+
+            if missing_or_invalid_pid:
+                #old_pid = info.get('process_pid')
+                info['process_pid'] = process_pid
+                #logging.debug('Replacing missing or invalid process pid {} with {} while parsing {}'.format(old_pid, process_pid, self.id))
+
+            if missing_or_invalid_name:
+                #old_name = info.get('process_name')
+                info['process_name'] = process_name
+                #logging.debug('Replacing missing or invalid process name {} with {} while parsing {}'.format(old_name, process_name, self.id))
+
+        for info in chain(self.__children_info, children_info):
+            if not info:
+                continue
+
+            missing_or_invalid_id = INVALID_ID in info.get('parent_id', INVALID_ID)
+            missing_or_invalid_uid = INVALID_ID in info.get('parent_unique_id', INVALID_ID)
+            missing_or_invalid_pid = info.get('parent_pid', INVALID_PID) == INVALID_PID
+            missing_or_invalid_name = info.get('parent_name', INVALID_NAME) == INVALID_NAME
+
+            if missing_or_invalid_id:
+                #old_id = info.get('parent_id')
+                info['parent_id'] = process_id
+                #logging.debug('Replacing missing or invalid process id {} with {} while parsing {}'.format(old_id, process_id, self.id))
+
+            if missing_or_invalid_uid:
+                #old_uid = info.get('parent_unique_id')
+                info['parent_unique_id'] = process_uid
+                #logging.debug('Replacing missing or invalid process uid {} with {} while parsing {}'.format(old_uid, process_uid, self.id))
+
+            if missing_or_invalid_pid:
+                #old_pid = info.get('parent_pid')
+                info['parent_pid'] = process_pid
+                #logging.debug('Replacing missing or invalid process pid {} with {} while parsing {}'.format(old_pid, process_pid, self.id))
+
+            if missing_or_invalid_name:
+                #old_name = info.get('parent_name')
+                info['parent_name'] = process_name
+                #logging.debug('Replacing missing or invalid process name {} with {} while parsing {}'.format(old_name, process_name, self.id))
+
+    def _parse_parent_vars(self, proc_info, parent_info, sibling_info, children_info):
+        INVALID_ID = self.__class__.invalid_id
+        INVALID_PID = self.__class__.invalid_pid
+        INVALID_NAME = self.__class__.invalid_name
+
+        # See _parse_proc_vars for rationalization and overview. This parsing function does basically
+        # the same thing, except for parent_* values in the process information object.
+
+        parent_ids = set([info.get('parent_id', INVALID_ID) for info in \
+                          chain([self._info, proc_info], self.__sibling_info, sibling_info)])
+        parent_uids = set([info.get('parent_unique_id', INVALID_ID) for info in \
+                           chain([self._info, proc_info], self.__sibling_info, sibling_info)])
+        parent_pids = set([info.get('parent_pid', INVALID_PID) for info in \
+                           chain([self._info, proc_info], self.__sibling_info, sibling_info)])
+        parent_names = set([info.get('parent_name', INVALID_NAME) for info in \
+                            chain([self._info, proc_info], self.__sibling_info, sibling_info)])
+
+        parent_ids = parent_ids.difference(set([parent_id for parent_id in parent_ids if INVALID_ID in parent_id]))
+        parent_uids = parent_uids.difference(set([parent_uid for parent_uid in parent_uids if INVALID_ID in parent_uid]))
+        parent_pids = parent_pids.difference(set([INVALID_PID]))
+        parent_names = parent_names.difference(set([INVALID_NAME]))
+
+        parent_id = proc_info.get('parent_id', INVALID_ID)
+        parent_uid = proc_info.get('parent_unique_id', INVALID_ID)
+        parent_pid = proc_info.get('parent_pid', INVALID_PID)
+        parent_name = proc_info.get('parent_name', INVALID_NAME)
+
+        if len(parent_ids) == 1:
+            parent_id = parent_ids.pop()
+        elif not parent_ids and parent_uids:
+            parent_id = parent_uids.pop()[:36]
+        #else:
+        #    logging.debug('Missing, invalid or ambiguous parent id(s) {} while parsing {}'.format(parent_ids, self.id))
+
+        parent_uid = '-'.join((parent_id, hex(self._default_segment)[2:].zfill(12)))
+
+        if len(parent_pids) == 1:
+            parent_pid = parent_pids.pop()
+        #else:
+        #    logging.debug('Missing, invalid or ambiguous parent pid(s) {} while parsing {}'.format(parent_pids, self.id))
+
+        if len(parent_names) == 1:
+            parent_name = parent_names.pop()
+        #else:
+        #    logging.debug('Missing, invalid or ambiguous parent name(s) {} while parsing {}'.format(parent_names, self.id))
+
+        for info in chain([self._info, proc_info], self.__sibling_info, sibling_info):
+            if not info:
+                continue
+
+            missing_or_invalid_id = INVALID_ID in info.get('parent_id', INVALID_ID)
+            missing_or_invalid_uid = INVALID_ID in info.get('parent_unique_id', INVALID_ID)
+            missing_or_invalid_pid = info.get('parent_pid', INVALID_PID) == INVALID_PID
+            missing_or_invalid_name = info.get('parent_name', INVALID_NAME) == INVALID_NAME
+
+            if missing_or_invalid_id:
+                #old_id = info.get('parent_id')
+                info['parent_id'] = parent_id
+                #logging.debug('Replacing missing or invalid parent id {} with {} while parsing {}'.format(old_id, parent_id, self.id))
+
+            if missing_or_invalid_uid:
+                #old_uid = info.get('parent_unique_id')
+                info['parent_unique_id'] = parent_uid
+                #logging.debug('Replacing missing or invalid parent uid {} with {} while parsing {}'.format(old_uid, parent_uid, self.id))
+
+            if missing_or_invalid_pid:
+                #old_pid = info.get('parent_pid')
+                info['parent_pid'] = parent_pid
+                #logging.debug('Replacing missing or invalid parent pid {} with {} while parsing {}'.format(old_pid, parent_pid, self.id))
+
+            if missing_or_invalid_name:
+                #old_name = info.get('parent_name')
+                info['parent_name'] = parent_name
+                #logging.debug('Replacing missing or invalid parent name {} with {} while parsing {}'.format(old_name, parent_name, self.id))
+
     def _parse(self, obj):
-        if "process" in obj:
-            self._info = obj.get("process", {})
-            self.__parent_info = obj.get("parent", {})
-            self.__children_info = obj.get("children", [])
-            self.__sibling_info = obj.get("siblings", [])
-        else:
-            self._info = obj.copy()
+        # Get information objects
+        proc_info = obj.get("process", obj)
+        parent_info = obj.get("parent", dict())
+        sibling_info = obj.get("siblings", list())
+        children_info = obj.get("children", list())
 
-        if self._info and (self._info.get("start", -1) == -1 or self._info.get("process_pid", -1) == -1):
-            log.debug("Process ID %s is invalid; start time or process PID are empty or -1." % self.id)
-            log.debug("Attempting to reverse-engineer start time and process PID from process GUID")
+        # Determine id, uid, pid, start for all information objects
+        self._parse_id_vars(proc_info, parent_info, sibling_info, children_info)
 
-            try:
-                sensor_id, proc_pid, start_time = parse_process_guid(self.id)
-                if proc_pid > 0 and proc_pid != -1:
-                    self._overrides["sensor_id"] = sensor_id
-                    self._overrides["process_pid"] = proc_pid
-                    self._overrides["start"] = convert_to_solr(start_time)
-                    log.debug("Recovered start time and process PID from GUID for process %s" % self.id)
+        # Determine process_id, process_uid, process_pid and process_name for process, sibling and children information objects
+        self._parse_proc_vars(proc_info, parent_info, sibling_info, children_info)
+
+        # Determine parent_id, parent_unique_id, parent_pid, parent_name for process and sibling information objects
+        self._parse_parent_vars(proc_info, parent_info, sibling_info, children_info)
+
+        # Avoid updating keys that are already populated and have no defaults with invalid values
+        if self._info.get('cmdline') and not proc_info.get('cmdline'):
+            proc_info.pop('cmdline', None)
+
+        if self._info.get('username') and not proc_info.get('username'):
+            proc_info.pop('username', None)
+
+        if self._info.get('start') and proc_info.get('start', int()) == -1:
+            proc_info.pop('start', None)
+
+        # Avoid updating keys that have defaults with invalid values
+        if len(proc_info.get('process_md5', str())) != 32:
+            proc_info.pop('process_md5', None)
+
+        # Update and extend
+        def update(old, new):
+            info_dict = {info['id']:info for info in old}
+
+            for info in new:
+                key = info['id']
+
+                if key not in info_dict:
+                    info_dict[key] = info
                 else:
-                    log.debug("Unable to recover start time and process PID for process %s, marking invalid" % self.id)
-                    self.valid_process = False
-            except Exception:
-                log.debug("Unable to parse process GUID %s, marking invalid" % self.id)
-                self.valid_process = False
+                    info_dict[key].update(info)
+
+            return list(info_dict.values())
+
+        self._info.update(proc_info)
+        self.__parent_info.update(parent_info)
+        self.__sibling_info = update(self.__sibling_info, sibling_info)
+        self.__children_info = update(self.__children_info, children_info)
+
+        # Set missing defaults
+        defaults = self.__class__.defaults
+
+        for kw in defaults:
+            if kw not in self._info:
+                self._info[kw] = defaults[kw]
 
     def walk_parents(self, callback, max_depth=0, depth=0):
         """

From 86cc59267cac508afd9cc17b0b42584d45a521fe Mon Sep 17 00:00:00 2001
From: Rafael Lukas Maers <rafael@mnemonic.no>
Date: Fri, 27 Sep 2019 12:52:49 +0200
Subject: [PATCH 04/13] Modify response.models.Process.children

Return a union of the children in the process summary and childprocs as
either one of the two might be incomplete. Furthermore, return as actual
processes instead of childproc events for consistency with the REST API
and the parents method.
---
 src/cbapi/response/models.py | 104 ++++++++++++++++++++---------------
 1 file changed, 60 insertions(+), 44 deletions(-)

diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py
index 65da60d1..d425d49b 100644
--- a/src/cbapi/response/models.py
+++ b/src/cbapi/response/models.py
@@ -16,6 +16,7 @@
 import logging
 import time
 
+from itertools import chain
 from cbapi.utils import convert_query_params
 from ..errors import InvalidObjectError, ApiError, TimeoutError
 from ..oldmodels import BaseModel, immutable
@@ -2856,64 +2857,79 @@ def crossprocs(self):
                 yield self._event_parser.parse_crossproc(i, raw_crossproc)
                 i += 1
 
+    @property
+    def childprocs(self):
+        """
+        Generator that returns :py:class:`CbChildProcEvent` objects associated with this process
+        """
+        self.require_events(segment=self.current_segment)
+
+        i = 0
+        for segment in self._events:
+            for raw_childproc in self._events[segment].get('childproc_complete', []):
+                yield self._event_parser.parse_childproc(i, raw_childproc)
+                i += 1
+
     @property
     def parents(self):
         current_process = self
         while True:
-            try:
-                parent = current_process.parent
-                if not(parent) or parent.get('process_pid', -1) == -1:
-                    break
-                yield parent
-                current_process = parent
-            except ObjectNotFoundError:
-                return
+                try :
+                    parent = current_process.parent
+                    if not(parent) or parent.get('process_pid',-1) == -1:
+                        break
+                    yield parent
+                    current_process = parent
+                except ObjectNotFoundError:
+                    return
         return
 
     @property
     def children(self):
         """
-        Generator that returns :py:class:`CbChildProcEvent` objects associated with this process
+        Generator that returns :py:class:`Process` objects of the children of this process
         """
 
-        # Try and take a shortcut; generate the list of children from the process "summary" rather than the list
-        # of raw events first.
-
-        # This is a little different from how the children() method worked before; the most noticeable differences are:
-        # - only one entry is present per child (before there were up to two; one for the spawn event, one for the
-        #   terminate event)
-        # - the timestamp is derived from the start time of the process, not the timestamp from the spawn event.
-        #   the two timestamps will be off by a few microseconds.
-
-        if self._children_info is not None:
-            for i, child in enumerate(self._children_info):
-                timestamp = convert_event_time(child.get("start") or "1970-01-01T00:00:00Z")
-                yield CbChildProcEvent(self, timestamp, i,
-                                       {
-                                           "procguid": child.get("unique_id", None),
-                                           "md5": child.get("process_md5", None),
-                                           "pid": child.get("process_pid", None),
-                                           "path": child.get("path", None),
-                                           "terminated": False
-                                       },
-                                       is_suppressed=child.get("is_suppressed", False),
-                                       proc_data=child)
-        else:
-            for cp in self.childprocs:
-                yield cp
+        # Generate a list of children as the union of the children in the process summary and childprocs, as either
+        # one of the two might be incomplete.
 
-    @property
-    def childprocs(self):
-        """
-        Generator that returns :py:class:`CbChildProcEvent` objects associated with this process
-        """
+        def parse_childinfo(childinfo):
+            procguid = childinfo['unique_id']
+            suppressed_process = childinfo.get('is_suppressed', False)
 
-        self.require_events()
+            childinfo['parent_id'] = self.id
+            childinfo['parent_unique_id'] = self._model_unique_id
+            childinfo['parent_md5'] = self.process_md5
+            childinfo['parent_pid'] = self.process_pid
+            childinfo['parent_name'] = self.process_name
+            childinfo['group'] = self._info['group']
+            childinfo['os_type'] = self._info['os_type']
+            childinfo['hostname'] = self._info['hostname']
+            childinfo['interface_ip'] = self._info['interface_ip']
 
-        i = 0
-        for raw_childproc in self._events.get(self.current_segment, {}).get('childproc_complete', []):
-            yield self._event_parser.parse_childproc(i, raw_childproc)
-            i += 1
+            if not childinfo.get('id'):
+                childinfo['id'] = '-'.join(procguid.split('-')[:5])
+
+            if not childinfo.get('start'):
+                proc_createtime = parse_process_guid(procguid)[2]
+                childinfo['start'] = convert_to_solr(proc_createtime)
+
+            if not childinfo.get('last_update') and childinfo.get('lastUpdate'):
+                childinfo['last_update'] = childinfo['lastUpdate']
+                del childinfo['lastUpdate']
+
+            return self._cb.select(Process, procguid, initial_data=childinfo, suppressed_process=suppressed_process)
+
+        childprocs = (childproc.process for childproc in self.childprocs)
+        childinfos = (parse_childinfo(childinfo) for childinfo in self._children_info)
+
+        ids = set()
+        for child in chain(childprocs, childinfos):
+            if child.id in ids:
+                continue
+
+            ids.add(child.id)
+            yield child
 
     @property
     def all_events_segment(self):

From 57b59758e2b8af6633fac3860abf65e896b270a0 Mon Sep 17 00:00:00 2001
From: Rafael Lukas Maers <rafael@mnemonic.no>
Date: Fri, 27 Sep 2019 12:55:39 +0200
Subject: [PATCH 05/13] Remove response.models.Process.all_* and
 require_all_events

These methods have been made obsolete by the recent changes to the
require_events and related events methods.
---
 src/cbapi/response/models.py | 174 -----------------------------------
 1 file changed, 174 deletions(-)

diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py
index d425d49b..34cae5b9 100644
--- a/src/cbapi/response/models.py
+++ b/src/cbapi/response/models.py
@@ -3213,180 +3213,6 @@ def username(self):
         """
         return self.get("username", None)
 
-    def require_all_events(self):
-        event_types = {
-                'filemod_count': 'filemod_complete',
-                'regmod_count': 'regmod_complete',
-                'modload_count': 'modload_complete',
-                'netconn_count': 'netconn_complete',
-                'crossproc_count': 'crossproc_complete',
-                'childproc_count': 'childproc_complete'
-                }
-
-        if not self.valid_process or self.suppressed_process:
-            return
-
-        # Get all events
-        res = self._cb.get_object("/api/{0}/process/{1}/{2}/event".format(self._process_event_api,
-                                                                          self.id, 0)).get("process", {})
-
-        self._events['all'] = {}
-        for (event_count, event_complete) in event_types.items():
-            complete_events = res.get(event_complete, 0)
-            if complete_events:
-                self._info[event_count] = len(complete_events)
-                self._info[event_complete] = complete_events
-                self._events['all'][event_count] = self._info[event_count]
-                self._events['all'][event_complete] = self._info[event_complete]
-
-        # total_events = sum([self._info[event_count] for event_count in event_types])
-
-        # Delete references in res.
-        if not self._full_init:
-            for k in event_types.keys():
-                try:
-                    del res[k]
-                except KeyError:
-                    pass
-
-            self._parse(res)
-            self._full_init = True
-
-        self.all_events_loaded = True
-
-    def all_childprocs(self):
-        if self._cb.cb_server_version < LooseVersion('6.0.0'):
-            self.get_segments()
-            segments = self._segments
-
-            i = 0
-            for segment in segments:
-                self.current_segment = segment
-                self.require_events()
-
-                for raw_childproc in self._events.get(self.current_segment, {}).get('childproc_complete', []):
-                    yield self._event_parser.parse_childproc(i, raw_childproc)
-                    i += 1
-        else:
-            if not self.all_events_loaded:
-                self.require_all_events()
-
-            i = 0
-            for raw_childproc in self._events.get('all', {}).get('childproc_complete', []):
-                yield self._event_parser.parse_childproc(i, raw_childproc)
-                i += 1
-
-    def all_modloads(self):
-        if self._cb.cb_server_version < LooseVersion('6.0.0'):
-            self.get_segments()
-            segments = self._segments
-
-            i = 0
-            for segment in segments:
-                self.current_segment = segment
-                self.require_events()
-
-                for raw_modload in self._events.get(self.current_segment, {}).get('modload_complete', []):
-                    yield self._event_parser.parse_modload(i, raw_modload)
-                    i += 1
-        else:
-            if not self.all_events_loaded:
-                self.require_all_events()
-
-            i = 0
-            for raw_modload in self._events.get('all', {}).get('modload_complete', []):
-                yield self._event_parser.parse_modload(i, raw_modload)
-                i += 1
-
-    def all_filemods(self):
-        if self._cb.cb_server_version < LooseVersion('6.0.0'):
-            self.get_segments()
-            segments = self._segments
-
-            i = 0
-            for segment in segments:
-                self.current_segment = segment
-                self.require_events()
-
-                for raw_filemod in self._events.get(self.current_segment, {}).get('filemod_complete', []):
-                    yield self._event_parser.parse_filemod(i, raw_filemod)
-                    i += 1
-        else:
-            if not self.all_events_loaded:
-                self.require_all_events()
-
-            i = 0
-            for raw_filemod in self._events.get('all', {}).get('filemod_complete', []):
-                yield self._event_parser.parse_filemod(i, raw_filemod)
-                i += 1
-
-    def all_regmods(self):
-        if self._cb.cb_server_version < LooseVersion('6.0.0'):
-            self.get_segments()
-            segments = self._segments
-
-            i = 0
-            for segment in segments:
-                self.current_segment = segment
-                self.require_events()
-
-                for raw_regmod in self._events.get(self.current_segment, {}).get('regmod_complete', []):
-                    yield self._event_parser.parse_regmod(i, raw_regmod)
-                    i += 1
-        else:
-            if not self.all_events_loaded:
-                self.require_all_events()
-
-            i = 0
-            for raw_regmod in self._events.get('all', {}).get('regmod_complete', []):
-                yield self._event_parser.parse_regmod(i, raw_regmod)
-                i += 1
-
-    def all_crossprocs(self):
-        if self._cb.cb_server_version < LooseVersion('6.0.0'):
-            self.get_segments()
-            segments = self._segments
-
-            i = 0
-            for segment in segments:
-                self.current_segment = segment
-                self.require_events()
-
-                for raw_crossproc in self._events.get(self.current_segment, {}).get('crossproc_complete', []):
-                    yield self._event_parser.parse_crossproc(i, raw_crossproc)
-                    i += 1
-        else:
-            if not self.all_events_loaded:
-                self.require_all_events()
-
-            i = 0
-            for raw_crossproc in self._events.get('all', {}).get('crossproc_complete', []):
-                yield self._event_parser.parse_crossproc(i, raw_crossproc)
-                i += 1
-
-    def all_netconns(self):
-        if self._cb.cb_server_version < LooseVersion('6.0.0'):
-            self.get_segments()
-            segments = self._segments
-
-            i = 0
-            for segment in segments:
-                self.current_segment = segment
-                self.require_events()
-
-                for raw_netconn in self._events.get(self.current_segment, {}).get('netconn_complete', []):
-                    yield self._event_parser.parse_netconn(i, raw_netconn)
-                    i += 1
-        else:
-            if not self.all_events_loaded:
-                self.require_all_events()
-
-            i = 0
-            for raw_netconn in self._events.get('all', {}).get('netconn_complete', []):
-                yield self._event_parser.parse_netconn(i, raw_netconn)
-                i += 1
-
-
 def get_constants(prefix):
     return dict((getattr(socket, n), n)
                 for n in dir(socket)

From 257f8ebd1190fcf363e3d43d8aa44f845c2415db Mon Sep 17 00:00:00 2001
From: Rafael Lukas Maers <rafael@mnemonic.no>
Date: Fri, 27 Sep 2019 12:59:49 +0200
Subject: [PATCH 06/13] Deduplicate process guid parsing code

---
 src/cbapi/response/models.py | 31 +++++++++++--------------------
 1 file changed, 11 insertions(+), 20 deletions(-)

diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py
index 34cae5b9..cab56f15 100644
--- a/src/cbapi/response/models.py
+++ b/src/cbapi/response/models.py
@@ -2145,7 +2145,8 @@ def new_object(cls, cb, item, max_children=15):
         return cb.select(Process, item['id'] or item['unique_id'], long(item['segment_id']),
                          max_children, initial_data=item)
 
-    def parse_guid(self, procguid):
+    @staticmethod
+    def parse_guid(procguid, cb_server_version):
         try:
             # old 4.x process IDs are integers.
             return int(procguid), None
@@ -2153,7 +2154,7 @@ def parse_guid(self, procguid):
             # new 5.x process IDs are hex strings with optional segment IDs.
             if len(procguid) == 45:
                 return procguid[:36], int(procguid[38:], 16)
-            elif len(procguid) == 49 and self._cb.cb_server_version >= LooseVersion('6.0.0'):
+            elif len(procguid) == 49 and cb_server_version >= LooseVersion('6.0.0'):
                 return procguid[:36], int(procguid[38:], 16)
             else:
                 return None, None
@@ -2181,23 +2182,13 @@ def __init__(self, cb, procguid, segment=None, max_children=15, initial_data=Non
         else:
             self._default_segment = 0
 
-        try:
-            # old 4.x process IDs are integers.
-            self.id = int(procguid)
-        except ValueError:
-            # new 5.x process IDs are hex strings with optional segment IDs.
-            if len(procguid) == 45:
-                self.id = procguid[:36]
-                self.current_segment = int(procguid[38:], 16)
-            elif len(procguid) == 49 and cb.cb_server_version >= LooseVersion('6.0.0'):
-                self.id = procguid[:36]
-                self.current_segment = int(procguid[38:], 16)
-            else:
-                self.id = procguid
-                if len(procguid) != 36:
-                    log.debug("Invalid process GUID: %s, declaring this process as invalid" % procguid)
-                    self.valid_process = False
-                    self._full_init = True
+        self.id, self.current_segment = Process.parse_guid(procguid, cb.cb_server_version)
+        if not (self.id and self.current_segment):
+            self.id = procguid
+            if len(procguid) != 36:
+                log.debug("Invalid process GUID: %s, declaring this process as invalid" % procguid)
+                self.valid_process = False
+                self._full_init = True
 
         if not self.current_segment:
             self.current_segment = self._default_segment
@@ -2953,7 +2944,7 @@ def get_segments(self):
                 log.debug("using segment API route for cb response server >= 6.0")
                 res = self._cb.get_object("/api/v1/process/{0}/segment".format(self.id))\
                     .get("process", {}).get("segments", {})
-                proclist = [self.parse_guid(x["unique_id"])[1] for x in res]
+                proclist = [Process.parse_guid(x["unique_id"], self._cb.cb_server_version)[1] for x in res]
 
             self._segments = proclist
 

From 8ea974864d4f168ed98ee24562494078a386855a Mon Sep 17 00:00:00 2001
From: Rafael Lukas Maers <rafael@mnemonic.no>
Date: Sat, 28 Sep 2019 23:53:25 +0200
Subject: [PATCH 07/13] Fix for response.models.ProcessQuery.num_children

If a selection method is called after the max_children method, then the
num_children member is reset to the default since it's not cloned. This
commit fixes the issue.
---
 src/cbapi/response/models.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py
index cab56f15..dc212d52 100644
--- a/src/cbapi/response/models.py
+++ b/src/cbapi/response/models.py
@@ -1388,6 +1388,14 @@ def __init__(self, doc_class, cb, query=None, raw_query=None):
         if cb._has_legacy_partitions:
             self._default_args["cb.legacy_5x_mode"] = True
 
+    def _clone(self):
+        nq = self.__class__(self._doc_class, self._cb, self._query, self._raw_query)
+        nq._default_args = copy.deepcopy(self._default_args)
+        nq.num_children = self.num_children
+        nq._batch_size = self._batch_size
+        nq._sort_by = self._sort_by
+        return nq
+
     def group_by(self, field_name):
         """Set the group-by field name for this query. Typically, you will want to set this to 'id' if you only want
         one result per process.

From 22fc720ced9ebf2d587d0d4a8a3af741cac4f713 Mon Sep 17 00:00:00 2001
From: Rafael Lukas Maers <rafael@mnemonic.no>
Date: Wed, 2 Oct 2019 08:22:59 +0200
Subject: [PATCH 08/13] Simplify response.utils.convert_from_solr

Use dateutil.parser as mentioned by the TODO and used by the other time
and date conversion functions.
---
 src/cbapi/response/utils.py | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/src/cbapi/response/utils.py b/src/cbapi/response/utils.py
index 6288befc..07d24c22 100644
--- a/src/cbapi/response/utils.py
+++ b/src/cbapi/response/utils.py
@@ -37,17 +37,11 @@ def convert_to_solr(dt):
     return dt.strftime(solr_datetime_format)
 
 
-# TODO: change these to use dateutil.parser (see http://labix.org/python-dateutil)
 def convert_from_solr(s):
     if s == -1:
-        # special case for invalid processes
-        return datetime.fromtimestamp(0)
-
-    try:
-        return datetime.strptime(s, solr_datetime_format)
-    except ValueError:
-        # try interpreting the timestamp without the milliseconds
-        return datetime.strptime(s, sign_datetime_format)
+        return dateutil.parser.parse("1970-01-01T00:00:00Z")
+
+    return dateutil.parser.parse(s)
 
 
 def convert_from_cb(s):

From d2291fc566450ad90715513c0c2f9297d0854f7f Mon Sep 17 00:00:00 2001
From: Rafael Lukas Maers <rafael@mnemonic.no>
Date: Wed, 2 Oct 2019 08:23:39 +0200
Subject: [PATCH 09/13] Small identation fix in response/util.py


From 2bb5153e4ba7d16ba4af1ca319d351bab13d3235 Mon Sep 17 00:00:00 2001
From: Rafael Lukas Maers <rafael@mnemonic.no>
Date: Wed, 2 Oct 2019 08:30:31 +0200
Subject: [PATCH 10/13] Fix response.models.Process.min/max_last_update

---
 src/cbapi/response/models.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py
index dc212d52..e7147c34 100644
--- a/src/cbapi/response/models.py
+++ b/src/cbapi/response/models.py
@@ -3170,19 +3170,20 @@ def last_update(self):
         """
         return convert_from_solr(self.get('last_update', -1))
 
+
     @property
     def min_last_update(self):
         """
         Returns a pretty version of the earliest event in this process segment
         """
-        return convert_from_solr(self.get('last_update', -1))
+        return convert_from_solr(self.get('min_last_update', -1))
 
     @property
     def max_last_update(self):
         """
         Returns a pretty version of the latest event in this process segment
         """
-        return convert_from_solr(self.get('last_update', -1))
+        return convert_from_solr(self.get('max_last_update', -1))
 
     @property
     def last_server_update(self):

From 666b53d02d700d04f9f0a6402d27cb5505a2582d Mon Sep 17 00:00:00 2001
From: Rafael Lukas Maers <rafael@mnemonic.no>
Date: Wed, 9 Oct 2019 15:35:17 +0200
Subject: [PATCH 11/13] Cleanup parent_md5 in response.models.Process

Deduplicate and move the removal of parent_md5 to the _parse method.
---
 src/cbapi/response/models.py | 21 ++++++---------------
 1 file changed, 6 insertions(+), 15 deletions(-)

diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py
index e7147c34..0e0076e4 100644
--- a/src/cbapi/response/models.py
+++ b/src/cbapi/response/models.py
@@ -2224,12 +2224,6 @@ def __init__(self, cb, procguid, segment=None, max_children=15, initial_data=Non
             # fill in data object for performance
             self._parse({"process": copy.deepcopy(initial_data)})
 
-        # 7/31/2018: remove parent_md5 from the _info array, as it's never set.
-        #  instead add an accessor to transparently request the process_md5 from the parent.
-        try:
-            del self._info["parent_md5"]
-        except KeyError:
-            pass
 
         if force_init:
             self.refresh()
@@ -2271,16 +2265,8 @@ def _build_api_request_uri(self):
         return "/api/v1/process/{0}/{1}?children={2}".format(self.id, self._default_segment, self.max_children)
 
     def _retrieve_cb_info(self, query_parameters=None):
-        if self.suppressed_process or not self.valid_process:
-            return
-        else:
+        if self.valid_process and not self.suppressed_process:
             super(Process, self)._retrieve_cb_info(query_parameters)
-            # 7/31/2018: remove parent_md5 from the _info array, as it's never set.
-            #  instead add an accessor to transparently request the process_md5 from the parent.
-            try:
-                del self._info["parent_md5"]
-            except KeyError:
-                pass
 
     def _parse_id_vars(self, proc_info, parent_info, sibling_info, children_info):
         INVALID_ID = self.__class__.invalid_id
@@ -2585,6 +2571,11 @@ def update(old, new):
             if kw not in self._info:
                 self._info[kw] = defaults[kw]
 
+        # 7/31/2018: remove parent_md5 from the _info array, as it's never set.
+        #  instead add an accessor to transparently request the process_md5 from the parent.
+        if self._info.get('parent_md5'):
+            del self._info['parent_md5']
+
     def walk_parents(self, callback, max_depth=0, depth=0):
         """
         Walk up the execution chain while calling the specified callback function at each depth.

From 5352ab80e5ad44160e62634da5dadf8bc8a5bbf6 Mon Sep 17 00:00:00 2001
From: Rafael Lukas Maers <rafael@mnemonic.no>
Date: Wed, 9 Oct 2019 15:41:05 +0200
Subject: [PATCH 12/13] Cleanup response.models.Process.__init__

Ensure that _full_init is True for suppressed processes by calling the
super class constructor before setting the value, and cleanup while we
are at it.
---
 src/cbapi/response/models.py | 85 +++++++++++++++++-------------------
 1 file changed, 41 insertions(+), 44 deletions(-)

diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py
index 0e0076e4..a375317e 100644
--- a/src/cbapi/response/models.py
+++ b/src/cbapi/response/models.py
@@ -2167,63 +2167,60 @@ def parse_guid(procguid, cb_server_version):
             else:
                 return None, None
 
-    def __init__(self, cb, procguid, segment=None, max_children=15, initial_data=None, force_init=False,
-                 suppressed_process=False):
-        self.max_children = max_children
-        self.current_segment = segment
-        self.suppressed_process = suppressed_process
-        if suppressed_process:
-            self._full_init = True
-            log.debug("{0} is suppressed".format(procguid))
-
-        self.valid_process = True
-
-        self._events_loaded = False
-        self._events = {}
-        self._segments = []
-        self.__parent_info = {}
-        self.__children_info = []
-        self.__sibling_info = []
-
-        if cb.cb_server_version < LooseVersion('6.0.0'):
-            self._default_segment = 1
-        else:
-            self._default_segment = 0
-
-        self.id, self.current_segment = Process.parse_guid(procguid, cb.cb_server_version)
-        if not (self.id and self.current_segment):
-            self.id = procguid
-            if len(procguid) != 36:
-                log.debug("Invalid process GUID: %s, declaring this process as invalid" % procguid)
-                self.valid_process = False
-                self._full_init = True
-
-        if not self.current_segment:
-            self.current_segment = self._default_segment
-
-        super(Process, self).__init__(cb, self.id)
-
+    def __init__(self, cb, procguid, segment=None, max_children=15, initial_data=None, force_init=False, suppressed_process=False):
+        # Set version specific members
+        self._default_segment = 1
+        self._process_event_api = 'v1'
         self._process_summary_api = 'v1'
+        self._event_parser = ProcessV1Parser(self)
 
         if cb.cb_server_version >= LooseVersion('6.0.0'):
-            self._process_summary_api = 'v2'
+            self._default_segment = 0
             self._process_event_api = 'v4'
+            self._process_summary_api = 'v2'
             self._event_parser = ProcessV4Parser(self)
         elif cb.cb_server_version >= LooseVersion('5.2.0'):
             self._process_event_api = 'v3'
             self._event_parser = ProcessV3Parser(self)
         elif cb.cb_server_version >= LooseVersion('5.1.0'):
-            # CbER 5.1.0 introduced an extended event API
             self._process_event_api = 'v2'
             self._event_parser = ProcessV2Parser(self)
-        else:
-            self._process_event_api = 'v1'
-            self._event_parser = ProcessV1Parser(self)
 
-        if initial_data:
-            # fill in data object for performance
-            self._parse({"process": copy.deepcopy(initial_data)})
+        # Set id and segment
+        self.id, self.current_segment = Process.parse_guid(procguid, cb.cb_server_version)
+
+        if not self.id:
+            self.id = procguid
+
+        if not self.current_segment:
+            self.current_segment = segment or self._default_segment
 
+        # NOTE: Sets self._full_init = False
+        super(Process, self).__init__(cb, self.id)
+
+        # Set object members
+        self._events = {}
+        self._segments = []
+        self.__parent_info = {}
+        self.__sibling_info = []
+        self.__children_info = []
+        self.valid_process = True
+        self._events_loaded = False
+        self.max_children = max_children
+        self.suppressed_process = suppressed_process
+
+        if len(self.id) != 36:
+            self._full_init = True
+            self.valid_process = False
+            log.debug('Invalid process GUID: {}, declaring this process as invalid'.format(self.id))
+
+        if self.suppressed_process:
+            self._full_init = True
+            log.debug('{0} is suppressed'.format(self.id))
+
+        # Parse data
+        if initial_data:
+            self._parse({'process': copy.deepcopy(initial_data)})
 
         if force_init:
             self.refresh()

From ca8abf81d71758d9b374c15f155f8bbe3038c5f2 Mon Sep 17 00:00:00 2001
From: Rafael Lukas Maers <rafael@mnemonic.no>
Date: Wed, 16 Oct 2019 10:26:16 +0200
Subject: [PATCH 13/13] Use the specified process summary API version

---
 src/cbapi/response/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cbapi/response/models.py b/src/cbapi/response/models.py
index a375317e..a824a68c 100644
--- a/src/cbapi/response/models.py
+++ b/src/cbapi/response/models.py
@@ -2259,7 +2259,7 @@ def _attribute(self, attrname, default=None):
         return super(Process, self)._attribute(attrname, default=default)
 
     def _build_api_request_uri(self):
-        return "/api/v1/process/{0}/{1}?children={2}".format(self.id, self._default_segment, self.max_children)
+        return "/api/{0}/process/{1}/{2}?children={3}".format(self._process_summary_api, self.id, self._default_segment, self.max_children)
 
     def _retrieve_cb_info(self, query_parameters=None):
         if self.valid_process and not self.suppressed_process:
