ET-Telemetry, limit logs shipped to rules received from Proofpoint.

security/etpro-telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/__init__.py

@@ -36,6 +36,7 @@ import ujson
 
 
 BASE_URL = 'https://opnsense.emergingthreats.net'
+RELATED_SIDS_FILE = '/usr/local/etc/suricata/rules/telemetry_sids.txt'
 
 
 def get_config(rule_update_config):
@@ -62,7 +63,30 @@ class EventCollector(object):
     def __init__(self):
         self._tmp_handle = tempfile.NamedTemporaryFile()
         self._local_networks = list()
+        self._our_sids = set()
         self._get_local_networks()
+        self._get_our_sids()
+
+    def _get_our_sids(self):
+        """ collect sids of interest, which are part of the ET-Telemetry delivery
+        :return: None
+        """
+        if os.path.isfile(RELATED_SIDS_FILE):
+            for line in open(RELATED_SIDS_FILE, 'r'):
+                if line.strip().isdigit():
+                    self._our_sids.add(int(line.strip()))
+
+    def _is_rule_of_interest(self, record):
+        """ check if rule is of interest for delivery
+        :param record: parsed eve log record
+        :return: boolean
+        """
+        if not self._our_sids:
+            return True
+        elif 'alert' in record and 'signature_id' in record['alert']:
+            if record['alert']['signature_id'] in self._our_sids:
+                return True
+        return False
 
     def _get_local_networks(self):
         """ collect local attached networks for anonymization purposes
@@ -103,26 +127,27 @@ class EventCollector(object):
         :param record: parsed eve log record
         :return: None
         """
-        to_push = dict()
-        for address in ['src_ip', 'dest_ip']:
-            if address in record:
-                if self.is_local_address(record[address]):
-                    if record[address].find(':') > -1:
-                        # replace local IPv6 address
-                        to_push[address] = 'xxxx:xxxx:%s' % ':'.join(record[address].split(':')[-2:])
+        if self._is_rule_of_interest(record):
+            to_push = dict()
+            for address in ['src_ip', 'dest_ip']:
+                if address in record:
+                    if self.is_local_address(record[address]):
+                        if record[address].find(':') > -1:
+                            # replace local IPv6 address
+                            to_push[address] = 'xxxx:xxxx:%s' % ':'.join(record[address].split(':')[-2:])
+                        else:
+                            to_push[address] = 'xxx.xxx.xxx.%s' % record[address].split('.')[-1]
                     else:
-                        to_push[address] = 'xxx.xxx.xxx.%s' % record[address].split('.')[-1]
-                else:
-                    # non local address
-                    to_push[address] = record[address]
+                        # non local address
+                        to_push[address] = record[address]
 
-        # unfiltered output fields
-        for attr in ["timestamp", "flow_id", "in_iface", "event_type",
-                     "vlan", "src_port", "dest_port", "proto", "alert"]:
-            if attr in record:
-                to_push[attr] = record[attr]
+            # unfiltered output fields
+            for attr in ["timestamp", "flow_id", "in_iface", "event_type",
+                         "vlan", "src_port", "dest_port", "proto", "alert"]:
+                if attr in record:
+                    to_push[attr] = record[attr]
 
-        self._tmp_handle.write(("%s\n" % ujson.dumps(to_push)).encode())
+            self._tmp_handle.write(("%s\n" % ujson.dumps(to_push)).encode())
 
     def get(self):
         """ fetch all data from temp

security/etpro-telemetry/src/opnsense/scripts/suricata/metadata/rules/et-telemetry.xml

@@ -6,6 +6,7 @@
     </headers>
     <version url="https://opnsense.emergingthreats.net/api/v1/ruleset/version"/>
     <files>
+        <file url="inline::rules/telemetry_sids.txt" required="true">telemetry_sids.txt</file>
         <file description="botcc.portgrouped" url="inline::rules/botcc.portgrouped.rules">botcc.portgrouped.rules</file>
         <file description="botcc" url="inline::rules/botcc.rules">botcc.rules</file>
         <file description="ciarmy" url="inline::rules/ciarmy.rules">ciarmy.rules</file>