ETPro Telemetry (#1487)

2026-05-22 18:44:07 -07:00 · 2019-09-03 17:21:39 +02:00
parent ad3c3e745a
commit 6370b5374a
7 changed files with 178 additions and 21 deletions
@@ -1,6 +1,6 @@
 PLUGIN_NAME=		etpro-telemetry
-PLUGIN_VERSION=		1.3
-PLUGIN_REVISION=	2
+PLUGIN_VERSION=		1.4
+PLUGIN_REVISION=	3
 PLUGIN_COMMENT=		ET Pro Telemetry Edition
 PLUGIN_MAINTAINER=	ad@opnsense.org
 PLUGIN_WWW=		https://docs.opnsense.org/manual/etpro_telemetry.html
@@ -6,5 +6,5 @@
 SHELL=/bin/sh
 PATH=/etc:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
 #minute	hour	mday	month	wday	who	command
-0,30	*	*	*	*	root    /usr/local/opnsense/scripts/etpro_telemetry/send_heartbeat.py
+0	*	*	*	*	root    /usr/local/opnsense/scripts/etpro_telemetry/send_heartbeat.py
 *	*	*	*	*	root    /usr/local/opnsense/scripts/etpro_telemetry/send_telemetry.py
@@ -33,13 +33,15 @@ import syslog
 import time
 import random
 import urllib3
+import json
 import telemetry
+import telemetry.system
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

 parser = argparse.ArgumentParser()
 parser.add_argument('-e', '--endpoint',
                    help='Endpoint url to reach',
-                    default="%s/api/v1/sensor" % telemetry.BASE_URL)
+                    default="%s/api/v1/telemetry" % telemetry.BASE_URL)
 parser.add_argument('-i', '--insecure',
                    help='Insecure, skip certificate validation',
                    action="store_true",
@@ -52,6 +54,10 @@ parser.add_argument('-D', '--direct',
                    help='do not sleep before send (disable traffic spread)',
                    action="store_true",
                    default=False)
+parser.add_argument('-t', '--test',
+                    help='test mode, output request/response to stdout',
+                    action="store_true",
+                    default=False)
 args = parser.parse_args()

 exit_code = -1
@@ -62,16 +68,22 @@ if cnf.token is not None:
        params['verify'] = False
    try:
        # spread traffic to remote host, usual cron interval is 30 minutes
-        if not args.direct:
+        if not args.direct and not args.test:
            time.sleep(random.randint(0, 1800))
-        r = requests.head(args.endpoint, **params)
-        if r.status_code == 200:
+        params['json'] = telemetry.system.Stats().get()
+        if args.test:
+            print("push to \t%s " % args.endpoint)
+            print("payload : \t%s" % json.dumps(params['json']))
+        r = requests.post(args.endpoint, **params)
+        if args.test:
+            print("response %d : \t%s " % (r.status_code, r.text))
+        if r.status_code == 201:
            # expected result, set exit code
            exit_code = 0
        else:
            syslog.syslog(syslog.LOG_ERR, 'unexpected result from %s (http_code %s)' % (args.endpoint, r.status_code))
    except requests.exceptions.ConnectionError:
-        syslog.syslog(syslog.LOG_ERR, 'connection error sending heardbeat to %s' % args.endpoint)
+        syslog.syslog(syslog.LOG_ERR, 'connection error sending heartbeat to %s' % args.endpoint)
 else:
    syslog.syslog(syslog.LOG_ERR, 'telemetry token missing in %s' % args.config)

@@ -65,24 +65,26 @@ def get_config(rule_update_config):
    return response


-class EventCollector(object):
+def telemetry_sids():
+    """ collect sids of interest, which are part of the ET-Telemetry delivery
+    :return: set
+    """
+    our_sids = set()
+    if os.path.isfile(RELATED_SIDS_FILE):
+        for line in open(RELATED_SIDS_FILE, 'r'):
+            if line.strip().isdigit():
+                our_sids.add(int(line.strip()))
+    return our_sids
+
+
+class EventCollector:
    """ Event collector, responsible for extracting and anonymising from an eve.json stream
    """
    def __init__(self):
        self._tmp_handle = tempfile.NamedTemporaryFile()
        self._local_networks = list()
-        self._our_sids = set()
+        self._our_sids = telemetry_sids()
        self._get_local_networks()
-        self._get_our_sids()
-
-    def _get_our_sids(self):
-        """ collect sids of interest, which are part of the ET-Telemetry delivery
-        :return: None
-        """
-        if os.path.isfile(RELATED_SIDS_FILE):
-            for line in open(RELATED_SIDS_FILE, 'r'):
-                if line.strip().isdigit():
-                    self._our_sids.add(int(line.strip()))

    def _is_rule_of_interest(self, record):
        """ check if rule is of interest for delivery
@@ -27,7 +27,7 @@ import fcntl
 import datetime


-class Telemetry(object):
+class Telemetry:
    def __init__(self, filename, init_last_days=2):
        self._filename = filename
        self._init_last_days = init_last_days
@@ -0,0 +1,142 @@
+"""
+    Copyright (c) 2018-2019 Ad Schellevis <ad@opnsense.org>
+    All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimer.
+
+    2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+
+    THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+    AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+    AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+    OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+    SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+    CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+    ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+    POSSIBILITY OF SUCH DAMAGE.
+"""
+import os
+import time
+import subprocess
+import ujson
+from . import telemetry_sids
+from .log import reverse_log_reader
+
+class Stats:
+    def __init__(self):
+        self._suricata_default_rule_path = '/usr/local/etc/suricata/opnsense.rules'
+        self._suricata_installed_rules = '/usr/local/etc/suricata/installed_rules.yaml'
+        self._our_sids = telemetry_sids()
+        self._installed_sids = self._fetch_installed_sids()
+
+    def _fetch_installed_sids(self):
+        installed_sids = set()
+        if os.path.isfile(self._suricata_installed_rules):
+            with open(self._suricata_installed_rules) as fin:
+                for line in fin:
+                    line = line.strip()
+                    if line.endswith('.rules') and line.startswith('- '):
+                        rule_path = '%s/%s' % (self._suricata_default_rule_path, line[2:].strip())
+                        if os.path.isfile(rule_path):
+                            with open(rule_path) as rf:
+                                for rline in rf:
+                                    rline = rline.strip()
+                                    if not rline.startswith('#'):
+                                        sid_ref = rline.rfind('sid:')
+                                        if sid_ref > 0:
+                                            sid = rline[sid_ref+4:].split(';')[0]
+                                            if sid.isdigit():
+                                                installed_sids.add(int(sid))
+        return installed_sids
+
+    @staticmethod
+    def software_version():
+        return subprocess.run(['/usr/local/sbin/opnsense-version', '-v'], capture_output=True, text=True).stdout.strip()
+
+    @staticmethod
+    def suricata_version():
+        tmp = subprocess.run(['/usr/local/bin/suricata', '-V'], capture_output=True, text=True).stdout.strip()
+        if tmp.find(' version '):
+            tmp = tmp[tmp.find(' version ')+9:]
+        return tmp
+
+    @staticmethod
+    def suricata_status():
+        sp = subprocess.run(['/usr/local/etc/rc.d/suricata', 'status'], capture_output=True, text=True)
+        return 'Running' if sp.returncode == 0 else 'Stopped'
+
+    @staticmethod
+    def system_time():
+        return int(time.time())
+
+    @staticmethod
+    def ruleset_version():
+        if os.path.isfile('/usr/local/etc/suricata/rules/telemetry_version.json'):
+            with open('/usr/local/etc/suricata/rules/telemetry_version.json') as f_in:
+                data = f_in.read()
+                if data.startswith('#@opnsense_downlo'):
+                    # strip download hash line
+                    data = data[data.find('\n')+1:]
+                data = ujson.loads(data)
+                if 'version' in data:
+                    return data['version']
+        return None
+
+    def total_enabled_rules(self):
+        return len(self._installed_sids)
+
+    def total_enabled_telemetry_rules(self):
+        return len(self._installed_sids & self._our_sids)
+
+    @staticmethod
+    def mode():
+        # quick scan config for inline (ips) mode
+        conf = '/usr/local/etc/suricata/suricata.yaml'
+        if os.path.isfile(conf):
+            with open(conf) as fin:
+                for line in fin:
+                    if line.startswith('  inline: true'):
+                        return "IPS"
+        return "IDS"
+
+    @staticmethod
+    def log_stats():
+        # tail stats.log, return statistcs of interest
+        result = dict()
+        stats_of_interest = ['capture.kernel_packets', 'decoder.pkts', 'decoder.bytes', 'decoder.ipv4', 'decoder.ipv6',
+                             'flow.tcp', 'flow.udp', 'detect.alert']
+        if os.path.isfile('/var/log/suricata/stats.log'):
+            for line in reverse_log_reader('/var/log/suricata/stats.log'):
+                if line.strip().startswith('------'):
+                    break
+                elif line.count('|') == 2:
+                    parts = [x.strip() for x in line.split('|')]
+                    if parts[0] in stats_of_interest:
+                        result[parts[0]] = int(parts[2]) if parts[2].isdigit() else parts[2]
+            # add empty values for stats_of_interest not found
+            for item in stats_of_interest:
+                if item not in result:
+                    result[item] = None
+
+        return result
+
+    def get(self):
+        result = dict()
+        for item in ['software_version', 'suricata_version', 'suricata_status', 'system_time',
+                     'ruleset_version', 'total_enabled_rules', 'total_enabled_telemetry_rules' ,'mode', 'log_stats']:
+            try:
+                value = getattr(self, item)()
+            except FileNotFoundError:
+                value = 'NOTFOUND'
+            except Exception as e:
+                value = 'ERROR'
+            result[item] = value
+        return result
@@ -7,6 +7,7 @@
    <version url="https://opnsense.emergingthreats.net/api/v1/ruleset/version"/>
    <files>
        <file url="inline::rules/telemetry_sids.txt" required="true">telemetry_sids.txt</file>
+        <file url="https://opnsense.emergingthreats.net/api/v1/ruleset/version" required="true">telemetry_version.json</file>
        <file description="botcc.portgrouped" url="inline::rules/botcc.portgrouped.rules">botcc.portgrouped.rules</file>
        <file description="botcc" url="inline::rules/botcc.rules">botcc.rules</file>
        <file description="ciarmy" url="inline::rules/ciarmy.rules">ciarmy.rules</file>