diff --git a/security/etpro-telemetry/Makefile b/security/etpro-telemetry/Makefile index afd7663dc..1b863e4fd 100644 --- a/security/etpro-telemetry/Makefile +++ b/security/etpro-telemetry/Makefile @@ -1,6 +1,6 @@ PLUGIN_NAME= etpro-telemetry -PLUGIN_VERSION= 1.3 -PLUGIN_REVISION= 2 +PLUGIN_VERSION= 1.4 +PLUGIN_REVISION= 3 PLUGIN_COMMENT= ET Pro Telemetry Edition PLUGIN_MAINTAINER= ad@opnsense.org PLUGIN_WWW= https://docs.opnsense.org/manual/etpro_telemetry.html diff --git a/security/etpro-telemetry/src/etc/cron.d/etpro-telemetry.cron b/security/etpro-telemetry/src/etc/cron.d/etpro-telemetry.cron index bd4d7b1e7..1d412a342 100644 --- a/security/etpro-telemetry/src/etc/cron.d/etpro-telemetry.cron +++ b/security/etpro-telemetry/src/etc/cron.d/etpro-telemetry.cron @@ -6,5 +6,5 @@ SHELL=/bin/sh PATH=/etc:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin #minute hour mday month wday who command -0,30 * * * * root /usr/local/opnsense/scripts/etpro_telemetry/send_heartbeat.py +0 * * * * root /usr/local/opnsense/scripts/etpro_telemetry/send_heartbeat.py * * * * * root /usr/local/opnsense/scripts/etpro_telemetry/send_telemetry.py diff --git a/security/etpro-telemetry/src/opnsense/scripts/etpro_telemetry/send_heartbeat.py b/security/etpro-telemetry/src/opnsense/scripts/etpro_telemetry/send_heartbeat.py index 2f0562f16..c4805b1e0 100755 --- a/security/etpro-telemetry/src/opnsense/scripts/etpro_telemetry/send_heartbeat.py +++ b/security/etpro-telemetry/src/opnsense/scripts/etpro_telemetry/send_heartbeat.py @@ -33,13 +33,15 @@ import syslog import time import random import urllib3 +import json import telemetry +import telemetry.system urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) parser = argparse.ArgumentParser() parser.add_argument('-e', '--endpoint', help='Endpoint url to reach', - default="%s/api/v1/sensor" % telemetry.BASE_URL) + default="%s/api/v1/telemetry" % telemetry.BASE_URL) parser.add_argument('-i', '--insecure', help='Insecure, skip certificate validation', action="store_true", @@ -52,6 +54,10 @@ parser.add_argument('-D', '--direct', help='do not sleep before send (disable traffic spread)', action="store_true", default=False) +parser.add_argument('-t', '--test', + help='test mode, output request/response to stdout', + action="store_true", + default=False) args = parser.parse_args() exit_code = -1 @@ -62,16 +68,22 @@ if cnf.token is not None: params['verify'] = False try: # spread traffic to remote host, usual cron interval is 30 minutes - if not args.direct: + if not args.direct and not args.test: time.sleep(random.randint(0, 1800)) - r = requests.head(args.endpoint, **params) - if r.status_code == 200: + params['json'] = telemetry.system.Stats().get() + if args.test: + print("push to \t%s " % args.endpoint) + print("payload : \t%s" % json.dumps(params['json'])) + r = requests.post(args.endpoint, **params) + if args.test: + print("response %d : \t%s " % (r.status_code, r.text)) + if r.status_code == 201: # expected result, set exit code exit_code = 0 else: syslog.syslog(syslog.LOG_ERR, 'unexpected result from %s (http_code %s)' % (args.endpoint, r.status_code)) except requests.exceptions.ConnectionError: - syslog.syslog(syslog.LOG_ERR, 'connection error sending heardbeat to %s' % args.endpoint) + syslog.syslog(syslog.LOG_ERR, 'connection error sending heartbeat to %s' % args.endpoint) else: syslog.syslog(syslog.LOG_ERR, 'telemetry token missing in %s' % args.config) diff --git a/security/etpro-telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/__init__.py b/security/etpro-telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/__init__.py index 049131999..04de204b6 100755 --- a/security/etpro-telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/__init__.py +++ b/security/etpro-telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/__init__.py @@ -65,24 +65,26 @@ def get_config(rule_update_config): return response -class EventCollector(object): +def telemetry_sids(): + """ collect sids of interest, which are part of the ET-Telemetry delivery + :return: set + """ + our_sids = set() + if os.path.isfile(RELATED_SIDS_FILE): + for line in open(RELATED_SIDS_FILE, 'r'): + if line.strip().isdigit(): + our_sids.add(int(line.strip())) + return our_sids + + +class EventCollector: """ Event collector, responsible for extracting and anonymising from an eve.json stream """ def __init__(self): self._tmp_handle = tempfile.NamedTemporaryFile() self._local_networks = list() - self._our_sids = set() + self._our_sids = telemetry_sids() self._get_local_networks() - self._get_our_sids() - - def _get_our_sids(self): - """ collect sids of interest, which are part of the ET-Telemetry delivery - :return: None - """ - if os.path.isfile(RELATED_SIDS_FILE): - for line in open(RELATED_SIDS_FILE, 'r'): - if line.strip().isdigit(): - self._our_sids.add(int(line.strip())) def _is_rule_of_interest(self, record): """ check if rule is of interest for delivery diff --git a/security/etpro-telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/state.py b/security/etpro-telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/state.py index 8aa76b1b8..40df17ae9 100755 --- a/security/etpro-telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/state.py +++ b/security/etpro-telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/state.py @@ -27,7 +27,7 @@ import fcntl import datetime -class Telemetry(object): +class Telemetry: def __init__(self, filename, init_last_days=2): self._filename = filename self._init_last_days = init_last_days diff --git a/security/etpro-telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/system.py b/security/etpro-telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/system.py new file mode 100644 index 000000000..8c544f866 --- /dev/null +++ b/security/etpro-telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/system.py @@ -0,0 +1,142 @@ +""" + Copyright (c) 2018-2019 Ad Schellevis + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. +""" +import os +import time +import subprocess +import ujson +from . import telemetry_sids +from .log import reverse_log_reader + +class Stats: + def __init__(self): + self._suricata_default_rule_path = '/usr/local/etc/suricata/opnsense.rules' + self._suricata_installed_rules = '/usr/local/etc/suricata/installed_rules.yaml' + self._our_sids = telemetry_sids() + self._installed_sids = self._fetch_installed_sids() + + def _fetch_installed_sids(self): + installed_sids = set() + if os.path.isfile(self._suricata_installed_rules): + with open(self._suricata_installed_rules) as fin: + for line in fin: + line = line.strip() + if line.endswith('.rules') and line.startswith('- '): + rule_path = '%s/%s' % (self._suricata_default_rule_path, line[2:].strip()) + if os.path.isfile(rule_path): + with open(rule_path) as rf: + for rline in rf: + rline = rline.strip() + if not rline.startswith('#'): + sid_ref = rline.rfind('sid:') + if sid_ref > 0: + sid = rline[sid_ref+4:].split(';')[0] + if sid.isdigit(): + installed_sids.add(int(sid)) + return installed_sids + + @staticmethod + def software_version(): + return subprocess.run(['/usr/local/sbin/opnsense-version', '-v'], capture_output=True, text=True).stdout.strip() + + @staticmethod + def suricata_version(): + tmp = subprocess.run(['/usr/local/bin/suricata', '-V'], capture_output=True, text=True).stdout.strip() + if tmp.find(' version '): + tmp = tmp[tmp.find(' version ')+9:] + return tmp + + @staticmethod + def suricata_status(): + sp = subprocess.run(['/usr/local/etc/rc.d/suricata', 'status'], capture_output=True, text=True) + return 'Running' if sp.returncode == 0 else 'Stopped' + + @staticmethod + def system_time(): + return int(time.time()) + + @staticmethod + def ruleset_version(): + if os.path.isfile('/usr/local/etc/suricata/rules/telemetry_version.json'): + with open('/usr/local/etc/suricata/rules/telemetry_version.json') as f_in: + data = f_in.read() + if data.startswith('#@opnsense_downlo'): + # strip download hash line + data = data[data.find('\n')+1:] + data = ujson.loads(data) + if 'version' in data: + return data['version'] + return None + + def total_enabled_rules(self): + return len(self._installed_sids) + + def total_enabled_telemetry_rules(self): + return len(self._installed_sids & self._our_sids) + + @staticmethod + def mode(): + # quick scan config for inline (ips) mode + conf = '/usr/local/etc/suricata/suricata.yaml' + if os.path.isfile(conf): + with open(conf) as fin: + for line in fin: + if line.startswith(' inline: true'): + return "IPS" + return "IDS" + + @staticmethod + def log_stats(): + # tail stats.log, return statistcs of interest + result = dict() + stats_of_interest = ['capture.kernel_packets', 'decoder.pkts', 'decoder.bytes', 'decoder.ipv4', 'decoder.ipv6', + 'flow.tcp', 'flow.udp', 'detect.alert'] + if os.path.isfile('/var/log/suricata/stats.log'): + for line in reverse_log_reader('/var/log/suricata/stats.log'): + if line.strip().startswith('------'): + break + elif line.count('|') == 2: + parts = [x.strip() for x in line.split('|')] + if parts[0] in stats_of_interest: + result[parts[0]] = int(parts[2]) if parts[2].isdigit() else parts[2] + # add empty values for stats_of_interest not found + for item in stats_of_interest: + if item not in result: + result[item] = None + + return result + + def get(self): + result = dict() + for item in ['software_version', 'suricata_version', 'suricata_status', 'system_time', + 'ruleset_version', 'total_enabled_rules', 'total_enabled_telemetry_rules' ,'mode', 'log_stats']: + try: + value = getattr(self, item)() + except FileNotFoundError: + value = 'NOTFOUND' + except Exception as e: + value = 'ERROR' + result[item] = value + return result diff --git a/security/etpro-telemetry/src/opnsense/scripts/suricata/metadata/rules/et-telemetry.xml b/security/etpro-telemetry/src/opnsense/scripts/suricata/metadata/rules/et-telemetry.xml index 226a0b295..d8357c5a9 100644 --- a/security/etpro-telemetry/src/opnsense/scripts/suricata/metadata/rules/et-telemetry.xml +++ b/security/etpro-telemetry/src/opnsense/scripts/suricata/metadata/rules/et-telemetry.xml @@ -7,6 +7,7 @@ telemetry_sids.txt + telemetry_version.json botcc.portgrouped.rules botcc.rules ciarmy.rules