From edb29455b304ffdea02d7fb5d02983786147ef27 Mon Sep 17 00:00:00 2001 From: Ad Schellevis Date: Tue, 2 Apr 2019 16:40:39 +0200 Subject: [PATCH] etpro-telemetry, keep documentation and test data with product for development/internal use --- security/etpro-telemetry/README.md | 95 +++++++++++++++++++ security/etpro-telemetry/requires.txt | 2 + .../etpro-telemetry/testdata/log/eve.json | 6 ++ .../testdata/rule-updater.config | 4 + 4 files changed, 107 insertions(+) create mode 100644 security/etpro-telemetry/README.md create mode 100644 security/etpro-telemetry/requires.txt create mode 100755 security/etpro-telemetry/testdata/log/eve.json create mode 100644 security/etpro-telemetry/testdata/rule-updater.config diff --git a/security/etpro-telemetry/README.md b/security/etpro-telemetry/README.md new file mode 100644 index 000000000..e2c5ba206 --- /dev/null +++ b/security/etpro-telemetry/README.md @@ -0,0 +1,95 @@ +# ETPRO telemetry edition + +Telemetry tools, depends on `rule-updater.config`, which contains the device registartion number provided by OPNsense. + +for example. +``` +# autogenerated, do not edit. +[__properties__] +et_telemetry.token=eeyahsei1nootu6aeJ5Eeghoohahxu4rahkiewu2bayeiv2eif +``` + +The scripts will be scheduled automatically using a cron job after installation of the ruleset / telemetry function. + +General +---------------- +All scripts described in the readme are available in the directory `src/opnsense/scripts/etpro_telemetry` + + +Requirements +-------------- + +Install requirements (default available on OPNsense) + +``` +pip install -r requires.txt +``` + + +send_telemetry.py +----------------- + +Send anonymized telemetry data from Suricata eve log file, for details how the anonimisation works see `EventCollector._get_local_networks()` and +`EventCollector.push()' where all local attached networks are used to strip off the first segments of the address before sending. + + +``` +usage: send_telemetry.py [-h] [-e ENDPOINT] [-i] [-c CONFIG] [-l LOG] + [-s STATE] + +optional arguments: + -h, --help show this help message and exit + -e ENDPOINT, --endpoint ENDPOINT + Endpoint url to reach + -i, --insecure Insecure, skip certificate validation + -c CONFIG, --config CONFIG + rule downloader configuration + -l LOG, --log LOG log directory containing eve.json files + -s STATE, --state STATE + persistent state (and lock) filename + -d DAYS, --days DAYS Maximum number of days to look back on initial run +``` + +to test using supplied configuration: + +``` +python send_telemetry.py -i -d 9999 -e https://endpoint -c testdata/rule-updater.config -l testdata/log/ +``` + +State is saved `/usr/local/var/run/et_telemetry.state` by default. + + +send_heartbeat.py +------------------ + +Send a small heartbeat to proofpoint to signal the device is still active. + +``` +usage: send_heartbeat.py [-h] [-e ENDPOINT] [-i] [-c CONFIG] + +optional arguments: + -h, --help show this help message and exit + -e ENDPOINT, --endpoint ENDPOINT + Endpoint url to reach + -i, --insecure Insecure, skip certificate validation + -c CONFIG, --config CONFIG + rule downloader configuration +``` + + +send_heartbeat.py +------------------ + +Dump suricata eve.log + +``` +usage: dump_data.py [-h] [-l LOG] [-t TIME] [-p] [-L LIMIT] + +optional arguments: + -h, --help show this help message and exit + -l LOG, --log LOG log directory containing eve.json files + -t TIME, --time TIME max seconds to read from now() + -p, --parsed show data as shipped using send_telemetry + -L LIMIT, --limit LIMIT + limit number of rows +``` diff --git a/security/etpro-telemetry/requires.txt b/security/etpro-telemetry/requires.txt new file mode 100644 index 000000000..2abd7d3a9 --- /dev/null +++ b/security/etpro-telemetry/requires.txt @@ -0,0 +1,2 @@ +requests +ujson diff --git a/security/etpro-telemetry/testdata/log/eve.json b/security/etpro-telemetry/testdata/log/eve.json new file mode 100755 index 000000000..164565c1c --- /dev/null +++ b/security/etpro-telemetry/testdata/log/eve.json @@ -0,0 +1,6 @@ +{"timestamp":"2018-05-21T13:05:42.646499+0200","flow_id":1612621304749411,"in_iface":"em1","event_type":"alert","src_ip":"fe80:0000:0000:0000:021c:42ff:fec0:d65f","dest_ip":"ff02:0000:0000:0000:0000:0000:0000:0016","proto":"IPV6-ICMP","icmp_type":143,"icmp_code":0,"alert":{"action":"allowed","gid":1,"signature_id":2200094,"rev":2,"signature":"SURICATA zero length padN option","category":"Generic Protocol Command Decode","severity":3}} +{"timestamp":"2018-05-21T13:05:43.252983+0200","flow_id":1612621304749411,"in_iface":"em1","event_type":"alert","src_ip":"fe80:0000:0000:0000:021c:42ff:fec0:d65f","dest_ip":"ff02:0000:0000:0000:0000:0000:0000:0016","proto":"IPV6-ICMP","icmp_type":143,"icmp_code":0,"alert":{"action":"allowed","gid":1,"signature_id":2200094,"rev":2,"signature":"SURICATA zero length padN option","category":"Generic Protocol Command Decode","severity":3}} +{"timestamp":"2018-05-21T13:05:48.646577+0200","flow_id":130587005083057,"in_iface":"em1","event_type":"alert","src_ip":"fe80:0000:0000:0000:021c:42ff:fe0d:9ba1","dest_ip":"ff02:0000:0000:0000:0000:0000:0000:0016","proto":"IPV6-ICMP","icmp_type":143,"icmp_code":0,"alert":{"action":"allowed","gid":1,"signature_id":2200094,"rev":2,"signature":"SURICATA zero length padN option","category":"Generic Protocol Command Decode","severity":3}} +{"timestamp":"2018-05-21T13:05:48.846691+0200","flow_id":130587005083057,"in_iface":"em1","event_type":"alert","src_ip":"fe80:0000:0000:0000:021c:42ff:fe0d:9ba1","dest_ip":"ff02:0000:0000:0000:0000:0000:0000:0016","proto":"IPV6-ICMP","icmp_type":143,"icmp_code":0,"alert":{"action":"allowed","gid":1,"signature_id":2200094,"rev":2,"signature":"SURICATA zero length padN option","category":"Generic Protocol Command Decode","severity":3}} +{"timestamp":"2018-05-21T13:05:49.846589+0200","flow_id":130587005083057,"in_iface":"em1","event_type":"alert","src_ip":"fe80:0000:0000:0000:021c:42ff:fe0d:9ba1","dest_ip":"ff02:0000:0000:0000:0000:0000:0000:0016","proto":"IPV6-ICMP","icmp_type":143,"icmp_code":0,"alert":{"action":"allowed","gid":1,"signature_id":2200094,"rev":2,"signature":"SURICATA zero length padN option","category":"Generic Protocol Command Decode","severity":3}} +{"timestamp":"2018-05-21T13:05:55.757441+0200","flow_id":1612621304749411,"in_iface":"em1","event_type":"alert","src_ip":"fe80:0000:0000:0000:021c:42ff:fec0:d65f","dest_ip":"ff02:0000:0000:0000:0000:0000:0000:0016","proto":"IPV6-ICMP","icmp_type":143,"icmp_code":0,"alert":{"action":"allowed","gid":1,"signature_id":2200094,"rev":2,"signature":"SURICATA zero length padN option","category":"Generic Protocol Command Decode","severity":3}} diff --git a/security/etpro-telemetry/testdata/rule-updater.config b/security/etpro-telemetry/testdata/rule-updater.config new file mode 100644 index 000000000..3b52d3afc --- /dev/null +++ b/security/etpro-telemetry/testdata/rule-updater.config @@ -0,0 +1,4 @@ +# autogenerated, do not edit. +[__properties__] +et_telemetry.deviceid=eeyahsei1nootu6aeJ5Eeghoohahxu4rahkiewu2bayeiv2eif +