diff --git a/security/q-feeds-connector/+POST_INSTALL.post b/security/q-feeds-connector/+POST_INSTALL.post new file mode 100755 index 000000000..47481ad32 --- /dev/null +++ b/security/q-feeds-connector/+POST_INSTALL.post @@ -0,0 +1,3 @@ +#!/bin/sh + +/usr/local/sbin/pluginctl -s cron restart diff --git a/security/q-feeds-connector/Makefile b/security/q-feeds-connector/Makefile new file mode 100644 index 000000000..dffd163a2 --- /dev/null +++ b/security/q-feeds-connector/Makefile @@ -0,0 +1,6 @@ +PLUGIN_NAME= q-feeds-connector +PLUGIN_VERSION= 1.0 +PLUGIN_COMMENT= Connector for Q-Feeds threat intel +PLUGIN_MAINTAINER= devel@qfeeds.com + +.include "../../Mk/plugins.mk" diff --git a/security/q-feeds-connector/README.md b/security/q-feeds-connector/README.md new file mode 100644 index 000000000..b18fb8cc5 --- /dev/null +++ b/security/q-feeds-connector/README.md @@ -0,0 +1,48 @@ +# plugin to connect to QFeeds threat platform + +Register for a token at : https://qfeeds.com/opnsense/ + +# command line control + +Settings are persisted in `/usr/local/etc/qfeeds.conf`, using the following format: + +``` +[api] +key=tip_xxxxxxxx +``` + +Our commandline tool contains all actions used by the UI, which is practical for debuggging. + +``` +usage: qfeedsctl.py [-h] [--target_dir TARGET_DIR] [-f] [-v] [{fetch_index,fetch,show_index,firewall_load,update,stats} ...] + +positional arguments: + {fetch_index,fetch,show_index,firewall_load,update,stats} + +options: + -h, --help show this help message and exit + --target_dir TARGET_DIR + -f forced (auto index) + -v verbose output +``` + +The index is the driver for most actions, which is a json encoded file in `/var/db/qfeeds-tables/index.json`. + +Actions supported: + +* fetch_index --> download the index file +* fetch --> download the lists +* firewall_load --> collect ip lists into pre-defined firewall tables +* update [sleep when almost time] --> run fetch_index --> fetch --> firewall_load (to be used by cron) +* show_index --> dumps the index +* stats --> dumps feed information +* logs --> dumps firewall log information for aliases offered by Q-Feeds + + +Example usage: + +``` +/usr/local/opnsense/scripts/qfeeds/qfeedsctl.py update +``` + +**Fetch index and update lists when updated remotely** diff --git a/security/q-feeds-connector/pkg-descr b/security/q-feeds-connector/pkg-descr new file mode 100644 index 000000000..c6c57300e --- /dev/null +++ b/security/q-feeds-connector/pkg-descr @@ -0,0 +1,8 @@ +Connector for Q-Feeds threat intel + +Plugin Changelog +================ + +1.0 + +* Intial release version diff --git a/security/q-feeds-connector/src/etc/inc/plugins.inc.d/qfeeds.inc b/security/q-feeds-connector/src/etc/inc/plugins.inc.d/qfeeds.inc new file mode 100644 index 000000000..c2fb7a2a6 --- /dev/null +++ b/security/q-feeds-connector/src/etc/inc/plugins.inc.d/qfeeds.inc @@ -0,0 +1,35 @@ +configdRun('template reload OPNsense/QFeeds')); + if (strtolower($res) != 'ok') { + throw new UserException(sprintf(gettext("Unable to update settings (%s)"), $res)); + } + $res = trim($backend->configdRun('qfeeds reconfigure')); + if (strpos($res, 'EXIT OK') === false) { + throw new UserException($res); + } + return ['status' => 'ok', 'output' => $res]; + } + + public function searchFeedsAction() + { + $records = []; + $data = json_decode((new Backend())->configdRun('qfeeds info') ?? '[]', true); + if (!empty($data) && !empty($data['feeds'])) { + $records = $data['feeds']; + foreach ($records as &$record) { + $record['licensed'] = $record['licensed'] ? '1' : '0'; + } + } + return $this->searchRecordsetBase($records); + } + + public function searchEventsAction() + { + $records = []; + $ifnames = []; + foreach (Config::getInstance()->object()->interfaces->children() as $key => $node) { + if (!empty((string)$node->if)) { + $ifnames[(string)$node->if] = (string)($node->descr ?? strtoupper($key)); + } + } + $data = json_decode((new Backend())->configdRun('qfeeds logs') ?? '[]', true); + if (!empty($data) && !empty($data['rows'])) { + foreach ($data['rows'] as $row) { + $records[] = [ + 'timestamp' => $row[0], + 'interface' => $ifnames[$row[1]] ?? $row[1], + 'direction' => $row[2], + 'source' => $row[3], + 'destination' => $row[4], + ]; + } + } + return $this->searchRecordsetBase($records); + } + + public function statsAction() + { + $stats = json_decode((new Backend())->configdRun('qfeeds stats'), true); + if (!empty($stats) && !empty($stats['feeds'])) { + $info = json_decode((new Backend())->configdRun('qfeeds info'), true); + if (!empty($info) && !empty($info['feeds'])) { + $feeds = []; + foreach ($info['feeds'] as $feed) { + $feeds[$feed['feed_type']] = $feed; + } + foreach ($stats['feeds'] as &$feed) { + if (isset($feeds[$feed['name']])) { + $tmp = $feeds[$feed['name']]; + $feed['updated_at'] = $tmp['updated_at']; + $feed['next_update'] = $tmp['next_update']; + $feed['licensed'] = $tmp['licensed']; + } + } + } + } + return $stats; + } +} \ No newline at end of file diff --git a/security/q-feeds-connector/src/opnsense/mvc/app/controllers/OPNsense/QFeeds/IndexController.php b/security/q-feeds-connector/src/opnsense/mvc/app/controllers/OPNsense/QFeeds/IndexController.php new file mode 100644 index 000000000..aec86b9a3 --- /dev/null +++ b/security/q-feeds-connector/src/opnsense/mvc/app/controllers/OPNsense/QFeeds/IndexController.php @@ -0,0 +1,40 @@ +view->formSettings = $this->getForm("settings"); + $this->view->pick('OPNsense/QFeeds/index'); + } +} \ No newline at end of file diff --git a/security/q-feeds-connector/src/opnsense/mvc/app/controllers/OPNsense/QFeeds/forms/settings.xml b/security/q-feeds-connector/src/opnsense/mvc/app/controllers/OPNsense/QFeeds/forms/settings.xml new file mode 100644 index 000000000..e27658d30 --- /dev/null +++ b/security/q-feeds-connector/src/opnsense/mvc/app/controllers/OPNsense/QFeeds/forms/settings.xml @@ -0,0 +1,12 @@ +
+ + header + + + + connect.general.apikey + + text + click here]]> + +
\ No newline at end of file diff --git a/security/q-feeds-connector/src/opnsense/mvc/app/library/OPNsense/System/Status/QfeedsStatus.php b/security/q-feeds-connector/src/opnsense/mvc/app/library/OPNsense/System/Status/QfeedsStatus.php new file mode 100644 index 000000000..5477729d6 --- /dev/null +++ b/security/q-feeds-connector/src/opnsense/mvc/app/library/OPNsense/System/Status/QfeedsStatus.php @@ -0,0 +1,61 @@ +internalPriority = 2; + $this->internalPersistent = true; + $this->internalIsBanner = true; + $this->internalTitle = gettext('QFeeds'); + $this->internalScope = [ + '/ui/q_feeds/' + ]; + } + + public function collectStatus() + { + $cnf = Config::getInstance()->object(); + if (!empty($cnf->system->maximumtableentries) && $cnf->system->maximumtableentries >= 2000000) { + return; + } + $this->internalStatus = SystemStatusCode::ERROR; + $this->internalMessage = gettext( + 'QFeeds requires additional memory to be reserved for aliases. ' . + 'Please increase `Firewall Maximum Table Entries` in `Firewall: Settings: Advanced` to at least' . + ' 2 million items.' + ); + } +} diff --git a/security/q-feeds-connector/src/opnsense/mvc/app/models/OPNsense/Firewall/DynamicAliases/QfeedsAliases.php b/security/q-feeds-connector/src/opnsense/mvc/app/models/OPNsense/Firewall/DynamicAliases/QfeedsAliases.php new file mode 100644 index 000000000..26eca705d --- /dev/null +++ b/security/q-feeds-connector/src/opnsense/mvc/app/models/OPNsense/Firewall/DynamicAliases/QfeedsAliases.php @@ -0,0 +1,56 @@ +configdRun('qfeeds index') ?? '', true) ?? []; + if (is_array($payload) && !empty($payload['feeds'])) { + foreach ($payload['feeds'] as $feed) { + if ($feed['type'] == 'ip' && !empty($feed['licensed'])) { + $name = '__qfeeds_'. $feed['feed_type']; + $result[$name] = [ + 'enabled' => '1', + 'counters' => '1', + 'name' => $name, + 'type' => 'external', + 'description' => $feed['feed_type'], + 'content' => '' + ]; + } + } + } + return $result; + } +} diff --git a/security/q-feeds-connector/src/opnsense/mvc/app/models/OPNsense/QFeeds/ACL/ACL.xml b/security/q-feeds-connector/src/opnsense/mvc/app/models/OPNsense/QFeeds/ACL/ACL.xml new file mode 100644 index 000000000..3c58e042a --- /dev/null +++ b/security/q-feeds-connector/src/opnsense/mvc/app/models/OPNsense/QFeeds/ACL/ACL.xml @@ -0,0 +1,9 @@ + + + Services: QFeeds + + ui/q_feeds/* + api/q_feeds/* + + + \ No newline at end of file diff --git a/security/q-feeds-connector/src/opnsense/mvc/app/models/OPNsense/QFeeds/Connector.php b/security/q-feeds-connector/src/opnsense/mvc/app/models/OPNsense/QFeeds/Connector.php new file mode 100644 index 000000000..eca13f960 --- /dev/null +++ b/security/q-feeds-connector/src/opnsense/mvc/app/models/OPNsense/QFeeds/Connector.php @@ -0,0 +1,38 @@ + + //OPNsense/QFeedsConnector + 1.0.0 + QFeeds connector + + + + + + \ No newline at end of file diff --git a/security/q-feeds-connector/src/opnsense/mvc/app/models/OPNsense/QFeeds/Menu/Menu.xml b/security/q-feeds-connector/src/opnsense/mvc/app/models/OPNsense/QFeeds/Menu/Menu.xml new file mode 100644 index 000000000..a91edf926 --- /dev/null +++ b/security/q-feeds-connector/src/opnsense/mvc/app/models/OPNsense/QFeeds/Menu/Menu.xml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/security/q-feeds-connector/src/opnsense/mvc/app/views/OPNsense/QFeeds/index.volt b/security/q-feeds-connector/src/opnsense/mvc/app/views/OPNsense/QFeeds/index.volt new file mode 100644 index 000000000..08dc8e4a5 --- /dev/null +++ b/security/q-feeds-connector/src/opnsense/mvc/app/views/OPNsense/QFeeds/index.volt @@ -0,0 +1,136 @@ +{# + +OPNsense® is Copyright © 2025 by Deciso B.V. +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, +OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +#} + + + + +
+
+ {{ partial("layout_partials/base_form",['fields':formSettings,'id':'frm_settings'])}} +
+
+ + + + + + + + + + + + +
{{ lang._('Description') }}{{ lang._('Type') }}{{ lang._('Updated at') }}{{ lang._('Next update') }}{{ lang._('Licensed') }}
+
+
+ + + + + + + + + + + + + +
{{ lang._('Timestamp') }}{{ lang._('Interface') }}{{ lang._('Direction') }}{{ lang._('Source') }}{{ lang._('Destination') }}
+
{{ lang._('Collected events from the firewall log for QFeed aliases') }}
+
+
+ +
+
+
+
+ +

+
+
+
\ No newline at end of file diff --git a/security/q-feeds-connector/src/opnsense/scripts/qfeeds/lib/__init__.py b/security/q-feeds-connector/src/opnsense/scripts/qfeeds/lib/__init__.py new file mode 100644 index 000000000..bf4c528aa --- /dev/null +++ b/security/q-feeds-connector/src/opnsense/scripts/qfeeds/lib/__init__.py @@ -0,0 +1,179 @@ +""" + Copyright (c) 2025 Deciso B.V. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. +""" +import os +import subprocess +import time +import ujson +from datetime import datetime +from lib.api import Api +from lib.log import PFLogCrawler +from lib.file import LockedFile + + +class QFeedsActions: + def __init__(self, target_dir, forced=False): + self._target_dir = target_dir + self._forced = forced + + @classmethod + def list_actions(cls): + return [ + 'fetch_index', + 'fetch', + 'show_index', + 'firewall_load', + 'update', + 'stats', + 'logs' + ] + + @property + def index_file(self): + return "%s/index.json" % self._target_dir + + @property + def index(self): + if not os.path.exists(self.index_file) and self._forced: + # require index file to get feeds + list(self.fetch_index()) + elif not os.path.exists(self.index_file): + return {} + data = ujson.load(open(self.index_file)) or {} + if type(data) is dict: + for feed in data.get('feeds', []): + feed['local_filename'] = "%s/%s.txt" % (self._target_dir, feed['feed_type']) + feed['updated_at_dt'] = datetime.fromisoformat(feed['updated_at']).timestamp() + feed['next_update_dt'] = datetime.fromisoformat(feed['next_update']).timestamp() + + return data + + def _file_stat(self, filename): + if not os.path.exists(filename): + return 0 + return os.stat(filename).st_mtime + + def fetch_index(self): + if not os.path.isdir(self._target_dir): + os.makedirs(self._target_dir) + with LockedFile(self.index_file) as f: + payload = Api().licenses() + f.truncate() + f.write(ujson.dumps(payload)) + yield 'downloaded index to %s' % f.filename + + def show_index(self): + yield ujson.dumps(self.index) + + def fetch(self): + for feed in self.index.get('feeds', []): + if feed['licensed'] and feed['updated_at_dt'] != self._file_stat(feed['local_filename']): + with LockedFile(feed['local_filename']) as f: + counter = 0 + for entry in Api().fetch(feed['feed_type']): + if counter == 0: + f.truncate() + f.write("%s\n" % entry) + counter += 1 + os.utime(feed['local_filename'], (feed['updated_at_dt'], feed['updated_at_dt'])) + yield "downloaded %d entries into %s [%s]" % (counter, feed['local_filename'], feed['updated_at']) + elif feed['licensed']: + yield "skipped %s [%s]" % (feed['local_filename'], feed['updated_at']) + + def firewall_load(self): + for feed in self.index.get('feeds', []): + if feed['licensed'] and os.path.exists(feed['local_filename']) and feed['type'] == 'ip': + table_name = '__qfeeds_%s' % feed['feed_type'] + sp = subprocess.run( + ['/sbin/pfctl', '-t', table_name, '-T', 'replace', '-f', feed['local_filename']], + capture_output=True, + text=True + ) + yield 'load feed %s [%s]' % (feed['feed_type'], sp.stderr.strip().replace("\n", " ")) + + def update(self): + update_sleep = 99999 + try: + index_payload = self.index + except TypeError: + # when the index can't be parsed, assume we have none while updating + index_payload = {} + do_update = len(index_payload.get('feeds', [])) == 0 + for feed in index_payload.get('feeds', []): + update_sleep = min(feed['next_update_dt'] - time.time(), update_sleep) + if feed['licensed'] and update_sleep <= 300: # 5 minute cron interval + do_update = True + if do_update: + if 0 < update_sleep <= 300: + time.sleep(update_sleep) + for action in ['fetch_index', 'fetch', 'firewall_load']: + yield from getattr(self, action)() + + def stats(self): + result = {'feeds': []} + for feed in self.index.get('feeds', []): + if feed['licensed'] and os.path.exists(feed['local_filename']) and feed['type'] == 'ip': + table_name = '__qfeeds_%s' % feed['feed_type'] + sp = subprocess.Popen( + ['/sbin/pfctl', '-t', table_name, '-vT', 'show'], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True + ) + record = { + 'name': feed['feed_type'], + 'total_entries': 0, + 'packets_blocked': 0, + 'bytes_blocked': 0, + 'addresses_blocked': 0 + } + while (line := sp.stdout.readline()): + if line.startswith(' '): + record['total_entries'] += 1 + elif 'Packets:' in line and 'Packets: 0 ' not in line: + parts = line.split() + if parts[3].isdigit() and parts[5].isdigit() and parts[0].lower().find('block') > 0: + record['packets_blocked'] += int(parts[3]) + record['bytes_blocked'] += int(parts[5]) + record['addresses_blocked'] += 1 + + result['feeds'].append(record) + result['totals'] = { + 'entries': sum(r['total_entries'] for r in result['feeds']), + # assumes no overlaps in datafeeds + 'addresses_blocked': sum(r['addresses_blocked'] for r in result['feeds']), + 'packets_blocked': sum(r['packets_blocked'] for r in result['feeds']), + 'bytes_blocked': sum(r['bytes_blocked'] for r in result['feeds']), + } + + yield ujson.dumps(result) + + def logs(self): + feeds = [] + for feed in self.index.get('feeds', []): + if feed['type'] == 'ip': + feeds.append('__qfeeds_%s' % feed['feed_type']) + + yield ujson.dumps({'rows': PFLogCrawler(feeds).find()}) diff --git a/security/q-feeds-connector/src/opnsense/scripts/qfeeds/lib/api.py b/security/q-feeds-connector/src/opnsense/scripts/qfeeds/lib/api.py new file mode 100644 index 000000000..3892c371f --- /dev/null +++ b/security/q-feeds-connector/src/opnsense/scripts/qfeeds/lib/api.py @@ -0,0 +1,72 @@ +""" + Copyright (c) 2025 Deciso B.V. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. +""" +import os +import requests +from configparser import ConfigParser + + +class QFeedsConfig: + api_key = None + + def __init__(self): + config_filename = '/usr/local/etc/qfeeds.conf' + if os.path.isfile(config_filename): + cnf = ConfigParser() + cnf.read(config_filename) + if cnf.has_section('api') and cnf.has_option('api', 'key'): + self.api_key = cnf.get('api', 'key') + + +class Api: + def __init__(self): + self.api_key = QFeedsConfig().api_key + + def licenses(self): + r = requests.get( + url='https://api.qfeeds.com/licenses.php', + auth=('api_token', self.api_key), + timeout=60, + headers={'User-Agent': 'Q-Feeds_OPNsense'} + ) + r.raise_for_status() + return r.json() + + def fetch(self, feed): + r = requests.get( + url='https://api.qfeeds.com/api.php', + params={'feed_type': feed}, + auth=('api_token', self.api_key), + headers={'User-Agent': 'Q-Feeds_OPNsense'}, + stream=True, + timeout=60 + ) + r.raise_for_status() + for line in r.raw: + entry = line.decode().strip() + if entry: + yield entry + + diff --git a/security/q-feeds-connector/src/opnsense/scripts/qfeeds/lib/file.py b/security/q-feeds-connector/src/opnsense/scripts/qfeeds/lib/file.py new file mode 100644 index 000000000..53d6b3ea6 --- /dev/null +++ b/security/q-feeds-connector/src/opnsense/scripts/qfeeds/lib/file.py @@ -0,0 +1,53 @@ +""" + Copyright (c) 2025 Deciso B.V. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. +""" +import fcntl + + +class LockedFile: + def __init__(self, filename): + self._filename = filename + self._fh = None + + def __enter__(self): + self._fh = open(self._filename, 'a+') + fcntl.flock(self._fh, fcntl.LOCK_EX | fcntl.LOCK_NB) + return self + + def __exit__(self, ex_type, ex_value, traceback): + if self._fh: + self._fh.close() + + def truncate(self): + self._fh.seek(0) + self._fh.truncate() + + def write(self, data): + self._fh.write(data) + + @property + def filename(self): + return self._filename + diff --git a/security/q-feeds-connector/src/opnsense/scripts/qfeeds/lib/log.py b/security/q-feeds-connector/src/opnsense/scripts/qfeeds/lib/log.py new file mode 100644 index 000000000..fed49e8e4 --- /dev/null +++ b/security/q-feeds-connector/src/opnsense/scripts/qfeeds/lib/log.py @@ -0,0 +1,77 @@ +""" + Copyright (c) 2025 Deciso B.V. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. +""" + +import glob +import time +import subprocess +import ipaddress + +def is_ip_address(value): + try: + ipaddress.ip_address(value) + return True + except ValueError: + return False + + +class PFLogCrawler: + def __init__(self, table_names:list=[]): + self._table_names = table_names + self._rule_ids = [] + self._collect_rule_ids() + + def _collect_rule_ids(self): + self._rule_ids = [] + sp = subprocess.run(['/sbin/pfctl', '-sr'], capture_output=True, text=True) + for line in sp.stdout.split("\n"): + for table in self._table_names: + if line.find("<%s>" % table) > 0: + self._rule_ids.append(line.split()[-1].strip('"')) + + @staticmethod + def _parse_log_line(line): + # quick scan for datetime, interface, direction, source, dest + parts = line.split() + fw_line = parts[-1].split(',') # strip syslog + return [parts[1], fw_line[4], fw_line[7]] + [x for x in fw_line if is_ip_address(x)] + + def find(self, max_time=60, max_results=50000): + result = [] + start_time = time.time() + rows_processed = 0 + for filename in sorted(glob.glob("/var/log/filter/filter_*.log"), reverse=True): + with open(filename) as f_in: + for idx, line in enumerate(f_in): + for rule_id in self._rule_ids: + if rule_id in line: + result.append(self._parse_log_line(line)) + rows_processed +=1 + continue + if (idx % 100000 == 0 and time.time() - start_time > max_time) or rows_processed >= max_results: + return result + + return result + diff --git a/security/q-feeds-connector/src/opnsense/scripts/qfeeds/qfeedsctl.py b/security/q-feeds-connector/src/opnsense/scripts/qfeeds/qfeedsctl.py new file mode 100755 index 000000000..737ec65e8 --- /dev/null +++ b/security/q-feeds-connector/src/opnsense/scripts/qfeeds/qfeedsctl.py @@ -0,0 +1,64 @@ +#!/usr/local/bin/python3 + +""" + Copyright (c) 2025 Deciso B.V. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. +""" + +import argparse +import sys +import ujson +from requests.exceptions import HTTPError, Timeout +from lib import QFeedsActions + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--target_dir', default='/var/db/qfeeds-tables') + parser.add_argument('-f', help='forced (auto index)' , default=False, action='store_true') + parser.add_argument('-v', help='verbose output' , default=False, action='store_true') + parser.add_argument("action", choices=QFeedsActions.list_actions(), nargs='*') + args = parser.parse_args() + if args.v: + # verbose mode + import http.client as http_client + http_client.HTTPConnection.debuglevel = 1 + try: + actions = QFeedsActions(args.target_dir, args.f) + for action in args.action: + for msg in getattr(actions, action)(): + print(msg) + except HTTPError as exc: + print('exit with HTTPError %d (%s)' % (exc.response.status_code, exc.response.text)) + sys.exit(-1) + except Timeout as exc: + print('timeout reaching api endpoint') + sys.exit(-1) + except IOError as e: + print("output filename locked or missing") + sys.exit(-1) + except ujson.JSONDecodeError: + print("JSON decode error") + sys.exit(-1) + diff --git a/security/q-feeds-connector/src/opnsense/service/conf/actions.d/actions_qfeeds.conf b/security/q-feeds-connector/src/opnsense/service/conf/actions.d/actions_qfeeds.conf new file mode 100644 index 000000000..b693285e2 --- /dev/null +++ b/security/q-feeds-connector/src/opnsense/service/conf/actions.d/actions_qfeeds.conf @@ -0,0 +1,40 @@ +[reconfigure] +command:/usr/local/opnsense/scripts/qfeeds/qfeedsctl.py fetch_index fetch firewall_load && echo 'EXIT OK' +parameters: +type:script_output +message:reconfigure QFeeds +errors:no + +[update] +command:/usr/local/opnsense/scripts/qfeeds/qfeedsctl.py update +parameters: +type:script_output +message:update QFeeds +errors:no + +[info] +command:/usr/local/opnsense/scripts/qfeeds/qfeedsctl.py show_index +parameters: +type:script_output +message:fetch QFeeds info + +[stats] +command:/usr/local/opnsense/scripts/qfeeds/qfeedsctl.py stats +parameters: +type:script_output +cache_ttl: 3600 +message:return Qfeeds local stats + +[logs] +command:/usr/local/opnsense/scripts/qfeeds/qfeedsctl.py logs +parameters: +type:script_output +cache_ttl: 300 +message:return Qfeeds log data + +[index] +command:cat /var/db/qfeeds-tables/index.json +parameters: +type:script_output +message:return raw QFeeds index file +cache_ttl: 60 diff --git a/security/q-feeds-connector/src/opnsense/service/templates/OPNsense/QFeeds/+TARGETS b/security/q-feeds-connector/src/opnsense/service/templates/OPNsense/QFeeds/+TARGETS new file mode 100644 index 000000000..da2b51f08 --- /dev/null +++ b/security/q-feeds-connector/src/opnsense/service/templates/OPNsense/QFeeds/+TARGETS @@ -0,0 +1 @@ +qfeeds.conf:/usr/local/etc/qfeeds.conf diff --git a/security/q-feeds-connector/src/opnsense/service/templates/OPNsense/QFeeds/qfeeds.conf b/security/q-feeds-connector/src/opnsense/service/templates/OPNsense/QFeeds/qfeeds.conf new file mode 100644 index 000000000..be8460cb8 --- /dev/null +++ b/security/q-feeds-connector/src/opnsense/service/templates/OPNsense/QFeeds/qfeeds.conf @@ -0,0 +1,4 @@ +{% if not helpers.empty('OPNsense.QFeedsConnector.general.apikey') %} +[api] +key={{OPNsense.QFeedsConnector.general.apikey}} +{% endif %} diff --git a/security/q-feeds-connector/src/opnsense/www/img/QFeeds.png b/security/q-feeds-connector/src/opnsense/www/img/QFeeds.png new file mode 100644 index 000000000..5dd5e38a5 Binary files /dev/null and b/security/q-feeds-connector/src/opnsense/www/img/QFeeds.png differ diff --git a/security/q-feeds-connector/src/opnsense/www/js/widgets/Metadata/QFeeds.xml b/security/q-feeds-connector/src/opnsense/www/js/widgets/Metadata/QFeeds.xml new file mode 100644 index 000000000..05f357452 --- /dev/null +++ b/security/q-feeds-connector/src/opnsense/www/js/widgets/Metadata/QFeeds.xml @@ -0,0 +1,20 @@ + + + QFeeds.js + + /api/q_feeds/settings/stats + + + Q-Feeds Threat Protection + Unable to contact information feed. + Installed Feeds + Database + Size + Blocked + Updated + Next + Licensed + Unlicensed + + + \ No newline at end of file diff --git a/security/q-feeds-connector/src/opnsense/www/js/widgets/QFeeds.js b/security/q-feeds-connector/src/opnsense/www/js/widgets/QFeeds.js new file mode 100644 index 000000000..6a9bb4be4 --- /dev/null +++ b/security/q-feeds-connector/src/opnsense/www/js/widgets/QFeeds.js @@ -0,0 +1,96 @@ +/* + * Copyright (C) 2025 Deciso B.V. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, + * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +export default class QFeeds extends BaseTableWidget { + constructor() { + super(); + } + + getMarkup() { + let $container = $('
'); + let $sysinfotable = this.createTable('qfeeds-table', { + headerPosition: 'left', + }); + $container.append($sysinfotable); + return $container; + } + + async onWidgetTick() { + return; + } + + async onMarkupRendered() { + let header = $("div.widget.widget-qfeeds").find('.widget-header'); + let title = $('#qfeeds-title'); + let divider = $("div.widget.widget-qfeeds").find('.panel-divider'); + header.css({ + 'background-image': 'URL("/ui/img/QFeeds.png")', + 'background-size': 'auto 50px', + 'background-position': 'center left', + 'margin-top': '0px', + 'mix-blend-mode': 'difference', + 'background-repeat': 'no-repeat' + }); + title.empty(); + title.css({ + 'height': '70px' + }) + divider.hide(); + $("#qfeeds-table").css({ + 'margin-top': '0px', + 'margin-bottom': '5px', + }); + + const data = await this.ajaxCall('/api/q_feeds/settings/stats'); + if (!data.feeds.length) { + $('#qfeeds-table').html(`${this.translations.no_feed}`); + return; + } + let rows = []; + let feeds = []; + for (let feed of data.feeds) { + feeds.push( + ` ${feed.name}`, + `
  ${this.translations.last_update}: ${feed.updated_at}
`, + `
  ${this.translations.next_update}: ${feed.next_update}
` + ); + if (feed.licensed) { + feeds.push(`
  ${this.translations.licensed}
`); + } else { + feeds.push(`
  ${this.translations.unlicensed}
`); + } + } + rows.push([[this.translations.installed_feeds], feeds]); + let db = [ + `
${this.translations.size}: ${data.totals.entries.toLocaleString()}
`, + `
${this.translations.blocked}: ${data.totals.addresses_blocked.toLocaleString()}
` + + ]; + rows.push([[this.translations.database], db]); + + super.updateTable('qfeeds-table', rows); + } +}