netbird/plugins
0
0
Fork 0
mirror of https://github.com/netbirdio/plugins.git synced 2026-05-22 18:44:07 -07:00
Code Issues Packages Projects Releases Wiki Activity

security/q-feeds-connector - add initial version (ref: https://forum.opnsense.org/index.php?topic=49123.0)

Browse Source
This commit is contained in:
Ad Schellevis
2025-10-11 09:07:04 +02:00
parent a9c5f61850
commit 27bd359a36
27 changed files with 1198 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
security/q-feeds-connector/+POST_INSTALL.post
Executable
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
/usr/local/sbin/pluginctl -s cron restart
security/q-feeds-connector/Makefile
+6
View File
@@ -0,0 +1,6 @@
PLUGIN_NAME= q-feeds-connector
PLUGIN_VERSION= 1.0
PLUGIN_COMMENT= Connector for Q-Feeds threat intel
PLUGIN_MAINTAINER= devel@qfeeds.com
.include "../../Mk/plugins.mk"
security/q-feeds-connector/README.md
+48
View File
@@ -0,0 +1,48 @@
# plugin to connect to QFeeds threat platform
Register for a token at : https://qfeeds.com/opnsense/
# command line control
Settings are persisted in `/usr/local/etc/qfeeds.conf`, using the following format:
```
[api]
key=tip_xxxxxxxx
```
Our commandline tool contains all actions used by the UI, which is practical for debuggging.
```
usage: qfeedsctl.py [-h] [--target_dir TARGET_DIR] [-f] [-v] [{fetch_index,fetch,show_index,firewall_load,update,stats} ...]
positional arguments:
{fetch_index,fetch,show_index,firewall_load,update,stats}
options:
-h, --help show this help message and exit
--target_dir TARGET_DIR
-f forced (auto index)
-v verbose output
```
The index is the driver for most actions, which is a json encoded file in `/var/db/qfeeds-tables/index.json`.
Actions supported:
* fetch_index --> download the index file
* fetch --> download the lists
* firewall_load --> collect ip lists into pre-defined firewall tables
* update [sleep when almost time] --> run fetch_index --> fetch --> firewall_load (to be used by cron)
* show_index --> dumps the index
* stats --> dumps feed information
* logs --> dumps firewall log information for aliases offered by Q-Feeds
Example usage:
```
/usr/local/opnsense/scripts/qfeeds/qfeedsctl.py update
```
**Fetch index and update lists when updated remotely**
security/q-feeds-connector/pkg-descr
+8
View File
@@ -0,0 +1,8 @@
Connector for Q-Feeds threat intel
Plugin Changelog
================
1.0
* Intial release version
security/q-feeds-connector/src/etc/inc/plugins.inc.d/qfeeds.inc
+35
View File
@@ -0,0 +1,35 @@
<?php
/*
* Copyright (C) 2025 Deciso B.V.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
* AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
* OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
function qfeeds_cron()
{
$jobs = [];
$jobs[]['autocron'] = ['/usr/local/sbin/configctl -d qfeeds update', '*/5'];
$jobs[]['autocron'] = ['/usr/local/sbin/configctl -d ! qfeeds stats', '*/15'];
return $jobs;
}
security/q-feeds-connector/src/etc/rc.syshook.d/start/99-qfeeds
Executable
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
# load already collected feeds
/usr/local/opnsense/scripts/qfeeds/qfeedsctl.py firewall_load
security/q-feeds-connector/src/opnsense/mvc/app/controllers/OPNsense/QFeeds/Api/SettingsController.php
+117
View File
@@ -0,0 +1,117 @@
<?php
/**
* Copyright (C) 2025 Deciso B.V.
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
* AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
* OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
*/
namespace OPNsense\QFeeds\Api;
use OPNsense\Base\ApiMutableModelControllerBase;
use OPNsense\Base\UserException;
use OPNsense\Core\Backend;
use OPNsense\Core\Config;
class SettingsController extends ApiMutableModelControllerBase
{
protected static $internalModelName = 'connect';
protected static $internalModelClass = 'OPNsense\QFeeds\Connector';
public function reconfigureAction()
{
$backend = new Backend();
$res = trim($backend->configdRun('template reload OPNsense/QFeeds'));
if (strtolower($res) != 'ok') {
throw new UserException(sprintf(gettext("Unable to update settings (%s)"), $res));
}
$res = trim($backend->configdRun('qfeeds reconfigure'));
if (strpos($res, 'EXIT OK') === false) {
throw new UserException($res);
}
return ['status' => 'ok', 'output' => $res];
}
public function searchFeedsAction()
{
$records = [];
$data = json_decode((new Backend())->configdRun('qfeeds info') ?? '[]', true);
if (!empty($data) && !empty($data['feeds'])) {
$records = $data['feeds'];
foreach ($records as &$record) {
$record['licensed'] = $record['licensed'] ? '1' : '0';
}
}
return $this->searchRecordsetBase($records);
}
public function searchEventsAction()
{
$records = [];
$ifnames = [];
foreach (Config::getInstance()->object()->interfaces->children() as $key => $node) {
if (!empty((string)$node->if)) {
$ifnames[(string)$node->if] = (string)($node->descr ?? strtoupper($key));
}
}
$data = json_decode((new Backend())->configdRun('qfeeds logs') ?? '[]', true);
if (!empty($data) && !empty($data['rows'])) {
foreach ($data['rows'] as $row) {
$records[] = [
'timestamp' => $row[0],
'interface' => $ifnames[$row[1]] ?? $row[1],
'direction' => $row[2],
'source' => $row[3],
'destination' => $row[4],
];
}
}
return $this->searchRecordsetBase($records);
}
public function statsAction()
{
$stats = json_decode((new Backend())->configdRun('qfeeds stats'), true);
if (!empty($stats) && !empty($stats['feeds'])) {
$info = json_decode((new Backend())->configdRun('qfeeds info'), true);
if (!empty($info) && !empty($info['feeds'])) {
$feeds = [];
foreach ($info['feeds'] as $feed) {
$feeds[$feed['feed_type']] = $feed;
}
foreach ($stats['feeds'] as &$feed) {
if (isset($feeds[$feed['name']])) {
$tmp = $feeds[$feed['name']];
$feed['updated_at'] = $tmp['updated_at'];
$feed['next_update'] = $tmp['next_update'];
$feed['licensed'] = $tmp['licensed'];
}
}
}
}
return $stats;
}
}
security/q-feeds-connector/src/opnsense/mvc/app/controllers/OPNsense/QFeeds/IndexController.php
+40
View File
@@ -0,0 +1,40 @@
<?php
/**
* Copyright (C) 2025 Deciso B.V.
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
* AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
* OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
*/
namespace OPNsense\QFeeds;
class IndexController extends \OPNsense\Base\IndexController
{
public function indexAction()
{
$this->view->formSettings = $this->getForm("settings");
$this->view->pick('OPNsense/QFeeds/index');
}
}
security/q-feeds-connector/src/opnsense/mvc/app/controllers/OPNsense/QFeeds/forms/settings.xml
+12
View File
@@ -0,0 +1,12 @@
<form>
<field>
<type>header</type>
<label>General Settings</label>
</field>
<field>
<id>connect.general.apikey</id>
<label>API key</label>
<type>text</type>
<help><![CDATA[API key to access Q-Feeds services, to apply for a key, <a target="_new" href="https://qfeeds.com/opnsense/">click here</a>]]></help>
</field>
</form>
security/q-feeds-connector/src/opnsense/mvc/app/library/OPNsense/System/Status/QfeedsStatus.php
+61
View File
@@ -0,0 +1,61 @@
<?php
/*
* Copyright (C) 2025 Deciso B.V.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
* AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
* OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
namespace OPNsense\System\Status;
use OPNsense\System\AbstractStatus;
use OPNsense\System\SystemStatusCode;
use OPNsense\Core\Config;
class QfeedsStatus extends AbstractStatus
{
public function __construct()
{
$this->internalPriority = 2;
$this->internalPersistent = true;
$this->internalIsBanner = true;
$this->internalTitle = gettext('QFeeds');
$this->internalScope = [
'/ui/q_feeds/'
];
}
public function collectStatus()
{
$cnf = Config::getInstance()->object();
if (!empty($cnf->system->maximumtableentries) && $cnf->system->maximumtableentries >= 2000000) {
return;
}
$this->internalStatus = SystemStatusCode::ERROR;
$this->internalMessage = gettext(
'QFeeds requires additional memory to be reserved for aliases. ' .
'Please increase `Firewall Maximum Table Entries` in `Firewall: Settings: Advanced` to at least' .
' 2 million items.'
);
}
}
security/q-feeds-connector/src/opnsense/mvc/app/models/OPNsense/Firewall/DynamicAliases/QfeedsAliases.php
+56
View File
@@ -0,0 +1,56 @@
<?php
/*
* Copyright (C) 2025 Deciso B.V.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
* AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
* OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
namespace OPNsense\Firewall\DynamicAliases;
use OPNsense\Core\Backend;
class QfeedsAliases
{
public function collect()
{
$result = [];
$payload = json_decode((new Backend())->configdRun('qfeeds index') ?? '', true) ?? [];
if (is_array($payload) && !empty($payload['feeds'])) {
foreach ($payload['feeds'] as $feed) {
if ($feed['type'] == 'ip' && !empty($feed['licensed'])) {
$name = '__qfeeds_'. $feed['feed_type'];
$result[$name] = [
'enabled' => '1',
'counters' => '1',
'name' => $name,
'type' => 'external',
'description' => $feed['feed_type'],
'content' => ''
];
}
}
}
return $result;
}
}
security/q-feeds-connector/src/opnsense/mvc/app/models/OPNsense/QFeeds/ACL/ACL.xml
+9
View File
@@ -0,0 +1,9 @@
<acl>
<page-firewall-qfeeds>
<name>Services: QFeeds</name>
<patterns>
<pattern>ui/q_feeds/*</pattern>
<pattern>api/q_feeds/*</pattern>
</patterns>
</page-firewall-qfeeds>
</acl>
security/q-feeds-connector/src/opnsense/mvc/app/models/OPNsense/QFeeds/Connector.php
+38
View File
@@ -0,0 +1,38 @@
<?php
/*
* Copyright (C) 2025 Deciso B.V.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
* AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
* OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
namespace OPNsense\QFeeds;
use OPNsense\Base\BaseModel;
/**
* Class Connector
* @package OPNsense\QFeeds
*/
class Connector extends BaseModel
{
}
security/q-feeds-connector/src/opnsense/mvc/app/models/OPNsense/QFeeds/Connector.xml
+10
View File
@@ -0,0 +1,10 @@
<model>
<mount>//OPNsense/QFeedsConnector</mount>
<version>1.0.0</version>
<description>QFeeds connector</description>
<items>
<general>
<apikey type="TextField"/>
</general>
</items>
</model>
security/q-feeds-connector/src/opnsense/mvc/app/models/OPNsense/QFeeds/Menu/Menu.xml
+9
View File
@@ -0,0 +1,9 @@
<menu>
<Security order="65" cssClass="fa fa-shield">
<QFeeds VisibleName="Q-Feeds Connect" cssClass="fa fa-heartbeat fa-fw">
<Settings order="10" url="/ui/q_feeds/#settings"/>
<Feeds order="20" url="/ui/q_feeds/#feeds"/>
<Events order="30" url="/ui/q_feeds/#events"/>
</QFeeds>
</Security>
</menu>
security/q-feeds-connector/src/opnsense/mvc/app/views/OPNsense/QFeeds/index.volt
+136
View File
@@ -0,0 +1,136 @@
{#
OPNsense® is Copyright © 2025 by Deciso B.V.
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
#}
<script>
$( document ).ready(function() {
let data_get_map = {'frm_settings':"/api/q_feeds/settings/get"};
mapDataToFormUI(data_get_map).done(function(){
formatTokenizersUI();
$('.selectpicker').selectpicker('refresh');
});
$("#reconfigureAct").SimpleActionButton({
onPreAction: function() {
const dfObj = new $.Deferred();
saveFormToEndpoint("/api/q_feeds/settings/set", 'frm_settings', function(){
dfObj.resolve();
});
return dfObj;
}
});
$('a[data-toggle="tab"]').on('shown.bs.tab', function (e) {
if (e.target.id === 'feeds_tab') {
if (!$("#grid-feeds").hasClass('tabulator')) {
$("#grid-feeds").UIBootgrid({
'search': '/api/q_feeds/settings/search_feeds/'
});
} else {
$("#grid-feeds").bootgrid("reload");
}
} else if (e.target.id === 'events_tab') {
if (!$("#grid-events").hasClass('tabulator')) {
$("#grid-events").UIBootgrid({
'search': '/api/q_feeds/settings/search_events/'
});
} else {
$("#grid-events").bootgrid("reload");
}
}
});
let selected_tab = window.location.hash != "" ? window.location.hash : "#settings";
$('a[href="' +selected_tab + '"]').tab('show');
$('.nav-tabs a').on('shown.bs.tab', function (e) {
history.pushState(null, null, e.target.hash);
});
$(window).on('hashchange', function(e) {
$('a[href="' + window.location.hash + '"]').click()
});
});
</script>
<ul class="nav nav-tabs" data-tabs="tabs" id="maintabs">
<li><a data-toggle="tab" href="#settings" id="settings_tab">{{ lang._('Settings') }}</a></li>
<li><a data-toggle="tab" href="#feeds" id="feeds_tab">{{ lang._('Feeds') }}</a></li>
<li><a data-toggle="tab" href="#events" id="events_tab">{{ lang._('Events') }}</a></li>
</ul>
<div class="tab-content content-box">
<div id="settings" class="tab-pane fade in">
{{ partial("layout_partials/base_form",['fields':formSettings,'id':'frm_settings'])}}
</div>
<div id="feeds" class="tab-pane fade in">
<table id="grid-feeds" class="table table-condensed table-hover table-striped table-responsive">
<thead>
<tr>
<th data-column-id="description" data-type="string">{{ lang._('Description') }}</th>
<th data-column-id="type" data-type="string">{{ lang._('Type') }}</th>
<th data-column-id="updated_at" data-type="string">{{ lang._('Updated at') }}</th>
<th data-column-id="next_update" data-type="string">{{ lang._('Next update') }}</th>
<th data-column-id="licensed" data-type="boolean" data-formatter="boolean">{{ lang._('Licensed') }}</th>
</tr>
</thead>
<tbody>
</tbody>
</table>
</div>
<div id="events" class="tab-pane fade in">
<table id="grid-events" class="table table-condensed table-hover table-striped table-responsive">
<thead>
<tr>
<th data-column-id="timestamp" data-type="string">{{ lang._('Timestamp') }}</th>
<th data-column-id="interface" data-type="string">{{ lang._('Interface') }}</th>
<th data-column-id="direction" data-type="string">{{ lang._('Direction') }}</th>
<th data-column-id="source" data-type="string">{{ lang._('Source') }}</th>
<th data-column-id="destination" data-type="string">{{ lang._('Destination') }}</th>
</tr>
</thead>
<tbody>
</tbody>
</table>
<div class="pull-right">{{ lang._('Collected events from the firewall log for QFeed aliases') }}</div>
</div>
</div>
<section class="page-content-main">
<div class="content-box">
<div class="col-md-12">
<br/>
<button class="btn btn-primary" id="reconfigureAct"
data-endpoint='/api/q_feeds/settings/reconfigure'
data-label="{{ lang._('Apply') }}"
data-error-title="{{ lang._('Error reconfiguring QFeeds connect') }}"
type="button"
></button>
<br/><br/>
</div>
</div>
</section>
security/q-feeds-connector/src/opnsense/scripts/qfeeds/lib/__init__.py
+179
View File
@@ -0,0 +1,179 @@
"""
Copyright (c) 2025 Deciso B.V.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
"""
import os
import subprocess
import time
import ujson
from datetime import datetime
from lib.api import Api
from lib.log import PFLogCrawler
from lib.file import LockedFile
class QFeedsActions:
def __init__(self, target_dir, forced=False):
self._target_dir = target_dir
self._forced = forced
@classmethod
def list_actions(cls):
return [
'fetch_index',
'fetch',
'show_index',
'firewall_load',
'update',
'stats',
'logs'
]
@property
def index_file(self):
return "%s/index.json" % self._target_dir
@property
def index(self):
if not os.path.exists(self.index_file) and self._forced:
# require index file to get feeds
list(self.fetch_index())
elif not os.path.exists(self.index_file):
return {}
data = ujson.load(open(self.index_file)) or {}
if type(data) is dict:
for feed in data.get('feeds', []):
feed['local_filename'] = "%s/%s.txt" % (self._target_dir, feed['feed_type'])
feed['updated_at_dt'] = datetime.fromisoformat(feed['updated_at']).timestamp()
feed['next_update_dt'] = datetime.fromisoformat(feed['next_update']).timestamp()
return data
def _file_stat(self, filename):
if not os.path.exists(filename):
return 0
return os.stat(filename).st_mtime
def fetch_index(self):
if not os.path.isdir(self._target_dir):
os.makedirs(self._target_dir)
with LockedFile(self.index_file) as f:
payload = Api().licenses()
f.truncate()
f.write(ujson.dumps(payload))
yield 'downloaded index to %s' % f.filename
def show_index(self):
yield ujson.dumps(self.index)
def fetch(self):
for feed in self.index.get('feeds', []):
if feed['licensed'] and feed['updated_at_dt'] != self._file_stat(feed['local_filename']):
with LockedFile(feed['local_filename']) as f:
counter = 0
for entry in Api().fetch(feed['feed_type']):
if counter == 0:
f.truncate()
f.write("%s\n" % entry)
counter += 1
os.utime(feed['local_filename'], (feed['updated_at_dt'], feed['updated_at_dt']))
yield "downloaded %d entries into %s [%s]" % (counter, feed['local_filename'], feed['updated_at'])
elif feed['licensed']:
yield "skipped %s [%s]" % (feed['local_filename'], feed['updated_at'])
def firewall_load(self):
for feed in self.index.get('feeds', []):
if feed['licensed'] and os.path.exists(feed['local_filename']) and feed['type'] == 'ip':
table_name = '__qfeeds_%s' % feed['feed_type']
sp = subprocess.run(
['/sbin/pfctl', '-t', table_name, '-T', 'replace', '-f', feed['local_filename']],
capture_output=True,
text=True
)
yield 'load feed %s [%s]' % (feed['feed_type'], sp.stderr.strip().replace("\n", " "))
def update(self):
update_sleep = 99999
try:
index_payload = self.index
except TypeError:
# when the index can't be parsed, assume we have none while updating
index_payload = {}
do_update = len(index_payload.get('feeds', [])) == 0
for feed in index_payload.get('feeds', []):
update_sleep = min(feed['next_update_dt'] - time.time(), update_sleep)
if feed['licensed'] and update_sleep <= 300: # 5 minute cron interval
do_update = True
if do_update:
if 0 < update_sleep <= 300:
time.sleep(update_sleep)
for action in ['fetch_index', 'fetch', 'firewall_load']:
yield from getattr(self, action)()
def stats(self):
result = {'feeds': []}
for feed in self.index.get('feeds', []):
if feed['licensed'] and os.path.exists(feed['local_filename']) and feed['type'] == 'ip':
table_name = '__qfeeds_%s' % feed['feed_type']
sp = subprocess.Popen(
['/sbin/pfctl', '-t', table_name, '-vT', 'show'],
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True
)
record = {
'name': feed['feed_type'],
'total_entries': 0,
'packets_blocked': 0,
'bytes_blocked': 0,
'addresses_blocked': 0
}
while (line := sp.stdout.readline()):
if line.startswith(' '):
record['total_entries'] += 1
elif 'Packets:' in line and 'Packets: 0 ' not in line:
parts = line.split()
if parts[3].isdigit() and parts[5].isdigit() and parts[0].lower().find('block') > 0:
record['packets_blocked'] += int(parts[3])
record['bytes_blocked'] += int(parts[5])
record['addresses_blocked'] += 1
result['feeds'].append(record)
result['totals'] = {
'entries': sum(r['total_entries'] for r in result['feeds']),
# assumes no overlaps in datafeeds
'addresses_blocked': sum(r['addresses_blocked'] for r in result['feeds']),
'packets_blocked': sum(r['packets_blocked'] for r in result['feeds']),
'bytes_blocked': sum(r['bytes_blocked'] for r in result['feeds']),
}
yield ujson.dumps(result)
def logs(self):
feeds = []
for feed in self.index.get('feeds', []):
if feed['type'] == 'ip':
feeds.append('__qfeeds_%s' % feed['feed_type'])
yield ujson.dumps({'rows': PFLogCrawler(feeds).find()})
security/q-feeds-connector/src/opnsense/scripts/qfeeds/lib/api.py
+72
View File
@@ -0,0 +1,72 @@
"""
Copyright (c) 2025 Deciso B.V.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
"""
import os
import requests
from configparser import ConfigParser
class QFeedsConfig:
api_key = None
def __init__(self):
config_filename = '/usr/local/etc/qfeeds.conf'
if os.path.isfile(config_filename):
cnf = ConfigParser()
cnf.read(config_filename)
if cnf.has_section('api') and cnf.has_option('api', 'key'):
self.api_key = cnf.get('api', 'key')
class Api:
def __init__(self):
self.api_key = QFeedsConfig().api_key
def licenses(self):
r = requests.get(
url='https://api.qfeeds.com/licenses.php',
auth=('api_token', self.api_key),
timeout=60,
headers={'User-Agent': 'Q-Feeds_OPNsense'}
)
r.raise_for_status()
return r.json()
def fetch(self, feed):
r = requests.get(
url='https://api.qfeeds.com/api.php',
params={'feed_type': feed},
auth=('api_token', self.api_key),
headers={'User-Agent': 'Q-Feeds_OPNsense'},
stream=True,
timeout=60
)
r.raise_for_status()
for line in r.raw:
entry = line.decode().strip()
if entry:
yield entry
security/q-feeds-connector/src/opnsense/scripts/qfeeds/lib/file.py
+53
View File
@@ -0,0 +1,53 @@
"""
Copyright (c) 2025 Deciso B.V.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
"""
import fcntl
class LockedFile:
def __init__(self, filename):
self._filename = filename
self._fh = None
def __enter__(self):
self._fh = open(self._filename, 'a+')
fcntl.flock(self._fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
return self
def __exit__(self, ex_type, ex_value, traceback):
if self._fh:
self._fh.close()
def truncate(self):
self._fh.seek(0)
self._fh.truncate()
def write(self, data):
self._fh.write(data)
@property
def filename(self):
return self._filename
security/q-feeds-connector/src/opnsense/scripts/qfeeds/lib/log.py
+77
View File
@@ -0,0 +1,77 @@
"""
Copyright (c) 2025 Deciso B.V.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
"""
import glob
import time
import subprocess
import ipaddress
def is_ip_address(value):
try:
ipaddress.ip_address(value)
return True
except ValueError:
return False
class PFLogCrawler:
def __init__(self, table_names:list=[]):
self._table_names = table_names
self._rule_ids = []
self._collect_rule_ids()
def _collect_rule_ids(self):
self._rule_ids = []
sp = subprocess.run(['/sbin/pfctl', '-sr'], capture_output=True, text=True)
for line in sp.stdout.split("\n"):
for table in self._table_names:
if line.find("<%s>" % table) > 0:
self._rule_ids.append(line.split()[-1].strip('"'))
@staticmethod
def _parse_log_line(line):
# quick scan for datetime, interface, direction, source, dest
parts = line.split()
fw_line = parts[-1].split(',') # strip syslog
return [parts[1], fw_line[4], fw_line[7]] + [x for x in fw_line if is_ip_address(x)]
def find(self, max_time=60, max_results=50000):
result = []
start_time = time.time()
rows_processed = 0
for filename in sorted(glob.glob("/var/log/filter/filter_*.log"), reverse=True):
with open(filename) as f_in:
for idx, line in enumerate(f_in):
for rule_id in self._rule_ids:
if rule_id in line:
result.append(self._parse_log_line(line))
rows_processed +=1
continue
if (idx % 100000 == 0 and time.time() - start_time > max_time) or rows_processed >= max_results:
return result
return result

Some files were not shown because too many files have changed in this diff Show More