nginx logging (#805)

* www/nginx: add parser for access log and error log

* www/nginx: add missing license headers; add basic logging endpoints

* www/nginx: PSR, typos etc.

* www/nginx: add initial log viewer

* www/nginx: add log viewer

* www/nginx: add filter functionality to log view

* www/nginx: adjust log scanner and some bug fixes in viewer

* www/nginx: logviewer ready
This commit is contained in:
Fabian Franz BSc
2018-08-29 18:38:04 +02:00
committed by GitHub
parent 5484d1ab3e
commit a1036dd5a2
38 changed files with 5368 additions and 1 deletions
@@ -0,0 +1,82 @@
<?php
/*
Copyright (C) 2018 Fabian Franz
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
*/
namespace OPNsense\Nginx\Api;
use OPNsense\Base\ApiControllerBase;
use OPNsense\Core\Backend;
use OPNsense\Nginx\Nginx;
class LogsController extends ApiControllerBase
{
private $nginx;
public function accessesAction($uuid = null) {
$this->nginx = new Nginx();
if (!isset($uuid)) {
// emulate REST API -> /accesses delivers a list of servers with access logs
return $this->list_vhosts();
} else {
// emulate REST call for a specific log /accesses/uuid
return $this->call_configd('access', $uuid);
}
}
public function errorsAction($uuid = null) {
$this->nginx = new Nginx();
if (!isset($uuid)) {
// emulate REST API -> /errors delivers a list of servers with error logs
return $this->list_vhosts();
} else {
// emulate REST call for a specific log /errors/uuid
return $this->call_configd('error', $uuid);
}
}
private function call_configd($type, $uuid) {
if (!$this->vhost_exists($uuid)) {
$this->response->setStatusCode(404, "Not Found");
}
$backend = new Backend();
$data = $backend->configdRun('nginx log ' . $type . ' ' . $uuid);
return json_decode($data, true);
}
private function list_vhosts() {
$data = [];
foreach ($this->nginx->http_server->__items as $item) {
$data[] = array('id' => $item->getAttributes()['uuid'], 'server_name' => (string)$item->servername);
}
return $data;
}
private function vhost_exists($uuid) {
$data = $this->nginx->getNodeByReference('http_server.'. $uuid);
return isset($data);
}
}
@@ -37,6 +37,10 @@ namespace OPNsense\Nginx;
*/
class IndexController extends \OPNsense\Base\IndexController
{
/**
* show the configuration page /ui/nginx
* @throws \Exception when a form cannot be loaded
*/
public function indexAction()
{
$this->view->settings = $this->getForm("settings");
@@ -52,4 +56,11 @@ class IndexController extends \OPNsense\Base\IndexController
$this->view->security_headers = $this->getForm("security_headers");
$this->view->pick('OPNsense/Nginx/index');
}
/**
* show the nginx logs page /ui/nginx/index/logs
*/
public function logsAction() {
$this->view->pick('OPNsense/Nginx/logs');
}
}
@@ -0,0 +1,43 @@
<?php
/*
Copyright (C) 2018 Fabian Franz
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
*/
namespace OPNsense\Nginx;
class AccessLogLine
{
public $remote_ip;
public $username;
public $time;
public $status;
public $size;
public $user_agent;
public $request_line;
public $forwarded_for;
public $http_referer;
}
@@ -0,0 +1,65 @@
<?php
/*
Copyright (C) 2018 Fabian Franz
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
*/
namespace OPNsense\Nginx;
class AccessLogParser
{
private $file_name;
private $lines;
private $result;
private const LogLineRegex = '/(\S+) - (\S+) \[([\d\sa-z\:\-\/\+]+)\] "([^"]+?)" (\d+) (\d+) "([^"]*?)" "([^"]*?)" "([^"]*?)"/i';
function __construct($file_name)
{
$this->file_name = $file_name;
$this->lines = file($this->file_name);
$this->result = array_map([$this, 'parse_line'], $this->lines);
}
private function parse_line($line) {
$container = new AccessLogLine();
if (preg_match(self::LogLineRegex, $line, $data)) {
$container->remote_ip = $data[1];
$container->username = $data[2];
$container->time = $data[3];
$container->request_line = $data[4];
$container->status = $data[5];
$container->size = $data[6];
$container->user_agent = $data[8];
$container->http_referer = $data[7];
$container->forwarded_for = $data[9];
}
return $container;
}
public function get_result() {
return $this->result;
}
}
@@ -0,0 +1,38 @@
<?php
/*
Copyright (C) 2018 Fabian Franz
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
*/
namespace OPNsense\Nginx;
class ErrorLogLine
{
public $date;
public $time;
public $severity;
public $number;
public $message;
}
@@ -0,0 +1,60 @@
<?php
/*
Copyright (C) 2018 Fabian Franz
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
*/
namespace OPNsense\Nginx;
class ErrorLogParser
{
private $file_name;
private $lines;
private $result;
private const LogLineRegex = '/(\S+) (\S+) \[([\d\sa-z\:\-\/\+\#]+)\] ([\S:]+): (.+)/i';
function __construct($file_name)
{
$this->file_name = $file_name;
$this->lines = file($this->file_name);
$this->result = array_map([$this, 'parse_line'], $this->lines);
}
private function parse_line($line) {
$container = new ErrorLogLine();
if (preg_match(self::LogLineRegex, $line, $data)) {
$container->date = $data[1];
$container->time = $data[2];
$container->severity = $data[3];
$container->number = $data[4];
$container->message = $data[5];
}
return $container;
}
public function get_result() {
return $this->result;
}
}
@@ -1,5 +1,8 @@
<menu>
<Services>
<Nginx cssClass="fa fa-shield fa-fw" url="/ui/nginx" />
<Nginx cssClass="fa fa-shield fa-fw">
<Configuration url="/ui/nginx" />
<Logs url="/ui/nginx/index/logs" />
</Nginx>
</Services>
</menu>
@@ -0,0 +1,32 @@
{#
# Copyright (C) 2017-2018 Fabian Franz
# Copyright (C) 2014-2015 Deciso B.V.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
# AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#}
<div id="logapplication"></div>
<script src="/ui/js/nginx/lib/lodash.min.js"></script>
<script src="/ui/js/nginx/lib/backbone-min.js"></script>
<script src="/ui/js/nginx/dist/bundle.js"></script>
@@ -1,5 +1,31 @@
<?php
/*
* Copyright (C) 2018 Fabian Franz
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
* AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
* OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
$log_file = '/var/log/nginx/csp_violations.log';
// make sure we don't have any formatting issues here
+74
View File
@@ -0,0 +1,74 @@
#!/usr/local/bin/php
<?php
/*
* Copyright (C) 2018 Fabian Franz
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
* AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
* OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
require_once 'config.inc';
use OPNsense\Nginx\Nginx;
use OPNsense\Nginx\ErrorLogParser;
use OPNsense\Nginx\AccessLogParser;
$log_prefix = '/var/log/nginx/';
$log_suffix = '.log';
if ($_SERVER['argc'] != 3) die('{"error": "Incorrect amount of parameters given"}');
// first parameter: error|access
$mode = $_SERVER['argv'][1];
// second parameter: uuid of server
$server = $_SERVER['argv'][2];
$nginx = new Nginx();
if ($data = $nginx->getNodeByReference('http_server.'. $server)) {
$server_names = (string)$data->servername;
if (empty($server_names)) {
die('{"error": "The server entry has no server name"}');
}
$lines = [];
foreach (explode(',', $server_names) as $server_name) {
$log_file_name = $log_prefix . basename($server_name) . '.' . $mode . $log_suffix;
// this entry has no log file, ignore it
if (!file_exists($log_file_name)) continue;
$logparser = null;
if ($mode == 'error') {
$logparser = new ErrorLogParser($log_file_name);
}
elseif ($mode == 'access') {
$logparser = new AccessLogParser($log_file_name);
}
// we cannot parse the file - something went wrong
if ($logparser == null) continue;
$lines = array_merge($lines, $logparser->get_result());
}
if (empty($lines)) {
$lines['error'] = 'no lines found';
}
echo json_encode($lines);
}
else {
die('{"error": "UUID not found"}');
}
@@ -1,6 +1,32 @@
#!/usr/local/bin/php
<?php
/*
* Copyright (C) 2018 Fabian Franz
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
* AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
* OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
require_once('config.inc');
require_once('certs.inc');
use \OPNsense\Nginx\Nginx;
@@ -25,3 +25,9 @@ type:script_output
command:/usr/local/etc/rc.d/php-fpm status; exit 0
parameters:
type:script_output
[log]
command:/usr/local/opnsense/scripts/nginx/read_log.php
parameters: %s %s
type:script_output
message:restarting nginx
@@ -79,6 +79,7 @@ server {
server_name {{ server.servername }};
charset {{ server.charset }};
access_log /var/log/nginx/{{ server.servername }}.access.log {{ server.access_log_format }};
error_log /var/log/nginx/{{ server.servername }}.error.log;
{% if server.root is defined and server.root != '' %}
root "{{server.root}}";
{% endif %}
@@ -0,0 +1,10 @@
{
"parserOptions": {
"ecmaVersion": 6,
"sourceType": "module"
},
"rules": {
"semi": "warn",
"no-extra-semi": "error"
}
}
@@ -0,0 +1 @@
node_modules
+3
View File
@@ -0,0 +1,3 @@
This file contains the build result of webpack.
Please do not delete it.
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,22 @@
Copyright (c) 2010-2017 Jeremy Ashkenas, DocumentCloud
Permission is hereby granted, free of charge, to any person
obtaining a copy of this software and associated documentation
files (the "Software"), to deal in the Software without
restriction, including without limitation the rights to use,
copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following
conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
File diff suppressed because one or more lines are too long

Some files were not shown because too many files have changed in this diff Show More