From 537da9cbe58d5291c5963df8e72fda5e7f770eba Mon Sep 17 00:00:00 2001 From: Fabian Franz Date: Mon, 17 Dec 2018 18:55:04 +0100 Subject: [PATCH 1/2] www/nginx: backend performance improvements don't read log file at once --- .../OPNsense/Nginx/Api/LogsController.php | 34 +++++++++++++------ .../OPNsense/Nginx/AccessLogParser.php | 16 +++++++-- .../OPNsense/Nginx/StreamAccessLogParser.php | 17 ++++++++-- 3 files changed, 51 insertions(+), 16 deletions(-) diff --git a/www/nginx/src/opnsense/mvc/app/controllers/OPNsense/Nginx/Api/LogsController.php b/www/nginx/src/opnsense/mvc/app/controllers/OPNsense/Nginx/Api/LogsController.php index dbbee6481..75bba032c 100644 --- a/www/nginx/src/opnsense/mvc/app/controllers/OPNsense/Nginx/Api/LogsController.php +++ b/www/nginx/src/opnsense/mvc/app/controllers/OPNsense/Nginx/Api/LogsController.php @@ -42,7 +42,7 @@ class LogsController extends ApiControllerBase return $this->list_vhosts(); } else { // emulate REST call for a specific log /accesses/uuid - return $this->call_configd('access', $uuid); + $this->call_configd('access', $uuid); } } @@ -54,9 +54,10 @@ class LogsController extends ApiControllerBase return $this->list_vhosts(); } else { // emulate REST call for a specific log /errors/uuid - return $this->call_configd('error', $uuid); + $this->call_configd('error', $uuid); } } + public function stream_accessesAction($uuid = null) { $this->nginx = new Nginx(); @@ -65,9 +66,10 @@ class LogsController extends ApiControllerBase return $this->list_streams(); } else { // emulate REST call for a specific log /stream_accesses/uuid - return $this->call_configd_stream('streamaccess', $uuid); + $this->call_configd_stream('streamaccess', $uuid); } } + public function stream_errorsAction($uuid = null) { $this->nginx = new Nginx(); @@ -76,19 +78,18 @@ class LogsController extends ApiControllerBase return $this->list_streams(); } else { // emulate REST call for a specific log /stream_errors/uuid - return $this->call_configd_stream('streamerror', $uuid); + $this->call_configd_stream('streamerror', $uuid); } } + private function call_configd($type, $uuid) { if (!$this->vhost_exists($uuid)) { $this->response->setStatusCode(404, "Not Found"); } - $backend = new Backend(); - $data = $backend->configdRun('nginx log ' . $type . ' ' . $uuid); - return json_decode($data, true); + return $this->sendConfigdToClient('nginx log ' . $type . ' ' . $uuid); } private function call_configd_stream($type, $uuid) { @@ -96,9 +97,7 @@ class LogsController extends ApiControllerBase $this->response->setStatusCode(404, "Not Found"); } - $backend = new Backend(); - $data = $backend->configdRun('nginx log ' . $type . ' ' . $uuid); - return json_decode($data, true); + return $this->sendConfigdToClient('nginx log ' . $type . ' ' . $uuid); } private function list_vhosts() @@ -128,4 +127,19 @@ class LogsController extends ApiControllerBase $data = $this->nginx->getNodeByReference('stream_server.'. $uuid); return isset($data); } + + /** + * @param $command String JSON generating configd command + * @return null + * @throws \Exception ? + */ + private function sendConfigdToClient($command) + { + $backend = new Backend(); + // must be passed directly -> OOM Problem + $this->response->setContent($backend->configdRun($command)); + $this->response->setStatusCode(200, "OK"); + $this->response->setContentType('application/json', 'UTF-8'); + return $this->response->send(); + } } diff --git a/www/nginx/src/opnsense/mvc/app/library/OPNsense/Nginx/AccessLogParser.php b/www/nginx/src/opnsense/mvc/app/library/OPNsense/Nginx/AccessLogParser.php index aa288caf9..8bf5c6c03 100644 --- a/www/nginx/src/opnsense/mvc/app/library/OPNsense/Nginx/AccessLogParser.php +++ b/www/nginx/src/opnsense/mvc/app/library/OPNsense/Nginx/AccessLogParser.php @@ -31,7 +31,6 @@ namespace OPNsense\Nginx; class AccessLogParser { private $file_name; - private $lines; private $result; private const LogLineRegex = '/(\S+) - (\S+) \[([\d\sa-z\:\-\/\+]+)\] "([^"]+?)" (\d+) (\d+) "([^"]*?)" "([^"]*?)" "([^"]*?)"/i'; @@ -39,9 +38,20 @@ class AccessLogParser function __construct($file_name) { $this->file_name = $file_name; - $this->lines = file($this->file_name); - $this->result = array_map([$this, 'parse_line'], $this->lines); + $this->result = array(); + $this->parse_file(); } + + private function parse_file() { + $handle = @fopen($this->file_name, 'r'); + if ($handle) { + while (($buffer = fgets($handle)) !== false) { + $this->result[] = $this->parse_line($buffer); + } + fclose($handle); + } + } + private function parse_line($line) { $container = new AccessLogLine(); diff --git a/www/nginx/src/opnsense/mvc/app/library/OPNsense/Nginx/StreamAccessLogParser.php b/www/nginx/src/opnsense/mvc/app/library/OPNsense/Nginx/StreamAccessLogParser.php index 940f25d3b..d897c94cb 100644 --- a/www/nginx/src/opnsense/mvc/app/library/OPNsense/Nginx/StreamAccessLogParser.php +++ b/www/nginx/src/opnsense/mvc/app/library/OPNsense/Nginx/StreamAccessLogParser.php @@ -31,17 +31,28 @@ namespace OPNsense\Nginx; class StreamAccessLogParser { private $file_name; - private $lines; private $result; private const LogLineRegex = '/(\S+) \[([\d\sa-z\:\-\/\+]+)\] (\S+?) (\d+) (\d+) (\d+) (\d+(?:\.\d+)?)/i'; + function __construct($file_name) { $this->file_name = $file_name; - $this->lines = file($this->file_name); - $this->result = array_map([$this, 'parse_line'], $this->lines); + $this->result = array(); + $this->parse_file(); } + + private function parse_file() { + $handle = @fopen($this->file_name, 'r'); + if ($handle) { + while (($buffer = fgets($handle)) !== false) { + $this->result[] = $this->parse_line($buffer); + } + fclose($handle); + } + } + private function parse_line($line) { $container = new StreamAccessLogLine(); From 94fb4f2dee11157c33edf3f676571d828e651db4 Mon Sep 17 00:00:00 2001 From: Fabian Franz Date: Mon, 17 Dec 2018 19:26:23 +0100 Subject: [PATCH 2/2] www/nginx: style -> put bracket in a separate line --- .../mvc/app/library/OPNsense/Nginx/AccessLogParser.php | 3 ++- .../mvc/app/library/OPNsense/Nginx/StreamAccessLogParser.php | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/www/nginx/src/opnsense/mvc/app/library/OPNsense/Nginx/AccessLogParser.php b/www/nginx/src/opnsense/mvc/app/library/OPNsense/Nginx/AccessLogParser.php index 8bf5c6c03..b5ab6bb67 100644 --- a/www/nginx/src/opnsense/mvc/app/library/OPNsense/Nginx/AccessLogParser.php +++ b/www/nginx/src/opnsense/mvc/app/library/OPNsense/Nginx/AccessLogParser.php @@ -42,7 +42,8 @@ class AccessLogParser $this->parse_file(); } - private function parse_file() { + private function parse_file() + { $handle = @fopen($this->file_name, 'r'); if ($handle) { while (($buffer = fgets($handle)) !== false) { diff --git a/www/nginx/src/opnsense/mvc/app/library/OPNsense/Nginx/StreamAccessLogParser.php b/www/nginx/src/opnsense/mvc/app/library/OPNsense/Nginx/StreamAccessLogParser.php index d897c94cb..3e617f1ef 100644 --- a/www/nginx/src/opnsense/mvc/app/library/OPNsense/Nginx/StreamAccessLogParser.php +++ b/www/nginx/src/opnsense/mvc/app/library/OPNsense/Nginx/StreamAccessLogParser.php @@ -43,7 +43,8 @@ class StreamAccessLogParser $this->parse_file(); } - private function parse_file() { + private function parse_file() + { $handle = @fopen($this->file_name, 'r'); if ($handle) { while (($buffer = fgets($handle)) !== false) {