From 3c5ae7dc9f183bf91ee9613a553adb7007d42af6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Kellerer?= Date: Tue, 13 Aug 2019 22:38:38 +0200 Subject: [PATCH 1/9] AcmeClient: Prevent mode 0750 on everything inside /var/etc/acme-client This change avoids that all contents in any subfolder is set to mode 0750 as this is illegal e.g. for private keys for SSH. --- .../src/opnsense/scripts/OPNsense/AcmeClient/setup.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/setup.sh b/security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/setup.sh index 583e3f9d6..af6039d44 100755 --- a/security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/setup.sh +++ b/security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/setup.sh @@ -1,11 +1,17 @@ #!/bin/sh -ACME_DIRS="/var/etc/acme-client /var/etc/acme-client/certs /var/etc/acme-client/keys /var/etc/acme-client/configs /var/etc/acme-client/challenges /var/etc/acme-client/home" +ACME_BASE="/var/etc/acme-client" +ACME_DIRS="/var/etc/acme-client/certs /var/etc/acme-client/keys /var/etc/acme-client/configs /var/etc/acme-client/challenges /var/etc/acme-client/home" +# Generating dirs if missing and setting owner and mode (recursively) for directory in ${ACME_DIRS}; do mkdir -p ${directory} chown -R root:wheel ${directory} chmod -R 750 ${directory} done +# Setting owner and mode for base and immediate children (non recursive) +chown root:wheel ${ACME_BASE} ${ACME_BASE}/* +chmod 750 ${ACME_BASE} ${ACME_BASE}/* + exit 0 From 16e0baff72aca795ac36fc468db87b466f6c0303 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Kellerer?= Date: Tue, 13 Aug 2019 22:40:10 +0200 Subject: [PATCH 2/9] AcmeClient: Added automation feature: cert upload via sftp --- .../AcmeClient/Api/ActionsController.php | 51 + .../AcmeClient/forms/dialogAction.xml | 62 + .../models/OPNsense/AcmeClient/AcmeClient.xml | 53 + .../views/OPNsense/AcmeClient/actions.volt | 130 +- .../OPNsense/AcmeClient/certhelper.php | 3 + .../OPNsense/AcmeClient/upload_sftp.php | 1454 +++++++++++++++++ .../conf/actions.d/actions_acmeclient.conf | 18 + 7 files changed, 1770 insertions(+), 1 deletion(-) create mode 100644 security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/upload_sftp.php diff --git a/security/acme-client/src/opnsense/mvc/app/controllers/OPNsense/AcmeClient/Api/ActionsController.php b/security/acme-client/src/opnsense/mvc/app/controllers/OPNsense/AcmeClient/Api/ActionsController.php index 48682f8b3..a9be0c84e 100644 --- a/security/acme-client/src/opnsense/mvc/app/controllers/OPNsense/AcmeClient/Api/ActionsController.php +++ b/security/acme-client/src/opnsense/mvc/app/controllers/OPNsense/AcmeClient/Api/ActionsController.php @@ -31,6 +31,7 @@ namespace OPNsense\AcmeClient\Api; use \OPNsense\Base\ApiMutableModelControllerBase; use \OPNsense\Base\UIModelGrid; +use \OPNsense\Core\Backend; use \OPNsense\Core\Config; use \OPNsense\AcmeClient\AcmeClient; @@ -73,4 +74,54 @@ class ActionsController extends ApiMutableModelControllerBase { return $this->searchBase('actions.action', array('enabled', 'name', 'description'), 'name'); } + + public function sftpGetIdentityAction() + { + $result = ["status" => "unavailable"]; + + if ($response = $this->callBackend(["show-sftp-identity"], ["sftp_identity_type", "sftp_host"])) { + $result["status"] = "ok"; + $result["identity"] = $response; + } + + return $result; + } + + public function sftpTestConnectionAction() + { + if ($response = $this->callBackend( + ["test-sftp-connection"], + ["sftp_host", "sftp_host_key", "sftp_port", "sftp_user", "sftp_identity_type", "sftp_remote_path"])) { + + return $response; + } + + return ["status" => "unavailable"]; + } + + private function callBackend(array $command, array $arguments = []) + { + if ($this->request->isPost()) { + $backend = new Backend(); + + foreach ($arguments as $name) { + $command[] = $this->request->getPost($name); + } + + $command = array_map(function ($value) { + return escapeshellarg(empty($value = trim($value)) ? "__default_value" : $value); + }, $command); + + if ($result = trim($backend->configdRun("acmeclient " . join(" ", $command)))) { + if (preg_match('/^\[.+\]$/ms', $result) || preg_match('/^\{.+\}$/ms', $result)) { + try { + $result = json_decode($result, true, 64, JSON_THROW_ON_ERROR); + } catch (\Exception $ignored) {/*pass as is when json parsing fails*/} + } + return $result; + } + } + + return false; + } } diff --git a/security/acme-client/src/opnsense/mvc/app/controllers/OPNsense/AcmeClient/forms/dialogAction.xml b/security/acme-client/src/opnsense/mvc/app/controllers/OPNsense/AcmeClient/forms/dialogAction.xml index 54669fa74..c9c4f3b42 100644 --- a/security/acme-client/src/opnsense/mvc/app/controllers/OPNsense/AcmeClient/forms/dialogAction.xml +++ b/security/acme-client/src/opnsense/mvc/app/controllers/OPNsense/AcmeClient/forms/dialogAction.xml @@ -40,6 +40,68 @@ text Access token for Highwinds API. + + + header + + + + action.sftp_host + + text + IP address or hostname of the SFTP server. + + + action.sftp_port + + text + SFTP server port. Leave blank to use default "22". + + + action.sftp_host_key + + text + SFTP server host key, formatted as in 'known_hosts'. + Leave blank to auto accept host key on first connect (not as secure as specifying it). + + + action.sftp_user + + text + The username to login to the SFTP server. + + + action.sftp_identity_type + + dropdown + The type of identify to present to the SFTP server for authorization. Select 'none' to use default "ECDSA". + + + action.sftp_remote_path + + text + Path on the SFTP server to change to after login. + The path can be absolute or relative to home and must exist. + Leave blank to not change path after login. + + + action.sftp_chmod + + text + Unix permission to apply to uploaded public keys. Leave blank to use default "0440". + + + action.sftp_chmod_key + + text + Unix permission to apply to uploaded private keys. Leave blank to use default "0400". + + + action.sftp_chgrp + + text + Unix group id to apply to all uploaded files. Leave blank to not change the group. + header diff --git a/security/acme-client/src/opnsense/mvc/app/models/OPNsense/AcmeClient/AcmeClient.xml b/security/acme-client/src/opnsense/mvc/app/models/OPNsense/AcmeClient/AcmeClient.xml index e66b9d041..6879922c5 100644 --- a/security/acme-client/src/opnsense/mvc/app/models/OPNsense/AcmeClient/AcmeClient.xml +++ b/security/acme-client/src/opnsense/mvc/app/models/OPNsense/AcmeClient/AcmeClient.xml @@ -738,6 +738,7 @@ Restart HAProxy (OPNsense plugin) Restart Nginx (OPNsense plugin) Upload certificate to Highwinds CDN + Upload certificate via SFTP System or Plugin Command (select below) @@ -751,6 +752,58 @@ /^.{1,1024}$/u Should be a string between 1 and 1024 characters. + + N + /^.{1,255}$/u + Should be a string between 1 and 255 characters. + + + N + + /^.+?\s(?:[a-z0-9+\/]{4})*(?:[a-z0-9+\/]{2}==|[a-z0-9+\/]{3}=)?(?:\s.+?)?$/i + Should be a valid public SSH host key (see "known_hosts"). + + + N + 1 + 49151 + 22 + Should be a valid port number between 1 and 49151. + + + N + /^.{1,128}$/u + Should be a string between 1 and 128 characters. + + + N + + ECDSA + RSA + ed25519 + + + + N + /^.{1,512}$/u + Should be a string between 1 and 512 characters. + + + N + /^[0-9]+$/u + Should be a numeric value. + + + N + /^0[0-9]{3}$/u + A unix permission, 4 digits (e.g. 0440). + + + N + /^0[0-9]{3}$/u + A unix permission, 4 digits (e.g. 0400). + /^(?!.*(Let\'s\ Encrypt|acme|[fF]irmware))([\S\s]{1,255})/ diff --git a/security/acme-client/src/opnsense/mvc/app/views/OPNsense/AcmeClient/actions.volt b/security/acme-client/src/opnsense/mvc/app/views/OPNsense/AcmeClient/actions.volt index dcb794f7e..b6b3f8843 100644 --- a/security/acme-client/src/opnsense/mvc/app/views/OPNsense/AcmeClient/actions.volt +++ b/security/acme-client/src/opnsense/mvc/app/views/OPNsense/AcmeClient/actions.volt @@ -60,7 +60,135 @@ POSSIBILITY OF SUCH DAMAGE. $(".method_table_"+$(this).val()).show(); }); $("#action\\.type").change(); - }) + }); + + // Helpers for extra buttons and status divs + function makeButton(label, buttonGroup, buttonClass) { + var button = $(''); + button.addClass(buttonClass || "btn-primary"); + $('.fa-spinner', button).hide(); + $('.btn-text', button).html(label); + + var targetContainer = $("#DialogAction .modal-footer"), + targetId = "method_table_" + buttonGroup, + target = $("." + targetId, targetContainer); + + if (!target.is('span')) { + target = $('') + .addClass(targetId) + .prependTo(targetContainer) + .hide(); + } + + return button.appendTo(target); + } + + function makeStatusDiv(anchor, statusClass) { + return $('') + .appendTo($(anchor).closest("table").find("thead th[colspan=3]").first()) + .addClass(statusClass || 'alert-info') + .hide(); + } + + // SFTP - Identity show button + (function ($identityType) { + var identityDiv = makeStatusDiv($identityType); + + makeButton("{{ lang._('Show Identity') }}", "upload_sftp", "btn-info") + .click(function () { + identityDiv.hide(); + var button = $(this); + button.prop('disabled', true).find(".fa-spinner").show(); + + ajaxCall("/api/acmeclient/actions/sftpGetIdentity", getFormData("DialogAction").action, function (data, status) { + button.prop('disabled', false).find(".fa-spinner").hide(); + + if (status === "success" && data.status === "ok") + identityDiv.text(data.identity).show(); + else + identityDiv.text("{{ lang._('Failed loading identity') }}").show(); + }); + }); + + // Hide when input changes that influences the identity. + $identityType.change(function() { + identityDiv.hide(); + }); + })($('#action\\.sftp_identity_type')); + + // SFTP - Connection test button + (function ($user) { + var statusDiv = makeStatusDiv($user, 'alert-success').html( + '
' + + '
' + + '
'); + + statusDiv.find(".detail-enabler").click(function() { + $(".detail", statusDiv).show(); + $(this).hide(); + }); + + var errors = [ + {cond: ["connect_failed", "invalid_parameters"], msg: "{{ lang._('Host or username not specified.') }}"}, + {cond: ["connect_failed", "host_not_trusted"], msg: "{{ lang._('Host cannot be trusted.') }}"}, + {cond: ["connect_failed", "host_not_resolved"], msg: "{{ lang._('Failed to resolve hostname.') }}"}, + {cond: ["connect_failed", "connection_refused"], msg: "{{ lang._('Connection to host refused.') }}"}, + {cond: ["connect_failed", "network_timeout"], msg: "{{ lang._('Connection timed out.') }}"}, + {cond: ["connect_failed", "network_unreachable"], msg: "{{ lang._('Host not reachable.') }}"}, + {cond: ["connect_failed", "permission_denied"], msg: "{{ lang._('Host does not permit a connection for the specified user & identity.') }}"}, + {cond: ["connect_failed"], msg: "{{ lang._('Failed to connect to host.') }}"}, + {cond: ["change_home_dir_failed"], msg: "{{ lang._('Failed to change the remote path.') }}"}, + {cond: ["permission_denied"], msg: "{{ lang._('Uploads are not allowed to the specified remote path.') }}"}, + {msg: "{{ lang._('Test failed, see details.') }}"}, + ]; + + makeButton("{{ lang._('Test Connection') }}", "upload_sftp") + .click(function () { + statusDiv.hide(); + var button = $(this); + button.prop('disabled', true).find(".fa-spinner").show(); + + ajaxCall("/api/acmeclient/actions/sftpTestConnection", getFormData("DialogAction").action, function (data, status) { + button.prop('disabled', false).find(".fa-spinner").hide(); + + var message = "", detail = "", statusClass = "alert-warning"; + + if (status === "success") { + if (data.success === true) { + statusClass = "alert-success"; + message = "{{ lang._('Connection and upload test succeeded.') }}" + } else { + detail = JSON.stringify(data, null, ' '); + + for (var i = 0; i < errors.length; i++) { + var error = errors[i], + matching = (error.cond || []).filter(function (condition) { + return data[condition] === true; + }); + + if (matching.length === error.cond.length) { + message = error.msg; + break; + } + } + } + } else { + message = "{{ lang._('Test not possible. Failed to talk to firewall backend.') }}"; + } + + $(".message", statusDiv).html(message); + $(".detail", statusDiv).text(detail).hide(); + $(".detail-enabler", statusDiv).toggle(detail !== ""); + + statusDiv.removeClass("alert-success alert-warning").addClass(statusClass).show(); + }); + }); + })($('#action\\.sftp_user')); + + // Eagerly hiding method tables to avoid contents popping up when opening the dialog for the first time. + $(".method_table").hide(); }); diff --git a/security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/certhelper.php b/security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/certhelper.php index c561c3bcd..7d4266995 100755 --- a/security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/certhelper.php +++ b/security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/certhelper.php @@ -1290,6 +1290,9 @@ function run_restart_actions($certlist, $modelObj) case 'upload_highwinds': $response = $backend->configdRun("acmeclient upload_highwinds ${cert_id} ${action_id}"); break; + case 'upload_sftp': + $response = $backend->configdRun("acmeclient upload-sftp ${cert_id} ${action_id}"); + break; case 'configd': // Make sure a configd command was specified. if (empty((string)$action->configd)) { diff --git a/security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/upload_sftp.php b/security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/upload_sftp.php new file mode 100644 index 000000000..03691ca72 --- /dev/null +++ b/security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/upload_sftp.php @@ -0,0 +1,1454 @@ +#!/usr/local/bin/php + [ + "description" => "transfers certificates to the specified target host", + "options" => [ + "host::", "port::", "host-key::", "user::", "identity-type::", "remote-path::", + "certificates::", "files::", "chgrp::", "chmod::", "chmod-key::"], + "implementation" => "commandUpload", + "default" => true, + ], + + "test-connection" => [ + "description" => "connects to the host and returns results as JSON", + "options" => [ + "host:", "port::", "host-key::", "user:", "remote-path::", "identity-type::"], + "implementation" => "commandTestConnection", + ], + + "show-identity" => [ + "description" => "prints the ssh client identity (publickey)", + "options" => ["identity-type::", "source-ip::", "host::", "unrestricted"], + "implementation" => "commandShowIdentity", + ], +]; + +const STATIC_OPTIONS = << 4096, "ecdsa" => 521]; +const IDENTITY_TYPES = ["rsa", "rsa_2048", "rsa_4096", "rsa_8192", "ecdsa", "ecdsa_256", "ecdsa_384", "ecdsa_521", "ed25519"]; +const DEFAULT_IDENTITY_TYPE = "ecdsa"; + +const EXITCODE_SUCCESS = 0; +const EXITCODE_ERROR = 1; +const EXITCODE_ERROR_NO_PERMISSION = 2; +const EXITCODE_ERROR_NOTHING_TO_UPLOAD = 4; +const EXITCODE_ERROR_UNKNOWN_COMMAND = 255; + +// Optional imports +@include_once("config.inc"); +@include_once("certs.inc"); +@include_once("util.inc"); + +// -------------------------------------------------------------------------------------------------------------------- +// Main script logic + +function commandShowIdentity(array &$options): int +{ + $identity_type = trim(($options["identity-type"] ?: DEFAULT_IDENTITY_TYPE)); + $source_ip = trim(($options["source-ip"] ?: "")); + $host = trim(($options["host"] ?: "")); + + $keys = new SSHKeys(configPath()); + if (($id_file = $keys->getIdentity($identity_type)) && is_readable($id_file)) { + + if (!isset($options["unrestricted"]) + && ($restrictions = SSHKeys::getIdentityRestrictions($host, $source_ip))) { + echo "$restrictions "; + } + + echo file_get_contents($id_file); + return EXITCODE_SUCCESS; + + } else { + logger()->error("Failed getting identity. See log output for details."); + } + return EXITCODE_ERROR; +} + +function commandTestConnection(array &$options): int +{ + $result = ["actions" => ["connecting"]]; + + // Testing connection + $sftp = connectWithServer($options, $error); + if ($result["success"] = ($sftp !== null)) { + $result["actions"][] = "connected"; + $result["remote"] = [ + "address" => $sftp->remote_address, + "path" => $sftp->pwd(), + ]; + } else { + $result = array_merge($result, $error); + } + + // Testing file upload + if ($result["success"]) { + $result["actions"][] = "upload-testing"; + $file = temporaryFile(); + $filename = "." . basename($file); + + if ($error = $sftp->put($file, $filename)->lastError(3)) { + $result["success"] = false; + $result = array_merge($result, $error); + } else if ($error = $sftp->rm($filename)->lastError(3)) { + logger()->error("Failed removing upload test file '$filename'. Cause: " . json_encode($error, JSON_UNESCAPED_SLASHES)); + } + + if ($result["success"]) $result["actions"][] = "upload-tested"; + } + + echo json_encode($result, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT) . PHP_EOL; + + return $result["success"] ? EXITCODE_SUCCESS : EXITCODE_ERROR; +} + +function commandUpload(array &$options): int +{ + if (isset($options["certificates"])) { + // Includes host, upload all certs to the same host. + if (isset($options["host"])) { + return uploadCertificatesToHost($options); + + } else { + // Find the actions associated with the given certs. + $tasks = []; + $cert_ids = preg_split('/[,;\s]+/', $options["certificates"] ?: "", 0, PREG_SPLIT_NO_EMPTY); + foreach (findCertificates($cert_ids, false) as $id => $cert) { + foreach ($cert["automations"] as $action_id) { + if (!isset($tasks[$action_id])) + $tasks[$action_id] = []; + $tasks[$action_id][] = $id; + } + } + + $result = 0; + foreach ($tasks as $action_id => $cert_list) { + if (!empty($cert_list) && ($task_options = getOptionsById($action_id, true))) { + $task_options = array_merge($options, $task_options, ["certificates" => join(",", $cert_list)]); + $result = uploadCertificatesToHost($task_options); + if ($result != EXITCODE_SUCCESS) + break; + } + } + + return $result; + } + + } else if (isset($options["host"])) { + return uploadCertificatesToHost($options); + + } else { + logger()->error("No work to do, neither --host nor --certificates is present."); + return EXITCODE_ERROR_NOTHING_TO_UPLOAD; + } +} + +function uploadCertificatesToHost(array $options): int +{ + $sftp = connectWithServer($options, $error); + if ($sftp === null) { + logger()->error("Aborting after connect failure."); + return $error["connect_failed"] ? EXITCODE_ERROR : EXITCODE_ERROR_NO_PERMISSION; + } + + $username = $options["user"]; + + $remote_home = $sftp->pwd(); + $remote_files = []; + $remote_path = "."; + + $chmod = isset($options["chmod"]) ? ($options["chmod"] ?: DEFAULT_CERT_MODE) : false; + $chmod_key = isset($options["chmod-key"]) ? ($options["chmod-key"] ?: DEFAULT_KEY_MODE) : false; + $chgrp = $options["chgrp"] ?: false; + + // Collecting files to upload (sorted by target to reduce remote directory changes) + $files_to_upload = getFilesToUpload($options); + usort($files_to_upload, function (&$a, &$b) { + return $a["target"] <=> $b["target"]; + }); + + // Uploading the files + foreach ($files_to_upload as $file) { + // Checking if source is valid. + if (!is_file($file["source"]) && is_readable($file["source"])) { + logger()->error("Skipping {$file["source"]}, it is not a file or not readable."); + continue; + } + + // Changing remote directory if required. + if (($dir = dirname($file["target"])) !== $remote_path) { + $target_dir = $sftp->resolve($dir, $remote_home); + + $error = $sftp->cd($target_dir)->lastError(); + if ($error["file_not_found"]) { + logger()->info("Creating remote directory: $target_dir"); + $sftp->clearError() + ->mkdir($target_dir) + ->cd($target_dir); + } + + if ($error = $sftp->lastError()) { + logger()->error("Failed to cd into '$dir'. Cause: " . json_encode($error, JSON_UNESCAPED_SLASHES)); + return EXITCODE_ERROR; + } else { + $remote_path = $dir; + $remote_files = $sftp->ls(); + if ($error = $sftp->lastError()) { + logger()->error("Failed listing remote files. Cause: " . json_encode($error, JSON_UNESCAPED_SLASHES)); + return EXITCODE_ERROR; + } + } + } + + // Preparing copy + $remote_filename = basename($file["target"]); + $remote_file = $remote_files[$remote_filename] ?: ["type" => "-", "owner" => $username]; + $remote_is_file = $remote_file["type"] === "-"; + $remote_is_readonly = preg_match('/^-r-.r-.+$/', $remote_file["permissions"] ?: ""); + + // Check if a folder/socket/symlink, etc is in the way + if (!$remote_is_file) { + logger()->error("Skipping file '{$file["source"]}' as there is a non-file in the way at '{$file["target"]}'."); + continue; + } + + $file_chmod = $file["is_key"] ? $chmod_key : $chmod; + $file_chmod = preg_match('/^0[0-9]{3}$/', $file_chmod) ? (string)$file_chmod : false; + $permission_change_retry_allowed = $file_chmod + && $remote_file["owner"] === $username + && isset($remote_files[$remote_filename]); + + // Initial upload when permissions are properly set. + if (!$remote_is_readonly) { + if ($error = $sftp->put($file["source"], $remote_filename)->lastError()) { + logger()->error("Failed uploading file '{$file["source"]}' to '{$file["target"]}'. Cause: " . json_encode($error, JSON_UNESCAPED_SLASHES)); + + if ($error["permission_denied"] !== true) + $permission_change_retry_allowed = false; + + if ($permission_change_retry_allowed) + logger()->info("Retrying file '{$file["source"]}' to '{$file["target"]}' with adjusted permissions."); + } else { + $permission_change_retry_allowed = false; + } + } + + // Second upload when initial failed or was skipped due to write protection (only possible if we have chmod defined to reset permissions later). + if ($permission_change_retry_allowed) { + $sftp->chmod($remote_filename, '0600'); + if ($error = $sftp->put($file["source"], $remote_filename)->lastError()) { + logger()->error("Failed uploading file '{$file["source"]}' to '{$file["target"]}'. Cause: " . json_encode($error, JSON_UNESCAPED_SLASHES)); + return EXITCODE_ERROR_NO_PERMISSION; + } + } else if ($remote_is_readonly) { + logger()->error("Failed uploading file '{$file["source"]}' to '{$file["target"]}'. Existing file is write protected and --chmod[-key] options are missing."); + return EXITCODE_ERROR_NO_PERMISSION; + } + + // Applying Chmod / chgrp if requested. + if ($file_chmod) { + if ($error = $sftp->chmod($remote_filename, $file_chmod)->lastError()) + logger()->error("Failed chmod ($file_chmod) for '{$file["target"]}'. Cause: " . json_encode($error, JSON_UNESCAPED_SLASHES)); + } + + if (!empty($chgrp)) { + if ($error = $sftp->chgrp($remote_filename, $chgrp)->lastError()) + logger()->error("Failed chgrp ($chgrp) for '{$file["target"]}'. Cause: " . json_encode($error, JSON_UNESCAPED_SLASHES)); + } + + $sftp->clearError(); + } + + return EXITCODE_SUCCESS; +} + +function connectWithServer(array $options, &$error): ?SftpClient +{ + $identity_type = trim(($options["identity-type"] ?: DEFAULT_IDENTITY_TYPE)); + $host = trim(($options["host"] ?: "")); + $host_key = ($options["host-key"] ?: ""); + $port = $options["port"] ?: 22; + $username = $options["user"]; + + $sftp = new SftpClient(configPath(), $identity_type); + + if (!$sftp->connect($host, $username, $host_key, $port)) { + $error = $sftp->lastError(); + $error["connect_failed"] = true; + return null; + } + + // Apply start path (if one was specified, defaults to home dir) + if (($remote_path = $options["remote-path"])) { + if ($err = $sftp->cd($remote_path)->lastError()) { + $error = $err; + $error["change_home_dir_failed"] = true; + logger()->error("Failed cd into '{$remote_path}'. Cause: " . json_encode($err, JSON_UNESCAPED_SLASHES)); + return null; + } + } + + return $sftp; +} + +function help() +{ + echo ABOUT . PHP_EOL + . "Usage: " . basename($GLOBALS["argv"][0]) . " [options] [--command=]COMMAND" . PHP_EOL + . PHP_EOL . STATIC_OPTIONS . PHP_EOL; + + foreach (COMMANDS as $name => $cmd) { + echo PHP_EOL . "COMMAND \"$name\" {$cmd["description"]}" . PHP_EOL . "Options:" . PHP_EOL; + foreach ($cmd["options"] as $option) { + $option = preg_replace(['/^([^:]+)$/', '/(.+)::$/', '/(.+):$/'], ['[$1]', '[$1=value]', '$1=value'], "--$option"); + echo " $option" . PHP_EOL; + } + } + + echo PHP_EOL . "Examples:" . PHP_EOL + . str_replace('/\r\n|\n|\r/g', PHP_EOL, EXAMPLES) + . PHP_EOL . PHP_EOL; +} + +function getCommand() +{ + $default = null; + $command = null; + $parsed_args = getopt("", ["command::"]); + foreach (COMMANDS as $name => $cmd) { + if (in_array($name, $GLOBALS["argv"]) || $parsed_args["command"] === $name) + $command = $cmd; + if ($cmd["default"] === true) + $default = $cmd; + } + + return $command ?: $default; +} + +function getActionById($automation_id) +{ + $config = OPNsense\Core\Config::getInstance()->object(); + $client = $config->OPNsense->AcmeClient; + + foreach ($client->actions->children() as $action) { + if ($automation_id === (string)$action->attributes()["uuid"] + || $automation_id === (string)$action->id) + return $action; + } + + return null; +} + +function getOptionsById($automation_id, $silent = false) +{ + if (!$silent) logger()->info("Reading options from automation: $automation_id"); + + if (is_object($action = getActionById($automation_id))) { + if ($action->enabled && "upload_sftp" === (string)$action->type) { + return [ + "host" => trim((string)$action->sftp_host), + "host-key" => trim((string)$action->sftp_host_key), + "port" => trim((string)$action->sftp_port), + "identity-type" => trim((string)$action->sftp_identity_type), + "user" => trim((string)$action->sftp_user), + "remote-path" => trim((string)$action->sftp_remote_path), + "chgrp" => trim((string)$action->sftp_chgrp), + "chmod" => trim((string)$action->sftp_chmod), + "chmod-key" => trim((string)$action->sftp_chmod_key), + ]; + } else if (!$silent) { + logger()->error("Ignoring disabled or invalid automation '$automation_id'"); + } + } else { + logger()->error("No upload automation found with uuid = '$automation_id'"); + } + + return false; +} + +function getFilesToUpload(array $options) +{ + $files = []; + $cert_ids = preg_split('/[,;\s]+/', $options["certificates"] ?: "", 0, PREG_SPLIT_NO_EMPTY); + + if (class_exists("OPNsense\\Core\\Config")) { + foreach (findCertificates($cert_ids) as $cert) { + if (isset($cert["content"])) { + foreach ($cert["content"] as $name => $content) { + $source = temporaryFile(); + $ok = file_put_contents($source, $content); + + if (!$ok) { + logger()->error("Ignoring upload for cert '{$cert["name"]}', since the content cannot be prepared or is empty."); + continue; + } + + if (($time = intval($cert["updated"])) && $time > 0) + touch($source, $time); + + // Sanitize user input (allow unicode chars, numbers and some special characters) + $context_path = preg_replace('/[^\w\d_\-@.]+/uim', "-", $cert["name"]); + + $file = ["source" => $source, "target" => "{$context_path}/{$name}.pem"]; + $file["is_key"] = ($name === "key"); + $files[] = $file; + } + } else { + logger()->error("Ignoring upload for cert '{$cert["name"]}', since it is not available in trust storage."); + } + } + } else { + // Allow to specify "--files" only if we're not running in opnsense environment (for development only). + if (isset($options["files"])) { + foreach (preg_split('/[,;\s]+/', $options["files"] ?: "", 0, PREG_SPLIT_NO_EMPTY) as $file) { + $files[] = ["source" => $file, "target" => "upload-test/" . basename($file)]; + }; + } + } + + if (empty($files)) + logger()->error("Didn't find any certificates to upload (cert-ids: " . (empty($cert_ids) ? "all" : join(", ", $cert_ids)) . ")."); + + return $files; +} + +function findCertificates(array $certificate_ids_or_names, $load_content = true): array +{ + if (!class_exists("OPNsense\\Core\\Config")) return []; + + $config = OPNsense\Core\Config::getInstance()->object(); + $client = $config->OPNsense->AcmeClient; + + $result = []; + $refids = []; + + foreach ($client->certificates->children() as $cert) { + $item = []; + $id = (string)$cert->id; + $name = (string)$cert->name; + + if (empty($certificate_ids_or_names) + || in_array($id, $certificate_ids_or_names) + || in_array($name, $certificate_ids_or_names)) { + + if ($cert->enabled == 0) { + if (!empty($certificate_ids_or_names)) + logger()->error("Certificate '{$name}' (id: $id) is disabled, skipping it."); + + continue; + } + + $item["name"] = $name; + $item["updated"] = intval($cert->lastUpdate); + $item["automations"] = preg_split('/[\s*,]+/', $cert->restartActions); + if (isset($cert->certRefId)) { + $refids[] = $item['content_id'] = (string)$cert->certRefId; + } + + $result[$id] = $item; + } + } + + if ($load_content && ($certificates = exportCertificates($refids))) { + foreach ($result as &$cert_info) { + $id = $cert_info["content_id"]; + if (isset($certificates[$id])) + $cert_info["content"] = $certificates[$id]; + } + } + + return $result; +} + +function exportCertificates(array $cert_refids) +{ + $result = []; + $config = OPNsense\Core\Config::getInstance()->object(); + foreach ($config->cert as $cert) { + $refid = (string)$cert->refid; + $item = []; + if (in_array($refid, $cert_refids)) { + $item["cert"] = str_replace(["\n\n", "\r"], ["\n", ""], base64_decode($cert->crt)); + $item["key"] = str_replace(["\n\n", "\r"], ["\n", ""], base64_decode($cert->prv)); + // check if a CA is linked + if (!empty((string)$cert->caref)) { + $cert = (array)$cert; + $item["ca"] = ca_chain($cert); + } + $result[$refid] = $item; + } + } + + return $result; +} + +function configPath(): string +{ + static $paths = [ + '/var/etc/acme-client', + __DIR__ + ]; + foreach ($paths as $path) { + if (is_dir($path)) return $path . DIRECTORY_SEPARATOR . 'sftp-config'; + } + die("Failed detecting config path"); +} + +function main() +{ + global $argv; + $command = getCommand(); + $options = ["help", "log", "no-error"]; + + $has_automation_id = preg_match('/--automation-id=\S+/', join(" ", $argv)); + if ($has_automation_id) { + $options = array_merge($options, ["automation-id:", "certificates::"]); + } else { + $options = array_merge($options, $command["options"]); + } + + $index = 0; + if ($options = getopt("h", $options, $index)) { + if (isset($options["h"]) || isset($options["help"])) { + help(); + } else { + if (isset($options["log"])) + logger(true)->info("Logging to stdout enabled"); + + register_shutdown_function(function () { + temporaryFile(true); + }); + + $options = array_filter($options, function ($value) { + return !is_string($value) + || (!empty($value = trim($value)) && $value !== "__default_value"); + }); + + if (isset($options["automation-id"])) + $options = array_merge($options, getOptionsById($options["automation-id"])); + + if (is_callable($runner = $command["implementation"])) { + $code = $runner($options); + if ($code != EXITCODE_SUCCESS) { + logger()->error("Command execution failed, exit code $code. Last input was: " . json_encode($options, JSON_UNESCAPED_SLASHES)); + } + exit(isset($options["no-error"]) ? EXITCODE_SUCCESS : $code); + } else { + exit(EXITCODE_ERROR_UNKNOWN_COMMAND); + } + } + } else { + if (count($argv) < 2) { + help(); + } else { + $cmd = join(" ", $argv); + logger()->error("Parsing of '$cmd' failed at argument '{$argv[$index]}'"); + } + exit(1); + } +} + + +// -------------------------------------------------------------------------------------------------------------------- +// Utility functions + +interface ILogger +{ + function info($message); + + function error($message); +} + +function &logger($reconfigure_to_stdout = false): ILogger +{ + static $logger; + if (!$logger || $reconfigure_to_stdout) { + if (!$reconfigure_to_stdout && function_exists("log_error")) { + $logger = new class implements ILogger + { + function info($message) { syslog(SYSLOG_INFO_LEVEL, basename(__FILE__) . ": INFO: $message"); } + + function error($message) { log_error($message); } + }; + } else { + $logger = new class implements ILogger + { + function info($message) { echo "INFO: {$message}" . PHP_EOL; } + + function error($message) { echo "ERROR: {$message}" . PHP_EOL; } + }; + } + } + return $logger; +} + +function requireThat($expression, $message) +{ + if (!$expression) { + logger()->error("FATAL: $message"); + exit(EXITCODE_ERROR); + } +} + +function resolvePath($file, $base = ".") +{ + if (!$base || $base[0] != DIRECTORY_SEPARATOR) + $base = realpath(($base ?: ".")); + + $path = []; + $combined_path = ((!empty($file) && $file[0] == DIRECTORY_SEPARATOR) ? $file : $base . DIRECTORY_SEPARATOR . $file); + foreach (explode(DIRECTORY_SEPARATOR, $combined_path) as $part) { + if (empty($part) || $part === '.') + continue; + if ($part !== '..') + array_push($path, $part); + else if (!empty($path)) + array_pop($path); + else + return false; + } + + return DIRECTORY_SEPARATOR . join(DIRECTORY_SEPARATOR, $path); +} + +function temporaryFile($delete_all = false) +{ + static $__temporary_files = []; + if ($delete_all) { + foreach ($__temporary_files as $file) + unlink($file); + $__temporary_files = []; + } else { + if ($file = tempnam(sys_get_temp_dir(), "sftp-upload-")) { + $file = realpath($file); + $__temporary_files[] = $file; + requireThat(chmod($file, 0600), "failed setting user-only permissions on '$file'."); + return $file; + }; + } + return false; +} + + +// -------------------------------------------------------------------------------------------------------------------- +// Classes + + +/** + * Wrapper around the 'sftp' commandline client. + */ +class SftpClient +{ + public $remote_address; + private $identity_type; + + /* @var false|array */ + private $failed_status; + /* @var SSHKeys */ + private $ssh_keys; + /* @var null|Process */ + private $process = null; + + public function __construct($config_path, $identity_type = DEFAULT_IDENTITY_TYPE) + { + $this->ssh_keys = new SSHKeys($config_path); + $this->identity_type = $identity_type; + } + + public function __destruct() + { + $this->close(); + } + + public function connect($host, $username, $host_key = "", $port = 22) + { + if (empty(trim($host)) || empty(trim($username))) { + $this->failed_status = ["invalid_parameters" => true]; + logger()->error("Failed connecting to '$host'. Hostname or username is missing."); + return false; + } + + $trust = $this->ssh_keys->trustHost($host, $host_key, $port); + if ($trust["ok"] !== true) { + logger()->error("Failed establishing trust in '$host'; Cause: {$trust["error"]}"); + unset($trust["ok"]); + $this->failed_status = array_merge($trust, ["host_not_trusted" => true]); + return false; + } else { + $host = $trust["host"]; + } + + // Building sftp command. + $cmd = [ + "sftp", + "-P", $port, + "-oUser=$username", + "-oUserKnownHostsFile={$this->ssh_keys->knownHostsFile()}", + ]; + + // Handle client side identity + $identity = $this->ssh_keys->getIdentity($this->identity_type, true); + if (is_file($identity) && is_readable($identity)) { + array_push($cmd, + "-i", $identity, + "-oPreferredAuthentications=publickey"); + } else { + logger()->error("Failed adding client identity ($identity). Connect will likely fail."); + } + + // Adding the host + array_push($cmd, "$host"); + + // Creating the sftp process + if ($this->process = Process::open($cmd)) { + $this->processAvailableInput(120, 1); + if ($error = $this->lastError()) { + logger()->error("Failed connecting to '$host' (user: '$username'). Cause: " . json_encode($error)); + return false; + } + $this->remote_address = strpos($host, ':') ? "[$host]:$port" : "$host:$port"; + return true; + } + return false; + } + + private function processAvailableInput(float $timeout = 0, $expected_lines = 0, Callable $lines_consumer = null) + { + requireThat($this->process !== null, "SFTP: process not connected"); + + static $expected_errors = [ + ["host_not_resolved", /* => */ '/.*not resolve.*/i'], + ["host_not_trusted", /* => */ '/.*IDENTIFICATION HAS CHANGED.*/i'], + ["connection_refused", /* => */ '/.*connection refused.*/i'], + ["connection_closed", /* => */ '/.*connection closed.*/i'], + ["network_timeout", /* => */ '/.*timed out.*/i'], + ["network_unreachable", /* => */ '/.*network.+unreachable.*/i'], + ["permission_denied", /* => */ '/.*permission denied.*/i'], + ["file_not_found", /* => */ '/.*(no such|not found).*/i'], + ["failure", /* => */ '/.*(error|failure|you must supply).*/i'], + ]; + + while (($line = $this->process->get($timeout)) !== false) { + if (--$expected_lines <= 0) + $timeout = 0; + + foreach ($expected_errors as $ee) { + if (preg_match($ee[1], $line)) { + $this->failed_status = [$ee[0] => true, "error" => trim($line)]; + break; + } + } + + $hide = ($lines_consumer && $lines_consumer($line) === true); + if (!$hide) + logger()->info("SFTP: " . rtrim($line)); + } + } + + public function close() + { + if (($p = $this->process) !== null) { + $p->put("exit"); + $p->closeInput(); + + $this->processAvailableInput(1.5); + $p->close(); + + $this->process = null; + + if ($this->failed_status && $this->failed_status["connection_closed"]) + $this->clearError(); + } + } + + public function lastError($timeout = 0.5) + { + if ($this->failed_status === false) + $this->processAvailableInput($timeout); + return $this->failed_status; + } + + public function clearError() + { + $this->failed_status = false; + return $this; + } + + public function ls() + { + $files = []; + $this->processAvailableInput(); + $this->process->put("ls -la"); + + $regex = '/^([bcdlsp\-][rwx\-]{9}[+@]?)\s+[0-9]+\s+([^\s]+)\s+([^\s]+)\s+([0-9]+)\s+(\w+\s+[0-9]+\s+[0-9:]+)\s+(.+)$/'; + $this->processAvailableInput(30, 2, function ($line) use (&$files, $regex) { + if (preg_match($regex, $line, $matches)) { + $files[trim($matches[6])] = [ + "type" => $matches[1][0], + "permissions" => $matches[1], + "owner" => $matches[2], + "group" => $matches[3], + "size" => intval($matches[4]), + "mtime" => strtotime($matches[5]) + ]; + return true; + } + return false; + }); + + return $files; + } + + public function pwd() + { + $remote_path = false; + $this->processAvailableInput(); + $this->process->put("pwd"); + $this->processAvailableInput(30, 2, function ($line) use (&$remote_path) { + if (preg_match('/^.+directory:\s(.+)$/i', $line, $matches)) + $remote_path = trim($matches[1]); + }); + return $remote_path; + } + + public function resolve($remote_path, $remote_pwd = null) + { + if (($pwd = ($remote_pwd ?: $this->pwd())) !== false) { + $remote_path = resolvePath($remote_path, str_replace("/", DIRECTORY_SEPARATOR, $pwd)); + $remote_path = str_replace("\\", "/", $remote_path); + return $remote_path; + } + return false; + } + + public function get($remote_file, $local_file = "") + { + $this->processAvailableInput(); + $this->process->put("get " . escapeshellarg($remote_file) . " " . (empty($local_file) ? "" : escapeshellarg($local_file))); + $this->processAvailableInput(30, 2); + return $this; + } + + public function put($local_file, $remote_file = "") + { + if (is_file($local_file)) { + $this->processAvailableInput(); + $this->process->put("put -p " . escapeshellarg($local_file) . " " . (empty($remote_file) ? "" : escapeshellarg($remote_file))); + $this->processAvailableInput(30, 2); + } else { + logger()->info("put: File $local_file doesn't exist."); + $this->failed_status = ["file_not_found" => true, "error" => $local_file]; + } + + return $this; + } + + public function mkdir($remote_path) + { + if (($remote_path = $this->resolve($remote_path)) !== false) { + $this->process->put("mkdir " . escapeshellarg($remote_path)); + $this->processAvailableInput(30, 1); + } + return $this; + } + + public function cd($remote_path) + { + if (($remote_path = $this->resolve($remote_path)) !== false) { + $this->clearError(); + $this->process->put("cd " . escapeshellarg($remote_path)); + + $this->processAvailableInput(30, 1); + $error = $this->lastError(); + $pwd = false; + + if ($error || $remote_path !== ($pwd = $this->pwd())) { + $this->failed_status = array_merge(($error ?: []), [ + "failure" => true, + "error" => "Failed changing path to '$remote_path' (pwd: '$pwd'); Cause: {$error["error"]}" + ]); + } + } + return $this; + } + + public function chmod($remote_file, $mode) + { + $this->processAvailableInput(); + $this->process->put("chmod " . escapeshellarg($mode) . " " . escapeshellarg($remote_file)); + $this->processAvailableInput(30, 2); + return $this; + } + + public function chgrp($remote_file, $group_id) + { + $this->processAvailableInput(); + $this->process->put("chgrp " . escapeshellarg($group_id) . " " . escapeshellarg($remote_file)); + $this->processAvailableInput(30, 2); + return $this; + } + + public function rm($remote_file) + { + $this->processAvailableInput(); + $this->process->put("rm " . escapeshellarg($remote_file)); + $this->processAvailableInput(30, 2); + return $this; + } +} + +/** + * Utility class for managing SSH host (known_hosts) and identity keys to be used in {@see SftpClient}. + */ +class SSHKeys +{ + private $config_path; + private $known_hosts_file; + + public function __construct($config_path) + { + if (!is_dir($config_path)) { + $dir_created = mkdir($config_path, CONFIG_PATH_CREATE_MODE, true); + requireThat($dir_created, "Failed creating directory '$config_path' with permission " . CONFIG_PATH_CREATE_MODE); + } + + $this->config_path = realpath($config_path); + $this->known_hosts_file = resolvePath("known_hosts", $this->config_path); + } + + public function knownHostsFile() + { + if (!is_file($this->known_hosts_file)) { + $file_created = + touch($this->known_hosts_file) + && chmod($this->known_hosts_file, KNOWN_HOSTS_FILE_CREATE_MODE); + + requireThat($file_created, "Failed creating file '{$this->known_hosts_file}' with permission " . KNOWN_HOSTS_FILE_CREATE_MODE); + } + + return $this->known_hosts_file; + } + + public function trustHost($host, $host_key = "", $port = 22, $no_modification_allowed = false): array + { + requireThat(!empty(trim($host)), "Hostname must not be empty."); + + // Convert the specified host_key to a data structure that can be compared + if (empty($host_key = trim($host_key))) { + $host_key = false; + } else { + $host_key = self::getHostKeyInfo($host_key); + if ($host_key === false) + return ["ok" => false, "error" => "Invalid host_key specified."]; + } + + + // Check our current known_host file + $addKeyInfo = function (array &$key_list) { + foreach ($key_list as &$item) { + $item["key_info"] = self::getHostKeyInfo($item["host_key"]); + } + return array_filter($key_list, function (&$item) { + return $item["key_info"] !== false; + }); + }; + + $known_keys = $addKeyInfo($this->getKnownHostKey($host)); + + // Find known_hosts item with same hostname + $known_by_host = array_reduce($known_keys, function ($found, $key) use ($host) { + return (!$found && !empty(trim($key["host"])) && strcasecmp(trim($host), trim($key["host"])) == 0) + ? $key + : $found; + }, false); + + // Find known_hosts item with same public host-key + $known_by_key = array_reduce($known_keys, function ($found, $key) use ($host_key) { + return (!$found && $host_key && $host_key === $key["key_info"]) + ? $key + : $found; + }, false); + + + // Updating $host and $host_key from known_hosts and check if we need to update known_hosts. + if ($host_key === false && $known_by_host) { + logger()->info("No host key specified, using existing known_hosts entry for '$host'"); + $host_key = $known_by_host["key_info"]; + } + + $is_key_known = false; + if ($known_by_host && $host_key && $host_key === $known_by_host["key_info"]) { + $is_key_known = true; + } else if ($known_by_key) { + if (strcasecmp(trim($host), trim($known_by_key["host"])) != 0) { + logger()->info("Host key is in known_hosts but hostname differs. Changing '$host' to '{$known_by_key["host"]}'."); + $host = $known_by_key["host"]; + } + $is_key_known = true; + } + + + // Check if we don't have a matching known_hosts entry and add or update it as required. + if (!$is_key_known && !$no_modification_allowed) { + $key_type = $host_key ? $host_key["key_type"] : DEFAULT_KEY_TYPE; + + $remote_host_keys = $addKeyInfo($this->queryHostKey($host, $key_type, $port)); + $matching_remote_host_keys = array_filter($remote_host_keys, function ($key) use ($host_key) { + return $key["key_info"] !== false && (!$host_key || $host_key === $key["key_info"]); + }); + + if (!empty($matching_remote_host_keys)) { + if ($known_by_host) { + logger()->info("Removing known_hosts entry with differing key for '{$known_by_host["host"]}' as it is in the way."); + $this->removeKnownHost($known_by_host["host"]); + } + + foreach ($matching_remote_host_keys as $key) { + logger()->info("Adding known_hosts entry: " . json_encode($key["key_info"], JSON_UNESCAPED_SLASHES)); + $ok = file_put_contents($this->knownHostsFile(), $key["host_key"] . PHP_EOL, FILE_APPEND); + if (!$ok) + logger()->error("Failed adding known_hosts entry {$key["host_key"]}"); + } + + // Verify that known_hosts contains the correct keys after adding them (using recursion). + return $this->trustHost($host, $matching_remote_host_keys[0]["host_key"], $port, true); + + } else { + if (empty($remote_host_keys)) { + $msg = "No connection to '$host'; Failed querying host key from server."; + } else { + $msg = "Key mismatch for '$host'; " + . "The expected key (" . json_encode($host_key) . ") was not found in (" . json_encode($remote_host_keys) . ")"; + } + return ["ok" => false, "error" => $msg]; + } + } + + + if ($is_key_known) { + return ["ok" => true, "host" => $host, "key_info" => $host_key]; + } else { + return ["ok" => false, "error" => "Host unknown and remote key cannot be queried."]; + } + } + + public static function getHostSearchList($host) + { + $search_list = [($host = strtolower($host))]; + + // Add IP-address to search list (IPv4 only) + $has_ip = ($ip = gethostbyname($host)) + && ($ip !== $host || preg_match('/^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$/', $ip)); + + if ($has_ip) + $search_list[] = strtolower($ip); + + // Add FQDN to search list if reverse lookup provides a valid one. + $has_fqdn = $has_ip + && ($reverse_fqdn = gethostbyaddr($ip)) + && $reverse_fqdn !== $ip + && gethostbyname($reverse_fqdn) === $ip; + + if ($has_fqdn && isset($reverse_fqdn)) + $search_list[] = strtolower($reverse_fqdn); + + // Build unique search list (dedup list) + $search_list = array_filter($search_list, function ($value, $index) use (&$search_list) { + return !empty(trim($value)) && array_search($value, $search_list) == $index; + }, ARRAY_FILTER_USE_BOTH); + + return $search_list; + } + + public static function queryHostKey($host, $key_type = DEFAULT_KEY_TYPE, $port = 22) + { + $keys = []; + $failed = false; + $names = join(",", self::getHostSearchList($host)); + + if (!empty($names) && ($p = Process::open(["ssh-keyscan", "-p", $port, "-t", $key_type, $names]))) { + $lines = []; + while (($line = $p->get(60)) !== false) { + $line = trim($line); + if (empty($line) || $line[0] == "#") + continue; + + if (preg_match('/.*(connect|write|broken|no route|not known).*/i', $line)) + $failed = true; + + $lines[] = $line; + } + + if ($p->close() == 0 && !$failed) { + foreach ($lines as $line) { + $keys[] = ["host_key" => $line]; + } + } else { + logger()->error("Failed querying public keys for [$names] ($host / $key_type). " + . "Exit code: {$p->exitCode} (failed flag: $failed) ; " . PHP_EOL + . "ssh-keyscan: " . join(PHP_EOL . "ssh-keyscan: ", $lines)); + } + } + + if (empty($keys)) + logger()->info("Couldn't fetch public host key ($key_type) from $host"); + + return $keys; + } + + + public function getKnownHostKey($host) + { + $keys = []; + foreach (self::getHostSearchList($host) as $name_or_ip) { + if ($p = Process::open(["ssh-keygen", "-F", $name_or_ip, "-f", $this->knownHostsFile()])) { + $lines = []; + while (($line = $p->get()) !== false) { + $line = trim($line); + if (empty($line) || $line[0] == "#") + continue; + + $lines[] = $line; + } + + if ($p->close() == 0) { + $keys[] = ["host" => $name_or_ip, "host_key" => $lines[0]]; + } else if ($p->exitCode != 1 /* 1 == NOT_FOUND */) { + logger()->error("Failed querying known hosts for $name_or_ip ($host). Return code was: {$p->exitCode}" + . " ; " . PHP_EOL . join(PHP_EOL, $lines)); + } + } + } + + if (empty($keys)) + logger()->info("Didn't find $host in known_hosts"); + + return $keys; + } + + public function removeKnownHost($host) + { + $ok = false; + if ($p = Process::open(["ssh-keygen", "-R", $host, "-f", $this->knownHostsFile()])) { + $ok = $p->close() === 0; + if (!$ok) + logger()->error("Failed removing known hosts for $host. Return code was: {$p->exitCode}"); + } + return $ok; + } + + public static function getHostKeyInfo($host_key) + { + if ($p = Process::open(["ssh-keygen", "-l", "-f", "-"])) { + $p->put($host_key); + $p->closeInput(); + + if (($hash = $p->get()) && preg_match('/^([0-9]+) (.+?) .+? \(([^()]+)\)$/', $hash, $matches)) { + return ["hash" => $matches[2], "key_type" => $matches[3], "key_length" => $matches[1]]; + } else { + logger()->error("Unsupported hash type: $hash"); + } + } + + logger()->error("Failed getting hash for host_key"); + return false; + } + + // Returns the path to the public identity key file, generating it if missing (optionally returns the private key path) + public function getIdentity($identity_type = DEFAULT_IDENTITY_TYPE, $private = false) + { + requireThat(in_array($identity_type, IDENTITY_TYPES), "Identity type $identity_type unknown."); + + list($key_type, $key_size) = explode('_', $identity_type, 2); + if (!$key_size && DEFAULT_IDENTITY_KEY_BITS[$key_type] > 0) + $key_size = DEFAULT_IDENTITY_KEY_BITS[$key_type]; + + $identity_path = "{$this->config_path}/id.{$identity_type}"; + + if (!file_exists($identity_path)) { + $generate_key = [ + "ssh-keygen", "-v", + "-f", $identity_path, + "-t", $key_type, + "-N", "", + ]; + + if (intval($key_size) > 0) + array_push($generate_key, "-b", $key_size); + + if ($p = Process::open($generate_key)) { + while (($line = $p->get(10)) !== false) { + logger()->info("SSH keygen: $line"); + } + + requireThat($p->close() == 0, + "Failed generating identity $identity_path: Error code: {$p->exitCode}" . PHP_EOL + . "Command: " . join(" ", $generate_key)); + } + } + + return $private ? $identity_path : "{$identity_path}.pub"; + } + + public static function getIdentityRestrictions($host = "", $source_ip = "") + { + $restrictions = ['restrict', 'command="internal-sftp"']; + + $restrict_ip = empty(trim($source_ip)) + ? (empty(trim($host)) ? false : self::getOutgoingIpFor($host)) + : $source_ip; + + if ($restrict_ip) + $restrictions[] = 'from="' . $restrict_ip . '"'; + + return join(",", $restrictions); + } + + public static function getOutgoingIpFor($host) + { + $ip = gethostbyname($host); + $interface = null; + + if ($p = Process::open(["route", "-n", "get", $ip])) { + while (($line = $p->get(10)) !== false) + if (preg_match('/\s*interface:\s*([^\s]+).*$/', $line, $matches)) { + $interface = $matches[1]; + } + } + + if ($interface && $p = Process::open(["ifconfig", $interface, "inet"])) { + while (($line = $p->get(10)) !== false) + if (preg_match('/\s*inet\s+([^\s]+)\s+netmask.*/', $line, $matches)) { + return $matches[1]; + } + } + + return false; + } +} + +/** + * Utility class to execute shell processes and handle their IO. + */ +class Process +{ + private $handle; + private $inputs; + private $outputs; + + public $exitCode = null; + + // Starts the specified process and returns an object to manage it. + public static function open(array $cmd, $cwd = null, $env = null): Process + { + global $__process_terminate_hook, $__open_processes; + if (!is_array($__open_processes)) + $__open_processes = []; + + // Ensure we never leave zombies around: Hooking into script shutdown and kill processes that are still running. + if (!$__process_terminate_hook) { + register_shutdown_function($__process_terminate_hook = function () { + global $__open_processes; + foreach ($__open_processes as $handle) { + if (is_resource($handle)) { + logger()->error("Terminating process: " . json_encode(proc_get_status($handle))); + @proc_terminate($handle); + } + } + }); + } + + $p = new Process($cmd, $cwd, $env); + return $p->isRunning() ? $p : null; + } + + public function __construct($cmd, $cwd = null, $env = null) + { + $cmd = join(" ", array_map(function ($v) { + return escapeshellarg($v); + }, $cmd)); + + $spec = [0 => ["pipe", "r"], 1 => ["pipe", "w"], 2 => ["pipe", "w"]]; + $this->handle = proc_open($cmd, $spec, $pipes, $cwd, $env); + + if (is_resource($this->handle)) { + $this->outputs = $pipes; + $this->inputs = [array_shift($this->outputs)]; + + foreach ($this->outputs as $stream) + stream_set_blocking($stream, false); + + global $__open_processes; + $__open_processes[] = $this->handle; + } else { + logger()->error("Failed opening '$cmd' in '$cwd'"); + } + } + + public function __destruct() + { + $this->close(); + if ($this->isRunning()) + $this->close(true); + } + + public function get($timeout = 5, $max_length = 8192, $ending = PHP_EOL) + { + $readables = array_filter($this->outputs, function ($stream) { + return is_resource($stream) && !feof($stream); + }); + + $micros = intval(($timeout - floor($timeout)) * 1000000); + $can_read = !empty($readables) && stream_select($readables, $w = [], $e = [], $timeout, $micros); + $stream = array_reduce(($can_read ? $readables : []), function ($a, $b) { + return is_resource($a) && !feof($a) ? $a : $b; + }, null); + + return is_resource($stream) + ? stream_get_line($stream, $max_length, $ending) + : false; + } + + public function put($data, $append = PHP_EOL) + { + if ($this->isRunning() && is_resource($stdin = $this->inputs[0]) && !feof($stdin)) { + fwrite($stdin, $data); + if ($append) + fwrite($stdin, $append); + } + } + + public function closeInput() + { + if (!feof($stdin = $this->inputs[0])) fclose($stdin); + } + + public function close($force = false) + { + global $__open_processes; + + // Read up-to 10k remaining lines from STDOUT/ERR to release locks before closing. + for ($i = 0; ($line = $this->get(0)) && $i < 10000; $i++) { + logger()->error("WARN: process: $line"); + }; + + if ($this->isRunning()) + $this->exitCode = ($force ? proc_terminate($this->handle) : proc_close($this->handle)); + + if (!$this->isRunning() && in_array($this->handle, $__open_processes)) + $__open_processes = array_diff($__open_processes, [$this->handle]); + + return $this->exitCode; + } + + public function isRunning() + { + $status = is_resource($this->handle) ? proc_get_status($this->handle) : false; + if (is_array($status)) { + if (!$this->exitCode && $this->exitCode !== 0 && !$status["running"]) + $this->exitCode = $status["exitcode"]; + return $status["running"]; + } + return false; + } +} + + +// Calling main if we have been called via CLI +if (isset($GLOBALS["argc"])) main(); diff --git a/security/acme-client/src/opnsense/service/conf/actions.d/actions_acmeclient.conf b/security/acme-client/src/opnsense/service/conf/actions.d/actions_acmeclient.conf index 99858b1aa..0637f185b 100644 --- a/security/acme-client/src/opnsense/service/conf/actions.d/actions_acmeclient.conf +++ b/security/acme-client/src/opnsense/service/conf/actions.d/actions_acmeclient.conf @@ -84,6 +84,24 @@ parameters:-c %s -a %s type:script message:uploading a certificate to highwinds +[upload-sftp] +command:/usr/local/opnsense/scripts/OPNsense/AcmeClient/upload_sftp.php +parameters:--certificates=%s --automation-id=%s +type:script +message:uploading a certificate to sftp server + +[test-sftp-connection] +command:/usr/local/opnsense/scripts/OPNsense/AcmeClient/upload_sftp.php +parameters:--host=%s --host-key=%s --port=%s --user=%s --identity-type=%s --remote-path=%s --no-error test-connection +type:script_output +message:testing connection to sftp server + +[show-sftp-identity] +command:/usr/local/opnsense/scripts/OPNsense/AcmeClient/upload_sftp.php +parameters:--identity-type=%s --host=%s show-identity +type:script_output +message:prints the public key used to connect to sftp server + [reset-acme-client] command:/usr/bin/find /var/etc/acme-client/home /var/etc/acme-client/configs /var/etc/acme-client/certs /var/etc/acme-client/keys /var/etc/acme-client/accounts -type f -delete parameters: From 2b08495f0c591adcb2983b48e8b99f30601cdc37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Kellerer?= Date: Thu, 15 Aug 2019 17:54:29 +0200 Subject: [PATCH 3/9] AcmeClient: Enhanced sftp upload with configurable filenames. Filename templates for cert, key and ca-chain can be configured in the UI and user input is sanitized to ensure valid file names are produced. New and some existing fields are using "advanced" markers to reduce amount of fields visible in the default UI. Also fixed minor bugs including one with unicode characters in names. --- .../AcmeClient/forms/dialogAction.xml | 31 +++ .../models/OPNsense/AcmeClient/AcmeClient.xml | 18 ++ .../OPNsense/AcmeClient/upload_sftp.php | 177 +++++++++++++----- 3 files changed, 178 insertions(+), 48 deletions(-) diff --git a/security/acme-client/src/opnsense/mvc/app/controllers/OPNsense/AcmeClient/forms/dialogAction.xml b/security/acme-client/src/opnsense/mvc/app/controllers/OPNsense/AcmeClient/forms/dialogAction.xml index c9c4f3b42..5d89e439f 100644 --- a/security/acme-client/src/opnsense/mvc/app/controllers/OPNsense/AcmeClient/forms/dialogAction.xml +++ b/security/acme-client/src/opnsense/mvc/app/controllers/OPNsense/AcmeClient/forms/dialogAction.xml @@ -56,6 +56,7 @@ text SFTP server port. Leave blank to use default "22". + true action.sftp_host_key @@ -89,18 +90,48 @@ text Unix permission to apply to uploaded public keys. Leave blank to use default "0440". + true action.sftp_chmod_key text Unix permission to apply to uploaded private keys. Leave blank to use default "0400". + true action.sftp_chgrp text Unix group id to apply to all uploaded files. Leave blank to not change the group. + true + + + action.sftp_filename_cert + + text + Name template for the public certificate. + Placeholders "{{name}}" and "%s" are replaced by the name of the certificate being uploaded. + Leave blank to use default "{{name}}/cert.pem". + true + + + action.sftp_filename_key + + text + Name template for the certificate's private key. + Placeholders "{{name}}" and "%s" are replaced by the name of the certificate being uploaded. + Leave blank to use default "{{name}}/key.pem". + true + + + action.sftp_filename_ca + + text + Name template for the public certificate chain file. + Placeholders "{{name}}" and "%s" are replaced by the name of the certificate being uploaded. + Leave blank to use default "{{name}}/ca.pem". + true diff --git a/security/acme-client/src/opnsense/mvc/app/models/OPNsense/AcmeClient/AcmeClient.xml b/security/acme-client/src/opnsense/mvc/app/models/OPNsense/AcmeClient/AcmeClient.xml index 6879922c5..c41a91337 100644 --- a/security/acme-client/src/opnsense/mvc/app/models/OPNsense/AcmeClient/AcmeClient.xml +++ b/security/acme-client/src/opnsense/mvc/app/models/OPNsense/AcmeClient/AcmeClient.xml @@ -804,6 +804,24 @@ /^0[0-9]{3}$/u A unix permission, 4 digits (e.g. 0400). + + N + /^(?![\/\\])[\w\d_\-@.\/{}%]{1,255}(?<![\/\\])$/ui + Should be a string between 1 and 255 characters. + Characters are limited to [a-z], [0-9] and [{}@./-_%] and the string must neither begin nor end with '/'. + + + N + /^(?![\/\\])[\w\d_\-@.\/{}%]{1,255}(?<![\/\\])$/ui + Should be a string between 1 and 255 characters. + Characters are limited to [a-z], [0-9] and [{}@./-_%] and the string must neither begin nor end with '/'. + + + N + /^(?![\/\\])[\w\d_\-@.\/{}%]{1,255}(?<![\/\\])$/ui + Should be a string between 1 and 255 characters. + Characters are limited to [a-z], [0-9] and [{}@./-_%] and the string must neither begin nor end with '/'. + /^(?!.*(Let\'s\ Encrypt|acme|[fF]irmware))([\S\s]{1,255})/ diff --git a/security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/upload_sftp.php b/security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/upload_sftp.php index 03691ca72..c04abaaa1 100644 --- a/security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/upload_sftp.php +++ b/security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/upload_sftp.php @@ -64,7 +64,8 @@ const COMMANDS = [ "description" => "transfers certificates to the specified target host", "options" => [ "host::", "port::", "host-key::", "user::", "identity-type::", "remote-path::", - "certificates::", "files::", "chgrp::", "chmod::", "chmod-key::"], + "certificates::", "files::", "chgrp::", "chmod::", "chmod-key::", + "cert-name::", "key-name::", "ca-name::"], "implementation" => "commandUpload", "default" => true, ], @@ -117,6 +118,13 @@ const KNOWN_HOSTS_FILE_CREATE_MODE = 0640; const DEFAULT_CERT_MODE = 0440; const DEFAULT_KEY_MODE = 0400; +// Names +const UPLOAD_NAME_TEMPLATES = [ + "cert" => ["default" => "{{name}}/cert.pem", "option" => "cert-name"], + "key" => ["default" => "{{name}}/key.pem", "option" => "key-name"], + "ca" => ["default" => "{{name}}/ca.pem", "option" => "ca-name"], +]; + // Keys & bits const DEFAULT_KEY_TYPE = "ecdsa"; @@ -249,7 +257,7 @@ function uploadCertificatesToHost(array $options): int $username = $options["user"]; - $remote_home = $sftp->pwd(); + $remote_base_path = $sftp->pwd(); $remote_files = []; $remote_path = "."; @@ -272,22 +280,37 @@ function uploadCertificatesToHost(array $options): int } // Changing remote directory if required. - if (($dir = dirname($file["target"])) !== $remote_path) { - $target_dir = $sftp->resolve($dir, $remote_home); + if (($target_dir = dirname($file["target"])) !== $remote_path) { - $error = $sftp->cd($target_dir)->lastError(); - if ($error["file_not_found"]) { - logger()->info("Creating remote directory: $target_dir"); - $sftp->clearError() - ->mkdir($target_dir) - ->cd($target_dir); + $absolute_target_dir = $sftp->resolve($target_dir, $remote_base_path); + requireThat( + $absolute_target_dir && strpos($absolute_target_dir, $remote_base_path) === 0, + "Illegal target directory '$absolute_target_dir' is not below '$remote_base_path'"); + + $dir_names = preg_split('-/+-', substr($absolute_target_dir, strlen($remote_base_path)), 0, PREG_SPLIT_NO_EMPTY); + if (count($dir_names) == 1) + $dir_names[0] = $absolute_target_dir; // Single dir: Use absolute path + else + $sftp->cd($remote_base_path); // None or multiple directories: Start from base path and create one by one as needed + + foreach ($dir_names as $dir) { + if ($error = $sftp->cd($dir)->lastError()) { + if ($error["file_not_found"]) { + logger()->info("Creating remote directory: $dir"); + $sftp->clearError() + ->mkdir($dir) + ->cd($dir); + } else { + break; + } + } } if ($error = $sftp->lastError()) { - logger()->error("Failed to cd into '$dir'. Cause: " . json_encode($error, JSON_UNESCAPED_SLASHES)); + logger()->error("Failed to cd into '$target_dir'. Cause: " . json_encode($error, JSON_UNESCAPED_SLASHES)); return EXITCODE_ERROR; } else { - $remote_path = $dir; + $remote_path = $target_dir; $remote_files = $sftp->ls(); if ($error = $sftp->lastError()) { logger()->error("Failed listing remote files. Cause: " . json_encode($error, JSON_UNESCAPED_SLASHES)); @@ -316,7 +339,8 @@ function uploadCertificatesToHost(array $options): int // Initial upload when permissions are properly set. if (!$remote_is_readonly) { - if ($error = $sftp->put($file["source"], $remote_filename)->lastError()) { + $preserve_times_and_mod = $file_chmod !== false; + if ($error = $sftp->put($file["source"], $remote_filename, $preserve_times_and_mod)->lastError()) { logger()->error("Failed uploading file '{$file["source"]}' to '{$file["target"]}'. Cause: " . json_encode($error, JSON_UNESCAPED_SLASHES)); if ($error["permission_denied"] !== true) @@ -451,6 +475,9 @@ function getOptionsById($automation_id, $silent = false) "chgrp" => trim((string)$action->sftp_chgrp), "chmod" => trim((string)$action->sftp_chmod), "chmod-key" => trim((string)$action->sftp_chmod_key), + "cert-name" => trim((string)$action->sftp_filename_cert), + "key-name" => trim((string)$action->sftp_filename_key), + "ca-name" => trim((string)$action->sftp_filename_ca), ]; } else if (!$silent) { logger()->error("Ignoring disabled or invalid automation '$automation_id'"); @@ -482,12 +509,40 @@ function getFilesToUpload(array $options) if (($time = intval($cert["updated"])) && $time > 0) touch($source, $time); - // Sanitize user input (allow unicode chars, numbers and some special characters) - $context_path = preg_replace('/[^\w\d_\-@.]+/uim', "-", $cert["name"]); + // Build the upload name + $target_path = requireThat(UPLOAD_NAME_TEMPLATES[$name], "No upload template defined for '{$name}'"); + $target_path = stripcslashes($options[$target_path["option"]] ?: $target_path["default"]); - $file = ["source" => $source, "target" => "{$context_path}/{$name}.pem"]; - $file["is_key"] = ($name === "key"); - $files[] = $file; + $target_path = join("/", array_map( + function ($path_part) use (&$cert) { + // Replace template params "{{.+}}" & "%s" + $path_part = preg_replace_callback( + ['/%s/', '/{{([^}]+?)}}/'], + function ($m) use (&$cert) { + $index = $m[0] == '%s' ? "name" : trim($m[1]); + + return in_array($index, ["name", "id", "updated"]) + ? stripcslashes($cert[$index]) + : "__unknown-template-param__{$index}__"; + }, + $path_part); + + // Sanitize user input. Allow unicode chars, numbers and some special characters [_-@.]. + // Also replace all ".." with "." to avoid upwards tree traversal. + return preg_replace(['/\.+/', '/[^\w\d_\-@.]+/uim'], ['.', '-'], trim($path_part)); + }, + preg_split('-[/\\\\]+-', $target_path, 0, PREG_SPLIT_NO_EMPTY))); + + // Add the file to upload (if valid) + if (!empty($target_path) + && preg_match('-^(?!/).+?(? $source, "target" => $target_path]; + $file["is_key"] = ($name === "key"); + $files[] = $file; + } else { + logger()->error("Cannot add '{$name}.pem' since the upload path '$target_path' is invalid."); + } } } else { logger()->error("Ignoring upload for cert '{$cert["name"]}', since it is not available in trust storage."); @@ -534,9 +589,10 @@ function findCertificates(array $certificate_ids_or_names, $load_content = true) continue; } + $item["id"] = $id; $item["name"] = $name; $item["updated"] = intval($cert->lastUpdate); - $item["automations"] = preg_split('/[\s*,]+/', $cert->restartActions); + $item["automations"] = preg_split('/[\s,]+/', $cert->restartActions); if (isset($cert->certRefId)) { $refids[] = $item['content_id'] = (string)$cert->certRefId; } @@ -684,6 +740,7 @@ function requireThat($expression, $message) logger()->error("FATAL: $message"); exit(EXITCODE_ERROR); } + return $expression; } function resolvePath($file, $base = ".") @@ -744,6 +801,8 @@ class SftpClient private $ssh_keys; /* @var null|Process */ private $process = null; + /* @var null|string */ + private $pwd = null; public function __construct($config_path, $identity_type = DEFAULT_IDENTITY_TYPE) { @@ -808,7 +867,7 @@ class SftpClient return false; } - private function processAvailableInput(float $timeout = 0, $expected_lines = 0, Callable $lines_consumer = null) + private function processAvailableInput(float $timeout = 0, $expected_lines = 0, Callable $lines_consumer = null, $remaining_timeout = 0) { requireThat($this->process !== null, "SFTP: process not connected"); @@ -825,9 +884,6 @@ class SftpClient ]; while (($line = $this->process->get($timeout)) !== false) { - if (--$expected_lines <= 0) - $timeout = 0; - foreach ($expected_errors as $ee) { if (preg_match($ee[1], $line)) { $this->failed_status = [$ee[0] => true, "error" => trim($line)]; @@ -835,9 +891,13 @@ class SftpClient } } - $hide = ($lines_consumer && $lines_consumer($line) === true); - if (!$hide) + $consumed = ($lines_consumer && $lines_consumer($line) === true); + if (!$consumed) logger()->info("SFTP: " . rtrim($line)); + + if (!$lines_consumer || $consumed) { + if (--$expected_lines <= 0) $timeout = $remaining_timeout; + } } } @@ -879,7 +939,8 @@ class SftpClient $regex = '/^([bcdlsp\-][rwx\-]{9}[+@]?)\s+[0-9]+\s+([^\s]+)\s+([^\s]+)\s+([0-9]+)\s+(\w+\s+[0-9]+\s+[0-9:]+)\s+(.+)$/'; $this->processAvailableInput(30, 2, function ($line) use (&$files, $regex) { if (preg_match($regex, $line, $matches)) { - $files[trim($matches[6])] = [ + $filename = trim(stripcslashes($matches[6])); // decodes octal UTF-8 sequences + $files[$filename] = [ "type" => $matches[1][0], "permissions" => $matches[1], "owner" => $matches[2], @@ -890,21 +951,30 @@ class SftpClient return true; } return false; - }); + }, 1); return $files; } public function pwd() { - $remote_path = false; - $this->processAvailableInput(); - $this->process->put("pwd"); - $this->processAvailableInput(30, 2, function ($line) use (&$remote_path) { - if (preg_match('/^.+directory:\s(.+)$/i', $line, $matches)) - $remote_path = trim($matches[1]); - }); - return $remote_path; + if ($this->pwd === null) { + $remote_path = false; + + $this->processAvailableInput(); + $this->process->put("pwd"); + $this->processAvailableInput(30, 1, function ($line) use (&$remote_path) { + if (preg_match('/^.+directory:\s(.+)$/i', $line, $matches)) { + $remote_path = trim(stripcslashes($matches[1])); + return true; + } + return false; + }); + + $this->pwd = $remote_path; + } + + return $this->pwd; } public function resolve($remote_path, $remote_pwd = null) @@ -920,16 +990,20 @@ class SftpClient public function get($remote_file, $local_file = "") { $this->processAvailableInput(); - $this->process->put("get " . escapeshellarg($remote_file) . " " . (empty($local_file) ? "" : escapeshellarg($local_file))); + $this->process->put("get " + . escapeshellarg($remote_file) + . (empty($local_file) ? "" : " " . escapeshellarg($local_file))); $this->processAvailableInput(30, 2); return $this; } - public function put($local_file, $remote_file = "") + public function put($local_file, $remote_file = "", $preserve = true) { if (is_file($local_file)) { $this->processAvailableInput(); - $this->process->put("put -p " . escapeshellarg($local_file) . " " . (empty($remote_file) ? "" : escapeshellarg($remote_file))); + $this->process->put("put " . ($preserve ? "-p " : "") + . escapeshellarg($local_file) + . (empty($remote_file) ? "" : " " . escapeshellarg($remote_file))); $this->processAvailableInput(30, 2); } else { logger()->info("put: File $local_file doesn't exist."); @@ -957,6 +1031,7 @@ class SftpClient $this->processAvailableInput(30, 1); $error = $this->lastError(); $pwd = false; + $this->pwd = null; if ($error || $remote_path !== ($pwd = $this->pwd())) { $this->failed_status = array_merge(($error ?: []), [ @@ -970,25 +1045,31 @@ class SftpClient public function chmod($remote_file, $mode) { - $this->processAvailableInput(); - $this->process->put("chmod " . escapeshellarg($mode) . " " . escapeshellarg($remote_file)); - $this->processAvailableInput(30, 2); + if (($remote_file = $this->resolve($remote_file)) !== false) { + $this->processAvailableInput(); + $this->process->put("chmod " . escapeshellarg($mode) . " " . escapeshellarg($remote_file)); + $this->processAvailableInput(30, 2); + } return $this; } public function chgrp($remote_file, $group_id) { - $this->processAvailableInput(); - $this->process->put("chgrp " . escapeshellarg($group_id) . " " . escapeshellarg($remote_file)); - $this->processAvailableInput(30, 2); + if (($remote_file = $this->resolve($remote_file)) !== false) { + $this->processAvailableInput(); + $this->process->put("chgrp " . escapeshellarg($group_id) . " " . escapeshellarg($remote_file)); + $this->processAvailableInput(30, 2); + } return $this; } public function rm($remote_file) { - $this->processAvailableInput(); - $this->process->put("rm " . escapeshellarg($remote_file)); - $this->processAvailableInput(30, 2); + if (($remote_file = $this->resolve($remote_file)) !== false) { + $this->processAvailableInput(); + $this->process->put("rm " . escapeshellarg($remote_file)); + $this->processAvailableInput(30, 2); + } return $this; } } @@ -1335,7 +1416,7 @@ class Process public $exitCode = null; // Starts the specified process and returns an object to manage it. - public static function open(array $cmd, $cwd = null, $env = null): Process + public static function open(array $cmd, $cwd = null, $env = null): ?Process { global $__process_terminate_hook, $__open_processes; if (!is_array($__open_processes)) From f6d8572b7edeaa9b7045ee9b3454d0ec67063a38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Kellerer?= Date: Thu, 15 Aug 2019 19:03:13 +0200 Subject: [PATCH 4/9] AcmeClient: Applied minor changes as requested. --- .../mvc/app/views/OPNsense/AcmeClient/actions.volt | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/security/acme-client/src/opnsense/mvc/app/views/OPNsense/AcmeClient/actions.volt b/security/acme-client/src/opnsense/mvc/app/views/OPNsense/AcmeClient/actions.volt index b6b3f8843..5edb1fed9 100644 --- a/security/acme-client/src/opnsense/mvc/app/views/OPNsense/AcmeClient/actions.volt +++ b/security/acme-client/src/opnsense/mvc/app/views/OPNsense/AcmeClient/actions.volt @@ -105,10 +105,11 @@ POSSIBILITY OF SUCH DAMAGE. ajaxCall("/api/acmeclient/actions/sftpGetIdentity", getFormData("DialogAction").action, function (data, status) { button.prop('disabled', false).find(".fa-spinner").hide(); - if (status === "success" && data.status === "ok") + if (status === "success" && data.status === "ok") { identityDiv.text(data.identity).show(); - else + } else { identityDiv.text("{{ lang._('Failed loading identity') }}").show(); + } }); }); @@ -153,7 +154,9 @@ POSSIBILITY OF SUCH DAMAGE. ajaxCall("/api/acmeclient/actions/sftpTestConnection", getFormData("DialogAction").action, function (data, status) { button.prop('disabled', false).find(".fa-spinner").hide(); - var message = "", detail = "", statusClass = "alert-warning"; + var message = "", + detail = "", + statusClass = "alert-warning"; if (status === "success") { if (data.success === true) { From 3f56755e81f975a49241341a35a0bead9a75027b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Kellerer?= Date: Sat, 17 Aug 2019 16:32:43 +0200 Subject: [PATCH 5/9] AcmeClient: Refactored SFTP upload script into multiple class files. --- .../library/OPNsense/AcmeClient/Process.php | 180 +++ .../library/OPNsense/AcmeClient/SSHKeys.php | 511 +++++++ .../OPNsense/AcmeClient/SftpClient.php | 335 +++++ .../OPNsense/AcmeClient/SftpUploader.php | 388 ++++++ .../app/library/OPNsense/AcmeClient/Utils.php | 143 ++ .../OPNsense/AcmeClient/upload_sftp.php | 1210 +++-------------- 6 files changed, 1722 insertions(+), 1045 deletions(-) create mode 100644 security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/Process.php create mode 100644 security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/SSHKeys.php create mode 100644 security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/SftpClient.php create mode 100644 security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/SftpUploader.php create mode 100644 security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/Utils.php diff --git a/security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/Process.php b/security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/Process.php new file mode 100644 index 000000000..e35df1d7b --- /dev/null +++ b/security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/Process.php @@ -0,0 +1,180 @@ +isRunning() ? $p : null; + } + + private static function manageOpenedProcess($process_handle, $release = false) + { + static $open_processes; + + if (!is_array($open_processes)) { + $open_processes = []; + + // Ensure we never leave zombies around: Hooking into script shutdown and kill processes that are still running. + register_shutdown_function(function () use (&$open_processes) { + foreach ($open_processes as $handle) { + if (is_resource($handle)) { + Utils::log()->error("Terminating process: " . json_encode(proc_get_status($handle))); + @proc_terminate($handle); + } + } + $open_processes = []; + }); + } + + if ($process_handle) { + if ($release) { + if (in_array($process_handle, $open_processes)) + $open_processes = array_diff($open_processes, [$process_handle]); + } else { + $open_processes[] = $process_handle; + } + } + } + + public function __construct($cmd, $cwd = null, $env = null) + { + $cmd = join(" ", array_map(function ($v) { + return escapeshellarg($v); + }, $cmd)); + + $spec = [0 => ["pipe", "r"], 1 => ["pipe", "w"], 2 => ["pipe", "w"]]; + $this->handle = proc_open($cmd, $spec, $pipes, $cwd, $env); + + if (is_resource($this->handle)) { + $this->outputs = $pipes; + $this->inputs = [array_shift($this->outputs)]; + + foreach ($this->outputs as $stream) + stream_set_blocking($stream, false); + + self::manageOpenedProcess($this->handle); + } else { + Utils::log()->error("Failed opening '$cmd' in '$cwd'"); + } + } + + public function __destruct() + { + $this->close(); + + if ($this->isRunning()) + $this->close(true); + } + + public function get($timeout = 5, $max_length = 8192, $ending = PHP_EOL) + { + $readables = array_filter($this->outputs, function ($stream) { + return is_resource($stream) && !feof($stream); + }); + + $micros = intval(($timeout - floor($timeout)) * 1000000); + $can_read = !empty($readables) && stream_select($readables, $w = [], $e = [], $timeout, $micros); + $stream = array_reduce(($can_read ? $readables : []), function ($a, $b) { + return is_resource($a) && !feof($a) ? $a : $b; + }, null); + + return is_resource($stream) + ? stream_get_line($stream, $max_length, $ending) + : false; + } + + public function put($data, $append = PHP_EOL) + { + if ($this->isRunning() && is_resource($stdin = $this->inputs[0]) && !feof($stdin)) { + fwrite($stdin, $data); + if ($append) + fwrite($stdin, $append); + } + } + + public function closeInput() + { + if (!feof($stdin = $this->inputs[0])) fclose($stdin); + } + + public function close($force = false) + { + // Read up-to 10k remaining lines from STDOUT/ERR to release locks before closing. + for ($i = 0; ($line = $this->get(0)) && $i < 10000; $i++) { + Utils::log()->error("WARN: process: $line"); + } + + if ($this->isRunning()) { + $this->exitCode = $force + ? proc_terminate($this->handle) + : proc_close($this->handle); + } + + if (!$this->isRunning()) { + self::manageOpenedProcess($this->handle, true); + } + + return $this->exitCode; + } + + public function isRunning() + { + $status = is_resource($this->handle) + ? proc_get_status($this->handle) + : false; + + if (is_array($status)) { + if (!$this->exitCode && $this->exitCode !== 0 && !$status["running"]) + $this->exitCode = $status["exitcode"]; + return $status["running"]; + } + + return false; + } +} diff --git a/security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/SSHKeys.php b/security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/SSHKeys.php new file mode 100644 index 000000000..7603f3612 --- /dev/null +++ b/security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/SSHKeys.php @@ -0,0 +1,511 @@ + 4096, + "ecdsa" => 521, + ]; + + const DEFAULT_PORT = 22; + + private $config_path; + private $known_hosts_file; + + public function __construct($config_path) + { + if (!is_dir($config_path)) { + $dir_created = mkdir($config_path, self::CONFIG_PATH_CREATE_MODE, true); + + Utils::requireThat($dir_created, + "Failed creating directory '$config_path' with permission " . self::CONFIG_PATH_CREATE_MODE); + } + + $this->config_path = realpath($config_path); + $this->known_hosts_file = Utils::resolvePath("known_hosts", $this->config_path); + } + + public function knownHostsFile() + { + if (!is_file($this->known_hosts_file)) { + $file_created = + touch($this->known_hosts_file) + && chmod($this->known_hosts_file, self::KNOWN_HOSTS_FILE_CREATE_MODE); + + Utils::requireThat($file_created, + "Failed creating file '{$this->known_hosts_file}' with permission " . self::KNOWN_HOSTS_FILE_CREATE_MODE); + } + + return $this->known_hosts_file; + } + + /** + * Establishes a trust in the specified host with the specified host-key. Fails if host-key mismatches or host cannot be reached. + * @param string $host The name or IP address of the host to trust. + * @param string $host_key The expected host-key required to establish a trust. Is auto-accepted on first connect and stored to "known_hosts" if omitted. + * @param int $port The port of the SSH server on the host to trust. + * @param bool $no_modification_allowed Indicates whether "known_hosts" may be modified to trust the host or not. + * @return array A status following the format ["ok"=>true/false, "error"=>"reason"]. + */ + public function trustHost(string $host, $host_key = "", $port = self::DEFAULT_PORT, $no_modification_allowed = false): array + { + Utils::requireThat(!empty(trim($host)), "Hostname must not be empty."); + + + // Convert the specified host_key to a data structure that can be compared + if (empty($host_key = trim($host_key))) { + $host_key = false; + } else { + $host_key = self::getHostKeyInfo($host_key); + if ($host_key === false) + return ["ok" => false, "error" => "Invalid host_key specified."]; + } + + + // Check our current known_host file + $addKeyInfo = function (array &$key_list) { + foreach ($key_list as &$item) { + $item["key_info"] = self::getHostKeyInfo($item["host_key"]); + } + return array_filter($key_list, function (&$item) { + return $item["key_info"] !== false; + }); + }; + + $known_keys = $addKeyInfo($this->getKnownHostKey($host, $port)); + + // Find known_hosts item with same hostname + $known_by_host = array_reduce($known_keys, function ($found, $key) use ($host) { + return (!$found && !empty(trim($key["host"])) && strcasecmp(trim($host), trim($key["host"])) == 0) + ? $key + : $found; + }, false); + + // Find known_hosts item with same public host-key + $known_by_key = array_reduce($known_keys, function ($found, $key) use ($host_key) { + return (!$found && $host_key && $host_key === $key["key_info"]) + ? $key + : $found; + }, false); + + + // Updating $host and $host_key from known_hosts and check if we need to update known_hosts. + if ($host_key === false && $known_by_host) { + Utils::log()->info("No host key specified, using existing known_hosts entry for '$host'"); + $host_key = $known_by_host["key_info"]; + } + + $is_key_known = false; + if ($known_by_host && $host_key && $host_key === $known_by_host["key_info"]) { + $is_key_known = true; + } else if ($known_by_key) { + if (strcasecmp(trim($host), trim($known_by_key["host"])) != 0) { + Utils::log()->info("Host key is in known_hosts but hostname differs. Changing '$host' to '{$known_by_key["host"]}'."); + $host = $known_by_key["host"]; + } + $is_key_known = true; + } + + + // Check if we don't have a matching known_hosts entry and add or update it as required. + if (!$is_key_known && !$no_modification_allowed) { + + // Query the key. + $key_type = $host_key ? $host_key["key_type"] : self::DEFAULT_KEY_TYPE; + $remote_host_keys = $addKeyInfo($this->queryHostKey($host, $key_type, $port, $query_error)); + + // Retry with ALTERNATE_DEFAULT_KEY_TYPE when DEFAULT_KEY_TYPE was applied in the first place. + if (empty($remote_host_keys) + && $query_error + && $query_error["connection_refused"] + && !$host_key + && self::ALTERNATE_DEFAULT_KEY_TYPE != self::DEFAULT_KEY_TYPE) { + + $key_type = self::ALTERNATE_DEFAULT_KEY_TYPE; + $remote_host_keys = $addKeyInfo($this->queryHostKey($host, $key_type, $port, $query_error)); + } + + $matching_remote_host_keys = array_filter($remote_host_keys, function ($key) use ($host_key) { + return $key["key_info"] !== false && (!$host_key || $host_key === $key["key_info"]); + }); + + if (!empty($matching_remote_host_keys)) { + if ($known_by_host) { + Utils::log()->info("Removing known_hosts entry with differing key for '{$known_by_host["host"]}' as it is in the way."); + $this->removeKnownHost($known_by_host["host"]); + } + + foreach ($matching_remote_host_keys as $key) { + Utils::log()->info("Adding known_hosts entry: " . json_encode($key["key_info"], JSON_UNESCAPED_SLASHES)); + $ok = file_put_contents($this->knownHostsFile(), $key["host_key"] . PHP_EOL, FILE_APPEND); + if (!$ok) + Utils::log()->error("Failed adding known_hosts entry {$key["host_key"]}"); + } + + // Verify that known_hosts contains the correct keys after adding them (using recursion). + return $this->trustHost($host, $matching_remote_host_keys[0]["host_key"], $port, true); + + } else { + if (empty($remote_host_keys)) { + $msg = "No connection to '$host'; Failed querying host key from server."; + } else { + $remote_infos = array_map(function ($key) { return $key["key_info"]; }, $remote_host_keys); + $msg = "Key mismatch for '$host'; " + . "The expected key (" . json_encode($host_key) . ") was not found in (" . json_encode($remote_infos) . ")"; + } + + return array_merge(["ok" => false, "error" => $msg], ($query_error ?: [])); + } + } + + + if ($is_key_known) { + return ["ok" => true, "host" => $host, "key_info" => $host_key]; + } else { + return ["ok" => false, "error" => "Host unknown and remote key cannot be queried."]; + } + } + + /** + * Returns a normalized list of names and IP addresses (IPv4) that point to the same host. + * @param string $host The name or IP address of the host. + * @param int $port Add port specific host names and addresses to the search-list when greater 0. + * @return array A list of hostnames / ip addresses pointing to the same host. + */ + public static function getHostSearchList(string $host, int $port = 0) + { + $host = strtolower($host); + $search_list = [$host]; + + // Add IP-address to search list (IPv4 only) + $has_ip = ($ip = gethostbyname($host)) + && ($ip !== $host || preg_match('/^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$/', $ip)); + + if ($has_ip) + $search_list[] = strtolower($ip); + + // Add FQDN to search list if reverse lookup provides a valid one. + $has_fqdn = $has_ip + && ($reverse_fqdn = gethostbyaddr($ip)) + && $reverse_fqdn !== $ip + && gethostbyname($reverse_fqdn) === $ip; + + if ($has_fqdn && isset($reverse_fqdn)) + $search_list[] = strtolower($reverse_fqdn); + + // Build unique search list (dedup list) + $search_list = array_filter($search_list, function ($value, $index) use (&$search_list) { + return !empty(trim($value)) && array_search($value, $search_list) == $index; + }, ARRAY_FILTER_USE_BOTH); + + // Add port specific items when a port is specified (reason: ssh-keyscan may create such entries). + if ($port > 0) { + foreach (array_values($search_list) as $item) { + $search_list[] = "[{$item}]:{$port}"; + } + } + + return $search_list; + } + + /** + * Queries the host-key from the SSH server running on the specified host. + * @param string $host The name or IP address of the host. + * @param string $key_type The type of host-key to query (one of "rsa", "ecdsa" or "ed25519"). + * @param int $port the port of the SSH server. + * @param array $error receives error details. + * @return array A list of host-keys returned by the query. + */ + public static function queryHostKey(string $host, $key_type = self::DEFAULT_KEY_TYPE, $port = self::DEFAULT_PORT, ?array &$error = []) + { + // Error list matching output from "ssh-keyscan". Keep names in sync with similar list in SftpClient. + static $expected_errors = [ + ["host_not_resolved", /* -> */ '/.*not known.*/i'], + ["network_unreachable", /* -> */ '/.*no route.*/i'], + ["failure", /* -> */ '/.*(connect|write|broken).*/i'], + ]; + + $keys = []; + $failed = false; + $names = join(",", self::getHostSearchList($host)); + + if (!empty($names) && ($p = Process::open(["ssh-keyscan", "-p", $port, "-t", $key_type, $names]))) { + $lines = []; + while (($line = $p->get(60)) !== false) { + $line = trim($line); + if (empty($line) || $line[0] == "#") + continue; + + if (!$failed) { + foreach ($expected_errors as $err) + if (preg_match($err[1], $line)) { + $error = [$err[0] => true]; + $failed = true; + break; + } + } + + $lines[] = $line; + } + + if ($p->close() == 0 && !$failed) { + foreach ($lines as $line) { + $keys[] = ["host_key" => $line]; + } + } else { + $marker = $failed ? "yes" : "no"; + $output = empty($lines) + ? "" + : PHP_EOL . "ssh-keyscan: " . join(PHP_EOL . "ssh-keyscan: ", $lines); + + Utils::log()->error("Failed querying public keys ($key_type) for [$names]. Exit code: {$p->exitCode} (error-marker: $marker) $output"); + } + } + + if (empty($keys)) { + Utils::log()->info("Couldn't fetch public host key ($key_type) from $host"); + + if (!is_array($error) || empty($error)) + $error = ["connection_refused" => true]; + } + + return $keys; + } + + /** + * Returns all host-keys from "known_hosts" that match the specified hostname. + * @param string $host The name of the host to lookup. + * @param int $port The port of the SSH server on the host (used to find port specific known host entries). + * @return array A list of all matching host keys. + */ + public function getKnownHostKey(string $host, int $port = self::DEFAULT_PORT) + { + $keys = []; + + foreach (self::getHostSearchList($host, $port) as $name_or_ip) { + if ($p = Process::open(["ssh-keygen", "-F", $name_or_ip, "-f", $this->knownHostsFile()])) { + $lines = []; + while (($line = $p->get()) !== false) { + $line = trim($line); + if (empty($line) || $line[0] == "#") + continue; + + $lines[] = $line; + } + + if ($p->close() == 0) { + // Removing port from name or ip before returning it. + $applied_host = preg_match('/^\[([^\]]+?)\]:\d+/', $name_or_ip, $matches) + ? $matches[1] + : $name_or_ip; + + $keys[] = ["host" => $applied_host, "host_key" => $lines[0]]; + + } else if ($p->exitCode != 1 /* 1 == NOT_FOUND */) { + $output = empty($lines) + ? "" + : PHP_EOL . join(PHP_EOL, $lines); + + Utils::log()->error("Failed querying known hosts for $name_or_ip ($host). Exit code: {$p->exitCode} $output"); + } + } + } + + if (empty($keys)) + Utils::log()->info("Didn't find $host in known_hosts"); + + return $keys; + } + + /** + * Removes a specific host from the "known_hosts" file. + * @param string $host The name of the host to remove. + * @return bool True on success. + */ + public function removeKnownHost(string $host) + { + $ok = false; + + if ($p = Process::open(["ssh-keygen", "-R", $host, "-f", $this->knownHostsFile()])) { + $ok = $p->close() === 0; + if (!$ok) + Utils::log()->error("Failed removing known hosts for $host. Return code was: {$p->exitCode}"); + } + + return $ok; + } + + /** + * Returns the key info (key hash, length, type) for a specified host key. + * @param string $host_key The host key as formatted in "authorized_keys" or "known_hosts". + * @return array|bool A host key info [hash=>.., key_type=>.., key_length=>..] or false on failure. + */ + public static function getHostKeyInfo(string $host_key) + { + if ($p = Process::open(["ssh-keygen", "-l", "-f", "-"])) { + $p->put($host_key); + $p->closeInput(); + + if (($hash = $p->get()) && preg_match('/^([0-9]+) (.+?) .+? \(([^()]+)\)$/', $hash, $matches)) { + return [ + "hash" => $matches[2], + "key_type" => $matches[3], + "key_length" => $matches[1] + ]; + } else { + Utils::log()->error("Unsupported hash type: $hash"); + } + } + + Utils::log()->error("Failed getting hash for host_key"); + return false; + } + + /** + * Returns the path to the public identity key file, generating it if missing. + * @param string $identity_type the type of identity to return {@see IDENTITY_TYPES}. + * @param bool $private Return the path to the private key file instead. + * @return string The path to the key file. + */ + public function getIdentity(string $identity_type = self::DEFAULT_IDENTITY_TYPE, $private = false): string + { + Utils::requireThat(in_array($identity_type, self::IDENTITY_TYPES), "Identity type '$identity_type' unknown."); + + list($key_type, $key_size) = explode('_', $identity_type, 2); + if (!$key_size && self::DEFAULT_IDENTITY_KEY_BITS[$key_type] > 0) + $key_size = self::DEFAULT_IDENTITY_KEY_BITS[$key_type]; + + $identity_path = "{$this->config_path}/id.{$identity_type}"; + + if (!file_exists($identity_path)) { + $generate_key = [ + "ssh-keygen", "-v", + "-f", $identity_path, + "-t", $key_type, + "-N", "", + ]; + + if (intval($key_size) > 0) + array_push($generate_key, "-b", $key_size); + + if ($p = Process::open($generate_key)) { + while (($line = $p->get(10)) !== false) { + Utils::log()->info("SSH keygen: $line"); + } + + Utils::requireThat($p->close() == 0, + "Failed generating identity $identity_path: Error code: {$p->exitCode}" . PHP_EOL + . "Command: " . join(" ", $generate_key)); + } + } + + return $private ? $identity_path : "{$identity_path}.pub"; + } + + /** + * Returns a restrictions comment to be used in "authorized_keys" to limit what an identity can be used for. + * @param string $host the SSH server host to create the restriction for. + * @param string $outgoing_ip the IP address of the interface that will be used to connect the SSH server or empty to autodetect. + * @param string $command the command to restrict the identity to. + * @return string A restriction comment to be used to prepend an "authorized_keys" entry. + */ + public static function getIdentityRestrictions($host = "", $outgoing_ip = "", $command = "internal-sftp"): string + { + $restrictions = ['restrict']; + + if ($command) + $restrictions[] = 'command="' . $command . '"'; + + $restrict_ip = empty(trim($outgoing_ip)) + ? (empty(trim($host)) ? false : self::getOutgoingIpFor($host)) + : $outgoing_ip; + + if ($restrict_ip) + $restrictions[] = 'from="' . $restrict_ip . '"'; + + + return count($restrictions) > 1 + ? join(",", $restrictions) + : ""; + } + + /** + * Returns the IP address of the interface that will be used when connecting to host. + * @param string $host the host to check outgoing IP address for. + * @return bool|mixed an IPv4 address when the route & interface was detected. + */ + public static function getOutgoingIpFor(string $host) + { + $ip = gethostbyname($host); + $interface = null; + + if ($p = Process::open(["route", "-n", "get", $ip])) { + while (($line = $p->get(10)) !== false) + if (preg_match('/\s*interface:\s*([^\s]+).*$/', $line, $matches)) { + $interface = $matches[1]; + } + } + + if ($interface && $p = Process::open(["ifconfig", $interface, "inet"])) { + while (($line = $p->get(10)) !== false) + if (preg_match('/\s*inet\s+([^\s]+)\s+netmask.*/', $line, $matches)) { + return $matches[1]; + } + } + + return false; + } +} diff --git a/security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/SftpClient.php b/security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/SftpClient.php new file mode 100644 index 000000000..e3ba387ec --- /dev/null +++ b/security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/SftpClient.php @@ -0,0 +1,335 @@ +ssh_keys = new SSHKeys($config_path); + $this->identity_type = $identity_type; + } + + public function __destruct() + { + $this->close(); + } + + public function connected(): ?array + { + return $this->process + ? $this->connection_info + : null; + } + + public function connect($host, $username, $host_key = "", $port = SSHKeys::DEFAULT_PORT) + { + if (empty(trim($host)) || empty(trim($username))) { + $this->failed_status = ["invalid_parameters" => true]; + Utils::log()->error("Failed connecting to '$host'. Hostname or username is missing."); + return false; + } + + $trust = $this->ssh_keys->trustHost($host, $host_key, $port); + if ($trust["ok"] !== true) { + Utils::log()->error("Failed establishing trust in '$host'; Cause: {$trust["error"]}"); + unset($trust["ok"]); + $this->failed_status = array_merge($trust, ["host_not_trusted" => true]); + return false; + } else { + $host = $trust["host"]; + } + + // Building sftp command. + $cmd = [ + "sftp", + "-P", $port, + "-oUser=$username", + "-oUserKnownHostsFile={$this->ssh_keys->knownHostsFile()}", + ]; + + // Handle client side identity + $identity = $this->ssh_keys->getIdentity($this->identity_type, true); + if (is_file($identity) && is_readable($identity)) { + array_push($cmd, + "-i", $identity, + "-oPreferredAuthentications=publickey"); + } else { + Utils::log()->error("Failed adding client identity ($identity). Connect will likely fail."); + } + + // Adding the host + array_push($cmd, "$host"); + + // Creating the sftp process + if ($this->process = Process::open($cmd)) { + $this->processAvailableInput(self::CONNECT_REPLY_TIMEOUT, 1, null, 0.75); + if ($error = $this->lastError()) { + Utils::log()->error("Failed connecting to '$host' (user: '$username')", $error); + return false; + } + $this->connection_info = ["host" => $host, "port" => $port, "user" => $username]; + return true; + } + return false; + } + + private function processAvailableInput(float $timeout = 0, $expected_lines = 0, Callable $lines_consumer = null, $remaining_timeout = 0) + { + Utils::requireThat($this->process !== null, "SFTP: process not connected"); + + // Error list matching output from "sftp". Keep names in sync with similar list in SSHKeys. + static $expected_errors = [ + ["host_not_resolved", /* -> */ '/.*not resolve.*/i'], + ["host_not_trusted", /* -> */ '/.*IDENTIFICATION HAS CHANGED.*/i'], + ["connection_refused", /* -> */ '/.*connection refused.*/i'], + ["connection_closed", /* -> */ '/.*connection closed.*/i'], + ["network_timeout", /* -> */ '/.*timed out.*/i'], + ["network_unreachable", /* -> */ '/.*network.+unreachable.*/i'], + ["permission_denied", /* -> */ '/.*permission denied.*/i'], + ["file_not_found", /* -> */ '/.*(no such|not found).*/i'], + ["failure", /* -> */ '/.*(error|failure|you must supply).*/i'], + ]; + + while (($line = $this->process->get($timeout)) !== false) { + foreach ($expected_errors as $ee) { + if (preg_match($ee[1], $line)) { + if (!$this->failed_status || $ee[0] !== "connection_closed") + $this->failed_status = [$ee[0] => true, "error" => trim($line)]; + break; + } + } + + $consumed = ($lines_consumer && $lines_consumer($line) === true); + if (!$consumed) + Utils::log()->info("SFTP: " . rtrim($line)); + + if (!$lines_consumer || $consumed) { + if (--$expected_lines <= 0) $timeout = $remaining_timeout; + } + } + } + + public function close() + { + if (($p = $this->process) !== null) { + $p->put("exit"); + $p->closeInput(); + + $this->processAvailableInput(1.5); + $p->close(); + + $this->process = null; + + if ($this->failed_status && $this->failed_status["connection_closed"]) + $this->clearError(); + } + } + + public function lastError($timeout = 0.5) + { + if ($this->failed_status === false) + $this->processAvailableInput($timeout); + return $this->failed_status; + } + + public function clearError() + { + $this->failed_status = false; + return $this; + } + + public function ls() + { + $files = []; + $this->processAvailableInput(); + $this->process->put("ls -la"); + + $regex = '/^([bcdlsp\-][rwx\-]{9}[+@]?)\s+[0-9]+\s+([^\s]+)\s+([^\s]+)\s+([0-9]+)\s+(\w+\s+[0-9]+\s+[0-9:]+)\s+(.+)$/'; + $this->processAvailableInput(self::COMMAND_REPLY_TIMEOUT, 2, function ($line) use (&$files, $regex) { + if (preg_match($regex, $line, $matches)) { + $filename = trim(stripcslashes($matches[6])); // decodes octal UTF-8 sequences + $files[$filename] = [ + "type" => $matches[1][0], + "permissions" => $matches[1], + "owner" => $matches[2], + "group" => $matches[3], + "size" => intval($matches[4]), + "mtime" => strtotime($matches[5]) + ]; + return true; + } + return false; + }, 1); + + return $files; + } + + public function pwd() + { + if ($this->pwd === null) { + $remote_path = false; + + $this->processAvailableInput(); + $this->process->put("pwd"); + $this->processAvailableInput(self::COMMAND_REPLY_TIMEOUT, 1, function ($line) use (&$remote_path) { + if (preg_match('/^.+directory:\s(.+)$/i', $line, $matches)) { + $remote_path = trim(stripcslashes($matches[1])); + return true; + } + return false; + }); + + $this->pwd = $remote_path; + } + + return $this->pwd; + } + + /** + * Returns the absolute remote path for the specified file. + * @param string $remote_path the relative path. + * @param string|null $remote_pwd the remote base directory. Omit to use current. + * @return bool|string An absolute path. + */ + public function resolve(string $remote_path, ?string $remote_pwd = null) + { + if (($pwd = ($remote_pwd ?: $this->pwd())) !== false) { + $remote_path = Utils::resolvePath($remote_path, str_replace("/", DIRECTORY_SEPARATOR, $pwd)); + $remote_path = str_replace("\\", "/", $remote_path); + return $remote_path; + } + return false; + } + + public function get($remote_file, $local_file = "") + { + $this->processAvailableInput(); + $this->process->put("get " + . escapeshellarg($remote_file) + . (empty($local_file) ? "" : " " . escapeshellarg($local_file))); + $this->processAvailableInput(self::COMMAND_REPLY_TIMEOUT, 2); + return $this; + } + + public function put($local_file, $remote_file = "", $preserve = true) + { + if (is_file($local_file)) { + $this->processAvailableInput(); + $this->process->put("put " . ($preserve ? "-p " : "") + . escapeshellarg($local_file) + . (empty($remote_file) ? "" : " " . escapeshellarg($remote_file))); + $this->processAvailableInput(self::COMMAND_REPLY_TIMEOUT, 2); + } else { + Utils::log()->info("put: File $local_file doesn't exist."); + $this->failed_status = ["file_not_found" => true, "error" => $local_file]; + } + + return $this; + } + + public function mkdir($remote_path) + { + if (($remote_path = $this->resolve($remote_path)) !== false) { + $this->process->put("mkdir " . escapeshellarg($remote_path)); + $this->processAvailableInput(self::COMMAND_REPLY_TIMEOUT, 1); + } + return $this; + } + + public function cd($remote_path) + { + if (($remote_path = $this->resolve($remote_path)) !== false) { + $this->clearError(); + $this->process->put("cd " . escapeshellarg($remote_path)); + + $this->processAvailableInput(self::COMMAND_REPLY_TIMEOUT, 1); + $error = $this->lastError(); + $pwd = false; + $this->pwd = null; + + if ($error || $remote_path !== ($pwd = $this->pwd())) { + $this->failed_status = array_merge(($error ?: []), [ + "failure" => true, + "error" => "Failed changing path to '$remote_path' (pwd: '$pwd'); Cause: {$error["error"]}" + ]); + } + } + return $this; + } + + public function chmod($remote_file, $mode) + { + if (($remote_file = $this->resolve($remote_file)) !== false && $remote_file !== $this->pwd()) { + $this->processAvailableInput(); + $this->process->put("chmod " . escapeshellarg($mode) . " " . escapeshellarg($remote_file)); + $this->processAvailableInput(self::COMMAND_REPLY_TIMEOUT, 2); + } + return $this; + } + + public function chgrp($remote_file, $group_id) + { + if (($remote_file = $this->resolve($remote_file)) !== false && $remote_file !== $this->pwd()) { + $this->processAvailableInput(); + $this->process->put("chgrp " . escapeshellarg($group_id) . " " . escapeshellarg($remote_file)); + $this->processAvailableInput(self::COMMAND_REPLY_TIMEOUT, 2); + } + return $this; + } + + public function rm($remote_file) + { + if (($remote_file = $this->resolve($remote_file)) !== false && $remote_file !== $this->pwd()) { + $this->processAvailableInput(); + $this->process->put("rm " . escapeshellarg($remote_file)); + $this->processAvailableInput(self::COMMAND_REPLY_TIMEOUT, 2); + } + return $this; + } +} diff --git a/security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/SftpUploader.php b/security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/SftpUploader.php new file mode 100644 index 000000000..83d072cd4 --- /dev/null +++ b/security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/SftpUploader.php @@ -0,0 +1,388 @@ +sftp = $sftp; + } + + public function __destruct() + { + $this->temporaryFile(true); + } + + /** + * Add a file to upload. + * + * @param string $local_file the path to the local file. + * @param string $remote_file the remote path to copy the file to or empty to use the files name. + * @param bool $chmod the 4 digit unix permission to apply or false to leave it unchanged. + * @param bool $chgrp the numeric group on the remote server to apply or false to leave it unchanged. + * @return string the name of the normalized local file. + */ + public function addFile(string $local_file, $remote_file = "", $chmod = false, $chgrp = false): string + { + Utils::requireThat(is_file($local_file) && is_readable($local_file), "Not a file or not readable: '$local_file'"); + $local_file = realpath($local_file); + + $this->deleteSourceIfRequested($local_file); + + $this->pending_files[$local_file] = [ + "source" => $local_file, + "target" => $remote_file, + "mode" => $chmod, + "group" => $chgrp + ]; + + return $local_file; + } + + /** + * Adds content to upload. + * + * @param string $content the binary content to upload. + * @param string $remote_file the remote path to copy the file to. + * @param int $content_last_modified the unix timestamp when the content was last modified (is preserved when chmod is also specified). + * @param bool $chmod the 4 digit unix permission to apply or false to leave it unchanged. + * @param bool $chgrp the numeric group on the remote server to apply or false to leave it unchanged. + * @return string the name of the remote file. + */ + public function addContent(string $content, string $remote_file = "", $content_last_modified = 0, $chmod = false, $chgrp = false): string + { + $local_file = $this->temporaryFile(); + Utils::requireThat($local_file, "Failed creating temporary file for '$remote_file'"); + + $content_written = file_put_contents($local_file, $content); + Utils::requireThat($content_written > 0, "Failed writing content of '$remote_file' to '$local_file', disk full?"); + + if (($time = intval($content_last_modified)) && $time > 0) + touch($local_file, $time); + + $remote_file = trim($remote_file); + if (empty($remote_file)) + $remote_file = basename($local_file); + + $local_file = $this->addFile($local_file, $remote_file, $chmod, $chgrp); + $this->pending_files[$local_file]["delete_source"] = true; + + return $remote_file; + } + + /** + * @return array a list of files pending an upload. + */ + public function pending(): array + { + return array_values($this->pending_files); + } + + /** + * @return array|null The current file being uploaded or failed to upload. Null when no upload was performed or when it succeeded. + */ + public function current(): ?array + { + return empty($this->current_file) || empty($this->pending_files) + ? null + : $this->pending_files[$this->current_file]; + } + + /** + * Uploads all files from the pending list. + * @return int a return code indicating whether upload was successful or stopped. + */ + public function upload(): int + { + $connection = Utils::requireThat($this->sftp->connected(), "The sftp client must be connected"); + $username = $connection["user"]; + + // Correct state when we are restarted after an error + if ($this->current_file) { + // Restore the remote path to where we originally have been. + if ($this->pending_base_path) { + $error = $this->sftp + ->clearError() + ->cd($this->pending_base_path) + ->lastError(); + + if ($error) { + Utils::log()->error("Cannot continue since changing to initial remote path '{$this->pending_base_path}' failed", $error); + return self::UPLOAD_ERROR; + } + } + + // Remove file that caused an error previously. + unset($this->pending_files[$this->current_file]); + } + + $this->pending_base_path = $remote_base_path = $this->sftp->pwd(); + $remote_files = []; + $remote_path = "."; + + // Collecting files to upload (sorted by target to reduce remote directory changes) + $files_to_upload = $this->pending(); + usort($files_to_upload, function (&$a, &$b) { + return $a["target"] <=> $b["target"]; + }); + + // Uploading the files + foreach ($files_to_upload as $file) { + + // Managing pending files. + $local_file = $this->current_file = $file["source"]; + + // Clear errors for processing next file. + $this->sftp->clearError(); + + try { + // Changing remote directory if required. + if (($target_dir = dirname($file["target"])) !== $remote_path) { + + $absolute_target_dir = $this->sftp->resolve($target_dir, $remote_base_path); + Utils::requireThat( + $absolute_target_dir && strpos($absolute_target_dir, $remote_base_path) === 0, + "Illegal target directory '$absolute_target_dir' is not below '$remote_base_path'"); + + $dir_names = preg_split('-/+-', substr($absolute_target_dir, strlen($remote_base_path)), 0, PREG_SPLIT_NO_EMPTY); + if (count($dir_names) == 1) { + $dir_names[0] = $absolute_target_dir; // Single dir: Use absolute path + } else { + $this->sftp->cd($remote_base_path); // None or multiple directories: Start from base path and create one by one as needed + } + + foreach ($dir_names as $dir) { + if ($error = $this->sftp->cd($dir)->lastError()) { + if ($error["file_not_found"]) { + Utils::log()->info("Creating remote directory: $dir"); + $this->sftp->clearError() + ->mkdir($dir) + ->cd($dir); + } else { + break; + } + } + } + + if ($error = $this->sftp->lastError()) { + Utils::log()->error("Failed to cd into '$target_dir'.", $error); + return self::UPLOAD_ERROR; + } + + $remote_path = $target_dir; + $remote_files = []; + } + + // Listing existing remote files + if (empty($remote_files)) { + $remote_files = $this->sftp->clearError()->ls(); + if ($error = $this->sftp->lastError()) { + Utils::log()->error("Failed listing remote files.", $error); + return self::UPLOAD_ERROR; + } + } + + // Preparing upload + $remote_filename = basename((empty($file["target"]) ? $local_file : $file["target"])); + $remote_file = $remote_files[$remote_filename] ?: ["type" => "-", "owner" => $username]; + $remote_is_file = $remote_file["type"] === "-"; + $remote_is_readonly = preg_match('/^-r-.r-.+$/', $remote_file["permissions"] ?: ""); + + // Check if a folder/socket/symlink, etc is in the way + if (!$remote_is_file) { + Utils::log()->error("Failed uploading file '{$local_file}' as there is a non-file in the way at '{$file["target"]}'"); + return self::UPLOAD_ERROR_NO_OVERWRITE; + } + + $chgrp = $file["group"] ?: ""; + $chgrp = preg_match('/^\d+$/', $chgrp) ? (string)$chgrp : false; + + $chmod = $file["mode"] ?: ""; + $chmod = preg_match('/^0\d{3}$/', $chmod) ? (string)$chmod : false; + + + // Initial upload when permissions are properly set. + $retry_with_permission_change = + $chmod !== false + && $remote_file["owner"] === $username + && isset($remote_files[$remote_filename]); + + if (!$remote_is_readonly) { + $preserve_times_and_mod = $chmod !== false; + + if ($error = $this->sftp->put($local_file, $remote_filename, $preserve_times_and_mod)->lastError()) { + Utils::log()->error("Failed uploading file '{$local_file}' to '{$file["target"]}'", $error); + + if ($error["permission_denied"] !== true) + $retry_with_permission_change = false; + + if ($retry_with_permission_change) { + Utils::log()->info("Retrying file '{$local_file}' to '{$file["target"]}' with adjusted permissions"); + } else { + return self::UPLOAD_ERROR_NO_PERMISSION; + } + } else { + $retry_with_permission_change = false; + } + } + + + // Second attempt when initial failed or was skipped due to write protection (only possible if we have chmod defined to reset permissions later) + if ($retry_with_permission_change) { + + $this->sftp->chmod($remote_filename, '0600'); + + if ($error = $this->sftp->put($local_file, $remote_filename)->lastError()) { + Utils::log()->error("Failed uploading file '{$local_file}' to '{$file["target"]}'", $error); + return self::UPLOAD_ERROR_NO_PERMISSION; + } + + } else if ($remote_is_readonly) { + Utils::log()->error("Failed uploading file '{$local_file}' to '{$file["target"]}'. Existing file is write protected."); + return self::UPLOAD_ERROR_NO_PERMISSION; + } + + + // Applying chmod / chgrp if requested. + if ($chmod) { + if ($error = $this->sftp->chmod($remote_filename, $chmod)->lastError()) { + Utils::log()->error("Failed chmod ($chmod) for '{$file["target"]}'", $error); + return self::UPLOAD_ERROR_CHMOD_FAILED; + } + } + + if ($chgrp) { + if ($error = $this->sftp->chgrp($remote_filename, $chgrp)->lastError()) { + Utils::log()->error("Failed chgrp ($chgrp) for '{$file["target"]}'", $error); + return self::UPLOAD_ERROR_CHGRP_FAILED; + } + } + } finally { + $this->deleteSourceIfRequested($local_file); + } + + unset($this->pending_files[$local_file]); + } + + $this->current_file = null; + + if (empty($this->pending_files)) + $this->temporaryFile(true); + + return self::UPLOAD_SUCCESS; + } + + private function deleteSourceIfRequested($file) + { + if (isset($this->pending_files[$file]) + && is_array($existing = $this->pending_files[$file]) + && $existing["delete_source"] === true) { + + unlink($existing["source"]); + } + } + + private function temporaryFile($delete_all = false) + { + static $shared_temporary_files; + static $shared_temporary_files_index_sequence = 0; + + // Maintain all generated files statically to ensure they are removed even when the destructor isn't called. + if (!is_array($shared_temporary_files)) { + $shared_temporary_files = []; + + register_shutdown_function(function () use (&$shared_temporary_files) { + $count = 0; + foreach ($shared_temporary_files as $temporary_files) { + if (!is_iterable($temporary_files)) + continue; + foreach ($temporary_files as $file) { + if (is_file($file)) { + unlink($file); + $count++; + } + } + } + + if ($count > 0) + Utils::log()->info("Removed $count files in shutdown hook instead of object destruction."); + + $shared_temporary_files = []; + }); + } + + $index = $this->temporary_files_index; + if ($index <= 0 || !is_array($shared_temporary_files[$index])) { + $index = $this->temporary_files_index = ++$shared_temporary_files_index_sequence; + $shared_temporary_files[$index] = []; + } + + $temporary_files = &$shared_temporary_files[$index]; + + + // Dealing with temp file creation or cleanup + if ($delete_all) { + foreach ($temporary_files as $file) { + if (is_file($file)) unlink($file); + } + + unset($shared_temporary_files[$index]); + + } else { + if ($file = tempnam(sys_get_temp_dir(), "sftp-upload-")) { + $file = realpath($file); + $temporary_files[] = $file; + Utils::requireThat(chmod($file, 0600), "failed setting user-only permissions on '$file'."); + return $file; + }; + } + + return false; + } +} diff --git a/security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/Utils.php b/security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/Utils.php new file mode 100644 index 000000000..89a89c5ae --- /dev/null +++ b/security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/Utils.php @@ -0,0 +1,143 @@ +error("FATAL: $message"); + throw new \AssertionError($message); + } + return $expression; + } + + /** + * Converts a relative to a normalized absolute path. + * @param string $file The relative file path to resolve against base. + * @param string $base The base path to use for resolving the file. + * @return bool|string The resolved path or false when impossible. + */ + public static function resolvePath(string $file, string $base = ".") + { + $combined_path = $file; + + if (empty($file) || $file[0] != DIRECTORY_SEPARATOR) { + + if (empty($base) || $base[0] != DIRECTORY_SEPARATOR) + $base = realpath(($base ?: ".")); + + $combined_path = $base . DIRECTORY_SEPARATOR . $file; + } + + $path = []; + + foreach (explode(DIRECTORY_SEPARATOR, $combined_path) as $part) { + if (empty($part) || $part === '.') + continue; + + if ($part !== '..') + array_push($path, $part); + else if (!empty($path)) + array_pop($path); + else + return false; + } + + return DIRECTORY_SEPARATOR . join(DIRECTORY_SEPARATOR, $path); + } +} diff --git a/security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/upload_sftp.php b/security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/upload_sftp.php index c04abaaa1..7e205490a 100644 --- a/security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/upload_sftp.php +++ b/security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/upload_sftp.php @@ -73,7 +73,8 @@ const COMMANDS = [ "test-connection" => [ "description" => "connects to the host and returns results as JSON", "options" => [ - "host:", "port::", "host-key::", "user:", "remote-path::", "identity-type::"], + "host:", "port::", "host-key::", "user:", "remote-path::", "identity-type::", + "chgrp::", "chmod::"], "implementation" => "commandTestConnection", ], @@ -108,13 +109,7 @@ const EXAMPLES = << ["default" => "{{name}}/ca.pem", "option" => "ca-name"], ]; -// Keys & bits -const DEFAULT_KEY_TYPE = "ecdsa"; - -const DEFAULT_IDENTITY_KEY_BITS = ["rsa" => 4096, "ecdsa" => 521]; -const IDENTITY_TYPES = ["rsa", "rsa_2048", "rsa_4096", "rsa_8192", "ecdsa", "ecdsa_256", "ecdsa_384", "ecdsa_521", "ed25519"]; -const DEFAULT_IDENTITY_TYPE = "ecdsa"; - +// Exit codes const EXITCODE_SUCCESS = 0; const EXITCODE_ERROR = 1; const EXITCODE_ERROR_NO_PERMISSION = 2; @@ -143,12 +132,23 @@ const EXITCODE_ERROR_UNKNOWN_COMMAND = 255; @include_once("certs.inc"); @include_once("util.inc"); -// -------------------------------------------------------------------------------------------------------------------- -// Main script logic +// Optional autoloader (for local dev environment) +if (!function_exists("log_error")) { + spl_autoload_register(function ($class_name) { + require_once(__DIR__ . "/../../../mvc/app/library/" . str_replace("\\", "/", $class_name) . ".php"); + }); +} +// Importing classes +use OPNsense\AcmeClient\SftpUploader; +use OPNsense\AcmeClient\SftpClient; +use OPNsense\AcmeClient\SSHKeys; +use OPNsense\AcmeClient\Utils; + +// Implementing logic function commandShowIdentity(array &$options): int { - $identity_type = trim(($options["identity-type"] ?: DEFAULT_IDENTITY_TYPE)); + $identity_type = trim(($options["identity-type"] ?: SSHKeys::DEFAULT_IDENTITY_TYPE)); $source_ip = trim(($options["source-ip"] ?: "")); $host = trim(($options["host"] ?: "")); @@ -164,7 +164,7 @@ function commandShowIdentity(array &$options): int return EXITCODE_SUCCESS; } else { - logger()->error("Failed getting identity. See log output for details."); + Utils::log()->error("Failed getting identity. See log output for details."); } return EXITCODE_ERROR; } @@ -175,12 +175,9 @@ function commandTestConnection(array &$options): int // Testing connection $sftp = connectWithServer($options, $error); - if ($result["success"] = ($sftp !== null)) { + if ($result["success"] = ($sftp !== null && $sftp->connected())) { $result["actions"][] = "connected"; - $result["remote"] = [ - "address" => $sftp->remote_address, - "path" => $sftp->pwd(), - ]; + $result["remote"] = array_merge($sftp->connected(), ["path" => $sftp->pwd()]); } else { $result = array_merge($result, $error); } @@ -188,17 +185,41 @@ function commandTestConnection(array &$options): int // Testing file upload if ($result["success"]) { $result["actions"][] = "upload-testing"; - $file = temporaryFile(); - $filename = "." . basename($file); - if ($error = $sftp->put($file, $filename)->lastError(3)) { - $result["success"] = false; - $result = array_merge($result, $error); - } else if ($error = $sftp->rm($filename)->lastError(3)) { - logger()->error("Failed removing upload test file '$filename'. Cause: " . json_encode($error, JSON_UNESCAPED_SLASHES)); + $uploader = new SftpUploader($sftp); + + $chgrp = $options["chgrp"] ?: false; + $chmod = isset($options["chmod"]) ? ($options["chmod"] ?: DEFAULT_CERT_MODE) : false; + $filename = $uploader->addContent("upload-test", "", 0, $chmod, $chgrp); + + $upload_result = $uploader->upload(); + $result["success"] = $upload_result === SftpUploader::UPLOAD_SUCCESS; + + if ($result["success"]) { + $result["actions"][] = "upload-tested"; + + } else { + if ($error = $sftp->lastError(3)) + $result = array_merge($result, $error); + + if ($upload_result === SftpUploader::UPLOAD_ERROR_CHGRP_FAILED) { + $result["chgrp_failed"] = true; + } else if ($upload_result === SftpUploader::UPLOAD_ERROR_CHMOD_FAILED) { + $result["chmod_failed"] = true; + } } - if ($result["success"]) $result["actions"][] = "upload-tested"; + $remove_file = in_array($upload_result, [ + SftpUploader::UPLOAD_SUCCESS, + SftpUploader::UPLOAD_ERROR_CHGRP_FAILED, + SftpUploader::UPLOAD_ERROR_CHMOD_FAILED]); + + if ($remove_file) { + if ($error = $sftp->clearError()->rm($filename)->lastError(3)) + Utils::log()->error("Failed removing upload test file '$filename'", $error); + } + + $sftp->close(); } echo json_encode($result, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT) . PHP_EOL; @@ -242,7 +263,7 @@ function commandUpload(array &$options): int return uploadCertificatesToHost($options); } else { - logger()->error("No work to do, neither --host nor --certificates is present."); + Utils::log()->error("No work to do, neither --host nor --certificates is present."); return EXITCODE_ERROR_NOTHING_TO_UPLOAD; } } @@ -251,132 +272,45 @@ function uploadCertificatesToHost(array $options): int { $sftp = connectWithServer($options, $error); if ($sftp === null) { - logger()->error("Aborting after connect failure."); - return $error["connect_failed"] ? EXITCODE_ERROR : EXITCODE_ERROR_NO_PERMISSION; + Utils::log()->error("Aborting after connect failure."); + return $error["connect_failed"] + ? EXITCODE_ERROR + : EXITCODE_ERROR_NO_PERMISSION; } - $username = $options["user"]; + try { + $uploader = new SftpUploader($sftp); - $remote_base_path = $sftp->pwd(); - $remote_files = []; - $remote_path = "."; + addFilesToUpload($options, $uploader); - $chmod = isset($options["chmod"]) ? ($options["chmod"] ?: DEFAULT_CERT_MODE) : false; - $chmod_key = isset($options["chmod-key"]) ? ($options["chmod-key"] ?: DEFAULT_KEY_MODE) : false; - $chgrp = $options["chgrp"] ?: false; + if (empty($uploader->pending())) + return EXITCODE_ERROR_NOTHING_TO_UPLOAD; - // Collecting files to upload (sorted by target to reduce remote directory changes) - $files_to_upload = getFilesToUpload($options); - usort($files_to_upload, function (&$a, &$b) { - return $a["target"] <=> $b["target"]; - }); + for ($max_restarts = 5; !empty($uploader->pending()) && $max_restarts > 0; $max_restarts--) { - // Uploading the files - foreach ($files_to_upload as $file) { - // Checking if source is valid. - if (!is_file($file["source"]) && is_readable($file["source"])) { - logger()->error("Skipping {$file["source"]}, it is not a file or not readable."); - continue; - } + $result = $uploader->upload(); - // Changing remote directory if required. - if (($target_dir = dirname($file["target"])) !== $remote_path) { + if ($result != SftpUploader::UPLOAD_SUCCESS) { + Utils::log()->error("Failed on " . json_encode($uploader->current(), JSON_UNESCAPED_SLASHES)); - $absolute_target_dir = $sftp->resolve($target_dir, $remote_base_path); - requireThat( - $absolute_target_dir && strpos($absolute_target_dir, $remote_base_path) === 0, - "Illegal target directory '$absolute_target_dir' is not below '$remote_base_path'"); + switch ($result) { + case SftpUploader::UPLOAD_ERROR_CHGRP_FAILED: + case SftpUploader::UPLOAD_ERROR_CHMOD_FAILED: + case SftpUploader::UPLOAD_ERROR_NO_OVERWRITE: + continue; - $dir_names = preg_split('-/+-', substr($absolute_target_dir, strlen($remote_base_path)), 0, PREG_SPLIT_NO_EMPTY); - if (count($dir_names) == 1) - $dir_names[0] = $absolute_target_dir; // Single dir: Use absolute path - else - $sftp->cd($remote_base_path); // None or multiple directories: Start from base path and create one by one as needed + case SftpUploader::UPLOAD_ERROR_NO_PERMISSION: + return EXITCODE_ERROR_NO_PERMISSION; - foreach ($dir_names as $dir) { - if ($error = $sftp->cd($dir)->lastError()) { - if ($error["file_not_found"]) { - logger()->info("Creating remote directory: $dir"); - $sftp->clearError() - ->mkdir($dir) - ->cd($dir); - } else { - break; - } + case SftpUploader::UPLOAD_ERROR: + return EXITCODE_ERROR; } - } - - if ($error = $sftp->lastError()) { - logger()->error("Failed to cd into '$target_dir'. Cause: " . json_encode($error, JSON_UNESCAPED_SLASHES)); - return EXITCODE_ERROR; } else { - $remote_path = $target_dir; - $remote_files = $sftp->ls(); - if ($error = $sftp->lastError()) { - logger()->error("Failed listing remote files. Cause: " . json_encode($error, JSON_UNESCAPED_SLASHES)); - return EXITCODE_ERROR; - } + break; } } - - // Preparing copy - $remote_filename = basename($file["target"]); - $remote_file = $remote_files[$remote_filename] ?: ["type" => "-", "owner" => $username]; - $remote_is_file = $remote_file["type"] === "-"; - $remote_is_readonly = preg_match('/^-r-.r-.+$/', $remote_file["permissions"] ?: ""); - - // Check if a folder/socket/symlink, etc is in the way - if (!$remote_is_file) { - logger()->error("Skipping file '{$file["source"]}' as there is a non-file in the way at '{$file["target"]}'."); - continue; - } - - $file_chmod = $file["is_key"] ? $chmod_key : $chmod; - $file_chmod = preg_match('/^0[0-9]{3}$/', $file_chmod) ? (string)$file_chmod : false; - $permission_change_retry_allowed = $file_chmod - && $remote_file["owner"] === $username - && isset($remote_files[$remote_filename]); - - // Initial upload when permissions are properly set. - if (!$remote_is_readonly) { - $preserve_times_and_mod = $file_chmod !== false; - if ($error = $sftp->put($file["source"], $remote_filename, $preserve_times_and_mod)->lastError()) { - logger()->error("Failed uploading file '{$file["source"]}' to '{$file["target"]}'. Cause: " . json_encode($error, JSON_UNESCAPED_SLASHES)); - - if ($error["permission_denied"] !== true) - $permission_change_retry_allowed = false; - - if ($permission_change_retry_allowed) - logger()->info("Retrying file '{$file["source"]}' to '{$file["target"]}' with adjusted permissions."); - } else { - $permission_change_retry_allowed = false; - } - } - - // Second upload when initial failed or was skipped due to write protection (only possible if we have chmod defined to reset permissions later). - if ($permission_change_retry_allowed) { - $sftp->chmod($remote_filename, '0600'); - if ($error = $sftp->put($file["source"], $remote_filename)->lastError()) { - logger()->error("Failed uploading file '{$file["source"]}' to '{$file["target"]}'. Cause: " . json_encode($error, JSON_UNESCAPED_SLASHES)); - return EXITCODE_ERROR_NO_PERMISSION; - } - } else if ($remote_is_readonly) { - logger()->error("Failed uploading file '{$file["source"]}' to '{$file["target"]}'. Existing file is write protected and --chmod[-key] options are missing."); - return EXITCODE_ERROR_NO_PERMISSION; - } - - // Applying Chmod / chgrp if requested. - if ($file_chmod) { - if ($error = $sftp->chmod($remote_filename, $file_chmod)->lastError()) - logger()->error("Failed chmod ($file_chmod) for '{$file["target"]}'. Cause: " . json_encode($error, JSON_UNESCAPED_SLASHES)); - } - - if (!empty($chgrp)) { - if ($error = $sftp->chgrp($remote_filename, $chgrp)->lastError()) - logger()->error("Failed chgrp ($chgrp) for '{$file["target"]}'. Cause: " . json_encode($error, JSON_UNESCAPED_SLASHES)); - } - - $sftp->clearError(); + } finally { + $sftp->close(); } return EXITCODE_SUCCESS; @@ -384,7 +318,7 @@ function uploadCertificatesToHost(array $options): int function connectWithServer(array $options, &$error): ?SftpClient { - $identity_type = trim(($options["identity-type"] ?: DEFAULT_IDENTITY_TYPE)); + $identity_type = trim(($options["identity-type"] ?: SSHKeys::DEFAULT_IDENTITY_TYPE)); $host = trim(($options["host"] ?: "")); $host_key = ($options["host-key"] ?: ""); $port = $options["port"] ?: 22; @@ -403,7 +337,7 @@ function connectWithServer(array $options, &$error): ?SftpClient if ($err = $sftp->cd($remote_path)->lastError()) { $error = $err; $error["change_home_dir_failed"] = true; - logger()->error("Failed cd into '{$remote_path}'. Cause: " . json_encode($err, JSON_UNESCAPED_SLASHES)); + Utils::log()->error("Failed cd into '{$remote_path}'", $err); return null; } } @@ -461,7 +395,7 @@ function getActionById($automation_id) function getOptionsById($automation_id, $silent = false) { - if (!$silent) logger()->info("Reading options from automation: $automation_id"); + if (!$silent) Utils::log()->info("Reading options from automation: $automation_id"); if (is_object($action = getActionById($automation_id))) { if ($action->enabled && "upload_sftp" === (string)$action->type) { @@ -478,89 +412,98 @@ function getOptionsById($automation_id, $silent = false) "cert-name" => trim((string)$action->sftp_filename_cert), "key-name" => trim((string)$action->sftp_filename_key), "ca-name" => trim((string)$action->sftp_filename_ca), + "certificates" => "", // defaults to all (= empty), may be overridden via CLI ]; } else if (!$silent) { - logger()->error("Ignoring disabled or invalid automation '$automation_id'"); + Utils::log()->error("Ignoring disabled or invalid automation '$automation_id'"); } } else { - logger()->error("No upload automation found with uuid = '$automation_id'"); + Utils::log()->error("No upload automation found with uuid = '$automation_id'"); } return false; } -function getFilesToUpload(array $options) +function addFilesToUpload(array $options, SftpUploader &$uploader) { - $files = []; - $cert_ids = preg_split('/[,;\s]+/', $options["certificates"] ?: "", 0, PREG_SPLIT_NO_EMPTY); + $chmod = isset($options["chmod"]) ? ($options["chmod"] ?: DEFAULT_CERT_MODE) : false; + $chmod_key = isset($options["chmod-key"]) ? ($options["chmod-key"] ?: DEFAULT_KEY_MODE) : false; + $chgrp = $options["chgrp"] ?: false; + + if (isset($options["certificates"])) { + $cert_ids = preg_split('/[,;\s]+/', $options["certificates"] ?: "", 0, PREG_SPLIT_NO_EMPTY); - if (class_exists("OPNsense\\Core\\Config")) { foreach (findCertificates($cert_ids) as $cert) { - if (isset($cert["content"])) { - foreach ($cert["content"] as $name => $content) { - $source = temporaryFile(); - $ok = file_put_contents($source, $content); - if (!$ok) { - logger()->error("Ignoring upload for cert '{$cert["name"]}', since the content cannot be prepared or is empty."); - continue; - } + if (!isset($cert["content"])) { + Utils::log()->error("Ignoring upload for cert '{$cert["name"]}', since it is not available in trust storage."); + continue; + } - if (($time = intval($cert["updated"])) && $time > 0) - touch($source, $time); + foreach ($cert["content"] as $name => $content) { - // Build the upload name - $target_path = requireThat(UPLOAD_NAME_TEMPLATES[$name], "No upload template defined for '{$name}'"); - $target_path = stripcslashes($options[$target_path["option"]] ?: $target_path["default"]); - - $target_path = join("/", array_map( - function ($path_part) use (&$cert) { - // Replace template params "{{.+}}" & "%s" - $path_part = preg_replace_callback( - ['/%s/', '/{{([^}]+?)}}/'], - function ($m) use (&$cert) { - $index = $m[0] == '%s' ? "name" : trim($m[1]); - - return in_array($index, ["name", "id", "updated"]) - ? stripcslashes($cert[$index]) - : "__unknown-template-param__{$index}__"; - }, - $path_part); - - // Sanitize user input. Allow unicode chars, numbers and some special characters [_-@.]. - // Also replace all ".." with "." to avoid upwards tree traversal. - return preg_replace(['/\.+/', '/[^\w\d_\-@.]+/uim'], ['.', '-'], trim($path_part)); - }, - preg_split('-[/\\\\]+-', $target_path, 0, PREG_SPLIT_NO_EMPTY))); - - // Add the file to upload (if valid) - if (!empty($target_path) - && preg_match('-^(?!/).+?(? $source, "target" => $target_path]; - $file["is_key"] = ($name === "key"); - $files[] = $file; - } else { - logger()->error("Cannot add '{$name}.pem' since the upload path '$target_path' is invalid."); - } + if (empty($content)) { + Utils::log()->error("Content for '{$name}.pem' in cert '{$cert["name"]}' is empty, skipping it."); + continue; + } + + // Build the upload name + $target_path = requireThat(UPLOAD_NAME_TEMPLATES[$name], "No upload template defined for '{$name}'"); + $target_path = stripcslashes($options[$target_path["option"]] ?: $target_path["default"]); + + $target_path = join("/", array_map( + function ($path_part) use (&$cert) { + // Replace template params "{{.+}}" & "%s" + $path_part = preg_replace_callback( + ['/%s/', '/{{([^}]+?)}}/'], + function ($m) use (&$cert) { + $index = $m[0] == '%s' ? "name" : trim($m[1]); + + return in_array($index, ["name", "id", "updated"]) + ? stripcslashes($cert[$index]) + : "__unknown-template-param__{$index}__"; + }, + $path_part); + + // Sanitize user input. Allow unicode chars, numbers and some special characters [_-@.]. + // Also replace all ".." with "." to avoid upwards tree traversal. + return preg_replace(['/\.+/', '/[^\w\d_\-@.]+/uim'], ['.', '-'], trim($path_part)); + }, + preg_split('-[/\\\\]+-', $target_path, 0, PREG_SPLIT_NO_EMPTY))); + + + // Add the file to upload (if valid) + if (!empty($target_path) + && preg_match('-^(?!/).+?(?addContent($content, $target_path, $cert["updated"], $mod, $chgrp); + + } else { + Utils::log()->error("Cannot add '{$name}.pem' since the upload path '$target_path' is invalid."); } - } else { - logger()->error("Ignoring upload for cert '{$cert["name"]}', since it is not available in trust storage."); } } + + if (empty($uploader->pending())) + Utils::log()->error("Didn't find any certificates to upload (cert-ids: " . (empty($cert_ids) ? "*all*" : join(", ", $cert_ids)) . ")."); + + } else if (isset($options["files"])) { + $files = preg_split('/[,;\s]+/', $options["files"] ?: "", 0, PREG_SPLIT_NO_EMPTY); + foreach ($files as $file) { + $uploader->addFile($file, "", $chmod, $chgrp); + }; + + if (empty($uploader->pending())) + Utils::log()->error("Didn't files to upload (files: " . join(", ", $files) . ")."); + } else { - // Allow to specify "--files" only if we're not running in opnsense environment (for development only). - if (isset($options["files"])) { - foreach (preg_split('/[,;\s]+/', $options["files"] ?: "", 0, PREG_SPLIT_NO_EMPTY) as $file) { - $files[] = ["source" => $file, "target" => "upload-test/" . basename($file)]; - }; - } + Utils::log()->error("Neither '--certificates' nor '--files' was specified. Have nothing to upload."); } - - if (empty($files)) - logger()->error("Didn't find any certificates to upload (cert-ids: " . (empty($cert_ids) ? "all" : join(", ", $cert_ids)) . ")."); - - return $files; } function findCertificates(array $certificate_ids_or_names, $load_content = true): array @@ -584,7 +527,7 @@ function findCertificates(array $certificate_ids_or_names, $load_content = true) if ($cert->enabled == 0) { if (!empty($certificate_ids_or_names)) - logger()->error("Certificate '{$name}' (id: $id) is disabled, skipping it."); + Utils::log()->error("Certificate '{$name}' (id: $id) is disabled, skipping it."); continue; } @@ -665,11 +608,7 @@ function main() help(); } else { if (isset($options["log"])) - logger(true)->info("Logging to stdout enabled"); - - register_shutdown_function(function () { - temporaryFile(true); - }); + Utils::log(true)->info("Logging to stdout enabled"); $options = array_filter($options, function ($value) { return !is_string($value) @@ -677,13 +616,15 @@ function main() }); if (isset($options["automation-id"])) - $options = array_merge($options, getOptionsById($options["automation-id"])); + $options = array_merge(getOptionsById($options["automation-id"]), $options); if (is_callable($runner = $command["implementation"])) { $code = $runner($options); + if ($code != EXITCODE_SUCCESS) { - logger()->error("Command execution failed, exit code $code. Last input was: " . json_encode($options, JSON_UNESCAPED_SLASHES)); + Utils::log()->error("Command execution failed, exit code $code. Last input was: " . json_encode($options, JSON_UNESCAPED_SLASHES)); } + exit(isset($options["no-error"]) ? EXITCODE_SUCCESS : $code); } else { exit(EXITCODE_ERROR_UNKNOWN_COMMAND); @@ -694,842 +635,21 @@ function main() help(); } else { $cmd = join(" ", $argv); - logger()->error("Parsing of '$cmd' failed at argument '{$argv[$index]}'"); + Utils::log()->error("Parsing of '$cmd' failed at argument '{$argv[$index]}'"); } exit(1); } } - -// -------------------------------------------------------------------------------------------------------------------- -// Utility functions - -interface ILogger -{ - function info($message); - - function error($message); -} - -function &logger($reconfigure_to_stdout = false): ILogger -{ - static $logger; - if (!$logger || $reconfigure_to_stdout) { - if (!$reconfigure_to_stdout && function_exists("log_error")) { - $logger = new class implements ILogger - { - function info($message) { syslog(SYSLOG_INFO_LEVEL, basename(__FILE__) . ": INFO: $message"); } - - function error($message) { log_error($message); } - }; - } else { - $logger = new class implements ILogger - { - function info($message) { echo "INFO: {$message}" . PHP_EOL; } - - function error($message) { echo "ERROR: {$message}" . PHP_EOL; } - }; - } - } - return $logger; -} - function requireThat($expression, $message) { - if (!$expression) { - logger()->error("FATAL: $message"); + try { + Utils::requireThat($message, $message); + } catch (\AssertionError $e) { exit(EXITCODE_ERROR); } return $expression; } -function resolvePath($file, $base = ".") -{ - if (!$base || $base[0] != DIRECTORY_SEPARATOR) - $base = realpath(($base ?: ".")); - - $path = []; - $combined_path = ((!empty($file) && $file[0] == DIRECTORY_SEPARATOR) ? $file : $base . DIRECTORY_SEPARATOR . $file); - foreach (explode(DIRECTORY_SEPARATOR, $combined_path) as $part) { - if (empty($part) || $part === '.') - continue; - if ($part !== '..') - array_push($path, $part); - else if (!empty($path)) - array_pop($path); - else - return false; - } - - return DIRECTORY_SEPARATOR . join(DIRECTORY_SEPARATOR, $path); -} - -function temporaryFile($delete_all = false) -{ - static $__temporary_files = []; - if ($delete_all) { - foreach ($__temporary_files as $file) - unlink($file); - $__temporary_files = []; - } else { - if ($file = tempnam(sys_get_temp_dir(), "sftp-upload-")) { - $file = realpath($file); - $__temporary_files[] = $file; - requireThat(chmod($file, 0600), "failed setting user-only permissions on '$file'."); - return $file; - }; - } - return false; -} - - -// -------------------------------------------------------------------------------------------------------------------- -// Classes - - -/** - * Wrapper around the 'sftp' commandline client. - */ -class SftpClient -{ - public $remote_address; - private $identity_type; - - /* @var false|array */ - private $failed_status; - /* @var SSHKeys */ - private $ssh_keys; - /* @var null|Process */ - private $process = null; - /* @var null|string */ - private $pwd = null; - - public function __construct($config_path, $identity_type = DEFAULT_IDENTITY_TYPE) - { - $this->ssh_keys = new SSHKeys($config_path); - $this->identity_type = $identity_type; - } - - public function __destruct() - { - $this->close(); - } - - public function connect($host, $username, $host_key = "", $port = 22) - { - if (empty(trim($host)) || empty(trim($username))) { - $this->failed_status = ["invalid_parameters" => true]; - logger()->error("Failed connecting to '$host'. Hostname or username is missing."); - return false; - } - - $trust = $this->ssh_keys->trustHost($host, $host_key, $port); - if ($trust["ok"] !== true) { - logger()->error("Failed establishing trust in '$host'; Cause: {$trust["error"]}"); - unset($trust["ok"]); - $this->failed_status = array_merge($trust, ["host_not_trusted" => true]); - return false; - } else { - $host = $trust["host"]; - } - - // Building sftp command. - $cmd = [ - "sftp", - "-P", $port, - "-oUser=$username", - "-oUserKnownHostsFile={$this->ssh_keys->knownHostsFile()}", - ]; - - // Handle client side identity - $identity = $this->ssh_keys->getIdentity($this->identity_type, true); - if (is_file($identity) && is_readable($identity)) { - array_push($cmd, - "-i", $identity, - "-oPreferredAuthentications=publickey"); - } else { - logger()->error("Failed adding client identity ($identity). Connect will likely fail."); - } - - // Adding the host - array_push($cmd, "$host"); - - // Creating the sftp process - if ($this->process = Process::open($cmd)) { - $this->processAvailableInput(120, 1); - if ($error = $this->lastError()) { - logger()->error("Failed connecting to '$host' (user: '$username'). Cause: " . json_encode($error)); - return false; - } - $this->remote_address = strpos($host, ':') ? "[$host]:$port" : "$host:$port"; - return true; - } - return false; - } - - private function processAvailableInput(float $timeout = 0, $expected_lines = 0, Callable $lines_consumer = null, $remaining_timeout = 0) - { - requireThat($this->process !== null, "SFTP: process not connected"); - - static $expected_errors = [ - ["host_not_resolved", /* => */ '/.*not resolve.*/i'], - ["host_not_trusted", /* => */ '/.*IDENTIFICATION HAS CHANGED.*/i'], - ["connection_refused", /* => */ '/.*connection refused.*/i'], - ["connection_closed", /* => */ '/.*connection closed.*/i'], - ["network_timeout", /* => */ '/.*timed out.*/i'], - ["network_unreachable", /* => */ '/.*network.+unreachable.*/i'], - ["permission_denied", /* => */ '/.*permission denied.*/i'], - ["file_not_found", /* => */ '/.*(no such|not found).*/i'], - ["failure", /* => */ '/.*(error|failure|you must supply).*/i'], - ]; - - while (($line = $this->process->get($timeout)) !== false) { - foreach ($expected_errors as $ee) { - if (preg_match($ee[1], $line)) { - $this->failed_status = [$ee[0] => true, "error" => trim($line)]; - break; - } - } - - $consumed = ($lines_consumer && $lines_consumer($line) === true); - if (!$consumed) - logger()->info("SFTP: " . rtrim($line)); - - if (!$lines_consumer || $consumed) { - if (--$expected_lines <= 0) $timeout = $remaining_timeout; - } - } - } - - public function close() - { - if (($p = $this->process) !== null) { - $p->put("exit"); - $p->closeInput(); - - $this->processAvailableInput(1.5); - $p->close(); - - $this->process = null; - - if ($this->failed_status && $this->failed_status["connection_closed"]) - $this->clearError(); - } - } - - public function lastError($timeout = 0.5) - { - if ($this->failed_status === false) - $this->processAvailableInput($timeout); - return $this->failed_status; - } - - public function clearError() - { - $this->failed_status = false; - return $this; - } - - public function ls() - { - $files = []; - $this->processAvailableInput(); - $this->process->put("ls -la"); - - $regex = '/^([bcdlsp\-][rwx\-]{9}[+@]?)\s+[0-9]+\s+([^\s]+)\s+([^\s]+)\s+([0-9]+)\s+(\w+\s+[0-9]+\s+[0-9:]+)\s+(.+)$/'; - $this->processAvailableInput(30, 2, function ($line) use (&$files, $regex) { - if (preg_match($regex, $line, $matches)) { - $filename = trim(stripcslashes($matches[6])); // decodes octal UTF-8 sequences - $files[$filename] = [ - "type" => $matches[1][0], - "permissions" => $matches[1], - "owner" => $matches[2], - "group" => $matches[3], - "size" => intval($matches[4]), - "mtime" => strtotime($matches[5]) - ]; - return true; - } - return false; - }, 1); - - return $files; - } - - public function pwd() - { - if ($this->pwd === null) { - $remote_path = false; - - $this->processAvailableInput(); - $this->process->put("pwd"); - $this->processAvailableInput(30, 1, function ($line) use (&$remote_path) { - if (preg_match('/^.+directory:\s(.+)$/i', $line, $matches)) { - $remote_path = trim(stripcslashes($matches[1])); - return true; - } - return false; - }); - - $this->pwd = $remote_path; - } - - return $this->pwd; - } - - public function resolve($remote_path, $remote_pwd = null) - { - if (($pwd = ($remote_pwd ?: $this->pwd())) !== false) { - $remote_path = resolvePath($remote_path, str_replace("/", DIRECTORY_SEPARATOR, $pwd)); - $remote_path = str_replace("\\", "/", $remote_path); - return $remote_path; - } - return false; - } - - public function get($remote_file, $local_file = "") - { - $this->processAvailableInput(); - $this->process->put("get " - . escapeshellarg($remote_file) - . (empty($local_file) ? "" : " " . escapeshellarg($local_file))); - $this->processAvailableInput(30, 2); - return $this; - } - - public function put($local_file, $remote_file = "", $preserve = true) - { - if (is_file($local_file)) { - $this->processAvailableInput(); - $this->process->put("put " . ($preserve ? "-p " : "") - . escapeshellarg($local_file) - . (empty($remote_file) ? "" : " " . escapeshellarg($remote_file))); - $this->processAvailableInput(30, 2); - } else { - logger()->info("put: File $local_file doesn't exist."); - $this->failed_status = ["file_not_found" => true, "error" => $local_file]; - } - - return $this; - } - - public function mkdir($remote_path) - { - if (($remote_path = $this->resolve($remote_path)) !== false) { - $this->process->put("mkdir " . escapeshellarg($remote_path)); - $this->processAvailableInput(30, 1); - } - return $this; - } - - public function cd($remote_path) - { - if (($remote_path = $this->resolve($remote_path)) !== false) { - $this->clearError(); - $this->process->put("cd " . escapeshellarg($remote_path)); - - $this->processAvailableInput(30, 1); - $error = $this->lastError(); - $pwd = false; - $this->pwd = null; - - if ($error || $remote_path !== ($pwd = $this->pwd())) { - $this->failed_status = array_merge(($error ?: []), [ - "failure" => true, - "error" => "Failed changing path to '$remote_path' (pwd: '$pwd'); Cause: {$error["error"]}" - ]); - } - } - return $this; - } - - public function chmod($remote_file, $mode) - { - if (($remote_file = $this->resolve($remote_file)) !== false) { - $this->processAvailableInput(); - $this->process->put("chmod " . escapeshellarg($mode) . " " . escapeshellarg($remote_file)); - $this->processAvailableInput(30, 2); - } - return $this; - } - - public function chgrp($remote_file, $group_id) - { - if (($remote_file = $this->resolve($remote_file)) !== false) { - $this->processAvailableInput(); - $this->process->put("chgrp " . escapeshellarg($group_id) . " " . escapeshellarg($remote_file)); - $this->processAvailableInput(30, 2); - } - return $this; - } - - public function rm($remote_file) - { - if (($remote_file = $this->resolve($remote_file)) !== false) { - $this->processAvailableInput(); - $this->process->put("rm " . escapeshellarg($remote_file)); - $this->processAvailableInput(30, 2); - } - return $this; - } -} - -/** - * Utility class for managing SSH host (known_hosts) and identity keys to be used in {@see SftpClient}. - */ -class SSHKeys -{ - private $config_path; - private $known_hosts_file; - - public function __construct($config_path) - { - if (!is_dir($config_path)) { - $dir_created = mkdir($config_path, CONFIG_PATH_CREATE_MODE, true); - requireThat($dir_created, "Failed creating directory '$config_path' with permission " . CONFIG_PATH_CREATE_MODE); - } - - $this->config_path = realpath($config_path); - $this->known_hosts_file = resolvePath("known_hosts", $this->config_path); - } - - public function knownHostsFile() - { - if (!is_file($this->known_hosts_file)) { - $file_created = - touch($this->known_hosts_file) - && chmod($this->known_hosts_file, KNOWN_HOSTS_FILE_CREATE_MODE); - - requireThat($file_created, "Failed creating file '{$this->known_hosts_file}' with permission " . KNOWN_HOSTS_FILE_CREATE_MODE); - } - - return $this->known_hosts_file; - } - - public function trustHost($host, $host_key = "", $port = 22, $no_modification_allowed = false): array - { - requireThat(!empty(trim($host)), "Hostname must not be empty."); - - // Convert the specified host_key to a data structure that can be compared - if (empty($host_key = trim($host_key))) { - $host_key = false; - } else { - $host_key = self::getHostKeyInfo($host_key); - if ($host_key === false) - return ["ok" => false, "error" => "Invalid host_key specified."]; - } - - - // Check our current known_host file - $addKeyInfo = function (array &$key_list) { - foreach ($key_list as &$item) { - $item["key_info"] = self::getHostKeyInfo($item["host_key"]); - } - return array_filter($key_list, function (&$item) { - return $item["key_info"] !== false; - }); - }; - - $known_keys = $addKeyInfo($this->getKnownHostKey($host)); - - // Find known_hosts item with same hostname - $known_by_host = array_reduce($known_keys, function ($found, $key) use ($host) { - return (!$found && !empty(trim($key["host"])) && strcasecmp(trim($host), trim($key["host"])) == 0) - ? $key - : $found; - }, false); - - // Find known_hosts item with same public host-key - $known_by_key = array_reduce($known_keys, function ($found, $key) use ($host_key) { - return (!$found && $host_key && $host_key === $key["key_info"]) - ? $key - : $found; - }, false); - - - // Updating $host and $host_key from known_hosts and check if we need to update known_hosts. - if ($host_key === false && $known_by_host) { - logger()->info("No host key specified, using existing known_hosts entry for '$host'"); - $host_key = $known_by_host["key_info"]; - } - - $is_key_known = false; - if ($known_by_host && $host_key && $host_key === $known_by_host["key_info"]) { - $is_key_known = true; - } else if ($known_by_key) { - if (strcasecmp(trim($host), trim($known_by_key["host"])) != 0) { - logger()->info("Host key is in known_hosts but hostname differs. Changing '$host' to '{$known_by_key["host"]}'."); - $host = $known_by_key["host"]; - } - $is_key_known = true; - } - - - // Check if we don't have a matching known_hosts entry and add or update it as required. - if (!$is_key_known && !$no_modification_allowed) { - $key_type = $host_key ? $host_key["key_type"] : DEFAULT_KEY_TYPE; - - $remote_host_keys = $addKeyInfo($this->queryHostKey($host, $key_type, $port)); - $matching_remote_host_keys = array_filter($remote_host_keys, function ($key) use ($host_key) { - return $key["key_info"] !== false && (!$host_key || $host_key === $key["key_info"]); - }); - - if (!empty($matching_remote_host_keys)) { - if ($known_by_host) { - logger()->info("Removing known_hosts entry with differing key for '{$known_by_host["host"]}' as it is in the way."); - $this->removeKnownHost($known_by_host["host"]); - } - - foreach ($matching_remote_host_keys as $key) { - logger()->info("Adding known_hosts entry: " . json_encode($key["key_info"], JSON_UNESCAPED_SLASHES)); - $ok = file_put_contents($this->knownHostsFile(), $key["host_key"] . PHP_EOL, FILE_APPEND); - if (!$ok) - logger()->error("Failed adding known_hosts entry {$key["host_key"]}"); - } - - // Verify that known_hosts contains the correct keys after adding them (using recursion). - return $this->trustHost($host, $matching_remote_host_keys[0]["host_key"], $port, true); - - } else { - if (empty($remote_host_keys)) { - $msg = "No connection to '$host'; Failed querying host key from server."; - } else { - $msg = "Key mismatch for '$host'; " - . "The expected key (" . json_encode($host_key) . ") was not found in (" . json_encode($remote_host_keys) . ")"; - } - return ["ok" => false, "error" => $msg]; - } - } - - - if ($is_key_known) { - return ["ok" => true, "host" => $host, "key_info" => $host_key]; - } else { - return ["ok" => false, "error" => "Host unknown and remote key cannot be queried."]; - } - } - - public static function getHostSearchList($host) - { - $search_list = [($host = strtolower($host))]; - - // Add IP-address to search list (IPv4 only) - $has_ip = ($ip = gethostbyname($host)) - && ($ip !== $host || preg_match('/^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$/', $ip)); - - if ($has_ip) - $search_list[] = strtolower($ip); - - // Add FQDN to search list if reverse lookup provides a valid one. - $has_fqdn = $has_ip - && ($reverse_fqdn = gethostbyaddr($ip)) - && $reverse_fqdn !== $ip - && gethostbyname($reverse_fqdn) === $ip; - - if ($has_fqdn && isset($reverse_fqdn)) - $search_list[] = strtolower($reverse_fqdn); - - // Build unique search list (dedup list) - $search_list = array_filter($search_list, function ($value, $index) use (&$search_list) { - return !empty(trim($value)) && array_search($value, $search_list) == $index; - }, ARRAY_FILTER_USE_BOTH); - - return $search_list; - } - - public static function queryHostKey($host, $key_type = DEFAULT_KEY_TYPE, $port = 22) - { - $keys = []; - $failed = false; - $names = join(",", self::getHostSearchList($host)); - - if (!empty($names) && ($p = Process::open(["ssh-keyscan", "-p", $port, "-t", $key_type, $names]))) { - $lines = []; - while (($line = $p->get(60)) !== false) { - $line = trim($line); - if (empty($line) || $line[0] == "#") - continue; - - if (preg_match('/.*(connect|write|broken|no route|not known).*/i', $line)) - $failed = true; - - $lines[] = $line; - } - - if ($p->close() == 0 && !$failed) { - foreach ($lines as $line) { - $keys[] = ["host_key" => $line]; - } - } else { - logger()->error("Failed querying public keys for [$names] ($host / $key_type). " - . "Exit code: {$p->exitCode} (failed flag: $failed) ; " . PHP_EOL - . "ssh-keyscan: " . join(PHP_EOL . "ssh-keyscan: ", $lines)); - } - } - - if (empty($keys)) - logger()->info("Couldn't fetch public host key ($key_type) from $host"); - - return $keys; - } - - - public function getKnownHostKey($host) - { - $keys = []; - foreach (self::getHostSearchList($host) as $name_or_ip) { - if ($p = Process::open(["ssh-keygen", "-F", $name_or_ip, "-f", $this->knownHostsFile()])) { - $lines = []; - while (($line = $p->get()) !== false) { - $line = trim($line); - if (empty($line) || $line[0] == "#") - continue; - - $lines[] = $line; - } - - if ($p->close() == 0) { - $keys[] = ["host" => $name_or_ip, "host_key" => $lines[0]]; - } else if ($p->exitCode != 1 /* 1 == NOT_FOUND */) { - logger()->error("Failed querying known hosts for $name_or_ip ($host). Return code was: {$p->exitCode}" - . " ; " . PHP_EOL . join(PHP_EOL, $lines)); - } - } - } - - if (empty($keys)) - logger()->info("Didn't find $host in known_hosts"); - - return $keys; - } - - public function removeKnownHost($host) - { - $ok = false; - if ($p = Process::open(["ssh-keygen", "-R", $host, "-f", $this->knownHostsFile()])) { - $ok = $p->close() === 0; - if (!$ok) - logger()->error("Failed removing known hosts for $host. Return code was: {$p->exitCode}"); - } - return $ok; - } - - public static function getHostKeyInfo($host_key) - { - if ($p = Process::open(["ssh-keygen", "-l", "-f", "-"])) { - $p->put($host_key); - $p->closeInput(); - - if (($hash = $p->get()) && preg_match('/^([0-9]+) (.+?) .+? \(([^()]+)\)$/', $hash, $matches)) { - return ["hash" => $matches[2], "key_type" => $matches[3], "key_length" => $matches[1]]; - } else { - logger()->error("Unsupported hash type: $hash"); - } - } - - logger()->error("Failed getting hash for host_key"); - return false; - } - - // Returns the path to the public identity key file, generating it if missing (optionally returns the private key path) - public function getIdentity($identity_type = DEFAULT_IDENTITY_TYPE, $private = false) - { - requireThat(in_array($identity_type, IDENTITY_TYPES), "Identity type $identity_type unknown."); - - list($key_type, $key_size) = explode('_', $identity_type, 2); - if (!$key_size && DEFAULT_IDENTITY_KEY_BITS[$key_type] > 0) - $key_size = DEFAULT_IDENTITY_KEY_BITS[$key_type]; - - $identity_path = "{$this->config_path}/id.{$identity_type}"; - - if (!file_exists($identity_path)) { - $generate_key = [ - "ssh-keygen", "-v", - "-f", $identity_path, - "-t", $key_type, - "-N", "", - ]; - - if (intval($key_size) > 0) - array_push($generate_key, "-b", $key_size); - - if ($p = Process::open($generate_key)) { - while (($line = $p->get(10)) !== false) { - logger()->info("SSH keygen: $line"); - } - - requireThat($p->close() == 0, - "Failed generating identity $identity_path: Error code: {$p->exitCode}" . PHP_EOL - . "Command: " . join(" ", $generate_key)); - } - } - - return $private ? $identity_path : "{$identity_path}.pub"; - } - - public static function getIdentityRestrictions($host = "", $source_ip = "") - { - $restrictions = ['restrict', 'command="internal-sftp"']; - - $restrict_ip = empty(trim($source_ip)) - ? (empty(trim($host)) ? false : self::getOutgoingIpFor($host)) - : $source_ip; - - if ($restrict_ip) - $restrictions[] = 'from="' . $restrict_ip . '"'; - - return join(",", $restrictions); - } - - public static function getOutgoingIpFor($host) - { - $ip = gethostbyname($host); - $interface = null; - - if ($p = Process::open(["route", "-n", "get", $ip])) { - while (($line = $p->get(10)) !== false) - if (preg_match('/\s*interface:\s*([^\s]+).*$/', $line, $matches)) { - $interface = $matches[1]; - } - } - - if ($interface && $p = Process::open(["ifconfig", $interface, "inet"])) { - while (($line = $p->get(10)) !== false) - if (preg_match('/\s*inet\s+([^\s]+)\s+netmask.*/', $line, $matches)) { - return $matches[1]; - } - } - - return false; - } -} - -/** - * Utility class to execute shell processes and handle their IO. - */ -class Process -{ - private $handle; - private $inputs; - private $outputs; - - public $exitCode = null; - - // Starts the specified process and returns an object to manage it. - public static function open(array $cmd, $cwd = null, $env = null): ?Process - { - global $__process_terminate_hook, $__open_processes; - if (!is_array($__open_processes)) - $__open_processes = []; - - // Ensure we never leave zombies around: Hooking into script shutdown and kill processes that are still running. - if (!$__process_terminate_hook) { - register_shutdown_function($__process_terminate_hook = function () { - global $__open_processes; - foreach ($__open_processes as $handle) { - if (is_resource($handle)) { - logger()->error("Terminating process: " . json_encode(proc_get_status($handle))); - @proc_terminate($handle); - } - } - }); - } - - $p = new Process($cmd, $cwd, $env); - return $p->isRunning() ? $p : null; - } - - public function __construct($cmd, $cwd = null, $env = null) - { - $cmd = join(" ", array_map(function ($v) { - return escapeshellarg($v); - }, $cmd)); - - $spec = [0 => ["pipe", "r"], 1 => ["pipe", "w"], 2 => ["pipe", "w"]]; - $this->handle = proc_open($cmd, $spec, $pipes, $cwd, $env); - - if (is_resource($this->handle)) { - $this->outputs = $pipes; - $this->inputs = [array_shift($this->outputs)]; - - foreach ($this->outputs as $stream) - stream_set_blocking($stream, false); - - global $__open_processes; - $__open_processes[] = $this->handle; - } else { - logger()->error("Failed opening '$cmd' in '$cwd'"); - } - } - - public function __destruct() - { - $this->close(); - if ($this->isRunning()) - $this->close(true); - } - - public function get($timeout = 5, $max_length = 8192, $ending = PHP_EOL) - { - $readables = array_filter($this->outputs, function ($stream) { - return is_resource($stream) && !feof($stream); - }); - - $micros = intval(($timeout - floor($timeout)) * 1000000); - $can_read = !empty($readables) && stream_select($readables, $w = [], $e = [], $timeout, $micros); - $stream = array_reduce(($can_read ? $readables : []), function ($a, $b) { - return is_resource($a) && !feof($a) ? $a : $b; - }, null); - - return is_resource($stream) - ? stream_get_line($stream, $max_length, $ending) - : false; - } - - public function put($data, $append = PHP_EOL) - { - if ($this->isRunning() && is_resource($stdin = $this->inputs[0]) && !feof($stdin)) { - fwrite($stdin, $data); - if ($append) - fwrite($stdin, $append); - } - } - - public function closeInput() - { - if (!feof($stdin = $this->inputs[0])) fclose($stdin); - } - - public function close($force = false) - { - global $__open_processes; - - // Read up-to 10k remaining lines from STDOUT/ERR to release locks before closing. - for ($i = 0; ($line = $this->get(0)) && $i < 10000; $i++) { - logger()->error("WARN: process: $line"); - }; - - if ($this->isRunning()) - $this->exitCode = ($force ? proc_terminate($this->handle) : proc_close($this->handle)); - - if (!$this->isRunning() && in_array($this->handle, $__open_processes)) - $__open_processes = array_diff($__open_processes, [$this->handle]); - - return $this->exitCode; - } - - public function isRunning() - { - $status = is_resource($this->handle) ? proc_get_status($this->handle) : false; - if (is_array($status)) { - if (!$this->exitCode && $this->exitCode !== 0 && !$status["running"]) - $this->exitCode = $status["exitcode"]; - return $status["running"]; - } - return false; - } -} - - -// Calling main if we have been called via CLI -if (isset($GLOBALS["argc"])) main(); +// Running the main script +main(); From 96537b436f0e47333c7ae0c09b54d42aae5b3928 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Kellerer?= Date: Sat, 17 Aug 2019 16:50:30 +0200 Subject: [PATCH 6/9] AcmeClient: Added chmod/chgrp verification to SFTP test connection. --- .../controllers/OPNsense/AcmeClient/Api/ActionsController.php | 2 +- .../opnsense/mvc/app/views/OPNsense/AcmeClient/actions.volt | 4 ++-- .../opnsense/service/conf/actions.d/actions_acmeclient.conf | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/security/acme-client/src/opnsense/mvc/app/controllers/OPNsense/AcmeClient/Api/ActionsController.php b/security/acme-client/src/opnsense/mvc/app/controllers/OPNsense/AcmeClient/Api/ActionsController.php index a9be0c84e..c40ced752 100644 --- a/security/acme-client/src/opnsense/mvc/app/controllers/OPNsense/AcmeClient/Api/ActionsController.php +++ b/security/acme-client/src/opnsense/mvc/app/controllers/OPNsense/AcmeClient/Api/ActionsController.php @@ -91,7 +91,7 @@ class ActionsController extends ApiMutableModelControllerBase { if ($response = $this->callBackend( ["test-sftp-connection"], - ["sftp_host", "sftp_host_key", "sftp_port", "sftp_user", "sftp_identity_type", "sftp_remote_path"])) { + ["sftp_host", "sftp_host_key", "sftp_port", "sftp_user", "sftp_identity_type", "sftp_remote_path", "sftp_chmod", "sftp_chgrp"])) { return $response; } diff --git a/security/acme-client/src/opnsense/mvc/app/views/OPNsense/AcmeClient/actions.volt b/security/acme-client/src/opnsense/mvc/app/views/OPNsense/AcmeClient/actions.volt index 5edb1fed9..1585373c1 100644 --- a/security/acme-client/src/opnsense/mvc/app/views/OPNsense/AcmeClient/actions.volt +++ b/security/acme-client/src/opnsense/mvc/app/views/OPNsense/AcmeClient/actions.volt @@ -133,11 +133,11 @@ POSSIBILITY OF SUCH DAMAGE. var errors = [ {cond: ["connect_failed", "invalid_parameters"], msg: "{{ lang._('Host or username not specified.') }}"}, - {cond: ["connect_failed", "host_not_trusted"], msg: "{{ lang._('Host cannot be trusted.') }}"}, {cond: ["connect_failed", "host_not_resolved"], msg: "{{ lang._('Failed to resolve hostname.') }}"}, {cond: ["connect_failed", "connection_refused"], msg: "{{ lang._('Connection to host refused.') }}"}, {cond: ["connect_failed", "network_timeout"], msg: "{{ lang._('Connection timed out.') }}"}, {cond: ["connect_failed", "network_unreachable"], msg: "{{ lang._('Host not reachable.') }}"}, + {cond: ["connect_failed", "host_not_trusted"], msg: "{{ lang._('Host cannot be trusted.') }}"}, {cond: ["connect_failed", "permission_denied"], msg: "{{ lang._('Host does not permit a connection for the specified user & identity.') }}"}, {cond: ["connect_failed"], msg: "{{ lang._('Failed to connect to host.') }}"}, {cond: ["change_home_dir_failed"], msg: "{{ lang._('Failed to change the remote path.') }}"}, @@ -163,7 +163,7 @@ POSSIBILITY OF SUCH DAMAGE. statusClass = "alert-success"; message = "{{ lang._('Connection and upload test succeeded.') }}" } else { - detail = JSON.stringify(data, null, ' '); + detail = JSON.stringify(data, null, ' ').replace(/\\"/g, "'"); for (var i = 0; i < errors.length; i++) { var error = errors[i], diff --git a/security/acme-client/src/opnsense/service/conf/actions.d/actions_acmeclient.conf b/security/acme-client/src/opnsense/service/conf/actions.d/actions_acmeclient.conf index 0637f185b..7c53c1cfd 100644 --- a/security/acme-client/src/opnsense/service/conf/actions.d/actions_acmeclient.conf +++ b/security/acme-client/src/opnsense/service/conf/actions.d/actions_acmeclient.conf @@ -92,7 +92,7 @@ message:uploading a certificate to sftp server [test-sftp-connection] command:/usr/local/opnsense/scripts/OPNsense/AcmeClient/upload_sftp.php -parameters:--host=%s --host-key=%s --port=%s --user=%s --identity-type=%s --remote-path=%s --no-error test-connection +parameters:--host=%s --host-key=%s --port=%s --user=%s --identity-type=%s --remote-path=%s --chmod=%s --chgrp=%s --no-error test-connection type:script_output message:testing connection to sftp server From 68e7d2936807ebcda5896761251c09d7f582bdd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Kellerer?= Date: Sat, 17 Aug 2019 18:04:47 +0200 Subject: [PATCH 7/9] AcmeClient: Fixed selecting SSH host-key for non-standard ports. --- .../library/OPNsense/AcmeClient/SSHKeys.php | 39 ++++++++++++------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/SSHKeys.php b/security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/SSHKeys.php index 7603f3612..aaefc5e2d 100644 --- a/security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/SSHKeys.php +++ b/security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/SSHKeys.php @@ -60,7 +60,7 @@ class SSHKeys "ecdsa" => 521, ]; - const DEFAULT_PORT = 22; + public const DEFAULT_PORT = 22; private $config_path; private $known_hosts_file; @@ -134,6 +134,8 @@ class SSHKeys : $found; }, false); + $known_by_host_matches_port = $known_by_host && ($port == self::DEFAULT_PORT || $known_by_host["host"] !== $known_by_host["host_query"]); + // Find known_hosts item with same public host-key $known_by_key = array_reduce($known_keys, function ($found, $key) use ($host_key) { return (!$found && $host_key && $host_key === $key["key_info"]) @@ -144,13 +146,18 @@ class SSHKeys // Updating $host and $host_key from known_hosts and check if we need to update known_hosts. if ($host_key === false && $known_by_host) { - Utils::log()->info("No host key specified, using existing known_hosts entry for '$host'"); - $host_key = $known_by_host["key_info"]; + if ($known_by_host_matches_port) { + Utils::log()->info("No host key specified, using existing known_hosts entry for '$host'"); + $host_key = $known_by_host["key_info"]; + } else { + Utils::log()->info("No host key specified and existing entry for '$host' cannot be used as isn't matching port $port."); + } } $is_key_known = false; if ($known_by_host && $host_key && $host_key === $known_by_host["key_info"]) { $is_key_known = true; + } else if ($known_by_key) { if (strcasecmp(trim($host), trim($known_by_key["host"])) != 0) { Utils::log()->info("Host key is in known_hosts but hostname differs. Changing '$host' to '{$known_by_key["host"]}'."); @@ -183,9 +190,9 @@ class SSHKeys }); if (!empty($matching_remote_host_keys)) { - if ($known_by_host) { - Utils::log()->info("Removing known_hosts entry with differing key for '{$known_by_host["host"]}' as it is in the way."); - $this->removeKnownHost($known_by_host["host"]); + if ($known_by_host && $known_by_host_matches_port) { + Utils::log()->info("Removing known_hosts entry with differing key for '{$known_by_host["host_query"]}' as it is in the way."); + $this->removeKnownHost($known_by_host["host_query"]); } foreach ($matching_remote_host_keys as $key) { @@ -251,10 +258,12 @@ class SSHKeys return !empty(trim($value)) && array_search($value, $search_list) == $index; }, ARRAY_FILTER_USE_BOTH); - // Add port specific items when a port is specified (reason: ssh-keyscan may create such entries). + // Insert port specific items at the beginning when a port was specified. + // Reason: Multiple SSH servers may be used on the same host but different ports. + // This ensures specific keys (with port) are selected first. if ($port > 0) { - foreach (array_values($search_list) as $item) { - $search_list[] = "[{$item}]:{$port}"; + foreach (array_reverse($search_list) as $item) { + array_unshift($search_list, "[{$item}]:{$port}"); } } @@ -311,12 +320,12 @@ class SSHKeys ? "" : PHP_EOL . "ssh-keyscan: " . join(PHP_EOL . "ssh-keyscan: ", $lines); - Utils::log()->error("Failed querying public keys ($key_type) for [$names]. Exit code: {$p->exitCode} (error-marker: $marker) $output"); + Utils::log()->error("Failed querying host keys ($key_type) for [$names] port $port. Exit code: {$p->exitCode} (error-marker: $marker) $output"); } } if (empty($keys)) { - Utils::log()->info("Couldn't fetch public host key ($key_type) from $host"); + Utils::log()->info("Couldn't fetch public host key ($key_type) from {$host}:{$port}"); if (!is_array($error) || empty($error)) $error = ["connection_refused" => true]; @@ -348,11 +357,15 @@ class SSHKeys if ($p->close() == 0) { // Removing port from name or ip before returning it. - $applied_host = preg_match('/^\[([^\]]+?)\]:\d+/', $name_or_ip, $matches) + $hostname = preg_match('/^\[([^\]]+?)\]:\d+/', $name_or_ip, $matches) ? $matches[1] : $name_or_ip; - $keys[] = ["host" => $applied_host, "host_key" => $lines[0]]; + $keys[] = [ + "host" => $hostname, + "host_key" => $lines[0], + "host_query" => $name_or_ip, + ]; } else if ($p->exitCode != 1 /* 1 == NOT_FOUND */) { $output = empty($lines) From b54007176cc6802f154fdb11d1daf00b066ea210 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Kellerer?= Date: Sat, 17 Aug 2019 22:33:42 +0200 Subject: [PATCH 8/9] AcmeClient: Fixed not detecting remote side closing connection. --- .../mvc/app/library/OPNsense/AcmeClient/SftpClient.php | 4 ++-- .../app/library/OPNsense/AcmeClient/SftpUploader.php | 10 +++++++--- .../scripts/OPNsense/AcmeClient/upload_sftp.php | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/SftpClient.php b/security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/SftpClient.php index e3ba387ec..0292fa166 100644 --- a/security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/SftpClient.php +++ b/security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/SftpClient.php @@ -62,7 +62,7 @@ class SftpClient public function connected(): ?array { - return $this->process + return $this->process && $this->process->isRunning() ? $this->connection_info : null; } @@ -109,7 +109,7 @@ class SftpClient // Creating the sftp process if ($this->process = Process::open($cmd)) { $this->processAvailableInput(self::CONNECT_REPLY_TIMEOUT, 1, null, 0.75); - if ($error = $this->lastError()) { + if (($error = $this->lastError()) || !$this->process->isRunning()) { Utils::log()->error("Failed connecting to '$host' (user: '$username')", $error); return false; } diff --git a/security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/SftpUploader.php b/security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/SftpUploader.php index 83d072cd4..b56bc2411 100644 --- a/security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/SftpUploader.php +++ b/security/acme-client/src/opnsense/mvc/app/library/OPNsense/AcmeClient/SftpUploader.php @@ -140,9 +140,6 @@ class SftpUploader */ public function upload(): int { - $connection = Utils::requireThat($this->sftp->connected(), "The sftp client must be connected"); - $username = $connection["user"]; - // Correct state when we are restarted after an error if ($this->current_file) { // Restore the remote path to where we originally have been. @@ -182,6 +179,12 @@ class SftpUploader $this->sftp->clearError(); try { + $connection = $this->sftp->connected(); + if (!$connection) { + Utils::log()->error("The sftp client is not connected, upload stopped."); + return self::UPLOAD_ERROR; + } + // Changing remote directory if required. if (($target_dir = dirname($file["target"])) !== $remote_path) { @@ -229,6 +232,7 @@ class SftpUploader } // Preparing upload + $username = $connection["user"]; $remote_filename = basename((empty($file["target"]) ? $local_file : $file["target"])); $remote_file = $remote_files[$remote_filename] ?: ["type" => "-", "owner" => $username]; $remote_is_file = $remote_file["type"] === "-"; diff --git a/security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/upload_sftp.php b/security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/upload_sftp.php index 7e205490a..aa86faa3c 100644 --- a/security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/upload_sftp.php +++ b/security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/upload_sftp.php @@ -179,7 +179,7 @@ function commandTestConnection(array &$options): int $result["actions"][] = "connected"; $result["remote"] = array_merge($sftp->connected(), ["path" => $sftp->pwd()]); } else { - $result = array_merge($result, $error); + $result = array_merge($result, ($error ?: [])); } // Testing file upload From a93b8020d7f4b9f05bb8ea6782d212d0b54f79e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Kellerer?= Date: Sat, 17 Aug 2019 23:19:39 +0200 Subject: [PATCH 9/9] AcmeClient: Fixed default chmod not getting applied (type problem). --- .../src/opnsense/scripts/OPNsense/AcmeClient/upload_sftp.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/upload_sftp.php b/security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/upload_sftp.php index aa86faa3c..5bef2ceac 100644 --- a/security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/upload_sftp.php +++ b/security/acme-client/src/opnsense/scripts/OPNsense/AcmeClient/upload_sftp.php @@ -110,8 +110,8 @@ const EXAMPLES = <<