Merge branch 'dev' of http://10.95.0.64/zerotier/ZeroTierOne into dev

This reverts commit e96515433d.

.drone.jsonnet

@@ -0,0 +1,68 @@
+
+local targets = [
+      //
+      // Render these into .drone.yaml by running "make drone"
+      //      
+      { "os": "linux", "name": "el9", "isas": [ "amd64", "arm64", "ppc64le", "s390x" ], "events": [ "tag" ] },
+      { "os": "linux", "name": "el8", "isas": [ "amd64", "arm64", "ppc64le", "s390x" ], "events": [ "tag" ] },
+      { "os": "linux", "name": "el7", "isas": [ "amd64", "ppc64le"], "events": [ "tag" ] },
+      { "os": "linux", "name": "el6", "isas": [ "amd64" ], "events": [ "tag" ] },
+      { "os": "linux", "name": "amzn2", "isas": [ "amd64", "arm64" ], "events": [ "tag" ] },
+      { "os": "linux", "name": "fc37", "isas": [ "amd64", "arm64", "ppc64le", "s390x" ], "events": [ "tag" ] },
+      { "os": "linux", "name": "fc36", "isas": [ "amd64", "arm64", "ppc64le", "s390x" ], "events": [ "tag" ] },
+      { "os": "linux", "name": "fc35", "isas": [ "amd64", "arm64", "ppc64le", "s390x" ], "events": [ "tag" ] },
+      { "os": "linux", "name": "jammy", "isas": [ "amd64", "arm64", "armv7", "riscv64", "ppc64le", "s390x" ], "events": [ "tag" ] },
+      { "os": "linux", "name": "focal", "isas": [ "amd64", "arm64", "armv7", "riscv64", "ppc64le"  ], "events": [ "tag" ] },
+      { "os": "linux", "name": "bionic", "isas": [ "amd64", "arm64", "386", "ppc64le", "s390x" ], "events": ["tag" ] },
+      { "os": "linux", "name": "xenial", "isas": [ "amd64", "arm64", "386" ], "events": [ "tag" ] },
+      { "os": "linux", "name": "sid", "isas": [ "386", "amd64", "arm64", "armv7", "riscv64", "mips64le", "ppc64le", "s390x" ], "events": [ "push", "tag" ] },
+      { "os": "linux", "name": "bookworm", "isas": [ "amd64", "arm64", "armv7", "386", "mips64le", "ppc64le", "s390x" ], "events": [ "tag" ] },
+      { "os": "linux", "name": "bullseye", "isas": [ "amd64", "arm64", "armv7", "386", "mips64le", "ppc64le", "s390x" ], "events": [ "tag" ] },
+      { "os": "linux", "name": "buster", "isas": [ "amd64", "arm64", "armv7", "386", "mips64le", "ppc64le", "s390x" ], "events": [ "tag" ] },
+      { "os": "linux", "name": "stretch", "isas": [ "amd64", "arm64", "386" ], "events": [ "tag" ] },
+      // { "os": "windows", "name": "win2k19", "isas": [ "amd64" ], "events": ["push", "tag" ] }
+];
+
+local Build(platform, os, isa, events) = {
+  "kind": "pipeline",
+  "type": "docker",
+  "pull": "always",
+  "name": platform + " " + isa + " " + "build",
+  "clone": { "depth": 1 },
+  "steps": [
+    {
+      "name": "build",
+      "image": "registry.sean.farm/honda-builder",
+      "commands": [ "./ci/scripts/build.sh " + platform + " " + isa + " " + "100.0.0+${DRONE_COMMIT_SHA:0:8}" + " " + "${DRONE_BUILD_EVENT}" ]
+    },
+    {
+      "name": "list",
+      "image": "registry.sean.farm/honda-builder",
+      "commands": [ "ls -la " + platform ]
+    },
+    {
+      "name": "notify-mattermost",
+      "image": "registry.sean.farm/mattermost-notify",
+      "environment": {
+        "token": { "from_secret": "mattermost-token" },
+        "host": { "from_secret": "mattermost-host" },
+        "channel": { "from_secret": "mattermost-channel" },
+        "maxRetry": 3,
+      },
+      "when": { "status": [ "failure" ] }
+    }
+  ],  
+  "image_pull_secrets": [ "dockerconfigjson" ],
+  [ if isa == "arm64" || isa == "armv7" then "platform" ]: { os: os, arch: "arm64" },
+  "trigger": { "event": events }
+};
+
+// puttin on the bits
+
+std.flattenArrays([
+  [
+     Build(p.name, p.os, isa, p.events)
+      for isa in p.isas
+  ]
+  for p in targets
+])

.gitignore

@@ -6,6 +6,11 @@
 /zerotier
 /nltest
 
+# IDE stuff
+/.idea
+/.nova
+/compile_commands.json
+
 # OS-created garbage files from various platforms
 .DS_Store
 .Apple*
@@ -30,7 +35,6 @@ Thumbs.db
 /windows/WebUIWrapper/obj
 /windows/lib
 /ext/installfiles/windows/ZeroTier One-SetupFiles
-/ext/installfiles/windows/Prerequisites
 /ext/installfiles/windows/*-cache
 /ZeroTier One.msi
 *.vcxproj.backup
@@ -58,7 +62,6 @@ zt1-src.tar.gz
 *.opensdf
 *.user
 *.cache
-*.obj
 *.tlog
 *.pid
 *.pkg
@@ -103,7 +106,6 @@ windows/ZeroTierOne/Debug/
 *.swp
 *~.nib
 DerivedData/
-build/
 *.pbxuser
 *.mode1v3
 *.mode2v3
@@ -114,7 +116,6 @@ build/
 !default.perspectivev3
 *.xccheckout
 xcuserdata/
-ext/librethinkdbxx/build
 .vscode
 __pycache__
 *~
@@ -123,7 +124,6 @@ attic/world/mkworld
 workspace/
 workspace2/
 zeroidc/target/
-tmp/
 
 #snapcraft specifics
 /parts/

Dockerfile.release

@@ -11,6 +11,8 @@ RUN curl -sSL -o zerotier-one.deb "${PACKAGE_BASEURL}/zerotier-one_${VERSION}_${
 
 FROM debian:buster
 
+RUN apt-get update -qq && apt-get install openssl libssl1.1 -y
+
 COPY --from=stage zerotier-one.deb .
 
 RUN dpkg -i zerotier-one.deb && rm -f zerotier-one.deb
@@ -20,5 +22,7 @@ RUN rm -rf /var/lib/zerotier-one
 COPY entrypoint.sh.release /entrypoint.sh
 RUN chmod 755 /entrypoint.sh
 
+HEALTHCHECK --interval=1s CMD bash /healthcheck.sh
+
 CMD []
 ENTRYPOINT ["/entrypoint.sh"]

Makefile

@@ -26,3 +26,7 @@ endif
 ifeq ($(OSTYPE),NetBSD)
 	include make-netbsd.mk
 endif
+
+drone:
+	@echo "rendering .drone.yaml from .drone.jsonnet"
+	drone jsonnet --format --stream

README.md

@@ -1,6 +1,7 @@
 ZeroTier - Global Area Networking
 ======
-This document is written for a software developer audience. For information on using ZeroTier, see the: [Website](https://www.zerotier.com), [Documentation Site](https://docs.zerotier.com), and [Discussion Forum](https://discuss.zerotier.com)
+
+*This document is written for a software developer audience. For information on using ZeroTier, see the: [Website](https://www.zerotier.com), [Documentation Site](https://docs.zerotier.com), and [Discussion Forum](https://discuss.zerotier.com).*
 
 ZeroTier is a smart programmable Ethernet switch for planet Earth. It allows all networked devices, VMs, containers, and applications to communicate as if they all reside in the same physical data center or cloud region.
 
@@ -42,24 +43,29 @@ The base path contains the ZeroTier One service main entry point (`one.cpp`), se
  - `rule-compiler/`: JavaScript rules language compiler for defining network-level rules.
  - `service/`: the ZeroTier One service, which wraps the ZeroTier core and provides VPN-like connectivity to virtual networks for desktops, laptops, servers, VMs, and containers.
  - `windows/`: Visual Studio solution files, Windows service code, and the Windows task bar app UI.
+ - `zeroidc/`: OIDC implementation used by ZeroTier service to log into SSO-enabled networks. (This part is written in Rust, and more Rust will be appearing in this repository in the future.)
 
 ### Build and Platform Notes
 
 To build on Mac and Linux just type `make`. On FreeBSD and OpenBSD `gmake` (GNU make) is required and can be installed from packages or ports. For Windows there is a Visual Studio solution in `windows/`.
 
  - **Mac**
-   - Xcode command line tools for OSX 10.8 or newer are required.
+   - Xcode command line tools for macOS 10.13 or newer are required.
+   - Rust for x86_64 and ARM64 targets *if SSO is enabled in the build*.
  - **Linux**
    - The minimum compiler versions required are GCC/G++ 4.9.3 or CLANG/CLANG++ 3.4.2. (Install `clang` on CentOS 7 as G++ is too old.)
    - Linux makefiles automatically detect and prefer clang/clang++ if present as it produces smaller and slightly faster binaries in most cases. You can override by supplying CC and CXX variables on the make command line.
+   - Rust for x86_64 and ARM64 targets *if SSO is enabled in the build*.
  - **Windows**
-   - Windows 7 or newer is supported. This *may* work on Vista but isn't officially supported there. It will not work on Windows XP.
-   - We build with Visual Studio 2017. Older versions may not work. Clang or MinGW will also probably work but may require some makefile hacking.
+   - Visual Studio 2022 on Windows 10 or newer.
+   - Rust for x86_64 and ARM64 targets *if SSO is enabled in the build*.
  - **FreeBSD**
    - GNU make is required. Type `gmake` to build.
+   - Rust for x86_64 and ARM64 targets *if SSO is enabled in the build*.
  - **OpenBSD**
    - There is a limit of four network memberships on OpenBSD as there are only four tap devices (`/dev/tap0` through `/dev/tap3`).
    - GNU make is required. Type `gmake` to build.
+   - Rust for x86_64 and ARM64 targets *if SSO is enabled in the build*.
 
 Typing `make selftest` will build a *zerotier-selftest* binary which unit tests various internals and reports on a few aspects of the build environment. It's a good idea to try this on novel platforms or architectures.
 
@@ -82,7 +88,7 @@ Here's where home folders live (by default) on each OS:
  * **Linux**: `/var/lib/zerotier-one`
  * **FreeBSD** / **OpenBSD**: `/var/db/zerotier-one`
  * **Mac**: `/Library/Application Support/ZeroTier/One`
- * **Windows**: `\ProgramData\ZeroTier\One` (That's for Windows 7. The base 'shared app data' folder might be different on different Windows versions.)
+ * **Windows**: `\ProgramData\ZeroTier\One` (That's the default. The base 'shared app data' folder might be different if Windows is installed with a non-standard drive letter assignment or layout.)
 
 ### Basic Troubleshooting
 

RELEASE-NOTES.md

@@ -1,6 +1,46 @@
 ZeroTier Release Notes
 ======
 
+# 2022-06-27 -- Version 1.10.1
+
+ * Fix an issue that could cause SSO clients to get "stuck" on stale auth URLs.
+ * A few other SSO related bug fixes.
+
+# 2022-06-07 -- Version 1.10.0
+
+ * Fix formatting problem in `zerotier-cli` when using SSO networks.
+ * Fix a few other minor bugs in SSO signin to prepare for general availability.
+ * Remove requirement for webview in desktop UI and instead just make everything available via the tray pulldown/menu. Use [libui-ng](https://github.com/libui-ng/libui-ng) for minor prompt dialogs. Saves space and eliminates installation headaches on Windows.
+ * Fix SSO "spam" bug in desktop UI.
+ * Use system default browser for SSO login so all your plugins, MFA devices, password managers, etc. will work as you have them configured.
+ * Minor fix for bonding/multipath.
+
+# 2022-05-10 -- Version 1.8.10
+
+ * Fixed a bug preventing SSO sign-on on Windows.
+
+# 2022-04-25 -- Version 1.8.9
+
+ * Fixed a long-standing and strange bug that was causing sporadic "phantom" packet authentication failures. Not a security problem but could be behind spordaic reports of link failures under some conditions.
+ * Fized a memory leak in SSO/OIDC support.
+ * Fixed SSO/OIDC display error on CLI.
+ * Fixed a bug causing nodes to sometimes fail to push certs to each other (primarily affects SSO/OIDC use cases).
+ * Fixed a deadlock bug on leaving SSO/OIDC managed networks.
+ * Added some new Linux distributions to the build subsystem.
+
+# 2022-04-11 -- Version 1.8.8
+
+ * Fix a local privilege escalation bug in the Windows installer.
+ * Dependency fix for some Ubuntu versions.
+ * No changes for other platforms. Windows upgrade recommended, everyone else optional.
+
+# 2022-03-30 -- Version 1.8.7
+
+ * Fix for dependency installations in Windows MSI package.
+ * Fix for desktop UI setup when run by a non-super-user.
+ * Bug fix in local OIDC / SSO support for auth0 and other providers.
+ * Other minor fixes for e.g. old Linux distributions.
+
 # 2022-03-04 -- Version 1.8.6
 
  * Fixed an issue that could cause the UI to be non-responsive if not joined to any networks.

ci/Dockerfile.deb

@@ -0,0 +1,10 @@
+ARG PLATFORM
+FROM registry.sean.farm/${PLATFORM}-builder as stage
+WORKDIR /work/build
+COPY . .
+RUN make debian
+RUN ls -ls /work
+
+FROM scratch AS export
+ARG PLATFORM
+COPY --from=stage /work/*.deb ./${PLATFORM}/

ci/Dockerfile.el6

@@ -0,0 +1,36 @@
+ARG DOCKER_ARCH
+FROM --platform=linux/${DOCKER_ARCH} alpine:edge AS builder
+
+RUN apk update
+RUN apk add curl
+RUN apk add bash
+RUN apk add file
+RUN apk add rust
+RUN apk add cargo
+RUN apk add make
+RUN apk add cmake
+RUN apk add clang
+RUN apk add openssl-dev
+RUN apk add linux-headers
+RUN apk add build-base
+RUN apk add openssl-libs-static
+
+COPY . .
+RUN ZT_STATIC=1 make one
+RUN ls -la
+
+ARG DOCKER_ARCH
+FROM --platform=linux/${DOCKER_ARCH} centos:6 AS stage
+WORKDIR /root/rpmbuild/BUILD
+COPY . .
+COPY --from=builder zerotier-one ./
+RUN curl https://gist.githubusercontent.com/someara/b363002ba6e57b3c474dd027d4daef85/raw/4ac5534139752fc92fbe1a53599a390214f69615/el6%2520vault --output /etc/yum.repos.d/CentOS-Base.repo
+RUN uname -a
+RUN yum -y install make gcc rpm-build
+RUN pwd
+RUN ls -la
+RUN make redhat
+
+FROM scratch AS export
+ARG PLATFORM
+COPY --from=stage /root/rpmbuild/RPMS/*/*.rpm ./${PLATFORM}/

ci/Dockerfile.rpm

@@ -0,0 +1,9 @@
+ARG PLATFORM
+FROM registry.sean.farm/${PLATFORM}-builder as stage
+WORKDIR /root/rpmbuild/BUILD
+COPY . .
+RUN make redhat
+
+FROM scratch AS export
+ARG PLATFORM
+COPY --from=stage /root/rpmbuild/RPMS/*/*.rpm ./${PLATFORM}/

ci/Dockerfile.sid

@@ -1,7 +0,0 @@
-FROM registry.sean.farm/sid-builder as stage
-COPY . .
-RUN /usr/bin/make -j 8
-
-FROM scratch AS export
-COPY --from=stage /zerotier-one .
-COPY --from=stage /zerotier-cli .

ci/scripts/build.sh

@@ -2,27 +2,115 @@
 set -euo pipefail
 IFS=$'\n\t'
 
-export GOOS=$1
-export GOARCH=$2
+export PLATFORM=$1
+export ZT_ISA=$2
 export VERSION=$3
-export DOCKER_BUILDKIT=1
+export EVENT=$4
 
-echo "nproc: $(nproc)"
-
-case $GOARCH in
-    armv5)
-        export ARCH=arm/v5
-        ;;
-    armv7)
-        export ARCH=arm/v7
-        ;;
-    arm64)
-        export ARCH=arm64/v8
+case $PLATFORM in
+    el*|fc*|amzn*)
+        export PKGFMT=rpm
         ;;
     *)
-        export ARCH=$GOARCH
+        export PKGFMT=deb
+esac
+
+# OSX
+# x86_64-apple-darwin
+# aarch64-apple-darwin
+
+# Windows
+# x86_64-pc-windows-msvc
+# i686-pc-windows-msvc
+# aarch64-pc-windows-msvc
+
+# Linux
+# i686-unknown-linux-gnu
+# x86_64-unknown-linux-gnu
+# arm-unknown-linux-gnueabi       ?
+# arm-unknown-linux-gnueabihf     ?
+# armv7-unknown-linux-gnueabihf
+# 
+
+case $ZT_ISA in
+    386)
+        export DOCKER_ARCH=386
+        export RUST_TRIPLET=i686-unknown-linux-gnu
+        ;;
+    amd64)
+        export DOCKER_ARCH=amd64
+        export RUST_TRIPLET=x86_64-unknown-linux-gnu
+        ;;
+    armv6)
+        export DOCKER_ARCH=arm/v6
+        export RUST_TRIPLET=arm-unknown-linux-gnueabi
+        ;;
+    armv7)        
+        export DOCKER_ARCH=arm/v7
+        export RUST_TRIPLET=arm-unknown-linux-gnueabihf
+        ;;
+    arm64)
+        export DOCKER_ARCH=arm64/v8
+        export RUST_TRIPLET=aarch64-unknown-linux-gnu
+        ;;
+    riscv64)
+        export DOCKER_ARCH=riscv64
+        export RUST_TRIPLET=riscv64gc-unknown-linux-gnu
+        ;;
+    ppc64le)
+        export DOCKER_ARCH=ppc64le
+        export RUST_TRIPLET=powerpc64le-unknown-linux-gnu
+        ;;    
+    mips64le)
+        export DOCKER_ARCH=mips64le
+        export RUST_TRIPLET=mips64el-unknown-linux-gnuabi64
+        ;;
+    s390x)
+        export DOCKER_ARCH=s390x
+        export RUST_TRIPLET=s390x-unknown-linux-gnu
+        ;;
+    *)        
+        echo "ERROR: could not determine architecture settings. PLEASE FIX ME"
+        exit 1
         ;;
 esac
 
+if [ -f "ci/Dockerfile.${PLATFORM}" ]; then
+    export DOCKERFILE="ci/Dockerfile.${PLATFORM}"
+else
+    export DOCKERFILE="ci/Dockerfile.${PKGFMT}"
+fi
+
+echo "#~~~~~~~~~~~~~~~~~~~~"
+echo "$0 variables:"
+echo "nproc: $(nproc)"
+echo "ZT_ISA: ${ZT_ISA}"
+echo "DOCKER_ARCH: ${DOCKER_ARCH}"
+echo "RUST_TRIPLET: ${RUST_TRIPLET}"
+echo "VERSION: ${VERSION}"
+echo "EVENT: ${EVENT}"
+echo "PKGFMT: ${PKGFMT}"
+echo "PWD: ${PWD}"
+echo "DOCKERFILE: ${DOCKERFILE}"
+echo "#~~~~~~~~~~~~~~~~~~~~"
+
+if [ ${EVENT} == "push" ]; then
+make munge_rpm zerotier-one.spec VERSION=${VERSION}
+make munge_deb debian/changelog VERSION=${VERSION}
+fi
+
+export DOCKER_BUILDKIT=1
 docker run --privileged --rm tonistiigi/binfmt --install all
-docker buildx build --platform ${GOOS}/${ARCH} -f ci/Dockerfile.sid --target export -t test . --output out/${GOOS}/${GOARCH}
+
+# docker pull --platform linux/${DOCKER_ARCH} registry.sean.farm/${PLATFORM}-builder
+
+docker buildx build \
+       --build-arg PLATFORM="${PLATFORM}" \
+       --build-arg RUST_TRIPLET="${RUST_TRIPLET}" \
+       --build-arg DOCKER_ARCH="${DOCKER_ARCH}" \
+       --platform linux/${DOCKER_ARCH} \
+       -f ${DOCKERFILE} \
+       -t build \
+       . \
+       --output type=local,dest=. \
+       --target export

ci/scripts/munge_debian_changelog.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+export FILE=$1
+export VERSION=$2
+export NAME=$3
+export MESSAGE=$4
+export DATE=$(date "+%a, %d %b %Y %T %z")
+# export DATE=$(date "+%a %b %d %Y")
+
+set +e
+grep --version | grep BSD &> /dev/null
+if [ $? == 0 ]; then BSDGREP=true ; else BSDGREP=false ; fi
+set -e
+
+# echo "#~~~~~~~~~~~~~~~~~~~~"
+# echo "$0 variables:"
+# echo "VERSION: ${VERSION}"
+# echo "NAME: ${NAME}"
+# echo "MESSAGE: ${MESSAGE}"
+# echo "DATE: ${DATE}"
+# echo "BSDGREP: ${BSDGREP}"
+# echo "#~~~~~~~~~~~~~~~~~~~~"
+# echo
+
+if $BSDGREP ; then    
+    sed -i '' s/^Version:.*/"Version:        ${VERSION}"/ ${FILE}
+else
+    sed -i s/^Version:.*/"Version:        ${VERSION}"/ ${FILE}
+fi
+
+awk -v version=${VERSION} -v date=${DATE} -v name=${NAME} -v message=${MESSAGE} \
+    'BEGIN{print "zerotier-one (" version ") unstable; urgency=medium\n\n  * " message "\n\n -- " name "  " date "\n" }{ print }' \
+    ${FILE} > ${FILE}.new
+
+mv ${FILE}.new ${FILE}

ci/scripts/munge_rpm_spec.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+export FILE=$1
+export VERSION=$2
+export NAME=$3
+export MESSAGE=$4
+export DATE=$(date "+%a %b %d %Y")
+
+set +e
+grep --version | grep BSD &> /dev/null
+if [ $? == 0 ]; then BSDGREP=true ; else BSDGREP=false ; fi
+set -e
+
+# echo "#~~~~~~~~~~~~~~~~~~~~"
+# echo "$0 variables:"
+# echo "VERSION: ${VERSION}"
+# echo "NAME: ${NAME}"
+# echo "MESSAGE: ${MESSAGE}"
+# echo "DATE: ${DATE}"
+# echo "BSDGREP: ${BSDGREP}"
+# echo "#~~~~~~~~~~~~~~~~~~~~"
+# echo
+
+if $BSDGREP ; then    
+    sed -i '' s/^Version:.*/"Version:        ${VERSION}"/ ${FILE}
+else
+    sed -i s/^Version:.*/"Version:        ${VERSION}"/ ${FILE}
+fi
+
+awk -v version=${VERSION} -v date=${DATE} -v name=${NAME} -v message=${MESSAGE} \
+    'FNR==NR{ if (/%changelog/) p=NR; next} 1; FNR==p{ print "* " date  " " name " - " version "\n- " message "\n" }' \
+    ${FILE} ${FILE} > ${FILE}.new
+
+mv ${FILE}.new ${FILE}

controller/DB.cpp

@@ -196,14 +196,6 @@ void DB::networks(std::set<uint64_t> &networks)
 		networks.insert(n->first);
 }
 
-void DB::networkMemberSSOHasExpired(uint64_t nwid, int64_t now) {
-	std::lock_guard<std::mutex> l(_networks_l);
-	auto nw = _networks.find(nwid);
-	if (nw != _networks.end()) {
-		nw->second->mostRecentDeauthTime = now;
-	}
-}
-
 void DB::_memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool notifyListeners)
 {
 	uint64_t memberId = 0;

controller/DB.hpp

@@ -33,7 +33,7 @@
 #include <set>
 #include <map>
 
-#include "../ext/json/json.hpp"
+#include <nlohmann/json.hpp>
 
 #define ZT_MEMBER_AUTH_TIMEOUT_NOTIFY_BEFORE 25000
 
@@ -135,7 +135,6 @@ public:
 	virtual void nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress) = 0;
 
 	virtual AuthInfo getSSOAuthInfo(const nlohmann::json &member, const std::string &redirectURL) { return AuthInfo(); }
-	virtual void networkMemberSSOHasExpired(uint64_t nwid, int64_t ts);
 
 	inline void addListener(DB::ChangeListener *const listener)
 	{

controller/DBMirrorSet.cpp

@@ -137,14 +137,6 @@ AuthInfo DBMirrorSet::getSSOAuthInfo(const nlohmann::json &member, const std::st
 	return AuthInfo();
 }
 
-void DBMirrorSet::networkMemberSSOHasExpired(uint64_t nwid, int64_t ts)
-{
-	std::lock_guard<std::mutex> l(_dbs_l);
-	for(auto d=_dbs.begin();d!=_dbs.end();++d) { 
-		(*d)->networkMemberSSOHasExpired(nwid, ts);
-	}
-}
-
 void DBMirrorSet::networks(std::set<uint64_t> &networks)
 {
 	std::lock_guard<std::mutex> l(_dbs_l);
@@ -248,47 +240,4 @@ void DBMirrorSet::onNetworkMemberDeauthorize(const void *db,uint64_t networkId,u
 	_listener->onNetworkMemberDeauthorize(this,networkId,memberId);
 }
 
-void DBMirrorSet::membersExpiring(std::set< std::pair<uint64_t, uint64_t> > &soon, std::set< std::pair<uint64_t, uint64_t> > &expired)
-{
-	std::unique_lock<std::mutex> l(_membersExpiringSoon_l);
-	int64_t now = OSUtils::now();
-	for(auto next=_membersExpiringSoon.begin();next!=_membersExpiringSoon.end();) {
-		if (next->first > now) {
-			const uint64_t nwid = next->second.first;
-			const uint64_t memberId = next->second.second;
-			nlohmann::json network, member;
-			if (this->get(nwid, network, memberId, member)) {
-				try {
-					const bool authorized = member["authorized"];
-					const bool ssoExempt = member["ssoExempt"];
-					const int64_t authenticationExpiryTime = member["authenticationExpiryTime"];
-					if ((authenticationExpiryTime == next->first)&&(authorized)&&(!ssoExempt)) {
-						if ((authenticationExpiryTime - now) > ZT_MEMBER_AUTH_TIMEOUT_NOTIFY_BEFORE) {
-							// Stop when we get to entries too far in the future.
-							break;
-						} else {
-							const bool ssoEnabled = network["ssoEnabled"];
-							if (ssoEnabled)
-								soon.insert(std::pair<uint64_t, uint64_t>(nwid, memberId));
-						}
-					} else {
-						// Obsolete entry, no longer authorized, or SSO exempt.
-					}
-				} catch ( ... ) {
-					// Invalid member object, erase.
-				}
-			} else {
-				// Not found.
-			}
-		}
-		_membersExpiringSoon.erase(next++);
-	}
-}
-
-void DBMirrorSet::memberWillExpire(int64_t expTime, uint64_t nwid, uint64_t memberId)
-{
-	std::unique_lock<std::mutex> l(_membersExpiringSoon_l);
-	_membersExpiringSoon.insert(std::pair< int64_t, std::pair< uint64_t, uint64_t > >(expTime, std::pair< uint64_t, uint64_t >(nwid, memberId)));
-}
-
 } // namespace ZeroTier

controller/DBMirrorSet.hpp

@@ -52,7 +52,6 @@ public:
 	virtual void onNetworkMemberDeauthorize(const void *db,uint64_t networkId,uint64_t memberId);
 
 	AuthInfo getSSOAuthInfo(const nlohmann::json &member, const std::string &redirectURL);
-	void networkMemberSSOHasExpired(uint64_t nwid, int64_t ts);
 
 	inline void addDB(const std::shared_ptr<DB> &db)
 	{
@@ -61,17 +60,12 @@ public:
 		_dbs.push_back(db);
 	}
 
-	void membersExpiring(std::set< std::pair<uint64_t, uint64_t> > &soon, std::set< std::pair<uint64_t, uint64_t> > &expired);
-	void memberWillExpire(int64_t expTime, uint64_t nwid, uint64_t memberId);
-
 private:
 	DB::ChangeListener *const _listener;
 	std::atomic_bool _running;
 	std::thread _syncCheckerThread;
 	std::vector< std::shared_ptr< DB > > _dbs;
 	mutable std::mutex _dbs_l;
-	std::set< std::pair< int64_t, std::pair<uint64_t, uint64_t> > > _membersExpiringSoon;
-	mutable std::mutex _membersExpiringSoon_l;
 };
 
 } // namespace ZeroTier

controller/EmbeddedNetworkController.cpp

@@ -1262,6 +1262,7 @@ void EmbeddedNetworkController::_request(
 	}
 	const bool newMember = ((!member.is_object())||(member.empty()));
 	DB::initMember(member);
+	_MemberStatusKey msk(nwid,identity.address().toInt());
 
 	{
 		const std::string haveIdStr(OSUtils::jsonString(member["identity"],""));
@@ -1335,43 +1336,21 @@ void EmbeddedNetworkController::_request(
 	// Should we check SSO Stuff?
 	// If network is configured with SSO, and the member is not marked exempt: yes
 	// Otherwise no, we use standard auth logic.
+	AuthInfo info;
+	int64_t authenticationExpiryTime = -1;
 	bool networkSSOEnabled = OSUtils::jsonBool(network["ssoEnabled"], false);
 	bool memberSSOExempt = OSUtils::jsonBool(member["ssoExempt"], false);
-	AuthInfo info;
 	if (networkSSOEnabled && !memberSSOExempt) {
-		// TODO:  Get expiry time if auth is still valid
-
-		// else get new auth info & stuff
+		authenticationExpiryTime = (int64_t)OSUtils::jsonInt(member["authenticationExpiryTime"], 0);
 		info = _db.getSSOAuthInfo(member, _ssoRedirectURL);
 		assert(info.enabled == networkSSOEnabled);
-
-		std::string memberId = member["id"];
-		//fprintf(stderr, "ssoEnabled && !ssoExempt %s-%s\n", nwids, memberId.c_str());
-		uint64_t authenticationExpiryTime = (int64_t)OSUtils::jsonInt(member["authenticationExpiryTime"], 0);
-		fprintf(stderr, "authExpiryTime: %lld\n", authenticationExpiryTime);
-		if (authenticationExpiryTime < now) {
-			fprintf(stderr, "Handling expired member\n");
+		if (authenticationExpiryTime <= now) {
 			if (info.version == 0) {
-				if (!info.authenticationURL.empty()) {
-					_db.networkMemberSSOHasExpired(nwid, now);
-					onNetworkMemberDeauthorize(&_db, nwid, identity.address().toInt());
-
-					Dictionary<4096> authInfo;
-					authInfo.add(ZT_AUTHINFO_DICT_KEY_VERSION, (uint64_t)0ULL);
-					authInfo.add(ZT_AUTHINFO_DICT_KEY_AUTHENTICATION_URL, info.authenticationURL.c_str());
-					//fprintf(stderr, "sending auth URL: %s\n", authenticationURL.c_str());
-
-					DB::cleanMember(member);
-					_db.save(member,true);
-
-					_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_AUTHENTICATION_REQUIRED, authInfo.data(), authInfo.sizeBytes());
-					return;
-				}
-			}
-			else if (info.version == 1) {
-				_db.networkMemberSSOHasExpired(nwid, now);
-				onNetworkMemberDeauthorize(&_db, nwid, identity.address().toInt());
-
+				Dictionary<4096> authInfo;
+				authInfo.add(ZT_AUTHINFO_DICT_KEY_VERSION, (uint64_t)0ULL);
+				authInfo.add(ZT_AUTHINFO_DICT_KEY_AUTHENTICATION_URL, info.authenticationURL.c_str());
+				_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_AUTHENTICATION_REQUIRED, authInfo.data(), authInfo.sizeBytes());
+			} else if (info.version == 1) {
 				Dictionary<8192> authInfo;
 				authInfo.add(ZT_AUTHINFO_DICT_KEY_VERSION, info.version);
 				authInfo.add(ZT_AUTHINFO_DICT_KEY_ISSUER_URL, info.issuerURL.c_str());
@@ -1379,20 +1358,11 @@ void EmbeddedNetworkController::_request(
 				authInfo.add(ZT_AUTHINFO_DICT_KEY_NONCE, info.ssoNonce.c_str());
 				authInfo.add(ZT_AUTHINFO_DICT_KEY_STATE, info.ssoState.c_str());
 				authInfo.add(ZT_AUTHINFO_DICT_KEY_CLIENT_ID, info.ssoClientID.c_str());
-
-				DB::cleanMember(member);
-				_db.save(member, true);
-
-				fprintf(stderr, "Sending NC_ERROR_AUTHENTICATION_REQUIRED\n");
 				_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_AUTHENTICATION_REQUIRED, authInfo.data(), authInfo.sizeBytes());
-				return;
 			}
-			else {
-				fprintf(stderr, "invalid sso info.version %llu\n", info.version);
-			}
-		} else if (authorized) {
-			fprintf(stderr, "Setting member will expire to: %lld\n", authenticationExpiryTime);
-			_db.memberWillExpire(authenticationExpiryTime, nwid, identity.address().toInt());
+			DB::cleanMember(member);
+			_db.save(member,true);
+			return;
 		}
 	}
 
@@ -1411,8 +1381,8 @@ void EmbeddedNetworkController::_request(
 
 			{
 				std::lock_guard<std::mutex> l(_memberStatus_l);
-				_MemberStatus &ms = _memberStatus[_MemberStatusKey(nwid,identity.address().toInt())];
-
+				_MemberStatus &ms = _memberStatus[msk];
+				ms.authenticationExpiryTime = authenticationExpiryTime;
 				ms.vMajor = (int)vMajor;
 				ms.vMinor = (int)vMinor;
 				ms.vRev = (int)vRev;
@@ -1420,9 +1390,13 @@ void EmbeddedNetworkController::_request(
 				ms.lastRequestMetaData = metaData;
 				ms.identity = identity;
 			}
-		}		
+
+			if (authenticationExpiryTime > 0) {
+				std::lock_guard<std::mutex> l(_expiringSoon_l);
+				_expiringSoon.insert(std::pair<int64_t, _MemberStatusKey>(authenticationExpiryTime, msk));
+			}
+		}
 	} else {
-		
 		// If they are not authorized, STOP!
 		DB::cleanMember(member);
 		_db.save(member,true);
@@ -1434,18 +1408,13 @@ void EmbeddedNetworkController::_request(
 	// If we made it this far, they are authorized (and authenticated).
 	// -------------------------------------------------------------------------
 
-	int64_t credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA;
-	if (now > ns.mostRecentDeauthTime) {
-		// If we recently de-authorized a member, shrink credential TTL/max delta to
-		// be below the threshold required to exclude it. Cap this to a min/max to
-		// prevent jitter or absurdly large values.
-		const uint64_t deauthWindow = now - ns.mostRecentDeauthTime;
-		if (deauthWindow < ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MIN_MAX_DELTA) {
-			credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MIN_MAX_DELTA;
-		} else if (deauthWindow < (ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA + 5000ULL)) {
-			credentialtmd = deauthWindow - 5000ULL;
-		}
+	// Default timeout: 15 minutes. Maximum: two hours. Can be specified by an optional field in the network config
+	// if something longer than 15 minutes is desired. Minimum is 5 minutes since shorter than that would be flaky.
+	int64_t credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_DFL_MAX_DELTA;
+	if (network.contains("certificateTimeoutWindowSize")) {
+		credentialtmd = (int64_t)network["certificateTimeoutWindowSize"];
 	}
+	credentialtmd = std::max(std::min(credentialtmd, ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA), ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MIN_MAX_DELTA);
 
 	std::unique_ptr<NetworkConfig> nc(new NetworkConfig());
 
@@ -1460,7 +1429,7 @@ void EmbeddedNetworkController::_request(
 	nc->mtu = std::max(std::min((unsigned int)OSUtils::jsonInt(network["mtu"],ZT_DEFAULT_MTU),(unsigned int)ZT_MAX_MTU),(unsigned int)ZT_MIN_MTU);
 	nc->multicastLimit = (unsigned int)OSUtils::jsonInt(network["multicastLimit"],32ULL);
 
-	nc->ssoEnabled = OSUtils::jsonBool(network["ssoEnabled"], false);
+	nc->ssoEnabled = networkSSOEnabled; //OSUtils::jsonBool(network["ssoEnabled"], false);
 	nc->ssoVersion = info.version;
 
 	if (info.version == 0) {
@@ -1478,7 +1447,9 @@ void EmbeddedNetworkController::_request(
 			Utils::scopy(nc->centralAuthURL, sizeof(nc->centralAuthURL), info.centralAuthURL.c_str());
 		}
 		if (!info.issuerURL.empty()) {
+			#ifdef ZT_DEBUG
 			fprintf(stderr, "copying issuerURL to nc: %s\n", info.issuerURL.c_str());
+			#endif
 			Utils::scopy(nc->issuerURL, sizeof(nc->issuerURL), info.issuerURL.c_str());
 		}
 		if (!info.ssoNonce.empty()) {
@@ -1858,6 +1829,8 @@ void EmbeddedNetworkController::_startThreads()
 	const long hwc = std::max((long)std::thread::hardware_concurrency(),(long)1);
 	for(long t=0;t<hwc;++t) {
 		_threads.emplace_back([this]() {
+			std::vector<_MemberStatusKey> expired;
+			nlohmann::json network, member;
 			for(;;) {
 				_RQEntry *qe = (_RQEntry *)0;
 				auto timedWaitResult = _queue.get(qe, 1000);
@@ -1876,28 +1849,31 @@ void EmbeddedNetworkController::_startThreads()
 					}
 				}
 
-				std::set< std::pair<uint64_t, uint64_t> > soon;
-				std::set< std::pair<uint64_t, uint64_t> > expired;
-				_db.membersExpiring(soon, expired);
-
-				for(auto s=soon.begin();s!=soon.end();++s) {
-					Identity identity;
-					Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> lastMetaData;
-					{
-						std::unique_lock<std::mutex> ll(_memberStatus_l);
-						auto ms = _memberStatus.find(_MemberStatusKey(s->first, s->second));
-						if (ms != _memberStatus.end()) {
-							lastMetaData = ms->second.lastRequestMetaData;
-							identity = ms->second.identity;
+				expired.clear();
+				int64_t now = OSUtils::now();
+				{
+					std::lock_guard<std::mutex> l(_expiringSoon_l);
+					for(auto s=_expiringSoon.begin();s!=_expiringSoon.end();) {
+						const int64_t when = s->first;
+						if (when <= now) {
+							// The user may have re-authorized, so we must actually look it up and check.
+							network.clear();
+							member.clear();
+							if (_db.get(s->second.networkId, network, s->second.nodeId, member)) {
+								int64_t authenticationExpiryTime = (int64_t)OSUtils::jsonInt(member["authenticationExpiryTime"], 0);
+								if (authenticationExpiryTime <= now) {
+									expired.push_back(s->second);
+								}
+							}
+							_expiringSoon.erase(s++);
+						} else {
+							// Don't bother going further into the future than necessary.
+							break;
 						}
 					}
-					if (identity) {
-						request(s->first,InetAddress(),0,identity,lastMetaData);
-					}
 				}
-
 				for(auto e=expired.begin();e!=expired.end();++e) {
-					onNetworkMemberDeauthorize(nullptr, e->first, e->second);
+					onNetworkMemberDeauthorize(nullptr, e->networkId, e->nodeId);
 				}
 			}
 		});

controller/EmbeddedNetworkController.hpp

@@ -35,7 +35,7 @@
 #include "../osdep/Thread.hpp"
 #include "../osdep/BlockingQueue.hpp"
 
-#include "../ext/json/json.hpp"
+#include <nlohmann/json.hpp>
 
 #include "DB.hpp"
 #include "DBMirrorSet.hpp"
@@ -109,6 +109,7 @@ private:
 			RQENTRY_TYPE_REQUEST = 0
 		} type;
 	};
+
 	struct _MemberStatusKey
 	{
 		_MemberStatusKey() : networkId(0),nodeId(0) {}
@@ -116,11 +117,13 @@ private:
 		uint64_t networkId;
 		uint64_t nodeId;
 		inline bool operator==(const _MemberStatusKey &k) const { return ((k.networkId == networkId)&&(k.nodeId == nodeId)); }
+		inline bool operator<(const _MemberStatusKey &k) const { return (k.networkId < networkId) || ((k.networkId == networkId)&&(k.nodeId < nodeId)); }
 	};
 	struct _MemberStatus
 	{
-		_MemberStatus() : lastRequestTime(0),vMajor(-1),vMinor(-1),vRev(-1),vProto(-1) {}
-		uint64_t lastRequestTime;
+		_MemberStatus() : lastRequestTime(0),authenticationExpiryTime(-1),vMajor(-1),vMinor(-1),vRev(-1),vProto(-1) {}
+		int64_t lastRequestTime;
+		int64_t authenticationExpiryTime;
 		int vMajor,vMinor,vRev,vProto;
 		Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> lastRequestMetaData;
 		Identity identity;
@@ -152,6 +155,9 @@ private:
 	std::unordered_map< _MemberStatusKey,_MemberStatus,_MemberStatusHash > _memberStatus;
 	std::mutex _memberStatus_l;
 
+	std::set< std::pair<int64_t, _MemberStatusKey> > _expiringSoon;
+	std::mutex _expiringSoon_l;
+
 	RedisConfig *_rc;
 	std::string _ssoRedirectURL;
 };