Merge branch 'dev' of http://git.int.zerotier.com/zerotier/ZeroTierOne into dev

20x20, 60x60, 1024x1024 pngs
getting ready for an iOS release

.gitignore

@@ -9,6 +9,8 @@
 .DS_Store
 .Apple*
 Thumbs.db
+@eaDir
+._*
 
 # Windows build droppings
 /windows/ZeroTierOne.sdf
@@ -25,10 +27,20 @@ Thumbs.db
 /windows/Release
 /windows/WebUIWrapper/bin
 /windows/WebUIWrapper/obj
+/windows/lib
 /ext/installfiles/windows/ZeroTier One-SetupFiles
 /ext/installfiles/windows/Prerequisites
 /ext/installfiles/windows/*-cache
 /ZeroTier One.msi
+/windows/.vs
+*.vcxproj.backup
+/windows/TapDriver6/Win7Debug
+/windows/TapDriver6/win7Release
+/windows/*.db
+/windows/*.opendb
+enc_temp_folder
+/windows/copyutil/bin
+/windows/copyutil/obj
 
 # *nix/Mac build droppings
 /build-*
@@ -49,7 +61,7 @@ zt1-src.tar.gz
 *.pid
 *.pkg
 *.o
-*.a
+/*.a
 *.dylib
 *.so
 *.so.*
@@ -59,16 +71,15 @@ zt1-src.tar.gz
 *.rpm
 *.autosave
 *.tmp
-doc/*.1
-doc/*.2
-doc/*.8
 .depend
 node_modules
+zt1_update_*
 debian/files
 debian/zerotier-one
 debian/zerotier-one*.debhelper
 debian/*.log
 debian/zerotier-one.substvars
+root-watcher/config.json
 
 # Java/Android/JNI build droppings
 java/obj/
@@ -82,3 +93,26 @@ java/build_win32/
 windows/WinUI/obj/
 windows/WinUI/bin/
 windows/ZeroTierOne/Debug/
+/ext/installfiles/windows/chocolatey/zerotier-one/*.nupkg
+
+# Miscellaneous mac/Xcode droppings
+.DS_Store
+.Trashes
+*.swp
+*~.nib
+DerivedData/
+build/
+*.pbxuser
+*.mode1v3
+*.mode2v3
+*.perspectivev3
+!default.pbxuser
+!default.mode1v3
+!default.mode2v3
+!default.perspectivev3
+*.xccheckout
+xcuserdata/
+ext/librethinkdbxx/build
+.vscode
+__pycache__
+*~

AUTHORS.md

@@ -25,13 +25,13 @@
 
 ## Third-Party Code
 
-These are included in ext/ for platforms that do not have them available in common repositories. Otherwise they may be linked and the package may ship with them as dependencies.
+ZeroTier includes the following third party code, either in ext/ or incorporated into the ZeroTier core.
 
  * LZ4 compression algorithm by Yann Collet
 
-   * Files: ext/lz4/*
+   * Files: node/Packet.cpp (bundled within anonymous namespace)
    * Home page: http://code.google.com/p/lz4/
-   * License grant: BSD attribution
+   * License grant: BSD 2-clause
 
  * http-parser by Joyent, Inc. (many authors)
 
@@ -39,11 +39,11 @@ These are included in ext/ for platforms that do not have them available in comm
    * Home page: https://github.com/joyent/http-parser/
    * License grant: MIT/Expat
 
- * json-parser by James McLaughlin
+ * C++11 json (nlohmann/json) by Niels Lohmann
 
-   * Files: ext/json-parser/*
-   * Home page: https://github.com/udp/json-parser/
-   * License grant: BSD attribution
+   * Files: ext/json/*
+   * Home page: https://github.com/nlohmann/json
+   * License grant: MIT
 
  * TunTapOSX by Mattias Nissler
 
@@ -55,26 +55,19 @@ These are included in ext/ for platforms that do not have them available in comm
  * tap-windows6 by the OpenVPN project
 
    * Files: windows/TapDriver6/*
-   * Home page:
-       https://github.com/OpenVPN/tap-windows6/
+   * Home page: https://github.com/OpenVPN/tap-windows6/
    * License grant: GNU GPL v2
    * ZeroTier Modifications: change name of driver to ZeroTier, add ioctl() to get L2 multicast memberships (source is in ext/ and modifications inherit GPL)
 
- * Salsa20 stream cipher, Curve25519 elliptic curve cipher, Ed25519
-   digital signature algorithm, and Poly1305 MAC algorithm, all by
-   Daniel J. Bernstein
+ * Salsa20 stream cipher, Curve25519 elliptic curve cipher, Ed25519 digital signature algorithm, and Poly1305 MAC algorithm, all by Daniel J. Bernstein
 
-   * Files:
-       node/Salsa20.hpp
-       node/C25519.hpp
-       node/Poly1305.hpp
+   * Files: node/Salsa20.* node/C25519.* node/Poly1305.*
    * Home page: http://cr.yp.to/
    * License grant: public domain
+   * ZeroTier Modifications: slight cryptographically-irrelevant modifications for inclusion into ZeroTier core
 
  * MiniUPNPC and libnatpmp by Thomas Bernard
 
-   * Files:
-       ext/libnatpmp/*
-       ext/miniupnpc/*
+   * Files: ext/libnatpmp/* ext/miniupnpc/*
    * Home page: http://miniupnp.free.fr/
    * License grant: BSD attribution no-endorsement

CMakeLists.txt

@@ -0,0 +1,12 @@
+# CMake build script for libzerotiercore.a
+
+cmake_minimum_required (VERSION 2.8)
+project (zerotiercore)
+
+set (PROJ_DIR ${PROJECT_SOURCE_DIR})
+set (ZT_DEFS -std=c++11)
+
+file(GLOB core_src_glob ${PROJ_DIR}/node/*.cpp)
+add_library(zerotiercore STATIC ${core_src_glob})
+
+target_compile_options(zerotiercore PRIVATE ${ZT_DEFS})

COPYING

@@ -1,5 +1,5 @@
 ZeroTier One, an endpoint server for the ZeroTier virtual network layer.
-Copyright © 2011–2016 ZeroTier, Inc.
+Copyright © 2011–2018 ZeroTier, Inc.
 
 ZeroTier One is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

Jenkinsfile

@@ -0,0 +1,84 @@
+#!/usr/bin/env groovy
+
+node('master') {
+    checkout scm
+    
+    def changelog = getChangeLog currentBuild
+
+    mattermostSend "Building ${env.JOB_NAME} #${env.BUILD_NUMBER} \n Change Log: \n ${changelog}"
+}
+
+parallel 'centos7': {
+    node('centos7') {
+        try {
+            checkout scm
+
+	        stage('Build Centos 7') {
+                sh 'make -f make-linux.mk'
+            }
+        }
+        catch (err) {
+            currentBuild.result = "FAILURE"
+            mattermostSend color: '#ff0000', message: "${env.JOB_NAME} broken on Centos 7 (<${env.BUILD_URL}|Open>)"
+
+            throw err
+        }
+    }
+}, 'android-ndk': {
+    node('android-ndk') {
+        try {
+            checkout scm
+	
+            stage('Build Android NDK') { 
+                sh "/android/android-ndk-r15b/ndk-build -C $WORKSPACE/java ZT1=${WORKSPACE}"
+            }
+        }
+        catch (err) {
+            currentBuild.result = "FAILURE"
+            mattermostSend color: '#ff0000', message: "${env.JOB_NAME} broken on Android NDK (<${env.BUILD_URL}|Open>)"
+
+            throw err
+        }
+    }
+}, 'macOS': {
+    node('macOS') {
+        try {
+            checkout scm
+
+            stage('Build macOS') {
+                sh 'make -f make-mac.mk'
+            }
+
+            stage('Build macOS UI') {
+                sh 'cd macui && xcodebuild -target "ZeroTier One" -configuration Debug'
+            }
+        }
+        catch (err) {
+            currentBuild.result = "FAILURE"
+            mattermostSend color: '#ff0000', message: "${env.JOB_NAME} broken on macOS (<${env.BUILD_URL}|Open>)"
+
+            throw err
+        }
+    }
+}, 'windows': {
+    node('windows') {
+        try {
+            checkout scm
+            
+            stage('Build Windows') {
+                bat '''CALL "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\vcvarsall.bat" amd64
+git clean -dfx
+msbuild windows\\ZeroTierOne.sln
+'''
+            }
+        }
+        catch (err) {
+            currentBuild.result = "FAILURE"
+            mattermostSend color: '#ff0000', message: "${env.JOB_NAME} broken on Windows (<${env.BUILD_URL}|Open>)"
+
+            throw err
+        }
+    }
+}
+
+mattermostSend color: "#00ff00", message: "${env.JOB_NAME} #${env.BUILD_NUMBER} Complete (<${env.BUILD_URL}|Show More...>)"

LICENSE.txt

@@ -0,0 +1,37 @@
+ZeroTier One - Network Virtualization Everywhere
+Copyright (C) 2011-2017  ZeroTier, Inc.  https://www.zerotier.com/
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+See LICENSE.GPL-3 for the full GNU GPL v3 license.
+
+--
+
+You can be released from the requirements of the license by purchasing
+a commercial license. Buying such a license is mandatory as soon as you
+develop commercial closed-source software that incorporates or links
+directly against ZeroTier software without disclosing the source code
+of your own application.
+
+--
+
+The above license does not apply to third party code included with or
+linked against by ZeroTier software. See the third party code section
+of the AUTHORS.md for an index of third party software included in
+this software repository.
+
+Licenses for third party code are all relatively permissive: MIT,
+BSD, and public domain. The only exception is the tap-windows driver
+which is under the GPLv2, but this is only needed to produce the
+binary tap device driver used by the ZeroTier service on Windows.

Makefile

@@ -11,8 +11,18 @@ ifeq ($(OSTYPE),Linux)
 endif
 
 ifeq ($(OSTYPE),FreeBSD)
-	include make-freebsd.mk
+	CC=clang
+	CXX=clang++
+	ZT_BUILD_PLATFORM=7
+	include make-bsd.mk
 endif
 ifeq ($(OSTYPE),OpenBSD)
-	include make-freebsd.mk
+	CC=egcc
+	CXX=eg++
+	ZT_BUILD_PLATFORM=9
+	include make-bsd.mk
+endif
+
+ifeq ($(OSTYPE),NetBSD)
+	include make-netbsd.mk
 endif

OFFICIAL-RELEASE-STEPS.md

@@ -0,0 +1,49 @@
+ZeroTier Official Release Steps
+======
+
+This is mostly for ZeroTier internal use, but others who want to do builds might find it helpful.
+
+Note: Many of these steps will require GPG and other signing keys that are kept in cold storage and must be mounted.
+
+# Bumping the Version and Preparing Installers
+
+The version must be incremented in all of the following files:
+
+    /version.h
+    /zerotier-one.spec
+    /debian/changelog
+    /ext/installfiles/mac/ZeroTier One.pkgproj
+    /ext/installfiles/windows/chocolatey/zerotier-one.nuspec
+    /ext/installfiles/windows/ZeroTier One.aip
+    /windows/WinUI/AboutView.xaml
+
+The final .AIP file can only be edited on Windows with [Advanced Installer Enterprise](http://www.advancedinstaller.com/). In addition to incrementing the version be sure that a new product code is generated. (The "upgrade code" GUID on the other hand must never change.)
+
+# Building for Supported Platforms
+
+## Macintosh
+
+Mac's easy. Just type:
+
+    make official
+
+You will need [Packages](http://s.sudre.free.fr/Software/Packages/about.html) and our release signing key in the keychain.
+
+## Linux
+
+See `LinuxBuild` environment on `linux-build` VM and use: `chroots/mount-build.sh`, `chroots/build.sh`, and the scripts in `build/` to make APT and RPM repositories.
+
+## Windows
+
+First load the Visual Studio solution and rebuild the UI and ZeroTier One in both x64 and i386 `Release` mode. Then load [Advanced Installer Enterprise](http://www.advancedinstaller.com/), check that the version is correct, and build. The build will fail if any build artifacts are missing, and Windows must have our product singing key (from DigiCert) available to sign the resulting MSI file. The MSI must then be tested on at least a few different CLEAN Windows VMs to ensure that the installer is valid and properly signed.
+
+*After the MSI is published to download.zerotier.com in the proper RELEASE/#.#.#/dist subfolder for its version* the Chocolatey package must be rebuilt and published. Open a command prompt, change to `ext/installfiles/windows/chocolatey`, and type `choco pack`. Then use `choco push` to push it to Chocolatey (API key required).
+
+    choco pack
+    choco push zerotier-one.#.#.#.nupkg -s https://chocolatey.org/
+
+Note that this does not cover rebuilding the drivers or their containing MSI projects, as this is typically not necessary and they are shipped in binary form in the repository for convenience.
+
+## iOS, Android
+
+... no docs here yet since this is done entirely out of band with regular installs.

README.md

@@ -1,52 +1,65 @@
 ZeroTier - A Planetary Ethernet Switch
 ======
 
-ZeroTier is a software-based managed Ethernet switch for planet Earth.
+ZeroTier is a smart programmable Ethernet switch for planet Earth.
 
-It erases the LAN/WAN distinction and makes VPNs, tunnels, proxies, and other kludges arising from the inflexible nature of physical networks obsolete. Everything is encrypted end-to-end and traffic takes the most direct (peer to peer) path available.
+It replaces the physical LAN/WAN boundary with a virtual one, allowing devices of any type at any location to be managed as if they all reside in the same cloud region or data center. All traffic is encrypted end-to-end and takes the most direct path available for minimum latency and maximum performance. The goals and design of ZeroTier are inspired by among other things the original [Google BeyondCorp](https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/43231.pdf) paper and the [Jericho Forum](https://en.wikipedia.org/wiki/Jericho_Forum).
 
-This repository contains ZeroTier One, a service that provides ZeroTier network connectivity to devices running Windows, Mac, Linux, iOS, Android, and FreeBSD and makes joining virtual networks as easy as joining IRC or Slack channels. It also contains the OS-independent core ZeroTier protocol implementation in [node/](node/).
-
-Visit [ZeroTier's site](https://www.zerotier.com/) for more information and [pre-built binary packages](https://www.zerotier.com/download.shtml). Apps for Android and iOS are available for free in the Google Play and Apple app stores.
+Visit [ZeroTier's site](https://www.zerotier.com/?pk_campaign=github_ZeroTierOne) for more information and [pre-built binary packages](https://www.zerotier.com/download.shtml?pk_campaign=github_ZeroTierOne). Apps for Android and iOS are available for free in the Google Play and Apple app stores.
 
 ### Getting Started
 
-ZeroTier's basic operation is easy to understand. Devices have 10-digit *ZeroTier addresses* like `89e92ceee5` and networks have 16-digit network IDs like `8056c2e21c000001`. All it takes for a device to join a network is its 16-digit ID, and all it takes for a network to authorize a device is its 10-digit address. Everything else is automatic.
+Everything in the ZeroTier world is controlled by two types of identifier: 40-bit/10-digit *ZeroTier addresses* and 64-bit/16-digit *network IDs*. A ZeroTier address identifies a node or "device" (laptop, phone, server, VM, app, etc.) while a network ID identifies a virtual Ethernet network that can be joined by devices.
 
-A "device" can be anything really: desktops, laptops, phones, servers, VMs/VPSes, containers, and even (soon) apps.
+Another way of thinking about it is that ZeroTier addresses are port numbers on a giant planetary-sized smart switch while network IDs are VLANs to which these ports can be assigned. For more details read about VL1 and VL2 in [the ZeroTier manual](https://www.zerotier.com/manual.shtml).
 
-For testing we provide a public virtual network called *Earth* with network ID `8056c2e21c000001`. On Linux and Mac you can do this with:
+*Network controllers* are ZeroTier nodes that act as access control certificate authorities and configuration managers for virtual networks. The first 40 bits (or 10 digits) of a network ID is the ZeroTier address of its controller. You can create networks with our [hosted controllers](https://my.zerotier.com/) and web UI/API or [host your own](controller/) if you don't mind posting some JSON configuration info or writing a script to do so.
 
-    sudo zerotier-cli join 8056c2e21c000001
+### Project Layout
 
-Now wait about 30 seconds and check your system with `ip addr list` or `ifconfig`. You'll see a new interface whose name starts with *zt* and it should quickly get an IPv4 and an IPv6 address. Once you see it get an IP, try pinging `earth.zerotier.net` at `29.209.112.93`. If you've joined Earth from more than one system, try pinging your other machine.
+ - `artwork/`: icons, logos, etc.
+ - `attic/`: old stuff and experimental code that we want to keep around for reference.
+ - `controller/`: the reference network controller implementation, which is built and included by default on desktop and server build targets.
+ - `debian/`: files for building Debian packages on Linux.
+ - `doc/`: manual pages and other documentation.
+ - `ext/`: third party libraries, binaries that we ship for convenience on some platforms (Mac and Windows), and installation support files.
+ - `include/`: include files for the ZeroTier core.
+ - `java/`: a JNI wrapper used with our Android mobile app. (The whole Android app is not open source but may be made so in the future.)
+ - `macui/`: a Macintosh menu-bar app for controlling ZeroTier One, written in Objective C.
+ - `node/`: the ZeroTier virtual Ethernet switch core, which is designed to be entirely separate from the rest of the code and able to be built as a stand-alone OS-independent library. Note to developers: do not use C++11 features in here, since we want this to build on old embedded platforms that lack C++11 support. C++11 can be used elsewhere.
+ - `osdep/`: code to support and integrate with OSes, including platform-specific stuff only built for certain targets.
+ - `service/`: the ZeroTier One service, which wraps the ZeroTier core and provides VPN-like connectivity to virtual networks for desktops, laptops, servers, VMs, and containers.
+ - `tcp-proxy/`: TCP proxy code run by ZeroTier, Inc. to provide TCP fallback (this will die soon!).
+ - `windows/`: Visual Studio solution files, Windows service code for ZeroTier One, and the Windows task bar app UI.
 
-*(IPv4 addresses for Earth are assigned from the block 28.0.0.0/7, which is not a part of the public Internet but is non-standard for private networks. It's used to avoid IP conflicts during testing. Your networks can run any IP addressing scheme you want.)*
+The base path contains the ZeroTier One service main entry point (`one.cpp`), self test code, makefiles, etc.
 
-If you don't want to belong to a giant Ethernet party line anymore, just type:
+### Build and Platform Notes
 
-    sudo zerotier-cli leave 8056c2e21c000001
+To build on Mac and Linux just type `make`. On FreeBSD and OpenBSD `gmake` (GNU make) is required and can be installed from packages or ports. For Windows there is a Visual Studio solution in `windows/'.
 
-The *zt* interface will disappear. You're no longer on the network.
-
-To create networks of your own you'll need a network controller. You can use [our hosted controller at my.zerotier.com](https://my.zerotier.com) which is free for up to 100 devices on an unlimited number of networks, or you can build your own controller and run it through its local JSON API. See [README.md in controller/](controller/) for more information.
-
-### Building from Source
-
-For Mac, Linux, and BSD, just type "make" (or "gmake" on BSD). You won't need much installed; here are the requirements for various platforms:
-
- * **Mac**: Xcode command line tools. It should build on OSX 10.7 or newer.
- * **Linux**: gcc/g++ (4.9 or newer recommended) or clang/clang++ (3.4 or newer recommended) Makefile will use clang by default if available. The Linux build will auto-detect the presence of development headers for *json-parser*, *http-parser*, *li8bnatpmp*, and *libminiupnpc* and will link against the system libraries for these if they are present and recent enough. Otherwise the bundled versions in [ext/](ext/) will be used. Type `make install` to install the binaries and other files on the system, though this will not create init.d or systemd links.
- * **FreeBSD**: C++ compiler (G++ usually) and GNU make (gmake).
-
-Each supported platform has its own *make-XXX.mk* file that contains the actual make rules for the platform. The right .mk file is included by the main Makefile based on the GNU make *OSTYPE* variable. Take a look at the .mk file for your platform for other targets, debug build rules, etc.
+ - **Mac**
+   - Xcode command line tools for OSX 10.7 or newer are required.
+   - Tap device driver kext source is in `ext/tap-mac` and a signed pre-built binary can be found in `ext/bin/tap-mac`. You should not need to build it yourself. It's a fork of [tuntaposx](http://tuntaposx.sourceforge.net) with device names changed to `zt#`, support for a larger MTU, and tun functionality removed.
+ - **Linux**
+   - The minimum compiler versions required are GCC/G++ 4.9.3 or CLANG/CLANG++ 3.4.2.
+   - Linux makefiles automatically detect and prefer clang/clang++ if present as it produces smaller and slightly faster binaries in most cases. You can override by supplying CC and CXX variables on the make command line.
+   - CentOS 7 ships with a version of GCC/G++ that is too old, but a new enough version of CLANG can be found in the *epel* repositories. Type `yum install epel-release` and then `yum install clang` to build there.
+ - **Windows**
+   - Windows 7 or newer is supported. This *may* work on Vista but isn't officially supported there. It will not work on Windows XP.
+   - We build with Visual Studio 2015. Older versions may not work. Clang or MinGW will also probably work but may require some makefile hacking.
+   - Pre-built signed Windows drivers are included in `ext/bin/tap-windows-ndis6`. The MSI files found there will install them on 32-bit and 64-bit systems. We don't recommend trying to build Windows drivers from scratch unless you know what you're doing. One does not simply "build" a Windows driver.
+ - **FreeBSD**
+   - Tested most recently on FreeBSD-11. Older versions may work but we're not sure.
+   - GCC/G++ 4.9 and gmake are required. These can be installed from packages or ports. Type `gmake` to build.
+ - **OpenBSD**
+   - There is a limit of four network memberships on OpenBSD as there are only four tap devices (`/dev/tap0` through `/dev/tap3`). We're not sure if this can be increased.
+   - OpenBSD lacks `getifmaddrs` (or any equivalent method) to get interface multicast memberships. As a result multicast will only work on OpenBSD for ARP and NDP (IP/MAC lookup) and not for other purposes.
+   - Only tested on OpenBSD 6.0. Older versions may not work.
+   - GCC/G++ 4.9 and gmake are required and can be installed using `pkg_add` or from ports. They get installed in `/usr/local/bin` as `egcc` and `eg++` and our makefile is pre-configured to use them on OpenBSD.
 
 Typing `make selftest` will build a *zerotier-selftest* binary which unit tests various internals and reports on a few aspects of the build environment. It's a good idea to try this on novel platforms or architectures.
 
-Windows, of course, is special. We build for Windows with Microsoft Visual Studio 2012 on Windows 7. A solution file is located in the *windows/* subfolder. Newer versions of Visual Studio (and Windows) may work but haven't been tested. Older versions almost certainly will not, since they lack things like *stdint.h* and certain STL features. MinGW or other ports of gcc/clang to Windows should also work but haven't been tested.
-
-32 and 64 bit X86 and ARM (e.g. Raspberry Pi, Android) are officially supported. Community members have built for MIPS and Sparc without issues.
-
 ### Running
 
 Running *zerotier-one* with -h will show help.
@@ -62,7 +75,7 @@ The service is controlled via the JSON API, which by default is available at 127
 Here's where home folders live (by default) on each OS:
 
  * **Linux**: `/var/lib/zerotier-one`
- * **FreeBSD**: `/var/db/zerotier-one`
+ * **FreeBSD** / **OpenBSD**: `/var/db/zerotier-one`
  * **Mac**: `/Library/Application Support/ZeroTier/One`
  * **Windows**: `\ProgramData\ZeroTier\One` (That's for Windows 7. The base 'shared app data' folder might be different on different Windows versions.)
 
@@ -88,7 +101,7 @@ ZeroTier One peers will automatically locate each other and communicate directly
 
 Users behind certain types of firewalls and "symmetric" NAT devices may not able able to connect to external peers directly at all. ZeroTier has limited support for port prediction and will *attempt* to traverse symmetric NATs, but this doesn't always work. If P2P connectivity fails you'll be bouncing UDP packets off our relay servers resulting in slower performance. Some NAT router(s) have a configurable NAT mode, and setting this to "full cone" will eliminate this problem. If you do this you may also see a magical improvement for things like VoIP phones, Skype, BitTorrent, WebRTC, certain games, etc., since all of these use NAT traversal techniques similar to ours.
 
-If you're interested, there's a [technical deep dive about NAT traversal on our blog](https://www.zerotier.com/blog/?p=226). A troubleshooting tool to help you diagnose NAT issues is planned for the future as are uPnP/IGD/NAT-PMP and IPv6 transport.
+If you're interested, there's a [technical deep dive about NAT traversal on our blog](https://www.zerotier.com/blog/?p=226?pk_campaign=github_ZeroTierOne). A troubleshooting tool to help you diagnose NAT issues is planned for the future as are uPnP/IGD/NAT-PMP and IPv6 transport.
 
 If a firewall between you and the Internet blocks ZeroTier's UDP traffic, you will fall back to last-resort TCP tunneling to rootservers over port 443 (https impersonation). This will work almost anywhere but is *very slow* compared to UDP or direct peer to peer connectivity.
 

RELEASE-NOTES.md

@@ -0,0 +1,187 @@
+ZeroTier Release Notes
+======
+
+# 2018-05-08 -- Version 1.2.10
+
+ * Fix bug loading `moons.d/` files for federated root operation.
+ * Fix compile problem with ZT_DEBUG on some versions of `clang`
+ * Fix slow network startup bug related to loading of `networks.d/` cache files
+
+# 2018-04-27 -- Version 1.2.8
+
+ * Linux version once again builds with PIE (position independent executable) flags
+ * Fixed bug in zerotier-idtool file sign and verify
+ * Fixed minor OSX app typo
+ * Merged alpha NetBSD support (mostly untested, so YMMV)
+ * Merged several minor typo and one-liner bug fixes
+
+# 2018-04-17 -- Version 1.2.6
+
+ * Features and Core Improvements
+    * Path selection has been overhauled to improve path stability, simplify code, and prepare for multi-path and trunking in the next major release.
+    * This version introduces remote tracing for remote diagnostics. Network controllers can set a node (usually the controller itself) to receive remote tracing events from all members of the network or from select members. Events are only sent if they pertain to a given network for security reasons.
+    * Multicast replication can now be done by designated multicast replicators on a network (flagged as such at the controller) rather than by the sender. Most users won't want this, but it's useful for specialized use cases on hub-and-spoke networks and for low-power devices.
+    * Cryptographic performance improvements on several platforms.
+    * Multithreaded performance improvements throughout the code base, including the use of an inline lightweight spinlock for low-contention resources.
+ * Bugs fixed
+    * Disappearing routes on Mac (GitHub issue #600)
+    * Route flapping and path instability in some dual-stack V4/V6 networks
+    * Blacklist (in local.conf) doesn't work reliably (GitHub issue #656)
+    * Connection instabilities due to unsigned integer overflows in timing comparisons (use int64_t instead of uint64_t)
+    * Binaries don't run on some older or lower-end 32-bit ARM chips (build problem)
+    * ARM NEON crypto code crashes (build problem)
+    * Fixed some lock ordering issues revealed by "valgrind" tool
+    * The "zerotier-idtool" command could not be accessed from "zerotier-one" via command line switch
+    * Leaking sockets on some platforms when uPnP/NAT-PMP is enabled
+    * Fixed two very rare multithreading issues that were only observed on certain systems
+ * Platform-Specific Changes
+    * MacOS
+        * Installer now loads the kernel extension right away so that High Sierra users will see the prompt to authorize it. This is done in the "Security & Privacy" preference pane and must be done driectly on the console (not via remote desktop). On High Sierra and newer kexts must be authorized at the console via security settings system preferences pane.
+    * Windows
+        * The Windows installer should now install the driver without requiring a special prompt in most cases. This should make it easier for our packages to be accepted into and updated in the Chocolatey repository and should make it easier to perform remote installs across groups of machines using IT management and provisioning tools.
+        * The Windows official packages are now signed with an EV certificate (with hardware key).
+        * The Windows UI can now log into ZeroTier Central and join networks via the Central API.
+        * The `zerotier-idtool` command should now work on Windows without ugly hacks.
+        * Upgraded the installer version.
+        * Made a few changes to hopefully fix sporadic "will not uninstall" problems, though we cannot duplicate these issues ourselves.
+    * Linux
+        * Device names are now generated deterministically based on network IDs for all newly joined networks.
+    * Android
+        * Multicast now works on Android in most cases! Android apps can send and receive multicast and subscribe to multicast group IPs. Note that in some cases the app must bind to the specific correct interface for this to work.
+        * IPv6 can be disabled in UI for cases where it causes problems.
+
+# 2017-04-20 -- Version 1.2.4
+
+ * Managed routes are now only bifurcated for the default route. This is a change in behavior, though few people will probably notice. Bifurcating all managed routes was causing more trouble than it was worth for most users.
+ * Up to 2X crypto speedup on x86-64 (except Windows, which will take some porting) and 32-bit ARM platforms due to integration of fast assembly language implementations of Salsa20/12 from the [supercop](http://bench.cr.yp.to/supercop.html) code base. These were written by Daniel J. Bernstein and are in the public domain. My Macbook Pro (Core i5 2.8ghz) now does almost 1.5GiB/sec Salsa20/12 per core and a Raspberry Pi got a 2X boost. 64-bit ARM support and Windows support will take some work but should not be too hard.
+ * Refactored code that manages credentials to greatly reduce memory use in most cases. This may also result in a small performance improvement.
+ * Reworked and simplified path selection and priority logic to fix path instability and dead path persistence edge cases. There have been some sporadic reports of persistent path instabilities and dead paths hanging around that take minutes to resolve. These have proven difficult to reproduce in house, but hopefully this will fix them. In any case it seems to speed up path establishment in our tests and it makes the code simpler and more readable.
+ * Eliminated some unused cruft from the code around path management and in the peer class.
+ * Fixed an issue causing build problems on some MIPS architecture systems.
+ * Fixed Windows forgetting routes on sleep/wake or in some other circumstances. (GitHub issue #465)
+
+# 2017-03-17 -- Version 1.2.2
+
+ * A bug causing unreliable multicast propagation (GitHub issue #461).
+ * A crash in ARM binaries due to a build chain and flags problem.
+ * A bug in the network controller preventing members from being listed (GitHub issue #460).
+
+# 2017-03-14 -- Version 1.2.0
+
+Version 1.2.0 is a major milestone release representing almost nine months of work. It includes our rules engine for distributed network packet filtering and security monitoring, federated roots, and many other architectural and UI improvements and bug fixes.
+
+## New Features in 1.2.0
+
+### The ZeroTier Rules Engine
+
+The largest new feature in 1.2.0, and the product of many months of work, is our advanced network rules engine. With this release we achieve traffic control, security monitoring, and micro-segmentation capability on par with many enterprise SDN solutions designed for use in advanced data centers and corporate networks.
+
+Rules allow you to filter packets on your network and vector traffic to security observers. Security observation can be performed in-band using REDIRECT or out of band using TEE.
+
+Tags and capabilites provide advanced methods for implementing fine grained permission structures and micro-segmentation schemes without bloating the size and complexity of your rules table.
+
+See the [rules engine announcement blog post](https://www.zerotier.com/blog/?p=927) for an in-depth discussion of theory and implementation. The [manual](https://www.zerotier.com/manual.shtml) contains detailed information on rule, tag, and capability use, and the `rule-compiler/` subfolder of the ZeroTier source tree contains a JavaScript function to compile rules in our human-readable rule definition language into rules suitable for import into a network controller. (ZeroTier Central uses this same script to compile rules on [my.zerotier.com](https://my.zerotier.com/).)
+
+### Root Server Federation
+
+It's now possible to create your own root servers and add them to the root server pool on your nodes. This is done by creating what's called a "moon," which is a signed enumeration of root servers and their stable points on the network. Refer to the [manual](https://www.zerotier.com/manual.shtml) for instructions.
+
+Federated roots achieve a number of things:
+
+ * You can deploy your own infrastructure to reduce dependency on ours.
+ * You can deploy roots *inside your LAN* to ensure that network connectivity inside your facility still works if the Internet goes down. This is the first step toward making ZeroTier viable as an in-house SDN solution.
+ * Roots can be deployed inside national boundaries for countries with data residency laws or "great firewalls." (As of 1.2.0 there is still no way to force all traffic to use these roots, but that will be easy to do in a later version.)
+ * Last but not least this makes ZeroTier somewhat less centralized by eliminating any hard dependency on ZeroTier, Inc.'s infrastructure.
+
+Our roots will of course remain and continue to provide zero-configuration instant-on deployment, a secure global authority for identities, and free traffic relaying for those who can't establish peer to peer connections.
+
+### Local Configuration
+
+An element of our design philosophy is "features are bugs." This isn't an absolute dogma but more of a guiding principle. We try as hard as we can to avoid adding features, especially "knobs" that must be tweaked by a user.
+
+As of 1.2.0 we've decided that certain knobs are unavoidable, and so there is now a `local.conf` file that can be used to configure them. See the ZeroTier One documentation for these. They include:
+
+ * Blacklisting interfaces you want to make sure ZeroTier doesn't use for network traffic, such as VPNs, slow links, or backplanes designated for only certain kinds of traffic.
+ * Turning uPnP/NAT-PMP on or off.
+ * Configuring software updates on Windows and Mac platforms.
+ * Defining trusted paths (the old trusted paths file is now deprecated)
+ * Setting the ZeroTier main port so it doesn't have to be changed on the command line, which is very inconvenient in many cases.
+
+### Improved In-Band Software Updates
+
+A good software update system for Windows and Mac clients has been a missing feature in previous versions. It does exist but we've been shy about using it so far due to its fragility in some environments.
+
+We've greatly improved this mechanism in 1.2.0. Not only does it now do a better job of actually invoking the update, but it also transfers updates in-band using the ZeroTier protocol. This means it can work in environments that do not allows http/https traffic or that force it through proxies. There's also now an update channel setting: `beta` or `release` (the default).
+
+Software updates are authenticated three ways:
+
+ 1. ZeroTier's own signing key is used to sign all updates and this signature is checked prior to installation. ZeroTier, Inc.'s signatures are performed on an air-gapped machine.
+
+ 2. Updates for Mac and Windows are signed using Apple and Microsoft (DigiCert EV) keys and will not install unless these signatures are also valid.
+
+ 3. The new in-band update mechanism also authenticates the source of the update via ZeroTier's built-in security features. This provides transport security, while 1 and 2 provide security of the update at rest.
+
+Updates are now configurable via `local.conf`. There are three options: `disable`, `download`, and `apply`. The third (apply) is the default for official builds on Windows and Mac, making updates happen silently and automatically as they do for popular browsers like Chrome and Firefox. Updates are disabled by default on Linux and other Unix-type systems as these are typically updated through package managers.
+
+### Path Link Quality Awareness
+
+Version 1.2.0 is now aware of the link quality of direct paths with other 1.2.0 nodes. This information isn't used yet but is visible through the JSON API. (Quality always shows as 100% with pre-1.2.0 nodes.) Quality is measured passively with no additional overhead using a counter based packet loss detection algorithm.
+
+This information is visible from the command line via `listpeers`:
+
+    200 listpeers XXXXXXXXXX 199.XXX.XXX.XXX/9993;10574;15250;1.00 48 1.2.0 LEAF
+    200 listpeers XXXXXXXXXX 195.XXX.XXX.XXX/45584;467;7608;0.44 290 1.2.0 LEAF
+
+The first peer's path is at 100% (1.00), while the second peer's path is suffering quite a bit of packet loss (0.44).
+
+Link quality awareness is a precursor to intelligent multi-path and QoS support, which will in future versions bring us to feature parity with SD-WAN products like Cisco iWAN.
+
+### Security Improvements
+
+Version 1.2.0 adds anti-DOS (denial of service) rate limits and other hardening for improved resiliency against a number of denial of service attack scenarios.
+
+It also adds a mechanism for instantaneous credential revocation. This can be used to revoke certificates of membership instantly to kick a node off a network (for private networks) and also to revoke capabilities and tags. The new controller sends revocations by default when a peer is de-authorized.
+
+Revocations propagate using a "rumor mill" peer to peer algorithm. This means that a controller need only successfully send a revocation to at least one member of a network with connections to other active members. At this point the revocation will flood through the network peer to peer very quickly. This helps make revocations more robust in the face of poor connectivity with the controller or attempts to incapacitate the controller with denial of service attacks, as well as making revocations faster on huge networks.
+
+### Windows and Macintosh UI Improvements (ZeroTier One)
+
+The Mac has a whole new UI built natively in Objective-C. It provides a pulldown similar in appearance and operation to the Mac WiFi task bar menu.
+
+The Windows UI has also been improved and now provides a task bar icon that can be right-clicked to manage networks. Both now expose managed route and IP permissions, allowing nodes to easily opt in to full tunnel operation if you have a router configured on your network.
+
+### Ad-Hoc Networks
+
+A special kind of public network called an ad-hoc network may be accessed by joining a network ID with the format:
+
+    ffSSSSEEEE000000
+    | |   |   |
+    | |   |   Reserved for future use, must be 0
+    | |   End of port range (hex)
+    | Start of port range (hex)
+    Reserved ZeroTier address prefix indicating a controller-less network
+
+Ad-hoc networks are public (no access control) networks that have no network controller. Instead their configuration and other credentials are generated locally. Ad-hoc networks permit only IPv6 UDP and TCP unicast traffic (no multicast or broadcast) using 6plane format NDP-emulated IPv6 addresses. In addition an ad-hoc network ID encodes an IP port range. UDP packets and TCP SYN (connection open) packets are only allowed to desintation ports within the encoded range.
+
+For example `ff00160016000000` is an ad-hoc network allowing only SSH, while `ff0000ffff000000` is an ad-hoc network allowing any UDP or TCP port.
+
+Keep in mind that these networks are public and anyone in the entire world can join them. Care must be taken to avoid exposing vulnerable services or sharing unwanted files or other resources.
+
+### Network Controller (Partial) Rewrite
+
+The network controller has been largely rewritten to use a simple in-filesystem JSON data store in place of SQLite, and it is now included by default in all Windows, Mac, Linux, and BSD builds. This means any desktop or server node running ZeroTier One can now be a controller with no recompilation needed.
+
+If you have data in an old SQLite3 controller we've included a NodeJS script in `controller/migrate-sqlite` to migrate data to the new format. If you don't migrate, members will start getting `NOT_FOUND` when they attempt to query for updates.
+
+## Major Bug Fixes in 1.2.0
+
+ * **The Windows HyperV 100% CPU bug is FINALLY DEAD**: This long-running problem turns out to have been an issue with Windows itself, but one we were triggering by placing invalid data into the Windows registry. Microsoft is aware of the issue but we've also fixed the triggering problem on our side. ZeroTier should now co-exist quite well with HyperV and should now be able to be bridged with a HyperV virtual switch.
+ * **Segmenation faults on musl-libc based Linux systems**: Alpine Linux and some embedded Linux systems that use musl libc (a minimal libc) experienced segmentation faults. These were due to a smaller default stack size. A work-around that sets the stack size for new threads has been added.
+ * **Windows firewall blocks local JSON API**: On some Windows systems the firewall likes to block 127.0.0.1:9993 for mysterious reasons. This is now fixed in the installer via the addition of another firewall exemption rule.
+ * **UI crash on embedded Windows due to missing fonts**: The MSI installer now ships fonts and will install them if they are not present, so this should be fixed.
+
+## Other Improvements in 1.2.0
+
+ * **Improved dead path detection**: ZeroTier is now more aggressive about expiring paths that do not seem to be active. If a path seems marginal it is re-confirmed before re-use.
+ * **Minor performance improvements**: We've reduced unnecessary memcpy's and made a few other performance improvements in the core.
+ * **Linux static binaries**: For our official packages (the ones in the download.zerotier.com apt and yum repositories) we now build Linux binaries with static linking. Hopefully this will stop all the bug reports relating to library inconsistencies, as well as allowing our deb packages to run on a wider variety of Debian-based distributions. (There are far too many of these to support officially!) The overhead for this is very small, especially since we built our static versions against musl-libc. Distribution maintainers are of course free to build dynamically linked versions for inclusion into distributions; this only affects our official binaries.

attic/Cluster.cpp

@@ -1,6 +1,6 @@
 /*
  * ZeroTier One - Network Virtualization Everywhere
- * Copyright (C) 2011-2016  ZeroTier, Inc.  https://www.zerotier.com/
+ * Copyright (C) 2011-2017  ZeroTier, Inc.  https://www.zerotier.com/
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -14,6 +14,14 @@
  *
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
  */
 
 #ifdef ZT_ENABLE_CLUSTER
@@ -44,6 +52,7 @@
 #include "Packet.hpp"
 #include "Switch.hpp"
 #include "Node.hpp"
+#include "Network.hpp"
 #include "Array.hpp"
 
 namespace ZeroTier {
@@ -248,13 +257,13 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 		memcpy(keytmp,_key,32);
 		for(int i=0;i<8;++i)
 			keytmp[i] ^= reinterpret_cast<const char *>(msg)[i];
-		Salsa20 s20(keytmp,256,reinterpret_cast<const char *>(msg) + 8);
+		Salsa20 s20(keytmp,reinterpret_cast<const char *>(msg) + 8);
 		Utils::burn(keytmp,sizeof(keytmp));
 
 		// One-time-use Poly1305 key from first 32 bytes of Salsa20 keystream (as per DJB/NaCl "standard")
 		char polykey[ZT_POLY1305_KEY_LEN];
 		memset(polykey,0,sizeof(polykey));
-		s20.encrypt12(polykey,polykey,sizeof(polykey));
+		s20.crypt12(polykey,polykey,sizeof(polykey));
 
 		// Compute 16-byte MAC
 		char mac[ZT_POLY1305_MAC_LEN];
@@ -266,7 +275,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 
 		// Decrypt!
 		dmsg.setSize(len - 24);
-		s20.decrypt12(reinterpret_cast<const char *>(msg) + 24,const_cast<void *>(dmsg.data()),dmsg.size());
+		s20.crypt12(reinterpret_cast<const char *>(msg) + 24,const_cast<void *>(dmsg.data()),dmsg.size());
 	}
 
 	if (dmsg.size() < 4)
@@ -341,17 +350,20 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 						Identity id;
 						ptr += id.deserialize(dmsg,ptr);
 						if (id) {
-							RR->topology->saveIdentity(id);
-
 							{
 								Mutex::Lock _l(_remotePeers_m);
-								_remotePeers[std::pair<Address,unsigned int>(id.address(),(unsigned int)fromMemberId)] = RR->node->now();
+								_RemotePeer &rp = _remotePeers[std::pair<Address,unsigned int>(id.address(),(unsigned int)fromMemberId)];
+								if (!rp.lastHavePeerReceived) {
+									RR->topology->saveIdentity((void *)0,id);
+									RR->identity.agree(id,rp.key,ZT_PEER_SECRET_KEY_LENGTH);
+								}
+								rp.lastHavePeerReceived = RR->node->now();
 							}
 
 							_ClusterSendQueueEntry *q[16384]; // 16384 is "tons"
 							unsigned int qc = _sendQueue->getByDest(id.address(),q,16384);
 							for(unsigned int i=0;i<qc;++i)
-								this->sendViaCluster(q[i]->fromPeerAddress,q[i]->toPeerAddress,q[i]->data,q[i]->len,q[i]->unite);
+								this->relayViaCluster(q[i]->fromPeerAddress,q[i]->toPeerAddress,q[i]->data,q[i]->len,q[i]->unite);
 							_sendQueue->returnToPool(q,qc);
 
 							TRACE("[%u] has %s (retried %u queued sends)",(unsigned int)fromMemberId,id.address().toString().c_str(),qc);
@@ -361,7 +373,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 					case CLUSTER_MESSAGE_WANT_PEER: {
 						const Address zeroTierAddress(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH;
 						SharedPtr<Peer> peer(RR->topology->getPeerNoCache(zeroTierAddress));
-						if ( (peer) && (peer->hasClusterOptimalPath(RR->node->now())) ) {
+						if ( (peer) && (peer->hasLocalClusterOptimalPath(RR->node->now())) ) {
 							Buffer<1024> buf;
 							peer->identity().serialize(buf);
 							Mutex::Lock _l2(_members[fromMemberId].lock);
@@ -396,7 +408,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 						SharedPtr<Peer> localPeer(RR->topology->getPeerNoCache(localPeerAddress));
 						if ((localPeer)&&(numRemotePeerPaths > 0)) {
 							InetAddress bestLocalV4,bestLocalV6;
-							localPeer->getBestActiveAddresses(now,bestLocalV4,bestLocalV6);
+							localPeer->getRendezvousAddresses(now,bestLocalV4,bestLocalV6);
 
 							InetAddress bestRemoteV4,bestRemoteV6;
 							for(unsigned int i=0;i<numRemotePeerPaths;++i) {
@@ -455,7 +467,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 									Mutex::Lock _l2(_members[fromMemberId].lock);
 									_send(fromMemberId,CLUSTER_MESSAGE_PROXY_SEND,rendezvousForRemote.data(),rendezvousForRemote.size());
 								}
-								RR->sw->send(rendezvousForLocal,true,0);
+								RR->sw->send((void *)0,rendezvousForLocal,true);
 							}
 						}
 					}	break;
@@ -466,9 +478,18 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 						const unsigned int len = dmsg.at<uint16_t>(ptr); ptr += 2;
 						Packet outp(rcpt,RR->identity.address(),verb);
 						outp.append(dmsg.field(ptr,len),len); ptr += len;
-						RR->sw->send(outp,true,0);
+						RR->sw->send((void *)0,outp,true);
 						//TRACE("[%u] proxy send %s to %s length %u",(unsigned int)fromMemberId,Packet::verbString(verb),rcpt.toString().c_str(),len);
 					}	break;
+
+					case CLUSTER_MESSAGE_NETWORK_CONFIG: {
+						const SharedPtr<Network> network(RR->node->network(dmsg.at<uint64_t>(ptr)));
+						if (network) {
+							// Copy into a Packet just to conform to Network API. Eventually
+							// will want to refactor.
+							network->handleConfigChunk((void *)0,0,Address(),Buffer<ZT_PROTO_MAX_PACKET_LENGTH>(dmsg),ptr);
+						}
+					}	break;
 				}
 			} catch ( ... ) {
 				TRACE("invalid message of size %u type %d (inner decode), discarding",mlen,mtype);
@@ -494,7 +515,84 @@ void Cluster::broadcastHavePeer(const Identity &id)
 	}
 }
 
-void Cluster::sendViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite)
+void Cluster::broadcastNetworkConfigChunk(const void *chunk,unsigned int len)
+{
+	Mutex::Lock _l(_memberIds_m);
+	for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
+		Mutex::Lock _l2(_members[*mid].lock);
+		_send(*mid,CLUSTER_MESSAGE_NETWORK_CONFIG,chunk,len);
+	}
+}
+
+int Cluster::checkSendViaCluster(const Address &toPeerAddress,uint64_t &mostRecentTs,void *peerSecret)
+{
+	const uint64_t now = RR->node->now();
+	mostRecentTs = 0;
+	int mostRecentMemberId = -1;
+	{
+		Mutex::Lock _l2(_remotePeers_m);
+		std::map< std::pair<Address,unsigned int>,_RemotePeer >::const_iterator rpe(_remotePeers.lower_bound(std::pair<Address,unsigned int>(toPeerAddress,0)));
+		for(;;) {
+			if ((rpe == _remotePeers.end())||(rpe->first.first != toPeerAddress))
+				break;
+			else if (rpe->second.lastHavePeerReceived > mostRecentTs) {
+				mostRecentTs = rpe->second.lastHavePeerReceived;
+				memcpy(peerSecret,rpe->second.key,ZT_PEER_SECRET_KEY_LENGTH);
+				mostRecentMemberId = (int)rpe->first.second;
+			}
+			++rpe;
+		}
+	}
+
+	const uint64_t ageOfMostRecentHavePeerAnnouncement = now - mostRecentTs;
+	if (ageOfMostRecentHavePeerAnnouncement >= (ZT_PEER_ACTIVITY_TIMEOUT / 3)) {
+		if (ageOfMostRecentHavePeerAnnouncement >= ZT_PEER_ACTIVITY_TIMEOUT)
+			mostRecentMemberId = -1;
+
+		bool sendWantPeer = true;
+		{
+			Mutex::Lock _l(_remotePeers_m);
+			_RemotePeer &rp = _remotePeers[std::pair<Address,unsigned int>(toPeerAddress,(unsigned int)_id)];
+			if ((now - rp.lastSentWantPeer) >= ZT_CLUSTER_WANT_PEER_EVERY) {
+				rp.lastSentWantPeer = now;
+			} else {
+				sendWantPeer = false; // don't flood WANT_PEER
+			}
+		}
+		if (sendWantPeer) {
+			char tmp[ZT_ADDRESS_LENGTH];
+			toPeerAddress.copyTo(tmp,ZT_ADDRESS_LENGTH);
+			{
+				Mutex::Lock _l(_memberIds_m);
+				for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
+					Mutex::Lock _l2(_members[*mid].lock);
+					_send(*mid,CLUSTER_MESSAGE_WANT_PEER,tmp,ZT_ADDRESS_LENGTH);
+				}
+			}
+		}
+	}
+
+	return mostRecentMemberId;
+}
+
+bool Cluster::sendViaCluster(int mostRecentMemberId,const Address &toPeerAddress,const void *data,unsigned int len)
+{
+	if ((mostRecentMemberId < 0)||(mostRecentMemberId >= ZT_CLUSTER_MAX_MEMBERS)) // sanity check
+		return false;
+	Mutex::Lock _l2(_members[mostRecentMemberId].lock);
+	for(std::vector<InetAddress>::const_iterator i1(_zeroTierPhysicalEndpoints.begin());i1!=_zeroTierPhysicalEndpoints.end();++i1) {
+		for(std::vector<InetAddress>::const_iterator i2(_members[mostRecentMemberId].zeroTierPhysicalEndpoints.begin());i2!=_members[mostRecentMemberId].zeroTierPhysicalEndpoints.end();++i2) {
+			if (i1->ss_family == i2->ss_family) {
+				TRACE("sendViaCluster sending %u bytes to %s by way of %u (%s->%s)",len,toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId,i1->toString().c_str(),i2->toString().c_str());
+				RR->node->putPacket((void *)0,*i1,*i2,data,len);
+				return true;
+			}
+		}
+	}
+	return false;
+}
+
+void Cluster::relayViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite)
 {
 	if (len > ZT_PROTO_MAX_PACKET_LENGTH) // sanity check
 		return;
@@ -502,87 +600,101 @@ void Cluster::sendViaCluster(const Address &fromPeerAddress,const Address &toPee
 	const uint64_t now = RR->node->now();
 
 	uint64_t mostRecentTs = 0;
-	unsigned int mostRecentMemberId = 0xffffffff;
+	int mostRecentMemberId = -1;
 	{
 		Mutex::Lock _l2(_remotePeers_m);
-		std::map< std::pair<Address,unsigned int>,uint64_t >::const_iterator rpe(_remotePeers.lower_bound(std::pair<Address,unsigned int>(toPeerAddress,0)));
+		std::map< std::pair<Address,unsigned int>,_RemotePeer >::const_iterator rpe(_remotePeers.lower_bound(std::pair<Address,unsigned int>(toPeerAddress,0)));
 		for(;;) {
 			if ((rpe == _remotePeers.end())||(rpe->first.first != toPeerAddress))
 				break;
-			else if (rpe->second > mostRecentTs) {
-				mostRecentTs = rpe->second;
-				mostRecentMemberId = rpe->first.second;
+			else if (rpe->second.lastHavePeerReceived > mostRecentTs) {
+				mostRecentTs = rpe->second.lastHavePeerReceived;
+				mostRecentMemberId = (int)rpe->first.second;
 			}
 			++rpe;
 		}
 	}
 
-	const uint64_t age = now - mostRecentTs;
-	if (age >= (ZT_PEER_ACTIVITY_TIMEOUT / 3)) {
-		const bool enqueueAndWait = ((age >= ZT_PEER_ACTIVITY_TIMEOUT)||(mostRecentMemberId > 0xffff));
+	const uint64_t ageOfMostRecentHavePeerAnnouncement = now - mostRecentTs;
+	if (ageOfMostRecentHavePeerAnnouncement >= (ZT_PEER_ACTIVITY_TIMEOUT / 3)) {
+		// Enqueue and wait if peer seems alive, but do WANT_PEER to refresh homing
+		const bool enqueueAndWait = ((ageOfMostRecentHavePeerAnnouncement >= ZT_PEER_ACTIVITY_TIMEOUT)||(mostRecentMemberId < 0));
 
 		// Poll everyone with WANT_PEER if the age of our most recent entry is
 		// approaching expiration (or has expired, or does not exist).
-		char tmp[ZT_ADDRESS_LENGTH];
-		toPeerAddress.copyTo(tmp,ZT_ADDRESS_LENGTH);
+		bool sendWantPeer = true;
 		{
-			Mutex::Lock _l(_memberIds_m);
-			for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
-				Mutex::Lock _l2(_members[*mid].lock);
-				_send(*mid,CLUSTER_MESSAGE_WANT_PEER,tmp,ZT_ADDRESS_LENGTH);
+			Mutex::Lock _l(_remotePeers_m);
+			_RemotePeer &rp = _remotePeers[std::pair<Address,unsigned int>(toPeerAddress,(unsigned int)_id)];
+			if ((now - rp.lastSentWantPeer) >= ZT_CLUSTER_WANT_PEER_EVERY) {
+				rp.lastSentWantPeer = now;
+			} else {
+				sendWantPeer = false; // don't flood WANT_PEER
+			}
+		}
+		if (sendWantPeer) {
+			char tmp[ZT_ADDRESS_LENGTH];
+			toPeerAddress.copyTo(tmp,ZT_ADDRESS_LENGTH);
+			{
+				Mutex::Lock _l(_memberIds_m);
+				for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
+					Mutex::Lock _l2(_members[*mid].lock);
+					_send(*mid,CLUSTER_MESSAGE_WANT_PEER,tmp,ZT_ADDRESS_LENGTH);
+				}
 			}
 		}
 
 		// If there isn't a good place to send via, then enqueue this for retrying
 		// later and return after having broadcasted a WANT_PEER.
 		if (enqueueAndWait) {
-			TRACE("sendViaCluster %s -> %s enqueueing to wait for HAVE_PEER",fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str());
+			TRACE("relayViaCluster %s -> %s enqueueing to wait for HAVE_PEER",fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str());
 			_sendQueue->enqueue(now,fromPeerAddress,toPeerAddress,data,len,unite);
 			return;
 		}
 	}
 
-	Buffer<1024> buf;
-	if (unite) {
-		InetAddress v4,v6;
-		if (fromPeerAddress) {
-			SharedPtr<Peer> fromPeer(RR->topology->getPeerNoCache(fromPeerAddress));
-			if (fromPeer)
-				fromPeer->getBestActiveAddresses(now,v4,v6);
-		}
-		uint8_t addrCount = 0;
-		if (v4)
-			++addrCount;
-		if (v6)
-			++addrCount;
-		if (addrCount) {
-			toPeerAddress.appendTo(buf);
-			fromPeerAddress.appendTo(buf);
-			buf.append(addrCount);
+	if (mostRecentMemberId >= 0) {
+		Buffer<1024> buf;
+		if (unite) {
+			InetAddress v4,v6;
+			if (fromPeerAddress) {
+				SharedPtr<Peer> fromPeer(RR->topology->getPeerNoCache(fromPeerAddress));
+				if (fromPeer)
+					fromPeer->getRendezvousAddresses(now,v4,v6);
+			}
+			uint8_t addrCount = 0;
 			if (v4)
-				v4.serialize(buf);
+				++addrCount;
 			if (v6)
-				v6.serialize(buf);
-		}
-	}
-
-	{
-		Mutex::Lock _l2(_members[mostRecentMemberId].lock);
-		if (buf.size() > 0)
-			_send(mostRecentMemberId,CLUSTER_MESSAGE_PROXY_UNITE,buf.data(),buf.size());
-
-		for(std::vector<InetAddress>::const_iterator i1(_zeroTierPhysicalEndpoints.begin());i1!=_zeroTierPhysicalEndpoints.end();++i1) {
-			for(std::vector<InetAddress>::const_iterator i2(_members[mostRecentMemberId].zeroTierPhysicalEndpoints.begin());i2!=_members[mostRecentMemberId].zeroTierPhysicalEndpoints.end();++i2) {
-				if (i1->ss_family == i2->ss_family) {
-					TRACE("sendViaCluster relaying %u bytes from %s to %s by way of %u (%s->%s)",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId,i1->toString().c_str(),i2->toString().c_str());
-					RR->node->putPacket(*i1,*i2,data,len);
-					return;
-				}
+				++addrCount;
+			if (addrCount) {
+				toPeerAddress.appendTo(buf);
+				fromPeerAddress.appendTo(buf);
+				buf.append(addrCount);
+				if (v4)
+					v4.serialize(buf);
+				if (v6)
+					v6.serialize(buf);
 			}
 		}
 
-		TRACE("sendViaCluster relaying %u bytes from %s to %s by way of %u failed: no common endpoints with the same address family!",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId);
-		return;
+		{
+			Mutex::Lock _l2(_members[mostRecentMemberId].lock);
+			if (buf.size() > 0)
+				_send(mostRecentMemberId,CLUSTER_MESSAGE_PROXY_UNITE,buf.data(),buf.size());
+
+			for(std::vector<InetAddress>::const_iterator i1(_zeroTierPhysicalEndpoints.begin());i1!=_zeroTierPhysicalEndpoints.end();++i1) {
+				for(std::vector<InetAddress>::const_iterator i2(_members[mostRecentMemberId].zeroTierPhysicalEndpoints.begin());i2!=_members[mostRecentMemberId].zeroTierPhysicalEndpoints.end();++i2) {
+					if (i1->ss_family == i2->ss_family) {
+						TRACE("relayViaCluster relaying %u bytes from %s to %s by way of %u (%s->%s)",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId,i1->toString().c_str(),i2->toString().c_str());
+						RR->node->putPacket((void *)0,*i1,*i2,data,len);
+						return;
+					}
+				}
+			}
+
+			TRACE("relayViaCluster relaying %u bytes from %s to %s by way of %u failed: no common endpoints with the same address family!",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId);
+		}
 	}
 }
 
@@ -644,8 +756,8 @@ void Cluster::doPeriodicTasks()
 		_lastCleanedRemotePeers = now;
 
 		Mutex::Lock _l(_remotePeers_m);
-		for(std::map< std::pair<Address,unsigned int>,uint64_t >::iterator rp(_remotePeers.begin());rp!=_remotePeers.end();) {
-			if ((now - rp->second) >= ZT_PEER_ACTIVITY_TIMEOUT)
+		for(std::map< std::pair<Address,unsigned int>,_RemotePeer >::iterator rp(_remotePeers.begin());rp!=_remotePeers.end();) {
+			if ((now - rp->second.lastHavePeerReceived) >= ZT_PEER_ACTIVITY_TIMEOUT)
 				_remotePeers.erase(rp++);
 			else ++rp;
 		}
@@ -719,7 +831,9 @@ bool Cluster::findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddr
 		std::vector<InetAddress> best;
 		const double currentDistance = _dist3d(_x,_y,_z,px,py,pz);
 		double bestDistance = (offload ? 2147483648.0 : currentDistance);
+#ifdef ZT_TRACE
 		unsigned int bestMember = _id;
+#endif
 		{
 			Mutex::Lock _l(_memberIds_m);
 			for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
@@ -731,7 +845,9 @@ bool Cluster::findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddr
 					const double mdist = _dist3d(m.x,m.y,m.z,px,py,pz);
 					if (mdist < bestDistance) {
 						bestDistance = mdist;
+#ifdef ZT_TRACE
 						bestMember = *mid;
+#endif
 						best = m.zeroTierPhysicalEndpoints;
 					}
 				}
@@ -754,6 +870,19 @@ bool Cluster::findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddr
 	}
 }
 
+bool Cluster::isClusterPeerFrontplane(const InetAddress &ip) const
+{
+	Mutex::Lock _l(_memberIds_m);
+	for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
+		Mutex::Lock _l2(_members[*mid].lock);
+		for(std::vector<InetAddress>::const_iterator i2(_members[*mid].zeroTierPhysicalEndpoints.begin());i2!=_members[*mid].zeroTierPhysicalEndpoints.end();++i2) {
+			if (ip == *i2)
+				return true;
+		}
+	}
+	return false;
+}
+
 void Cluster::status(ZT_ClusterStatus &status) const
 {
 	const uint64_t now = RR->node->now();
@@ -827,16 +956,16 @@ void Cluster::_flush(uint16_t memberId)
 		memcpy(keytmp,m.key,32);
 		for(int i=0;i<8;++i)
 			keytmp[i] ^= m.q[i];
-		Salsa20 s20(keytmp,256,m.q.field(8,8));
+		Salsa20 s20(keytmp,m.q.field(8,8));
 		Utils::burn(keytmp,sizeof(keytmp));
 
 		// One-time-use Poly1305 key from first 32 bytes of Salsa20 keystream (as per DJB/NaCl "standard")
 		char polykey[ZT_POLY1305_KEY_LEN];
 		memset(polykey,0,sizeof(polykey));
-		s20.encrypt12(polykey,polykey,sizeof(polykey));
+		s20.crypt12(polykey,polykey,sizeof(polykey));
 
 		// Encrypt m.q in place
-		s20.encrypt12(reinterpret_cast<const char *>(m.q.data()) + 24,const_cast<char *>(reinterpret_cast<const char *>(m.q.data())) + 24,m.q.size() - 24);
+		s20.crypt12(reinterpret_cast<const char *>(m.q.data()) + 24,const_cast<char *>(reinterpret_cast<const char *>(m.q.data())) + 24,m.q.size() - 24);
 
 		// Add MAC for authentication (encrypt-then-MAC)
 		char mac[ZT_POLY1305_MAC_LEN];
@@ -860,7 +989,7 @@ void Cluster::_flush(uint16_t memberId)
 void Cluster::_doREMOTE_WHOIS(uint64_t fromMemberId,const Packet &remotep)
 {
 	if (remotep.payloadLength() >= ZT_ADDRESS_LENGTH) {
-		Identity queried(RR->topology->getIdentity(Address(remotep.payload(),ZT_ADDRESS_LENGTH)));
+		Identity queried(RR->topology->getIdentity((void *)0,Address(remotep.payload(),ZT_ADDRESS_LENGTH)));
 		if (queried) {
 			Buffer<1024> routp;
 			remotep.source().appendTo(routp);

attic/Cluster.hpp

@@ -1,6 +1,6 @@
 /*
  * ZeroTier One - Network Virtualization Everywhere
- * Copyright (C) 2011-2016  ZeroTier, Inc.  https://www.zerotier.com/
+ * Copyright (C) 2011-2017  ZeroTier, Inc.  https://www.zerotier.com/
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -14,6 +14,14 @@
  *
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
  */
 
 #ifndef ZT_CLUSTER_HPP
@@ -88,6 +96,11 @@
  */
 #define ZT_CLUSTER_SEND_QUEUE_DATA_MAX 1500
 
+/**
+ * We won't send WANT_PEER to other members more than every (ms) per recipient
+ */
+#define ZT_CLUSTER_WANT_PEER_EVERY 1000
+
 namespace ZeroTier {
 
 class RuntimeEnvironment;
@@ -216,14 +229,13 @@ public:
 
 		/**
 		 * Replicate a network config for a network we belong to:
-		 *   <[8] 64-bit network ID>
-		 *   <[2] 16-bit length of network config>
-		 *   <[...] serialized network config>
+		 *   <[...] network config chunk>
 		 *
 		 * This is used by clusters to avoid every member having to query
 		 * for the same netconf for networks all members belong to.
 		 *
-		 * TODO: not implemented yet!
+		 * The first field of a network config chunk is the network ID,
+		 * so this can be checked to look up the network on receipt.
 		 */
 		CLUSTER_MESSAGE_NETWORK_CONFIG = 7
 	};
@@ -268,7 +280,38 @@ public:
 	void broadcastHavePeer(const Identity &id);
 
 	/**
-	 * Send this packet via another node in this cluster if another node has this peer
+	 * Broadcast a network config chunk to other members of cluster
+	 *
+	 * @param chunk Chunk data
+	 * @param len Length of chunk
+	 */
+	void broadcastNetworkConfigChunk(const void *chunk,unsigned int len);
+
+	/**
+	 * If the cluster has this peer, prepare the packet to send via cluster
+	 *
+	 * Note that outp is only armored (or modified at all) if the return value is a member ID.
+	 *
+	 * @param toPeerAddress Value of outp.destination(), simply to save additional lookup
+	 * @param ts Result: set to time of last HAVE_PEER from the cluster
+	 * @param peerSecret Result: Buffer to fill with peer secret on valid return value, must be at least ZT_PEER_SECRET_KEY_LENGTH bytes
+	 * @return -1 if cluster does not know this peer, or a member ID to pass to sendViaCluster()
+	 */
+	int checkSendViaCluster(const Address &toPeerAddress,uint64_t &mostRecentTs,void *peerSecret);
+
+	/**
+	 * Send data via cluster front plane (packet head or fragment)
+	 *
+	 * @param haveMemberId Member ID that has this peer as returned by prepSendviaCluster()
+	 * @param toPeerAddress Destination peer address
+	 * @param data Packet or packet fragment data
+	 * @param len Length of packet or fragment
+	 * @return True if packet was sent (and outp was modified via armoring)
+	 */
+	bool sendViaCluster(int haveMemberId,const Address &toPeerAddress,const void *data,unsigned int len);
+
+	/**
+	 * Relay a packet via the cluster
 	 *
 	 * This is used in the outgoing packet and relaying logic in Switch to
 	 * relay packets to other cluster members. It isn't PROXY_SEND-- that is
@@ -280,7 +323,7 @@ public:
 	 * @param len Length of packet or fragment
 	 * @param unite If true, also request proxy unite across cluster
 	 */
-	void sendViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite);
+	void relayViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite);
 
 	/**
 	 * Send a distributed query to other cluster members
@@ -323,6 +366,12 @@ public:
 	 */
 	bool findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload);
 
+	/**
+	 * @param ip Address to check
+	 * @return True if this is a cluster frontplane address (excluding our addresses)
+	 */
+	bool isClusterPeerFrontplane(const InetAddress &ip) const;
+
 	/**
 	 * Fill out ZT_ClusterStatus structure (from core API)
 	 *
@@ -391,7 +440,15 @@ private:
 	std::vector<uint16_t> _memberIds;
 	Mutex _memberIds_m;
 
-	std::map< std::pair<Address,unsigned int>,uint64_t > _remotePeers; // we need ordered behavior and lower_bound here
+	struct _RemotePeer
+	{
+		_RemotePeer() : lastHavePeerReceived(0),lastSentWantPeer(0) {}
+		~_RemotePeer() { Utils::burn(key,ZT_PEER_SECRET_KEY_LENGTH); }
+		uint64_t lastHavePeerReceived;
+		uint64_t lastSentWantPeer;
+		uint8_t key[ZT_PEER_SECRET_KEY_LENGTH]; // secret key from identity agreement
+	};
+	std::map< std::pair<Address,unsigned int>,_RemotePeer > _remotePeers; // we need ordered behavior and lower_bound here
 	Mutex _remotePeers_m;
 
 	uint64_t _lastFlushed;

attic/ClusterDefinition.hpp

@@ -1,6 +1,6 @@
 /*
  * ZeroTier One - Network Virtualization Everywhere
- * Copyright (C) 2011-2016  ZeroTier, Inc.  https://www.zerotier.com/
+ * Copyright (C) 2011-2017  ZeroTier, Inc.  https://www.zerotier.com/
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -14,6 +14,14 @@
  *
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
  */
 
 #ifndef ZT_CLUSTERDEFINITION_HPP
@@ -64,11 +72,11 @@ public:
 			return;
 
 		char myAddressStr[64];
-		Utils::snprintf(myAddressStr,sizeof(myAddressStr),"%.10llx",myAddress);
+		Utils::ztsnprintf(myAddressStr,sizeof(myAddressStr),"%.10llx",myAddress);
 
-		std::vector<std::string> lines(Utils::split(cf.c_str(),"\r\n","",""));
+		std::vector<std::string> lines(OSUtils::split(cf.c_str(),"\r\n","",""));
 		for(std::vector<std::string>::iterator l(lines.begin());l!=lines.end();++l) {
-			std::vector<std::string> fields(Utils::split(l->c_str()," \t","",""));
+			std::vector<std::string> fields(OSUtils::split(l->c_str()," \t","",""));
 			if ((fields.size() < 5)||(fields[0][0] == '#')||(fields[0] != myAddressStr))
 				continue;
 
@@ -93,7 +101,7 @@ public:
 
 			md.id = (unsigned int)id;
 			if (fields.size() >= 6) {
-				std::vector<std::string> xyz(Utils::split(fields[5].c_str(),",","",""));
+				std::vector<std::string> xyz(OSUtils::split(fields[5].c_str(),",","",""));
 				md.x = (xyz.size() > 0) ? Utils::strToInt(xyz[0].c_str()) : 0;
 				md.y = (xyz.size() > 1) ? Utils::strToInt(xyz[1].c_str()) : 0;
 				md.z = (xyz.size() > 2) ? Utils::strToInt(xyz[2].c_str()) : 0;
@@ -102,7 +110,7 @@ public:
 			md.clusterEndpoint.fromString(fields[3]);
 			if (!md.clusterEndpoint)
 				continue;
-			std::vector<std::string> zips(Utils::split(fields[4].c_str(),",","",""));
+			std::vector<std::string> zips(OSUtils::split(fields[4].c_str(),",","",""));
 			for(std::vector<std::string>::iterator zip(zips.begin());zip!=zips.end();++zip) {
 				InetAddress i;
 				i.fromString(*zip);

attic/ClusterGeoIpService.cpp

@@ -1,6 +1,6 @@
 /*
  * ZeroTier One - Network Virtualization Everywhere
- * Copyright (C) 2011-2016  ZeroTier, Inc.  https://www.zerotier.com/
+ * Copyright (C) 2011-2017  ZeroTier, Inc.  https://www.zerotier.com/
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -14,6 +14,14 @@
  *
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
  */
 
 #ifdef ZT_ENABLE_CLUSTER
@@ -101,7 +109,7 @@ bool ClusterGeoIpService::locate(const InetAddress &ip,int &x,int &y,int &z)
 
 void ClusterGeoIpService::_parseLine(const char *line,std::vector<_V4E> &v4db,std::vector<_V6E> &v6db,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn)
 {
-	std::vector<std::string> ls(Utils::split(line,",\t","\\","\"'"));
+	std::vector<std::string> ls(OSUtils::split(line,",\t","\\","\"'"));
 	if ( ((ipStartColumn >= 0)&&(ipStartColumn < (int)ls.size()))&&
 	     ((ipEndColumn >= 0)&&(ipEndColumn < (int)ls.size()))&&
 	     ((latitudeColumn >= 0)&&(latitudeColumn < (int)ls.size()))&&

attic/ClusterGeoIpService.hpp

@@ -1,6 +1,6 @@
 /*
  * ZeroTier One - Network Virtualization Everywhere
- * Copyright (C) 2011-2016  ZeroTier, Inc.  https://www.zerotier.com/
+ * Copyright (C) 2011-2017  ZeroTier, Inc.  https://www.zerotier.com/
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -14,6 +14,14 @@
  *
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
  */
 
 #ifndef ZT_CLUSTERGEOIPSERVICE_HPP

attic/FCV.hpp

@@ -0,0 +1,101 @@
+/*
+ * ZeroTier One - Network Virtualization Everywhere
+ * Copyright (C) 2011-2018  ZeroTier, Inc.  https://www.zerotier.com/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
+ */
+
+#include "Constants.hpp"
+
+namespace ZeroTier {
+
+/**
+ * A really simple fixed capacity vector
+ *
+ * This class does no bounds checking, so the user must ensure that
+ * no more than C elements are ever added and that accesses are in
+ * bounds.
+ *
+ * @tparam T Type to contain
+ * @tparam C Capacity of vector
+ */
+template<typename T,unsigned long C>
+class FCV
+{
+public:
+	FCV() : _s(0) {}
+	~FCV() { clear(); }
+
+	FCV(const FCV &v) :
+		_s(v._s)
+	{
+		for(unsigned long i=0;i<_s;++i) {
+			new (reinterpret_cast<T *>(_mem + (sizeof(T) * i))) T(reinterpret_cast<const T *>(v._mem)[i]);
+		}
+	}
+
+	inline FCV &operator=(const FCV &v)
+	{
+		clear();
+		_s = v._s;
+		for(unsigned long i=0;i<_s;++i) {
+			new (reinterpret_cast<T *>(_mem + (sizeof(T) * i))) T(reinterpret_cast<const T *>(v._mem)[i]);
+		}
+		return *this;
+	}
+
+	typedef T * iterator;
+	typedef const T * const_iterator;
+	typedef unsigned long size_type;
+
+	inline iterator begin() { return (T *)_mem; }
+	inline iterator end() { return (T *)(_mem + (sizeof(T) * _s)); }
+	inline iterator begin() const { return (const T *)_mem; }
+	inline iterator end() const { return (const T *)(_mem + (sizeof(T) * _s)); }
+
+	inline T &operator[](const size_type i) { return reinterpret_cast<T *>(_mem)[i]; }
+	inline const T &operator[](const size_type i) const { return reinterpret_cast<const T *>(_mem)[i]; }
+
+	inline T &front() { return reinterpret_cast<T *>(_mem)[0]; }
+	inline const T &front() const { return reinterpret_cast<const T *>(_mem)[0]; }
+	inline T &back() { return reinterpret_cast<T *>(_mem)[_s - 1]; }
+	inline const T &back() const { return reinterpret_cast<const T *>(_mem)[_s - 1]; }
+
+	inline void push_back(const T &v) { new (reinterpret_cast<T *>(_mem + (sizeof(T) * _s++))) T(v); }
+	inline void pop_back() { reinterpret_cast<T *>(_mem + (sizeof(T) * --_s))->~T(); }
+
+	inline size_type size() const { return _s; }
+	inline size_type capacity() const { return C; }
+
+	inline void clear()
+	{
+		for(unsigned long i=0;i<_s;++i)
+			reinterpret_cast<T *>(_mem + (sizeof(T) * i))->~T();
+		_s = 0;
+	}
+
+private:
+	char _mem[sizeof(T) * C];
+	unsigned long _s;
+};
+
+} // namespace ZeroTier