Merge branch 'dev' of http://git.int.zerotier.com/zerotier/ZeroTierOne into dev

20x20, 60x60, 1024x1024 pngs
getting ready for an iOS release

.gitignore

@@ -1,10 +1,16 @@
 # Main binaries created in *nix builds
-/zerotier-*
+/zerotier-one
+/zerotier-idtool
+/zerotier-cli
+/zerotier-selftest
+/zerotier
 
 # OS-created garbage files from various platforms
 .DS_Store
 .Apple*
 Thumbs.db
+@eaDir
+._*
 
 # Windows build droppings
 /windows/ZeroTierOne.sdf
@@ -21,10 +27,20 @@ Thumbs.db
 /windows/Release
 /windows/WebUIWrapper/bin
 /windows/WebUIWrapper/obj
+/windows/lib
 /ext/installfiles/windows/ZeroTier One-SetupFiles
 /ext/installfiles/windows/Prerequisites
 /ext/installfiles/windows/*-cache
 /ZeroTier One.msi
+/windows/.vs
+*.vcxproj.backup
+/windows/TapDriver6/Win7Debug
+/windows/TapDriver6/win7Release
+/windows/*.db
+/windows/*.opendb
+enc_temp_folder
+/windows/copyutil/bin
+/windows/copyutil/obj
 
 # *nix/Mac build droppings
 /build-*
@@ -33,6 +49,7 @@ Thumbs.db
 /examples/docker/test-*.env
 /world/mkworld
 /world/*.c25519
+zt1-src.tar.gz
 
 # Miscellaneous temporaries, build files, etc.
 *.log
@@ -44,7 +61,7 @@ Thumbs.db
 *.pid
 *.pkg
 *.o
-*.a
+/*.a
 *.dylib
 *.so
 *.so.*
@@ -56,16 +73,13 @@ Thumbs.db
 *.tmp
 .depend
 node_modules
-cluster-geo/cluster-geo/config.js
-cluster-geo/cluster-geo/cache.*
-tests/http/zerotier-one
-tests/http/big-test-hosts
-netcon/httpstub
-
-# MacGap wrapper build files
-/ext/mac-ui-macgap1-wrapper/src/MacGap.xcodeproj/project.xcworkspace/xcuserdata/*
-/ext/mac-ui-macgap1-wrapper/src/MacGap.xcodeproj/xcuserdata/*
-/ext/mac-ui-macgap1-wrapper/src/build
+zt1_update_*
+debian/files
+debian/zerotier-one
+debian/zerotier-one*.debhelper
+debian/*.log
+debian/zerotier-one.substvars
+root-watcher/config.json
 
 # Java/Android/JNI build droppings
 java/obj/
@@ -79,3 +93,26 @@ java/build_win32/
 windows/WinUI/obj/
 windows/WinUI/bin/
 windows/ZeroTierOne/Debug/
+/ext/installfiles/windows/chocolatey/zerotier-one/*.nupkg
+
+# Miscellaneous mac/Xcode droppings
+.DS_Store
+.Trashes
+*.swp
+*~.nib
+DerivedData/
+build/
+*.pbxuser
+*.mode1v3
+*.mode2v3
+*.perspectivev3
+!default.pbxuser
+!default.mode1v3
+!default.mode2v3
+!default.perspectivev3
+*.xccheckout
+xcuserdata/
+ext/librethinkdbxx/build
+.vscode
+__pycache__
+*~

AUTHORS.md

@@ -1,44 +1,73 @@
-## Authors
+## Primary Authors
 
- * ZeroTier protocol design and core network virtualization engine, ZeroTier One service, React web UI, packaging for most platforms, kitchen sink...<br>
+ * ZeroTier Core and ZeroTier One virtual networking service<br>
    Adam Ierymenko / adam.ierymenko@zerotier.com
 
  * Java JNI Interface to enable Android application development, and Android app itself (code for that is elsewhere)<br>
    Grant Limberg / glimberg@gmail.com
 
-## Contributors
+ * ZeroTier SDK (formerly known as Network Containers)<br>
+   Joseph Henry / joseph.henry@zerotier.com
+
+## Third Party Contributors
 
  * A number of fixes and improvements to the new controller, other stuff.<br>
-   Kees Bos / https://github.com/keesbos
+   Kees Bos / https://github.com/keesbos/
 
  * Debugging and testing, OpenWRT support fixes.<br>
    Moritz Warning / moritzwarning@web.de
 
+ * Debian GNU/Linux packaging, manual pages, and license compliance edits.<br>
+   Ben Finney <ben+zerotier@benfinney.id.au>
+
  * Several others made smaller contributions, which GitHub tracks here:<br>
-   https://github.com/zerotier/ZeroTierOne/graphs/contributors
+   https://github.com/zerotier/ZeroTierOne/graphs/contributors/
 
-## Third Party Code
+## Third-Party Code
 
- * LZ4 compression algorithm by Yann Collet (BSD license)<br>
-   http://code.google.com/p/lz4/
+ZeroTier includes the following third party code, either in ext/ or incorporated into the ZeroTier core.
 
- * http-parser by many authors (MIT license)<br>
-   https://github.com/joyent/http-parser
+ * LZ4 compression algorithm by Yann Collet
 
- * json-parser by James McLaughlin (BSD license)<br>
-   https://github.com/udp/json-parser
+   * Files: node/Packet.cpp (bundled within anonymous namespace)
+   * Home page: http://code.google.com/p/lz4/
+   * License grant: BSD 2-clause
 
- * TunTapOSX by Mattias Nissler (BSD license)<br>
-   http://tuntaposx.sourceforge.net
+ * http-parser by Joyent, Inc. (many authors)
 
- * tap-windows and tap-windows6 by the OpenVPN project (GPL)<br>
-   https://github.com/OpenVPN/tap-windows<br>
-   https://github.com/OpenVPN/tap-windows6
+   * Files: ext/http-parser/*
+   * Home page: https://github.com/joyent/http-parser/
+   * License grant: MIT/Expat
 
- * Salsa20 stream cipher, Curve25519 elliptic curve cipher, Ed25519
-   digital signature algorithm, and Poly1305 MAC algorithm, all by
-   Daniel J. Bernstein (public domain)<br>
-   http://cr.yp.to/
+ * C++11 json (nlohmann/json) by Niels Lohmann
 
- * MiniUPNPC by Thomas Bernard [BSD]
-   http://miniupnp.free.fr
+   * Files: ext/json/*
+   * Home page: https://github.com/nlohmann/json
+   * License grant: MIT
+
+ * TunTapOSX by Mattias Nissler
+
+   * Files: ext/tap-mac/tuntap/*
+   * Home page: http://tuntaposx.sourceforge.net/
+   * License grant: BSD attribution no-endorsement
+   * ZeroTier Modifications: change interface name to zt#, increase max MTU, increase max devices
+
+ * tap-windows6 by the OpenVPN project
+
+   * Files: windows/TapDriver6/*
+   * Home page: https://github.com/OpenVPN/tap-windows6/
+   * License grant: GNU GPL v2
+   * ZeroTier Modifications: change name of driver to ZeroTier, add ioctl() to get L2 multicast memberships (source is in ext/ and modifications inherit GPL)
+
+ * Salsa20 stream cipher, Curve25519 elliptic curve cipher, Ed25519 digital signature algorithm, and Poly1305 MAC algorithm, all by Daniel J. Bernstein
+
+   * Files: node/Salsa20.* node/C25519.* node/Poly1305.*
+   * Home page: http://cr.yp.to/
+   * License grant: public domain
+   * ZeroTier Modifications: slight cryptographically-irrelevant modifications for inclusion into ZeroTier core
+
+ * MiniUPNPC and libnatpmp by Thomas Bernard
+
+   * Files: ext/libnatpmp/* ext/miniupnpc/*
+   * Home page: http://miniupnp.free.fr/
+   * License grant: BSD attribution no-endorsement

CMakeLists.txt

@@ -0,0 +1,12 @@
+# CMake build script for libzerotiercore.a
+
+cmake_minimum_required (VERSION 2.8)
+project (zerotiercore)
+
+set (PROJ_DIR ${PROJECT_SOURCE_DIR})
+set (ZT_DEFS -std=c++11)
+
+file(GLOB core_src_glob ${PROJ_DIR}/node/*.cpp)
+add_library(zerotiercore STATIC ${core_src_glob})
+
+target_compile_options(zerotiercore PRIVATE ${ZT_DEFS})

COPYING

@@ -0,0 +1,17 @@
+ZeroTier One, an endpoint server for the ZeroTier virtual network layer.
+Copyright © 2011–2018 ZeroTier, Inc.
+
+ZeroTier One is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or (at
+your option) any later version.
+
+See the file ‘LICENSE.GPL-3’ for the text of the GNU GPL version 3.
+If that file is not present, see <http://www.gnu.org/licenses/>.
+
+..
+    Local variables:
+    coding: utf-8
+    mode: text
+    End:
+    vim: fileencoding=utf-8 filetype=text :

Jenkinsfile

@@ -0,0 +1,84 @@
+#!/usr/bin/env groovy
+
+node('master') {
+    checkout scm
+    
+    def changelog = getChangeLog currentBuild
+
+    mattermostSend "Building ${env.JOB_NAME} #${env.BUILD_NUMBER} \n Change Log: \n ${changelog}"
+}
+
+parallel 'centos7': {
+    node('centos7') {
+        try {
+            checkout scm
+
+	        stage('Build Centos 7') {
+                sh 'make -f make-linux.mk'
+            }
+        }
+        catch (err) {
+            currentBuild.result = "FAILURE"
+            mattermostSend color: '#ff0000', message: "${env.JOB_NAME} broken on Centos 7 (<${env.BUILD_URL}|Open>)"
+
+            throw err
+        }
+    }
+}, 'android-ndk': {
+    node('android-ndk') {
+        try {
+            checkout scm
+	
+            stage('Build Android NDK') { 
+                sh "/android/android-ndk-r15b/ndk-build -C $WORKSPACE/java ZT1=${WORKSPACE}"
+            }
+        }
+        catch (err) {
+            currentBuild.result = "FAILURE"
+            mattermostSend color: '#ff0000', message: "${env.JOB_NAME} broken on Android NDK (<${env.BUILD_URL}|Open>)"
+
+            throw err
+        }
+    }
+}, 'macOS': {
+    node('macOS') {
+        try {
+            checkout scm
+
+            stage('Build macOS') {
+                sh 'make -f make-mac.mk'
+            }
+
+            stage('Build macOS UI') {
+                sh 'cd macui && xcodebuild -target "ZeroTier One" -configuration Debug'
+            }
+        }
+        catch (err) {
+            currentBuild.result = "FAILURE"
+            mattermostSend color: '#ff0000', message: "${env.JOB_NAME} broken on macOS (<${env.BUILD_URL}|Open>)"
+
+            throw err
+        }
+    }
+}, 'windows': {
+    node('windows') {
+        try {
+            checkout scm
+            
+            stage('Build Windows') {
+                bat '''CALL "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\vcvarsall.bat" amd64
+git clean -dfx
+msbuild windows\\ZeroTierOne.sln
+'''
+            }
+        }
+        catch (err) {
+            currentBuild.result = "FAILURE"
+            mattermostSend color: '#ff0000', message: "${env.JOB_NAME} broken on Windows (<${env.BUILD_URL}|Open>)"
+
+            throw err
+        }
+    }
+}
+
+mattermostSend color: "#00ff00", message: "${env.JOB_NAME} #${env.BUILD_NUMBER} Complete (<${env.BUILD_URL}|Show More...>)"

LICENSE.GPL-2

@@ -0,0 +1,339 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+                            NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.

LICENSE.txt

@@ -1,13 +1,37 @@
-ZeroTier One is licensed under the terms of the GNU General Public License
-version 3, which are available here:
+ZeroTier One - Network Virtualization Everywhere
+Copyright (C) 2011-2017  ZeroTier, Inc.  https://www.zerotier.com/
 
-http://gplv3.fsf.org/
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
 
-Modification and redistribution of ZeroTier One is permitted in source form.
-Binary distribution is permitted provided all copyright notices remain
-intact and any modifications to the source code are also distributed.
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
 
-ZeroTier One may not be embedded into any closed-source application (e.g. via
-linking), nor may closed-source derivatives be created, without a separately
-negotiated license from ZeroTier Networks LLC. See the terms of the GPLv3 for
-details.
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+See LICENSE.GPL-3 for the full GNU GPL v3 license.
+
+--
+
+You can be released from the requirements of the license by purchasing
+a commercial license. Buying such a license is mandatory as soon as you
+develop commercial closed-source software that incorporates or links
+directly against ZeroTier software without disclosing the source code
+of your own application.
+
+--
+
+The above license does not apply to third party code included with or
+linked against by ZeroTier software. See the third party code section
+of the AUTHORS.md for an index of third party software included in
+this software repository.
+
+Licenses for third party code are all relatively permissive: MIT,
+BSD, and public domain. The only exception is the tap-windows driver
+which is under the GPLv2, but this is only needed to produce the
+binary tap device driver used by the ZeroTier service on Windows.

Makefile

@@ -11,8 +11,18 @@ ifeq ($(OSTYPE),Linux)
 endif
 
 ifeq ($(OSTYPE),FreeBSD)
-	include make-freebsd.mk
+	CC=clang
+	CXX=clang++
+	ZT_BUILD_PLATFORM=7
+	include make-bsd.mk
 endif
 ifeq ($(OSTYPE),OpenBSD)
-	include make-freebsd.mk
+	CC=egcc
+	CXX=eg++
+	ZT_BUILD_PLATFORM=9
+	include make-bsd.mk
+endif
+
+ifeq ($(OSTYPE),NetBSD)
+	include make-netbsd.mk
 endif

OFFICIAL-RELEASE-STEPS.md

@@ -0,0 +1,49 @@
+ZeroTier Official Release Steps
+======
+
+This is mostly for ZeroTier internal use, but others who want to do builds might find it helpful.
+
+Note: Many of these steps will require GPG and other signing keys that are kept in cold storage and must be mounted.
+
+# Bumping the Version and Preparing Installers
+
+The version must be incremented in all of the following files:
+
+    /version.h
+    /zerotier-one.spec
+    /debian/changelog
+    /ext/installfiles/mac/ZeroTier One.pkgproj
+    /ext/installfiles/windows/chocolatey/zerotier-one.nuspec
+    /ext/installfiles/windows/ZeroTier One.aip
+    /windows/WinUI/AboutView.xaml
+
+The final .AIP file can only be edited on Windows with [Advanced Installer Enterprise](http://www.advancedinstaller.com/). In addition to incrementing the version be sure that a new product code is generated. (The "upgrade code" GUID on the other hand must never change.)
+
+# Building for Supported Platforms
+
+## Macintosh
+
+Mac's easy. Just type:
+
+    make official
+
+You will need [Packages](http://s.sudre.free.fr/Software/Packages/about.html) and our release signing key in the keychain.
+
+## Linux
+
+See `LinuxBuild` environment on `linux-build` VM and use: `chroots/mount-build.sh`, `chroots/build.sh`, and the scripts in `build/` to make APT and RPM repositories.
+
+## Windows
+
+First load the Visual Studio solution and rebuild the UI and ZeroTier One in both x64 and i386 `Release` mode. Then load [Advanced Installer Enterprise](http://www.advancedinstaller.com/), check that the version is correct, and build. The build will fail if any build artifacts are missing, and Windows must have our product singing key (from DigiCert) available to sign the resulting MSI file. The MSI must then be tested on at least a few different CLEAN Windows VMs to ensure that the installer is valid and properly signed.
+
+*After the MSI is published to download.zerotier.com in the proper RELEASE/#.#.#/dist subfolder for its version* the Chocolatey package must be rebuilt and published. Open a command prompt, change to `ext/installfiles/windows/chocolatey`, and type `choco pack`. Then use `choco push` to push it to Chocolatey (API key required).
+
+    choco pack
+    choco push zerotier-one.#.#.#.nupkg -s https://chocolatey.org/
+
+Note that this does not cover rebuilding the drivers or their containing MSI projects, as this is typically not necessary and they are shipped in binary form in the repository for convenience.
+
+## iOS, Android
+
+... no docs here yet since this is done entirely out of band with regular installs.

README.md

@@ -1,35 +1,64 @@
-ZeroTier One
+ZeroTier - A Planetary Ethernet Switch
 ======
 
-ZeroTier is a software defined networking layer for Earth.
+ZeroTier is a smart programmable Ethernet switch for planet Earth.
 
-It can be used for on-premise network virtualization, as a peer to peer VPN for mobile teams, for hybrid or multi-data-center cloud deployments, or just about anywhere else secure software defined virtual networking is useful.
+It replaces the physical LAN/WAN boundary with a virtual one, allowing devices of any type at any location to be managed as if they all reside in the same cloud region or data center. All traffic is encrypted end-to-end and takes the most direct path available for minimum latency and maximum performance. The goals and design of ZeroTier are inspired by among other things the original [Google BeyondCorp](https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/43231.pdf) paper and the [Jericho Forum](https://en.wikipedia.org/wiki/Jericho_Forum).
 
-ZeroTier One is our OS-level client service. It allows Mac, Linux, Windows, FreeBSD, and soon other types of clients to join ZeroTier virtual networks like conventional VPNs or VLANs. It can run on native systems, VMs, or containers (Docker, OpenVZ, etc.).
+Visit [ZeroTier's site](https://www.zerotier.com/?pk_campaign=github_ZeroTierOne) for more information and [pre-built binary packages](https://www.zerotier.com/download.shtml?pk_campaign=github_ZeroTierOne). Apps for Android and iOS are available for free in the Google Play and Apple app stores.
 
-Visit [ZeroTier's site](https://www.zerotier.com/) for more information. You can also download professionally packaged binary installers/packages for a variety of supported OSes there if you don't want to build ZeroTier One from source.
+### Getting Started
 
-### Building from Source
+Everything in the ZeroTier world is controlled by two types of identifier: 40-bit/10-digit *ZeroTier addresses* and 64-bit/16-digit *network IDs*. A ZeroTier address identifies a node or "device" (laptop, phone, server, VM, app, etc.) while a network ID identifies a virtual Ethernet network that can be joined by devices.
 
-For Mac, Linux, and BSD, just type "make" (or "gmake" on BSD). You won't need much installed; here are the requirements for various platforms:
+Another way of thinking about it is that ZeroTier addresses are port numbers on a giant planetary-sized smart switch while network IDs are VLANs to which these ports can be assigned. For more details read about VL1 and VL2 in [the ZeroTier manual](https://www.zerotier.com/manual.shtml).
 
- * Mac: Xcode command line tools, and [Packages](http://s.sudre.free.fr/Software/Packages/about.html) if you want to build an OSX .pkg installer ("make mac-dist-pkg"). It should build on OSX 10.7 or newer.
- * Linux: gcc/g++ or clang/clang++ (Makefile will use clang by default if available.)
- * FreeBSD (and other BSD): C++ compiler (G++ usually) and GNU make (gmake).
+*Network controllers* are ZeroTier nodes that act as access control certificate authorities and configuration managers for virtual networks. The first 40 bits (or 10 digits) of a network ID is the ZeroTier address of its controller. You can create networks with our [hosted controllers](https://my.zerotier.com/) and web UI/API or [host your own](controller/) if you don't mind posting some JSON configuration info or writing a script to do so.
 
-Each supported platform has its own *make-XXX.mk* file that contains the actual make rules for the platform. The right .mk file is included by the main Makefile based on the GNU make *OSTYPE* variable. Take a look at the .mk file for your platform for other targets, debug build rules, etc.
+### Project Layout
 
-Windows, of course, is special. We build for Windows with Microsoft Visual Studio 2012 on Windows 7. A solution file is located in the *windows* subfolder. Newer versions of Visual Studio (and Windows) may work but haven't been tested. Older versions almost certainly will not, since they lack things like *stdint.h* and certain STL features. MinGW or other ports of gcc/clang to Windows should also work but haven't been tested. Build steps for Windows are a bit more complicated. For the moment you are on your own there.
+ - `artwork/`: icons, logos, etc.
+ - `attic/`: old stuff and experimental code that we want to keep around for reference.
+ - `controller/`: the reference network controller implementation, which is built and included by default on desktop and server build targets.
+ - `debian/`: files for building Debian packages on Linux.
+ - `doc/`: manual pages and other documentation.
+ - `ext/`: third party libraries, binaries that we ship for convenience on some platforms (Mac and Windows), and installation support files.
+ - `include/`: include files for the ZeroTier core.
+ - `java/`: a JNI wrapper used with our Android mobile app. (The whole Android app is not open source but may be made so in the future.)
+ - `macui/`: a Macintosh menu-bar app for controlling ZeroTier One, written in Objective C.
+ - `node/`: the ZeroTier virtual Ethernet switch core, which is designed to be entirely separate from the rest of the code and able to be built as a stand-alone OS-independent library. Note to developers: do not use C++11 features in here, since we want this to build on old embedded platforms that lack C++11 support. C++11 can be used elsewhere.
+ - `osdep/`: code to support and integrate with OSes, including platform-specific stuff only built for certain targets.
+ - `service/`: the ZeroTier One service, which wraps the ZeroTier core and provides VPN-like connectivity to virtual networks for desktops, laptops, servers, VMs, and containers.
+ - `tcp-proxy/`: TCP proxy code run by ZeroTier, Inc. to provide TCP fallback (this will die soon!).
+ - `windows/`: Visual Studio solution files, Windows service code for ZeroTier One, and the Windows task bar app UI.
 
-Mobile versions are in progress. They don't work yet, and in any case only the glue code will be included in this repository. The full mobile apps are in private repositories on our own git server.
+The base path contains the ZeroTier One service main entry point (`one.cpp`), self test code, makefiles, etc.
 
-### Supported Platforms
+### Build and Platform Notes
 
-CPU architecture shouldn't matter unless it's smaller than 32-bit or something really bizarre like a "middle-endian" processor. We have reports of ZeroTier One running on arm32, arm64, and MIPS. It builds and runs out of the box on Raspberry Pi, BeagleBone, BananaPi, and other ARM-based developer/hobbyist boards.
+To build on Mac and Linux just type `make`. On FreeBSD and OpenBSD `gmake` (GNU make) is required and can be installed from packages or ports. For Windows there is a Visual Studio solution in `windows/'.
 
-ZeroTier is written in C and C++ (C++03 / ISO/IEC 14882:2003) and uses data structures and algorithms from the C++03 STL. We do not use any C++11 features (yet), since we want to support a few old and embedded platforms that don't have C++11 compilers. You *will* require a compiler and headers new enough to support 64-bit integers (long long) and the *stdint.h* header. The latter could also be faked by adding defines for things like *uint32\_t*, *int64\_t*, etc.
+ - **Mac**
+   - Xcode command line tools for OSX 10.7 or newer are required.
+   - Tap device driver kext source is in `ext/tap-mac` and a signed pre-built binary can be found in `ext/bin/tap-mac`. You should not need to build it yourself. It's a fork of [tuntaposx](http://tuntaposx.sourceforge.net) with device names changed to `zt#`, support for a larger MTU, and tun functionality removed.
+ - **Linux**
+   - The minimum compiler versions required are GCC/G++ 4.9.3 or CLANG/CLANG++ 3.4.2.
+   - Linux makefiles automatically detect and prefer clang/clang++ if present as it produces smaller and slightly faster binaries in most cases. You can override by supplying CC and CXX variables on the make command line.
+   - CentOS 7 ships with a version of GCC/G++ that is too old, but a new enough version of CLANG can be found in the *epel* repositories. Type `yum install epel-release` and then `yum install clang` to build there.
+ - **Windows**
+   - Windows 7 or newer is supported. This *may* work on Vista but isn't officially supported there. It will not work on Windows XP.
+   - We build with Visual Studio 2015. Older versions may not work. Clang or MinGW will also probably work but may require some makefile hacking.
+   - Pre-built signed Windows drivers are included in `ext/bin/tap-windows-ndis6`. The MSI files found there will install them on 32-bit and 64-bit systems. We don't recommend trying to build Windows drivers from scratch unless you know what you're doing. One does not simply "build" a Windows driver.
+ - **FreeBSD**
+   - Tested most recently on FreeBSD-11. Older versions may work but we're not sure.
+   - GCC/G++ 4.9 and gmake are required. These can be installed from packages or ports. Type `gmake` to build.
+ - **OpenBSD**
+   - There is a limit of four network memberships on OpenBSD as there are only four tap devices (`/dev/tap0` through `/dev/tap3`). We're not sure if this can be increased.
+   - OpenBSD lacks `getifmaddrs` (or any equivalent method) to get interface multicast memberships. As a result multicast will only work on OpenBSD for ARP and NDP (IP/MAC lookup) and not for other purposes.
+   - Only tested on OpenBSD 6.0. Older versions may not work.
+   - GCC/G++ 4.9 and gmake are required and can be installed using `pkg_add` or from ports. They get installed in `/usr/local/bin` as `egcc` and `eg++` and our makefile is pre-configured to use them on OpenBSD.
 
-Typing "make selftest" will build a *zerotier-selftest* binary which unit tests various internals and reports on a few aspects of the build environment. It's a good idea to try this on novel platforms or architectures.
+Typing `make selftest` will build a *zerotier-selftest* binary which unit tests various internals and reports on a few aspects of the build environment. It's a good idea to try this on novel platforms or architectures.
 
 ### Running
 
@@ -45,10 +74,10 @@ The service is controlled via the JSON API, which by default is available at 127
 
 Here's where home folders live (by default) on each OS:
 
- * Linux: /var/lib/zerotier-one
- * BSD: /var/db/zerotier-one
- * Mac: /Library/Application Support/ZeroTier/One
- * Windows: \\ProgramData\\ZeroTier\\One (That's for Windows 7. The base 'shared app data' folder might be different on different Windows versions.)
+ * **Linux**: `/var/lib/zerotier-one`
+ * **FreeBSD** / **OpenBSD**: `/var/db/zerotier-one`
+ * **Mac**: `/Library/Application Support/ZeroTier/One`
+ * **Windows**: `\ProgramData\ZeroTier\One` (That's for Windows 7. The base 'shared app data' folder might be different on different Windows versions.)
 
 Running ZeroTier One on a Mac is the same, but OSX requires a kernel extension. We ship a signed binary build of the ZeroTier tap device driver, which can be installed on Mac with:
 
@@ -56,119 +85,29 @@ Running ZeroTier One on a Mac is the same, but OSX requires a kernel extension.
 
 This will create the home folder for Mac, place *tap.kext* there, and set its modes correctly to enable ZeroTier One to manage it with *kextload* and *kextunload*.
 
-We recommend using our binary packages on Windows, since there are several prerequisites such as a tap driver that must be installed on the system *and* in the home folder.
-
-### Joining A Network
-
-ZeroTier virtual networks are identified by 16-digit hexadecimal network IDs, while devices are identified by 10-digit addresses. To get your address run:
-
-    sudo zerotier-cli status
-
-(Use ./zerotier-cli if you're running it right from your build folder.)
-
-You should see something like:
-
-    200 info ########## ONLINE #.#.#
-
-That 10-digit hex code is you. It's derived via a one-way proof of work function from your cryptographic public key. Your public key can be found in *identity.public* in ZeroTier's home folder, while *identity.secret* contains your full identity including the secret portion of the key pair.
-
-(The identity files define your device's *identity*. Moving them to another system will move that identity. Be careful when cloning virtual machines that have identities stored on them. If two devices have the same identity, they'll "fight" over it and you won't know which device will receive network packets.)
-
-If you want to do a quick test, you can join [Earth](https://www.zerotier.com/earth.html). It's a global public network that anyone can join. Type:
-
-    sudo zerotier-cli join 8056c2e21c000001
-
-Then:
-
-    sudo zerotier-cli listnetworks
-
-At first it'll be in *REQUESTING\_CONFIGURATION* state. In a few seconds to a minute you should see something like:
-
-    200 listnetworks 8056c2e21c000001 earth.zerotier.net ##:##:##:##:##:## OK PUBLIC zt0 ##.##.##.##/##
-
-Earth will assign you an IP address in the "unofficially available" globally unrouted 28.0.0.0/7 IP block so as to avoid conflicts with local networks. (Your networks can use any IP scheme, or can even leave IP addresses unmanaged.) Once you get an IP, you should be able to ping something:
-
-    ping earth.zerotier.net
-
-Go to [http://earth.zerotier.net/](http://earth.zerotier.net/) to see a short little welcome page that will tell you your IP and Ethernet MAC address.
-
-Earth is a public place. If you don't want to stick around run:
-
-    sudo zerotier-cli leave 8056c2e21c000001
-
-The network (and associated interface) should be gone.
-
-Networks are created and administrated by network controllers. Most users will want to use our hosted controllers. Visit [our web site](https://www.zerotier.com/) for more information. Later in this README there are brief instructions about building ZeroTier One with network controller support for those who want to try running their own.
-
-Macintosh and Windows installers also install a GUI application.
-
-### Installing
-
-We don't have a "make install" rule quite yet. On Linux you can type:
-
-    make installer
-
-This will build a binary that, when run, will install ZeroTier One on most current Linux distributions. We also have RPM and DEB build files in *ext/installfiles/linux* that wrap this installer in packages for RedHat/CentOS and Debian/Ubuntu derived distributions. If *rpmbuild* is present on the system, the RPM will be built. If *dpkg-deb* is present, the DEB package will be built.
-
-On Mac the best way is to install [Packages](http://s.sudre.free.fr/Software/Packages/about.html) and use:
-
-    make mac-dist-pkg
-
-This builds a .pkg file that can be installed.
-
-In FreeBSD there is now an official .pkg in the FreeBSD repository. Type "pkg install zerotier". It can also be built and installed from source.
-
-Linux/BSD and Mac installations have an *uninstall.sh* file in their ZeroTier home folder that cleanly removes ZeroTier One from the system. Run this with:
-
-    sudo /path/to/ZeroTier/home/folder/uninstall.sh
-
-Windows installers are insane. We build our .MSI installers with [Advanced Installer Enterprise](http://www.advancedinstaller.com). The Advanced Installer project file is in *ext/installfiles/windows*. To avoid lasting psychological trauma we recommend leaving Windows installers alone and using the pre-built Windows binaries on our web site.
-
-### Using ZeroTier One in Docker Containers
-
-To run the ZeroTier One service in a Docker container, run it with "--device=/dev/net/tun --cap-add=NET_ADMIN". This will allow ZeroTier One to open a "tap" virtual network port inside the container.
-
-Alternately, you can use Ethernet bridging to bridge the *docker0* device on your system to a ZeroTier virtual network. This allows you to run ZeroTier One on the host and bridge the entire Docker network backplane to a virtual network or other hosts.
-
-We're working on better "official" Docker support. In the meantime there is a [user-contributed project here](https://github.com/davide/docker-zerotier).
-
-### Building with Network Controller Support
-
-**Warning: as of beta version 1.0.3 the new network controller is not heavily tested. We recommend waiting for 1.0.4 to deploy this in production.**
-
-Network controllers are nodes responsible for issuing configurations and certificates to members of ZeroTier virtual networks. Most users won't need to run their own, so this code is by default not included in the ZeroTier One binary.
-
-You can build a network controller on Linux or Mac with:
-
-    make ZT_ENABLE_NETWORK_CONTROLLER=1
-
-This will build a version that contains the Sqlite-backed network controller and associated extensions to the JSON local service control API. You will need the development headers for sqlite3 installed. On Mac these ship as part of Xcode, while on Linux they'll be found in packages for the various distributions.
-
-See the JSON API documentation in [service/](service/) for more information about how to control controllers.
-
 ### Troubleshooting
 
 For most users, it just works.
 
 If you are running a local system firewall, we recommend adding a rule permitting UDP port 9993 inbound and outbound. If you installed binaries for Windows this should be done automatically. Other platforms might require manual editing of local firewall rules depending on your configuration.
 
-The Mac firewall can be founder under "Security" in System Preferences. Linux has a variety of firewall configuration systems and tools. If you're using Ubuntu's *ufw*, you can do this:
+The Mac firewall can be found under "Security" in System Preferences. Linux has a variety of firewall configuration systems and tools. If you're using Ubuntu's *ufw*, you can do this:
 
     sudo ufw allow 9993/udp
 
-On CentOS check */etc/sysconfig/iptables* for IPTables rules. For other distributions consult your distribution's documentation. You'll also have to check the UIs or documentation for commercial third party firewall applications like Little Snitch (Mac), McAfee Firewall Enterprise (Windows), etc. if you are running any of those. Some corporate environments might have centrally managed firewall software, so you might also have to contact IT.
+On CentOS check `/etc/sysconfig/iptables` for IPTables rules. For other distributions consult your distribution's documentation. You'll also have to check the UIs or documentation for commercial third party firewall applications like Little Snitch (Mac), McAfee Firewall Enterprise (Windows), etc. if you are running any of those. Some corporate environments might have centrally managed firewall software, so you might also have to contact IT.
 
 ZeroTier One peers will automatically locate each other and communicate directly over a local wired LAN *if UDP port 9993 inbound is open*. If that port is filtered, they won't be able to see each others' LAN announcement packets. If you're experiencing poor performance between devices on the same physical network, check their firewall settings. Without LAN auto-location peers must attempt "loopback" NAT traversal, which sometimes fails and in any case requires that every packet traverse your external router twice.
 
 Users behind certain types of firewalls and "symmetric" NAT devices may not able able to connect to external peers directly at all. ZeroTier has limited support for port prediction and will *attempt* to traverse symmetric NATs, but this doesn't always work. If P2P connectivity fails you'll be bouncing UDP packets off our relay servers resulting in slower performance. Some NAT router(s) have a configurable NAT mode, and setting this to "full cone" will eliminate this problem. If you do this you may also see a magical improvement for things like VoIP phones, Skype, BitTorrent, WebRTC, certain games, etc., since all of these use NAT traversal techniques similar to ours.
 
-If you're interested, there's a [technical deep dive about NAT traversal on our blog](https://www.zerotier.com/blog/?p=226). A troubleshooting tool to help you diagnose NAT issues is planned for the future as are uPnP/IGD/NAT-PMP and IPv6 transport.
+If you're interested, there's a [technical deep dive about NAT traversal on our blog](https://www.zerotier.com/blog/?p=226?pk_campaign=github_ZeroTierOne). A troubleshooting tool to help you diagnose NAT issues is planned for the future as are uPnP/IGD/NAT-PMP and IPv6 transport.
 
 If a firewall between you and the Internet blocks ZeroTier's UDP traffic, you will fall back to last-resort TCP tunneling to rootservers over port 443 (https impersonation). This will work almost anywhere but is *very slow* compared to UDP or direct peer to peer connectivity.
 
 ### Contributing
 
-There are three main branches: **edge**, **test**, and **master**. Other branches may be for specific features, tests, or use cases. In general **edge** is "bleeding" and may or may not work, while **test** should be relatively stable and **master** is the latest tagged release. Pull requests should generally be done against **test** or **edge**, since pull requests against **master** may be working against a branch that is somewhat out of date.
+Please make pull requests against the `dev` branch. The `master` branch is release, and `edge` is for unstable and work in progress changes and is not likely to work.
 
 ### License
 

RELEASE-NOTES.md

@@ -0,0 +1,187 @@
+ZeroTier Release Notes
+======
+
+# 2018-05-08 -- Version 1.2.10
+
+ * Fix bug loading `moons.d/` files for federated root operation.
+ * Fix compile problem with ZT_DEBUG on some versions of `clang`
+ * Fix slow network startup bug related to loading of `networks.d/` cache files
+
+# 2018-04-27 -- Version 1.2.8
+
+ * Linux version once again builds with PIE (position independent executable) flags
+ * Fixed bug in zerotier-idtool file sign and verify
+ * Fixed minor OSX app typo
+ * Merged alpha NetBSD support (mostly untested, so YMMV)
+ * Merged several minor typo and one-liner bug fixes
+
+# 2018-04-17 -- Version 1.2.6
+
+ * Features and Core Improvements
+    * Path selection has been overhauled to improve path stability, simplify code, and prepare for multi-path and trunking in the next major release.
+    * This version introduces remote tracing for remote diagnostics. Network controllers can set a node (usually the controller itself) to receive remote tracing events from all members of the network or from select members. Events are only sent if they pertain to a given network for security reasons.
+    * Multicast replication can now be done by designated multicast replicators on a network (flagged as such at the controller) rather than by the sender. Most users won't want this, but it's useful for specialized use cases on hub-and-spoke networks and for low-power devices.
+    * Cryptographic performance improvements on several platforms.
+    * Multithreaded performance improvements throughout the code base, including the use of an inline lightweight spinlock for low-contention resources.
+ * Bugs fixed
+    * Disappearing routes on Mac (GitHub issue #600)
+    * Route flapping and path instability in some dual-stack V4/V6 networks
+    * Blacklist (in local.conf) doesn't work reliably (GitHub issue #656)
+    * Connection instabilities due to unsigned integer overflows in timing comparisons (use int64_t instead of uint64_t)
+    * Binaries don't run on some older or lower-end 32-bit ARM chips (build problem)
+    * ARM NEON crypto code crashes (build problem)
+    * Fixed some lock ordering issues revealed by "valgrind" tool
+    * The "zerotier-idtool" command could not be accessed from "zerotier-one" via command line switch
+    * Leaking sockets on some platforms when uPnP/NAT-PMP is enabled
+    * Fixed two very rare multithreading issues that were only observed on certain systems
+ * Platform-Specific Changes
+    * MacOS
+        * Installer now loads the kernel extension right away so that High Sierra users will see the prompt to authorize it. This is done in the "Security & Privacy" preference pane and must be done driectly on the console (not via remote desktop). On High Sierra and newer kexts must be authorized at the console via security settings system preferences pane.
+    * Windows
+        * The Windows installer should now install the driver without requiring a special prompt in most cases. This should make it easier for our packages to be accepted into and updated in the Chocolatey repository and should make it easier to perform remote installs across groups of machines using IT management and provisioning tools.
+        * The Windows official packages are now signed with an EV certificate (with hardware key).
+        * The Windows UI can now log into ZeroTier Central and join networks via the Central API.
+        * The `zerotier-idtool` command should now work on Windows without ugly hacks.
+        * Upgraded the installer version.
+        * Made a few changes to hopefully fix sporadic "will not uninstall" problems, though we cannot duplicate these issues ourselves.
+    * Linux
+        * Device names are now generated deterministically based on network IDs for all newly joined networks.
+    * Android
+        * Multicast now works on Android in most cases! Android apps can send and receive multicast and subscribe to multicast group IPs. Note that in some cases the app must bind to the specific correct interface for this to work.
+        * IPv6 can be disabled in UI for cases where it causes problems.
+
+# 2017-04-20 -- Version 1.2.4
+
+ * Managed routes are now only bifurcated for the default route. This is a change in behavior, though few people will probably notice. Bifurcating all managed routes was causing more trouble than it was worth for most users.
+ * Up to 2X crypto speedup on x86-64 (except Windows, which will take some porting) and 32-bit ARM platforms due to integration of fast assembly language implementations of Salsa20/12 from the [supercop](http://bench.cr.yp.to/supercop.html) code base. These were written by Daniel J. Bernstein and are in the public domain. My Macbook Pro (Core i5 2.8ghz) now does almost 1.5GiB/sec Salsa20/12 per core and a Raspberry Pi got a 2X boost. 64-bit ARM support and Windows support will take some work but should not be too hard.
+ * Refactored code that manages credentials to greatly reduce memory use in most cases. This may also result in a small performance improvement.
+ * Reworked and simplified path selection and priority logic to fix path instability and dead path persistence edge cases. There have been some sporadic reports of persistent path instabilities and dead paths hanging around that take minutes to resolve. These have proven difficult to reproduce in house, but hopefully this will fix them. In any case it seems to speed up path establishment in our tests and it makes the code simpler and more readable.
+ * Eliminated some unused cruft from the code around path management and in the peer class.
+ * Fixed an issue causing build problems on some MIPS architecture systems.
+ * Fixed Windows forgetting routes on sleep/wake or in some other circumstances. (GitHub issue #465)
+
+# 2017-03-17 -- Version 1.2.2
+
+ * A bug causing unreliable multicast propagation (GitHub issue #461).
+ * A crash in ARM binaries due to a build chain and flags problem.
+ * A bug in the network controller preventing members from being listed (GitHub issue #460).
+
+# 2017-03-14 -- Version 1.2.0
+
+Version 1.2.0 is a major milestone release representing almost nine months of work. It includes our rules engine for distributed network packet filtering and security monitoring, federated roots, and many other architectural and UI improvements and bug fixes.
+
+## New Features in 1.2.0
+
+### The ZeroTier Rules Engine
+
+The largest new feature in 1.2.0, and the product of many months of work, is our advanced network rules engine. With this release we achieve traffic control, security monitoring, and micro-segmentation capability on par with many enterprise SDN solutions designed for use in advanced data centers and corporate networks.
+
+Rules allow you to filter packets on your network and vector traffic to security observers. Security observation can be performed in-band using REDIRECT or out of band using TEE.
+
+Tags and capabilites provide advanced methods for implementing fine grained permission structures and micro-segmentation schemes without bloating the size and complexity of your rules table.
+
+See the [rules engine announcement blog post](https://www.zerotier.com/blog/?p=927) for an in-depth discussion of theory and implementation. The [manual](https://www.zerotier.com/manual.shtml) contains detailed information on rule, tag, and capability use, and the `rule-compiler/` subfolder of the ZeroTier source tree contains a JavaScript function to compile rules in our human-readable rule definition language into rules suitable for import into a network controller. (ZeroTier Central uses this same script to compile rules on [my.zerotier.com](https://my.zerotier.com/).)
+
+### Root Server Federation
+
+It's now possible to create your own root servers and add them to the root server pool on your nodes. This is done by creating what's called a "moon," which is a signed enumeration of root servers and their stable points on the network. Refer to the [manual](https://www.zerotier.com/manual.shtml) for instructions.
+
+Federated roots achieve a number of things:
+
+ * You can deploy your own infrastructure to reduce dependency on ours.
+ * You can deploy roots *inside your LAN* to ensure that network connectivity inside your facility still works if the Internet goes down. This is the first step toward making ZeroTier viable as an in-house SDN solution.
+ * Roots can be deployed inside national boundaries for countries with data residency laws or "great firewalls." (As of 1.2.0 there is still no way to force all traffic to use these roots, but that will be easy to do in a later version.)
+ * Last but not least this makes ZeroTier somewhat less centralized by eliminating any hard dependency on ZeroTier, Inc.'s infrastructure.
+
+Our roots will of course remain and continue to provide zero-configuration instant-on deployment, a secure global authority for identities, and free traffic relaying for those who can't establish peer to peer connections.
+
+### Local Configuration
+
+An element of our design philosophy is "features are bugs." This isn't an absolute dogma but more of a guiding principle. We try as hard as we can to avoid adding features, especially "knobs" that must be tweaked by a user.
+
+As of 1.2.0 we've decided that certain knobs are unavoidable, and so there is now a `local.conf` file that can be used to configure them. See the ZeroTier One documentation for these. They include:
+
+ * Blacklisting interfaces you want to make sure ZeroTier doesn't use for network traffic, such as VPNs, slow links, or backplanes designated for only certain kinds of traffic.
+ * Turning uPnP/NAT-PMP on or off.
+ * Configuring software updates on Windows and Mac platforms.
+ * Defining trusted paths (the old trusted paths file is now deprecated)
+ * Setting the ZeroTier main port so it doesn't have to be changed on the command line, which is very inconvenient in many cases.
+
+### Improved In-Band Software Updates
+
+A good software update system for Windows and Mac clients has been a missing feature in previous versions. It does exist but we've been shy about using it so far due to its fragility in some environments.
+
+We've greatly improved this mechanism in 1.2.0. Not only does it now do a better job of actually invoking the update, but it also transfers updates in-band using the ZeroTier protocol. This means it can work in environments that do not allows http/https traffic or that force it through proxies. There's also now an update channel setting: `beta` or `release` (the default).
+
+Software updates are authenticated three ways:
+
+ 1. ZeroTier's own signing key is used to sign all updates and this signature is checked prior to installation. ZeroTier, Inc.'s signatures are performed on an air-gapped machine.
+
+ 2. Updates for Mac and Windows are signed using Apple and Microsoft (DigiCert EV) keys and will not install unless these signatures are also valid.
+
+ 3. The new in-band update mechanism also authenticates the source of the update via ZeroTier's built-in security features. This provides transport security, while 1 and 2 provide security of the update at rest.
+
+Updates are now configurable via `local.conf`. There are three options: `disable`, `download`, and `apply`. The third (apply) is the default for official builds on Windows and Mac, making updates happen silently and automatically as they do for popular browsers like Chrome and Firefox. Updates are disabled by default on Linux and other Unix-type systems as these are typically updated through package managers.
+
+### Path Link Quality Awareness
+
+Version 1.2.0 is now aware of the link quality of direct paths with other 1.2.0 nodes. This information isn't used yet but is visible through the JSON API. (Quality always shows as 100% with pre-1.2.0 nodes.) Quality is measured passively with no additional overhead using a counter based packet loss detection algorithm.
+
+This information is visible from the command line via `listpeers`:
+
+    200 listpeers XXXXXXXXXX 199.XXX.XXX.XXX/9993;10574;15250;1.00 48 1.2.0 LEAF
+    200 listpeers XXXXXXXXXX 195.XXX.XXX.XXX/45584;467;7608;0.44 290 1.2.0 LEAF
+
+The first peer's path is at 100% (1.00), while the second peer's path is suffering quite a bit of packet loss (0.44).
+
+Link quality awareness is a precursor to intelligent multi-path and QoS support, which will in future versions bring us to feature parity with SD-WAN products like Cisco iWAN.
+
+### Security Improvements
+
+Version 1.2.0 adds anti-DOS (denial of service) rate limits and other hardening for improved resiliency against a number of denial of service attack scenarios.
+
+It also adds a mechanism for instantaneous credential revocation. This can be used to revoke certificates of membership instantly to kick a node off a network (for private networks) and also to revoke capabilities and tags. The new controller sends revocations by default when a peer is de-authorized.
+
+Revocations propagate using a "rumor mill" peer to peer algorithm. This means that a controller need only successfully send a revocation to at least one member of a network with connections to other active members. At this point the revocation will flood through the network peer to peer very quickly. This helps make revocations more robust in the face of poor connectivity with the controller or attempts to incapacitate the controller with denial of service attacks, as well as making revocations faster on huge networks.
+
+### Windows and Macintosh UI Improvements (ZeroTier One)
+
+The Mac has a whole new UI built natively in Objective-C. It provides a pulldown similar in appearance and operation to the Mac WiFi task bar menu.
+
+The Windows UI has also been improved and now provides a task bar icon that can be right-clicked to manage networks. Both now expose managed route and IP permissions, allowing nodes to easily opt in to full tunnel operation if you have a router configured on your network.
+
+### Ad-Hoc Networks
+
+A special kind of public network called an ad-hoc network may be accessed by joining a network ID with the format:
+
+    ffSSSSEEEE000000
+    | |   |   |
+    | |   |   Reserved for future use, must be 0
+    | |   End of port range (hex)
+    | Start of port range (hex)
+    Reserved ZeroTier address prefix indicating a controller-less network
+
+Ad-hoc networks are public (no access control) networks that have no network controller. Instead their configuration and other credentials are generated locally. Ad-hoc networks permit only IPv6 UDP and TCP unicast traffic (no multicast or broadcast) using 6plane format NDP-emulated IPv6 addresses. In addition an ad-hoc network ID encodes an IP port range. UDP packets and TCP SYN (connection open) packets are only allowed to desintation ports within the encoded range.
+
+For example `ff00160016000000` is an ad-hoc network allowing only SSH, while `ff0000ffff000000` is an ad-hoc network allowing any UDP or TCP port.
+
+Keep in mind that these networks are public and anyone in the entire world can join them. Care must be taken to avoid exposing vulnerable services or sharing unwanted files or other resources.
+
+### Network Controller (Partial) Rewrite
+
+The network controller has been largely rewritten to use a simple in-filesystem JSON data store in place of SQLite, and it is now included by default in all Windows, Mac, Linux, and BSD builds. This means any desktop or server node running ZeroTier One can now be a controller with no recompilation needed.
+
+If you have data in an old SQLite3 controller we've included a NodeJS script in `controller/migrate-sqlite` to migrate data to the new format. If you don't migrate, members will start getting `NOT_FOUND` when they attempt to query for updates.
+
+## Major Bug Fixes in 1.2.0
+
+ * **The Windows HyperV 100% CPU bug is FINALLY DEAD**: This long-running problem turns out to have been an issue with Windows itself, but one we were triggering by placing invalid data into the Windows registry. Microsoft is aware of the issue but we've also fixed the triggering problem on our side. ZeroTier should now co-exist quite well with HyperV and should now be able to be bridged with a HyperV virtual switch.
+ * **Segmenation faults on musl-libc based Linux systems**: Alpine Linux and some embedded Linux systems that use musl libc (a minimal libc) experienced segmentation faults. These were due to a smaller default stack size. A work-around that sets the stack size for new threads has been added.
+ * **Windows firewall blocks local JSON API**: On some Windows systems the firewall likes to block 127.0.0.1:9993 for mysterious reasons. This is now fixed in the installer via the addition of another firewall exemption rule.
+ * **UI crash on embedded Windows due to missing fonts**: The MSI installer now ships fonts and will install them if they are not present, so this should be fixed.
+
+## Other Improvements in 1.2.0
+
+ * **Improved dead path detection**: ZeroTier is now more aggressive about expiring paths that do not seem to be active. If a path seems marginal it is re-confirmed before re-use.
+ * **Minor performance improvements**: We've reduced unnecessary memcpy's and made a few other performance improvements in the core.
+ * **Linux static binaries**: For our official packages (the ones in the download.zerotier.com apt and yum repositories) we now build Linux binaries with static linking. Hopefully this will stop all the bug reports relating to library inconsistencies, as well as allowing our deb packages to run on a wider variety of Debian-based distributions. (There are far too many of these to support officially!) The overhead for this is very small, especially since we built our static versions against musl-libc. Distribution maintainers are of course free to build dynamically linked versions for inclusion into distributions; this only affects our official binaries.

attic/BandwidthAccount.hpp

@@ -1,153 +0,0 @@
-/*
- * ZeroTier One - Network Virtualization Everywhere
- * Copyright (C) 2011-2015  ZeroTier, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * --
- *
- * ZeroTier may be used and distributed under the terms of the GPLv3, which
- * are available at: http://www.gnu.org/licenses/gpl-3.0.html
- *
- * If you would like to embed ZeroTier into a commercial application or
- * redistribute it in a modified binary form, please contact ZeroTier Networks
- * LLC. Start here: http://www.zerotier.com/
- */
-
-#ifndef ZT_BWACCOUNT_HPP
-#define ZT_BWACCOUNT_HPP
-
-#include "Constants.hpp"
-
-#include <algorithm>
-
-#include <stdint.h>
-#include <math.h>
-
-#include "Utils.hpp"
-
-#ifdef __WINDOWS__
-#define	round(x) ((x-floor(x))>0.5 ? ceil(x) : floor(x))
-#endif
-
-namespace ZeroTier {
-
-/**
- * Bandwidth account used for rate limiting multicast groups
- *
- * This is used to apply a bank account model to multicast groups. Each
- * multicast packet counts against a balance, which accrues at a given
- * rate in bytes per second. Debt is possible. These parameters are
- * configurable.
- *
- * A bank account model permits bursting behavior, which correctly models
- * how OSes and apps typically use multicast. It's common for things to
- * spew lots of multicast messages at once, wait a while, then do it
- * again. A consistent bandwidth limit model doesn't fit.
- */
-class BandwidthAccount
-{
-public:
-	/**
-	 * Create an uninitialized account
-	 *
-	 * init() must be called before this is used.
-	 */
-	BandwidthAccount() throw() {}
-
-	/**
-	 * Create and initialize
-	 *
-	 * @param preload Initial balance to place in account
-	 * @param maxb Maximum allowed balance (> 0)
-	 * @param acc Rate of accrual in bytes per second
-	 * @param now Current time
-	 */
-	BandwidthAccount(uint32_t preload,uint32_t maxb,uint32_t acc,uint64_t now)
-		throw()
-	{
-		init(preload,maxb,acc,now);
-	}
-
-	/**
-	 * Initialize or re-initialize account
-	 *
-	 * @param preload Initial balance to place in account
-	 * @param maxb Maximum allowed balance (> 0)
-	 * @param acc Rate of accrual in bytes per second
-	 * @param now Current time
-	 */
-	inline void init(uint32_t preload,uint32_t maxb,uint32_t acc,uint64_t now)
-		throw()
-	{
-		_lastTime = ((double)now / 1000.0);
-		_balance = preload;
-		_maxBalance = maxb;
-		_accrual = acc;
-	}
-
-	/**
-	 * Update and retrieve balance of this account
-	 *
-	 * @param now Current time
-	 * @return New balance updated from current clock
-	 */
-	inline uint32_t update(uint64_t now)
-		throw()
-	{
-		double lt = _lastTime;
-		double nowf = ((double)now / 1000.0);
-		_lastTime = nowf;
-		return (_balance = std::min(_maxBalance,(uint32_t)round((double)_balance + ((double)_accrual * (nowf - lt)))));
-	}
-
-	/**
-	 * Update balance and conditionally deduct
-	 *
-	 * If the deduction amount fits, it is deducted after update. Otherwise
-	 * balance is updated and false is returned.
-	 *
-	 * @param amt Amount to deduct
-	 * @param now Current time
-	 * @return True if amount fit within balance and was deducted
-	 */
-	inline bool deduct(uint32_t amt,uint64_t now)
-		throw()
-	{
-		if (update(now) >= amt) {
-			_balance -= amt;
-			return true;
-		}
-		return false;
-	}
-
-	/**
-	 * @return Most recent balance without update
-	 */
-	inline uint32_t balance() const
-		throw()
-	{
-		return _balance;
-	}
-
-private:
-	double _lastTime;
-	uint32_t _balance;
-	uint32_t _maxBalance;
-	uint32_t _accrual;
-};
-
-} // namespace ZeroTier
-
-#endif

attic/Cluster.cpp

@@ -1,6 +1,6 @@
 /*
  * ZeroTier One - Network Virtualization Everywhere
- * Copyright (C) 2011-2016  ZeroTier, Inc.  https://www.zerotier.com/
+ * Copyright (C) 2011-2017  ZeroTier, Inc.  https://www.zerotier.com/
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -14,6 +14,14 @@
  *
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
  */
 
 #ifdef ZT_ENABLE_CLUSTER
@@ -44,6 +52,7 @@
 #include "Packet.hpp"
 #include "Switch.hpp"
 #include "Node.hpp"
+#include "Network.hpp"
 #include "Array.hpp"
 
 namespace ZeroTier {
@@ -248,13 +257,13 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 		memcpy(keytmp,_key,32);
 		for(int i=0;i<8;++i)
 			keytmp[i] ^= reinterpret_cast<const char *>(msg)[i];
-		Salsa20 s20(keytmp,256,reinterpret_cast<const char *>(msg) + 8);
+		Salsa20 s20(keytmp,reinterpret_cast<const char *>(msg) + 8);
 		Utils::burn(keytmp,sizeof(keytmp));
 
 		// One-time-use Poly1305 key from first 32 bytes of Salsa20 keystream (as per DJB/NaCl "standard")
 		char polykey[ZT_POLY1305_KEY_LEN];
 		memset(polykey,0,sizeof(polykey));
-		s20.encrypt12(polykey,polykey,sizeof(polykey));
+		s20.crypt12(polykey,polykey,sizeof(polykey));
 
 		// Compute 16-byte MAC
 		char mac[ZT_POLY1305_MAC_LEN];
@@ -266,7 +275,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 
 		// Decrypt!
 		dmsg.setSize(len - 24);
-		s20.decrypt12(reinterpret_cast<const char *>(msg) + 24,const_cast<void *>(dmsg.data()),dmsg.size());
+		s20.crypt12(reinterpret_cast<const char *>(msg) + 24,const_cast<void *>(dmsg.data()),dmsg.size());
 	}
 
 	if (dmsg.size() < 4)
@@ -341,17 +350,20 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 						Identity id;
 						ptr += id.deserialize(dmsg,ptr);
 						if (id) {
-							RR->topology->saveIdentity(id);
-
 							{
 								Mutex::Lock _l(_remotePeers_m);
-								_remotePeers[std::pair<Address,unsigned int>(id.address(),(unsigned int)fromMemberId)] = RR->node->now();
+								_RemotePeer &rp = _remotePeers[std::pair<Address,unsigned int>(id.address(),(unsigned int)fromMemberId)];
+								if (!rp.lastHavePeerReceived) {
+									RR->topology->saveIdentity((void *)0,id);
+									RR->identity.agree(id,rp.key,ZT_PEER_SECRET_KEY_LENGTH);
+								}
+								rp.lastHavePeerReceived = RR->node->now();
 							}
 
 							_ClusterSendQueueEntry *q[16384]; // 16384 is "tons"
 							unsigned int qc = _sendQueue->getByDest(id.address(),q,16384);
 							for(unsigned int i=0;i<qc;++i)
-								this->sendViaCluster(q[i]->fromPeerAddress,q[i]->toPeerAddress,q[i]->data,q[i]->len,q[i]->unite);
+								this->relayViaCluster(q[i]->fromPeerAddress,q[i]->toPeerAddress,q[i]->data,q[i]->len,q[i]->unite);
 							_sendQueue->returnToPool(q,qc);
 
 							TRACE("[%u] has %s (retried %u queued sends)",(unsigned int)fromMemberId,id.address().toString().c_str(),qc);
@@ -361,7 +373,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 					case CLUSTER_MESSAGE_WANT_PEER: {
 						const Address zeroTierAddress(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH;
 						SharedPtr<Peer> peer(RR->topology->getPeerNoCache(zeroTierAddress));
-						if ( (peer) && (peer->hasClusterOptimalPath(RR->node->now())) ) {
+						if ( (peer) && (peer->hasLocalClusterOptimalPath(RR->node->now())) ) {
 							Buffer<1024> buf;
 							peer->identity().serialize(buf);
 							Mutex::Lock _l2(_members[fromMemberId].lock);
@@ -396,7 +408,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 						SharedPtr<Peer> localPeer(RR->topology->getPeerNoCache(localPeerAddress));
 						if ((localPeer)&&(numRemotePeerPaths > 0)) {
 							InetAddress bestLocalV4,bestLocalV6;
-							localPeer->getBestActiveAddresses(now,bestLocalV4,bestLocalV6);
+							localPeer->getRendezvousAddresses(now,bestLocalV4,bestLocalV6);
 
 							InetAddress bestRemoteV4,bestRemoteV6;
 							for(unsigned int i=0;i<numRemotePeerPaths;++i) {
@@ -455,7 +467,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 									Mutex::Lock _l2(_members[fromMemberId].lock);
 									_send(fromMemberId,CLUSTER_MESSAGE_PROXY_SEND,rendezvousForRemote.data(),rendezvousForRemote.size());
 								}
-								RR->sw->send(rendezvousForLocal,true,0);
+								RR->sw->send((void *)0,rendezvousForLocal,true);
 							}
 						}
 					}	break;
@@ -466,9 +478,18 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 						const unsigned int len = dmsg.at<uint16_t>(ptr); ptr += 2;
 						Packet outp(rcpt,RR->identity.address(),verb);
 						outp.append(dmsg.field(ptr,len),len); ptr += len;
-						RR->sw->send(outp,true,0);
+						RR->sw->send((void *)0,outp,true);
 						//TRACE("[%u] proxy send %s to %s length %u",(unsigned int)fromMemberId,Packet::verbString(verb),rcpt.toString().c_str(),len);
 					}	break;
+
+					case CLUSTER_MESSAGE_NETWORK_CONFIG: {
+						const SharedPtr<Network> network(RR->node->network(dmsg.at<uint64_t>(ptr)));
+						if (network) {
+							// Copy into a Packet just to conform to Network API. Eventually
+							// will want to refactor.
+							network->handleConfigChunk((void *)0,0,Address(),Buffer<ZT_PROTO_MAX_PACKET_LENGTH>(dmsg),ptr);
+						}
+					}	break;
 				}
 			} catch ( ... ) {
 				TRACE("invalid message of size %u type %d (inner decode), discarding",mlen,mtype);
@@ -494,7 +515,84 @@ void Cluster::broadcastHavePeer(const Identity &id)
 	}
 }
 
-void Cluster::sendViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite)
+void Cluster::broadcastNetworkConfigChunk(const void *chunk,unsigned int len)
+{
+	Mutex::Lock _l(_memberIds_m);
+	for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
+		Mutex::Lock _l2(_members[*mid].lock);
+		_send(*mid,CLUSTER_MESSAGE_NETWORK_CONFIG,chunk,len);
+	}
+}
+
+int Cluster::checkSendViaCluster(const Address &toPeerAddress,uint64_t &mostRecentTs,void *peerSecret)
+{
+	const uint64_t now = RR->node->now();
+	mostRecentTs = 0;
+	int mostRecentMemberId = -1;
+	{
+		Mutex::Lock _l2(_remotePeers_m);
+		std::map< std::pair<Address,unsigned int>,_RemotePeer >::const_iterator rpe(_remotePeers.lower_bound(std::pair<Address,unsigned int>(toPeerAddress,0)));
+		for(;;) {
+			if ((rpe == _remotePeers.end())||(rpe->first.first != toPeerAddress))
+				break;
+			else if (rpe->second.lastHavePeerReceived > mostRecentTs) {
+				mostRecentTs = rpe->second.lastHavePeerReceived;
+				memcpy(peerSecret,rpe->second.key,ZT_PEER_SECRET_KEY_LENGTH);
+				mostRecentMemberId = (int)rpe->first.second;
+			}
+			++rpe;
+		}
+	}
+
+	const uint64_t ageOfMostRecentHavePeerAnnouncement = now - mostRecentTs;
+	if (ageOfMostRecentHavePeerAnnouncement >= (ZT_PEER_ACTIVITY_TIMEOUT / 3)) {
+		if (ageOfMostRecentHavePeerAnnouncement >= ZT_PEER_ACTIVITY_TIMEOUT)
+			mostRecentMemberId = -1;
+
+		bool sendWantPeer = true;
+		{
+			Mutex::Lock _l(_remotePeers_m);
+			_RemotePeer &rp = _remotePeers[std::pair<Address,unsigned int>(toPeerAddress,(unsigned int)_id)];
+			if ((now - rp.lastSentWantPeer) >= ZT_CLUSTER_WANT_PEER_EVERY) {
+				rp.lastSentWantPeer = now;
+			} else {
+				sendWantPeer = false; // don't flood WANT_PEER
+			}
+		}
+		if (sendWantPeer) {
+			char tmp[ZT_ADDRESS_LENGTH];
+			toPeerAddress.copyTo(tmp,ZT_ADDRESS_LENGTH);
+			{
+				Mutex::Lock _l(_memberIds_m);
+				for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
+					Mutex::Lock _l2(_members[*mid].lock);
+					_send(*mid,CLUSTER_MESSAGE_WANT_PEER,tmp,ZT_ADDRESS_LENGTH);
+				}
+			}
+		}
+	}
+
+	return mostRecentMemberId;
+}
+
+bool Cluster::sendViaCluster(int mostRecentMemberId,const Address &toPeerAddress,const void *data,unsigned int len)
+{
+	if ((mostRecentMemberId < 0)||(mostRecentMemberId >= ZT_CLUSTER_MAX_MEMBERS)) // sanity check
+		return false;
+	Mutex::Lock _l2(_members[mostRecentMemberId].lock);
+	for(std::vector<InetAddress>::const_iterator i1(_zeroTierPhysicalEndpoints.begin());i1!=_zeroTierPhysicalEndpoints.end();++i1) {
+		for(std::vector<InetAddress>::const_iterator i2(_members[mostRecentMemberId].zeroTierPhysicalEndpoints.begin());i2!=_members[mostRecentMemberId].zeroTierPhysicalEndpoints.end();++i2) {
+			if (i1->ss_family == i2->ss_family) {
+				TRACE("sendViaCluster sending %u bytes to %s by way of %u (%s->%s)",len,toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId,i1->toString().c_str(),i2->toString().c_str());
+				RR->node->putPacket((void *)0,*i1,*i2,data,len);
+				return true;
+			}
+		}
+	}
+	return false;
+}
+
+void Cluster::relayViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite)
 {
 	if (len > ZT_PROTO_MAX_PACKET_LENGTH) // sanity check
 		return;
@@ -502,77 +600,100 @@ void Cluster::sendViaCluster(const Address &fromPeerAddress,const Address &toPee
 	const uint64_t now = RR->node->now();
 
 	uint64_t mostRecentTs = 0;
-	unsigned int mostRecentMemberId = 0xffffffff;
+	int mostRecentMemberId = -1;
 	{
 		Mutex::Lock _l2(_remotePeers_m);
-		std::map< std::pair<Address,unsigned int>,uint64_t >::const_iterator rpe(_remotePeers.lower_bound(std::pair<Address,unsigned int>(toPeerAddress,0)));
+		std::map< std::pair<Address,unsigned int>,_RemotePeer >::const_iterator rpe(_remotePeers.lower_bound(std::pair<Address,unsigned int>(toPeerAddress,0)));
 		for(;;) {
 			if ((rpe == _remotePeers.end())||(rpe->first.first != toPeerAddress))
 				break;
-			else if (rpe->second > mostRecentTs) {
-				mostRecentTs = rpe->second;
-				mostRecentMemberId = rpe->first.second;
+			else if (rpe->second.lastHavePeerReceived > mostRecentTs) {
+				mostRecentTs = rpe->second.lastHavePeerReceived;
+				mostRecentMemberId = (int)rpe->first.second;
 			}
 			++rpe;
 		}
 	}
 
-	const uint64_t age = now - mostRecentTs;
-	if (age >= (ZT_PEER_ACTIVITY_TIMEOUT / 3)) {
-		const bool enqueueAndWait = ((age >= ZT_PEER_ACTIVITY_TIMEOUT)||(mostRecentMemberId > 0xffff));
+	const uint64_t ageOfMostRecentHavePeerAnnouncement = now - mostRecentTs;
+	if (ageOfMostRecentHavePeerAnnouncement >= (ZT_PEER_ACTIVITY_TIMEOUT / 3)) {
+		// Enqueue and wait if peer seems alive, but do WANT_PEER to refresh homing
+		const bool enqueueAndWait = ((ageOfMostRecentHavePeerAnnouncement >= ZT_PEER_ACTIVITY_TIMEOUT)||(mostRecentMemberId < 0));
 
 		// Poll everyone with WANT_PEER if the age of our most recent entry is
 		// approaching expiration (or has expired, or does not exist).
-		char tmp[ZT_ADDRESS_LENGTH];
-		toPeerAddress.copyTo(tmp,ZT_ADDRESS_LENGTH);
+		bool sendWantPeer = true;
 		{
-			Mutex::Lock _l(_memberIds_m);
-			for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
-				Mutex::Lock _l2(_members[*mid].lock);
-				_send(*mid,CLUSTER_MESSAGE_WANT_PEER,tmp,ZT_ADDRESS_LENGTH);
+			Mutex::Lock _l(_remotePeers_m);
+			_RemotePeer &rp = _remotePeers[std::pair<Address,unsigned int>(toPeerAddress,(unsigned int)_id)];
+			if ((now - rp.lastSentWantPeer) >= ZT_CLUSTER_WANT_PEER_EVERY) {
+				rp.lastSentWantPeer = now;
+			} else {
+				sendWantPeer = false; // don't flood WANT_PEER
+			}
+		}
+		if (sendWantPeer) {
+			char tmp[ZT_ADDRESS_LENGTH];
+			toPeerAddress.copyTo(tmp,ZT_ADDRESS_LENGTH);
+			{
+				Mutex::Lock _l(_memberIds_m);
+				for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
+					Mutex::Lock _l2(_members[*mid].lock);
+					_send(*mid,CLUSTER_MESSAGE_WANT_PEER,tmp,ZT_ADDRESS_LENGTH);
+				}
 			}
 		}
 
 		// If there isn't a good place to send via, then enqueue this for retrying
 		// later and return after having broadcasted a WANT_PEER.
 		if (enqueueAndWait) {
-			TRACE("sendViaCluster %s -> %s enqueueing to wait for HAVE_PEER",fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str());
+			TRACE("relayViaCluster %s -> %s enqueueing to wait for HAVE_PEER",fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str());
 			_sendQueue->enqueue(now,fromPeerAddress,toPeerAddress,data,len,unite);
 			return;
 		}
 	}
 
-	Buffer<1024> buf;
-	if (unite) {
-		InetAddress v4,v6;
-		if (fromPeerAddress) {
-			SharedPtr<Peer> fromPeer(RR->topology->getPeerNoCache(fromPeerAddress));
-			if (fromPeer)
-				fromPeer->getBestActiveAddresses(now,v4,v6);
-		}
-		uint8_t addrCount = 0;
-		if (v4)
-			++addrCount;
-		if (v6)
-			++addrCount;
-		if (addrCount) {
-			toPeerAddress.appendTo(buf);
-			fromPeerAddress.appendTo(buf);
-			buf.append(addrCount);
+	if (mostRecentMemberId >= 0) {
+		Buffer<1024> buf;
+		if (unite) {
+			InetAddress v4,v6;
+			if (fromPeerAddress) {
+				SharedPtr<Peer> fromPeer(RR->topology->getPeerNoCache(fromPeerAddress));
+				if (fromPeer)
+					fromPeer->getRendezvousAddresses(now,v4,v6);
+			}
+			uint8_t addrCount = 0;
 			if (v4)
-				v4.serialize(buf);
+				++addrCount;
 			if (v6)
-				v6.serialize(buf);
+				++addrCount;
+			if (addrCount) {
+				toPeerAddress.appendTo(buf);
+				fromPeerAddress.appendTo(buf);
+				buf.append(addrCount);
+				if (v4)
+					v4.serialize(buf);
+				if (v6)
+					v6.serialize(buf);
+			}
 		}
-	}
 
-	{
-		Mutex::Lock _l2(_members[mostRecentMemberId].lock);
-		if (buf.size() > 0)
-			_send(mostRecentMemberId,CLUSTER_MESSAGE_PROXY_UNITE,buf.data(),buf.size());
-		if (_members[mostRecentMemberId].zeroTierPhysicalEndpoints.size() > 0) {
-			TRACE("sendViaCluster relaying %u bytes from %s to %s by way of %u",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId);
-			RR->node->putPacket(InetAddress(),_members[mostRecentMemberId].zeroTierPhysicalEndpoints.front(),data,len);
+		{
+			Mutex::Lock _l2(_members[mostRecentMemberId].lock);
+			if (buf.size() > 0)
+				_send(mostRecentMemberId,CLUSTER_MESSAGE_PROXY_UNITE,buf.data(),buf.size());
+
+			for(std::vector<InetAddress>::const_iterator i1(_zeroTierPhysicalEndpoints.begin());i1!=_zeroTierPhysicalEndpoints.end();++i1) {
+				for(std::vector<InetAddress>::const_iterator i2(_members[mostRecentMemberId].zeroTierPhysicalEndpoints.begin());i2!=_members[mostRecentMemberId].zeroTierPhysicalEndpoints.end();++i2) {
+					if (i1->ss_family == i2->ss_family) {
+						TRACE("relayViaCluster relaying %u bytes from %s to %s by way of %u (%s->%s)",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId,i1->toString().c_str(),i2->toString().c_str());
+						RR->node->putPacket((void *)0,*i1,*i2,data,len);
+						return;
+					}
+				}
+			}
+
+			TRACE("relayViaCluster relaying %u bytes from %s to %s by way of %u failed: no common endpoints with the same address family!",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId);
 		}
 	}
 }
@@ -635,8 +756,8 @@ void Cluster::doPeriodicTasks()
 		_lastCleanedRemotePeers = now;
 
 		Mutex::Lock _l(_remotePeers_m);
-		for(std::map< std::pair<Address,unsigned int>,uint64_t >::iterator rp(_remotePeers.begin());rp!=_remotePeers.end();) {
-			if ((now - rp->second) >= ZT_PEER_ACTIVITY_TIMEOUT)
+		for(std::map< std::pair<Address,unsigned int>,_RemotePeer >::iterator rp(_remotePeers.begin());rp!=_remotePeers.end();) {
+			if ((now - rp->second.lastHavePeerReceived) >= ZT_PEER_ACTIVITY_TIMEOUT)
 				_remotePeers.erase(rp++);
 			else ++rp;
 		}
@@ -701,7 +822,7 @@ bool Cluster::findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddr
 		// Pick based on location if it can be determined
 		int px = 0,py = 0,pz = 0;
 		if (_addressToLocationFunction(_addressToLocationFunctionArg,reinterpret_cast<const struct sockaddr_storage *>(&peerPhysicalAddress),&px,&py,&pz) == 0) {
-			TRACE("no geolocation data for %s (geo-lookup is lazy/async so it may work next time)",peerPhysicalAddress.toIpString().c_str());
+			TRACE("no geolocation data for %s",peerPhysicalAddress.toIpString().c_str());
 			return false;
 		}
 
@@ -710,7 +831,9 @@ bool Cluster::findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddr
 		std::vector<InetAddress> best;
 		const double currentDistance = _dist3d(_x,_y,_z,px,py,pz);
 		double bestDistance = (offload ? 2147483648.0 : currentDistance);
+#ifdef ZT_TRACE
 		unsigned int bestMember = _id;
+#endif
 		{
 			Mutex::Lock _l(_memberIds_m);
 			for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
@@ -722,7 +845,9 @@ bool Cluster::findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddr
 					const double mdist = _dist3d(m.x,m.y,m.z,px,py,pz);
 					if (mdist < bestDistance) {
 						bestDistance = mdist;
+#ifdef ZT_TRACE
 						bestMember = *mid;
+#endif
 						best = m.zeroTierPhysicalEndpoints;
 					}
 				}
@@ -745,6 +870,19 @@ bool Cluster::findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddr
 	}
 }
 
+bool Cluster::isClusterPeerFrontplane(const InetAddress &ip) const
+{
+	Mutex::Lock _l(_memberIds_m);
+	for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
+		Mutex::Lock _l2(_members[*mid].lock);
+		for(std::vector<InetAddress>::const_iterator i2(_members[*mid].zeroTierPhysicalEndpoints.begin());i2!=_members[*mid].zeroTierPhysicalEndpoints.end();++i2) {
+			if (ip == *i2)
+				return true;
+		}
+	}
+	return false;
+}
+
 void Cluster::status(ZT_ClusterStatus &status) const
 {
 	const uint64_t now = RR->node->now();
@@ -818,16 +956,16 @@ void Cluster::_flush(uint16_t memberId)
 		memcpy(keytmp,m.key,32);
 		for(int i=0;i<8;++i)
 			keytmp[i] ^= m.q[i];
-		Salsa20 s20(keytmp,256,m.q.field(8,8));
+		Salsa20 s20(keytmp,m.q.field(8,8));
 		Utils::burn(keytmp,sizeof(keytmp));
 
 		// One-time-use Poly1305 key from first 32 bytes of Salsa20 keystream (as per DJB/NaCl "standard")
 		char polykey[ZT_POLY1305_KEY_LEN];
 		memset(polykey,0,sizeof(polykey));
-		s20.encrypt12(polykey,polykey,sizeof(polykey));
+		s20.crypt12(polykey,polykey,sizeof(polykey));
 
 		// Encrypt m.q in place
-		s20.encrypt12(reinterpret_cast<const char *>(m.q.data()) + 24,const_cast<char *>(reinterpret_cast<const char *>(m.q.data())) + 24,m.q.size() - 24);
+		s20.crypt12(reinterpret_cast<const char *>(m.q.data()) + 24,const_cast<char *>(reinterpret_cast<const char *>(m.q.data())) + 24,m.q.size() - 24);
 
 		// Add MAC for authentication (encrypt-then-MAC)
 		char mac[ZT_POLY1305_MAC_LEN];
@@ -851,7 +989,7 @@ void Cluster::_flush(uint16_t memberId)
 void Cluster::_doREMOTE_WHOIS(uint64_t fromMemberId,const Packet &remotep)
 {
 	if (remotep.payloadLength() >= ZT_ADDRESS_LENGTH) {
-		Identity queried(RR->topology->getIdentity(Address(remotep.payload(),ZT_ADDRESS_LENGTH)));
+		Identity queried(RR->topology->getIdentity((void *)0,Address(remotep.payload(),ZT_ADDRESS_LENGTH)));
 		if (queried) {
 			Buffer<1024> routp;
 			remotep.source().appendTo(routp);

attic/Cluster.hpp

@@ -1,6 +1,6 @@
 /*
  * ZeroTier One - Network Virtualization Everywhere
- * Copyright (C) 2011-2016  ZeroTier, Inc.  https://www.zerotier.com/
+ * Copyright (C) 2011-2017  ZeroTier, Inc.  https://www.zerotier.com/
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -14,6 +14,14 @@
  *
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
  */
 
 #ifndef ZT_CLUSTER_HPP
@@ -47,22 +55,22 @@
 /**
  * Desired period between doPeriodicTasks() in milliseconds
  */
-#define ZT_CLUSTER_PERIODIC_TASK_PERIOD 50
+#define ZT_CLUSTER_PERIODIC_TASK_PERIOD 20
 
 /**
  * How often to flush outgoing message queues (maximum interval)
  */
-#define ZT_CLUSTER_FLUSH_PERIOD 100
+#define ZT_CLUSTER_FLUSH_PERIOD ZT_CLUSTER_PERIODIC_TASK_PERIOD
 
 /**
  * Maximum number of queued outgoing packets per sender address
  */
-#define ZT_CLUSTER_MAX_QUEUE_PER_SENDER 8
+#define ZT_CLUSTER_MAX_QUEUE_PER_SENDER 16
 
 /**
  * Expiration time for send queue entries
  */
-#define ZT_CLUSTER_QUEUE_EXPIRATION 5000
+#define ZT_CLUSTER_QUEUE_EXPIRATION 3000
 
 /**
  * Chunk size for allocating queue entries
@@ -85,11 +93,13 @@
 
 /**
  * Max data per queue entry
- *
- * If we ever support larger transport MTUs this must be increased. The plus
- * 16 is just a small margin and has no special meaning.
  */
-#define ZT_CLUSTER_SEND_QUEUE_DATA_MAX (ZT_UDP_DEFAULT_PAYLOAD_MTU + 16)
+#define ZT_CLUSTER_SEND_QUEUE_DATA_MAX 1500
+
+/**
+ * We won't send WANT_PEER to other members more than every (ms) per recipient
+ */
+#define ZT_CLUSTER_WANT_PEER_EVERY 1000
 
 namespace ZeroTier {
 
@@ -219,14 +229,13 @@ public:
 
 		/**
 		 * Replicate a network config for a network we belong to:
-		 *   <[8] 64-bit network ID>
-		 *   <[2] 16-bit length of network config>
-		 *   <[...] serialized network config>
+		 *   <[...] network config chunk>
 		 *
 		 * This is used by clusters to avoid every member having to query
 		 * for the same netconf for networks all members belong to.
 		 *
-		 * TODO: not implemented yet!
+		 * The first field of a network config chunk is the network ID,
+		 * so this can be checked to look up the network on receipt.
 		 */
 		CLUSTER_MESSAGE_NETWORK_CONFIG = 7
 	};
@@ -271,7 +280,38 @@ public:
 	void broadcastHavePeer(const Identity &id);
 
 	/**
-	 * Send this packet via another node in this cluster if another node has this peer
+	 * Broadcast a network config chunk to other members of cluster
+	 *
+	 * @param chunk Chunk data
+	 * @param len Length of chunk
+	 */
+	void broadcastNetworkConfigChunk(const void *chunk,unsigned int len);
+
+	/**
+	 * If the cluster has this peer, prepare the packet to send via cluster
+	 *
+	 * Note that outp is only armored (or modified at all) if the return value is a member ID.
+	 *
+	 * @param toPeerAddress Value of outp.destination(), simply to save additional lookup
+	 * @param ts Result: set to time of last HAVE_PEER from the cluster
+	 * @param peerSecret Result: Buffer to fill with peer secret on valid return value, must be at least ZT_PEER_SECRET_KEY_LENGTH bytes
+	 * @return -1 if cluster does not know this peer, or a member ID to pass to sendViaCluster()
+	 */
+	int checkSendViaCluster(const Address &toPeerAddress,uint64_t &mostRecentTs,void *peerSecret);
+
+	/**
+	 * Send data via cluster front plane (packet head or fragment)
+	 *
+	 * @param haveMemberId Member ID that has this peer as returned by prepSendviaCluster()
+	 * @param toPeerAddress Destination peer address
+	 * @param data Packet or packet fragment data
+	 * @param len Length of packet or fragment
+	 * @return True if packet was sent (and outp was modified via armoring)
+	 */
+	bool sendViaCluster(int haveMemberId,const Address &toPeerAddress,const void *data,unsigned int len);
+
+	/**
+	 * Relay a packet via the cluster
 	 *
 	 * This is used in the outgoing packet and relaying logic in Switch to
 	 * relay packets to other cluster members. It isn't PROXY_SEND-- that is
@@ -283,7 +323,7 @@ public:
 	 * @param len Length of packet or fragment
 	 * @param unite If true, also request proxy unite across cluster
 	 */
-	void sendViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite);
+	void relayViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite);
 
 	/**
 	 * Send a distributed query to other cluster members
@@ -326,6 +366,12 @@ public:
 	 */
 	bool findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload);
 
+	/**
+	 * @param ip Address to check
+	 * @return True if this is a cluster frontplane address (excluding our addresses)
+	 */
+	bool isClusterPeerFrontplane(const InetAddress &ip) const;
+
 	/**
 	 * Fill out ZT_ClusterStatus structure (from core API)
 	 *
@@ -394,7 +440,15 @@ private:
 	std::vector<uint16_t> _memberIds;
 	Mutex _memberIds_m;
 
-	std::map< std::pair<Address,unsigned int>,uint64_t > _remotePeers; // we need ordered behavior and lower_bound here
+	struct _RemotePeer
+	{
+		_RemotePeer() : lastHavePeerReceived(0),lastSentWantPeer(0) {}
+		~_RemotePeer() { Utils::burn(key,ZT_PEER_SECRET_KEY_LENGTH); }
+		uint64_t lastHavePeerReceived;
+		uint64_t lastSentWantPeer;
+		uint8_t key[ZT_PEER_SECRET_KEY_LENGTH]; // secret key from identity agreement
+	};
+	std::map< std::pair<Address,unsigned int>,_RemotePeer > _remotePeers; // we need ordered behavior and lower_bound here
 	Mutex _remotePeers_m;
 
 	uint64_t _lastFlushed;

attic/ClusterDefinition.hpp

@@ -1,6 +1,6 @@
 /*
  * ZeroTier One - Network Virtualization Everywhere
- * Copyright (C) 2011-2016  ZeroTier, Inc.  https://www.zerotier.com/
+ * Copyright (C) 2011-2017  ZeroTier, Inc.  https://www.zerotier.com/
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -14,6 +14,14 @@
  *
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
  */
 
 #ifndef ZT_CLUSTERDEFINITION_HPP
@@ -26,14 +34,17 @@
 
 #include "../node/Constants.hpp"
 #include "../node/Utils.hpp"
+#include "../node/NonCopyable.hpp"
 #include "../osdep/OSUtils.hpp"
 
+#include "ClusterGeoIpService.hpp"
+
 namespace ZeroTier {
 
 /**
  * Parser for cluster definition file
  */
-class ClusterDefinition
+class ClusterDefinition : NonCopyable
 {
 public:
 	struct MemberDefinition
@@ -47,6 +58,13 @@ public:
 		std::vector<InetAddress> zeroTierEndpoints;
 	};
 
+	/**
+	 * Load and initialize cluster definition and GeoIP data if any
+	 *
+	 * @param myAddress My ZeroTier address
+	 * @param pathToClusterFile Path to cluster definition file
+	 * @throws std::runtime_error Invalid cluster definition or unable to load data
+	 */
 	ClusterDefinition(uint64_t myAddress,const char *pathToClusterFile)
 	{
 		std::string cf;
@@ -54,22 +72,36 @@ public:
 			return;
 
 		char myAddressStr[64];
-		Utils::snprintf(myAddressStr,sizeof(myAddressStr),"%.10llx",myAddress);
+		Utils::ztsnprintf(myAddressStr,sizeof(myAddressStr),"%.10llx",myAddress);
 
-		std::vector<std::string> lines(Utils::split(cf.c_str(),"\r\n","",""));
+		std::vector<std::string> lines(OSUtils::split(cf.c_str(),"\r\n","",""));
 		for(std::vector<std::string>::iterator l(lines.begin());l!=lines.end();++l) {
-			std::vector<std::string> fields(Utils::split(l->c_str()," \t","",""));
+			std::vector<std::string> fields(OSUtils::split(l->c_str()," \t","",""));
 			if ((fields.size() < 5)||(fields[0][0] == '#')||(fields[0] != myAddressStr))
 				continue;
 
+			// <address> geo <CSV path> <ip start column> <ip end column> <latitutde column> <longitude column>
+			if (fields[1] == "geo") {
+				if ((fields.size() >= 7)&&(OSUtils::fileExists(fields[2].c_str()))) {
+					int ipStartColumn = Utils::strToInt(fields[3].c_str());
+					int ipEndColumn = Utils::strToInt(fields[4].c_str());
+					int latitudeColumn = Utils::strToInt(fields[5].c_str());
+					int longitudeColumn = Utils::strToInt(fields[6].c_str());
+					if (_geo.load(fields[2].c_str(),ipStartColumn,ipEndColumn,latitudeColumn,longitudeColumn) <= 0)
+						throw std::runtime_error(std::string("failed to load geo-ip data from ")+fields[2]);
+				}
+				continue;
+			}
+
+			// <address> <ID> <name> <backplane IP/port(s)> <ZT frontplane IP/port(s)> <x,y,z>
 			int id = Utils::strToUInt(fields[1].c_str());
 			if ((id < 0)||(id > ZT_CLUSTER_MAX_MEMBERS))
-				continue;
+				throw std::runtime_error(std::string("invalid cluster member ID: ")+fields[1]);
 			MemberDefinition &md = _md[id];
 
 			md.id = (unsigned int)id;
 			if (fields.size() >= 6) {
-				std::vector<std::string> xyz(Utils::split(fields[5].c_str(),",","",""));
+				std::vector<std::string> xyz(OSUtils::split(fields[5].c_str(),",","",""));
 				md.x = (xyz.size() > 0) ? Utils::strToInt(xyz[0].c_str()) : 0;
 				md.y = (xyz.size() > 1) ? Utils::strToInt(xyz[1].c_str()) : 0;
 				md.z = (xyz.size() > 2) ? Utils::strToInt(xyz[2].c_str()) : 0;
@@ -78,7 +110,7 @@ public:
 			md.clusterEndpoint.fromString(fields[3]);
 			if (!md.clusterEndpoint)
 				continue;
-			std::vector<std::string> zips(Utils::split(fields[4].c_str(),",","",""));
+			std::vector<std::string> zips(OSUtils::split(fields[4].c_str(),",","",""));
 			for(std::vector<std::string>::iterator zip(zips.begin());zip!=zips.end();++zip) {
 				InetAddress i;
 				i.fromString(*zip);
@@ -92,10 +124,29 @@ public:
 		std::sort(_ids.begin(),_ids.end());
 	}
 
+	/**
+	 * @return All member definitions in this cluster by ID (ID is array index)
+	 */
 	inline const MemberDefinition &operator[](unsigned int id) const throw() { return _md[id]; }
+
+	/**
+	 * @return Number of members in this cluster
+	 */
 	inline unsigned int size() const throw() { return (unsigned int)_ids.size(); }
+
+	/**
+	 * @return IDs of members in this cluster sorted by ID
+	 */
 	inline const std::vector<unsigned int> &ids() const throw() { return _ids; }
 
+	/**
+	 * @return GeoIP service for this cluster
+	 */
+	inline ClusterGeoIpService &geo() throw() { return _geo; }
+
+	/**
+	 * @return A vector (new copy) containing all cluster members
+	 */
 	inline std::vector<MemberDefinition> members() const
 	{
 		std::vector<MemberDefinition> m;
@@ -107,6 +158,7 @@ public:
 private:
 	MemberDefinition _md[ZT_CLUSTER_MAX_MEMBERS];
 	std::vector<unsigned int> _ids;
+	ClusterGeoIpService _geo;
 };
 
 } // namespace ZeroTier