join webhost to ZT network and configure separate scraper jobs for it + ZT controller node

Dockerfile.webhost

@@ -3,5 +3,6 @@ FROM debian:stable
 RUN apt-get update
 RUN apt-get install curl nginx -y
 RUN curl -s https://install.zerotier.com | bash
+COPY ./webhost-init.sh /
 
-CMD ["nginx", "-g", "daemon off;"]
+CMD ["/webhost-init.sh"]

README.md

@@ -1,5 +1,18 @@
 # ZeroTier Observability Metrics Tutorial
 
+This repository has an example docker-compose configuration and setup scripts to run a ZeroTier network along with an observability stack based on Prometheus and Grafana.
+
+To run this locally, you'll need the following tools installed:
+
+- Docker + docker-compose
+- jq
+
+We've tested this demo on MacOS and Linux; on Windows, you'll need to use WSL2 for a local Linux environment. (Note: pull requests to add PowerShell support for the setup script are welcome!)
+
+You can use this type of monitoring pipeline to measure the health and activity of your networks, whether you're using our [managed environment](https://my.zerotier.com) or hosting your own controller. For this example, we're running a standalone controller, which allows us to avoid capturing and injecting a valid ZeroTier Central API token into the demo environment, but the same observability tools and configuration methods will work for managed networks as well.
+
+For more information on the full set of metrics we expose and how to configure your own Prometheus instance to extract them for visualization and alerting, see the [ZeroTierOne README](https://github.com/zerotier/ZeroTierOne#prometheus-metrics).
+
 ## Preparation
 
 ```
@@ -13,12 +26,14 @@ $ ./setup.sh
 
 Next, open `http://localhost:3000/` in your browser and follow the steps below to configure dashboards.
 
+TODO: add screenshots for each step below
+
 1. Login (admin/admin; set a new password or click 'skip')
 2. Connections -> Data Sources -> New Data Source
 3. Select 'Prometheus' (first option)
 4. Set the prometheus server URL: `http://metrics-tutorial-prometheus-1:9090`
 5. Click 'Save and test'; then 'Explore view'
-6. 
+6. Select 'zt_packet' in the 'Metric' dropdown, then 'Last 15 minutes' in the range picker in the upper-right. Click 'Run query' and the graph will be populated in the bottom half of the screen.
 
 ## Cleanup
 
@@ -26,4 +41,15 @@ Next, open `http://localhost:3000/` in your browser and follow the steps below t
 
 $ ./cleanup.sh
 
-```
+```
+
+## Next Steps
+
+Now that you've set up a basic pipeline, you can try creating a custom dashboard reporting ZeroTier metrics.
+
+TODO: find good Grafana setup tutorial
+
+The following metrics in particular may prove useful for general health + activity monitoring:
+
+- zt_packet (labeled by rx/tx direction, peer node ID, network ID, etc.)
+- zt_peer_path (active paths to known network peers)

docker-compose.yaml

@@ -29,6 +29,7 @@ services:
       - internal
 
   webhost:
+    privileged: true
     build:
       dockerfile: ./Dockerfile.webhost
     ports:
@@ -37,6 +38,8 @@ services:
       - webhost-data:/var/lib/zerotier-one
     networks:
       - internal
+    devices:
+      - /dev/net/tun
 
 volumes:
   zerotier-data:

setup.sh

@@ -1,20 +1,43 @@
 #!/bin/bash
 
-set -euxo pipefail
+set -euo pipefail
 
-ZT_CONTAINER=metrics-tutorial-zerotier-1
+_=$(jq --version)
+if [ $? != 0 ]; then
+  echo "Missing 'jq' in path; exiting"
+fi
+
+CTRL_CONTAINER=metrics-tutorial-zerotier-1
 WEB_CONTAINER=metrics-tutorial-webhost-1
 
 PROM_CONFIG_PATH=./config/prometheus/prometheus.yml
 
-export ZT_METRICS_TOKEN=$(docker exec $ZT_CONTAINER \
-    cat /var/lib/zerotier-one/metricstoken.secret)
+get_zt_node_id() {
+  CONTAINER=$1
+  echo $(docker exec $CONTAINER zerotier-cli info | cut -d ' ' -f 3)
+}
 
-export ZT_ADMIN_TOKEN=$(docker exec $ZT_CONTAINER \
-    cat /var/lib/zerotier-one/authtoken.secret)
+get_zt_token() {
+  CONTAINER=$1
+  TOKEN_KIND=${2:='metrics'}
+  TOKEN_PATH="/var/lib/zerotier-one/${TOKEN_KIND}token.secret"
 
-export ZT_NODE_ID=$(docker exec $ZT_CONTAINER \
-    zerotier-cli info | cut -d ' ' -f 3)
+  echo $(docker exec $CONTAINER cat $TOKEN_PATH)
+}
+
+get_zt_ipaddr() {
+  CONTAINER=$1
+  NWID=$2
+
+  echo $(docker exec $CONTAINER zerotier-cli get $NWID ip | egrep '^\d\d\d.')
+}
+
+export CTRL_NODE_ID=$(get_zt_node_id $CTRL_CONTAINER)
+export WEB_NODE_ID=$(get_zt_node_id $WEB_CONTAINER)
+
+export CTRL_METRICS_TOKEN=$(get_zt_token $CTRL_CONTAINER 'metrics')
+export CTRL_ADMIN_TOKEN=$(get_zt_token $CTRL_CONTAINER 'auth')
+export WEB_METRICS_TOKEN=$(get_zt_token $WEB_CONTAINER 'metrics')
 
 NETWORK_CONFIG_JSON=$(cat <<END
 {
@@ -30,31 +53,57 @@ NETWORK_CONFIG_JSON=$(cat <<END
 }
 END)
 
-NETWORK_INFO=$(docker exec $ZT_CONTAINER curl -H "X-ZT1-Auth: $ZT_ADMIN_TOKEN" \
+NETWORK_INFO=$(docker exec $CTRL_CONTAINER curl -H "X-ZT1-Auth: $CTRL_ADMIN_TOKEN" \
   -d "$NETWORK_CONFIG_JSON" \
-  http://localhost:9993/controller/network/${ZT_NODE_ID}______)
+  http://localhost:9993/controller/network/${CTRL_NODE_ID}______)
 
 NETWORK_ID=$(echo "$NETWORK_INFO" | jq -r .nwid)
-docker exec $ZT_CONTAINER zerotier-cli join $NETWORK_ID
+docker exec $CTRL_CONTAINER zerotier-cli join $NETWORK_ID
 docker exec $WEB_CONTAINER zerotier-cli join $NETWORK_ID
 
+export CTRL_ZT_IPADDR=$(get_zt_ipaddr $CTRL_CONTAINER $NETWORK_ID)
+export WEB_ZT_IPADDR=$(get_zt_ipaddr $WEB_CONTAINER $NETWORK_ID)
+
 PROM_CONFIG=$(cat <<END
 scrape_configs:
-- job_name: zerotier-one
+- job_name: zt-controller
   honor_labels: true
   scrape_interval: 15s
-  metrics_path: /metrics
   static_configs:
   - targets:
-    - $ZT_CONTAINER:9993
+    - $CTRL_CONTAINER:9993
     labels:
       group: zerotier-one
-      node_id: $ZT_NODE_ID
+      node_id: $CTRL_NODE_ID
+      network_id: $NETWORK_ID
   authorization:
-    credentials: $ZT_METRICS_TOKEN
+    credentials: $CTRL_METRICS_TOKEN
+- job_name: webhost
+  honor_labels: true
+  scrape_interval: 15s
+  static_configs:
+  - targets:
+    - $WEB_CONTAINER:9993
+    labels:
+      group: zerotier-one
+      node_id: $WEB_NODE_ID
+      network_id: $NETWORK_ID
+  authorization:
+    credentials: $WEB_METRICS_TOKEN
 END)
 
 echo "$PROM_CONFIG" > $PROM_CONFIG_PATH
 docker cp $PROM_CONFIG_PATH metrics-tutorial-prometheus-1:/prometheus/prometheus.yml
 sleep 1
 docker-compose restart prometheus
+
+HOST_INFO=$(cat <<END
+controller:
+  node_id: $CTRL_NODE_ID
+  zt_ip: $CTRL_ZT_IPADDR
+webhost:
+  node_id: $WEB_NODE_ID
+  zt_ip: $WEB_ZT_IPADDR
+END)
+
+echo "$HOST_INFO"

webhost-init.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+service zerotier-one start
+nginx -g "daemon off;"