Merge pull request #1108 from dqminh/etcd-storage

Add etcd backed storage
2026-05-22 18:43:53 -07:00 · 2017-11-06 08:36:43 -08:00
parent e623bd626e e617197871
commit ccf85a7269
58 changed files with 28531 additions and 6 deletions
@@ -6,18 +6,20 @@ go:
  - 1.8
  - 1.9

+go_import_path: github.com/coreos/dex
+
 services:
  - postgresql
  - docker

 env:
-  - DEX_POSTGRES_DATABASE=postgres DEX_POSTGRES_USER=postgres DEX_POSTGRES_HOST="localhost" DEX_LDAP_TESTS=1 DEBIAN_FRONTEND=noninteractive
-
+  - DEX_POSTGRES_DATABASE=postgres DEX_POSTGRES_USER=postgres DEX_POSTGRES_HOST="localhost" DEX_ETCD_ENDPOINTS=http://localhost:2379 DEX_LDAP_TESTS=1 DEBIAN_FRONTEND=noninteractive

 install:
  - go get -u github.com/golang/lint/golint
  - sudo -E apt-get install -y --force-yes slapd time ldap-utils
  - sudo /etc/init.d/slapd stop
+  - docker run -d --net=host gcr.io/etcd-development/etcd:v3.2.9


 script:
@@ -4,6 +4,38 @@ Dex requires persisting state to perform various tasks such as track refresh tok

 Storage breaches are serious as they can affect applications that rely on dex. Dex saves sensitive data in its backing storage, including signing keys and bcrypt'd passwords. As such, transport security and database ACLs should both be used, no matter which storage option is chosen.

+## Etcd
+
+Dex supports persisting state to [etcd v3](https://github.com/coreos/etcd).
+
+An example etcd configuration is using these values:
+
+```
+storage:
+  type: etcd
+  config:
+    # list of etcd endpoints we should connect to
+    endpoints:
+      - http://localhost:2379
+    namespace: my-etcd-namespace/
+```
+
+Etcd storage can be customized further using the following options:
+
+* `endpoints`: list of etcd endpoints we should connect to
+* `namespace`: etcd namespace to be set for the connection. All keys created by
+  etcd storage will be prefixed with the namespace. This is useful when you
+  share your etcd cluster amongst several applications. Another approach for
+  setting namespace is to use [etcd proxy](https://coreos.com/etcd/docs/latest/op-guide/grpc_proxy.html#namespacing)
+* `username`: username for etcd authentication
+* `password`: password for etcd authentication
+* `ssl`: ssl setup for etcd connection
+  * `serverName`: ensures that the certificate matches the given hostname the
+    client is connecting to.
+  * `caFile`: path to the ca
+  * `keyFile`: path to the private key
+  * `certFile`: path to the certificate
+
 ## Kubernetes custom resource definitions (CRDs)

 __NOTE:__ CRDs are only supported by Kubernetes version 1.7+.
@@ -11,6 +11,7 @@ import (

 	"github.com/coreos/dex/server"
 	"github.com/coreos/dex/storage"
+	"github.com/coreos/dex/storage/etcd"
 	"github.com/coreos/dex/storage/kubernetes"
 	"github.com/coreos/dex/storage/memory"
 	"github.com/coreos/dex/storage/sql"
@@ -124,6 +125,7 @@ type StorageConfig interface {
 }

 var storages = map[string]func() StorageConfig{
+	"etcd":       func() StorageConfig { return new(etcd.Etcd) },
 	"kubernetes": func() StorageConfig { return new(kubernetes.Config) },
 	"memory":     func() StorageConfig { return new(memory.Config) },
 	"sqlite3":    func() StorageConfig { return new(sql.SQLite3) },
@@ -1,5 +1,5 @@
-hash: e83f57fea75d4a509d812fab12785599fdab16db950cc3738cec59a6c73f93af
-updated: 2017-07-25T13:48:39.899190846-07:00
+hash: 322a63b37abfe42807c2249c7d7b18dfb508c5548853732cf55c0c36a7688da5
+updated: 2017-10-31T14:32:36.104478036Z
 imports:
 - name: github.com/beevik/etree
  version: 4cd0dd976db869f817248477718071a28e978df0
@@ -7,6 +7,17 @@ imports:
  version: 31611c0501c812f437d4861d87d117053967c955
  subpackages:
  - crdb
+- name: github.com/coreos/etcd
+  version: f1d7dd87da3e8feab4aaf675b8e29c6a5ed5f58b
+  subpackages:
+  - auth/authpb
+  - clientv3
+  - clientv3/namespace
+  - etcdserver/api/v3rpc/rpctypes
+  - etcdserver/etcdserverpb
+  - mvcc/mvccpb
+  - pkg/tlsutil
+  - pkg/transport
 - name: github.com/coreos/go-oidc
  version: be73733bb8cc830d0205609b95d125215f8e9c70
 - name: github.com/ghodss/yaml
@@ -109,6 +109,12 @@ import:
  subpackages:
  - crdb

+# etcd driver
+- package: github.com/coreos/etcd
+  version: v3.2.9
+  subpackages:
+  - clientv3
+
 # gRPC and protobuf are use for the API. Also import x/net/http2 stack.
 - package: google.golang.org/grpc
  version: 0e8b58d22f34640cb17dbbed1c8aef3b8dcc0e97
@@ -628,9 +628,21 @@ func testConnectorCRUD(t *testing.T, s storage.Storage) {
 	c1.Type = "oidc"
 	getAndCompare(id1, c1)

-	if _, err := s.ListConnectors(); err != nil {
-		t.Fatalf("failed to list connectors: %v", err)
+	connectorList := []storage.Connector{c1, c2}
+	listAndCompare := func(want []storage.Connector) {
+		connectors, err := s.ListConnectors()
+		if err != nil {
+			t.Errorf("list connectors: %v", err)
+			return
+		}
+		sort.Slice(connectors, func(i, j int) bool {
+			return connectors[i].Name < connectors[j].Name
+		})
+		if diff := pretty.Compare(want, connectors); diff != "" {
+			t.Errorf("password list retrieved from storage did not match: %s", diff)
+		}
 	}
+	listAndCompare(connectorList)

 	if err := s.DeleteConnector(c1.ID); err != nil {
 		t.Fatalf("failed to delete connector: %v", err)
@@ -0,0 +1,92 @@
+package etcd
+
+import (
+	"time"
+
+	"github.com/coreos/dex/storage"
+	"github.com/coreos/etcd/clientv3"
+	"github.com/coreos/etcd/clientv3/namespace"
+	"github.com/coreos/etcd/pkg/transport"
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	defaultDialTimeout = 2 * time.Second
+)
+
+// SSL represents SSL options for etcd databases.
+type SSL struct {
+	ServerName string `json:"serverName" yaml:"serverName"`
+	CAFile     string `json:"caFile" yaml:"caFile"`
+	KeyFile    string `json:"keyFile" yaml:"keyFile"`
+	CertFile   string `json:"certFile" yaml:"certFile"`
+}
+
+// Etcd options for connecting to etcd databases.
+// If you are using a shared etcd cluster for storage, it might be useful to
+// configure an etcd namespace either via Namespace field or using `etcd grpc-proxy
+// --namespace=<prefix>`
+type Etcd struct {
+	Endpoints []string `json:"endpoints" yaml:"endpoints"`
+	Namespace string   `json:"namespace" yaml:"namespace"`
+	Username  string   `json:"username" yaml:"username"`
+	Password  string   `json:"password" yaml:"password"`
+	SSL       SSL      `json:"ssl" yaml:"ssl"`
+}
+
+// Open creates a new storage implementation backed by Etcd
+func (p *Etcd) Open(logger logrus.FieldLogger) (storage.Storage, error) {
+	return p.open(logger)
+}
+
+func (p *Etcd) open(logger logrus.FieldLogger) (*conn, error) {
+	cfg := clientv3.Config{
+		Endpoints:   p.Endpoints,
+		DialTimeout: defaultDialTimeout * time.Second,
+		Username:    p.Username,
+		Password:    p.Password,
+	}
+
+	var cfgtls *transport.TLSInfo
+	tlsinfo := transport.TLSInfo{}
+	if p.SSL.CertFile != "" {
+		tlsinfo.CertFile = p.SSL.CertFile
+		cfgtls = &tlsinfo
+	}
+
+	if p.SSL.KeyFile != "" {
+		tlsinfo.KeyFile = p.SSL.KeyFile
+		cfgtls = &tlsinfo
+	}
+
+	if p.SSL.CAFile != "" {
+		tlsinfo.CAFile = p.SSL.CAFile
+		cfgtls = &tlsinfo
+	}
+
+	if p.SSL.ServerName != "" {
+		tlsinfo.ServerName = p.SSL.ServerName
+		cfgtls = &tlsinfo
+	}
+
+	if cfgtls != nil {
+		clientTLS, err := cfgtls.ClientConfig()
+		if err != nil {
+			return nil, err
+		}
+		cfg.TLS = clientTLS
+	}
+
+	db, err := clientv3.New(cfg)
+	if err != nil {
+		return nil, err
+	}
+	if len(p.Namespace) > 0 {
+		db.KV = namespace.NewKV(db.KV, p.Namespace)
+	}
+	c := &conn{
+		db:     db,
+		logger: logger,
+	}
+	return c, nil
+}
@@ -0,0 +1,94 @@
+package etcd
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/coreos/dex/storage"
+	"github.com/coreos/dex/storage/conformance"
+	"github.com/coreos/etcd/clientv3"
+	"github.com/sirupsen/logrus"
+)
+
+func withTimeout(t time.Duration, f func()) {
+	c := make(chan struct{})
+	defer close(c)
+
+	go func() {
+		select {
+		case <-c:
+		case <-time.After(t):
+			// Dump a stack trace of the program. Useful for debugging deadlocks.
+			buf := make([]byte, 2<<20)
+			fmt.Fprintf(os.Stderr, "%s\n", buf[:runtime.Stack(buf, true)])
+			panic("test took too long")
+		}
+	}()
+
+	f()
+}
+
+func cleanDB(c *conn) error {
+	ctx := context.TODO()
+	for _, prefix := range []string{
+		clientPrefix,
+		authCodePrefix,
+		refreshTokenPrefix,
+		authRequestPrefix,
+		passwordPrefix,
+		offlineSessionPrefix,
+		connectorPrefix,
+	} {
+		_, err := c.db.Delete(ctx, prefix, clientv3.WithPrefix())
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+var logger = &logrus.Logger{
+	Out:       os.Stderr,
+	Formatter: &logrus.TextFormatter{DisableColors: true},
+	Level:     logrus.DebugLevel,
+}
+
+func TestEtcd(t *testing.T) {
+	testEtcdEnv := "DEX_ETCD_ENDPOINTS"
+	endpointsStr := os.Getenv(testEtcdEnv)
+	if endpointsStr == "" {
+		t.Skipf("test environment variable %q not set, skipping", testEtcdEnv)
+		return
+	}
+	endpoints := strings.Split(endpointsStr, ",")
+
+	newStorage := func() storage.Storage {
+		s := &Etcd{
+			Endpoints: endpoints,
+		}
+		conn, err := s.open(logger)
+		if err != nil {
+			fmt.Fprintln(os.Stdout, err)
+			t.Fatal(err)
+		}
+
+		if err := cleanDB(conn); err != nil {
+			fmt.Fprintln(os.Stdout, err)
+			t.Fatal(err)
+		}
+		return conn
+	}
+
+	withTimeout(time.Second*10, func() {
+		conformance.RunTests(t, newStorage)
+	})
+
+	withTimeout(time.Minute*1, func() {
+		conformance.RunTransactionTests(t, newStorage)
+	})
+}
@@ -0,0 +1,109 @@
+#!/bin/bash
+
+if [ "$EUID" -ne 0 ]
+  then echo "Please run as root"
+  exit
+fi
+
+function usage {
+  cat << EOF >> /dev/stderr
+Usage: sudo ./standup.sh [create|destroy] [etcd]
+
+This is a script for standing up test databases. It uses systemd to daemonize
+rkt containers running on a local loopback IP.
+
+The general workflow is to create a daemonized container, use the output to set
+the test environment variables, run the tests, then destroy the container.
+
+	sudo ./standup.sh create etcd
+	# Copy environment variables and run tests.
+	go test -v -i # always install test dependencies
+	go test -v
+	sudo ./standup.sh destroy etcd
+
+EOF
+  exit 2
+}
+
+function main {
+  if [ "$#" -ne 2 ]; then
+    usage
+    exit 2
+  fi
+
+  case "$1" in
+  "create")
+     case "$2" in
+     "etcd")
+        create_etcd;;
+     *)
+       usage
+       exit 2
+       ;;
+     esac
+     ;;
+  "destroy")
+     case "$2" in
+     "etcd")
+        destroy_etcd;;
+     *)
+       usage
+       exit 2
+       ;;
+     esac
+     ;;
+  *)
+    usage
+    exit 2
+    ;;
+  esac
+}
+
+function wait_for_file {
+  while [ ! -f $1 ]; do
+    sleep 1
+  done
+}
+
+function wait_for_container {
+  while [ -z "$( rkt list --full | grep $1 | grep running )" ]; do
+    sleep 1
+  done
+}
+
+function create_etcd {
+  UUID_FILE=/tmp/dex-etcd-uuid
+  if [ -f $UUID_FILE ]; then
+    echo "etcd database already exists, try ./standup.sh destroy etcd"
+    exit 2
+  fi
+
+  echo "Starting etcd . To view progress run:"
+  echo ""
+  echo "  journalctl -fu dex-etcd"
+  echo ""
+  UNIFIED_CGROUP_HIERARCHY=no \
+  systemd-run --unit=dex-etcd \
+      rkt run --uuid-file-save=$UUID_FILE --insecure-options=image \
+	  --net=host \
+	  docker://quay.io/coreos/etcd:v3.2.9
+
+  wait_for_file $UUID_FILE
+
+  UUID=$( cat $UUID_FILE )
+  wait_for_container $UUID
+  echo "To run tests export the following environment variables:"
+  echo ""
+  echo "  export DEX_ETCD_ENDPOINTS=http://localhost:2379"
+  echo ""
+}
+
+function destroy_etcd {
+  UUID_FILE=/tmp/dex-etcd-uuid
+  systemctl stop dex-etcd
+  rkt rm --uuid-file=$UUID_FILE
+  rm $UUID_FILE
+}
+
+
+main $@
@@ -0,0 +1,229 @@
+package etcd
+
+import (
+	"time"
+
+	"github.com/coreos/dex/storage"
+	jose "gopkg.in/square/go-jose.v2"
+)
+
+// AuthCode is a mirrored struct from storage with JSON struct tags
+type AuthCode struct {
+	ID          string   `json:"ID"`
+	ClientID    string   `json:"clientID"`
+	RedirectURI string   `json:"redirectURI"`
+	Nonce       string   `json:"nonce,omitempty"`
+	Scopes      []string `json:"scopes,omitempty"`
+
+	ConnectorID   string `json:"connectorID,omitempty"`
+	ConnectorData []byte `json:"connectorData,omitempty"`
+	Claims        Claims `json:"claims,omitempty"`
+
+	Expiry time.Time `json:"expiry"`
+}
+
+func fromStorageAuthCode(a storage.AuthCode) AuthCode {
+	return AuthCode{
+		ID:            a.ID,
+		ClientID:      a.ClientID,
+		RedirectURI:   a.RedirectURI,
+		ConnectorID:   a.ConnectorID,
+		ConnectorData: a.ConnectorData,
+		Nonce:         a.Nonce,
+		Scopes:        a.Scopes,
+		Claims:        fromStorageClaims(a.Claims),
+		Expiry:        a.Expiry,
+	}
+}
+
+// AuthRequest is a mirrored struct from storage with JSON struct tags
+type AuthRequest struct {
+	ID       string `json:"id"`
+	ClientID string `json:"client_id"`
+
+	ResponseTypes []string `json:"response_types"`
+	Scopes        []string `json:"scopes"`
+	RedirectURI   string   `json:"redirect_uri"`
+	Nonce         string   `json:"nonce"`
+	State         string   `json:"state"`
+
+	ForceApprovalPrompt bool `json:"force_approval_prompt"`
+
+	Expiry time.Time `json:"expiry"`
+
+	LoggedIn bool `json:"logged_in"`
+
+	Claims Claims `json:"claims"`
+
+	ConnectorID   string `json:"connector_id"`
+	ConnectorData []byte `json:"connector_data"`
+}
+
+func fromStorageAuthRequest(a storage.AuthRequest) AuthRequest {
+	return AuthRequest{
+		ID:                  a.ID,
+		ClientID:            a.ClientID,
+		ResponseTypes:       a.ResponseTypes,
+		Scopes:              a.Scopes,
+		RedirectURI:         a.RedirectURI,
+		Nonce:               a.Nonce,
+		State:               a.State,
+		ForceApprovalPrompt: a.ForceApprovalPrompt,
+		Expiry:              a.Expiry,
+		LoggedIn:            a.LoggedIn,
+		Claims:              fromStorageClaims(a.Claims),
+		ConnectorID:         a.ConnectorID,
+		ConnectorData:       a.ConnectorData,
+	}
+}
+
+func toStorageAuthRequest(a AuthRequest) storage.AuthRequest {
+	return storage.AuthRequest{
+		ID:                  a.ID,
+		ClientID:            a.ClientID,
+		ResponseTypes:       a.ResponseTypes,
+		Scopes:              a.Scopes,
+		RedirectURI:         a.RedirectURI,
+		Nonce:               a.Nonce,
+		State:               a.State,
+		ForceApprovalPrompt: a.ForceApprovalPrompt,
+		LoggedIn:            a.LoggedIn,
+		ConnectorID:         a.ConnectorID,
+		ConnectorData:       a.ConnectorData,
+		Expiry:              a.Expiry,
+		Claims:              toStorageClaims(a.Claims),
+	}
+}
+
+// RefreshToken is a mirrored struct from storage with JSON struct tags
+type RefreshToken struct {
+	ID string `json:"id"`
+
+	Token string `json:"token"`
+
+	CreatedAt time.Time `json:"created_at"`
+	LastUsed  time.Time `json:"last_used"`
+
+	ClientID string `json:"client_id"`
+
+	ConnectorID   string `json:"connector_id"`
+	ConnectorData []byte `json:"connector_data"`
+	Claims        Claims `json:"claims"`
+
+	Scopes []string `json:"scopes"`
+
+	Nonce string `json:"nonce"`
+}
+
+func toStorageRefreshToken(r RefreshToken) storage.RefreshToken {
+	return storage.RefreshToken{
+		ID:            r.ID,
+		Token:         r.Token,
+		CreatedAt:     r.CreatedAt,
+		LastUsed:      r.LastUsed,
+		ClientID:      r.ClientID,
+		ConnectorID:   r.ConnectorID,
+		ConnectorData: r.ConnectorData,
+		Scopes:        r.Scopes,
+		Nonce:         r.Nonce,
+		Claims:        toStorageClaims(r.Claims),
+	}
+}
+
+func fromStorageRefreshToken(r storage.RefreshToken) RefreshToken {
+	return RefreshToken{
+		ID:            r.ID,
+		Token:         r.Token,
+		CreatedAt:     r.CreatedAt,
+		LastUsed:      r.LastUsed,
+		ClientID:      r.ClientID,
+		ConnectorID:   r.ConnectorID,
+		ConnectorData: r.ConnectorData,
+		Scopes:        r.Scopes,
+		Nonce:         r.Nonce,
+		Claims:        fromStorageClaims(r.Claims),
+	}
+}
+
+// Claims is a mirrored struct from storage with JSON struct tags.
+type Claims struct {
+	UserID        string   `json:"userID"`
+	Username      string   `json:"username"`
+	Email         string   `json:"email"`
+	EmailVerified bool     `json:"emailVerified"`
+	Groups        []string `json:"groups,omitempty"`
+}
+
+func fromStorageClaims(i storage.Claims) Claims {
+	return Claims{
+		UserID:        i.UserID,
+		Username:      i.Username,
+		Email:         i.Email,
+		EmailVerified: i.EmailVerified,
+		Groups:        i.Groups,
+	}
+}
+
+func toStorageClaims(i Claims) storage.Claims {
+	return storage.Claims{
+		UserID:        i.UserID,
+		Username:      i.Username,
+		Email:         i.Email,
+		EmailVerified: i.EmailVerified,
+		Groups:        i.Groups,
+	}
+}
+
+// Keys is a mirrored struct from storage with JSON struct tags
+type Keys struct {
+	SigningKey       *jose.JSONWebKey          `json:"signing_key,omitempty"`
+	SigningKeyPub    *jose.JSONWebKey          `json:"signing_key_pub,omitempty"`
+	VerificationKeys []storage.VerificationKey `json:"verification_keys"`
+	NextRotation     time.Time                 `json:"next_rotation"`
+}
+
+func fromStorageKeys(keys storage.Keys) Keys {
+	return Keys{
+		SigningKey:       keys.SigningKey,
+		SigningKeyPub:    keys.SigningKeyPub,
+		VerificationKeys: keys.VerificationKeys,
+		NextRotation:     keys.NextRotation,
+	}
+}
+
+func toStorageKeys(keys Keys) storage.Keys {
+	return storage.Keys{
+		SigningKey:       keys.SigningKey,
+		SigningKeyPub:    keys.SigningKeyPub,
+		VerificationKeys: keys.VerificationKeys,
+		NextRotation:     keys.NextRotation,
+	}
+}
+
+// OfflineSessions is a mirrored struct from storage with JSON struct tags
+type OfflineSessions struct {
+	UserID  string                              `json:"user_id,omitempty"`
+	ConnID  string                              `json:"conn_id,omitempty"`
+	Refresh map[string]*storage.RefreshTokenRef `json:"refresh,omitempty"`
+}
+
+func fromStorageOfflineSessions(o storage.OfflineSessions) OfflineSessions {
+	return OfflineSessions{
+		UserID:  o.UserID,
+		ConnID:  o.ConnID,
+		Refresh: o.Refresh,
+	}
+}
+
+func toStorageOfflineSessions(o OfflineSessions) storage.OfflineSessions {
+	s := storage.OfflineSessions{
+		UserID:  o.UserID,
+		ConnID:  o.ConnID,
+		Refresh: o.Refresh,
+	}
+	if s.Refresh == nil {
+		// Server code assumes this will be non-nil.
+		s.Refresh = make(map[string]*storage.RefreshTokenRef)
+	}
+	return s
+}
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
@@ -0,0 +1,5 @@
+CoreOS Project
+Copyright 2014 CoreOS, Inc
+
+This product includes software developed at CoreOS, Inc.
+(http://www.coreos.com/).
@@ -0,0 +1,222 @@
+// Copyright 2016 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package clientv3
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/coreos/etcd/auth/authpb"
+	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
+	"golang.org/x/net/context"
+	"google.golang.org/grpc"
+)
+
+type (
+	AuthEnableResponse               pb.AuthEnableResponse
+	AuthDisableResponse              pb.AuthDisableResponse
+	AuthenticateResponse             pb.AuthenticateResponse
+	AuthUserAddResponse              pb.AuthUserAddResponse
+	AuthUserDeleteResponse           pb.AuthUserDeleteResponse
+	AuthUserChangePasswordResponse   pb.AuthUserChangePasswordResponse
+	AuthUserGrantRoleResponse        pb.AuthUserGrantRoleResponse
+	AuthUserGetResponse              pb.AuthUserGetResponse
+	AuthUserRevokeRoleResponse       pb.AuthUserRevokeRoleResponse
+	AuthRoleAddResponse              pb.AuthRoleAddResponse
+	AuthRoleGrantPermissionResponse  pb.AuthRoleGrantPermissionResponse
+	AuthRoleGetResponse              pb.AuthRoleGetResponse
+	AuthRoleRevokePermissionResponse pb.AuthRoleRevokePermissionResponse
+	AuthRoleDeleteResponse           pb.AuthRoleDeleteResponse
+	AuthUserListResponse             pb.AuthUserListResponse
+	AuthRoleListResponse             pb.AuthRoleListResponse
+
+	PermissionType authpb.Permission_Type
+	Permission     authpb.Permission
+)
+
+const (
+	PermRead      = authpb.READ
+	PermWrite     = authpb.WRITE
+	PermReadWrite = authpb.READWRITE
+)
+
+type Auth interface {
+	// AuthEnable enables auth of an etcd cluster.
+	AuthEnable(ctx context.Context) (*AuthEnableResponse, error)
+
+	// AuthDisable disables auth of an etcd cluster.
+	AuthDisable(ctx context.Context) (*AuthDisableResponse, error)
+
+	// UserAdd adds a new user to an etcd cluster.
+	UserAdd(ctx context.Context, name string, password string) (*AuthUserAddResponse, error)
+
+	// UserDelete deletes a user from an etcd cluster.
+	UserDelete(ctx context.Context, name string) (*AuthUserDeleteResponse, error)
+
+	// UserChangePassword changes a password of a user.
+	UserChangePassword(ctx context.Context, name string, password string) (*AuthUserChangePasswordResponse, error)
+
+	// UserGrantRole grants a role to a user.
+	UserGrantRole(ctx context.Context, user string, role string) (*AuthUserGrantRoleResponse, error)
+
+	// UserGet gets a detailed information of a user.
+	UserGet(ctx context.Context, name string) (*AuthUserGetResponse, error)
+
+	// UserList gets a list of all users.
+	UserList(ctx context.Context) (*AuthUserListResponse, error)
+
+	// UserRevokeRole revokes a role of a user.
+	UserRevokeRole(ctx context.Context, name string, role string) (*AuthUserRevokeRoleResponse, error)
+
+	// RoleAdd adds a new role to an etcd cluster.
+	RoleAdd(ctx context.Context, name string) (*AuthRoleAddResponse, error)
+
+	// RoleGrantPermission grants a permission to a role.
+	RoleGrantPermission(ctx context.Context, name string, key, rangeEnd string, permType PermissionType) (*AuthRoleGrantPermissionResponse, error)
+
+	// RoleGet gets a detailed information of a role.
+	RoleGet(ctx context.Context, role string) (*AuthRoleGetResponse, error)
+
+	// RoleList gets a list of all roles.
+	RoleList(ctx context.Context) (*AuthRoleListResponse, error)
+
+	// RoleRevokePermission revokes a permission from a role.
+	RoleRevokePermission(ctx context.Context, role string, key, rangeEnd string) (*AuthRoleRevokePermissionResponse, error)
+
+	// RoleDelete deletes a role.
+	RoleDelete(ctx context.Context, role string) (*AuthRoleDeleteResponse, error)
+}
+
+type auth struct {
+	remote pb.AuthClient
+}
+
+func NewAuth(c *Client) Auth {
+	return &auth{remote: pb.NewAuthClient(c.ActiveConnection())}
+}
+
+func (auth *auth) AuthEnable(ctx context.Context) (*AuthEnableResponse, error) {
+	resp, err := auth.remote.AuthEnable(ctx, &pb.AuthEnableRequest{}, grpc.FailFast(false))
+	return (*AuthEnableResponse)(resp), toErr(ctx, err)
+}
+
+func (auth *auth) AuthDisable(ctx context.Context) (*AuthDisableResponse, error) {
+	resp, err := auth.remote.AuthDisable(ctx, &pb.AuthDisableRequest{}, grpc.FailFast(false))
+	return (*AuthDisableResponse)(resp), toErr(ctx, err)
+}
+
+func (auth *auth) UserAdd(ctx context.Context, name string, password string) (*AuthUserAddResponse, error) {
+	resp, err := auth.remote.UserAdd(ctx, &pb.AuthUserAddRequest{Name: name, Password: password})
+	return (*AuthUserAddResponse)(resp), toErr(ctx, err)
+}
+
+func (auth *auth) UserDelete(ctx context.Context, name string) (*AuthUserDeleteResponse, error) {
+	resp, err := auth.remote.UserDelete(ctx, &pb.AuthUserDeleteRequest{Name: name})
+	return (*AuthUserDeleteResponse)(resp), toErr(ctx, err)
+}
+
+func (auth *auth) UserChangePassword(ctx context.Context, name string, password string) (*AuthUserChangePasswordResponse, error) {
+	resp, err := auth.remote.UserChangePassword(ctx, &pb.AuthUserChangePasswordRequest{Name: name, Password: password})
+	return (*AuthUserChangePasswordResponse)(resp), toErr(ctx, err)
+}
+
+func (auth *auth) UserGrantRole(ctx context.Context, user string, role string) (*AuthUserGrantRoleResponse, error) {
+	resp, err := auth.remote.UserGrantRole(ctx, &pb.AuthUserGrantRoleRequest{User: user, Role: role})
+	return (*AuthUserGrantRoleResponse)(resp), toErr(ctx, err)
+}
+
+func (auth *auth) UserGet(ctx context.Context, name string) (*AuthUserGetResponse, error) {
+	resp, err := auth.remote.UserGet(ctx, &pb.AuthUserGetRequest{Name: name}, grpc.FailFast(false))
+	return (*AuthUserGetResponse)(resp), toErr(ctx, err)
+}
+
+func (auth *auth) UserList(ctx context.Context) (*AuthUserListResponse, error) {
+	resp, err := auth.remote.UserList(ctx, &pb.AuthUserListRequest{}, grpc.FailFast(false))
+	return (*AuthUserListResponse)(resp), toErr(ctx, err)
+}
+
+func (auth *auth) UserRevokeRole(ctx context.Context, name string, role string) (*AuthUserRevokeRoleResponse, error) {
+	resp, err := auth.remote.UserRevokeRole(ctx, &pb.AuthUserRevokeRoleRequest{Name: name, Role: role})
+	return (*AuthUserRevokeRoleResponse)(resp), toErr(ctx, err)
+}
+
+func (auth *auth) RoleAdd(ctx context.Context, name string) (*AuthRoleAddResponse, error) {
+	resp, err := auth.remote.RoleAdd(ctx, &pb.AuthRoleAddRequest{Name: name})
+	return (*AuthRoleAddResponse)(resp), toErr(ctx, err)
+}
+
+func (auth *auth) RoleGrantPermission(ctx context.Context, name string, key, rangeEnd string, permType PermissionType) (*AuthRoleGrantPermissionResponse, error) {
+	perm := &authpb.Permission{
+		Key:      []byte(key),
+		RangeEnd: []byte(rangeEnd),
+		PermType: authpb.Permission_Type(permType),
+	}
+	resp, err := auth.remote.RoleGrantPermission(ctx, &pb.AuthRoleGrantPermissionRequest{Name: name, Perm: perm})
+	return (*AuthRoleGrantPermissionResponse)(resp), toErr(ctx, err)
+}
+
+func (auth *auth) RoleGet(ctx context.Context, role string) (*AuthRoleGetResponse, error) {
+	resp, err := auth.remote.RoleGet(ctx, &pb.AuthRoleGetRequest{Role: role}, grpc.FailFast(false))
+	return (*AuthRoleGetResponse)(resp), toErr(ctx, err)
+}
+
+func (auth *auth) RoleList(ctx context.Context) (*AuthRoleListResponse, error) {
+	resp, err := auth.remote.RoleList(ctx, &pb.AuthRoleListRequest{}, grpc.FailFast(false))
+	return (*AuthRoleListResponse)(resp), toErr(ctx, err)
+}
+
+func (auth *auth) RoleRevokePermission(ctx context.Context, role string, key, rangeEnd string) (*AuthRoleRevokePermissionResponse, error) {
+	resp, err := auth.remote.RoleRevokePermission(ctx, &pb.AuthRoleRevokePermissionRequest{Role: role, Key: key, RangeEnd: rangeEnd})
+	return (*AuthRoleRevokePermissionResponse)(resp), toErr(ctx, err)
+}
+
+func (auth *auth) RoleDelete(ctx context.Context, role string) (*AuthRoleDeleteResponse, error) {
+	resp, err := auth.remote.RoleDelete(ctx, &pb.AuthRoleDeleteRequest{Role: role})
+	return (*AuthRoleDeleteResponse)(resp), toErr(ctx, err)
+}
+
+func StrToPermissionType(s string) (PermissionType, error) {
+	val, ok := authpb.Permission_Type_value[strings.ToUpper(s)]
+	if ok {
+		return PermissionType(val), nil
+	}
+	return PermissionType(-1), fmt.Errorf("invalid permission type: %s", s)
+}
+
+type authenticator struct {
+	conn   *grpc.ClientConn // conn in-use
+	remote pb.AuthClient
+}
+
+func (auth *authenticator) authenticate(ctx context.Context, name string, password string) (*AuthenticateResponse, error) {
+	resp, err := auth.remote.Authenticate(ctx, &pb.AuthenticateRequest{Name: name, Password: password}, grpc.FailFast(false))
+	return (*AuthenticateResponse)(resp), toErr(ctx, err)
+}
+
+func (auth *authenticator) close() {
+	auth.conn.Close()
+}
+
+func newAuthenticator(endpoint string, opts []grpc.DialOption) (*authenticator, error) {
+	conn, err := grpc.Dial(endpoint, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return &authenticator{
+		conn:   conn,
+		remote: pb.NewAuthClient(conn),
+	}, nil
+}
@@ -0,0 +1,356 @@
+// Copyright 2016 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package clientv3
+
+import (
+	"net/url"
+	"strings"
+	"sync"
+
+	"golang.org/x/net/context"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+)
+
+// ErrNoAddrAvilable is returned by Get() when the balancer does not have
+// any active connection to endpoints at the time.
+// This error is returned only when opts.BlockingWait is true.
+var ErrNoAddrAvilable = grpc.Errorf(codes.Unavailable, "there is no address available")
+
+// simpleBalancer does the bare minimum to expose multiple eps
+// to the grpc reconnection code path
+type simpleBalancer struct {
+	// addrs are the client's endpoints for grpc
+	addrs []grpc.Address
+	// notifyCh notifies grpc of the set of addresses for connecting
+	notifyCh chan []grpc.Address
+
+	// readyc closes once the first connection is up
+	readyc    chan struct{}
+	readyOnce sync.Once
+
+	// mu protects upEps, pinAddr, and connectingAddr
+	mu sync.RWMutex
+
+	// upc closes when upEps transitions from empty to non-zero or the balancer closes.
+	upc chan struct{}
+
+	// downc closes when grpc calls down() on pinAddr
+	downc chan struct{}
+
+	// stopc is closed to signal updateNotifyLoop should stop.
+	stopc chan struct{}
+
+	// donec closes when all goroutines are exited
+	donec chan struct{}
+
+	// updateAddrsC notifies updateNotifyLoop to update addrs.
+	updateAddrsC chan struct{}
+
+	// grpc issues TLS cert checks using the string passed into dial so
+	// that string must be the host. To recover the full scheme://host URL,
+	// have a map from hosts to the original endpoint.
+	host2ep map[string]string
+
+	// pinAddr is the currently pinned address; set to the empty string on
+	// intialization and shutdown.
+	pinAddr string
+
+	closed bool
+}
+
+func newSimpleBalancer(eps []string) *simpleBalancer {
+	notifyCh := make(chan []grpc.Address, 1)
+	addrs := make([]grpc.Address, len(eps))
+	for i := range eps {
+		addrs[i].Addr = getHost(eps[i])
+	}
+	sb := &simpleBalancer{
+		addrs:        addrs,
+		notifyCh:     notifyCh,
+		readyc:       make(chan struct{}),
+		upc:          make(chan struct{}),
+		stopc:        make(chan struct{}),
+		downc:        make(chan struct{}),
+		donec:        make(chan struct{}),
+		updateAddrsC: make(chan struct{}, 1),
+		host2ep:      getHost2ep(eps),
+	}
+	close(sb.downc)
+	go sb.updateNotifyLoop()
+	return sb
+}
+
+func (b *simpleBalancer) Start(target string, config grpc.BalancerConfig) error { return nil }
+
+func (b *simpleBalancer) ConnectNotify() <-chan struct{} {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.upc
+}
+
+func (b *simpleBalancer) getEndpoint(host string) string {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.host2ep[host]
+}
+
+func getHost2ep(eps []string) map[string]string {
+	hm := make(map[string]string, len(eps))
+	for i := range eps {
+		_, host, _ := parseEndpoint(eps[i])
+		hm[host] = eps[i]
+	}
+	return hm
+}
+
+func (b *simpleBalancer) updateAddrs(eps []string) {
+	np := getHost2ep(eps)
+
+	b.mu.Lock()
+
+	match := len(np) == len(b.host2ep)
+	for k, v := range np {
+		if b.host2ep[k] != v {
+			match = false
+			break
+		}
+	}
+	if match {
+		// same endpoints, so no need to update address
+		b.mu.Unlock()
+		return
+	}
+
+	b.host2ep = np
+
+	addrs := make([]grpc.Address, 0, len(eps))
+	for i := range eps {
+		addrs = append(addrs, grpc.Address{Addr: getHost(eps[i])})
+	}
+	b.addrs = addrs
+
+	// updating notifyCh can trigger new connections,
+	// only update addrs if all connections are down
+	// or addrs does not include pinAddr.
+	update := !hasAddr(addrs, b.pinAddr)
+	b.mu.Unlock()
+
+	if update {
+		select {
+		case b.updateAddrsC <- struct{}{}:
+		case <-b.stopc:
+		}
+	}
+}
+
+func hasAddr(addrs []grpc.Address, targetAddr string) bool {
+	for _, addr := range addrs {
+		if targetAddr == addr.Addr {
+			return true
+		}
+	}
+	return false
+}
+
+func (b *simpleBalancer) updateNotifyLoop() {
+	defer close(b.donec)
+
+	for {
+		b.mu.RLock()
+		upc, downc, addr := b.upc, b.downc, b.pinAddr
+		b.mu.RUnlock()
+		// downc or upc should be closed
+		select {
+		case <-downc:
+			downc = nil
+		default:
+		}
+		select {
+		case <-upc:
+			upc = nil
+		default:
+		}
+		switch {
+		case downc == nil && upc == nil:
+			// stale
+			select {
+			case <-b.stopc:
+				return
+			default:
+			}
+		case downc == nil:
+			b.notifyAddrs()
+			select {
+			case <-upc:
+			case <-b.updateAddrsC:
+				b.notifyAddrs()
+			case <-b.stopc:
+				return
+			}
+		case upc == nil:
+			select {
+			// close connections that are not the pinned address
+			case b.notifyCh <- []grpc.Address{{Addr: addr}}:
+			case <-downc:
+			case <-b.stopc:
+				return
+			}
+			select {
+			case <-downc:
+			case <-b.updateAddrsC:
+			case <-b.stopc:
+				return
+			}
+			b.notifyAddrs()
+		}
+	}
+}
+
+func (b *simpleBalancer) notifyAddrs() {
+	b.mu.RLock()
+	addrs := b.addrs
+	b.mu.RUnlock()
+	select {
+	case b.notifyCh <- addrs:
+	case <-b.stopc:
+	}
+}
+
+func (b *simpleBalancer) Up(addr grpc.Address) func(error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	// gRPC might call Up after it called Close. We add this check
+	// to "fix" it up at application layer. Or our simplerBalancer
+	// might panic since b.upc is closed.
+	if b.closed {
+		return func(err error) {}
+	}
+	// gRPC might call Up on a stale address.
+	// Prevent updating pinAddr with a stale address.
+	if !hasAddr(b.addrs, addr.Addr) {
+		return func(err error) {}
+	}
+	if b.pinAddr != "" {
+		return func(err error) {}
+	}
+	// notify waiting Get()s and pin first connected address
+	close(b.upc)
+	b.downc = make(chan struct{})
+	b.pinAddr = addr.Addr
+	// notify client that a connection is up
+	b.readyOnce.Do(func() { close(b.readyc) })
+	return func(err error) {
+		b.mu.Lock()
+		b.upc = make(chan struct{})
+		close(b.downc)
+		b.pinAddr = ""
+		b.mu.Unlock()
+	}
+}
+
+func (b *simpleBalancer) Get(ctx context.Context, opts grpc.BalancerGetOptions) (grpc.Address, func(), error) {
+	var (
+		addr   string
+		closed bool
+	)
+
+	// If opts.BlockingWait is false (for fail-fast RPCs), it should return
+	// an address it has notified via Notify immediately instead of blocking.
+	if !opts.BlockingWait {
+		b.mu.RLock()
+		closed = b.closed
+		addr = b.pinAddr
+		b.mu.RUnlock()
+		if closed {
+			return grpc.Address{Addr: ""}, nil, grpc.ErrClientConnClosing
+		}
+		if addr == "" {
+			return grpc.Address{Addr: ""}, nil, ErrNoAddrAvilable
+		}
+		return grpc.Address{Addr: addr}, func() {}, nil
+	}
+
+	for {
+		b.mu.RLock()
+		ch := b.upc
+		b.mu.RUnlock()
+		select {
+		case <-ch:
+		case <-b.donec:
+			return grpc.Address{Addr: ""}, nil, grpc.ErrClientConnClosing
+		case <-ctx.Done():
+			return grpc.Address{Addr: ""}, nil, ctx.Err()
+		}
+		b.mu.RLock()
+		closed = b.closed
+		addr = b.pinAddr
+		b.mu.RUnlock()
+		// Close() which sets b.closed = true can be called before Get(), Get() must exit if balancer is closed.
+		if closed {
+			return grpc.Address{Addr: ""}, nil, grpc.ErrClientConnClosing
+		}
+		if addr != "" {
+			break
+		}
+	}
+	return grpc.Address{Addr: addr}, func() {}, nil
+}
+
+func (b *simpleBalancer) Notify() <-chan []grpc.Address { return b.notifyCh }
+
+func (b *simpleBalancer) Close() error {
+	b.mu.Lock()
+	// In case gRPC calls close twice. TODO: remove the checking
+	// when we are sure that gRPC wont call close twice.
+	if b.closed {
+		b.mu.Unlock()
+		<-b.donec
+		return nil
+	}
+	b.closed = true
+	close(b.stopc)
+	b.pinAddr = ""
+
+	// In the case of following scenario:
+	//	1. upc is not closed; no pinned address
+	// 	2. client issues an rpc, calling invoke(), which calls Get(), enters for loop, blocks
+	// 	3. clientconn.Close() calls balancer.Close(); closed = true
+	// 	4. for loop in Get() never exits since ctx is the context passed in by the client and may not be canceled
+	// we must close upc so Get() exits from blocking on upc
+	select {
+	case <-b.upc:
+	default:
+		// terminate all waiting Get()s
+		close(b.upc)
+	}
+
+	b.mu.Unlock()
+
+	// wait for updateNotifyLoop to finish
+	<-b.donec
+	close(b.notifyCh)
+
+	return nil
+}
+
+func getHost(ep string) string {
+	url, uerr := url.Parse(ep)
+	if uerr != nil || !strings.Contains(ep, "://") {
+		return ep
+	}
+	return url.Host
+}
@@ -0,0 +1,100 @@
+// Copyright 2016 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package clientv3
+
+import (
+	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
+	"golang.org/x/net/context"
+	"google.golang.org/grpc"
+)
+
+type (
+	Member               pb.Member
+	MemberListResponse   pb.MemberListResponse
+	MemberAddResponse    pb.MemberAddResponse
+	MemberRemoveResponse pb.MemberRemoveResponse
+	MemberUpdateResponse pb.MemberUpdateResponse
+)
+
+type Cluster interface {
+	// MemberList lists the current cluster membership.
+	MemberList(ctx context.Context) (*MemberListResponse, error)
+
+	// MemberAdd adds a new member into the cluster.
+	MemberAdd(ctx context.Context, peerAddrs []string) (*MemberAddResponse, error)
+
+	// MemberRemove removes an existing member from the cluster.
+	MemberRemove(ctx context.Context, id uint64) (*MemberRemoveResponse, error)
+
+	// MemberUpdate updates the peer addresses of the member.
+	MemberUpdate(ctx context.Context, id uint64, peerAddrs []string) (*MemberUpdateResponse, error)
+}
+
+type cluster struct {
+	remote pb.ClusterClient
+}
+
+func NewCluster(c *Client) Cluster {
+	return &cluster{remote: RetryClusterClient(c)}
+}
+
+func NewClusterFromClusterClient(remote pb.ClusterClient) Cluster {
+	return &cluster{remote: remote}
+}
+
+func (c *cluster) MemberAdd(ctx context.Context, peerAddrs []string) (*MemberAddResponse, error) {
+	r := &pb.MemberAddRequest{PeerURLs: peerAddrs}
+	resp, err := c.remote.MemberAdd(ctx, r)
+	if err != nil {
+		return nil, toErr(ctx, err)
+	}
+	return (*MemberAddResponse)(resp), nil
+}
+
+func (c *cluster) MemberRemove(ctx context.Context, id uint64) (*MemberRemoveResponse, error) {
+	r := &pb.MemberRemoveRequest{ID: id}
+	resp, err := c.remote.MemberRemove(ctx, r)
+	if err != nil {
+		return nil, toErr(ctx, err)
+	}
+	return (*MemberRemoveResponse)(resp), nil
+}
+
+func (c *cluster) MemberUpdate(ctx context.Context, id uint64, peerAddrs []string) (*MemberUpdateResponse, error) {
+	// it is safe to retry on update.
+	for {
+		r := &pb.MemberUpdateRequest{ID: id, PeerURLs: peerAddrs}
+		resp, err := c.remote.MemberUpdate(ctx, r, grpc.FailFast(false))
+		if err == nil {
+			return (*MemberUpdateResponse)(resp), nil
+		}
+		if isHaltErr(ctx, err) {
+			return nil, toErr(ctx, err)
+		}
+	}
+}
+
+func (c *cluster) MemberList(ctx context.Context) (*MemberListResponse, error) {
+	// it is safe to retry on list.
+	for {
+		resp, err := c.remote.MemberList(ctx, &pb.MemberListRequest{}, grpc.FailFast(false))
+		if err == nil {
+			return (*MemberListResponse)(resp), nil
+		}
+		if isHaltErr(ctx, err) {
+			return nil, toErr(ctx, err)
+		}
+	}
+}
@@ -0,0 +1,53 @@
+// Copyright 2016 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package clientv3
+
+import (
+	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
+)
+
+// CompactOp represents a compact operation.
+type CompactOp struct {
+	revision int64
+	physical bool
+}
+
+// CompactOption configures compact operation.
+type CompactOption func(*CompactOp)
+
+func (op *CompactOp) applyCompactOpts(opts []CompactOption) {
+	for _, opt := range opts {
+		opt(op)
+	}
+}
+
+// OpCompact wraps slice CompactOption to create a CompactOp.
+func OpCompact(rev int64, opts ...CompactOption) CompactOp {
+	ret := CompactOp{revision: rev}
+	ret.applyCompactOpts(opts)
+	return ret
+}
+
+func (op CompactOp) toRequest() *pb.CompactionRequest {
+	return &pb.CompactionRequest{Revision: op.revision, Physical: op.physical}
+}
+
+// WithCompactPhysical makes compact RPC call wait until
+// the compaction is physically applied to the local database
+// such that compacted entries are totally removed from the
+// backend database.
+func WithCompactPhysical() CompactOption {
+	return func(op *CompactOp) { op.physical = true }
+}
@@ -0,0 +1,110 @@
+// Copyright 2016 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package clientv3
+
+import (
+	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
+)
+
+type CompareTarget int
+type CompareResult int
+
+const (
+	CompareVersion CompareTarget = iota
+	CompareCreated
+	CompareModified
+	CompareValue
+)
+
+type Cmp pb.Compare
+
+func Compare(cmp Cmp, result string, v interface{}) Cmp {
+	var r pb.Compare_CompareResult
+
+	switch result {
+	case "=":
+		r = pb.Compare_EQUAL
+	case "!=":
+		r = pb.Compare_NOT_EQUAL
+	case ">":
+		r = pb.Compare_GREATER
+	case "<":
+		r = pb.Compare_LESS
+	default:
+		panic("Unknown result op")
+	}
+
+	cmp.Result = r
+	switch cmp.Target {
+	case pb.Compare_VALUE:
+		val, ok := v.(string)
+		if !ok {
+			panic("bad compare value")
+		}
+		cmp.TargetUnion = &pb.Compare_Value{Value: []byte(val)}
+	case pb.Compare_VERSION:
+		cmp.TargetUnion = &pb.Compare_Version{Version: mustInt64(v)}
+	case pb.Compare_CREATE:
+		cmp.TargetUnion = &pb.Compare_CreateRevision{CreateRevision: mustInt64(v)}
+	case pb.Compare_MOD:
+		cmp.TargetUnion = &pb.Compare_ModRevision{ModRevision: mustInt64(v)}
+	default:
+		panic("Unknown compare type")
+	}
+	return cmp
+}
+
+func Value(key string) Cmp {
+	return Cmp{Key: []byte(key), Target: pb.Compare_VALUE}
+}
+
+func Version(key string) Cmp {
+	return Cmp{Key: []byte(key), Target: pb.Compare_VERSION}
+}
+
+func CreateRevision(key string) Cmp {
+	return Cmp{Key: []byte(key), Target: pb.Compare_CREATE}
+}
+
+func ModRevision(key string) Cmp {
+	return Cmp{Key: []byte(key), Target: pb.Compare_MOD}
+}
+
+// KeyBytes returns the byte slice holding with the comparison key.
+func (cmp *Cmp) KeyBytes() []byte { return cmp.Key }
+
+// WithKeyBytes sets the byte slice for the comparison key.
+func (cmp *Cmp) WithKeyBytes(key []byte) { cmp.Key = key }
+
+// ValueBytes returns the byte slice holding the comparison value, if any.
+func (cmp *Cmp) ValueBytes() []byte {
+	if tu, ok := cmp.TargetUnion.(*pb.Compare_Value); ok {
+		return tu.Value
+	}
+	return nil
+}
+
+// WithValueBytes sets the byte slice for the comparison's value.
+func (cmp *Cmp) WithValueBytes(v []byte) { cmp.TargetUnion.(*pb.Compare_Value).Value = v }
+
+func mustInt64(val interface{}) int64 {
+	if v, ok := val.(int64); ok {
+		return v
+	}
+	if v, ok := val.(int); ok {
+		return int64(v)
+	}
+	panic("bad value")
+}
--- a/Show More
+++ b/Show More