mirror of
https://github.com/netbirdio/dex.git
synced 2026-05-22 18:43:53 -07:00
storage: add etcd storage
This patch adds etcd storage implementation. This should be useful in environments where - we dont want to depends on a separate, hard to maintain SQL cluster - we dont want to incur the overhead of talking to kubernetes apiservers - kubernetes is not available yet, or if kubernetes depends on dex to perform authentication and the operator would like to remove any circular dependency if possible.
This commit is contained in:
Daniel Dao
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
"github.com/coreos/dex/server"
|
||||
"github.com/coreos/dex/storage"
|
||||
"github.com/coreos/dex/storage/etcd"
|
||||
"github.com/coreos/dex/storage/kubernetes"
|
||||
"github.com/coreos/dex/storage/memory"
|
||||
"github.com/coreos/dex/storage/sql"
|
||||
@@ -124,6 +125,7 @@ type StorageConfig interface {
|
||||
}
|
||||
|
||||
var storages = map[string]func() StorageConfig{
|
||||
"etcd": func() StorageConfig { return new(etcd.Etcd) },
|
||||
"kubernetes": func() StorageConfig { return new(kubernetes.Config) },
|
||||
"memory": func() StorageConfig { return new(memory.Config) },
|
||||
"sqlite3": func() StorageConfig { return new(sql.SQLite3) },
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
package etcd
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/coreos/dex/storage"
|
||||
"github.com/coreos/etcd/clientv3"
|
||||
"github.com/coreos/etcd/clientv3/namespace"
|
||||
"github.com/coreos/etcd/pkg/transport"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
var (
|
||||
defaultDialTimeout = 2 * time.Second
|
||||
)
|
||||
|
||||
// SSL represents SSL options for etcd databases.
|
||||
type SSL struct {
|
||||
ServerName string
|
||||
CAFile string
|
||||
KeyFile string
|
||||
CertFile string
|
||||
}
|
||||
|
||||
// Etcd options for connecting to etcd databases.
|
||||
// If you are using a shared etcd cluster for storage, it might be useful to
|
||||
// configure an etcd namespace either via Namespace field or using `etcd grpc-proxy
|
||||
// --namespace=<prefix>`
|
||||
type Etcd struct {
|
||||
Endpoints []string
|
||||
Namespace string
|
||||
Username string
|
||||
Password string
|
||||
SSL SSL
|
||||
}
|
||||
|
||||
// Open creates a new storage implementation backed by Etcd
|
||||
func (p *Etcd) Open(logger logrus.FieldLogger) (storage.Storage, error) {
|
||||
return p.open(logger)
|
||||
}
|
||||
|
||||
func (p *Etcd) open(logger logrus.FieldLogger) (*conn, error) {
|
||||
cfg := clientv3.Config{
|
||||
Endpoints: p.Endpoints,
|
||||
DialTimeout: defaultDialTimeout * time.Second,
|
||||
Username: p.Username,
|
||||
Password: p.Password,
|
||||
}
|
||||
|
||||
var cfgtls *transport.TLSInfo
|
||||
tlsinfo := transport.TLSInfo{}
|
||||
if p.SSL.CertFile != "" {
|
||||
tlsinfo.CertFile = p.SSL.CertFile
|
||||
cfgtls = &tlsinfo
|
||||
}
|
||||
|
||||
if p.SSL.KeyFile != "" {
|
||||
tlsinfo.KeyFile = p.SSL.KeyFile
|
||||
cfgtls = &tlsinfo
|
||||
}
|
||||
|
||||
if p.SSL.CAFile != "" {
|
||||
tlsinfo.CAFile = p.SSL.CAFile
|
||||
cfgtls = &tlsinfo
|
||||
}
|
||||
|
||||
if p.SSL.ServerName != "" {
|
||||
tlsinfo.ServerName = p.SSL.ServerName
|
||||
cfgtls = &tlsinfo
|
||||
}
|
||||
|
||||
if cfgtls != nil {
|
||||
clientTLS, err := cfgtls.ClientConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg.TLS = clientTLS
|
||||
}
|
||||
|
||||
db, err := clientv3.New(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(p.Namespace) > 0 {
|
||||
db.KV = namespace.NewKV(db.KV, p.Namespace)
|
||||
}
|
||||
c := &conn{
|
||||
db: db,
|
||||
logger: logger,
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,94 @@
|
||||
package etcd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/coreos/dex/storage"
|
||||
"github.com/coreos/dex/storage/conformance"
|
||||
"github.com/coreos/etcd/clientv3"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func withTimeout(t time.Duration, f func()) {
|
||||
c := make(chan struct{})
|
||||
defer close(c)
|
||||
|
||||
go func() {
|
||||
select {
|
||||
case <-c:
|
||||
case <-time.After(t):
|
||||
// Dump a stack trace of the program. Useful for debugging deadlocks.
|
||||
buf := make([]byte, 2<<20)
|
||||
fmt.Fprintf(os.Stderr, "%s\n", buf[:runtime.Stack(buf, true)])
|
||||
panic("test took too long")
|
||||
}
|
||||
}()
|
||||
|
||||
f()
|
||||
}
|
||||
|
||||
func cleanDB(c *conn) error {
|
||||
ctx := context.TODO()
|
||||
for _, prefix := range []string{
|
||||
clientPrefix,
|
||||
authCodePrefix,
|
||||
refreshTokenPrefix,
|
||||
authRequestPrefix,
|
||||
passwordPrefix,
|
||||
offlineSessionPrefix,
|
||||
connectorPrefix,
|
||||
} {
|
||||
_, err := c.db.Delete(ctx, prefix, clientv3.WithPrefix())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var logger = &logrus.Logger{
|
||||
Out: os.Stderr,
|
||||
Formatter: &logrus.TextFormatter{DisableColors: true},
|
||||
Level: logrus.DebugLevel,
|
||||
}
|
||||
|
||||
func TestEtcd(t *testing.T) {
|
||||
testEtcdEnv := "DEX_ETCD_ENDPOINTS"
|
||||
endpointsStr := os.Getenv(testEtcdEnv)
|
||||
if endpointsStr == "" {
|
||||
t.Skipf("test environment variable %q not set, skipping", testEtcdEnv)
|
||||
return
|
||||
}
|
||||
endpoints := strings.Split(endpointsStr, ",")
|
||||
|
||||
newStorage := func() storage.Storage {
|
||||
s := &Etcd{
|
||||
Endpoints: endpoints,
|
||||
}
|
||||
conn, err := s.open(logger)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stdout, err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := cleanDB(conn); err != nil {
|
||||
fmt.Fprintln(os.Stdout, err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
return conn
|
||||
}
|
||||
|
||||
withTimeout(time.Second*10, func() {
|
||||
conformance.RunTests(t, newStorage)
|
||||
})
|
||||
|
||||
withTimeout(time.Minute*1, func() {
|
||||
conformance.RunTransactionTests(t, newStorage)
|
||||
})
|
||||
}
|
||||
Executable
+109
@@ -0,0 +1,109 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [ "$EUID" -ne 0 ]
|
||||
then echo "Please run as root"
|
||||
exit
|
||||
fi
|
||||
|
||||
function usage {
|
||||
cat << EOF >> /dev/stderr
|
||||
Usage: sudo ./standup.sh [create|destroy] [etcd]
|
||||
|
||||
This is a script for standing up test databases. It uses systemd to daemonize
|
||||
rkt containers running on a local loopback IP.
|
||||
|
||||
The general workflow is to create a daemonized container, use the output to set
|
||||
the test environment variables, run the tests, then destroy the container.
|
||||
|
||||
sudo ./standup.sh create etcd
|
||||
# Copy environment variables and run tests.
|
||||
go test -v -i # always install test dependencies
|
||||
go test -v
|
||||
sudo ./standup.sh destroy etcd
|
||||
|
||||
EOF
|
||||
exit 2
|
||||
}
|
||||
|
||||
function main {
|
||||
if [ "$#" -ne 2 ]; then
|
||||
usage
|
||||
exit 2
|
||||
fi
|
||||
|
||||
case "$1" in
|
||||
"create")
|
||||
case "$2" in
|
||||
"etcd")
|
||||
create_etcd;;
|
||||
*)
|
||||
usage
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
"destroy")
|
||||
case "$2" in
|
||||
"etcd")
|
||||
destroy_etcd;;
|
||||
*)
|
||||
usage
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
function wait_for_file {
|
||||
while [ ! -f $1 ]; do
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
function wait_for_container {
|
||||
while [ -z "$( rkt list --full | grep $1 | grep running )" ]; do
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
function create_etcd {
|
||||
UUID_FILE=/tmp/dex-etcd-uuid
|
||||
if [ -f $UUID_FILE ]; then
|
||||
echo "etcd database already exists, try ./standup.sh destroy etcd"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
echo "Starting etcd . To view progress run:"
|
||||
echo ""
|
||||
echo " journalctl -fu dex-etcd"
|
||||
echo ""
|
||||
UNIFIED_CGROUP_HIERARCHY=no \
|
||||
systemd-run --unit=dex-etcd \
|
||||
rkt run --uuid-file-save=$UUID_FILE --insecure-options=image \
|
||||
--net=host \
|
||||
docker://quay.io/coreos/etcd:v3.2.9
|
||||
|
||||
wait_for_file $UUID_FILE
|
||||
|
||||
UUID=$( cat $UUID_FILE )
|
||||
wait_for_container $UUID
|
||||
echo "To run tests export the following environment variables:"
|
||||
echo ""
|
||||
echo " export DEX_ETCD_ENDPOINTS=http://localhost:2379"
|
||||
echo ""
|
||||
}
|
||||
|
||||
function destroy_etcd {
|
||||
UUID_FILE=/tmp/dex-etcd-uuid
|
||||
systemctl stop dex-etcd
|
||||
rkt rm --uuid-file=$UUID_FILE
|
||||
rm $UUID_FILE
|
||||
}
|
||||
|
||||
|
||||
main $@
|
||||
@@ -0,0 +1,229 @@
|
||||
package etcd
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/coreos/dex/storage"
|
||||
jose "gopkg.in/square/go-jose.v2"
|
||||
)
|
||||
|
||||
// AuthCode is a mirrored struct from storage with JSON struct tags
|
||||
type AuthCode struct {
|
||||
ID string `json:"ID"`
|
||||
ClientID string `json:"clientID"`
|
||||
RedirectURI string `json:"redirectURI"`
|
||||
Nonce string `json:"nonce,omitempty"`
|
||||
Scopes []string `json:"scopes,omitempty"`
|
||||
|
||||
ConnectorID string `json:"connectorID,omitempty"`
|
||||
ConnectorData []byte `json:"connectorData,omitempty"`
|
||||
Claims Claims `json:"claims,omitempty"`
|
||||
|
||||
Expiry time.Time `json:"expiry"`
|
||||
}
|
||||
|
||||
func fromStorageAuthCode(a storage.AuthCode) AuthCode {
|
||||
return AuthCode{
|
||||
ID: a.ID,
|
||||
ClientID: a.ClientID,
|
||||
RedirectURI: a.RedirectURI,
|
||||
ConnectorID: a.ConnectorID,
|
||||
ConnectorData: a.ConnectorData,
|
||||
Nonce: a.Nonce,
|
||||
Scopes: a.Scopes,
|
||||
Claims: fromStorageClaims(a.Claims),
|
||||
Expiry: a.Expiry,
|
||||
}
|
||||
}
|
||||
|
||||
// AuthRequest is a mirrored struct from storage with JSON struct tags
|
||||
type AuthRequest struct {
|
||||
ID string `json:"id"`
|
||||
ClientID string `json:"client_id"`
|
||||
|
||||
ResponseTypes []string `json:"response_types"`
|
||||
Scopes []string `json:"scopes"`
|
||||
RedirectURI string `json:"redirect_uri"`
|
||||
Nonce string `json:"nonce"`
|
||||
State string `json:"state"`
|
||||
|
||||
ForceApprovalPrompt bool `json:"force_approval_prompt"`
|
||||
|
||||
Expiry time.Time `json:"expiry"`
|
||||
|
||||
LoggedIn bool `json:"logged_in"`
|
||||
|
||||
Claims Claims `json:"claims"`
|
||||
|
||||
ConnectorID string `json:"connector_id"`
|
||||
ConnectorData []byte `json:"connector_data"`
|
||||
}
|
||||
|
||||
func fromStorageAuthRequest(a storage.AuthRequest) AuthRequest {
|
||||
return AuthRequest{
|
||||
ID: a.ID,
|
||||
ClientID: a.ClientID,
|
||||
ResponseTypes: a.ResponseTypes,
|
||||
Scopes: a.Scopes,
|
||||
RedirectURI: a.RedirectURI,
|
||||
Nonce: a.Nonce,
|
||||
State: a.State,
|
||||
ForceApprovalPrompt: a.ForceApprovalPrompt,
|
||||
Expiry: a.Expiry,
|
||||
LoggedIn: a.LoggedIn,
|
||||
Claims: fromStorageClaims(a.Claims),
|
||||
ConnectorID: a.ConnectorID,
|
||||
ConnectorData: a.ConnectorData,
|
||||
}
|
||||
}
|
||||
|
||||
func toStorageAuthRequest(a AuthRequest) storage.AuthRequest {
|
||||
return storage.AuthRequest{
|
||||
ID: a.ID,
|
||||
ClientID: a.ClientID,
|
||||
ResponseTypes: a.ResponseTypes,
|
||||
Scopes: a.Scopes,
|
||||
RedirectURI: a.RedirectURI,
|
||||
Nonce: a.Nonce,
|
||||
State: a.State,
|
||||
ForceApprovalPrompt: a.ForceApprovalPrompt,
|
||||
LoggedIn: a.LoggedIn,
|
||||
ConnectorID: a.ConnectorID,
|
||||
ConnectorData: a.ConnectorData,
|
||||
Expiry: a.Expiry,
|
||||
Claims: toStorageClaims(a.Claims),
|
||||
}
|
||||
}
|
||||
|
||||
// RefreshToken is a mirrored struct from storage with JSON struct tags
|
||||
type RefreshToken struct {
|
||||
ID string `json:"id"`
|
||||
|
||||
Token string `json:"token"`
|
||||
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
LastUsed time.Time `json:"last_used"`
|
||||
|
||||
ClientID string `json:"client_id"`
|
||||
|
||||
ConnectorID string `json:"connector_id"`
|
||||
ConnectorData []byte `json:"connector_data"`
|
||||
Claims Claims `json:"claims"`
|
||||
|
||||
Scopes []string `json:"scopes"`
|
||||
|
||||
Nonce string `json:"nonce"`
|
||||
}
|
||||
|
||||
func toStorageRefreshToken(r RefreshToken) storage.RefreshToken {
|
||||
return storage.RefreshToken{
|
||||
ID: r.ID,
|
||||
Token: r.Token,
|
||||
CreatedAt: r.CreatedAt,
|
||||
LastUsed: r.LastUsed,
|
||||
ClientID: r.ClientID,
|
||||
ConnectorID: r.ConnectorID,
|
||||
ConnectorData: r.ConnectorData,
|
||||
Scopes: r.Scopes,
|
||||
Nonce: r.Nonce,
|
||||
Claims: toStorageClaims(r.Claims),
|
||||
}
|
||||
}
|
||||
|
||||
func fromStorageRefreshToken(r storage.RefreshToken) RefreshToken {
|
||||
return RefreshToken{
|
||||
ID: r.ID,
|
||||
Token: r.Token,
|
||||
CreatedAt: r.CreatedAt,
|
||||
LastUsed: r.LastUsed,
|
||||
ClientID: r.ClientID,
|
||||
ConnectorID: r.ConnectorID,
|
||||
ConnectorData: r.ConnectorData,
|
||||
Scopes: r.Scopes,
|
||||
Nonce: r.Nonce,
|
||||
Claims: fromStorageClaims(r.Claims),
|
||||
}
|
||||
}
|
||||
|
||||
// Claims is a mirrored struct from storage with JSON struct tags.
|
||||
type Claims struct {
|
||||
UserID string `json:"userID"`
|
||||
Username string `json:"username"`
|
||||
Email string `json:"email"`
|
||||
EmailVerified bool `json:"emailVerified"`
|
||||
Groups []string `json:"groups,omitempty"`
|
||||
}
|
||||
|
||||
func fromStorageClaims(i storage.Claims) Claims {
|
||||
return Claims{
|
||||
UserID: i.UserID,
|
||||
Username: i.Username,
|
||||
Email: i.Email,
|
||||
EmailVerified: i.EmailVerified,
|
||||
Groups: i.Groups,
|
||||
}
|
||||
}
|
||||
|
||||
func toStorageClaims(i Claims) storage.Claims {
|
||||
return storage.Claims{
|
||||
UserID: i.UserID,
|
||||
Username: i.Username,
|
||||
Email: i.Email,
|
||||
EmailVerified: i.EmailVerified,
|
||||
Groups: i.Groups,
|
||||
}
|
||||
}
|
||||
|
||||
// Keys is a mirrored struct from storage with JSON struct tags
|
||||
type Keys struct {
|
||||
SigningKey *jose.JSONWebKey `json:"signing_key,omitempty"`
|
||||
SigningKeyPub *jose.JSONWebKey `json:"signing_key_pub,omitempty"`
|
||||
VerificationKeys []storage.VerificationKey `json:"verification_keys"`
|
||||
NextRotation time.Time `json:"next_rotation"`
|
||||
}
|
||||
|
||||
func fromStorageKeys(keys storage.Keys) Keys {
|
||||
return Keys{
|
||||
SigningKey: keys.SigningKey,
|
||||
SigningKeyPub: keys.SigningKeyPub,
|
||||
VerificationKeys: keys.VerificationKeys,
|
||||
NextRotation: keys.NextRotation,
|
||||
}
|
||||
}
|
||||
|
||||
func toStorageKeys(keys Keys) storage.Keys {
|
||||
return storage.Keys{
|
||||
SigningKey: keys.SigningKey,
|
||||
SigningKeyPub: keys.SigningKeyPub,
|
||||
VerificationKeys: keys.VerificationKeys,
|
||||
NextRotation: keys.NextRotation,
|
||||
}
|
||||
}
|
||||
|
||||
// OfflineSessions is a mirrored struct from storage with JSON struct tags
|
||||
type OfflineSessions struct {
|
||||
UserID string `json:"user_id,omitempty"`
|
||||
ConnID string `json:"conn_id,omitempty"`
|
||||
Refresh map[string]*storage.RefreshTokenRef `json:"refresh,omitempty"`
|
||||
}
|
||||
|
||||
func fromStorageOfflineSessions(o storage.OfflineSessions) OfflineSessions {
|
||||
return OfflineSessions{
|
||||
UserID: o.UserID,
|
||||
ConnID: o.ConnID,
|
||||
Refresh: o.Refresh,
|
||||
}
|
||||
}
|
||||
|
||||
func toStorageOfflineSessions(o OfflineSessions) storage.OfflineSessions {
|
||||
s := storage.OfflineSessions{
|
||||
UserID: o.UserID,
|
||||
ConnID: o.ConnID,
|
||||
Refresh: o.Refresh,
|
||||
}
|
||||
if s.Refresh == nil {
|
||||
// Server code assumes this will be non-nil.
|
||||
s.Refresh = make(map[string]*storage.RefreshTokenRef)
|
||||
}
|
||||
return s
|
||||
}
|
||||
Reference in New Issue
Block a user