Merge pull request #577 from coreos/dev-sql

dev branch: add SQL storage implementation
This commit is contained in:
Eric Chiang
2016-10-03 12:52:14 -07:00
committed by GitHub
143 changed files with 237614 additions and 131 deletions
+7
View File
@@ -3,6 +3,13 @@ language: go
go:
- 1.7
services:
- postgresql
env:
- DEX_POSTGRES_DATABASE=postgres DEX_POSTGRES_USER=postgres DEX_POSTGRES_HOST="localhost"
install:
- go get -u github.com/golang/lint/golint
+37
View File
@@ -0,0 +1,37 @@
# Running database tests
Running database tests locally require:
* A systemd based Linux distro.
* A recent version of [rkt](https://github.com/coreos/rkt) installed.
The `standup.sh` script in the SQL directory is used to run databases in
containers with systemd daemonizing the process.
```
$ sudo ./storage/sql/standup.sh create postgres
Starting postgres. To view progress run
journalctl -fu dex-postgres
Running as unit dex-postgres.service.
To run tests export the following environment variables:
export DEX_POSTGRES_DATABASE=postgres; export DEX_POSTGRES_USER=postgres; export DEX_POSTGRES_PASSWORD=postgres; export DEX_POSTGRES_HOST=172.16.28.3:5432
```
Exporting the variables will cause the database tests to be run, rather than
skipped.
```
$ # sqlite takes forever to compile, be sure to install test dependencies
$ go test -v -i ./storage/sql
$ go test -v ./storage/sql
```
When you're done, tear down the unit using the `standup.sh` script.
```
$ sudo ./storage/sql/standup.sh destroy postgres
```
+5 -4
View File
@@ -10,7 +10,6 @@ DOCKER_IMAGE=$(DOCKER_REPO):$(VERSION)
export GOBIN=$(PWD)/bin
export GO15VENDOREXPERIMENT=1
export CGO_ENABLED:=0
LD_FLAGS="-w -X $(REPO_PATH)/version.Version=$(VERSION)"
@@ -26,10 +25,12 @@ bin/example-app: FORCE
@go install -ldflags $(LD_FLAGS) $(REPO_PATH)/cmd/example-app
test:
@go test $(shell go list ./... | grep -v '/vendor/')
@go test -v -i $(shell go list ./... | grep -v '/vendor/')
@go test -v $(shell go list ./... | grep -v '/vendor/')
testrace:
@CGO_ENABLED=1 go test --race $(shell go list ./... | grep -v '/vendor/')
@go test -v -i --race $(shell go list ./... | grep -v '/vendor/')
@go test -v --race $(shell go list ./... | grep -v '/vendor/')
vet:
@go vet $(shell go list ./... | grep -v '/vendor/')
@@ -39,7 +40,7 @@ fmt:
lint:
@for package in $(shell go list ./... | grep -v '/vendor/' | grep -v 'api/apipb'); do \
golint $$package; \
golint -set_exit_status $$package; \
done
server/templates_default.go: $(wildcard web/templates/**)
+13
View File
@@ -12,6 +12,7 @@ import (
"github.com/coreos/dex/storage"
"github.com/coreos/dex/storage/kubernetes"
"github.com/coreos/dex/storage/memory"
"github.com/coreos/dex/storage/sql"
)
// Config is the config format for the main application.
@@ -71,6 +72,18 @@ func (s *Storage) UnmarshalYAML(unmarshal func(interface{}) error) error {
}
err = unmarshal(&config)
s.Config = &config.Config
case "sqlite3":
var config struct {
Config sql.SQLite3 `yaml:"config"`
}
err = unmarshal(&config)
s.Config = &config.Config
case "postgres":
var config struct {
Config sql.Postgres `yaml:"config"`
}
err = unmarshal(&config)
s.Config = &config.Config
default:
return fmt.Errorf("unknown storage type %q", storageMeta.Type)
}
+1
View File
@@ -0,0 +1 @@
*.db
+3 -2
View File
@@ -1,7 +1,8 @@
issuer: http://127.0.0.1:5556
storage:
# NOTE(ericchiang): This will be replaced by sqlite3 in the future.
type: memory
type: sqlite3
config:
file: examples/dex.db
web:
http: 127.0.0.1:5556
Generated
+16 -4
View File
@@ -1,8 +1,14 @@
hash: 2af4a276277d2ab2ba9de9b0fd67ab7d6b70c07f4171a9efb225f30306d6f3eb
updated: 2016-08-08T11:20:44.300140564-07:00
hash: 149c717bc83cc279ab6192364776b8c1b6bad8a620ce9d64c56d946276630437
updated: 2016-09-30T21:01:57.607704513-07:00
imports:
- name: github.com/cockroachdb/cockroach-go
version: 31611c0501c812f437d4861d87d117053967c955
subpackages:
- crdb
- name: github.com/ericchiang/oidc
version: 1907f0e61549f9081f26bdf269f11603496c9dee
- name: github.com/go-sql-driver/mysql
version: 0b58b37b664c21f3010e836f1b931e1d0b0b0685
- name: github.com/golang/protobuf
version: 874264fbbb43f4d91e999fecb4b40143ed611400
subpackages:
@@ -21,6 +27,12 @@ imports:
subpackages:
- diff
- pretty
- name: github.com/lib/pq
version: 50761b0867bd1d9d069276790bcd4a3bccf2324a
subpackages:
- oid
- name: github.com/mattn/go-sqlite3
version: 3fb7a0e792edd47bf0cf1e919dfc14e2be412e15
- name: github.com/mitchellh/go-homedir
version: 756f7b183b7ab78acdbbee5c7f392838ed459dda
- name: github.com/pquerna/cachecontrol
@@ -48,13 +60,13 @@ imports:
- name: google.golang.org/appengine
version: 267c27e7492265b84fc6719503b14a1e17975d79
subpackages:
- urlfetch
- internal
- internal/urlfetch
- internal/base
- internal/datastore
- internal/log
- internal/remote_api
- internal/urlfetch
- urlfetch
- name: gopkg.in/asn1-ber.v1
version: 4e86f4367175e39f69d9358a5f17b4dda270378d
- name: gopkg.in/ldap.v2
+12
View File
@@ -77,3 +77,15 @@ import:
- diff
- pretty
version: eadb3ce320cbab8393bea5ca17bebac3f78a021b
# SQL drivers
- package: github.com/mattn/go-sqlite3
version: 3fb7a0e792edd47bf0cf1e919dfc14e2be412e15
- package: github.com/lib/pq
version: 50761b0867bd1d9d069276790bcd4a3bccf2324a
- package: github.com/go-sql-driver/mysql
version: 0b58b37b664c21f3010e836f1b931e1d0b0b0685
- package: github.com/cockroachdb/cockroach-go
version: 31611c0501c812f437d4861d87d117053967c955
subpackages:
- crdb
+5 -4
View File
@@ -264,7 +264,8 @@ func (s *Server) finalizeLogin(identity connector.Identity, authReqID, connector
}
updater := func(a storage.AuthRequest) (storage.AuthRequest, error) {
a.Claims = &claims
a.LoggedIn = true
a.Claims = claims
a.ConnectorID = connectorID
a.ConnectorData = identity.ConnectorData
return a, nil
@@ -282,7 +283,7 @@ func (s *Server) handleApproval(w http.ResponseWriter, r *http.Request) {
s.renderError(w, http.StatusInternalServerError, errServerError, "")
return
}
if authReq.Claims == nil {
if !authReq.LoggedIn {
log.Printf("Auth request does not have an identity for approval")
s.renderError(w, http.StatusInternalServerError, errServerError, "")
return
@@ -341,7 +342,7 @@ func (s *Server) sendCodeResponse(w http.ResponseWriter, r *http.Request, authRe
ConnectorID: authReq.ConnectorID,
Nonce: authReq.Nonce,
Scopes: authReq.Scopes,
Claims: *authReq.Claims,
Claims: authReq.Claims,
Expiry: s.now().Add(time.Minute * 5),
RedirectURI: authReq.RedirectURI,
}
@@ -358,7 +359,7 @@ func (s *Server) sendCodeResponse(w http.ResponseWriter, r *http.Request, authRe
}
q.Set("code", code.ID)
case responseTypeToken:
idToken, expiry, err := s.newIDToken(authReq.ClientID, *authReq.Claims, authReq.Scopes, authReq.Nonce)
idToken, expiry, err := s.newIDToken(authReq.ClientID, authReq.Claims, authReq.Scopes, authReq.Nonce)
if err != nil {
log.Printf("failed to create ID token: %v", err)
tokenErr(w, errServerError, "", http.StatusInternalServerError)
+226
View File
@@ -0,0 +1,226 @@
// +build go1.7
// Package conformance provides conformance tests for storage implementations.
package conformance
import (
"reflect"
"testing"
"time"
"github.com/coreos/dex/storage"
"github.com/kylelemons/godebug/pretty"
)
// ensure that values being tested on never expire.
var neverExpire = time.Now().UTC().Add(time.Hour * 24 * 365 * 100)
// StorageFactory is a method for creating a new storage. The returned storage sould be initialized
// but shouldn't have any existing data in it.
type StorageFactory func() storage.Storage
// RunTestSuite runs a set of conformance tests against a storage.
func RunTestSuite(t *testing.T, sf StorageFactory) {
tests := []struct {
name string
run func(t *testing.T, s storage.Storage)
}{
{"AuthCodeCRUD", testAuthCodeCRUD},
{"AuthRequestCRUD", testAuthRequestCRUD},
{"ClientCRUD", testClientCRUD},
{"RefreshTokenCRUD", testRefreshTokenCRUD},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
test.run(t, sf())
})
}
}
func mustBeErrNotFound(t *testing.T, kind string, err error) {
switch {
case err == nil:
t.Errorf("deleting non-existant %s should return an error", kind)
case err != storage.ErrNotFound:
t.Errorf("deleting %s expected storage.ErrNotFound, got %v", kind, err)
}
}
func testAuthRequestCRUD(t *testing.T, s storage.Storage) {
a := storage.AuthRequest{
ID: storage.NewID(),
ClientID: "foobar",
ResponseTypes: []string{"code"},
Scopes: []string{"openid", "email"},
RedirectURI: "https://localhost:80/callback",
Nonce: "foo",
State: "bar",
ForceApprovalPrompt: true,
LoggedIn: true,
Expiry: neverExpire,
ConnectorID: "ldap",
ConnectorData: []byte(`{"some":"data"}`),
Claims: storage.Claims{
UserID: "1",
Username: "jane",
Email: "jane.doe@example.com",
EmailVerified: true,
Groups: []string{"a", "b"},
},
}
identity := storage.Claims{Email: "foobar"}
if err := s.CreateAuthRequest(a); err != nil {
t.Fatalf("failed creating auth request: %v", err)
}
if err := s.UpdateAuthRequest(a.ID, func(old storage.AuthRequest) (storage.AuthRequest, error) {
old.Claims = identity
old.ConnectorID = "connID"
return old, nil
}); err != nil {
t.Fatalf("failed to update auth request: %v", err)
}
got, err := s.GetAuthRequest(a.ID)
if err != nil {
t.Fatalf("failed to get auth req: %v", err)
}
if !reflect.DeepEqual(got.Claims, identity) {
t.Fatalf("update failed, wanted identity=%#v got %#v", identity, got.Claims)
}
}
func testAuthCodeCRUD(t *testing.T, s storage.Storage) {
a := storage.AuthCode{
ID: storage.NewID(),
ClientID: "foobar",
RedirectURI: "https://localhost:80/callback",
Nonce: "foobar",
Scopes: []string{"openid", "email"},
Expiry: neverExpire,
ConnectorID: "ldap",
ConnectorData: []byte(`{"some":"data"}`),
Claims: storage.Claims{
UserID: "1",
Username: "jane",
Email: "jane.doe@example.com",
EmailVerified: true,
Groups: []string{"a", "b"},
},
}
if err := s.CreateAuthCode(a); err != nil {
t.Fatalf("failed creating auth code: %v", err)
}
got, err := s.GetAuthCode(a.ID)
if err != nil {
t.Fatalf("failed to get auth req: %v", err)
}
if a.Expiry.Unix() != got.Expiry.Unix() {
t.Errorf("auth code expiry did not match want=%s vs got=%s", a.Expiry, got.Expiry)
}
got.Expiry = a.Expiry // time fields do not compare well
if diff := pretty.Compare(a, got); diff != "" {
t.Errorf("auth code retrieved from storage did not match: %s", diff)
}
if err := s.DeleteAuthCode(a.ID); err != nil {
t.Fatalf("delete auth code: %v", err)
}
_, err = s.GetAuthCode(a.ID)
mustBeErrNotFound(t, "auth code", err)
}
func testClientCRUD(t *testing.T, s storage.Storage) {
id := storage.NewID()
c := storage.Client{
ID: id,
Secret: "foobar",
RedirectURIs: []string{"foo://bar.com/", "https://auth.example.com"},
Name: "dex client",
LogoURL: "https://goo.gl/JIyzIC",
}
err := s.DeleteClient(id)
mustBeErrNotFound(t, "client", err)
if err := s.CreateClient(c); err != nil {
t.Fatalf("create client: %v", err)
}
getAndCompare := func(id string, want storage.Client) {
gc, err := s.GetClient(id)
if err != nil {
t.Errorf("get client: %v", err)
return
}
if diff := pretty.Compare(want, gc); diff != "" {
t.Errorf("client retrieved from storage did not match: %s", diff)
}
}
getAndCompare(id, c)
newSecret := "barfoo"
err = s.UpdateClient(id, func(old storage.Client) (storage.Client, error) {
old.Secret = newSecret
return old, nil
})
if err != nil {
t.Errorf("update client: %v", err)
}
c.Secret = newSecret
getAndCompare(id, c)
if err := s.DeleteClient(id); err != nil {
t.Fatalf("delete client: %v", err)
}
_, err = s.GetClient(id)
mustBeErrNotFound(t, "client", err)
}
func testRefreshTokenCRUD(t *testing.T, s storage.Storage) {
id := storage.NewID()
refresh := storage.RefreshToken{
RefreshToken: id,
ClientID: "client_id",
ConnectorID: "client_secret",
Scopes: []string{"openid", "email", "profile"},
Claims: storage.Claims{
UserID: "1",
Username: "jane",
Email: "jane.doe@example.com",
EmailVerified: true,
Groups: []string{"a", "b"},
},
}
if err := s.CreateRefresh(refresh); err != nil {
t.Fatalf("create refresh token: %v", err)
}
getAndCompare := func(id string, want storage.RefreshToken) {
gr, err := s.GetRefresh(id)
if err != nil {
t.Errorf("get refresh: %v", err)
return
}
if diff := pretty.Compare(want, gr); diff != "" {
t.Errorf("refresh token retrieved from storage did not match: %s", diff)
}
}
getAndCompare(id, refresh)
if err := s.DeleteRefresh(id); err != nil {
t.Fatalf("failed to delete refresh request: %v", err)
}
if _, err := s.GetRefresh(id); err != storage.ErrNotFound {
t.Errorf("after deleting refresh expected storage.ErrNotFound, got %v", err)
}
}
+6 -2
View File
@@ -4,7 +4,8 @@ import (
"os"
"testing"
"github.com/coreos/dex/storage/storagetest"
"github.com/coreos/dex/storage"
"github.com/coreos/dex/storage/conformance"
)
func TestLoadClient(t *testing.T) {
@@ -73,5 +74,8 @@ func TestURLFor(t *testing.T) {
func TestStorage(t *testing.T) {
client := loadClient(t)
storagetest.RunTestSuite(t, client)
conformance.RunTestSuite(t, func() storage.Storage {
// TODO(erichiang): Tear down namespaces between each iteration.
return client
})
}
+7 -9
View File
@@ -118,9 +118,11 @@ type AuthRequest struct {
// attempts.
ForceApprovalPrompt bool `json:"forceApprovalPrompt,omitempty"`
LoggedIn bool `json:"loggedIn"`
// The identity of the end user. Generally nil until the user authenticates
// with a backend.
Claims *Claims `json:"claims,omitempty"`
Claims Claims `json:"claims,omitempty"`
// The connector used to login the user. Set when the user authenticates.
ConnectorID string `json:"connectorID,omitempty"`
ConnectorData []byte `json:"connectorData,omitempty"`
@@ -145,13 +147,11 @@ func toStorageAuthRequest(req AuthRequest) storage.AuthRequest {
Nonce: req.Nonce,
State: req.State,
ForceApprovalPrompt: req.ForceApprovalPrompt,
LoggedIn: req.LoggedIn,
ConnectorID: req.ConnectorID,
ConnectorData: req.ConnectorData,
Expiry: req.Expiry,
}
if req.Claims != nil {
i := toStorageClaims(*req.Claims)
a.Claims = &i
Claims: toStorageClaims(req.Claims),
}
return a
}
@@ -172,14 +172,12 @@ func (cli *client) fromStorageAuthRequest(a storage.AuthRequest) AuthRequest {
RedirectURI: a.RedirectURI,
Nonce: a.Nonce,
State: a.State,
LoggedIn: a.LoggedIn,
ForceApprovalPrompt: a.ForceApprovalPrompt,
ConnectorID: a.ConnectorID,
ConnectorData: a.ConnectorData,
Expiry: a.Expiry,
}
if a.Claims != nil {
i := fromStorageClaims(*a.Claims)
req.Claims = &i
Claims: fromStorageClaims(a.Claims),
}
return req
}
+2 -3
View File
@@ -3,10 +3,9 @@ package memory
import (
"testing"
"github.com/coreos/dex/storage/storagetest"
"github.com/coreos/dex/storage/conformance"
)
func TestStorage(t *testing.T) {
s := New()
storagetest.RunTestSuite(t, s)
conformance.RunTestSuite(t, New)
}
+113
View File
@@ -0,0 +1,113 @@
package sql
import (
"database/sql"
"fmt"
"net/url"
"strconv"
"github.com/coreos/dex/storage"
)
// SQLite3 options for creating an SQL db.
type SQLite3 struct {
// File to
File string `yaml:"file"`
}
// Open creates a new storage implementation backed by SQLite3
func (s *SQLite3) Open() (storage.Storage, error) {
return s.open()
}
func (s *SQLite3) open() (*conn, error) {
db, err := sql.Open("sqlite3", s.File)
if err != nil {
return nil, err
}
if s.File == ":memory:" {
// sqlite3 uses file locks to coordinate concurrent access. In memory
// doesn't support this, so limit the number of connections to 1.
db.SetMaxOpenConns(1)
}
c := &conn{db, flavorSQLite3}
if _, err := c.migrate(); err != nil {
return nil, fmt.Errorf("failed to perform migrations: %v", err)
}
return c, nil
}
const (
sslDisable = "disable"
sslRequire = "require"
sslVerifyCA = "verify-ca"
sslVerifyFull = "verify-full"
)
// PostgresSSL represents SSL options for Postgres databases.
type PostgresSSL struct {
Mode string
CAFile string
// Files for client auth.
KeyFile string
CertFile string
}
// Postgres options for creating an SQL db.
type Postgres struct {
Database string
User string
Password string
Host string
SSL PostgresSSL `json:"ssl" yaml:"ssl"`
ConnectionTimeout int // Seconds
}
// Open creates a new storage implementation backed by Postgres.
func (p *Postgres) Open() (storage.Storage, error) {
return p.open()
}
func (p *Postgres) open() (*conn, error) {
v := url.Values{}
set := func(key, val string) {
if val != "" {
v.Set(key, val)
}
}
set("connect_timeout", strconv.Itoa(p.ConnectionTimeout))
set("sslkey", p.SSL.KeyFile)
set("sslcert", p.SSL.CertFile)
set("sslrootcert", p.SSL.CAFile)
if p.SSL.Mode == "" {
// Assume the strictest mode if unspecified.
p.SSL.Mode = sslVerifyFull
}
set("sslmode", p.SSL.Mode)
u := url.URL{
Scheme: "postgres",
Host: p.Host,
Path: "/" + p.Database,
RawQuery: v.Encode(),
}
if p.User != "" {
if p.Password != "" {
u.User = url.UserPassword(p.User, p.Password)
} else {
u.User = url.User(p.User)
}
}
db, err := sql.Open("postgres", u.String())
if err != nil {
return nil, err
}
c := &conn{db, flavorPostgres}
if _, err := c.migrate(); err != nil {
return nil, fmt.Errorf("failed to perform migrations: %v", err)
}
return c, nil
}
+90
View File
@@ -0,0 +1,90 @@
package sql
import (
"fmt"
"os"
"runtime"
"testing"
"time"
"github.com/coreos/dex/storage"
"github.com/coreos/dex/storage/conformance"
)
func withTimeout(t time.Duration, f func()) {
c := make(chan struct{})
defer close(c)
go func() {
select {
case <-c:
case <-time.After(t):
// Dump a stack trace of the program. Useful for debugging deadlocks.
buf := make([]byte, 2<<20)
fmt.Fprintf(os.Stderr, "%s\n", buf[:runtime.Stack(buf, true)])
panic("test took too long")
}
}()
f()
}
func cleanDB(c *conn) error {
_, err := c.Exec(`
delete from client;
delete from auth_request;
delete from auth_code;
delete from refresh_token;
delete from keys;
`)
return err
}
func TestSQLite3(t *testing.T) {
newStorage := func() storage.Storage {
// NOTE(ericchiang): In memory means we only get one connection at a time. If we
// ever write tests that require using multiple connections, for instance to test
// transactions, we need to move to a file based system.
s := &SQLite3{":memory:"}
conn, err := s.open()
if err != nil {
t.Fatal(err)
}
return conn
}
withTimeout(time.Second*10, func() {
conformance.RunTestSuite(t, newStorage)
})
}
func TestPostgres(t *testing.T) {
if os.Getenv("DEX_POSTGRES_HOST") == "" {
t.Skip("postgres envs not set, skipping tests")
}
p := Postgres{
Database: os.Getenv("DEX_POSTGRES_DATABASE"),
User: os.Getenv("DEX_POSTGRES_USER"),
Password: os.Getenv("DEX_POSTGRES_PASSWORD"),
Host: os.Getenv("DEX_POSTGRES_HOST"),
SSL: PostgresSSL{
Mode: sslDisable, // Postgres container doesn't support SSL.
},
ConnectionTimeout: 5,
}
conn, err := p.open()
if err != nil {
t.Fatal(err)
}
defer conn.Close()
newStorage := func() storage.Storage {
if err := cleanDB(conn); err != nil {
t.Fatal(err)
}
return conn
}
withTimeout(time.Minute*1, func() {
conformance.RunTestSuite(t, newStorage)
})
}
+487
View File
@@ -0,0 +1,487 @@
package sql
import (
"database/sql"
"database/sql/driver"
"encoding/json"
"errors"
"fmt"
"github.com/coreos/dex/storage"
)
// TODO(ericchiang): The update, insert, and select methods queries are all
// very repetivite. Consider creating them programatically.
// keysRowID is the ID of the only row we expect to populate the "keys" table.
const keysRowID = "keys"
// encoder wraps the underlying value in a JSON marshaler which is automatically
// called by the database/sql package.
//
// s := []string{"planes", "bears"}
// err := db.Exec(`insert into t1 (id, things) values (1, $1)`, encoder(s))
// if err != nil {
// // handle error
// }
//
// var r []byte
// err = db.QueryRow(`select things from t1 where id = 1;`).Scan(&r)
// if err != nil {
// // handle error
// }
// fmt.Printf("%s\n", r) // ["planes","bears"]
//
func encoder(i interface{}) driver.Valuer {
return jsonEncoder{i}
}
// decoder wraps the underlying value in a JSON unmarshaler which can then be passed
// to a database Scan() method.
func decoder(i interface{}) sql.Scanner {
return jsonDecoder{i}
}
type jsonEncoder struct {
i interface{}
}
func (j jsonEncoder) Value() (driver.Value, error) {
b, err := json.Marshal(j.i)
if err != nil {
return nil, fmt.Errorf("marshal: %v", err)
}
return b, nil
}
type jsonDecoder struct {
i interface{}
}
func (j jsonDecoder) Scan(dest interface{}) error {
if dest == nil {
return errors.New("nil value")
}
b, ok := dest.([]byte)
if !ok {
return fmt.Errorf("expected []byte got %T", dest)
}
if err := json.Unmarshal(b, &j.i); err != nil {
return fmt.Errorf("unmarshal: %v", err)
}
return nil
}
// Abstract conn vs trans.
type querier interface {
QueryRow(query string, args ...interface{}) *sql.Row
}
// Abstract row vs rows.
type scanner interface {
Scan(dest ...interface{}) error
}
func (c *conn) CreateAuthRequest(a storage.AuthRequest) error {
_, err := c.Exec(`
insert into auth_request (
id, client_id, response_types, scopes, redirect_uri, nonce, state,
force_approval_prompt, logged_in,
claims_user_id, claims_username, claims_email, claims_email_verified,
claims_groups,
connector_id, connector_data,
expiry
)
values (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17
);
`,
a.ID, a.ClientID, encoder(a.ResponseTypes), encoder(a.Scopes), a.RedirectURI, a.Nonce, a.State,
a.ForceApprovalPrompt, a.LoggedIn,
a.Claims.UserID, a.Claims.Username, a.Claims.Email, a.Claims.EmailVerified,
encoder(a.Claims.Groups),
a.ConnectorID, a.ConnectorData,
a.Expiry,
)
if err != nil {
return fmt.Errorf("insert auth request: %v", err)
}
return nil
}
func (c *conn) UpdateAuthRequest(id string, updater func(a storage.AuthRequest) (storage.AuthRequest, error)) error {
return c.ExecTx(func(tx *trans) error {
r, err := getAuthRequest(tx, id)
if err != nil {
return err
}
a, err := updater(r)
if err != nil {
return err
}
_, err = tx.Exec(`
update auth_request
set
client_id = $1, response_types = $2, scopes = $3, redirect_uri = $4,
nonce = $5, state = $6, force_approval_prompt = $7, logged_in = $8,
claims_user_id = $9, claims_username = $10, claims_email = $11,
claims_email_verified = $12,
claims_groups = $13,
connector_id = $14, connector_data = $15,
expiry = $16
where id = $17;
`,
a.ClientID, encoder(a.ResponseTypes), encoder(a.Scopes), a.RedirectURI, a.Nonce, a.State,
a.ForceApprovalPrompt, a.LoggedIn,
a.Claims.UserID, a.Claims.Username, a.Claims.Email, a.Claims.EmailVerified,
encoder(a.Claims.Groups),
a.ConnectorID, a.ConnectorData,
a.Expiry, a.ID,
)
if err != nil {
return fmt.Errorf("update auth request: %v", err)
}
return nil
})
}
func (c *conn) GetAuthRequest(id string) (storage.AuthRequest, error) {
return getAuthRequest(c, id)
}
func getAuthRequest(q querier, id string) (a storage.AuthRequest, err error) {
err = q.QueryRow(`
select
id, client_id, response_types, scopes, redirect_uri, nonce, state,
force_approval_prompt, logged_in,
claims_user_id, claims_username, claims_email, claims_email_verified,
claims_groups,
connector_id, connector_data, expiry
from auth_request where id = $1;
`, id).Scan(
&a.ID, &a.ClientID, decoder(&a.ResponseTypes), decoder(&a.Scopes), &a.RedirectURI, &a.Nonce, &a.State,
&a.ForceApprovalPrompt, &a.LoggedIn,
&a.Claims.UserID, &a.Claims.Username, &a.Claims.Email, &a.Claims.EmailVerified,
decoder(&a.Claims.Groups),
&a.ConnectorID, &a.ConnectorData, &a.Expiry,
)
if err != nil {
if err == sql.ErrNoRows {
return a, storage.ErrNotFound
}
return a, fmt.Errorf("select auth request: %v", err)
}
return a, nil
}
func (c *conn) CreateAuthCode(a storage.AuthCode) error {
_, err := c.Exec(`
insert into auth_code (
id, client_id, scopes, nonce, redirect_uri,
claims_user_id, claims_username,
claims_email, claims_email_verified, claims_groups,
connector_id, connector_data,
expiry
)
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13);
`,
a.ID, a.ClientID, encoder(a.Scopes), a.Nonce, a.RedirectURI, a.Claims.UserID,
a.Claims.Username, a.Claims.Email, a.Claims.EmailVerified, encoder(a.Claims.Groups),
a.ConnectorID, a.ConnectorData, a.Expiry,
)
return err
}
func (c *conn) GetAuthCode(id string) (a storage.AuthCode, err error) {
err = c.QueryRow(`
select
id, client_id, scopes, nonce, redirect_uri,
claims_user_id, claims_username,
claims_email, claims_email_verified, claims_groups,
connector_id, connector_data,
expiry
from auth_code where id = $1;
`, id).Scan(
&a.ID, &a.ClientID, decoder(&a.Scopes), &a.Nonce, &a.RedirectURI, &a.Claims.UserID,
&a.Claims.Username, &a.Claims.Email, &a.Claims.EmailVerified, decoder(&a.Claims.Groups),
&a.ConnectorID, &a.ConnectorData, &a.Expiry,
)
if err != nil {
if err == sql.ErrNoRows {
return a, storage.ErrNotFound
}
return a, fmt.Errorf("select auth code: %v", err)
}
return a, nil
}
func (c *conn) CreateRefresh(r storage.RefreshToken) error {
_, err := c.Exec(`
insert into refresh_token (
id, client_id, scopes, nonce,
claims_user_id, claims_username, claims_email, claims_email_verified,
claims_groups,
connector_id, connector_data
)
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11);
`,
r.RefreshToken, r.ClientID, encoder(r.Scopes), r.Nonce,
r.Claims.UserID, r.Claims.Username, r.Claims.Email, r.Claims.EmailVerified,
encoder(r.Claims.Groups),
r.ConnectorID, r.ConnectorData,
)
if err != nil {
return fmt.Errorf("insert refresh_token: %v", err)
}
return nil
}
func (c *conn) GetRefresh(id string) (storage.RefreshToken, error) {
return scanRefresh(c.QueryRow(`
select
id, client_id, scopes, nonce,
claims_user_id, claims_username, claims_email, claims_email_verified,
claims_groups,
connector_id, connector_data
from refresh_token where id = $1;
`, id))
}
func (c *conn) ListRefreshTokens() ([]storage.RefreshToken, error) {
rows, err := c.Query(`
select
id, client_id, scopes, nonce,
claims_user_id, claims_username, claims_email, claims_email_verified,
claims_groups,
connector_id, connector_data
from refresh_token;
`)
if err != nil {
return nil, fmt.Errorf("query: %v", err)
}
var tokens []storage.RefreshToken
for rows.Next() {
r, err := scanRefresh(rows)
if err != nil {
return nil, err
}
tokens = append(tokens, r)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("scan: %v", err)
}
return tokens, nil
}
func scanRefresh(s scanner) (r storage.RefreshToken, err error) {
err = s.Scan(
&r.RefreshToken, &r.ClientID, decoder(&r.Scopes), &r.Nonce,
&r.Claims.UserID, &r.Claims.Username, &r.Claims.Email, &r.Claims.EmailVerified,
decoder(&r.Claims.Groups),
&r.ConnectorID, &r.ConnectorData,
)
if err != nil {
if err == sql.ErrNoRows {
return r, storage.ErrNotFound
}
return r, fmt.Errorf("scan refresh_token: %v", err)
}
return r, nil
}
func (c *conn) UpdateKeys(updater func(old storage.Keys) (storage.Keys, error)) error {
return c.ExecTx(func(tx *trans) error {
firstUpdate := false
// TODO(ericchiang): errors may cause a transaction be rolled back by the SQL
// server. Test this, and consider adding a COUNT() command beforehand.
old, err := getKeys(tx)
if err != nil {
if err != storage.ErrNotFound {
return fmt.Errorf("get keys: %v", err)
}
firstUpdate = true
old = storage.Keys{}
}
nk, err := updater(old)
if err != nil {
return err
}
if firstUpdate {
_, err = tx.Exec(`
insert into keys (
id, verification_keys, signing_key, signing_key_pub, next_rotation
)
values ($1, $2, $3, $4, $5);
`,
keysRowID, encoder(nk.VerificationKeys), encoder(nk.SigningKey),
encoder(nk.SigningKeyPub), nk.NextRotation,
)
if err != nil {
return fmt.Errorf("insert: %v", err)
}
} else {
_, err = tx.Exec(`
update keys
set
verification_keys = $1,
signing_key = $2,
singing_key_pub = $3,
next_rotation = $4
where id = $5;
`,
encoder(nk.VerificationKeys), encoder(nk.SigningKey),
encoder(nk.SigningKeyPub), nk.NextRotation, keysRowID,
)
if err != nil {
return fmt.Errorf("update: %v", err)
}
}
return nil
})
}
func (c *conn) GetKeys() (keys storage.Keys, err error) {
return getKeys(c)
}
func getKeys(q querier) (keys storage.Keys, err error) {
err = q.QueryRow(`
select
verification_keys, signing_key, signing_key_pub, next_rotation
from keys
where id=$q
`, keysRowID).Scan(
decoder(&keys.VerificationKeys), decoder(&keys.SigningKey),
decoder(&keys.SigningKeyPub), &keys.NextRotation,
)
if err != nil {
if err == sql.ErrNoRows {
return keys, storage.ErrNotFound
}
return keys, fmt.Errorf("query keys: %v", err)
}
return keys, nil
}
func (c *conn) UpdateClient(id string, updater func(old storage.Client) (storage.Client, error)) error {
return c.ExecTx(func(tx *trans) error {
cli, err := getClient(tx, id)
if err != nil {
return err
}
nc, err := updater(cli)
if err != nil {
return err
}
_, err = tx.Exec(`
update client
set
secret = $1,
redirect_uris = $2,
trusted_peers = $3,
public = $4,
name = $5,
logo_url = $6
where id = $7;
`, nc.Secret, encoder(nc.RedirectURIs), encoder(nc.TrustedPeers), nc.Public, nc.Name, nc.LogoURL, id,
)
if err != nil {
return fmt.Errorf("update client: %v", err)
}
return nil
})
}
func (c *conn) CreateClient(cli storage.Client) error {
_, err := c.Exec(`
insert into client (
id, secret, redirect_uris, trusted_peers, public, name, logo_url
)
values ($1, $2, $3, $4, $5, $6, $7);
`,
cli.ID, cli.Secret, encoder(cli.RedirectURIs), encoder(cli.TrustedPeers),
cli.Public, cli.Name, cli.LogoURL,
)
if err != nil {
return fmt.Errorf("insert client: %v", err)
}
return nil
}
func getClient(q querier, id string) (storage.Client, error) {
return scanClient(q.QueryRow(`
select
id, secret, redirect_uris, trusted_peers, public, name, logo_url
from client where id = $1;
`, id))
}
func (c *conn) GetClient(id string) (storage.Client, error) {
return getClient(c, id)
}
func (c *conn) ListClients() ([]storage.Client, error) {
rows, err := c.Query(`
select
id, secret, redirect_uris, trusted_peers, public, name, logo_url
from client;
`)
if err != nil {
return nil, err
}
var clients []storage.Client
for rows.Next() {
cli, err := scanClient(rows)
if err != nil {
return nil, err
}
clients = append(clients, cli)
}
if err := rows.Err(); err != nil {
return nil, err
}
return clients, nil
}
func scanClient(s scanner) (cli storage.Client, err error) {
err = s.Scan(
&cli.ID, &cli.Secret, decoder(&cli.RedirectURIs), decoder(&cli.TrustedPeers),
&cli.Public, &cli.Name, &cli.LogoURL,
)
if err != nil {
if err == sql.ErrNoRows {
return cli, storage.ErrNotFound
}
return cli, fmt.Errorf("get client: %v", err)
}
return cli, nil
}
func (c *conn) DeleteAuthRequest(id string) error { return c.delete("auth_request", id) }
func (c *conn) DeleteAuthCode(id string) error { return c.delete("auth_code", id) }
func (c *conn) DeleteClient(id string) error { return c.delete("client", id) }
func (c *conn) DeleteRefresh(id string) error { return c.delete("refresh_token", id) }
// Do NOT call directly. Does not escape table.
func (c *conn) delete(table, id string) error {
result, err := c.Exec(`delete from `+table+` where id = $1`, id)
if err != nil {
return fmt.Errorf("delete %s: %v", table, id)
}
// For now mandate that the driver implements RowsAffected. If we ever need to support
// a driver that doesn't implement this, we can run this in a transaction with a get beforehand.
n, err := result.RowsAffected()
if err != nil {
return fmt.Errorf("rows affected: %v", err)
}
if n < 1 {
return storage.ErrNotFound
}
return nil
}
+55
View File
@@ -0,0 +1,55 @@
package sql
import (
"database/sql"
"reflect"
"testing"
)
func TestDecoder(t *testing.T) {
db, err := sql.Open("sqlite3", ":memory:")
if err != nil {
t.Fatal(err)
}
defer db.Close()
if _, err := db.Exec(`create table foo ( id integer primary key, bar blob );`); err != nil {
t.Fatal(err)
}
if _, err := db.Exec(`insert into foo ( id, bar ) values (1, ?);`, []byte(`["a", "b"]`)); err != nil {
t.Fatal(err)
}
var got []string
if err := db.QueryRow(`select bar from foo where id = 1;`).Scan(decoder(&got)); err != nil {
t.Fatal(err)
}
want := []string{"a", "b"}
if !reflect.DeepEqual(got, want) {
t.Errorf("wanted %q got %q", want, got)
}
}
func TestEncoder(t *testing.T) {
db, err := sql.Open("sqlite3", ":memory:")
if err != nil {
t.Fatal(err)
}
defer db.Close()
if _, err := db.Exec(`create table foo ( id integer primary key, bar blob );`); err != nil {
t.Fatal(err)
}
put := []string{"a", "b"}
if _, err := db.Exec(`insert into foo ( id, bar ) values (1, ?)`, encoder(put)); err != nil {
t.Fatal(err)
}
var got []byte
if err := db.QueryRow(`select bar from foo where id = 1;`).Scan(&got); err != nil {
t.Fatal(err)
}
want := []byte(`["a","b"]`)
if !reflect.DeepEqual(got, want) {
t.Errorf("wanted %q got %q", want, got)
}
}
+24
View File
@@ -0,0 +1,24 @@
package sql
import (
"fmt"
"time"
)
type gc struct {
now func() time.Time
conn *conn
}
var tablesWithGC = []string{"auth_request", "auth_code"}
func (gc gc) run() error {
for _, table := range tablesWithGC {
_, err := gc.conn.Exec(`delete from `+table+` where expiry < $1`, gc.now())
if err != nil {
return fmt.Errorf("gc %s: %v", table, err)
}
// TODO(ericchiang): when we have levelled logging print how many rows were gc'd
}
return nil
}
+53
View File
@@ -0,0 +1,53 @@
package sql
import (
"testing"
"time"
"github.com/coreos/dex/storage"
)
func TestGC(t *testing.T) {
// TODO(ericchiang): Add a GarbageCollect method to the storage interface so
// we can write conformance tests instead of directly testing each implementation.
s := &SQLite3{":memory:"}
conn, err := s.open()
if err != nil {
t.Fatal(err)
}
defer conn.Close()
clock := time.Now()
now := func() time.Time { return clock }
runGC := (gc{now, conn}).run
a := storage.AuthRequest{
ID: storage.NewID(),
Expiry: now().Add(time.Second),
}
if err := conn.CreateAuthRequest(a); err != nil {
t.Fatal(err)
}
if err := runGC(); err != nil {
t.Errorf("gc failed: %v", err)
}
if _, err := conn.GetAuthRequest(a.ID); err != nil {
t.Errorf("failed to get auth request after gc: %v", err)
}
clock = clock.Add(time.Minute)
if err := runGC(); err != nil {
t.Errorf("gc failed: %v", err)
}
if _, err := conn.GetAuthRequest(a.ID); err == nil {
t.Errorf("expected error after gc'ing auth request: %v", err)
} else if err != storage.ErrNotFound {
t.Errorf("expected error storage.NotFound got: %v", err)
}
}
+151
View File
@@ -0,0 +1,151 @@
package sql
import (
"database/sql"
"fmt"
)
func (c *conn) migrate() (int, error) {
_, err := c.Exec(`
create table if not exists migrations (
num integer not null,
at timestamp not null
);
`)
if err != nil {
return 0, fmt.Errorf("creating migration table: %v", err)
}
i := 0
done := false
for {
err := c.ExecTx(func(tx *trans) error {
// Within a transaction, perform a single migration.
var (
num sql.NullInt64
n int
)
if err := tx.QueryRow(`select max(num) from migrations;`).Scan(&num); err != nil {
return fmt.Errorf("select max migration: %v", err)
}
if num.Valid {
n = int(num.Int64)
}
if n >= len(migrations) {
done = true
return nil
}
migrationNum := n + 1
m := migrations[n]
if _, err := tx.Exec(m.stmt); err != nil {
return fmt.Errorf("migration %d failed: %v", migrationNum, err)
}
q := `insert into migrations (num, at) values ($1, now());`
if _, err := tx.Exec(q, migrationNum); err != nil {
return fmt.Errorf("update migration table: %v", err)
}
return nil
})
if err != nil {
return i, err
}
if done {
break
}
i++
}
return i, nil
}
type migration struct {
stmt string
// TODO(ericchiang): consider adding additional fields like "forDrivers"
}
// All SQL flavors share migration strategies.
var migrations = []migration{
{
stmt: `
create table client (
id text not null primary key,
secret text not null,
redirect_uris bytea not null, -- JSON array of strings
trusted_peers bytea not null, -- JSON array of strings
public boolean not null,
name text not null,
logo_url text not null
);
create table auth_request (
id text not null primary key,
client_id text not null,
response_types bytea not null, -- JSON array of strings
scopes bytea not null, -- JSON array of strings
redirect_uri text not null,
nonce text not null,
state text not null,
force_approval_prompt boolean not null,
logged_in boolean not null,
claims_user_id text not null,
claims_username text not null,
claims_email text not null,
claims_email_verified boolean not null,
claims_groups bytea not null, -- JSON array of strings
connector_id text not null,
connector_data bytea,
expiry timestamp not null
);
create table auth_code (
id text not null primary key,
client_id text not null,
scopes bytea not null, -- JSON array of strings
nonce text not null,
redirect_uri text not null,
claims_user_id text not null,
claims_username text not null,
claims_email text not null,
claims_email_verified boolean not null,
claims_groups bytea not null, -- JSON array of strings
connector_id text not null,
connector_data bytea,
expiry timestamp not null
);
create table refresh_token (
id text not null primary key,
client_id text not null,
scopes bytea not null, -- JSON array of strings
nonce text not null,
claims_user_id text not null,
claims_username text not null,
claims_email text not null,
claims_email_verified boolean not null,
claims_groups bytea not null, -- JSON array of strings
connector_id text not null,
connector_data bytea
);
-- keys is a weird table because we only ever expect there to be a single row
create table keys (
id text not null primary key,
verification_keys bytea not null, -- JSON array
signing_key bytea not null, -- JSON object
signing_key_pub bytea not null, -- JSON object
next_rotation timestamp not null
);
`,
},
}

Some files were not shown because too many files have changed in this diff Show More