diff --git a/selection.go b/selection.go
index a9fec49..a8e718b 100644
--- a/selection.go
+++ b/selection.go
@@ -27,7 +27,12 @@ type controllingSelector struct {
 func (s *controllingSelector) Start() {
 	s.startTime = time.Now()
 	go func() {
-		time.Sleep(s.agent.candidateSelectionTimeout)
+		select {
+		case <-s.agent.done:
+			return
+		case <-time.After(s.agent.candidateSelectionTimeout):
+		}
+
 		err := s.agent.run(func(a *Agent) {
 			if s.nominatedPair == nil {
 				p := s.agent.getBestValidCandidatePair()