diff --git a/selection.go b/selection.go
index 2137c17..a9fec49 100644
--- a/selection.go
+++ b/selection.go
@@ -190,11 +190,14 @@ func (s *controllingSelector) PingCandidate(local, remote Candidate) {
 }
 
 type controlledSelector struct {
-	agent *Agent
-	log   logging.LeveledLogger
+	startTime time.Time
+	agent     *Agent
+	log       logging.LeveledLogger
 }
 
-func (s *controlledSelector) Start() {}
+func (s *controlledSelector) Start() {
+	s.startTime = time.Now()
+}
 
 func (s *controlledSelector) ContactCandidates() {
 	if s.agent.selectedPair != nil {
@@ -203,8 +206,13 @@ func (s *controlledSelector) ContactCandidates() {
 			s.agent.checkKeepalive()
 		}
 	} else {
-		s.log.Trace("pinging all candidates")
-		s.agent.pingAllCandidates()
+		if time.Since(s.startTime) > s.agent.candidateSelectionTimeout {
+			s.log.Trace("check timeout reached and no valid candidate pair found, marking connection as failed")
+			s.agent.updateConnectionState(ConnectionStateFailed)
+		} else {
+			s.log.Trace("pinging all candidates")
+			s.agent.pingAllCandidates()
+		}
 	}
 }