diff --git a/agent.go b/agent.go
index 8ce8fd1..82d6cbb 100644
--- a/agent.go
+++ b/agent.go
@@ -654,12 +654,16 @@ func (a *Agent) handleInbound(m *stun.Message, local *Candidate, remote net.Addr
 	}
 }
 
-// noSTUNSeen processes non STUN traffic from a remote candidate
-func (a *Agent) noSTUNSeen(local *Candidate, remote net.Addr) {
+// noSTUNSeen processes non STUN traffic from a remote candidate,
+// and returns true if it is an actual remote candidate
+func (a *Agent) noSTUNSeen(local *Candidate, remote net.Addr) bool {
 	remoteCandidate := a.findRemoteCandidate(local.NetworkType, remote)
-	if remoteCandidate != nil {
-		remoteCandidate.seen(false)
+	if remoteCandidate == nil {
+		return false
 	}
+
+	remoteCandidate.seen(false)
+	return true
 }
 
 func (a *Agent) getBestPair() (*candidatePair, error) {
diff --git a/candidate.go b/candidate.go
index 3edab15..65d37f0 100644
--- a/candidate.go
+++ b/candidate.go
@@ -188,13 +188,17 @@ func (c *Candidate) recvLoop() {
 			}
 
 			continue
-		} else {
-			err = c.agent.run(func(agent *Agent) {
-				agent.noSTUNSeen(c, srcAddr)
-			})
-			if err != nil {
-				log.Warnf("Failed to handle message: %v", err)
-			}
+		}
+
+		isValidRemoteCandidate := make(chan bool, 1)
+		err = c.agent.run(func(agent *Agent) {
+			isValidRemoteCandidate <- agent.noSTUNSeen(c, srcAddr)
+		})
+
+		if err != nil {
+			log.Warnf("Failed to handle message: %v", err)
+		} else if !<-isValidRemoteCandidate {
+			log.Warnf("Discarded message from %s, not a valid remote candidate", c.addr())
 		}
 
 		// NOTE This will return packetio.ErrFull if the buffer ever manages to fill up.