diff --git a/pkg/remote/sshclient.go b/pkg/remote/sshclient.go
index e8c4c0f0..a97b28f9 100644
--- a/pkg/remote/sshclient.go
+++ b/pkg/remote/sshclient.go
@@ -111,10 +111,6 @@ func createPublicKeyCallback(connCtx context.Context, sshKeywords *SshKeywords,
 		}
 
 		unencryptedPrivateKey, err := ssh.ParseRawPrivateKey(privateKey)
-		if _, ok := err.(*ssh.PassphraseMissingError); !ok {
-			// skip this key and try with the next
-			return createDummySigner()
-		}
 		if err == nil {
 			signer, err := ssh.NewSignerFromKey(unencryptedPrivateKey)
 			if err == nil {
@@ -126,6 +122,10 @@ func createPublicKeyCallback(connCtx context.Context, sshKeywords *SshKeywords,
 				return []ssh.Signer{signer}, err
 			}
 		}
+		if _, ok := err.(*ssh.PassphraseMissingError); !ok {
+			// skip this key and try with the next
+			return createDummySigner()
+		}
 
 		// batch mode deactivates user input
 		if sshKeywords.BatchMode {