detectors: emit unverified results instead of dropping them on verify…

pkg/detectors/algoliaadminkey/algoliaadminkey.go

@@ -68,30 +68,27 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	// Test matches.
 	for key := range keyMatches {
 		for id := range idMatches {
-			if invalidHosts.Exists(id) {
-				logger.V(3).Info("Skipping application id: no such host", "host", id)
-				delete(idMatches, id)
-				continue
-			}
-
 			r := detectors.Result{
 				DetectorType: detector_typepb.DetectorType_AlgoliaAdminKey,
 				Raw:          []byte(key),
 				RawV2:        []byte(id + ":" + key),
 			}
 
 			if verify {
-				// Verify if the key is a valid Algolia Admin Key.
-				isVerified, extraData, verificationErr := verifyMatch(ctx, id, key)
-				r.Verified = isVerified
-				r.ExtraData = extraData
-				if verificationErr != nil {
-					if errors.Is(verificationErr, errNoHost) {
-						invalidHosts.Set(id, struct{}{})
-						continue
+				if invalidHosts.Exists(id) {
+					logger.V(3).Info("Skipping verification: cached no such host", "host", id)
+					r.SetVerificationError(errNoHost, key)
+				} else {
+					// Verify if the key is a valid Algolia Admin Key.
+					isVerified, extraData, verificationErr := verifyMatch(ctx, id, key)
+					r.Verified = isVerified
+					r.ExtraData = extraData
+					if verificationErr != nil {
+						if errors.Is(verificationErr, errNoHost) {
+							invalidHosts.Set(id, struct{}{})
+						}
+						r.SetVerificationError(verificationErr, key)
 					}
-
-					r.SetVerificationError(verificationErr, key)
 				}
 			}
 

pkg/detectors/artifactory/artifactory.go

@@ -77,28 +77,24 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 	for token := range uniqueTokens {
 		for url := range uniqueUrls {
-			if invalidHosts.Exists(url) {
-				delete(uniqueUrls, url)
-				continue
-			}
-
 			s1 := detectors.Result{
 				DetectorType: detector_typepb.DetectorType_ArtifactoryAccessToken,
 				Raw:          []byte(token),
 				RawV2:        []byte(token + url),
 			}
 
 			if verify {
-				isVerified, verificationErr := verifyArtifactory(ctx, s.getClient(), url, token)
-				s1.Verified = isVerified
-				if verificationErr != nil {
-					if errors.Is(verificationErr, errNoHost) {
-						invalidHosts.Set(url, struct{}{})
-						continue
+				if invalidHosts.Exists(url) {
+					s1.SetVerificationError(errNoHost, token)
+				} else {
+					isVerified, verificationErr := verifyArtifactory(ctx, s.getClient(), url, token)
+					s1.Verified = isVerified
+					if verificationErr != nil {
+						if errors.Is(verificationErr, errNoHost) {
+							invalidHosts.Set(url, struct{}{})
+						}
+						s1.SetVerificationError(verificationErr, token)
 					}
-
-					s1.SetVerificationError(verificationErr, token)
-
 					if isVerified {
 						s1.AnalysisInfo = map[string]string{
 							"domain": url,

pkg/detectors/azure_cosmosdb/azure_cosmosdb.go

@@ -74,11 +74,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 	for key := range uniqueKeyMatches {
 		for accountUrl := range uniqueAccountMatches {
-			if invalidHosts.Exists(accountUrl) {
-				delete(uniqueAccountMatches, accountUrl)
-				continue
-			}
-
 			s1 := detectors.Result{
 				DetectorType: detector_typepb.DetectorType_AzureCosmosDBKeyIdentifiable,
 				Raw:          []byte(key),
@@ -87,29 +82,31 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 
 			if verify {
-				var verified bool
-				var verificationErr error
-
-				client := s.getClient()
-
-				// perform verification based on db type
-				if strings.Contains(accountUrl, ".documents.azure.com") {
-					verified, verificationErr = verifyCosmosDocumentDB(client, accountUrl, key)
-					s1.ExtraData["DB Type"] = "Document"
-
-				} else if strings.Contains(accountUrl, ".table.cosmos.azure.com") {
-					verified, verificationErr = verifyCosmosTableDB(client, accountUrl, key)
-					s1.ExtraData["DB Type"] = "Table"
-				}
-
-				s1.Verified = verified
-				if verificationErr != nil {
-					if errors.Is(verificationErr, errNoHost) {
-						invalidHosts.Set(accountUrl, struct{}{})
-						continue
+				if invalidHosts.Exists(accountUrl) {
+					s1.SetVerificationError(errNoHost)
+				} else {
+					var verified bool
+					var verificationErr error
+
+					client := s.getClient()
+
+					// perform verification based on db type
+					if strings.Contains(accountUrl, ".documents.azure.com") {
+						verified, verificationErr = verifyCosmosDocumentDB(client, accountUrl, key)
+						s1.ExtraData["DB Type"] = "Document"
+
+					} else if strings.Contains(accountUrl, ".table.cosmos.azure.com") {
+						verified, verificationErr = verifyCosmosTableDB(client, accountUrl, key)
+						s1.ExtraData["DB Type"] = "Table"
 					}
 
-					s1.SetVerificationError(verificationErr)
+					s1.Verified = verified
+					if verificationErr != nil {
+						if errors.Is(verificationErr, errNoHost) {
+							invalidHosts.Set(accountUrl, struct{}{})
+						}
+						s1.SetVerificationError(verificationErr)
+					}
 				}
 			}
 

pkg/detectors/azureapimanagement/repositorykey/repositorykey.go

@@ -59,7 +59,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		uniquePasswordMatches[strings.TrimSpace(matches[1])] = struct{}{}
 	}
 
-EndpointLoop:
 	for urlMatch := range uniqueUrlsMatches {
 		for passwordMatch := range uniquePasswordMatches {
 			s1 := detectors.Result{
@@ -70,18 +69,17 @@ EndpointLoop:
 
 			if verify {
 				if invalidHosts.Exists(urlMatch) {
-					logger.V(3).Info("Skipping invalid registry", "url", urlMatch)
-					continue EndpointLoop
-				}
-
-				isVerified, err := verifyUrlPassword(ctx, urlMatch, azureGitUsername, passwordMatch)
-				s1.Verified = isVerified
-				if err != nil {
-					if errors.Is(err, noSuchHostErr) {
-						invalidHosts.Set(urlMatch, struct{}{})
-						continue EndpointLoop
+					logger.V(3).Info("Skipping verification: cached no such host", "url", urlMatch)
+					s1.SetVerificationError(noSuchHostErr, urlMatch)
+				} else {
+					isVerified, err := verifyUrlPassword(ctx, urlMatch, azureGitUsername, passwordMatch)
+					s1.Verified = isVerified
+					if err != nil {
+						if errors.Is(err, noSuchHostErr) {
+							invalidHosts.Set(urlMatch, struct{}{})
+						}
+						s1.SetVerificationError(err, urlMatch)
 					}
-					s1.SetVerificationError(err, urlMatch)
 				}
 			}
 			results = append(results, s1)

pkg/detectors/azureapimanagementsubscriptionkey/azureapimanagementsubscriptionkey.go

@@ -54,7 +54,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		keyMatchesUnique[strings.TrimSpace(keyMatch[1])] = struct{}{}
 	}
 
-EndpointLoop:
 	for baseUrl := range urlMatchesUnique {
 		for key := range keyMatchesUnique {
 			s1 := detectors.Result{
@@ -65,23 +64,22 @@ EndpointLoop:
 
 			if verify {
 				if invalidHosts.Exists(baseUrl) {
-					logger.V(3).Info("Skipping invalid registry", "baseUrl", baseUrl)
-					continue EndpointLoop
-				}
-
-				client := s.client
-				if client == nil {
-					client = defaultClient
-				}
+					logger.V(3).Info("Skipping verification: cached no such host", "baseUrl", baseUrl)
+					s1.SetVerificationError(noSuchHostErr, baseUrl)
+				} else {
+					client := s.client
+					if client == nil {
+						client = defaultClient
+					}
 
-				isVerified, verificationErr := s.verifyMatch(ctx, client, baseUrl, key)
-				s1.Verified = isVerified
-				if verificationErr != nil {
-					if errors.Is(verificationErr, noSuchHostErr) {
-						invalidHosts.Set(baseUrl, struct{}{})
-						continue EndpointLoop
+					isVerified, verificationErr := s.verifyMatch(ctx, client, baseUrl, key)
+					s1.Verified = isVerified
+					if verificationErr != nil {
+						if errors.Is(verificationErr, noSuchHostErr) {
+							invalidHosts.Set(baseUrl, struct{}{})
+						}
+						s1.SetVerificationError(verificationErr, baseUrl)
 					}
-					s1.SetVerificationError(verificationErr, baseUrl)
 				}
 			}
 

pkg/detectors/azurecontainerregistry/azurecontainerregistry.go

@@ -73,7 +73,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		passwordMatches[p] = struct{}{}
 	}
 
-EndpointLoop:
 	for username := range registryMatches {
 		for password := range passwordMatches {
 			r := detectors.Result{
@@ -85,26 +84,25 @@ EndpointLoop:
 
 			if verify {
 				if invalidHosts.Exists(username) {
-					logger.V(3).Info("Skipping invalid registry", "username", username)
-					continue EndpointLoop
-				}
-
-				client := s.client
-				if client == nil {
-					client = defaultClient
-				}
+					logger.V(3).Info("Skipping verification: cached no such host", "username", username)
+					r.SetVerificationError(noSuchHostErr, password)
+				} else {
+					client := s.client
+					if client == nil {
+						client = defaultClient
+					}
 
-				isVerified, verificationErr := verifyMatch(ctx, client, username, password)
-				if isVerified {
-					delete(passwordMatches, password)
-					r.Verified = true
-				}
-				if verificationErr != nil {
-					if errors.Is(verificationErr, noSuchHostErr) {
-						invalidHosts.Set(username, struct{}{})
-						continue EndpointLoop
+					isVerified, verificationErr := verifyMatch(ctx, client, username, password)
+					if isVerified {
+						delete(passwordMatches, password)
+						r.Verified = true
+					}
+					if verificationErr != nil {
+						if errors.Is(verificationErr, noSuchHostErr) {
+							invalidHosts.Set(username, struct{}{})
+						}
+						r.SetVerificationError(verificationErr, password)
 					}
-					r.SetVerificationError(verificationErr, password)
 				}
 			}
 

pkg/detectors/azuredirectmanagementkey/azuredirectmanagementkey.go

@@ -60,7 +60,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		keyMatchesUnique[strings.TrimSpace(keyMatch[1])] = struct{}{}
 	}
 
-EndpointLoop:
 	for baseUrl, serviceName := range urlMatchesUnique {
 		for key := range keyMatchesUnique {
 			s1 := detectors.Result{
@@ -71,23 +70,22 @@ EndpointLoop:
 
 			if verify {
 				if invalidHosts.Exists(baseUrl) {
-					logger.V(3).Info("Skipping invalid registry", "baseUrl", baseUrl)
-					continue EndpointLoop
-				}
-
-				client := s.client
-				if client == nil {
-					client = defaultClient
-				}
+					logger.V(3).Info("Skipping verification: cached no such host", "baseUrl", baseUrl)
+					s1.SetVerificationError(noSuchHostErr, baseUrl)
+				} else {
+					client := s.client
+					if client == nil {
+						client = defaultClient
+					}
 
-				isVerified, verificationErr := s.verifyMatch(ctx, client, baseUrl, serviceName, key)
-				s1.Verified = isVerified
-				if verificationErr != nil {
-					if errors.Is(verificationErr, noSuchHostErr) {
-						invalidHosts.Set(baseUrl, struct{}{})
-						continue EndpointLoop
+					isVerified, verificationErr := s.verifyMatch(ctx, client, baseUrl, serviceName, key)
+					s1.Verified = isVerified
+					if verificationErr != nil {
+						if errors.Is(verificationErr, noSuchHostErr) {
+							invalidHosts.Set(baseUrl, struct{}{})
+						}
+						s1.SetVerificationError(verificationErr, baseUrl)
 					}
-					s1.SetVerificationError(verificationErr, baseUrl)
 				}
 			}
 

pkg/detectors/azuresastoken/azuresastoken.go

@@ -73,7 +73,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	}
 
 	// Check results.
-UrlLoop:
 	for url, storageAccount := range urlMatchesUnique {
 		for key := range keyMatchesUnique {
 			s1 := detectors.Result{
@@ -84,24 +83,23 @@ UrlLoop:
 
 			if verify {
 				if invalidStorageAccounts.Exists(storageAccount) {
-					logger.V(3).Info("Skipping invalid storage account", "storage account", storageAccount)
-					break
-				}
-
-				client := s.client
-				if client == nil {
-					client = defaultClient
-				}
+					logger.V(3).Info("Skipping verification: cached invalid storage account", "storage account", storageAccount)
+					s1.SetVerificationError(noSuchHostErr, key)
+				} else {
+					client := s.client
+					if client == nil {
+						client = defaultClient
+					}
 
-				isVerified, verificationErr := verifyMatch(ctx, client, url, key, true)
-				s1.Verified = isVerified
+					isVerified, verificationErr := verifyMatch(ctx, client, url, key, true)
+					s1.Verified = isVerified
 
-				if verificationErr != nil {
-					if errors.Is(verificationErr, noSuchHostErr) {
-						invalidStorageAccounts.Set(storageAccount, struct{}{})
-						continue UrlLoop
+					if verificationErr != nil {
+						if errors.Is(verificationErr, noSuchHostErr) {
+							invalidStorageAccounts.Set(storageAccount, struct{}{})
+						}
+						s1.SetVerificationError(verificationErr, key)
 					}
-					s1.SetVerificationError(verificationErr, key)
 				}
 			}