ROX-32269: migrate to 2.0 NVD API

.github/workflows/cron.yml

@@ -19,7 +19,7 @@ jobs:
       uses: actions/checkout@v3
 
     - name: Run croncheck
-      run: go run croncheck/*
+      run: go run ./croncheck
 
     - name: The job has failed
       if: ${{ failure() }}

.github/workflows/pr.yml

@@ -25,4 +25,4 @@ jobs:
       run: go test -v ./...
 
     - name: Run croncheck
-      run: go run croncheck/*
+      run: go run ./croncheck

croncheck/main.go

@@ -8,7 +8,6 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
-	"time"
 
 	"github.com/ghodss/yaml"
 	"github.com/google/go-github/v68/github"
@@ -208,18 +207,18 @@ func main() {
 		delete(issueLinks, knownMissing)
 	}
 
-	// NVD API rate limits requests to 5 per 30-second window.
-	// See https://nvd.nist.gov/developers/start-here for more
-	// information.
-	l := rate.NewLimiter(rate.Every(6*time.Second), 1)
+	interval := nvdRateLimitInterval()
+	log.Printf("Limiting NVD API requests to 1 every: %s", interval)
+
+	l := rate.NewLimiter(rate.Every(interval), 1)
 
 	// Iterate over missing issue links and see if CVE is valid
 	for link, linkRef := range issueLinks {
-		err := l.Wait(context.Background())
-		if err != nil {
-			log.Fatalf("waiting for rate limit: %v", err)
-		}
 		if linkRef.stillNeeded && linkRef.cve != "" {
+			err := l.Wait(context.Background())
+			if err != nil {
+				log.Fatalf("waiting for rate limit: %v", err)
+			}
 			valid, err := validateNVDCVEIsEvaluated(linkRef.cve)
 			if err != nil {
 				log.Fatalf("could not validate NVD CVE %s: %v", linkRef.cve, err)

croncheck/nvd.go

@@ -6,70 +6,98 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"os"
+	"time"
 
-	"github.com/facebookincubator/nvdtools/cvefeed/nvd/schema"
+	"github.com/facebookincubator/nvdtools/cveapi/nvd/schema"
 )
 
 // baseURL is NVD's endpoint base URL.
 var baseURL *url.URL
 
 func init() {
-    var err error
-    baseURL, err = url.Parse("https://services.nvd.nist.gov/rest/json/cves/2.0")
-    if err != nil {
-        panic(err)
-    }
+	var err error
+	baseURL, err = url.Parse("https://services.nvd.nist.gov/rest/json/cves/2.0")
+	if err != nil {
+		panic(err)
+	}
 }
 
-
 func validateNVDCVEIsEvaluated(cve string) (bool, error) {
 	// Create a copy of the baseURL and update its query parameters.
 	q := baseURL.Query()
 	q.Set("cveId", cve)
 	u := *baseURL
 	u.RawQuery = q.Encode()
 
-	resp, err := http.Get(u.String())
+	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
 	if err != nil {
 		return false, err
 	}
-	defer resp.Body.Close()
 
-	respMap := make(map[string]interface{})
+	if apiKey, found := os.LookupEnv("NVD_API_KEY"); found {
+		req.Header.Add("apiKey", apiKey)
+	}
 
-	data, err := io.ReadAll(resp.Body)
+	client := &http.Client{}
+	resp, err := client.Do(req)
 	if err != nil {
 		return false, err
 	}
-	if err := json.Unmarshal(data, &respMap); err != nil {
-		return false, fmt.Errorf("unmarshalling NVD response body: %w", err)
-	}
-	data, err = json.Marshal(respMap["result"])
+	defer resp.Body.Close()
+
+	data, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return false, err
 	}
 
-	var result schema.NVDCVEFeedJSON10
+	var result schema.CVEAPIJSON20
 	if err := json.Unmarshal(data, &result); err != nil {
 		return false, fmt.Errorf("unmarshalling NVD response result: %w", err)
 	}
 	// CVE not found
-	if len(result.CVEItems) == 0 {
+	if result.TotalResults == 0 {
 		return false, nil
 	}
-	if len(result.CVEItems) > 1 {
-		return false, fmt.Errorf("unexpected number of CVE items (%d) for %s", len(result.CVEItems), cve)
+	if result.TotalResults > 1 {
+		return false, fmt.Errorf("unexpected number of CVE items (%d) for %s", result.TotalResults, cve)
 	}
-	fullCVE := result.CVEItems[0]
-	if fullCVE.Impact.BaseMetricV2 == nil && fullCVE.Impact.BaseMetricV3 == nil {
+	fullCVE := result.Vulnerabilities[0].CVE
+	if fullCVE.Metrics == nil || (len(fullCVE.Metrics.CvssMetricV2) == 0 &&
+		len(fullCVE.Metrics.CvssMetricV30) == 0 &&
+		len(fullCVE.Metrics.CvssMetricV31) == 0) {
 		return false, nil
 	}
 
-	if fullCVE.Impact.BaseMetricV2 != nil && fullCVE.Impact.BaseMetricV2.CVSSV2 != nil {
-		return true, nil
+	// Verify CVSS data exists.
+	for _, metric := range fullCVE.Metrics.CvssMetricV2 {
+		if metric.CvssData != nil {
+			return true, nil
+		}
 	}
-	if fullCVE.Impact.BaseMetricV3 != nil && fullCVE.Impact.BaseMetricV3.CVSSV3 != nil {
-		return true, nil
+	for _, metric := range fullCVE.Metrics.CvssMetricV30 {
+		if metric.CvssData != nil {
+			return true, nil
+		}
 	}
+	for _, metric := range fullCVE.Metrics.CvssMetricV31 {
+		if metric.CvssData != nil {
+			return true, nil
+		}
+	}
+
 	return false, nil
 }
+
+// nvdRateLimitInterval returns an interval to limit rate of requests
+// to the NVD API. NVD allows 50 requests per 30-second window
+// with an API key or 5 per 30 seconds without an API key.
+// See https://nvd.nist.gov/developers/start-here for more information.
+func nvdRateLimitInterval() time.Duration {
+	window := 30 * time.Second
+	if _, found := os.LookupEnv("NVD_API_KEY"); found {
+		return window / 50 // 50 per 30 seconds (one request per 600ms)
+	}
+
+	return window / 5 // 5 per 30 seconds (one req per 6s)
+}

croncheck/nvd_test.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNVDRateLimitInterval(t *testing.T) {
+	t.Run("With NVD API key", func(t *testing.T) {
+		t.Setenv("NVD_API_KEY", "SOMETHING")
+
+		interval := nvdRateLimitInterval()
+
+		// 50 per 30 seconds
+		assert.Equal(t, 30*time.Second, (interval * 50))
+		assert.Equal(t, 600*time.Millisecond, interval)
+	})
+
+	t.Run("Without NVD API key", func(t *testing.T) {
+		interval := nvdRateLimitInterval()
+
+		// 5 per 30 seconds
+		assert.Equal(t, 30*time.Second, (interval * 5))
+		assert.Equal(t, 6*time.Second, interval)
+	})
+}

go.mod

@@ -17,3 +17,5 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
+
+replace github.com/facebookincubator/nvdtools => github.com/stackrox/nvdtools v0.0.0-20231111002313-57e262e4797e

go.sum

@@ -106,8 +106,6 @@ github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPO
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws=
 github.com/facebookincubator/flog v0.0.0-20190930132826-d2511d0ce33c/go.mod h1:QGzNH9ujQ2ZUr/CjDGZGWeDAVStrWNjHeEcjJL96Nuk=
-github.com/facebookincubator/nvdtools v0.1.5 h1:jbmDT1nd6+k+rlvKhnkgMokrCAzHoASWE5LtHbX2qFQ=
-github.com/facebookincubator/nvdtools v0.1.5/go.mod h1:Kh55SAWnjckS96TBSrXI99KrEKH4iB0OJby3N8GRJO4=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
@@ -325,6 +323,8 @@ github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM=
+github.com/stackrox/nvdtools v0.0.0-20231111002313-57e262e4797e h1:hLHHK1pGLpRvEbER2cJZekLeMnL+nMn3C37CVUhameM=
+github.com/stackrox/nvdtools v0.0.0-20231111002313-57e262e4797e/go.mod h1:Kh55SAWnjckS96TBSrXI99KrEKH4iB0OJby3N8GRJO4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=