From 94156f66748b1c4eb6aa18680a29ce91fd704b88 Mon Sep 17 00:00:00 2001 From: David Caravello <119438707+dcaravel@users.noreply.github.com> Date: Thu, 11 Dec 2025 12:45:38 -0600 Subject: [PATCH 1/5] migrate to 2.0 NVD API --- croncheck/main.go | 16 ++++++++---- croncheck/nvd.go | 65 +++++++++++++++++++++++++++++------------------ go.mod | 2 ++ go.sum | 4 +-- 4 files changed, 55 insertions(+), 32 deletions(-) diff --git a/croncheck/main.go b/croncheck/main.go index 52d5705..73bde77 100644 --- a/croncheck/main.go +++ b/croncheck/main.go @@ -211,15 +211,21 @@ func main() { // NVD API rate limits requests to 5 per 30-second window. // See https://nvd.nist.gov/developers/start-here for more // information. - l := rate.NewLimiter(rate.Every(6*time.Second), 1) + interval := 6 * time.Second // 5 per 30 seconds + if _, found := os.LookupEnv("NVD_API_KEY"); found { + interval = 50 / 30 * time.Second // 50 requests per seconds + } + log.Printf("Limiting NVD API requests to 1 every: %s", interval) + + l := rate.NewLimiter(rate.Every(interval), 1) // Iterate over missing issue links and see if CVE is valid for link, linkRef := range issueLinks { - err := l.Wait(context.Background()) - if err != nil { - log.Fatalf("waiting for rate limit: %v", err) - } if linkRef.stillNeeded && linkRef.cve != "" { + err := l.Wait(context.Background()) + if err != nil { + log.Fatalf("waiting for rate limit: %v", err) + } valid, err := validateNVDCVEIsEvaluated(linkRef.cve) if err != nil { log.Fatalf("could not validate NVD CVE %s: %v", linkRef.cve, err) diff --git a/croncheck/nvd.go b/croncheck/nvd.go index 1e6f27f..f51f63d 100644 --- a/croncheck/nvd.go +++ b/croncheck/nvd.go @@ -6,22 +6,22 @@ import ( "io" "net/http" "net/url" + "os" - "github.com/facebookincubator/nvdtools/cvefeed/nvd/schema" + "github.com/facebookincubator/nvdtools/cveapi/nvd/schema" ) // baseURL is NVD's endpoint base URL. var baseURL *url.URL func init() { - var err error - baseURL, err = url.Parse("https://services.nvd.nist.gov/rest/json/cves/2.0") - if err != nil { - panic(err) - } + var err error + baseURL, err = url.Parse("https://services.nvd.nist.gov/rest/json/cves/2.0") + if err != nil { + panic(err) + } } - func validateNVDCVEIsEvaluated(cve string) (bool, error) { // Create a copy of the baseURL and update its query parameters. q := baseURL.Query() @@ -29,47 +29,62 @@ func validateNVDCVEIsEvaluated(cve string) (bool, error) { u := *baseURL u.RawQuery = q.Encode() - resp, err := http.Get(u.String()) + req, err := http.NewRequest(http.MethodGet, u.String(), nil) if err != nil { return false, err } - defer resp.Body.Close() - respMap := make(map[string]interface{}) + if apiKey, found := os.LookupEnv("NVD_API_KEY"); found { + req.Header.Add("apiKey", apiKey) + } - data, err := io.ReadAll(resp.Body) + client := &http.Client{} + // resp, err := http.Get(u.String()) + resp, err := client.Do(req) if err != nil { return false, err } - if err := json.Unmarshal(data, &respMap); err != nil { - return false, fmt.Errorf("unmarshalling NVD response body: %w", err) - } - data, err = json.Marshal(respMap["result"]) + defer resp.Body.Close() + + data, err := io.ReadAll(resp.Body) if err != nil { return false, err } - var result schema.NVDCVEFeedJSON10 + var result schema.CVEAPIJSON20 if err := json.Unmarshal(data, &result); err != nil { return false, fmt.Errorf("unmarshalling NVD response result: %w", err) } // CVE not found - if len(result.CVEItems) == 0 { + if result.TotalResults == 0 { return false, nil } - if len(result.CVEItems) > 1 { - return false, fmt.Errorf("unexpected number of CVE items (%d) for %s", len(result.CVEItems), cve) + if result.TotalResults > 1 { + return false, fmt.Errorf("unexpected number of CVE items (%d) for %s", result.TotalResults, cve) } - fullCVE := result.CVEItems[0] - if fullCVE.Impact.BaseMetricV2 == nil && fullCVE.Impact.BaseMetricV3 == nil { + fullCVE := result.Vulnerabilities[0].CVE + if fullCVE.Metrics == nil || (len(fullCVE.Metrics.CvssMetricV2) == 0 && + len(fullCVE.Metrics.CvssMetricV30) == 0 && + len(fullCVE.Metrics.CvssMetricV31) == 0) { return false, nil } - if fullCVE.Impact.BaseMetricV2 != nil && fullCVE.Impact.BaseMetricV2.CVSSV2 != nil { - return true, nil + // Verify CVSS data exists. + for _, metric := range fullCVE.Metrics.CvssMetricV2 { + if metric.CvssData != nil { + return true, nil + } + } + for _, metric := range fullCVE.Metrics.CvssMetricV30 { + if metric.CvssData != nil { + return true, nil + } } - if fullCVE.Impact.BaseMetricV3 != nil && fullCVE.Impact.BaseMetricV3.CVSSV3 != nil { - return true, nil + for _, metric := range fullCVE.Metrics.CvssMetricV31 { + if metric.CvssData != nil { + return true, nil + } } + return false, nil } diff --git a/go.mod b/go.mod index 8a39387..f529196 100644 --- a/go.mod +++ b/go.mod @@ -17,3 +17,5 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +replace github.com/facebookincubator/nvdtools => github.com/stackrox/nvdtools v0.0.0-20231111002313-57e262e4797e diff --git a/go.sum b/go.sum index fb49c8b..efa157c 100644 --- a/go.sum +++ b/go.sum @@ -106,8 +106,6 @@ github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPO github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= github.com/facebookincubator/flog v0.0.0-20190930132826-d2511d0ce33c/go.mod h1:QGzNH9ujQ2ZUr/CjDGZGWeDAVStrWNjHeEcjJL96Nuk= -github.com/facebookincubator/nvdtools v0.1.5 h1:jbmDT1nd6+k+rlvKhnkgMokrCAzHoASWE5LtHbX2qFQ= -github.com/facebookincubator/nvdtools v0.1.5/go.mod h1:Kh55SAWnjckS96TBSrXI99KrEKH4iB0OJby3N8GRJO4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= @@ -325,6 +323,8 @@ github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= +github.com/stackrox/nvdtools v0.0.0-20231111002313-57e262e4797e h1:hLHHK1pGLpRvEbER2cJZekLeMnL+nMn3C37CVUhameM= +github.com/stackrox/nvdtools v0.0.0-20231111002313-57e262e4797e/go.mod h1:Kh55SAWnjckS96TBSrXI99KrEKH4iB0OJby3N8GRJO4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= From 2619c14969ce1c61850b1c2633b2fe02d31d083a Mon Sep 17 00:00:00 2001 From: David Caravello <119438707+dcaravel@users.noreply.github.com> Date: Thu, 11 Dec 2025 12:54:07 -0600 Subject: [PATCH 2/5] cleanup comments / typos --- croncheck/main.go | 2 +- croncheck/nvd.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/croncheck/main.go b/croncheck/main.go index 73bde77..da743ba 100644 --- a/croncheck/main.go +++ b/croncheck/main.go @@ -213,7 +213,7 @@ func main() { // information. interval := 6 * time.Second // 5 per 30 seconds if _, found := os.LookupEnv("NVD_API_KEY"); found { - interval = 50 / 30 * time.Second // 50 requests per seconds + interval = 50 / 30 * time.Second // 50 requests per 30 seconds } log.Printf("Limiting NVD API requests to 1 every: %s", interval) diff --git a/croncheck/nvd.go b/croncheck/nvd.go index f51f63d..b055065 100644 --- a/croncheck/nvd.go +++ b/croncheck/nvd.go @@ -39,7 +39,6 @@ func validateNVDCVEIsEvaluated(cve string) (bool, error) { } client := &http.Client{} - // resp, err := http.Get(u.String()) resp, err := client.Do(req) if err != nil { return false, err From 300e8ab8911758185352e406339a3d93cacdf1b8 Mon Sep 17 00:00:00 2001 From: David Caravello <119438707+dcaravel@users.noreply.github.com> Date: Thu, 11 Dec 2025 12:55:00 -0600 Subject: [PATCH 3/5] more cleanup --- croncheck/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/croncheck/main.go b/croncheck/main.go index da743ba..b7a4164 100644 --- a/croncheck/main.go +++ b/croncheck/main.go @@ -213,7 +213,7 @@ func main() { // information. interval := 6 * time.Second // 5 per 30 seconds if _, found := os.LookupEnv("NVD_API_KEY"); found { - interval = 50 / 30 * time.Second // 50 requests per 30 seconds + interval = 50 / 30 * time.Second // 50 per 30 seconds } log.Printf("Limiting NVD API requests to 1 every: %s", interval) From 863a5217f036417eded9ad5f8808c2a6dca9febc Mon Sep 17 00:00:00 2001 From: David Caravello <119438707+dcaravel@users.noreply.github.com> Date: Fri, 12 Dec 2025 16:20:19 -0600 Subject: [PATCH 4/5] fix interval calc --- croncheck/main.go | 20 +++++++++++++------- croncheck/main_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 7 deletions(-) create mode 100644 croncheck/main_test.go diff --git a/croncheck/main.go b/croncheck/main.go index b7a4164..8b9e914 100644 --- a/croncheck/main.go +++ b/croncheck/main.go @@ -208,13 +208,7 @@ func main() { delete(issueLinks, knownMissing) } - // NVD API rate limits requests to 5 per 30-second window. - // See https://nvd.nist.gov/developers/start-here for more - // information. - interval := 6 * time.Second // 5 per 30 seconds - if _, found := os.LookupEnv("NVD_API_KEY"); found { - interval = 50 / 30 * time.Second // 50 per 30 seconds - } + interval := rateLimitInterval() log.Printf("Limiting NVD API requests to 1 every: %s", interval) l := rate.NewLimiter(rate.Every(interval), 1) @@ -249,3 +243,15 @@ func main() { } } } + +// NVD API rate limits requests to 5 per 30-second window without an API +// key or 50 per 30-second window with an API key. +// See https://nvd.nist.gov/developers/start-here for more information. +func rateLimitInterval() time.Duration { + window := 30 * time.Second + if _, found := os.LookupEnv("NVD_API_KEY"); found { + return window / 50 // 50 per 30 seconds (one request per 600ms) + } + + return window / 5 // 5 per 30 seconds (one req per 6s) +} diff --git a/croncheck/main_test.go b/croncheck/main_test.go new file mode 100644 index 0000000..7c50f21 --- /dev/null +++ b/croncheck/main_test.go @@ -0,0 +1,30 @@ +package main + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestRateLimitIntervale(t *testing.T) { + + t.Run("With NVD API key", func(t *testing.T) { + t.Setenv("NVD_API_KEY", "SOMETHING") + + interval := rateLimitInterval() + + // 50 per 30 seconds + assert.Equal(t, 30*time.Second, (interval * 50)) + assert.Equal(t, 600*time.Millisecond, interval) + }) + + t.Run("Without NVD API key", func(t *testing.T) { + interval := rateLimitInterval() + + // 5 per 30 seconds + assert.Equal(t, 30*time.Second, (interval * 5)) + assert.Equal(t, 6*time.Second, interval) + }) + +} From f599bd584f82411126a71c03a314c5754bdd5662 Mon Sep 17 00:00:00 2001 From: David Caravello <119438707+dcaravel@users.noreply.github.com> Date: Fri, 12 Dec 2025 16:27:29 -0600 Subject: [PATCH 5/5] fix broken tests --- .github/workflows/cron.yml | 2 +- .github/workflows/pr.yml | 2 +- croncheck/main.go | 15 +-------------- croncheck/nvd.go | 14 ++++++++++++++ croncheck/{main_test.go => nvd_test.go} | 8 +++----- 5 files changed, 20 insertions(+), 21 deletions(-) rename croncheck/{main_test.go => nvd_test.go} (79%) diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index 4077fbc..553f2da 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v3 - name: Run croncheck - run: go run croncheck/* + run: go run ./croncheck - name: The job has failed if: ${{ failure() }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 8fe9122..91b6ad7 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -25,4 +25,4 @@ jobs: run: go test -v ./... - name: Run croncheck - run: go run croncheck/* + run: go run ./croncheck diff --git a/croncheck/main.go b/croncheck/main.go index 8b9e914..1cb687f 100644 --- a/croncheck/main.go +++ b/croncheck/main.go @@ -8,7 +8,6 @@ import ( "path/filepath" "regexp" "strings" - "time" "github.com/ghodss/yaml" "github.com/google/go-github/v68/github" @@ -208,7 +207,7 @@ func main() { delete(issueLinks, knownMissing) } - interval := rateLimitInterval() + interval := nvdRateLimitInterval() log.Printf("Limiting NVD API requests to 1 every: %s", interval) l := rate.NewLimiter(rate.Every(interval), 1) @@ -243,15 +242,3 @@ func main() { } } } - -// NVD API rate limits requests to 5 per 30-second window without an API -// key or 50 per 30-second window with an API key. -// See https://nvd.nist.gov/developers/start-here for more information. -func rateLimitInterval() time.Duration { - window := 30 * time.Second - if _, found := os.LookupEnv("NVD_API_KEY"); found { - return window / 50 // 50 per 30 seconds (one request per 600ms) - } - - return window / 5 // 5 per 30 seconds (one req per 6s) -} diff --git a/croncheck/nvd.go b/croncheck/nvd.go index b055065..47e2799 100644 --- a/croncheck/nvd.go +++ b/croncheck/nvd.go @@ -7,6 +7,7 @@ import ( "net/http" "net/url" "os" + "time" "github.com/facebookincubator/nvdtools/cveapi/nvd/schema" ) @@ -87,3 +88,16 @@ func validateNVDCVEIsEvaluated(cve string) (bool, error) { return false, nil } + +// nvdRateLimitInterval returns an interval to limit rate of requests +// to the NVD API. NVD allows 50 requests per 30-second window +// with an API key or 5 per 30 seconds without an API key. +// See https://nvd.nist.gov/developers/start-here for more information. +func nvdRateLimitInterval() time.Duration { + window := 30 * time.Second + if _, found := os.LookupEnv("NVD_API_KEY"); found { + return window / 50 // 50 per 30 seconds (one request per 600ms) + } + + return window / 5 // 5 per 30 seconds (one req per 6s) +} diff --git a/croncheck/main_test.go b/croncheck/nvd_test.go similarity index 79% rename from croncheck/main_test.go rename to croncheck/nvd_test.go index 7c50f21..13b1f32 100644 --- a/croncheck/main_test.go +++ b/croncheck/nvd_test.go @@ -7,12 +7,11 @@ import ( "github.com/stretchr/testify/assert" ) -func TestRateLimitIntervale(t *testing.T) { - +func TestNVDRateLimitInterval(t *testing.T) { t.Run("With NVD API key", func(t *testing.T) { t.Setenv("NVD_API_KEY", "SOMETHING") - interval := rateLimitInterval() + interval := nvdRateLimitInterval() // 50 per 30 seconds assert.Equal(t, 30*time.Second, (interval * 50)) @@ -20,11 +19,10 @@ func TestRateLimitIntervale(t *testing.T) { }) t.Run("Without NVD API key", func(t *testing.T) { - interval := rateLimitInterval() + interval := nvdRateLimitInterval() // 5 per 30 seconds assert.Equal(t, 30*time.Second, (interval * 5)) assert.Equal(t, 6*time.Second, interval) }) - }