diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index d6e1c21..e64e70d 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -21,6 +21,6 @@ jobs: - name: Run golangci-lint uses: golangci/golangci-lint-action@aaa42aa0628b4ae2578232a66b541047968fac86 # v6.1.0 with: - version: latest - skip-go-installation: true + version: v2.10.1 args: --timeout=5m + only-new-issues: true diff --git a/.golangci.yml b/.golangci.yml index f28e966..fd43fbf 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,9 +1,11 @@ +version: "2" + linters-settings: dupl: threshold: 100 funlen: lines: 150 - statements: 50 + statements: 80 goconst: min-len: 2 min-occurrences: 2 @@ -22,16 +24,17 @@ linters-settings: - wrapperFunc gocyclo: min-complexity: 15 - goimports: - local-prefixes: github.com/golangci/golangci-lint - golint: - min-confidence: 0 - gomnd: - settings: - mnd: - checks: argument,case,condition,return + revive: + confidence: 0 + mnd: + checks: + - argument + - case + - condition + - return govet: - shadow: true + enable: + - shadow settings: printf: funcs: @@ -41,8 +44,6 @@ linters-settings: - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf lll: line-length: 140 - maligned: - suggest-new: true misspell: locale: US @@ -54,35 +55,22 @@ linters: - dupl - errcheck - copyloopvar - - funlen - gochecknoinits - goconst - gocritic - gocyclo - - gofmt - - goimports - goprintffuncname - - gosec - - gosimple - govet - ineffassign - - lll - misspell - nakedret - - revive - rowserrcheck - staticcheck - - stylecheck - unconvert - unparam - unused - whitespace - don't enable: - - typecheck - - typechecker - - depguard - issues: exclude-dirs: - test/testdata_etc @@ -94,7 +82,3 @@ issues: linters: - revive -service: - golangci-lint-version: 1.63.4 # Updated to match latest stable version - prepare: - - echo "Custom preparation commands go here" diff --git a/internal/app/export/transform.go b/internal/app/export/transform.go index 439a689..fbababf 100644 --- a/internal/app/export/transform.go +++ b/internal/app/export/transform.go @@ -76,19 +76,20 @@ func TransformXMLInstallationMappings(installationMappings *soap.GetInstallation } for _, e := range installationMappings.GetInstallationSettingsResult.InstallationSettingsList.InstallationSetting { - if e.Name == InstallationEngineServiceName { + switch e.Name { + case InstallationEngineServiceName: out = append(out, &common.InstallationMapping{ Name: InstallationEngineServiceName, Version: e.Version, Hotfix: e.Hotfix, }) - } else if e.Name == installationScansManagerName { + case installationScansManagerName: scansManager = &common.InstallationMapping{ Name: InstallationEngineServiceName, Version: e.Version, Hotfix: e.Hotfix, } - } else if e.Name == installationContentPackName { + case installationContentPackName: out = append(out, &common.InstallationMapping{ Name: e.Name, Version: e.Version, diff --git a/internal/app/permissions/permissions.go b/internal/app/permissions/permissions.go index ec0e30d..2c89be6 100644 --- a/internal/app/permissions/permissions.go +++ b/internal/app/permissions/permissions.go @@ -34,11 +34,12 @@ func GetFromExportOptions(exportOptions []string) []interface{} { resultsPermissions := []string{useOdataPermission, generateScanReportPermission, viewResults} for _, exportOption := range exportOptions { - if exportOption == export.UsersOption { + switch exportOption { + case export.UsersOption: output = append(output, usersPermissions...) - } else if exportOption == export.TeamsOption { + case export.TeamsOption: output = append(output, teamsPermissions...) - } else if exportOption == export.ResultsOption { + case export.ResultsOption: output = append(output, resultsPermissions...) } } diff --git a/internal/app/resultsmapping/generate_csv.go b/internal/app/resultsmapping/generate_csv.go index 07ee8a5..2c284dc 100644 --- a/internal/app/resultsmapping/generate_csv.go +++ b/internal/app/resultsmapping/generate_csv.go @@ -52,6 +52,6 @@ func WriteAllToSanitizedCsv(records [][]string) []byte { } func sanitize(cell string) string { - escapedCell := strings.Replace(cell, `"`, `""`, -1) + escapedCell := strings.ReplaceAll(cell, `"`, `""`) return fmt.Sprintf(`"'%s"`, escapedCell) } diff --git a/internal/integration/rest/apiclient.go b/internal/integration/rest/apiclient.go index 951322b..f2ac874 100644 --- a/internal/integration/rest/apiclient.go +++ b/internal/integration/rest/apiclient.go @@ -112,7 +112,8 @@ func (c *APIClient) Authenticate(username, password string) error { Int("statusCode", resp.StatusCode). Logger() - if resp.StatusCode == http.StatusOK { + switch resp.StatusCode { + case http.StatusOK: responseBody, ioErr := io.ReadAll(resp.Body) if ioErr != nil { logger.Debug().Err(ioErr).Msg("authenticate ok failed read response") @@ -128,7 +129,7 @@ func (c *APIClient) Authenticate(username, password string) error { return fmt.Errorf("authentication error - could not decode response") } return nil - } else if resp.StatusCode == http.StatusBadRequest { + case http.StatusBadRequest: responseBody, ioErr := io.ReadAll(resp.Body) if ioErr != nil { logger.Debug().Err(ioErr).Msg("authenticate bad request failed to read response") diff --git a/internal/process.go b/internal/process.go index e6ee265..c72e6c9 100644 --- a/internal/process.go +++ b/internal/process.go @@ -2,6 +2,7 @@ package internal import ( "bytes" + "context" "encoding/json" "encoding/xml" "fmt" @@ -47,12 +48,13 @@ import ( ) const ( - scansFileName = "%d.xml" - scansMetadataFileName = "%d.json" - resultsPageLimit = 10000 - httpRetryWaitMin = 1 * time.Second //nolint:revive - httpRetryWaitMax = 30 * time.Second - httpRetryMax = 4 + scansFileName = "%d.xml" + scansMetadataFileName = "%d.json" + resultsPageLimit = 10000 + httpRetryWaitMin = 1 * time.Second //nolint:revive + httpRetryWaitMax = 30 * time.Second + httpRetryMax = 8 + httpRateLimitRetryDelay = 200 * time.Millisecond scanReportCreateAttempts = 10 scanReportCreateMinSleep = 1 * time.Second @@ -1068,8 +1070,8 @@ func getRetryHTTPClient() *retryablehttp.Client { RetryWaitMin: httpRetryWaitMin, RetryWaitMax: httpRetryWaitMax, RetryMax: httpRetryMax, - CheckRetry: retryablehttp.DefaultRetryPolicy, - Backoff: retryablehttp.DefaultBackoff, + CheckRetry: customRetryPolicy, + Backoff: customBackoff, RequestLogHook: func(_ retryablehttp.Logger, request *http.Request, i int) { log.Debug(). Str("method", request.Method). @@ -1087,6 +1089,73 @@ func getRetryHTTPClient() *retryablehttp.Client { } } +// customRetryPolicy determines whether a request should be retried, with special handling for rate limiting +func customRetryPolicy(ctx context.Context, resp *http.Response, err error) (bool, error) { + // Use default policy first + shouldRetry, checkErr := retryablehttp.DefaultRetryPolicy(ctx, resp, err) + + // If default policy says retry, return that + if shouldRetry { + return true, checkErr + } + + // Additionally check for 429 Too Many Requests + if resp != nil && resp.StatusCode == http.StatusTooManyRequests { + log.Debug(). + Str("url", resp.Request.URL.String()). + Msg("Rate limited (429), will retry with backoff") + return true, nil + } + + return shouldRetry, checkErr +} + +// customBackoff provides exponential backoff with special handling for rate limiting +func customBackoff(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration { + // If we got a 429, check for Retry-After header first + if resp != nil && resp.StatusCode == http.StatusTooManyRequests { + // Check for Retry-After header (can be in seconds or HTTP date) + if retryAfter := resp.Header.Get("Retry-After"); retryAfter != "" { + // Try parsing as seconds first + if seconds, err := strconv.Atoi(retryAfter); err == nil && seconds > 0 { + delay := time.Duration(seconds) * time.Second + if delay > max { + delay = max + } + log.Debug(). + Dur("delay", delay). + Int("attempt", attemptNum). + Str("retry_after", retryAfter). + Msg("Rate limit backoff delay from Retry-After header") + return delay + } + } + + // If no Retry-After header, use exponential backoff starting at httpRateLimitRetryDelay + // Cap attemptNum to prevent overflow + safeAttempt := attemptNum + if safeAttempt > 10 { + safeAttempt = 10 + } + baseDelay := httpRateLimitRetryDelay * time.Duration(1< max { + return max + } + + log.Debug(). + Dur("delay", baseDelay). + Int("attempt", attemptNum). + Msg("Rate limit backoff delay") + + return baseDelay + } + + // Otherwise use default exponential backoff + return retryablehttp.DefaultBackoff(min, max, attemptNum, resp) +} + func addQueryMappingFile(queryMappingProvider interfaces.QueryMappingRepo, exporter export2.Exporter) error { mapping := querymapping.MapSource{ Mappings: queryMappingProvider.GetMapping(), @@ -1176,7 +1245,9 @@ func fetchCustomExtensions(args *Args, exporter export2.Exporter) error { // Split the input string by space to get language, extension and language group parts := strings.Split(args.CustomExtensions, " ") if len(parts) < 3 || len(parts)%3 != 0 { - return fmt.Errorf("invalid custom extensions format. Expected: Language Extension LanguageGroup [Language Extension LanguageGroup ...]") + return fmt.Errorf( + "invalid custom extensions format. Expected: Language Extension LanguageGroup [...]", + ) } customExtensions := &CustomExtensionsList{ @@ -1214,7 +1285,12 @@ func fetchCustomExtensions(args *Args, exporter export2.Exporter) error { return nil } -func fetchProjectExcludeSettings(client rest.Client, exporter export2.Exporter, projects []*rest.Project, projectIDs string) error { +func fetchProjectExcludeSettings( + client rest.Client, + exporter export2.Exporter, + projects []*rest.Project, + projectIDs string, +) error { var allExcludeSettings []*rest.ProjectExcludeSettings // If specific project IDs are provided, filter the projects @@ -1233,7 +1309,10 @@ func fetchProjectExcludeSettings(client rest.Client, exporter export2.Exporter, log.Info().Int("projectID", project.ID).Msg("collecting project exclude settings") excludeSettings, err := client.GetProjectExcludeSettings(project.ID) if err != nil { - return errors.Wrapf(err, "failed to get exclude settings for project %d", project.ID) + log.Error().Err(err). + Int("projectID", project.ID). + Msg("Skipping project due to error getting exclude settings") + continue } allExcludeSettings = append(allExcludeSettings, excludeSettings) } @@ -1244,7 +1323,10 @@ func fetchProjectExcludeSettings(client rest.Client, exporter export2.Exporter, log.Info().Int("projectID", project.ID).Msg("collecting project exclude settings") excludeSettings, err := client.GetProjectExcludeSettings(project.ID) if err != nil { - return errors.Wrapf(err, "failed to get exclude settings for project %d", project.ID) + log.Error().Err(err). + Int("projectID", project.ID). + Msg("Skipping project due to error getting exclude settings") + continue } allExcludeSettings = append(allExcludeSettings, excludeSettings) }