diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 0000000..01b4c9d --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,17 @@ +changelog: + exclude: + authors: + - dependabot + categories: + - title: 🎉 Features + labels: + - "Type: Enhancement" + - title: 🐞 Bugs + labels: + - "Type: Bug" + - title: 🔨 Maintenance + labels: + - "Type: Maintenance" + - title: Other Changes + labels: + - "*" \ No newline at end of file diff --git a/.github/stale.yml b/.github/stale.yml new file mode 100644 index 0000000..381a372 --- /dev/null +++ b/.github/stale.yml @@ -0,0 +1,26 @@ +# Number of days of inactivity before an issue becomes stale +daysUntilStale: 14 + +# Number of days of inactivity before a stale issue is closed +daysUntilClose: 30 + +# Issues with these labels will never be considered stale +# exemptLabels: +# - pinned +# - security + +# Only issues or pull requests with all of these labels are check if stale. +onlyLabels: + - "Status: Abandoned" + - "Type: Question" + +# Label to use when marking as stale +staleLabel: stale + +# Comment to post when marking an issue as stale. Set to `false` to disable +markComment: > + This issue has been automatically marked as stale because it has not had + activity in the past 2 weeks. It will be closed if no further activity occurs. Thank you + for your contributions. +# Comment to post when closing a stale issue. Set to `false` to disable +closeComment: This issue is being closed after 30 days of inactivity. Thank you. \ No newline at end of file diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 0dc2651..c253ab9 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -16,7 +16,7 @@ jobs: os: [ubuntu-latest-16-cores, windows-latest-8-cores, macOS-latest] steps: - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: 1.19 @@ -31,6 +31,10 @@ jobs: run: go test ./... working-directory: . + - name: Running example + run: go run . + working-directory: examples/ + - name: Integration Tests Linux, macOS if: runner.os == 'Linux' || runner.os == 'macOS' env: diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index 7d2e49b..e7114a7 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -16,7 +16,7 @@ jobs: os: [ubuntu-latest-16-cores, windows-latest-8-cores, macOS-latest] steps: - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: 1.19 diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 0fdf3ea..a108128 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -13,13 +13,13 @@ jobs: runs-on: ubuntu-latest-16-cores steps: - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: 1.19 - name: Checkout code uses: actions/checkout@v3 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v3.4.0 + uses: golangci/golangci-lint-action@v3.6.0 with: version: latest args: --timeout 5m diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index 2dc5a16..11c8bff 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -16,7 +16,7 @@ jobs: fetch-depth: 0 - name: "Set up Go" - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: 1.19 diff --git a/Dockerfile b/Dockerfile index 101d761..ee83651 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Base -FROM golang:1.20.2-alpine AS builder +FROM golang:1.20.5-alpine AS builder RUN apk add --no-cache git build-base gcc musl-dev WORKDIR /app @@ -7,9 +7,9 @@ COPY . /app RUN go mod download RUN go build ./cmd/httpx -FROM alpine:3.17.2 +FROM alpine:3.18.2 RUN apk -U upgrade --no-cache \ - && apk add --no-cache bind-tools ca-certificates + && apk add --no-cache bind-tools ca-certificates chromium COPY --from=builder /app/httpx /usr/local/bin/ ENTRYPOINT ["httpx"] \ No newline at end of file diff --git a/README.md b/README.md index 0c47187..4355b1c 100644 --- a/README.md +++ b/README.md @@ -109,15 +109,19 @@ PROBES: -cdn display cdn in use -probe display probe status +HEADLESS: + -ss, -screenshot enable saving screenshot of the page using headless browser + -system-chrome enable using local installed chrome for screenshot + MATCHERS: -mc, -match-code string match response with specified status code (-mc 200,302) -ml, -match-length string match response with specified content length (-ml 100,102) -mlc, -match-line-count string match response body with specified line count (-mlc 423,532) -mwc, -match-word-count string match response body with specified word count (-mwc 43,55) -mfc, -match-favicon string[] match response with specified favicon hash (-mfc 1494302000) - -ms, -match-string string match response with specified string (case insensitive) (-ms admin) + -ms, -match-string string match response with specified string (-ms admin) -mr, -match-regex string match response with specified regex (-mr admin) - -mcdn, -match-cdn string[] match host with specified cdn provider (oracle, google, azure, cloudflare, cloudfront, fastly, incapsula, leaseweb, akamai, sucuri) + -mcdn, -match-cdn string[] match host with specified cdn provider (incapsula, oracle, google, azure, cloudflare, cloudfront, fastly, akamai, sucuri, leaseweb) -mrt, -match-response-time string match response with specified response time in seconds (-mrt '< 1') -mdc, -match-condition string match response with dsl expression condition @@ -133,7 +137,7 @@ FILTERS: -ffc, -filter-favicon string[] filter response with specified favicon hash (-mfc 1494302000) -fs, -filter-string string filter response with specified string (-fs admin) -fe, -filter-regex string filter response with specified regex (-fe admin) - -fcdn, -filter-cdn string[] filter host with specified cdn provider (oracle, google, azure, cloudflare, cloudfront, fastly, incapsula, leaseweb, akamai, sucuri) + -fcdn, -filter-cdn string[] filter host with specified cdn provider (incapsula, oracle, google, azure, cloudflare, cloudfront, fastly, akamai, sucuri, leaseweb) -frt, -filter-response-time string filter response with specified response time in seconds (-frt '> 1') -fdc, -filter-condition string filter response with dsl expression condition @@ -154,6 +158,10 @@ MISCELLANEOUS: -vhost probe and display server supporting VHOST -ldv, -list-dsl-variables list json output field keys name that support dsl matcher/filter +UPDATE: + -up, -update update httpx to latest version + -duc, -disable-update-check disable automatic httpx update check + OUTPUT: -o, -output string file to write output results -sr, -store-response store http response to output directory @@ -184,8 +192,11 @@ CONFIGURATIONS: -body string post body to include in http request -s, -stream stream mode - start elaborating input targets without sorting -sd, -skip-dedupe disable dedupe input items (only used with stream mode) - -ldp, -leave-default-ports leave default http/https ports in host header (eg. http://host:80 - https//host:443 + -ldp, -leave-default-ports leave default http/https ports in host header (eg. http://host:80 - https://host:443 -ztls use ztls library with autofallback to standard one for tls13 + -no-decode avoid decoding body + -tlsi, -tls-impersonate enable random tls client (ja3) impersonation (experimental) + -no-stdin Disable Stdin processing DEBUG: -health-check, -hc run diagnostic check up @@ -471,55 +482,76 @@ https://docs.hackerone.com https://support.hackerone.com ``` -### Using `httpx` as a library -`httpx` can be used as a library by creating an instance of the `Option` struct and populating it with the same options that would be specified via CLI. Once validated, the struct should be passed to a runner instance (to be closed at the end of the program) and the `RunEnumeration` method should be called. Here follows a minimal example of how to do it: +### Screenshot -```go -package main +Latest addition to the project, the addition of the `-screenshot` option in httpx, a powerful new feature that allows users to take screenshots of target URLs, pages, or endpoints along with the rendered DOM. This functionality enables the **visual content discovery process**, providing a comprehensive view of the target's visual appearance. -import ( - "log" +Rendered DOM body is also included in json line output when `-screenshot` option is used with `-json` option. - "github.com/projectdiscovery/goflags" - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/gologger/levels" - "github.com/projectdiscovery/httpx/runner" -) +#### 🚩 Usage -func main() { - gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) // increase the verbosity (optional) +To use the screenshot feature, simply add the `-screenshot` flag to your httpx command: - options := runner.Options{ - Methods: "GET", - InputTargetHost: goflags.StringSlice{"scanme.sh", "projectdiscovery.io"}, - //InputFile: "./targetDomains.txt", // path to file containing the target domains list - } +```console +httpx -screenshot -u https://example.com +``` - if err := options.ValidateOptions(); err != nil { - log.Fatal(err) - } +🎯 Domain, Subdomain, and Path Support +The `-screenshot` option is versatile and can be used to capture screenshots for domains, subdomains, and even specific paths when used in conjunction with the `-path` option: - httpxRunner, err := runner.New(&options) - if err != nil { - log.Fatal(err) - } - defer httpxRunner.Close() +```console +httpx -screenshot -u example.com +httpx -screenshot -u https://example.com/login +httpx -screenshot -path fuzz_path.txt -u https://example.com +``` - httpxRunner.RunEnumeration() -} +Using with other tools: + +```console +subfinder -d example.com | httpx -screenshot ``` +#### 🌐 System Chrome + +By default, httpx will use the go-rod library to install and manage Chrome for taking screenshots. However, if you prefer to use your locally installed system Chrome, add the `-system-chrome` flag: + +```console +httpx -screenshot -system-chrome -u https://example.com +``` + +#### 📁 Output Directory + +Screenshots are stored in the output/screenshot directory by default. To specify a custom output directory, use the `-srd` option: + +```console +httpx -screenshot -srd /path/to/custom/directory -u https://example.com +``` + +#### ⏳ Performance Considerations + +Please note that since screenshots are captured using a headless browser, httpx runs will be slower when using the `-screenshot` option. + +### Using `httpx` as a library +`httpx` can be used as a library by creating an instance of the `Option` struct and populating it with the same options that would be specified via CLI. Once validated, the struct should be passed to a runner instance (to be closed at the end of the program) and the `RunEnumeration` method should be called. A minimal example of how to do it is in the [examples](examples/) folder # Notes -- As default, `httpx` checks for **HTTPS** probe and fall-back to **HTTP** only if **HTTPS** is not reachable. -- The `-no-fallback` flag can be used to display both **HTTP** and **HTTPS** results +- As default, `httpx` probe with **HTTPS** scheme and fall-back to **HTTP** only if **HTTPS** is not reachable. +- The `-no-fallback` flag can be used to probe and display both **HTTP** and **HTTPS** result. - Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443` -- The following flags should be used for specific use cases instead of running them as default with other probes: - * `-favicon`,`-vhost`, `-http2`, `-pipeline`, `-ports`, `-csp-probe`, `-tls-probe`, `-path` -- When using the `-json` flag, all the default probe results are included in the JSON output. - Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`) -- Invalid custom resolvers/files are ignored. +- The following flags should be used for specific use cases instead of running them as default with other probes: + - `-ports` + - `-path` + - `-vhost` + - `-screenshot` + - `-csp-probe` + - `-tls-probe` + - `-favicon` + - `-http2` + - `-pipeline` + - `-tls-impersonate` + # Acknowledgement diff --git a/cmd/functional-test/testcases.txt b/cmd/functional-test/testcases.txt index 7a525c9..e7553c7 100644 --- a/cmd/functional-test/testcases.txt +++ b/cmd/functional-test/testcases.txt @@ -16,5 +16,7 @@ scanme.sh {{binary}} -silent -body 'a=b' scanme.sh {{binary}} -silent -exclude-cdn scanme.sh {{binary}} -silent -ports https:443 scanme.sh {{binary}} -silent -ztls +scanme.sh {{binary}} -silent -jarm https://scanme.sh?a=1*1 {{binary}} -silent -https://scanme.sh:443 {{binary}} -asn \ No newline at end of file +https://scanme.sh:443 {{binary}} -asn +scanme.sh {{binary}} -silent -tls-impersonate \ No newline at end of file diff --git a/common/fileutil/fileutil.go b/common/fileutil/fileutil.go index 8efe89f..994587a 100644 --- a/common/fileutil/fileutil.go +++ b/common/fileutil/fileutil.go @@ -80,3 +80,10 @@ func LoadCidrsFromSliceOrFileWithMaxRecursion(option string, splitchar string, m return } + +func AbsPathOrDefault(p string) string { + if absPath, err := filepath.Abs(p); err == nil { + return absPath + } + return p +} diff --git a/common/httpx/cdn.go b/common/httpx/cdn.go index 073a3e6..b117c52 100644 --- a/common/httpx/cdn.go +++ b/common/httpx/cdn.go @@ -11,5 +11,5 @@ func (h *HTTPX) CdnCheck(ip string) (bool, string, error) { return false, "", fmt.Errorf("cdn client not configured") } - return h.cdn.Check(net.ParseIP((ip))) + return h.cdn.CheckCDN(net.ParseIP((ip))) } diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 56224d9..e962762 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -4,6 +4,7 @@ import ( "crypto/tls" "fmt" "io" + "net" "net/http" "net/url" "strconv" @@ -14,9 +15,10 @@ import ( "github.com/microcosm-cc/bluemonday" "github.com/projectdiscovery/cdncheck" "github.com/projectdiscovery/fastdialer/fastdialer" - "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/fastdialer/fastdialer/ja3/impersonate" "github.com/projectdiscovery/rawhttp" retryablehttp "github.com/projectdiscovery/retryablehttp-go" + "github.com/projectdiscovery/utils/generic" pdhttputil "github.com/projectdiscovery/utils/http" stringsutil "github.com/projectdiscovery/utils/strings" urlutil "github.com/projectdiscovery/utils/url" @@ -105,8 +107,13 @@ func New(options *Options) (*HTTPX, error) { } } transport := &http.Transport{ - DialContext: httpx.Dialer.Dial, - DialTLSContext: httpx.Dialer.DialTLS, + DialContext: httpx.Dialer.Dial, + DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + if options.TlsImpersonate { + return httpx.Dialer.DialTLSWithConfigImpersonate(ctx, network, addr, &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS10}, impersonate.Random, nil) + } + return httpx.Dialer.DialTLS(ctx, network, addr) + }, MaxIdleConnsPerHost: -1, TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, @@ -151,10 +158,7 @@ func New(options *Options) (*HTTPX, error) { httpx.htmlPolicy = bluemonday.NewPolicy() httpx.CustomHeaders = httpx.Options.CustomHeaders if options.CdnCheck || options.ExcludeCdn { - httpx.cdn, err = cdncheck.NewWithCache() - if err != nil { - gologger.Error().Msgf("could not create cdn check: %v", err) - } + httpx.cdn = cdncheck.New() } return httpx, nil @@ -204,8 +208,10 @@ get_response: resp.Raw = string(rawResp) resp.RawHeaders = string(headers) var respbody []byte - // websockets don't have a readable body - if httpresp.StatusCode != http.StatusSwitchingProtocols { + // body shouldn't be read with the following status codes + // 101 - Switching Protocols => websockets don't have a readable body + // 304 - Not Modified => no body the response terminates with latest header newline + if !generic.EqualsAny(httpresp.StatusCode, http.StatusSwitchingProtocols, http.StatusNotModified) { var err error respbody, err = io.ReadAll(io.LimitReader(httpresp.Body, h.Options.MaxResponseBodySizeToRead)) if err != nil && !shouldIgnoreBodyErrors { @@ -218,6 +224,10 @@ get_response: return nil, closeErr } + // Todo: replace with https://github.com/projectdiscovery/utils/issues/110 + resp.RawData = make([]byte, len(respbody)) + copy(resp.RawData, respbody) + respbody, err = DecodeData(respbody, httpresp.Header) if err != nil && !shouldIgnoreBodyErrors { return nil, closeErr diff --git a/common/httpx/option.go b/common/httpx/option.go index bec97ed..352b985 100644 --- a/common/httpx/option.go +++ b/common/httpx/option.go @@ -42,6 +42,7 @@ type Options struct { Resolvers []string customCookies []*http.Cookie SniName string + TlsImpersonate bool } // DefaultOptions contains the default options diff --git a/common/httpx/response.go b/common/httpx/response.go index bf0e15f..816551f 100644 --- a/common/httpx/response.go +++ b/common/httpx/response.go @@ -12,7 +12,8 @@ import ( type Response struct { StatusCode int Headers map[string][]string - Data []byte + RawData []byte // undecoded data + Data []byte // decoded data ContentLength int Raw string RawHeaders string diff --git a/examples/example.go b/examples/example.go new file mode 100644 index 0000000..ae2bb4e --- /dev/null +++ b/examples/example.go @@ -0,0 +1,41 @@ +package main + +import ( + "fmt" + "log" + + "github.com/projectdiscovery/goflags" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/gologger/levels" + "github.com/projectdiscovery/httpx/runner" +) + +func main() { + gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) // increase the verbosity (optional) + + options := runner.Options{ + Methods: "GET", + InputTargetHost: goflags.StringSlice{"scanme.sh", "projectdiscovery.io", "localhost"}, + //InputFile: "./targetDomains.txt", // path to file containing the target domains list + OnResult: func(r runner.Result) { + // handle error + if r.Err != nil { + fmt.Printf("[Err] %s: %s\n", r.Input, r.Err) + return + } + fmt.Printf("%s %s %d\n", r.Input, r.Host, r.StatusCode) + }, + } + + if err := options.ValidateOptions(); err != nil { + log.Fatal(err) + } + + httpxRunner, err := runner.New(&options) + if err != nil { + log.Fatal(err) + } + defer httpxRunner.Close() + + httpxRunner.RunEnumeration() +} diff --git a/go.mod b/go.mod index 75ecc8f..ed40202 100644 --- a/go.mod +++ b/go.mod @@ -4,80 +4,89 @@ go 1.19 require ( github.com/akrylysov/pogreb v0.10.1 // indirect - github.com/bluele/gcache v0.0.2 github.com/corpix/uarand v0.2.0 github.com/golang/snappy v0.0.4 // indirect github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf github.com/julienschmidt/httprouter v1.3.0 github.com/logrusorgru/aurora v2.0.3+incompatible - github.com/microcosm-cc/bluemonday v1.0.23 - github.com/miekg/dns v1.1.50 // indirect + github.com/microcosm-cc/bluemonday v1.0.24 + github.com/miekg/dns v1.1.54 // indirect github.com/pkg/errors v0.9.1 - github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 - github.com/projectdiscovery/clistats v0.0.12 + github.com/projectdiscovery/cdncheck v1.0.9 + github.com/projectdiscovery/clistats v0.0.18 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goconfig v0.0.1 - github.com/projectdiscovery/goflags v0.1.8 - github.com/projectdiscovery/gologger v1.1.8 - github.com/projectdiscovery/hmap v0.0.10 - github.com/projectdiscovery/iputil v0.0.2 // indirect - github.com/projectdiscovery/mapcidr v1.1.0 - github.com/projectdiscovery/rawhttp v0.1.10 - github.com/projectdiscovery/retryablehttp-go v1.0.13 - github.com/projectdiscovery/stringsutil v0.0.2 // indirect - github.com/projectdiscovery/wappalyzergo v0.0.86 + github.com/projectdiscovery/goflags v0.1.10 + github.com/projectdiscovery/gologger v1.1.10 + github.com/projectdiscovery/hmap v0.0.13 + github.com/projectdiscovery/mapcidr v1.1.2 + github.com/projectdiscovery/rawhttp v0.1.15 + github.com/projectdiscovery/retryablehttp-go v1.0.17 + github.com/projectdiscovery/wappalyzergo v0.0.101 github.com/remeh/sizedwaitgroup v1.0.0 - github.com/rs/xid v1.4.0 + github.com/rs/xid v1.5.0 go.etcd.io/bbolt v1.3.7 // indirect - golang.org/x/net v0.8.0 - golang.org/x/sys v0.6.0 // indirect - golang.org/x/text v0.8.0 + golang.org/x/net v0.11.0 + golang.org/x/sys v0.9.0 // indirect + golang.org/x/text v0.10.0 ) require github.com/spaolacci/murmur3 v1.1.0 require ( + github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 github.com/PuerkitoBio/goquery v1.8.1 github.com/bxcodec/faker/v4 v4.0.0-beta.3 + github.com/go-rod/rod v0.113.3 github.com/hdm/jarm-go v0.0.7 github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 github.com/mitchellh/mapstructure v1.5.0 - github.com/projectdiscovery/asnmap v1.0.2 - github.com/projectdiscovery/dsl v0.0.3 - github.com/projectdiscovery/fastdialer v0.0.24 - github.com/projectdiscovery/ratelimit v0.0.6 - github.com/projectdiscovery/tlsx v1.0.4 - github.com/projectdiscovery/utils v0.0.16 - github.com/stretchr/testify v1.8.2 - go.uber.org/multierr v1.10.0 - golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 + github.com/projectdiscovery/asnmap v1.0.4 + github.com/projectdiscovery/dsl v0.0.10 + github.com/projectdiscovery/fastdialer v0.0.31 + github.com/projectdiscovery/ratelimit v0.0.8 + github.com/projectdiscovery/tlsx v1.1.0 + github.com/projectdiscovery/utils v0.0.38 + github.com/stretchr/testify v1.8.4 + go.uber.org/multierr v1.11.0 + golang.org/x/exp v0.0.0-20230420155640-133eef4313cb ) require ( aead.dev/minisign v0.2.0 // indirect - github.com/Knetic/govaluate v3.0.0+incompatible // indirect - github.com/Masterminds/semver/v3 v3.2.0 // indirect + github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect + github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect + github.com/Mzack9999/gostruct v0.0.0-20230415193108-30b70932da81 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/alecthomas/chroma v0.10.0 // indirect + github.com/andybalholm/brotli v1.0.4 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect - github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect + github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/charmbracelet/glamour v0.6.0 // indirect github.com/cheggaaa/pb/v3 v3.1.2 // indirect + github.com/cloudflare/cfssl v1.6.4 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dlclark/regexp2 v1.8.1 // indirect github.com/dsnet/compress v0.0.1 // indirect github.com/fatih/color v1.14.1 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/gaukas/godicttls v0.0.3 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/certificate-transparency-go v1.1.4 // indirect github.com/google/go-github/v30 v30.1.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/gorilla/css v1.0.0 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kataras/jwt v0.1.8 // indirect + github.com/klauspost/compress v1.15.15 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect @@ -87,41 +96,53 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.15.1 // indirect - github.com/nwaples/rardecode v1.1.0 // indirect + github.com/nwaples/rardecode v1.1.3 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect - github.com/pierrec/lz4 v2.6.0+incompatible // indirect + github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/projectdiscovery/blackrock v0.0.0-20221025011524-9e4efe804fb4 // indirect - github.com/projectdiscovery/freeport v0.0.4 // indirect - github.com/projectdiscovery/networkpolicy v0.0.4 // indirect - github.com/projectdiscovery/retryabledns v1.0.21 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/projectdiscovery/blackrock v0.0.1 // indirect + github.com/projectdiscovery/freeport v0.0.5 // indirect + github.com/projectdiscovery/networkpolicy v0.0.6 // indirect + github.com/projectdiscovery/retryabledns v1.0.30 // indirect + github.com/refraction-networking/utls v1.3.2 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect + github.com/sashabaranov/go-openai v1.11.2 // indirect + github.com/shirou/gopsutil/v3 v3.23.5 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/tidwall/btree v1.6.0 // indirect - github.com/tidwall/buntdb v1.2.10 // indirect + github.com/tidwall/buntdb v1.3.0 // indirect github.com/tidwall/gjson v1.14.4 // indirect github.com/tidwall/grect v0.1.4 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/rtred v0.1.2 // indirect github.com/tidwall/tinyqueue v0.1.1 // indirect + github.com/tklauser/go-sysconf v0.3.11 // indirect + github.com/tklauser/numcpus v0.6.0 // indirect github.com/ulikunitz/xz v0.5.11 // indirect github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect - github.com/weppos/publicsuffix-go v0.20.0 // indirect + github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/yl2chen/cidranger v1.0.2 // indirect + github.com/ysmood/fetchup v0.2.3 // indirect + github.com/ysmood/goob v0.4.0 // indirect + github.com/ysmood/got v0.34.1 // indirect + github.com/ysmood/gson v0.7.3 // indirect + github.com/ysmood/leakless v0.8.0 // indirect github.com/yuin/goldmark v1.5.4 // indirect github.com/yuin/goldmark-emoji v1.0.1 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect - github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4 // indirect github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101 // indirect - golang.org/x/crypto v0.7.0 // indirect - golang.org/x/mod v0.8.0 // indirect - golang.org/x/oauth2 v0.5.0 // indirect - golang.org/x/tools v0.6.0 // indirect + golang.org/x/crypto v0.10.0 // indirect + golang.org/x/mod v0.10.0 // indirect + golang.org/x/oauth2 v0.9.0 // indirect + golang.org/x/tools v0.8.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/protobuf v1.29.1 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 8d7e09a..b76269f 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,16 @@ aead.dev/minisign v0.2.0 h1:kAWrq/hBRu4AARY6AlciO83xhNnW9UaC8YipS2uhLPk= aead.dev/minisign v0.2.0/go.mod h1:zdq6LdSd9TbuSxchxwhpA9zEb9YXcVGoE8JakuiGaIQ= -github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg= -github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= -github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw= +github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 h1:KFac3SiGbId8ub47e7kd2PLZeACxc1LkiiNoDOFRClE= +github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057/go.mod h1:iLB2pivrPICvLOuROKmlqURtFIEsoJZaMidQfCG1+D4= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= +github.com/Mzack9999/gostruct v0.0.0-20230415193108-30b70932da81 h1:rwHZjxG8Cx3+FNujiZRuJbYTLHmW8U9+6xIoTseKA/I= +github.com/Mzack9999/gostruct v0.0.0-20230415193108-30b70932da81/go.mod h1:iXPMmoXMc0ZsSmbbHqhWCWd8w7FkXM7DU2IBf5OS+5g= github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/RumbleDiscovery/rumble-tools v0.0.0-20201105153123-f2adbb3244d2/go.mod h1:jD2+mU+E2SZUuAOHZvZj4xP4frlOo+N/YrXDvASFhkE= @@ -15,30 +20,31 @@ github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= +github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= -github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= -github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aymanbagabas/go-osc52 v1.0.3/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bits-and-blooms/bitset v1.3.1 h1:y+qrlmq3XsWi+xZqSaueaE8ry8Y127iMxlMfqcK8p0g= -github.com/bits-and-blooms/bloom/v3 v3.3.1 h1:K2+A19bXT8gJR5mU7y+1yW6hsKfNCjcP2uNfLFKncjQ= -github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= -github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= +github.com/bits-and-blooms/bloom/v3 v3.4.0 h1:9zesenPR5M3SLIJ/esQ84o1eSVFY5Rw5d+pa1tiXQNA= github.com/bxcodec/faker/v4 v4.0.0-beta.3 h1:gqYNBvN72QtzKkYohNDKQlm+pg+uwBDVMN28nWHS18k= github.com/bxcodec/faker/v4 v4.0.0-beta.3/go.mod h1:m6+Ch1Lj3fqW/unZmvkXIdxWS5+XQWPWxcbbQW2X+Ho= github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc= github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc= github.com/cheggaaa/pb/v3 v3.1.2 h1:FIxT3ZjOj9XJl0U4o2XbEhjFfZl7jCVCDOGq1ZAB7wQ= github.com/cheggaaa/pb/v3 v3.1.2/go.mod h1:SNjnd0yKcW+kw0brSusraeDd5Bf1zBfxAzTL2ss3yQ4= +github.com/cloudflare/cfssl v1.6.4 h1:NMOvfrEjFfC63K3SGXgAnFdsgkmiq4kATme5BfcqrO8= +github.com/cloudflare/cfssl v1.6.4/go.mod h1:8b3CQMxfWPAeom3zBnGJ6sd+G1NkL5TXqmDXacb+1J0= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= github.com/corpix/uarand v0.2.0 h1:U98xXwud/AVuCpkpgfPF7J5TQgr7R5tqT8VZP5KWbzE= github.com/corpix/uarand v0.2.0/go.mod h1:/3Z1QIqWkDIhf6XWn/08/uMHoQ8JUoTIKc2iPchBOmM= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -53,34 +59,46 @@ github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdf github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= +github.com/gaukas/godicttls v0.0.3 h1:YNDIf0d9adcxOijiLrEzpfZGAkNwLRzPaG6OjU7EITk= +github.com/gaukas/godicttls v0.0.3/go.mod h1:l6EenT4TLWgTdwslVb4sEMOCf7Bv0JAK67deKr9/NCI= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-rod/rod v0.113.3 h1:oLiKZW721CCMwA5g7977cWfcAKQ+FuosP47Zf1QiDrA= +github.com/go-rod/rod v0.113.3/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/certificate-transparency-go v1.1.4 h1:hCyXHDbtqlr/lMXU0D4WgbalXL0Zk4dSWWMbPV8VrqY= +github.com/google/certificate-transparency-go v1.1.4/go.mod h1:D6lvbfwckhNrbM9WVl1EVeMOyzC19mpIjMOI4nxBHtQ= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo= github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQFEufcolZ95JfU8= +github.com/google/go-github/v50 v50.1.0/go.mod h1:Ev4Tre8QoKiolvbpOSG3FIi4Mlon3S2Nt9W5JYqKiwA= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= -github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU= github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6457fz1+xXYIWDxdGc7HdkLS9aJ1skk= github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf/go.mod h1:V99KdStnMHZsvVOwIvhfcUzYgYkRZeQWUtumtL+SKxA= github.com/hdm/jarm-go v0.0.7 h1:Eq0geenHrBSYuKrdVhrBdMMzOmA+CAMLzN2WrF3eL6A= @@ -90,20 +108,24 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kataras/jwt v0.1.8 h1:u71baOsYD22HWeSOg32tCHbczPjdCk7V4MMeJqTtmGk= +github.com/kataras/jwt v0.1.8/go.mod h1:Q5j2IkcIHnfwy+oNY3TVWuEBJNw0ADgCcXK9CaZwV4o= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= +github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -118,11 +140,11 @@ github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/p github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU= github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU= github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= -github.com/microcosm-cc/bluemonday v1.0.23 h1:SMZe2IGa0NuHvnVNAZ+6B38gsTbi5e4sViiWJyDDqFY= -github.com/microcosm-cc/bluemonday v1.0.23/go.mod h1:mN70sk7UkkF8TUr2IGBpNN0jAgStuPzlK76QuruE/z4= +github.com/microcosm-cc/bluemonday v1.0.24 h1:NGQoPtwGVcbGkKfvyYk1yRqknzBuoMiUrO6R7uFTPlw= +github.com/microcosm-cc/bluemonday v1.0.24/go.mod h1:ArQySAMps0790cHSkdPEJ7bGkF2VePWH773hsJNSHf8= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= -github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/miekg/dns v1.1.54 h1:5jon9mWcb0sFJGpnI99tOMhCPyJ+RPVz5b63MQG0VWI= +github.com/miekg/dns v1.1.54/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU= github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -138,8 +160,8 @@ github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKt github.com/muesli/termenv v0.13.0/go.mod h1:sP1+uffeLaEYpyOTb8pLCUctGcGLnoFjSn4YJK5e2bc= github.com/muesli/termenv v0.15.1 h1:UzuTb/+hhlBugQz28rpzey4ZuKcZ03MeKsoG7IJZIxs= github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ= -github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= -github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= +github.com/nwaples/rardecode v1.1.3 h1:cWCaZwfM5H7nAD6PyEdcVnczzV8i/JtotnyW/dD9lEc= +github.com/nwaples/rardecode v1.1.3/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= @@ -149,70 +171,78 @@ github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A= -github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= +github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/projectdiscovery/asnmap v1.0.2 h1:2+8tqzJeFVpJS7u27YH7kMK7edDAr7OsmSxs92aWFNc= -github.com/projectdiscovery/asnmap v1.0.2/go.mod h1:64YfriVxyRQvqc+1iPMHMf+i/of2jr+Qx7geCIm4ZsU= -github.com/projectdiscovery/blackrock v0.0.0-20221025011524-9e4efe804fb4 h1:EsrQ/zkotVodSJLOch3pV/UYt1vQcwyIs5HX0sm1ljE= -github.com/projectdiscovery/blackrock v0.0.0-20221025011524-9e4efe804fb4/go.mod h1:5tNGQP9kOfW+X5+40pZP8aqPYLHs45nJkFaSHLxdeH8= -github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 h1:QtTPPx0uu42AsQJiXT86/wqdHS7/iVcgz1VM38tjv20= -github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1/go.mod h1:EevMeCG1ogBoUJYaa0Mv9R1VUboDm/DiynId7DboKy0= -github.com/projectdiscovery/clistats v0.0.12 h1:KLYJxpiwEFidduU4PbcwEcCQ2L7c5wrf7DI5IN5fZ+8= -github.com/projectdiscovery/clistats v0.0.12/go.mod h1:9luKJj+7Hjq3+a7g129sKWRYx4SbTdkUWZQxabn3H5Y= -github.com/projectdiscovery/dsl v0.0.3 h1:oWlZZaSADqoyfJdUHWqAzpB65NpvLukZQGFv1uTtU3g= -github.com/projectdiscovery/dsl v0.0.3/go.mod h1:ST66slxtp7fsFqTOq3k6+1TDV7RH1Moz3sPBqUGn4Fg= -github.com/projectdiscovery/fastdialer v0.0.24 h1:yEyYALCmDQpPYWttZ4uo9AJseqt4mYWcyx3s9WYzqW8= -github.com/projectdiscovery/fastdialer v0.0.24/go.mod h1:X7zZy3BGdGoprR6CftHKeJyV86a3OjSAlJcNU7FL26E= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/projectdiscovery/asnmap v1.0.4 h1:dmXrls7Y0Sdeb6cLlKGwdYX8h1K9q2iYOGXioD4U2AY= +github.com/projectdiscovery/asnmap v1.0.4/go.mod h1:iTLDyYsblEwYHcLiKZCRx8Et+xV7NlvgGLusANpgegc= +github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= +github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= +github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQOocUvrssFlg= +github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs= +github.com/projectdiscovery/clistats v0.0.18 h1:WLQNqLXsKvjoieDwXJO/1jlnxR0x9vdFaRUAR3gXfKQ= +github.com/projectdiscovery/clistats v0.0.18/go.mod h1:YUnUrMHFw+FHwUTIKr1KDUwz81x+SFjPU3xfLqXfzf0= +github.com/projectdiscovery/dsl v0.0.10 h1:soBL/dgyCYC9cf3BY8YzVs0sI/dFVztOOQPaaza5TfQ= +github.com/projectdiscovery/dsl v0.0.10/go.mod h1:bpJD7YUHBx2D0obqI4jdRVepBGTJZ8p+86j3WNb0QXs= +github.com/projectdiscovery/fastdialer v0.0.31 h1:eu0wTBCWjT8dXChmBtnQaAxoFpkLdvq0VroRxZoe/M8= +github.com/projectdiscovery/fastdialer v0.0.31/go.mod h1:ttLvt0xnpNQAStYYQ6ElIBHfSXHuPEiXBkLH/OLbYlc= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= -github.com/projectdiscovery/freeport v0.0.4 h1:H4VrK/7hUcC1zbg46zv9iSMBACBDpUqcHkV+FUyXISw= -github.com/projectdiscovery/freeport v0.0.4/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE= +github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= +github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE= github.com/projectdiscovery/goconfig v0.0.1 h1:36m3QjohZvemqh9bkJAakaHsm9iEZ2AcQSS18+0QX/s= github.com/projectdiscovery/goconfig v0.0.1/go.mod h1:CPO25zR+mzTtyBrsygqsHse0sp/4vB/PjaHi9upXlDw= -github.com/projectdiscovery/goflags v0.1.8 h1:Urhm2Isq2BdRt8h4h062lHKYXO65RHRjGTDSkUwex/g= -github.com/projectdiscovery/goflags v0.1.8/go.mod h1:Yxi9tclgwGczzDU65ntrwaIql5cXeTvW5j2WxFuF+Jk= -github.com/projectdiscovery/gologger v1.1.8 h1:CFlCzGlqAhPqWIrAXBt1OVh5jkMs1qgoR/z4xhdzLNE= -github.com/projectdiscovery/gologger v1.1.8/go.mod h1:bNyVaC1U/NpJtFkJltcesn01NR3K8Hg6RsLVce6yvrw= -github.com/projectdiscovery/hmap v0.0.10 h1:O6ALGW3BK+FmknLXW7ENwQevLs+faRJuoRbDtakZZus= -github.com/projectdiscovery/hmap v0.0.10/go.mod h1:xdtyejCgl5LJW7yz7nf/ut32tWuV/l7FjUzItiCtJIg= -github.com/projectdiscovery/iputil v0.0.2 h1:f6IGnZF4RImJLysPSPG3D84jyTH34q3lihCFeP+eZzI= -github.com/projectdiscovery/iputil v0.0.2/go.mod h1:J3Pcz1q51pi4/JL871mQztg0KOzyWDPxnPLOYJm2pVQ= -github.com/projectdiscovery/mapcidr v1.1.0 h1:Yeb+CGVsRYvHmZ9YSHb9iy4tzY9YuOm3oTFX/xzGhVU= -github.com/projectdiscovery/mapcidr v1.1.0/go.mod h1:hck0bWXka5ZkUaBG+TWt99bzLy+4hAg9oANhEmm3GNs= -github.com/projectdiscovery/networkpolicy v0.0.4 h1:zcGjEqZbyECZEdyCy1jVuwOS7Ww1mzgCefQU75XqdJA= -github.com/projectdiscovery/networkpolicy v0.0.4/go.mod h1:DIXwKs3sQyfCoWHKRLQiRrEorSQW4Zrh4ftu7oDVK6w= -github.com/projectdiscovery/ratelimit v0.0.6 h1:SAD2ArdT9F8NmbkAIZpl7DjNnbiXdUQLnMZt5dbVmZ0= -github.com/projectdiscovery/ratelimit v0.0.6/go.mod h1:WFL6gIggPLTwYwDbxqQODuWrz/lcMP2E5ofKSAz3YwI= -github.com/projectdiscovery/rawhttp v0.1.10 h1:wkQk/lpMVzi4AAELRDaBQEgMqyerpkz3Kks7QgDF274= -github.com/projectdiscovery/rawhttp v0.1.10/go.mod h1:cIlAWs3Nu8CTBArx/8GU1baimR5T1eO62TJFG2rAnSc= -github.com/projectdiscovery/retryabledns v1.0.21 h1:vOpPQR1q8Z824uoA8JXCI/RyvDAssPeD68Onz9hP/ds= -github.com/projectdiscovery/retryabledns v1.0.21/go.mod h1:6oTPKMRlKZ7lIIEzTH723K6RvNRjmm6fe9br4Dom3UI= -github.com/projectdiscovery/retryablehttp-go v1.0.13 h1:gKxd/J08Dxc8a/LFvTz9+JUedEvivH3PoDnQQEHAY4M= -github.com/projectdiscovery/retryablehttp-go v1.0.13/go.mod h1:L5HwtGSvc0E3dNVtVqPACWOmr21Bbop2ZhpbCPYEeYU= +github.com/projectdiscovery/goflags v0.1.10 h1:Gompf8JDy8y+5c4eWlc70KKtPuDH/hqFB3tMeHcMiKk= +github.com/projectdiscovery/goflags v0.1.10/go.mod h1:MHEkqm3XgxBf5fK4gr3IXsj6VeLTq4qJYGC/4JRYQ74= +github.com/projectdiscovery/gologger v1.1.10 h1:XNRdtzLTdxiFGuK9gutoL752mykzXDoii4P2yDovqck= +github.com/projectdiscovery/gologger v1.1.10/go.mod h1:VqANHK7qcEq3i6/vV5HNWwdyv2aFPSrlaVDU4Ogrc6U= +github.com/projectdiscovery/hmap v0.0.13 h1:8v5j99Pz0S7V1YrTeWp7xtr1yNOffKQ/KusHZfB+mrI= +github.com/projectdiscovery/hmap v0.0.13/go.mod h1:Ymc9xjbfhswpmI/gOx5hyR4+OvqguSq1SDJTH197gWg= +github.com/projectdiscovery/mapcidr v1.1.2 h1:Mmq/nPqvVc7fjvH/kJVK0IBOny/LrJIxZ4tQsLPCrsA= +github.com/projectdiscovery/mapcidr v1.1.2/go.mod h1:Aoq0x/wJl6KDbtQ8OcPkjIDCqx2iEyx5ty1nzso8wXM= +github.com/projectdiscovery/networkpolicy v0.0.6 h1:yDvm0XCrS9HeemRrBS+J+22surzVczM94W5nHiOy/1o= +github.com/projectdiscovery/networkpolicy v0.0.6/go.mod h1:8HJQ/33Pi7v3a3MRWIQGXzpj+zHw2d60TysEL4qdoQk= +github.com/projectdiscovery/ratelimit v0.0.8 h1:K6S/DCr48xNxTXHRmU82wl1mj7j0VrXnAKr8sKTacHI= +github.com/projectdiscovery/ratelimit v0.0.8/go.mod h1:JJAtj8Rd5DNqN5FgwyMHWIi4BHivOw1+8gDrpsBf8Ic= +github.com/projectdiscovery/rawhttp v0.1.15 h1:wW6U+M98NHtD0ZlSFJ49vS24gpSNZ6KZV3TZNPVCpAc= +github.com/projectdiscovery/rawhttp v0.1.15/go.mod h1:f57f8nG7oV8PqrhKmI1duKIT28mdpZauytslt8gP/7s= +github.com/projectdiscovery/retryabledns v1.0.30 h1:7bc8Lq3r/qzw4LdXXAxKtQa52iGiEx1WasZLVCO6Oj0= +github.com/projectdiscovery/retryabledns v1.0.30/go.mod h1:+Aqc0TjKGcTtP0HtXE8o1GzrjAHhSno6hSF+L63TBtI= +github.com/projectdiscovery/retryablehttp-go v1.0.17 h1:oppnrypatWsHxcMU5RuAcUsUu3nxBhId2CF3OBj9XJA= +github.com/projectdiscovery/retryablehttp-go v1.0.17/go.mod h1:zJh8bQdxhIsaEGnxsacvMbgiCKT4UAOr4T1kZBnSa68= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= -github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= -github.com/projectdiscovery/tlsx v1.0.4 h1:aVkoxl1Vq6Uh7Ietpv2X5TfXPRRBj3VGCtoelhj9BE0= -github.com/projectdiscovery/tlsx v1.0.4/go.mod h1:N50JpF12eeLg+rcDRwIZ8r974MOvJZeV7M0rIxHWV2g= -github.com/projectdiscovery/utils v0.0.16 h1:7vmi3haCyM3vk0yXSLjoid4p2/7bo042rcmG4Dtk+Sk= -github.com/projectdiscovery/utils v0.0.16/go.mod h1:Cu216AlQ7rAYa8aDBqB2OgNfu5p24Uj+tG9RxV8Wbfs= -github.com/projectdiscovery/wappalyzergo v0.0.86 h1:w4UP+F1emlhoDXRHZanSzWQNYm/tv/A0ocGOpeuParo= -github.com/projectdiscovery/wappalyzergo v0.0.86/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= +github.com/projectdiscovery/tlsx v1.1.0 h1:6L5VKpHaoqvIHN6lH9zi7jIvph1JwYMYZOIpWBJBG6I= +github.com/projectdiscovery/tlsx v1.1.0/go.mod h1:C9xTbU2t54Anmvuq+4jxevR5rzqpp6XUUtV7G9J5CTE= +github.com/projectdiscovery/utils v0.0.38 h1:EIAgaP3imfcQY+laxNOU9LXh7VZNAbmiwXsQN0mAxdQ= +github.com/projectdiscovery/utils v0.0.38/go.mod h1:5+WAxSV7yGl6SDCtR1qiOyiEMCIo3jIff+A5OiYTCgM= +github.com/projectdiscovery/wappalyzergo v0.0.101 h1:g3vEDqdsfIvsbp1YsXCdVMGurucdK+j72trncROMVjo= +github.com/projectdiscovery/wappalyzergo v0.0.101/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA= +github.com/refraction-networking/utls v1.3.2 h1:o+AkWB57mkcoW36ET7uJ002CpBWHu0KPxi6vzxvPnv8= +github.com/refraction-networking/utls v1.3.2/go.mod h1:fmoaOww2bxzzEpIKOebIsnBvjQpqP7L2vcm/9KUfm/E= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= -github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= +github.com/sashabaranov/go-openai v1.11.2 h1:HuMf+18eldSKbqVblyeCQbtcqSpGVfqTshvi8Bn6zes= +github.com/sashabaranov/go-openai v1.11.2/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= +github.com/shirou/gopsutil/v3 v3.23.5 h1:5SgDCeQ0KW0S4N0znjeM/eFHXXOKyv2dVNgRq/c9P6Y= +github.com/shirou/gopsutil/v3 v3.23.5/go.mod h1:Ng3Maa27Q2KARVJ0SPZF5NdrQSC3XHKP8IIWrHgMeLY= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= @@ -227,16 +257,16 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg= github.com/tidwall/btree v1.6.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= -github.com/tidwall/buntdb v1.2.10 h1:U/ebfkmYPBnyiNZIirUiWFcxA/mgzjbKlyPynFsPtyM= -github.com/tidwall/buntdb v1.2.10/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU= +github.com/tidwall/buntdb v1.3.0 h1:gdhWO+/YwoB2qZMeAU9JcWWsHSYU3OvcieYgFRS0zwA= +github.com/tidwall/buntdb v1.3.0/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -252,40 +282,57 @@ github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8= github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ= github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw= +github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= +github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= +github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= +github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= -github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 h1:TtyC78WMafNW8QFfv3TeP3yWNDG+uxNkk9vOrnDu6JA= github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6/go.mod h1:h8272+G2omSmi30fBXiZDMkmHuOgonplfKIKjQWzlfs= github.com/weppos/publicsuffix-go v0.12.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37/go.mod h1:5ZC/Uv3fIEUE0eP6o9+Yg4+5+W8V0/BieMi05feGXVA= -github.com/weppos/publicsuffix-go v0.20.0 h1:59ypvSUbW3Dunc6zVm+v+MmXf2Q6cGiNDkxgRIzEnaA= -github.com/weppos/publicsuffix-go v0.20.0/go.mod h1:5ZC/Uv3fIEUE0eP6o9+Yg4+5+W8V0/BieMi05feGXVA= +github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db h1:/WcxBne+5CbtbgWd/sV2wbravmr4sT7y52ifQaCgoLs= +github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db/go.mod h1:aiQaH1XpzIfgrJq3S1iw7w+3EDbRP7mF5fmwUhWyRUs= github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220704091424-e0182326a282/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g= +github.com/ysmood/fetchup v0.2.3 h1:ulX+SonA0Vma5zUFXtv52Kzip/xe7aj4vqT5AJwQ+ZQ= +github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns= +github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ= +github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18= +github.com/ysmood/gop v0.0.2 h1:VuWweTmXK+zedLqYufJdh3PlxDNBOfFHjIZlPT2T5nw= +github.com/ysmood/gop v0.0.2/go.mod h1:rr5z2z27oGEbyB787hpEcx4ab8cCiPnKxn0SUHt6xzk= +github.com/ysmood/got v0.34.1 h1:IrV2uWLs45VXNvZqhJ6g2nIhY+pgIG1CUoOcqfXFl1s= +github.com/ysmood/got v0.34.1/go.mod h1:yddyjq/PmAf08RMLSwDjPyCvHvYed+WjHnQxpH851LM= +github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY= +github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM= +github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE= +github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg= +github.com/ysmood/leakless v0.8.0 h1:BzLrVoiwxikpgEQR0Lk8NyBN5Cit2b1z+u0mgL4ZJak= +github.com/ysmood/leakless v0.8.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.5.2/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark-emoji v1.0.1 h1:ctuWEyzGBwiucEqxzwe0SOYDXPAucOrE9NQC18Wa1os= github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGjRIBbIZQFqkQ= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 h1:Nzukz5fNOBIHOsnP+6I79kPx3QhLv8nBy2mfFhBRq30= github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= -github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4 h1:17HHAgFKlLcZsDOjBOUrd5hDihb1ggf+1a5dTbkgkIY= github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101 h1:QuLjRpIBjqene8VvB+VhQ4eTcQGCQ7JDuk0/Fp4sLLw= github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101/go.mod h1:bRZdjnJaHWVXKEwrfAZMd0gfRjZGNhTbZwzp07s0Abw= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= -go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -293,17 +340,19 @@ golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= -golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 h1:Jvc7gsqn21cJHCmAWx0LiimpP18LZmUxkT5Mp7EZ1mI= -golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/exp v0.0.0-20230420155640-133eef4313cb h1:rhjz/8Mbfa8xROFiH+MQphmAmgqRM0bOMnytznhWEXk= +golang.org/x/exp v0.0.0-20230420155640-133eef4313cb/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -314,81 +363,89 @@ golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.9.0 h1:BPpt2kU7oMRq3kCHAA1tbSEshXRw1LpG2ztgDwrzuAs= +golang.org/x/oauth2 v0.9.0/go.mod h1:qYgFZaFiu6Wg24azG8bdV52QJXJGbZzIIsRCdVKzbLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= +golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM= +google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/djherbis/times.v1 v1.3.0 h1:uxMS4iMtH6Pwsxog094W0FYldiNnfY/xba00vq6C2+o= gopkg.in/djherbis/times.v1 v1.3.0/go.mod h1:AQlg6unIsrsCEdQYhTzERy542dz6SFdQFZFv6mUY0P8= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -401,6 +458,5 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/runner/banner.go b/runner/banner.go index e81804e..3b26ef1 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -16,7 +16,7 @@ const banner = ` ` // Version is the current version of httpx -const version = `v1.2.9` +const version = `v1.3.3` // showBanner is used to show the banner to the user func showBanner() { diff --git a/runner/headless.go b/runner/headless.go new file mode 100644 index 0000000..cdf082e --- /dev/null +++ b/runner/headless.go @@ -0,0 +1,129 @@ +package runner + +import ( + "fmt" + "os" + "time" + + "github.com/go-rod/rod" + "github.com/go-rod/rod/lib/launcher" + "github.com/go-rod/rod/lib/proto" + "github.com/pkg/errors" + fileutil "github.com/projectdiscovery/utils/file" + osutils "github.com/projectdiscovery/utils/os" + processutil "github.com/projectdiscovery/utils/process" +) + +// MustDisableSandbox determines if the current os and user needs sandbox mode disabled +func MustDisableSandbox() bool { + // linux with root user needs "--no-sandbox" option + // https://github.com/chromium/chromium/blob/c4d3c31083a2e1481253ff2d24298a1dfe19c754/chrome/test/chromedriver/client/chromedriver.py#L209 + return osutils.IsLinux() && os.Geteuid() == 0 +} + +type Browser struct { + tempDir string + engine *rod.Browser + pids map[int32]struct{} +} + +func NewBrowser(proxy string, useLocal bool) (*Browser, error) { + dataStore, err := os.MkdirTemp("", "nuclei-*") + if err != nil { + return nil, errors.Wrap(err, "could not create temporary directory") + } + + pids := processutil.FindProcesses(processutil.IsChromeProcess) + + chromeLauncher := launcher.New(). + Leakless(false). + Set("disable-gpu", "true"). + Set("ignore-certificate-errors", "true"). + Set("ignore-certificate-errors", "1"). + Set("disable-crash-reporter", "true"). + Set("disable-notifications", "true"). + Set("hide-scrollbars", "true"). + Set("window-size", fmt.Sprintf("%d,%d", 1080, 1920)). + Set("mute-audio", "true"). + Set("incognito", "true"). + Delete("use-mock-keychain"). + Headless(true). + UserDataDir(dataStore) + + if MustDisableSandbox() { + chromeLauncher = chromeLauncher.NoSandbox(true) + } + + executablePath, err := os.Executable() + if err != nil { + return nil, err + } + + // if musl is used, most likely we are on alpine linux which is not supported by go-rod, so we fallback to default chrome + useMusl, _ := fileutil.UseMusl(executablePath) + if useLocal || useMusl { + if chromePath, hasChrome := launcher.LookPath(); hasChrome { + chromeLauncher.Bin(chromePath) + } else { + return nil, errors.New("the chrome browser is not installed") + } + } + + if proxy != "" { + chromeLauncher = chromeLauncher.Proxy(proxy) + } + launcherURL, err := chromeLauncher.Launch() + if err != nil { + return nil, err + } + + browser := rod.New().ControlURL(launcherURL) + if browserErr := browser.Connect(); browserErr != nil { + return nil, browserErr + } + + engine := &Browser{ + tempDir: dataStore, + engine: browser, + pids: pids, + } + return engine, nil +} + +func (b *Browser) ScreenshotWithBody(url string, timeout time.Duration) ([]byte, string, error) { + page, err := b.engine.Page(proto.TargetCreateTarget{}) + if err != nil { + return nil, "", err + } + page = page.Timeout(timeout) + defer page.Close() + + if err := page.Navigate(url); err != nil { + return nil, "", err + } + + page.Timeout(2 * time.Second).WaitNavigation(proto.PageLifecycleEventNameFirstMeaningfulPaint)() + + if err := page.WaitLoad(); err != nil { + return nil, "", err + } + _ = page.WaitIdle(1 * time.Second) + + screenshot, err := page.Screenshot(true, &proto.PageCaptureScreenshot{}) + if err != nil { + return nil, "", err + } + + body, err := page.HTML() + if err != nil { + return screenshot, "", err + } + + return screenshot, body, nil +} + +func (b *Browser) Close() { + b.engine.Close() + os.RemoveAll(b.tempDir) + processutil.CloseProcesses(processutil.IsChromeProcess, b.pids) +} diff --git a/runner/options.go b/runner/options.go index d231c5d..d2b30e0 100644 --- a/runner/options.go +++ b/runner/options.go @@ -34,12 +34,10 @@ const ( DefaultOutputDirectory = "output" ) -var defaultProviders = strings.Join(cdncheck.GetDefaultProviders(), ", ") - // OnResultCallback (hostResult) type OnResultCallback func(Result) -type scanOptions struct { +type ScanOptions struct { Methods []string StoreResponseDirectory string RequestURI string @@ -85,10 +83,13 @@ type scanOptions struct { OutputLinesCount bool OutputWordsCount bool Hashes string + Screenshot bool + UseInstalledChrome bool + DisableStdini bool } -func (s *scanOptions) Clone() *scanOptions { - return &scanOptions{ +func (s *ScanOptions) Clone() *ScanOptions { + return &ScanOptions{ Methods: s.Methods, StoreResponseDirectory: s.StoreResponseDirectory, RequestURI: s.RequestURI, @@ -131,6 +132,8 @@ func (s *scanOptions) Clone() *scanOptions { OutputLinesCount: s.OutputLinesCount, OutputWordsCount: s.OutputWordsCount, Hashes: s.Hashes, + Screenshot: s.Screenshot, + UseInstalledChrome: s.UseInstalledChrome, } } @@ -260,8 +263,14 @@ type Options struct { ListDSLVariable bool OutputFilterCondition string OutputMatchCondition string - OnResult OnResultCallback - DisableUpdateCheck bool + //The OnResult callback function is invoked for each result. It is important to check for errors in the result before using Result.Err. + OnResult OnResultCallback + DisableUpdateCheck bool + NoDecode bool + Screenshot bool + UseInstalledChrome bool + TlsImpersonate bool + DisableStdin bool } // ParseOptions parses the command line options for application @@ -300,6 +309,11 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.Probe, "probe", false, "display probe status"), ) + flagSet.CreateGroup("headless", "Headless", + flagSet.BoolVarP(&options.Screenshot, "screenshot", "ss", false, "enable saving screenshot of the page using headless browser"), + flagSet.BoolVar(&options.UseInstalledChrome, "system-chrome", false, "enable using local installed chrome for screenshot"), + ) + flagSet.CreateGroup("matchers", "Matchers", flagSet.StringVarP(&options.OutputMatchStatusCode, "match-code", "mc", "", "match response with specified status code (-mc 200,302)"), flagSet.StringVarP(&options.OutputMatchContentLength, "match-length", "ml", "", "match response with specified content length (-ml 100,102)"), @@ -308,7 +322,7 @@ func ParseOptions() *Options { flagSet.StringSliceVarP(&options.OutputMatchFavicon, "match-favicon", "mfc", nil, "match response with specified favicon hash (-mfc 1494302000)", goflags.NormalizedStringSliceOptions), flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms", "", "match response with specified string (-ms admin)"), flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "match response with specified regex (-mr admin)"), - flagSet.StringSliceVarP(&options.OutputMatchCdn, "match-cdn", "mcdn", nil, fmt.Sprintf("match host with specified cdn provider (%s)", defaultProviders), goflags.NormalizedStringSliceOptions), + flagSet.StringSliceVarP(&options.OutputMatchCdn, "match-cdn", "mcdn", nil, fmt.Sprintf("match host with specified cdn provider (%s)", cdncheck.DefaultCDNProviders), goflags.NormalizedStringSliceOptions), flagSet.StringVarP(&options.OutputMatchResponseTime, "match-response-time", "mrt", "", "match response with specified response time in seconds (-mrt '< 1')"), flagSet.StringVarP(&options.OutputMatchCondition, "match-condition", "mdc", "", "match response with dsl expression condition"), ) @@ -326,7 +340,7 @@ func ParseOptions() *Options { flagSet.StringSliceVarP(&options.OutputFilterFavicon, "filter-favicon", "ffc", nil, "filter response with specified favicon hash (-mfc 1494302000)", goflags.NormalizedStringSliceOptions), flagSet.StringVarP(&options.OutputFilterString, "filter-string", "fs", "", "filter response with specified string (-fs admin)"), flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "filter response with specified regex (-fe admin)"), - flagSet.StringSliceVarP(&options.OutputFilterCdn, "filter-cdn", "fcdn", nil, fmt.Sprintf("filter host with specified cdn provider (%s)", defaultProviders), goflags.NormalizedStringSliceOptions), + flagSet.StringSliceVarP(&options.OutputFilterCdn, "filter-cdn", "fcdn", nil, fmt.Sprintf("filter host with specified cdn provider (%s)", cdncheck.DefaultCDNProviders), goflags.NormalizedStringSliceOptions), flagSet.StringVarP(&options.OutputFilterResponseTime, "filter-response-time", "frt", "", "filter response with specified response time in seconds (-frt '> 1')"), flagSet.StringVarP(&options.OutputFilterCondition, "filter-condition", "fdc", "", "filter response with dsl expression condition"), ) @@ -386,8 +400,11 @@ func ParseOptions() *Options { flagSet.StringVar(&options.RequestBody, "body", "", "post body to include in http request"), flagSet.BoolVarP(&options.Stream, "stream", "s", false, "stream mode - start elaborating input targets without sorting"), flagSet.BoolVarP(&options.SkipDedupe, "skip-dedupe", "sd", false, "disable dedupe input items (only used with stream mode)"), - flagSet.BoolVarP(&options.LeaveDefaultPorts, "leave-default-ports", "ldp", false, "leave default http/https ports in host header (eg. http://host:80 - https//host:443"), + flagSet.BoolVarP(&options.LeaveDefaultPorts, "leave-default-ports", "ldp", false, "leave default http/https ports in host header (eg. http://host:80 - https://host:443"), flagSet.BoolVar(&options.ZTLS, "ztls", false, "use ztls library with autofallback to standard one for tls13"), + flagSet.BoolVar(&options.NoDecode, "no-decode", false, "avoid decoding body"), + flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"), + flagSet.BoolVar(&options.DisableStdin, "no-stdin", false, "Disable Stdin processing"), ) flagSet.CreateGroup("debug", "Debug", @@ -410,7 +427,7 @@ func ParseOptions() *Options { flagSet.IntVarP(&options.HostMaxErrors, "max-host-error", "maxhr", 30, "max error count per host before skipping remaining path/s"), flagSet.BoolVarP(&options.ExcludeCDN, "exclude-cdn", "ec", false, "skip full port scans for CDNs (only checks for 80,443)"), flagSet.IntVar(&options.Retries, "retries", 0, "number of retries"), - flagSet.IntVar(&options.Timeout, "timeout", 5, "timeout in seconds"), + flagSet.IntVar(&options.Timeout, "timeout", 10, "timeout in seconds"), flagSet.DurationVar(&options.Delay, "delay", -1, "duration between each http request (eg: 200ms, 1s)"), flagSet.IntVarP(&options.MaxResponseBodySizeToSave, "response-size-to-save", "rsts", math.MaxInt32, "max response size to save in bytes"), flagSet.IntVarP(&options.MaxResponseBodySizeToRead, "response-size-to-read", "rstr", math.MaxInt32, "max response size to read in bytes"), @@ -452,7 +469,7 @@ func ParseOptions() *Options { } if !options.DisableUpdateCheck { - latestVersion, err := updateutils.GetVersionCheckCallback("httpx")() + latestVersion, err := updateutils.GetToolVersionCallback("httpx", version)() if err != nil { if options.Verbose { gologger.Error().Msgf("httpx version check failed: %v", err.Error()) @@ -555,6 +572,10 @@ func (options *Options) ValidateOptions() error { gologger.Debug().Msgf("Using resolvers: %s\n", strings.Join(options.Resolvers, ",")) } + if options.Screenshot && !options.StoreResponse { + gologger.Debug().Msgf("automatically enabling store response") + options.StoreResponse = true + } if options.StoreResponse && options.StoreResponseDir == "" { gologger.Debug().Msgf("Store response directory not specified, using \"%s\"\n", DefaultOutputDirectory) options.StoreResponseDir = DefaultOutputDirectory @@ -563,6 +584,7 @@ func (options *Options) ValidateOptions() error { gologger.Debug().Msgf("Store response directory specified, enabling \"sr\" flag automatically\n") options.StoreResponse = true } + if options.Hashes != "" { for _, hashType := range strings.Split(options.Hashes, ",") { if !slice.StringSliceContains([]string{"md5", "sha1", "sha256", "sha512", "mmh3", "simhash"}, strings.ToLower(hashType)) { diff --git a/runner/runner.go b/runner/runner.go index 0736cd6..2ce2710 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -26,14 +26,12 @@ import ( "github.com/PuerkitoBio/goquery" asnmap "github.com/projectdiscovery/asnmap/libs" dsl "github.com/projectdiscovery/dsl" - "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/httpx/common/customextract" "github.com/projectdiscovery/httpx/common/hashes/jarm" "github.com/projectdiscovery/mapcidr/asn" errorutil "github.com/projectdiscovery/utils/errors" - mapsutil "github.com/projectdiscovery/utils/maps" - "github.com/bluele/gcache" + "github.com/Mzack9999/gcache" "github.com/logrusorgru/aurora" "github.com/pkg/errors" @@ -71,12 +69,12 @@ type Runner struct { options *Options hp *httpx.HTTPX wappalyzer *wappalyzer.Wappalyze - fastdialer *fastdialer.Dialer - scanopts scanOptions + scanopts ScanOptions hm *hybrid.HybridMap stats clistats.StatisticsClient ratelimiter ratelimit.Limiter - HostErrorsCache gcache.Cache + HostErrorsCache gcache.Cache[string, int] + browser *Browser } // New creates a new client for running enumeration process. @@ -92,20 +90,9 @@ func New(options *Options) (*Runner, error) { return nil, errors.Wrap(err, "could not create wappalyzer client") } if options.StoreResponseDir != "" { - os.RemoveAll(filepath.Join(options.StoreResponseDir, "index.txt")) + os.RemoveAll(filepath.Join(options.StoreResponseDir, "response", "index.txt")) + os.RemoveAll(filepath.Join(options.StoreResponseDir, "screenshot", "index_screenshot.txt")) } - dialerOpts := fastdialer.DefaultOptions - dialerOpts.WithDialerHistory = true - dialerOpts.MaxRetries = 3 - dialerOpts.DialerTimeout = time.Duration(options.Timeout) * time.Second - if len(options.Resolvers) > 0 { - dialerOpts.BaseResolvers = options.Resolvers - } - fastDialer, err := fastdialer.NewDialer(dialerOpts) - if err != nil { - return nil, errors.Wrap(err, "could not create dialer") - } - runner.fastdialer = fastDialer httpxOptions := httpx.DefaultOptions // Enables automatically tlsgrab if tlsprobe is requested @@ -135,6 +122,7 @@ func New(options *Options) (*Runner, error) { httpxOptions.MaxResponseBodySizeToSave = httpxOptions.MaxResponseBodySizeToRead } httpxOptions.Resolvers = options.Resolvers + httpxOptions.TlsImpersonate = options.TlsImpersonate var key, value string httpxOptions.CustomHeaders = make(map[string]string) @@ -161,7 +149,7 @@ func New(options *Options) (*Runner, error) { gologger.Fatal().Msgf("Could not create httpx instance: %s\n", err) } - var scanopts scanOptions + var scanopts ScanOptions if options.InputRawRequest != "" { var rawRequest []byte @@ -243,6 +231,15 @@ func New(options *Options) (*Runner, error) { scanopts.MaxResponseBodySizeToSave = options.MaxResponseBodySizeToSave scanopts.MaxResponseBodySizeToRead = options.MaxResponseBodySizeToRead scanopts.extractRegexps = make(map[string]*regexp.Regexp) + if options.Screenshot { + browser, err := NewBrowser(options.HTTPProxy, options.UseInstalledChrome) + if err != nil { + return nil, err + } + runner.browser = browser + } + scanopts.Screenshot = options.Screenshot + scanopts.UseInstalledChrome = options.UseInstalledChrome if options.OutputExtractRegexs != nil { for _, regex := range options.OutputExtractRegexs { @@ -290,8 +287,6 @@ func New(options *Options) (*Runner, error) { } } - hmapOptions := hybrid.DefaultDiskOptions - hmapOptions.DBType = hybrid.PogrebDB hm, err := hybrid.New(hybrid.DefaultDiskOptions) if err != nil { return nil, err @@ -307,7 +302,7 @@ func New(options *Options) (*Runner, error) { } if options.HostMaxErrors >= 0 { - gc := gcache.New(1000). + gc := gcache.New[string, int](1000). ARC(). Build() runner.HostErrorsCache = gc @@ -366,7 +361,7 @@ func (r *Runner) prepareInput() { numHosts += numTargetsFile } } - if fileutil.HasStdin() { + if !r.options.DisableStdin && fileutil.HasStdin() { numTargetsStdin, err := r.loadAndCloseFile(os.Stdin) if err != nil { gologger.Fatal().Msgf("Could not read input from stdin: %s\n", err) @@ -379,11 +374,18 @@ func (r *Runner) prepareInput() { r.stats.AddCounter("hosts", 0) r.stats.AddStatic("startedAt", time.Now()) r.stats.AddCounter("requests", 0) - - err := r.stats.Start(makePrintCallback(), time.Duration(r.options.StatsInterval)*time.Second) + r.stats.AddDynamic("summary", makePrintCallback()) + err := r.stats.Start() if err != nil { gologger.Warning().Msgf("Could not create statistics: %s\n", err) } + + r.stats.GetStatResponse(time.Duration(r.options.StatsInterval)*time.Second, func(s string, err error) error { + if err != nil && r.options.Verbose { + gologger.Error().Msgf("Could not read statistics: %s\n", err) + } + return nil + }) } } @@ -502,9 +504,9 @@ var ( lastRequestsCount float64 ) -func makePrintCallback() func(stats clistats.StatisticsClient) { +func makePrintCallback() func(stats clistats.StatisticsClient) interface{} { builder := &strings.Builder{} - return func(stats clistats.StatisticsClient) { + return func(stats clistats.StatisticsClient) interface{} { startedAt, _ := stats.GetStatic("startedAt") duration := time.Since(startedAt.(time.Time)) @@ -539,11 +541,12 @@ func makePrintCallback() func(stats clistats.StatisticsClient) { builder.WriteRune(')') builder.WriteRune('\n') - - fmt.Fprintf(os.Stderr, "%s", builder.String()) + statString := builder.String() + fmt.Fprintf(os.Stderr, "%s", statString) builder.Reset() lastRequestsCount = currentRequests + return statString } } @@ -552,18 +555,33 @@ func (r *Runner) Close() { // nolint:errcheck // ignore r.hm.Close() r.hp.Dialer.Close() + r.ratelimiter.Stop() if r.options.HostMaxErrors >= 0 { r.HostErrorsCache.Purge() } + if r.options.Screenshot { + r.browser.Close() + } } // RunEnumeration on targets for httpx client func (r *Runner) RunEnumeration() { - // Try to create output folder if it doesn't exist + // Try to create output folders if it doesn't exist if r.options.StoreResponse && !fileutil.FolderExists(r.options.StoreResponseDir) { + // main folder if err := os.MkdirAll(r.options.StoreResponseDir, os.ModePerm); err != nil { gologger.Fatal().Msgf("Could not create output directory '%s': %s\n", r.options.StoreResponseDir, err) } + // response folder + responseFolder := filepath.Join(r.options.StoreResponseDir, "response") + if err := os.MkdirAll(responseFolder, os.ModePerm); err != nil { + gologger.Fatal().Msgf("Could not create output response directory '%s': %s\n", r.options.StoreResponseDir, err) + } + // screenshot folder + screenshotFolder := filepath.Join(r.options.StoreResponseDir, "screenshot") + if err := os.MkdirAll(screenshotFolder, os.ModePerm); err != nil { + gologger.Fatal().Msgf("Could not create output screenshot directory '%s': %s\n", r.options.StoreResponseDir, err) + } } r.prepareInputPaths() @@ -591,7 +609,7 @@ func (r *Runner) RunEnumeration() { go func(output chan Result) { defer wgoutput.Done() - var f, indexFile *os.File + var f, indexFile, indexScreenshotFile *os.File if r.options.Output != "" { var err error @@ -627,7 +645,7 @@ func (r *Runner) RunEnumeration() { } if r.options.StoreResponseDir != "" { var err error - indexPath := filepath.Join(r.options.StoreResponseDir, "index.txt") + indexPath := filepath.Join(r.options.StoreResponseDir, "response", "index.txt") if r.options.Resume { indexFile, err = os.OpenFile(indexPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) } else { @@ -638,14 +656,34 @@ func (r *Runner) RunEnumeration() { } defer indexFile.Close() //nolint } + if r.options.Screenshot { + var err error + indexScreenshotPath := filepath.Join(r.options.StoreResponseDir, "screenshot", "index_screenshot.txt") + if r.options.Resume { + indexScreenshotFile, err = os.OpenFile(indexScreenshotPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) + } else { + indexScreenshotFile, err = os.Create(indexScreenshotPath) + } + if err != nil { + gologger.Fatal().Msgf("Could not open/create index screenshot file '%s': %s\n", r.options.Output, err) + } + defer indexScreenshotFile.Close() //nolint + } for resp := range output { - if resp.err != nil { + + // call the callback function if any + // be careful and check for result.Err + if r.options.OnResult != nil { + r.options.OnResult(resp) + } + + if resp.Err != nil { // Change the error message if any port value passed explicitly if url, err := r.parseURL(resp.URL); err == nil && url.Port() != "" { - resp.err = errors.New(strings.ReplaceAll(resp.err.Error(), "address", "port")) + resp.Err = errors.New(strings.ReplaceAll(resp.Err.Error(), "address", "port")) } - gologger.Debug().Msgf("Failed '%s': %s\n", resp.URL, resp.err) + gologger.Debug().Msgf("Failed '%s': %s\n", resp.URL, resp.Err) } if resp.str == "" { continue @@ -655,6 +693,10 @@ func (r *Runner) RunEnumeration() { indexData := fmt.Sprintf("%s %s (%d %s)\n", resp.StoredResponsePath, resp.URL, resp.StatusCode, http.StatusText(resp.StatusCode)) _, _ = indexFile.WriteString(indexData) } + if indexScreenshotFile != nil && resp.ScreenshotPath != "" { + indexData := fmt.Sprintf("%s %s (%d %s)\n", resp.ScreenshotPath, resp.URL, resp.StatusCode, http.StatusText(resp.StatusCode)) + _, _ = indexScreenshotFile.WriteString(indexData) + } // apply matchers and filters if r.options.OutputFilterCondition != "" || r.options.OutputMatchCondition != "" { @@ -663,11 +705,11 @@ func (r *Runner) RunEnumeration() { gologger.Warning().Msgf("Could not decode response: %s\n", err) continue } - - flatMap := make(map[string]any) - mapsutil.Walk(rawMap, func(k string, v any) { - flatMap[k] = v - }) + dslVars, _ := dslVariables() + flatMap := make(map[string]interface{}) + for _, v := range dslVars { + flatMap[v] = rawMap[v] + } if r.options.OutputMatchCondition != "" { res, err := dsl.EvalExpr(r.options.OutputMatchCondition, flatMap) @@ -705,10 +747,10 @@ func (r *Runner) RunEnumeration() { if len(r.options.filterWordsCount) > 0 && slice.IntSliceContains(r.options.filterWordsCount, resp.Words) { continue } - if r.options.filterRegex != nil && r.options.filterRegex.MatchString(resp.raw) { + if r.options.filterRegex != nil && r.options.filterRegex.MatchString(resp.Raw) { continue } - if r.options.OutputFilterString != "" && strings.Contains(strings.ToLower(resp.raw), strings.ToLower(r.options.OutputFilterString)) { + if r.options.OutputFilterString != "" && strings.Contains(strings.ToLower(resp.Raw), strings.ToLower(r.options.OutputFilterString)) { continue } if len(r.options.OutputFilterFavicon) > 0 && stringsutil.EqualFoldAny(resp.FavIconMMH3, r.options.OutputFilterFavicon...) { @@ -720,10 +762,10 @@ func (r *Runner) RunEnumeration() { if len(r.options.matchContentLength) > 0 && !slice.IntSliceContains(r.options.matchContentLength, resp.ContentLength) { continue } - if r.options.matchRegex != nil && !r.options.matchRegex.MatchString(resp.raw) { + if r.options.matchRegex != nil && !r.options.matchRegex.MatchString(resp.Raw) { continue } - if r.options.OutputMatchString != "" && !strings.Contains(strings.ToLower(resp.raw), strings.ToLower(r.options.OutputMatchString)) { + if r.options.OutputMatchString != "" && !strings.Contains(strings.ToLower(resp.Raw), strings.ToLower(r.options.OutputMatchString)) { continue } if len(r.options.OutputMatchFavicon) > 0 && !stringsutil.EqualFoldAny(resp.FavIconMMH3, r.options.OutputMatchFavicon...) { @@ -870,15 +912,15 @@ func (r *Runner) RunEnumeration() { wgoutput.Wait() } -func (r *Runner) GetScanOpts() scanOptions { +func (r *Runner) GetScanOpts() ScanOptions { return r.scanopts } -func (r *Runner) Process(t string, wg *sizedwaitgroup.SizedWaitGroup, protocol string, scanopts *scanOptions, output chan Result) { +func (r *Runner) Process(t string, wg *sizedwaitgroup.SizedWaitGroup, protocol string, scanopts *ScanOptions, output chan Result) { r.process(t, wg, r.hp, protocol, scanopts, output) } -func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.HTTPX, protocol string, scanopts *scanOptions, output chan Result) { +func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.HTTPX, protocol string, scanopts *ScanOptions, output chan Result) { protocols := []string{protocol} if scanopts.NoFallback || protocol == httpx.HTTPandHTTPS { protocols = []string{httpx.HTTPS, httpx.HTTP} @@ -923,6 +965,13 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. } for port, wantedProtocolForPort := range customport.Ports { + // NoFallbackScheme overrides custom ports scheme + // Example: httpx -u https://www.example.com -ports http:8080,https:443 --no-fallback-scheme + // In this case, the requests will be created with the target scheme (ignoring the custom ports scheme) + // Examples: https://www.example.com:8080 and https://www.example.com:443 + if scanopts.NoFallbackScheme { + wantedProtocolForPort = protocol + } wantedProtocols := []string{wantedProtocolForPort} if wantedProtocolForPort == httpx.HTTPandHTTPS { wantedProtocols = []string{httpx.HTTPS, httpx.HTTP} @@ -938,7 +987,7 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. gologger.Warning().Msgf("failed to update port of %v got %v", target.Host, err) } else { urlx.UpdatePort(fmt.Sprint(port)) - target.Host = urlx.Host + target.Host = urlx.String() } result := r.analyze(hp, protocol, target, method, t, scanopts) output <- result @@ -1021,7 +1070,7 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { return results } -func (r *Runner) analyze(hp *httpx.HTTPX, protocol string, target httpx.Target, method, origInput string, scanopts *scanOptions) Result { +func (r *Runner) analyze(hp *httpx.HTTPX, protocol string, target httpx.Target, method, origInput string, scanopts *ScanOptions) Result { origProtocol := protocol if protocol == httpx.HTTPorHTTPS || protocol == httpx.HTTPandHTTPS { protocol = httpx.HTTPS @@ -1033,22 +1082,22 @@ retry: } URL, err := r.parseURL(target.Host) if err != nil { - return Result{URL: target.Host, Input: origInput, err: err} + return Result{URL: target.Host, Input: origInput, Err: err} } // check if we have to skip the host:port as a result of a previous failure hostPort := net.JoinHostPort(URL.Host, URL.Port()) if r.options.HostMaxErrors >= 0 && r.HostErrorsCache.Has(hostPort) { numberOfErrors, err := r.HostErrorsCache.GetIFPresent(hostPort) - if err == nil && numberOfErrors.(int) >= r.options.HostMaxErrors { - return Result{URL: target.Host, err: errors.New("skipping as previously unresponsive")} + if err == nil && numberOfErrors >= r.options.HostMaxErrors { + return Result{URL: target.Host, Err: errors.New("skipping as previously unresponsive")} } } // check if the combination host:port should be skipped if belonging to a cdn if r.skipCDNPort(URL.Host, URL.Port()) { gologger.Debug().Msgf("Skipping cdn target: %s:%s\n", URL.Host, URL.Port()) - return Result{URL: target.Host, Input: origInput, err: errors.New("cdn target only allows ports 80 and 443")} + return Result{URL: target.Host, Input: origInput, Err: errors.New("cdn target only allows ports 80 and 443")} } URL.Scheme = protocol @@ -1076,7 +1125,7 @@ retry: req, err = hp.NewRequest(method, URL.String()) } if err != nil { - return Result{URL: URL.String(), Input: origInput, err: err} + return Result{URL: URL.String(), Input: origInput, Err: err} } if target.CustomHost != "" { @@ -1117,7 +1166,7 @@ retry: var errDump error requestDump, errDump = rawhttp.DumpRequestRaw(req.Method, req.URL.String(), reqURI, req.Header, req.Body, rawhttp.DefaultOptions) if errDump != nil { - return Result{URL: URL.String(), Input: origInput, err: errDump} + return Result{URL: URL.String(), Input: origInput, Err: errDump} } } else { // Create a copy on the fly of the request body @@ -1128,7 +1177,7 @@ retry: var errDump error requestDump, errDump = httputil.DumpRequestOut(req.Request, true) if errDump != nil { - return Result{URL: URL.String(), Input: origInput, err: errDump} + return Result{URL: URL.String(), Input: origInput, Err: errDump} } // The original req.Body gets modified indirectly by httputil.DumpRequestOut so we set it again to nil if it was empty // Otherwise redirects like 307/308 would fail (as they require the body to be sent along) @@ -1140,7 +1189,7 @@ retry: // fix the final output url fullURL := req.URL.String() if parsedURL, errParse := r.parseURL(fullURL); errParse != nil { - return Result{URL: URL.String(), Input: origInput, err: errParse} + return Result{URL: URL.String(), Input: origInput, Err: errParse} } else { if r.options.Unsafe { parsedURL.Path = reqURI @@ -1200,17 +1249,17 @@ retry: // mark the host:port as failed to avoid further checks if r.options.HostMaxErrors >= 0 { errorCount, err := r.HostErrorsCache.GetIFPresent(hostPort) - if err != nil || errorCount == nil { + if err != nil || errorCount == 0 { _ = r.HostErrorsCache.Set(hostPort, 1) - } else if errorCount != nil { - _ = r.HostErrorsCache.Set(hostPort, errorCount.(int)+1) + } else if errorCount > 0 { + _ = r.HostErrorsCache.Set(hostPort, errorCount+1) } } if r.options.Probe { - return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), err: err, Failed: err != nil, Error: errString, str: builder.String()} + return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), Err: err, Failed: err != nil, Error: errString, str: builder.String()} } else { - return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), err: err} + return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), Err: err} } } @@ -1301,17 +1350,25 @@ retry: builder.WriteString(fmt.Sprintf(" [%s]", serverHeader)) } - var serverResponseRaw string - var request string - var rawResponseHeader string - var responseHeader map[string]interface{} - if scanopts.ResponseInStdout { - serverResponseRaw = string(resp.Data) + var ( + serverResponseRaw string + request string + rawResponseHeader string + responseHeader map[string]interface{} + ) + + respData := string(resp.Data) + if r.options.NoDecode { + respData = string(resp.RawData) + } + + if scanopts.ResponseInStdout || r.options.OutputMatchCondition != "" || r.options.OutputFilterCondition != "" { + serverResponseRaw = string(respData) request = string(requestDump) responseHeader = normalizeHeaders(resp.Headers) rawResponseHeader = resp.RawHeaders } else if scanopts.Base64ResponseInStdout { - serverResponseRaw = stringz.Base64(resp.Data) + serverResponseRaw = stringz.Base64([]byte(respData)) request = stringz.Base64(requestDump) responseHeader = normalizeHeaders(resp.Headers) rawResponseHeader = stringz.Base64([]byte(resp.RawHeaders)) @@ -1546,7 +1603,7 @@ retry: } jarmhash := "" if r.options.Jarm { - jarmhash = jarm.Jarm(r.fastdialer, fullURL, r.options.Timeout) + jarmhash = jarm.Jarm(r.hp.Dialer, fullURL, r.options.Timeout) builder.WriteString(" [") if !scanopts.OutputWithNoColor { builder.WriteString(aurora.Magenta(jarmhash).String()) @@ -1566,15 +1623,21 @@ retry: } // store responses or chain in directory - var responsePath string + domainFile := URL.EscapedString() + hash := hashes.Sha1([]byte(domainFile)) + domainResponseFile := fmt.Sprintf("%s.txt", hash) + screenshotResponseFile := fmt.Sprintf("%s.png", hash) + hostFilename := strings.ReplaceAll(URL.Host, ":", "_") + domainResponseBaseDir := filepath.Join(scanopts.StoreResponseDirectory, "response") + domainScreenshotBaseDir := filepath.Join(scanopts.StoreResponseDirectory, "screenshot") + responseBaseDir := filepath.Join(domainResponseBaseDir, hostFilename) + screenshotBaseDir := filepath.Join(domainScreenshotBaseDir, hostFilename) + + var responsePath, screenshotPath string + // store response if scanopts.StoreResponse || scanopts.StoreChain { + responsePath = fileutilz.AbsPathOrDefault(filepath.Join(responseBaseDir, domainResponseFile)) // URL.EscapedString returns that can be used as filename - domainFile := URL.EscapedString() - hash := hashes.Sha1([]byte(domainFile)) - domainFile = fmt.Sprintf("%s.txt", hash) - domainBaseDir := filepath.Join(scanopts.StoreResponseDirectory, URL.Host) - // store response - responsePath = filepath.Join(domainBaseDir, domainFile) respRaw := resp.Raw reqRaw := requestDump if len(respRaw) > scanopts.MaxResponseBodySizeToSave { @@ -1582,14 +1645,12 @@ retry: } data := append([]byte(fullURL), append([]byte("\n\n"), reqRaw...)...) data = append(data, append([]byte("\n"), respRaw...)...) - _ = fileutil.CreateFolder(domainBaseDir) + _ = fileutil.CreateFolder(responseBaseDir) writeErr := os.WriteFile(responsePath, data, 0644) if writeErr != nil { gologger.Error().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr) } if scanopts.StoreChain && resp.HasChain() { - domainFile = strings.ReplaceAll(domainFile, ".txt", ".chain.txt") - responsePath = filepath.Join(domainBaseDir, domainFile) writeErr := os.WriteFile(responsePath, []byte(resp.GetChain()), 0644) if writeErr != nil { gologger.Warning().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr) @@ -1599,7 +1660,7 @@ retry: parsed, err := r.parseURL(fullURL) if err != nil { - return Result{URL: fullURL, Input: origInput, err: errors.Wrap(err, "could not parse url")} + return Result{URL: fullURL, Input: origInput, Err: errors.Wrap(err, "could not parse url")} } finalPort := parsed.Port() @@ -1623,6 +1684,26 @@ retry: chainItems = append(chainItems, resp.GetChainAsSlice()...) } + // screenshot + var ( + screenshotBytes []byte + headlessBody string + ) + if scanopts.Screenshot { + var err error + screenshotBytes, headlessBody, err = r.browser.ScreenshotWithBody(fullURL, r.hp.Options.Timeout) + if err != nil { + gologger.Warning().Msgf("Could not take screenshot '%s': %s", fullURL, err) + } else { + screenshotPath = fileutilz.AbsPathOrDefault(filepath.Join(screenshotBaseDir, screenshotResponseFile)) + _ = fileutil.CreateFolder(screenshotBaseDir) + err := os.WriteFile(screenshotPath, screenshotBytes, 0644) + if err != nil { + gologger.Error().Msgf("Could not write screenshot at path '%s', to disk: %s", screenshotPath, err) + } + } + } + result := Result{ Timestamp: time.Now(), Request: request, @@ -1631,7 +1712,7 @@ retry: Scheme: parsed.Scheme, Port: finalPort, Path: finalPath, - raw: resp.Raw, + Raw: resp.Raw, URL: fullURL, Input: origInput, ContentLength: resp.ContentLength, @@ -1669,9 +1750,9 @@ retry: ASN: asnResponse, ExtractRegex: extractRegex, StoredResponsePath: responsePath, - } - if r.options.OnResult != nil { - r.options.OnResult(result) + ScreenshotBytes: screenshotBytes, + ScreenshotPath: screenshotPath, + HeadlessBody: headlessBody, } return result } @@ -1753,7 +1834,7 @@ func (r *Runner) SaveResumeConfig() error { } // JSON the result -func (r Result) JSON(scanopts *scanOptions) string { //nolint +func (r Result) JSON(scanopts *ScanOptions) string { //nolint if scanopts != nil && len(r.ResponseBody) > scanopts.MaxResponseBodySizeToSave { r.ResponseBody = r.ResponseBody[:scanopts.MaxResponseBodySizeToSave] } @@ -1788,7 +1869,7 @@ func (r Result) CSVHeader() string { //nolint } // CSVRow the CSV Row -func (r Result) CSVRow(scanopts *scanOptions) string { //nolint +func (r Result) CSVRow(scanopts *ScanOptions) string { //nolint if scanopts != nil && len(r.ResponseBody) > scanopts.MaxResponseBodySizeToSave { r.ResponseBody = r.ResponseBody[:scanopts.MaxResponseBodySizeToSave] } diff --git a/runner/types.go b/runner/types.go index 3cf7414..ac05f39 100644 --- a/runner/types.go +++ b/runner/types.go @@ -25,20 +25,20 @@ func (o AsnResponse) String() string { // Result of a scan type Result struct { - Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp"` - ASN *AsnResponse `json:"asn,omitempty" csv:"asn"` - err error + Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp"` + ASN *AsnResponse `json:"asn,omitempty" csv:"asn"` + Err error `json:"-" csv:"-"` CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"` TLSData *clients.Response `json:"tls,omitempty" csv:"tls"` Hashes map[string]interface{} `json:"hash,omitempty" csv:"hash"` ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"` CDNName string `json:"cdn_name,omitempty" csv:"cdn_name"` Port string `json:"port,omitempty" csv:"port"` - raw string - URL string `json:"url,omitempty" csv:"url"` - Input string `json:"input,omitempty" csv:"input"` - Location string `json:"location,omitempty" csv:"location"` - Title string `json:"title,omitempty" csv:"title"` + Raw string `json:"-" csv:"-"` + URL string `json:"url,omitempty" csv:"url"` + Input string `json:"input,omitempty" csv:"input"` + Location string `json:"location,omitempty" csv:"location"` + Title string `json:"title,omitempty" csv:"title"` str string Scheme string `json:"scheme,omitempty" csv:"scheme"` Error string `json:"error,omitempty" csv:"error"` @@ -72,13 +72,16 @@ type Result struct { CDN bool `json:"cdn,omitempty" csv:"cdn"` HTTP2 bool `json:"http2,omitempty" csv:"http2"` Pipeline bool `json:"pipeline,omitempty" csv:"pipeline"` + HeadlessBody string `json:"headless_body,omitempty" csv:"headless_body"` + ScreenshotBytes []byte `json:"screenshot_bytes,omitempty" csv:"screenshot_bytes"` StoredResponsePath string `json:"stored_response_path,omitempty" csv:"stored_response_path"` + ScreenshotPath string `json:"screenshot_path,omitempty" csv:"screenshot_path"` } // function to get dsl variables from result struct func dslVariables() ([]string, error) { fakeResult := Result{} - fieldsToIgnore := []string{"Hashes", "ResponseHeader"} + fieldsToIgnore := []string{"Hashes", "ResponseHeader", "Err"} if err := faker.FakeData(&fakeResult, options.WithFieldsToIgnore(fieldsToIgnore...)); err != nil { return nil, err }